From a14e4cce3a60aabed1e29d17fd7c00e84b4f09c7 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 13 Sep 2022 02:01:33 +0530
Subject: [PATCH 001/310] Add `Lexer` impl and `TypeDefintion` expr

---
 server/src/engine/mod.rs   |  30 +++
 server/src/engine/query.rs | 443 +++++++++++++++++++++++++++++++++++++
 server/src/main.rs         |   1 +
 3 files changed, 474 insertions(+)
 create mode 100644 server/src/engine/mod.rs
 create mode 100644 server/src/engine/query.rs

diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
new file mode 100644
index 00000000..5e74bdfd
--- /dev/null
+++ b/server/src/engine/mod.rs
@@ -0,0 +1,30 @@
+/*
+ * Created on Mon Sep 12 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#![allow(dead_code)]
+#![allow(unused_macros)]
+
+mod query;
diff --git a/server/src/engine/query.rs b/server/src/engine/query.rs
new file mode 100644
index 00000000..6138628e
--- /dev/null
+++ b/server/src/engine/query.rs
@@ -0,0 +1,443 @@
+/*
+ * Created on Mon Sep 12 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+macro_rules! boxed {
+    ([] $ty:ty) => {
+        ::std::boxed::Box::<[$ty]>
+    };
+}
+
+/*
+    Definitions
+*/
+
+use crate::util::compiler;
+use std::{
+    fmt,
+    marker::PhantomData,
+    mem::{self, transmute},
+    ops::Deref,
+    slice, str,
+};
+
+/// An unsafe, C-like slice that holds a ptr and length. Construction and usage is at the risk of the user
+pub struct RawSlice {
+    ptr: *const u8,
+    len: usize,
+}
+
+impl RawSlice {
+    const _EALIGN: () = assert!(mem::align_of::<Self>() == mem::align_of::<&[u8]>());
+    const unsafe fn new(ptr: *const u8, len: usize) -> Self {
+        Self { ptr, len }
+    }
+    unsafe fn as_slice(&self) -> &[u8] {
+        slice::from_raw_parts(self.ptr, self.len)
+    }
+}
+
+#[cfg(debug_assertions)]
+impl fmt::Debug for RawSlice {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list()
+            .entries(unsafe {
+                // UNSAFE(@ohsayan): Note, the caller is responsible for ensuring validity as long the
+                // slice is used. also note, the Debug impl only exists for Debug builds so we never use
+                // this in release builds
+                self.as_slice()
+            })
+            .finish()
+    }
+}
+
+#[cfg(debug_assertions)]
+impl PartialEq for RawSlice {
+    fn eq(&self, other: &Self) -> bool {
+        unsafe {
+            // UNSAFE(@ohsayan): Callers must ensure validity during usage
+            self.as_slice() == other.as_slice()
+        }
+    }
+}
+
+#[cfg(debug_assertions)]
+impl<U> PartialEq<U> for RawSlice
+where
+    U: Deref<Target = [u8]>,
+{
+    fn eq(&self, other: &U) -> bool {
+        unsafe {
+            // UNSAFE(@ohsayan): Callers must ensure validity during usage
+            self.as_slice() == other.deref()
+        }
+    }
+}
+
+/*
+    Lang errors
+*/
+
+type LangResult<T> = Result<T, LangError>;
+
+#[derive(Debug, PartialEq)]
+pub enum LangError {
+    InvalidNumericLiteral,
+    InvalidStringLiteral,
+    UnexpectedChar,
+    InvalidTypeExpression,
+}
+
+/*
+    Lex meta
+*/
+
+#[derive(Debug)]
+#[cfg_attr(debug_assertions, derive(PartialEq))]
+pub enum Token {
+    OpenParen,            // (
+    CloseParen,           // )
+    OpenAngular,          // <
+    CloseAngular,         // >
+    OpenSqBracket,        // [
+    CloseSqBracket,       // ]
+    Comma,                // ,
+    Colon,                // :
+    Period,               // .
+    LitString(String),    // str lit
+    Identifier(RawSlice), // ident
+    LitNum(u64),          // num lit
+    Keyword(Kw),          // kw
+    OperatorEq,           // =
+    OperatorAdd,          // +
+}
+
+impl From<Kw> for Token {
+    fn from(kw: Kw) -> Self {
+        Self::Keyword(kw)
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Kw {
+    Use,
+    Create,
+    Drop,
+    Inspect,
+    Alter,
+    Space,
+    Model,
+    Force,
+    Type(Ty),
+}
+
+impl From<Ty> for Kw {
+    fn from(ty: Ty) -> Self {
+        Self::Type(ty)
+    }
+}
+
+impl Kw {
+    // FIXME(@ohsayan): Use our pf hack
+    pub fn try_from_slice(s: &[u8]) -> Option<Kw> {
+        let r = match s.to_ascii_lowercase().as_slice() {
+            b"use" => Self::Use,
+            b"create" => Self::Create,
+            b"drop" => Self::Drop,
+            b"inspect" => Self::Inspect,
+            b"alter" => Self::Alter,
+            b"space" => Self::Space,
+            b"model" => Self::Model,
+            b"force" => Self::Force,
+            b"string" => Self::Type(Ty::String),
+            b"binary" => Self::Type(Ty::Binary),
+            b"list" => Self::Type(Ty::Ls),
+            _ => return None,
+        };
+        return Some(r);
+    }
+}
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+#[repr(u8)]
+pub enum Ty {
+    String = 0_u8,
+    Binary = 1_u8,
+    Ls = 2_u8,
+}
+
+/*
+    Lexer impl
+*/
+
+#[inline(always)]
+fn dptr(s: *const u8, e: *const u8) -> usize {
+    e as usize - s as usize
+}
+
+pub struct Lexer<'a> {
+    c: *const u8,
+    eptr: *const u8,
+    last_error: Option<LangError>,
+    tokens: Vec<Token>,
+    _lt: PhantomData<&'a [u8]>,
+}
+
+impl<'a> Lexer<'a> {
+    pub const fn new(src: &'a [u8]) -> Self {
+        unsafe {
+            Self {
+                c: src.as_ptr(),
+                eptr: src.as_ptr().add(src.len()),
+                last_error: None,
+                tokens: Vec::new(),
+                _lt: PhantomData,
+            }
+        }
+    }
+}
+
+// meta
+impl<'a> Lexer<'a> {
+    #[inline(always)]
+    const fn cursor(&self) -> *const u8 {
+        self.c
+    }
+    #[inline(always)]
+    const fn data_end_ptr(&self) -> *const u8 {
+        self.eptr
+    }
+    #[inline(always)]
+    fn not_exhausted(&self) -> bool {
+        self.data_end_ptr() > self.cursor()
+    }
+    #[inline(always)]
+    fn exhausted(&self) -> bool {
+        self.cursor() == self.data_end_ptr()
+    }
+    unsafe fn deref_cursor(&self) -> u8 {
+        *self.cursor()
+    }
+    #[inline(always)]
+    unsafe fn incr_cursor_by(&mut self, by: usize) {
+        debug_assert!(self.not_exhausted());
+        self.c = self.cursor().add(by)
+    }
+    #[inline(always)]
+    unsafe fn incr_cursor(&mut self) {
+        self.incr_cursor_by(1)
+    }
+    #[inline(always)]
+    unsafe fn incr_cursor_if(&mut self, iff: bool) {
+        self.incr_cursor_by(iff as usize)
+    }
+    #[inline(always)]
+    fn push_token(&mut self, token: Token) {
+        self.tokens.push(token)
+    }
+    #[inline(always)]
+    fn peek_is(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
+        self.not_exhausted() && unsafe { f(self.deref_cursor()) }
+    }
+    #[inline(always)]
+    fn peek_is_and_forward(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
+        self.not_exhausted() && unsafe { f(self.deref_cursor()) }
+    }
+    #[inline(always)]
+    fn peek_eq_and_forward_or_eof(&mut self, eq: u8) -> bool {
+        unsafe {
+            let eq = self.not_exhausted() && self.deref_cursor() == eq;
+            self.incr_cursor_if(eq);
+            eq | self.exhausted()
+        }
+    }
+    #[inline(always)]
+    fn peek_neq(&self, b: u8) -> bool {
+        self.not_exhausted() && unsafe { self.deref_cursor() != b }
+    }
+    #[inline(always)]
+    fn peek_eq_and_forward(&mut self, b: u8) -> bool {
+        unsafe {
+            let r = self.not_exhausted() && self.deref_cursor() == b;
+            self.incr_cursor_if(r);
+            r
+        }
+    }
+    #[inline(always)]
+    unsafe fn check_escaped(&self, b: u8) -> bool {
+        debug_assert!(self.not_exhausted());
+        self.deref_cursor() == b'\\' && { self.not_exhausted() && self.deref_cursor() == b }
+    }
+    #[inline(always)]
+    fn trim_ahead(&mut self) {
+        while self.peek_is_and_forward(|b| b == b' ' || b == b'\t' || b == b'\n') {}
+    }
+}
+
+impl<'a> Lexer<'a> {
+    fn scan_ident(&mut self) -> RawSlice {
+        let s = self.cursor();
+        unsafe {
+            while self.peek_is(|b| b.is_ascii_alphanumeric() || b == b'_') {
+                self.incr_cursor();
+            }
+            RawSlice::new(s, dptr(s, self.cursor()))
+        }
+    }
+    fn scan_ident_or_keyword(&mut self) {
+        let s = self.scan_ident();
+        match Kw::try_from_slice(unsafe { s.as_slice() }) {
+            Some(kw) => self.tokens.push(kw.into()),
+            None => self.tokens.push(Token::Identifier(s)),
+        }
+    }
+    fn scan_number(&mut self) {
+        let s = self.cursor();
+        unsafe {
+            while self.peek_is(|b| b.is_ascii_digit()) {
+                self.incr_cursor();
+            }
+            let wseof = self.peek_eq_and_forward_or_eof(b' ');
+            match str::from_utf8_unchecked(slice::from_raw_parts(s, dptr(s, self.cursor()))).parse()
+            {
+                Ok(num) if compiler::likely(wseof) => self.tokens.push(Token::LitNum(num)),
+                _ => self.last_error = Some(LangError::InvalidNumericLiteral),
+            }
+        }
+    }
+    fn scan_quoted_string(&mut self, quote_style: u8) {
+        debug_assert!(
+            unsafe { self.deref_cursor() } == quote_style,
+            "illegal call to scan_quoted_string"
+        );
+        unsafe { self.incr_cursor() }
+        let mut buf = Vec::new();
+        unsafe {
+            while self.peek_neq(quote_style) {
+                let esc_backslash = self.check_escaped(b'\\');
+                let esc_quote = self.check_escaped(quote_style);
+                // mutually exclusive
+                self.incr_cursor_if(esc_backslash | esc_quote);
+                buf.push(self.deref_cursor());
+                self.incr_cursor();
+            }
+            let eq = self.peek_eq_and_forward(quote_style);
+            match String::from_utf8(buf) {
+                Ok(st) if eq => self.tokens.push(Token::LitString(st)),
+                _ => self.last_error = Some(LangError::InvalidStringLiteral),
+            }
+        }
+    }
+    fn scan_byte(&mut self, byte: u8) {
+        let b = match byte {
+            b':' => Token::Colon,
+            b'(' => Token::OpenParen,
+            b')' => Token::CloseParen,
+            b'<' => Token::OpenAngular,
+            b'>' => Token::CloseAngular,
+            b'[' => Token::OpenSqBracket,
+            b']' => Token::CloseSqBracket,
+            b',' => Token::Comma,
+            b'.' => Token::Period,
+            b'=' => Token::OperatorEq,
+            b'+' => Token::OperatorAdd,
+            _ => {
+                self.last_error = Some(LangError::UnexpectedChar);
+                return;
+            }
+        };
+        self.tokens.push(b)
+    }
+
+    fn _lex(&mut self) {
+        while self.not_exhausted() && self.last_error.is_none() {
+            match unsafe { self.deref_cursor() } {
+                byte if byte.is_ascii_alphabetic() => self.scan_ident_or_keyword(),
+                byte if byte.is_ascii_digit() => self.scan_number(),
+                qs @ (b'\'' | b'"') => self.scan_quoted_string(qs),
+                b' ' => self.trim_ahead(),
+                b => self.scan_byte(b),
+            }
+        }
+    }
+
+    pub fn lex(src: &'a [u8]) -> LangResult<Vec<Token>> {
+        let mut slf = Self::new(src);
+        slf._lex();
+        match slf.last_error {
+            None => Ok(slf.tokens),
+            Some(e) => Err(e),
+        }
+    }
+}
+
+/*
+    AST
+*/
+
+#[derive(Debug, PartialEq)]
+pub struct TypeExpr(Vec<Ty>);
+
+#[repr(u8)]
+#[derive(Debug)]
+enum FTy {
+    String = 0,
+    Binary = 1,
+}
+
+#[derive(Debug)]
+struct TypeDefintion {
+    d: usize,
+    b: Ty,
+    f: FTy,
+}
+
+impl TypeDefintion {
+    const PRIM: usize = 1;
+    pub fn eval(s: TypeExpr) -> LangResult<Self> {
+        let TypeExpr(ex) = s;
+        let l = ex.len();
+        #[inline(always)]
+        fn ls(t: &Ty) -> bool {
+            *t == Ty::Ls
+        }
+        let d = ex.iter().map(|v| ls(v) as usize).sum::<usize>();
+        let v = (l == 1 && ex[0] != Ty::Ls) || (l > 1 && (d == l - 1) && ex[l - 1] != Ty::Ls);
+        if compiler::likely(v) {
+            unsafe {
+                Ok(Self {
+                    d: d + 1,
+                    b: ex[0],
+                    f: transmute(ex[l - 1]),
+                })
+            }
+        } else {
+            compiler::cold_err(Err(LangError::InvalidTypeExpression))
+        }
+    }
+    pub const fn is_prim(&self) -> bool {
+        self.d == Self::PRIM
+    }
+}
diff --git a/server/src/main.rs b/server/src/main.rs
index 421a6009..9dad3e91 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -53,6 +53,7 @@ mod config;
 mod corestore;
 mod dbnet;
 mod diskstore;
+mod engine;
 mod kvengine;
 mod protocol;
 mod queryengine;

From 9d0486452facf22e0099c25ba4045f631776da32 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 13 Sep 2022 21:10:10 +0530
Subject: [PATCH 002/310] Implement basic AST, add tests and fix lexing

---
 server/src/engine/mod.rs                    |   2 +-
 server/src/engine/ql/ast.rs                 | 256 +++++++++++++++++
 server/src/engine/{query.rs => ql/lexer.rs} | 301 ++++++++------------
 server/src/engine/ql/mod.rs                 | 122 ++++++++
 server/src/engine/ql/schema.rs              |  80 ++++++
 server/src/engine/ql/tests.rs               | 148 ++++++++++
 6 files changed, 721 insertions(+), 188 deletions(-)
 create mode 100644 server/src/engine/ql/ast.rs
 rename server/src/engine/{query.rs => ql/lexer.rs} (58%)
 create mode 100644 server/src/engine/ql/mod.rs
 create mode 100644 server/src/engine/ql/schema.rs
 create mode 100644 server/src/engine/ql/tests.rs

diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 5e74bdfd..6edec42a 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -27,4 +27,4 @@
 #![allow(dead_code)]
 #![allow(unused_macros)]
 
-mod query;
+mod ql;
diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
new file mode 100644
index 00000000..80d2cf43
--- /dev/null
+++ b/server/src/engine/ql/ast.rs
@@ -0,0 +1,256 @@
+/*
+ * Created on Tue Sep 13 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{
+        dptr,
+        lexer::{Kw, Lexer, Stmt, Token, Ty},
+        schema, LangError, LangResult, RawSlice,
+    },
+    crate::util::{compiler, Life},
+    core::{marker::PhantomData, mem::transmute},
+};
+
+/*
+    AST
+*/
+
+#[derive(Debug, PartialEq)]
+pub struct TypeExpr(Vec<Ty>);
+
+#[repr(u8)]
+#[derive(Debug)]
+enum FTy {
+    String = 0,
+    Binary = 1,
+}
+
+#[derive(Debug)]
+struct TypeDefintion {
+    d: usize,
+    b: Ty,
+    f: FTy,
+}
+
+impl TypeDefintion {
+    const PRIM: usize = 1;
+    pub fn eval(s: TypeExpr) -> LangResult<Self> {
+        let TypeExpr(ex) = s;
+        let l = ex.len();
+        #[inline(always)]
+        fn ls(t: &Ty) -> bool {
+            *t == Ty::Ls
+        }
+        let d = ex.iter().map(|v| ls(v) as usize).sum::<usize>();
+        let v = (l == 1 && ex[0] != Ty::Ls) || (l > 1 && (d == l - 1) && ex[l - 1] != Ty::Ls);
+        if compiler::likely(v) {
+            unsafe {
+                Ok(Self {
+                    d: d + 1,
+                    b: ex[0],
+                    f: transmute(ex[l - 1]),
+                })
+            }
+        } else {
+            compiler::cold_err(Err(LangError::InvalidTypeExpression))
+        }
+    }
+    pub const fn is_prim(&self) -> bool {
+        self.d == Self::PRIM
+    }
+}
+
+pub enum Entity {
+    Current(RawSlice),
+    Partial(RawSlice),
+    Full(RawSlice, RawSlice),
+}
+
+impl Entity {
+    fn parse(cm: &mut Compiler) -> LangResult<Self> {
+        let a = cm.nxtok_nofw_opt();
+        let b = cm.nxtok_nofw_opt();
+        let c = cm.nxtok_nofw_opt();
+        match (a, b, c) {
+            (Some(Token::Ident(ks)), Some(Token::Period), Some(Token::Ident(tbl))) => unsafe {
+                let r = Ok(Entity::Full(ks.raw_clone(), tbl.raw_clone()));
+                cm.incr_cursor_by(3);
+                r
+            },
+            (Some(Token::Ident(ident)), _, _) => unsafe {
+                let r = Ok(Entity::Current(ident.raw_clone()));
+                cm.incr_cursor();
+                r
+            },
+            (Some(Token::Colon), Some(Token::Ident(tbl)), _) => unsafe {
+                let r = Ok(Entity::Partial(tbl.raw_clone()));
+                cm.incr_cursor_by(2);
+                r
+            },
+            _ => Err(LangError::UnexpectedToken),
+        }
+    }
+}
+
+pub enum Statement {}
+
+pub struct Compiler<'a> {
+    c: *const Token,
+    e: *const Token,
+    _lt: PhantomData<&'a [u8]>,
+}
+
+impl<'a> Compiler<'a> {
+    pub fn compile(src: &'a [u8]) -> LangResult<Life<'a, Statement>> {
+        let token_stream = Lexer::lex(src)?;
+        Self::new(&token_stream).compile_link_lt()
+    }
+    #[inline(always)]
+    const fn new(token_stream: &[Token]) -> Self {
+        unsafe {
+            Self {
+                c: token_stream.as_ptr(),
+                e: token_stream.as_ptr().add(token_stream.len()),
+                _lt: PhantomData,
+            }
+        }
+    }
+    #[inline(always)]
+    fn compile_link_lt(mut self) -> LangResult<Life<'a, Statement>> {
+        match self.stage0() {
+            Ok(t) if self.exhausted() => Ok(Life::new(t)),
+            Err(e) => Err(e),
+            _ => Err(LangError::UnexpectedToken),
+        }
+    }
+    #[inline(always)]
+    fn stage0(&mut self) -> Result<Statement, LangError> {
+        match self.nxtok_opt() {
+            Some(Token::Keyword(Kw::Stmt(stmt))) => match stmt {
+                Stmt::Create => self.create0(),
+                Stmt::Drop => self.drop0(),
+                Stmt::Alter => self.alter0(),
+                Stmt::Inspect => self.inspect0(),
+                Stmt::Use => self.use0(),
+            },
+            _ => Err(LangError::ExpectedStatement),
+        }
+    }
+    #[inline(always)]
+    fn create0(&mut self) -> Result<Statement, LangError> {
+        match self.nxtok_opt() {
+            Some(Token::Keyword(Kw::Model)) => self.c_model0(),
+            Some(Token::Keyword(Kw::Space)) => self.c_space0(),
+            _ => Err(LangError::UnexpectedEndofStatement),
+        }
+    }
+    #[inline(always)]
+    fn drop0(&mut self) -> Result<Statement, LangError> {
+        todo!()
+    }
+    #[inline(always)]
+    fn alter0(&mut self) -> Result<Statement, LangError> {
+        todo!()
+    }
+    #[inline(always)]
+    fn inspect0(&mut self) -> Result<Statement, LangError> {
+        todo!()
+    }
+    #[inline(always)]
+    fn use0(&mut self) -> Result<Statement, LangError> {
+        todo!()
+    }
+    #[inline(always)]
+    fn c_model0(&mut self) -> Result<Statement, LangError> {
+        let model_name = match self.nxtok_opt() {
+            Some(Token::Ident(model)) => unsafe { model.raw_clone() },
+            _ => return Err(LangError::UnexpectedEndofStatement),
+        };
+        schema::parse_schema(self, model_name)
+    }
+    #[inline(always)]
+    fn c_space0(&mut self) -> Result<Statement, LangError> {
+        todo!()
+    }
+}
+
+impl<'a> Compiler<'a> {
+    #[inline(always)]
+    pub(super) fn nxtok_opt(&mut self) -> Option<&Token> {
+        if self.not_exhausted() {
+            unsafe {
+                let r = Some(&*self.c);
+                self.incr_cursor();
+                r
+            }
+        } else {
+            None
+        }
+    }
+    #[inline(always)]
+    fn nxtok_nofw_opt(&self) -> Option<&Token> {
+        if self.not_exhausted() {
+            unsafe { Some(&*self.c) }
+        } else {
+            None
+        }
+    }
+    #[inline(always)]
+    pub(super) fn not_exhausted(&self) -> bool {
+        self.c != self.e
+    }
+    #[inline(always)]
+    pub(super) fn exhausted(&self) -> bool {
+        self.c == self.e
+    }
+    #[inline(always)]
+    pub(super) fn remaining(&self) -> usize {
+        dptr(self.c, self.e)
+    }
+    unsafe fn deref_cursor(&self) -> &Token {
+        &*self.c
+    }
+    pub(super) fn peek_eq_and_forward(&mut self, t: &Token) -> bool {
+        self.not_exhausted() && unsafe { self.deref_cursor() == t }
+    }
+    #[inline(always)]
+    pub(super) unsafe fn incr_cursor(&mut self) {
+        self.incr_cursor_by(1)
+    }
+    #[inline(always)]
+    pub(super) unsafe fn incr_cursor_by(&mut self, by: usize) {
+        debug_assert!(self.remaining() >= by);
+        self.c = self.c.add(by);
+    }
+    #[inline(always)]
+    pub(super) unsafe fn decr_cursor_by(&mut self, by: usize) {
+        debug_assert!(
+            self.remaining().checked_sub(by).is_some(),
+            "cursor crossed e"
+        );
+        self.c = self.c.sub(by);
+    }
+}
diff --git a/server/src/engine/query.rs b/server/src/engine/ql/lexer.rs
similarity index 58%
rename from server/src/engine/query.rs
rename to server/src/engine/ql/lexer.rs
index 6138628e..64aaa094 100644
--- a/server/src/engine/query.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -1,5 +1,5 @@
 /*
- * Created on Mon Sep 12 2022
+ * Created on Tue Sep 13 2022
  *
  * This file is a part of Skytable
  * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@@ -24,133 +24,94 @@
  *
 */
 
-macro_rules! boxed {
-    ([] $ty:ty) => {
-        ::std::boxed::Box::<[$ty]>
-    };
-}
+use crate::util::Life;
+
+use {
+    super::{dptr, LangError, LangResult, RawSlice},
+    crate::util::compiler,
+    core::{marker::PhantomData, slice, str},
+};
 
 /*
-    Definitions
+    Lex meta
 */
 
-use crate::util::compiler;
-use std::{
-    fmt,
-    marker::PhantomData,
-    mem::{self, transmute},
-    ops::Deref,
-    slice, str,
-};
-
-/// An unsafe, C-like slice that holds a ptr and length. Construction and usage is at the risk of the user
-pub struct RawSlice {
-    ptr: *const u8,
-    len: usize,
+#[derive(Debug)]
+#[cfg_attr(debug_assertions, derive(PartialEq))]
+#[repr(u8)]
+pub enum Token {
+    OpenParen,       // (
+    CloseParen,      // )
+    OpenAngular,     // <
+    CloseAngular,    // >
+    OpenSqBracket,   // [
+    CloseSqBracket,  // ]
+    OpenBrace,       // {
+    CloseBrace,      // }
+    Comma,           // ,
+    Colon,           // :
+    Period,          // .
+    Ident(RawSlice), // ident
+    Keyword(Kw),     // kw
+    OperatorEq,      // =
+    OperatorAdd,     // +
+    Lit(Lit),        // literal
 }
 
-impl RawSlice {
-    const _EALIGN: () = assert!(mem::align_of::<Self>() == mem::align_of::<&[u8]>());
-    const unsafe fn new(ptr: *const u8, len: usize) -> Self {
-        Self { ptr, len }
-    }
-    unsafe fn as_slice(&self) -> &[u8] {
-        slice::from_raw_parts(self.ptr, self.len)
+impl From<Kw> for Token {
+    fn from(kw: Kw) -> Self {
+        Self::Keyword(kw)
     }
 }
 
-#[cfg(debug_assertions)]
-impl fmt::Debug for RawSlice {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_list()
-            .entries(unsafe {
-                // UNSAFE(@ohsayan): Note, the caller is responsible for ensuring validity as long the
-                // slice is used. also note, the Debug impl only exists for Debug builds so we never use
-                // this in release builds
-                self.as_slice()
-            })
-            .finish()
-    }
+#[derive(Debug, PartialEq)]
+pub enum Lit {
+    Str(String),
+    Bool(bool),
+    Num(u64),
 }
 
-#[cfg(debug_assertions)]
-impl PartialEq for RawSlice {
-    fn eq(&self, other: &Self) -> bool {
-        unsafe {
-            // UNSAFE(@ohsayan): Callers must ensure validity during usage
-            self.as_slice() == other.as_slice()
-        }
+impl From<Lit> for Token {
+    fn from(l: Lit) -> Self {
+        Self::Lit(l)
     }
 }
 
-#[cfg(debug_assertions)]
-impl<U> PartialEq<U> for RawSlice
-where
-    U: Deref<Target = [u8]>,
-{
-    fn eq(&self, other: &U) -> bool {
-        unsafe {
-            // UNSAFE(@ohsayan): Callers must ensure validity during usage
-            self.as_slice() == other.deref()
-        }
+impl From<u64> for Lit {
+    fn from(n: u64) -> Self {
+        Self::Num(n)
     }
 }
 
-/*
-    Lang errors
-*/
-
-type LangResult<T> = Result<T, LangError>;
-
-#[derive(Debug, PartialEq)]
-pub enum LangError {
-    InvalidNumericLiteral,
-    InvalidStringLiteral,
-    UnexpectedChar,
-    InvalidTypeExpression,
-}
-
-/*
-    Lex meta
-*/
-
-#[derive(Debug)]
-#[cfg_attr(debug_assertions, derive(PartialEq))]
-pub enum Token {
-    OpenParen,            // (
-    CloseParen,           // )
-    OpenAngular,          // <
-    CloseAngular,         // >
-    OpenSqBracket,        // [
-    CloseSqBracket,       // ]
-    Comma,                // ,
-    Colon,                // :
-    Period,               // .
-    LitString(String),    // str lit
-    Identifier(RawSlice), // ident
-    LitNum(u64),          // num lit
-    Keyword(Kw),          // kw
-    OperatorEq,           // =
-    OperatorAdd,          // +
+impl From<String> for Lit {
+    fn from(s: String) -> Self {
+        Self::Str(s)
+    }
 }
 
-impl From<Kw> for Token {
-    fn from(kw: Kw) -> Self {
-        Self::Keyword(kw)
+impl From<bool> for Lit {
+    fn from(b: bool) -> Self {
+        Self::Bool(b)
     }
 }
 
-#[derive(Debug, PartialEq)]
-pub enum Kw {
+#[derive(Debug, PartialEq, Clone, Copy)]
+#[repr(u8)]
+pub enum Stmt {
     Use,
     Create,
     Drop,
     Inspect,
     Alter,
+}
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+#[repr(u8)]
+pub enum Kw {
+    Stmt(Stmt),
+    Type(Ty),
     Space,
     Model,
-    Force,
-    Type(Ty),
 }
 
 impl From<Ty> for Kw {
@@ -161,19 +122,20 @@ impl From<Ty> for Kw {
 
 impl Kw {
     // FIXME(@ohsayan): Use our pf hack
-    pub fn try_from_slice(s: &[u8]) -> Option<Kw> {
+    pub fn try_from_slice(s: &[u8]) -> Option<Token> {
         let r = match s.to_ascii_lowercase().as_slice() {
-            b"use" => Self::Use,
-            b"create" => Self::Create,
-            b"drop" => Self::Drop,
-            b"inspect" => Self::Inspect,
-            b"alter" => Self::Alter,
-            b"space" => Self::Space,
-            b"model" => Self::Model,
-            b"force" => Self::Force,
-            b"string" => Self::Type(Ty::String),
-            b"binary" => Self::Type(Ty::Binary),
-            b"list" => Self::Type(Ty::Ls),
+            b"use" => Self::Stmt(Stmt::Use).into(),
+            b"create" => Self::Stmt(Stmt::Create).into(),
+            b"drop" => Self::Stmt(Stmt::Drop).into(),
+            b"inspect" => Self::Stmt(Stmt::Inspect).into(),
+            b"alter" => Self::Stmt(Stmt::Alter).into(),
+            b"space" => Self::Space.into(),
+            b"model" => Self::Model.into(),
+            b"string" => Self::Type(Ty::String).into(),
+            b"binary" => Self::Type(Ty::Binary).into(),
+            b"list" => Self::Type(Ty::Ls).into(),
+            b"true" => Token::Lit(Lit::Bool(true)),
+            b"false" => Token::Lit(Lit::Bool(false)),
             _ => return None,
         };
         return Some(r);
@@ -192,14 +154,9 @@ pub enum Ty {
     Lexer impl
 */
 
-#[inline(always)]
-fn dptr(s: *const u8, e: *const u8) -> usize {
-    e as usize - s as usize
-}
-
 pub struct Lexer<'a> {
     c: *const u8,
-    eptr: *const u8,
+    e: *const u8,
     last_error: Option<LangError>,
     tokens: Vec<Token>,
     _lt: PhantomData<&'a [u8]>,
@@ -210,7 +167,7 @@ impl<'a> Lexer<'a> {
         unsafe {
             Self {
                 c: src.as_ptr(),
-                eptr: src.as_ptr().add(src.len()),
+                e: src.as_ptr().add(src.len()),
                 last_error: None,
                 tokens: Vec::new(),
                 _lt: PhantomData,
@@ -227,7 +184,7 @@ impl<'a> Lexer<'a> {
     }
     #[inline(always)]
     const fn data_end_ptr(&self) -> *const u8 {
-        self.eptr
+        self.e
     }
     #[inline(always)]
     fn not_exhausted(&self) -> bool {
@@ -237,12 +194,16 @@ impl<'a> Lexer<'a> {
     fn exhausted(&self) -> bool {
         self.cursor() == self.data_end_ptr()
     }
+    #[inline(always)]
+    fn remaining(&self) -> usize {
+        dptr(self.c, self.e)
+    }
     unsafe fn deref_cursor(&self) -> u8 {
         *self.cursor()
     }
     #[inline(always)]
     unsafe fn incr_cursor_by(&mut self, by: usize) {
-        debug_assert!(self.not_exhausted());
+        debug_assert!(self.remaining() >= by);
         self.c = self.cursor().add(by)
     }
     #[inline(always)]
@@ -263,7 +224,11 @@ impl<'a> Lexer<'a> {
     }
     #[inline(always)]
     fn peek_is_and_forward(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
-        self.not_exhausted() && unsafe { f(self.deref_cursor()) }
+        let did_fw = self.not_exhausted() && unsafe { f(self.deref_cursor()) };
+        unsafe {
+            self.incr_cursor_if(did_fw);
+        }
+        did_fw
     }
     #[inline(always)]
     fn peek_eq_and_forward_or_eof(&mut self, eq: u8) -> bool {
@@ -286,11 +251,6 @@ impl<'a> Lexer<'a> {
         }
     }
     #[inline(always)]
-    unsafe fn check_escaped(&self, b: u8) -> bool {
-        debug_assert!(self.not_exhausted());
-        self.deref_cursor() == b'\\' && { self.not_exhausted() && self.deref_cursor() == b }
-    }
-    #[inline(always)]
     fn trim_ahead(&mut self) {
         while self.peek_is_and_forward(|b| b == b' ' || b == b'\t' || b == b'\n') {}
     }
@@ -309,8 +269,8 @@ impl<'a> Lexer<'a> {
     fn scan_ident_or_keyword(&mut self) {
         let s = self.scan_ident();
         match Kw::try_from_slice(unsafe { s.as_slice() }) {
-            Some(kw) => self.tokens.push(kw.into()),
-            None => self.tokens.push(Token::Identifier(s)),
+            Some(kw) => self.tokens.push(kw),
+            None => self.tokens.push(Token::Ident(s)),
         }
     }
     fn scan_number(&mut self) {
@@ -322,7 +282,7 @@ impl<'a> Lexer<'a> {
             let wseof = self.peek_eq_and_forward_or_eof(b' ');
             match str::from_utf8_unchecked(slice::from_raw_parts(s, dptr(s, self.cursor()))).parse()
             {
-                Ok(num) if compiler::likely(wseof) => self.tokens.push(Token::LitNum(num)),
+                Ok(num) if compiler::likely(wseof) => self.tokens.push(Token::Lit(Lit::Num(num))),
                 _ => self.last_error = Some(LangError::InvalidNumericLiteral),
             }
         }
@@ -336,16 +296,30 @@ impl<'a> Lexer<'a> {
         let mut buf = Vec::new();
         unsafe {
             while self.peek_neq(quote_style) {
-                let esc_backslash = self.check_escaped(b'\\');
-                let esc_quote = self.check_escaped(quote_style);
-                // mutually exclusive
-                self.incr_cursor_if(esc_backslash | esc_quote);
-                buf.push(self.deref_cursor());
+                match self.deref_cursor() {
+                    b if b != b'\\' => {
+                        buf.push(b);
+                    }
+                    _ => {
+                        self.incr_cursor();
+                        if self.exhausted() {
+                            break;
+                        }
+                        let b = self.deref_cursor();
+                        let quote = b == quote_style;
+                        let bs = b == b'\\';
+                        if quote | bs {
+                            buf.push(b);
+                        } else {
+                            break; // what on good earth is that escape?
+                        }
+                    }
+                }
                 self.incr_cursor();
             }
-            let eq = self.peek_eq_and_forward(quote_style);
+            let terminated = self.peek_eq_and_forward(quote_style);
             match String::from_utf8(buf) {
-                Ok(st) if eq => self.tokens.push(Token::LitString(st)),
+                Ok(st) if terminated => self.tokens.push(Token::Lit(st.into())),
                 _ => self.last_error = Some(LangError::InvalidStringLiteral),
             }
         }
@@ -359,6 +333,8 @@ impl<'a> Lexer<'a> {
             b'>' => Token::CloseAngular,
             b'[' => Token::OpenSqBracket,
             b']' => Token::CloseSqBracket,
+            b'{' => Token::OpenBrace,
+            b'}' => Token::CloseBrace,
             b',' => Token::Comma,
             b'.' => Token::Period,
             b'=' => Token::OperatorEq,
@@ -383,61 +359,12 @@ impl<'a> Lexer<'a> {
         }
     }
 
-    pub fn lex(src: &'a [u8]) -> LangResult<Vec<Token>> {
+    pub fn lex(src: &'a [u8]) -> LangResult<Life<'a, Vec<Token>>> {
         let mut slf = Self::new(src);
         slf._lex();
         match slf.last_error {
-            None => Ok(slf.tokens),
+            None => Ok(Life::new(slf.tokens)),
             Some(e) => Err(e),
         }
     }
 }
-
-/*
-    AST
-*/
-
-#[derive(Debug, PartialEq)]
-pub struct TypeExpr(Vec<Ty>);
-
-#[repr(u8)]
-#[derive(Debug)]
-enum FTy {
-    String = 0,
-    Binary = 1,
-}
-
-#[derive(Debug)]
-struct TypeDefintion {
-    d: usize,
-    b: Ty,
-    f: FTy,
-}
-
-impl TypeDefintion {
-    const PRIM: usize = 1;
-    pub fn eval(s: TypeExpr) -> LangResult<Self> {
-        let TypeExpr(ex) = s;
-        let l = ex.len();
-        #[inline(always)]
-        fn ls(t: &Ty) -> bool {
-            *t == Ty::Ls
-        }
-        let d = ex.iter().map(|v| ls(v) as usize).sum::<usize>();
-        let v = (l == 1 && ex[0] != Ty::Ls) || (l > 1 && (d == l - 1) && ex[l - 1] != Ty::Ls);
-        if compiler::likely(v) {
-            unsafe {
-                Ok(Self {
-                    d: d + 1,
-                    b: ex[0],
-                    f: transmute(ex[l - 1]),
-                })
-            }
-        } else {
-            compiler::cold_err(Err(LangError::InvalidTypeExpression))
-        }
-    }
-    pub const fn is_prim(&self) -> bool {
-        self.d == Self::PRIM
-    }
-}
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
new file mode 100644
index 00000000..a1bf6be1
--- /dev/null
+++ b/server/src/engine/ql/mod.rs
@@ -0,0 +1,122 @@
+/*
+ * Created on Tue Sep 13 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+pub(super) mod ast;
+pub(super) mod lexer;
+pub(super) mod schema;
+#[cfg(test)]
+mod tests;
+
+use core::{fmt, mem, ops::Deref, slice};
+
+/*
+    Lang errors
+*/
+
+pub type LangResult<T> = Result<T, LangError>;
+
+#[derive(Debug, PartialEq)]
+#[repr(u8)]
+pub enum LangError {
+    InvalidNumericLiteral,
+    InvalidStringLiteral,
+    UnexpectedChar,
+    InvalidTypeExpression,
+    ExpectedStatement,
+    UnexpectedEndofStatement,
+    UnexpectedToken,
+}
+
+/*
+    Utils
+*/
+
+/// An unsafe, C-like slice that holds a ptr and length. Construction and usage is at the risk of the user
+pub struct RawSlice {
+    ptr: *const u8,
+    len: usize,
+}
+
+impl RawSlice {
+    const _EALIGN: () = assert!(mem::align_of::<Self>() == mem::align_of::<&[u8]>());
+    const unsafe fn new(ptr: *const u8, len: usize) -> Self {
+        Self { ptr, len }
+    }
+    unsafe fn as_slice(&self) -> &[u8] {
+        slice::from_raw_parts(self.ptr, self.len)
+    }
+    unsafe fn raw_clone(&self) -> Self {
+        Self::new(self.ptr, self.len)
+    }
+}
+
+#[cfg(debug_assertions)]
+impl fmt::Debug for RawSlice {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list()
+            .entries(unsafe {
+                // UNSAFE(@ohsayan): Note, the caller is responsible for ensuring validity as long the
+                // slice is used. also note, the Debug impl only exists for Debug builds so we never use
+                // this in release builds
+                self.as_slice()
+            })
+            .finish()
+    }
+}
+
+#[cfg(debug_assertions)]
+impl PartialEq for RawSlice {
+    fn eq(&self, other: &Self) -> bool {
+        unsafe {
+            // UNSAFE(@ohsayan): Callers must ensure validity during usage
+            self.as_slice() == other.as_slice()
+        }
+    }
+}
+
+#[cfg(debug_assertions)]
+impl<U> PartialEq<U> for RawSlice
+where
+    U: Deref<Target = [u8]>,
+{
+    fn eq(&self, other: &U) -> bool {
+        unsafe {
+            // UNSAFE(@ohsayan): Callers must ensure validity during usage
+            self.as_slice() == other.deref()
+        }
+    }
+}
+
+impl From<&'static str> for RawSlice {
+    fn from(st: &'static str) -> Self {
+        unsafe { Self::new(st.as_bytes().as_ptr(), st.as_bytes().len()) }
+    }
+}
+
+#[inline(always)]
+fn dptr<T>(s: *const T, e: *const T) -> usize {
+    e as usize - s as usize
+}
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
new file mode 100644
index 00000000..60e4aea5
--- /dev/null
+++ b/server/src/engine/ql/schema.rs
@@ -0,0 +1,80 @@
+/*
+ * Created on Tue Sep 13 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{
+        ast::{Compiler, Statement},
+        lexer::{Lit, Ty},
+        LangResult, RawSlice,
+    },
+    std::collections::HashMap,
+};
+
+macro_rules! boxed {
+    ([] $ty:ty) => {
+        ::std::boxed::Box::<[$ty]>
+    };
+}
+
+/*
+    Meta
+*/
+
+struct FieldMeta {
+    field_name: Option<RawSlice>,
+    unprocessed_layers: boxed![[] TypeConfig],
+}
+
+type Dictionary = HashMap<String, Lit>;
+
+struct TypeConfig {
+    ty: Ty,
+    dict: Dictionary,
+}
+
+struct CreateStatement {
+    entity: RawSlice,
+}
+
+/*
+    Validation
+*/
+
+fn parse_dictionary(_c: &mut Compiler) -> LangResult<Dictionary> {
+    todo!()
+}
+
+fn parse_field(_c: &mut Compiler) -> LangResult<FieldMeta> {
+    todo!()
+}
+
+fn parse_type_definition(_c: &mut Compiler) -> LangResult<boxed![[] TypeConfig]> {
+    todo!()
+}
+
+pub(super) fn parse_schema(_c: &mut Compiler, _model: RawSlice) -> LangResult<Statement> {
+    todo!()
+}
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
new file mode 100644
index 00000000..37901458
--- /dev/null
+++ b/server/src/engine/ql/tests.rs
@@ -0,0 +1,148 @@
+/*
+ * Created on Tue Sep 13 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::lexer;
+
+mod lexer_tests {
+    use crate::engine::ql::LangError;
+
+    use super::lexer::{Lexer, Lit, Token};
+
+    macro_rules! v(
+        ($e:literal) => {{
+            $e.as_bytes().to_vec()
+        }};
+        ($($e:literal),* $(,)?) => {{
+            ($(v!($e)),*)
+        }};
+    );
+
+    #[test]
+    fn lex_ident() {
+        let src = v!("hello");
+        assert_eq!(
+            Lexer::lex(&src).unwrap(),
+            vec![Token::Ident("hello".into())]
+        );
+    }
+
+    // literals
+    #[test]
+    fn lex_number() {
+        let number = v!("123456");
+        assert_eq!(
+            Lexer::lex(&number).unwrap(),
+            vec![Token::Lit(Lit::Num(123456))]
+        );
+    }
+    #[test]
+    fn lex_bool() {
+        let (t, f) = v!("true", "false");
+        assert_eq!(Lexer::lex(&t).unwrap(), vec![Token::Lit(Lit::Bool(true))]);
+        assert_eq!(Lexer::lex(&f).unwrap(), vec![Token::Lit(Lit::Bool(false))]);
+    }
+    #[test]
+    fn lex_string() {
+        let s = br#" "hello, world" "#;
+        assert_eq!(
+            Lexer::lex(s).unwrap(),
+            vec![Token::Lit(Lit::Str("hello, world".into()))]
+        );
+        let s = br#" 'hello, world' "#;
+        assert_eq!(
+            Lexer::lex(s).unwrap(),
+            vec![Token::Lit(Lit::Str("hello, world".into()))]
+        );
+    }
+    #[test]
+    fn lex_string_test_escape_quote() {
+        let s = br#" "\"hello world\"" "#; // == "hello world"
+        assert_eq!(
+            Lexer::lex(s).unwrap(),
+            vec![Token::Lit(Lit::Str("\"hello world\"".into()))]
+        );
+        let s = br#" '\'hello world\'' "#; // == 'hello world'
+        assert_eq!(
+            Lexer::lex(s).unwrap(),
+            vec![Token::Lit(Lit::Str("'hello world'".into()))]
+        );
+    }
+    #[test]
+    fn lex_string_use_different_quote_style() {
+        let s = br#" "he's on it" "#;
+        assert_eq!(
+            Lexer::lex(s).unwrap(),
+            vec![Token::Lit(Lit::Str("he's on it".into()))]
+        );
+        let s = br#" 'he thinks that "that girl" fixed it' "#;
+        assert_eq!(
+            Lexer::lex(s).unwrap(),
+            vec![Token::Lit(Lit::Str(
+                "he thinks that \"that girl\" fixed it".into()
+            ))]
+        )
+    }
+    #[test]
+    fn lex_string_escape_bs() {
+        let s = v!(r#" "windows has c:\\" "#);
+        assert_eq!(
+            Lexer::lex(&s).unwrap(),
+            vec![Token::Lit(Lit::Str("windows has c:\\".into()))]
+        );
+        let s = v!(r#" 'windows has c:\\' "#);
+        assert_eq!(
+            Lexer::lex(&s).unwrap(),
+            vec![Token::Lit(Lit::Str("windows has c:\\".into()))]
+        );
+        let lol = v!(r#"'\\\\\\\\\\'"#);
+        assert_eq!(
+            Lexer::lex(&lol).unwrap(),
+            vec![Token::Lit(Lit::Str("\\".repeat(5)))],
+            "lol"
+        )
+    }
+    #[test]
+    fn lex_string_bad_escape() {
+        let wth = br#" '\a should be an alert on windows apparently' "#;
+        assert_eq!(
+            Lexer::lex(wth).unwrap_err(),
+            LangError::InvalidStringLiteral
+        );
+    }
+    #[test]
+    fn lex_string_unclosed() {
+        let wth = br#" 'omg where did the end go "#;
+        assert_eq!(
+            Lexer::lex(wth).unwrap_err(),
+            LangError::InvalidStringLiteral
+        );
+        let wth = br#" 'see, we escaped the end\' "#;
+        assert_eq!(
+            Lexer::lex(wth).unwrap_err(),
+            LangError::InvalidStringLiteral
+        );
+    }
+}

From 7d982a78bf2604acbb4b0b07b4a6f75143ee9507 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 14 Sep 2022 15:23:06 +0530
Subject: [PATCH 003/310] Add dictionary folding from tokens

---
 server/src/engine/ql/ast.rs    |  25 +++++-
 server/src/engine/ql/lexer.rs  |  17 +++-
 server/src/engine/ql/mod.rs    |   2 +
 server/src/engine/ql/schema.rs | 155 +++++++++++++++++++++++++++++----
 server/src/engine/ql/tests.rs  |  42 ++++++++-
 5 files changed, 215 insertions(+), 26 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 80d2cf43..9d31246d 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use core::slice;
+
 use {
     super::{
         dptr,
@@ -129,7 +131,7 @@ impl<'a> Compiler<'a> {
         Self::new(&token_stream).compile_link_lt()
     }
     #[inline(always)]
-    const fn new(token_stream: &[Token]) -> Self {
+    pub(super) const fn new(token_stream: &[Token]) -> Self {
         unsafe {
             Self {
                 c: token_stream.as_ptr(),
@@ -211,7 +213,11 @@ impl<'a> Compiler<'a> {
         }
     }
     #[inline(always)]
-    fn nxtok_nofw_opt(&self) -> Option<&Token> {
+    pub(super) const fn cursor(&self) -> *const Token {
+        self.c
+    }
+    #[inline(always)]
+    pub(super) fn nxtok_nofw_opt(&self) -> Option<&Token> {
         if self.not_exhausted() {
             unsafe { Some(&*self.c) }
         } else {
@@ -219,6 +225,10 @@ impl<'a> Compiler<'a> {
         }
     }
     #[inline(always)]
+    pub(super) fn remslice(&'a self) -> &'a [Token] {
+        unsafe { slice::from_raw_parts(self.c, self.remaining()) }
+    }
+    #[inline(always)]
     pub(super) fn not_exhausted(&self) -> bool {
         self.c != self.e
     }
@@ -230,16 +240,23 @@ impl<'a> Compiler<'a> {
     pub(super) fn remaining(&self) -> usize {
         dptr(self.c, self.e)
     }
-    unsafe fn deref_cursor(&self) -> &Token {
+    pub(super) unsafe fn deref_cursor(&self) -> &Token {
         &*self.c
     }
     pub(super) fn peek_eq_and_forward(&mut self, t: &Token) -> bool {
-        self.not_exhausted() && unsafe { self.deref_cursor() == t }
+        let did_fw = self.not_exhausted() && unsafe { self.deref_cursor() == t };
+        unsafe {
+            self.incr_cursor_if(did_fw);
+        }
+        did_fw
     }
     #[inline(always)]
     pub(super) unsafe fn incr_cursor(&mut self) {
         self.incr_cursor_by(1)
     }
+    pub(super) unsafe fn incr_cursor_if(&mut self, did_fw: bool) {
+        self.incr_cursor_by(did_fw as _)
+    }
     #[inline(always)]
     pub(super) unsafe fn incr_cursor_by(&mut self, by: usize) {
         debug_assert!(self.remaining() >= by);
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 64aaa094..e40cc949 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -64,7 +64,7 @@ impl From<Kw> for Token {
     }
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum Lit {
     Str(String),
     Bool(bool),
@@ -279,7 +279,15 @@ impl<'a> Lexer<'a> {
             while self.peek_is(|b| b.is_ascii_digit()) {
                 self.incr_cursor();
             }
-            let wseof = self.peek_eq_and_forward_or_eof(b' ');
+            /*
+                1234; // valid
+                1234} // valid
+                1234{ // invalid
+                1234, // valid
+                1234a // invalid
+            */
+            static TERMINAL_CHAR: [u8; 6] = [b';', b'}', b',', b' ', b'\n', b'\t'];
+            let wseof = self.peek_is(|b| TERMINAL_CHAR.contains(&b));
             match str::from_utf8_unchecked(slice::from_raw_parts(s, dptr(s, self.cursor()))).parse()
             {
                 Ok(num) if compiler::likely(wseof) => self.tokens.push(Token::Lit(Lit::Num(num))),
@@ -344,6 +352,9 @@ impl<'a> Lexer<'a> {
                 return;
             }
         };
+        unsafe {
+            self.incr_cursor();
+        }
         self.tokens.push(b)
     }
 
@@ -353,7 +364,7 @@ impl<'a> Lexer<'a> {
                 byte if byte.is_ascii_alphabetic() => self.scan_ident_or_keyword(),
                 byte if byte.is_ascii_digit() => self.scan_number(),
                 qs @ (b'\'' | b'"') => self.scan_quoted_string(qs),
-                b' ' => self.trim_ahead(),
+                b' ' | b'\n' | b'\t' => self.trim_ahead(),
                 b => self.scan_byte(b),
             }
         }
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index a1bf6be1..87a0ba2c 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -48,6 +48,7 @@ pub enum LangError {
     ExpectedStatement,
     UnexpectedEndofStatement,
     UnexpectedToken,
+    InvalidDictionaryExpression,
 }
 
 /*
@@ -55,6 +56,7 @@ pub enum LangError {
 */
 
 /// An unsafe, C-like slice that holds a ptr and length. Construction and usage is at the risk of the user
+#[cfg_attr(not(debug_assertions), derive(Debug))]
 pub struct RawSlice {
     ptr: *const u8,
     len: usize,
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 60e4aea5..3f561f5b 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -24,13 +24,22 @@
  *
 */
 
+/*
+    Most grammar tools are pretty much "off the shelf" which makes some things incredible hard to achieve (such
+    as custom error injection logic). To make things icier, Rust's integration with these tools (like lex) is not
+    very "refined." Hence, it is best for us to implement our own parsers. In the future, I plan to optimize our
+    rule checkers but that's not a concern at the moment.
+
+    -- Sayan (Sept. 14, 2022)
+*/
+
 use {
     super::{
         ast::{Compiler, Statement},
-        lexer::{Lit, Ty},
-        LangResult, RawSlice,
+        lexer::{Lit, Token, Ty},
+        LangError, LangResult, RawSlice,
     },
-    std::collections::HashMap,
+    std::{collections::HashMap, mem::MaybeUninit, str},
 };
 
 macro_rules! boxed {
@@ -43,35 +52,151 @@ macro_rules! boxed {
     Meta
 */
 
+#[derive(Debug)]
 struct FieldMeta {
     field_name: Option<RawSlice>,
     unprocessed_layers: boxed![[] TypeConfig],
 }
 
-type Dictionary = HashMap<String, Lit>;
-
+#[derive(Debug)]
 struct TypeConfig {
     ty: Ty,
-    dict: Dictionary,
-}
-
-struct CreateStatement {
-    entity: RawSlice,
+    dict: Dict,
 }
 
 /*
-    Validation
+    Dictionary
 */
 
-fn parse_dictionary(_c: &mut Compiler) -> LangResult<Dictionary> {
-    todo!()
+pub type Dict = HashMap<String, DictEntry>;
+
+#[derive(Debug, PartialEq)]
+pub enum DictEntry {
+    Lit(Lit),
+    Map(HashMap<String, Self>),
 }
 
-fn parse_field(_c: &mut Compiler) -> LangResult<FieldMeta> {
-    todo!()
+type NxFlag = u8;
+const EXP_COLON: NxFlag = 0x00;
+const EXP_LIT_OR_OBRC: NxFlag = 0x01;
+const EXP_COMMA_OR_CBRC: NxFlag = 0x02;
+const EXP_IDENT_OR_CBRC: NxFlag = 0x03;
+const EXP_EOF: NxFlag = 0x04;
+const EXP_START: NxFlag = 0x05;
+const HIBIT: u64 = 1 << 63;
+
+pub(super) fn parse_dictionary(c: &mut Compiler) -> LangResult<Dict> {
+    let mut dict = Dict::new();
+    let r = self::fold_dict(EXP_START, c.remslice(), &mut dict);
+    unsafe {
+        // IFEVERBROKEN: When the silicon guys decide to build a new chip with > 48 AS, lmk
+        c.incr_cursor_by((r & !HIBIT) as _);
+    }
+    if r & HIBIT == HIBIT {
+        Ok(dict)
+    } else {
+        Err(LangError::InvalidDictionaryExpression)
+    }
+}
+
+fn fold_dict<'a>(mut next: NxFlag, src: &'a [Token], dict: &mut Dict) -> u64 {
+    /*
+        NOTE: Assume respective <lit> validity and other character set rules
+
+        <lbrace> ::= "{"
+        <rbrace> ::= "}"
+        <colon> ::= ":"
+        <ident> ::= (<sym_us> | <alpha>) (<alphanum> | <sym_us>)*
+        <dict> ::= <lbrace> (<ident> <colon> (<lit> | <dict> ) <comma>)* <rbrace>
+    */
+    let mut i = 0;
+    let mut okay = true;
+    let mut tmp = MaybeUninit::uninit();
+    while i < src.len() && okay {
+        match (&src[i], next) {
+            // as a future optimization, note that this is just a single call
+            (Token::OpenBrace, EXP_START) => {
+                next = EXP_IDENT_OR_CBRC;
+            }
+            (Token::Ident(id), EXP_IDENT_OR_CBRC) => {
+                next = EXP_COLON;
+                tmp = MaybeUninit::new(unsafe {
+                    // UNSAFE(@ohsayan): If the token is an ident, the lexer guarantees that is valid unicode
+                    str::from_utf8_unchecked(id.as_slice())
+                });
+            }
+            (Token::Colon, EXP_COLON) => next = EXP_LIT_OR_OBRC,
+            (Token::Lit(lit), EXP_LIT_OR_OBRC) => {
+                okay &= dict
+                    .insert(
+                        unsafe {
+                            // UNSAFE(@ohsayan): This is completely safe because the transition and correctness
+                            // of this function makes it a consequence
+                            tmp.assume_init_ref()
+                        }
+                        .to_string(),
+                        DictEntry::Lit(lit.clone()),
+                    )
+                    .is_none();
+                next = EXP_COMMA_OR_CBRC;
+            }
+            (Token::OpenBrace, EXP_LIT_OR_OBRC) => {
+                // fold tokens
+                let mut this_dict = Dict::new();
+                let read = self::fold_dict(EXP_IDENT_OR_CBRC, &src[i..], &mut this_dict);
+                i += (read & !HIBIT) as usize;
+                okay &= (read & HIBIT) == HIBIT;
+                okay &= dict
+                    .insert(
+                        unsafe {
+                            // UNSAFE(@ohsayan): See above comment for context to know why this is safe
+                            tmp.assume_init_ref()
+                        }
+                        .to_string(),
+                        DictEntry::Map(this_dict),
+                    )
+                    .is_none();
+                next = EXP_COMMA_OR_CBRC;
+            }
+            (Token::Comma, EXP_COMMA_OR_CBRC) => {
+                next = EXP_IDENT_OR_CBRC;
+            }
+            (Token::CloseBrace, EXP_COMMA_OR_CBRC | EXP_IDENT_OR_CBRC) => {
+                // graceful exit
+                next = EXP_EOF;
+                i += 1;
+                break;
+            }
+            _ => {
+                okay = false;
+                break;
+            }
+        }
+        i += 1;
+    }
+    okay &= next == EXP_EOF;
+    ((okay as u64) << 63) | i as u64
 }
 
 fn parse_type_definition(_c: &mut Compiler) -> LangResult<boxed![[] TypeConfig]> {
+    /*
+        NOTE: Assume correct rules in context
+
+        <lbrace> ::= "{"
+        <rbrace> ::= "}"
+        <langle> ::= "<"
+        <rangle> ::= ">"
+        <colon> ::= ":"
+        <comma> ::= ","
+        <tydef_simple> ::= <type>
+        <tydef_nest> ::= <type> (<langle> <tydef_nest> <rangle>)*
+        <tydef_dict> ::= <type> <lbrace> ( ((<ident> <colon> <lit>) | <tydef_dict>) <comma>)* <rbrace>
+        <tydef> ::= <tydef_simple> | <tydef_nest> | <tydef_dict>
+    */
+    todo!()
+}
+
+fn parse_field(_c: &mut Compiler) -> LangResult<FieldMeta> {
     todo!()
 }
 
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 37901458..13ee1b98 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -24,12 +24,9 @@
  *
 */
 
-use super::lexer;
-
 mod lexer_tests {
     use crate::engine::ql::LangError;
-
-    use super::lexer::{Lexer, Lit, Token};
+    use super::super::lexer::{Lexer, Lit, Token};
 
     macro_rules! v(
         ($e:literal) => {{
@@ -146,3 +143,40 @@ mod lexer_tests {
         );
     }
 }
+
+mod schema_tests {
+    use super::super::{
+        ast::Compiler,
+        lexer::{Lexer, Lit},
+        schema::{parse_dictionary, Dict, DictEntry},
+    };
+    #[test]
+    fn dict_read() {
+        let d = Lexer::lex(
+            br#"
+            {
+                name: "sayan",
+                verified: true,
+                burgers: 152
+            }
+        "#,
+        )
+        .unwrap();
+        let mut c = Compiler::new(&d);
+        let dict = parse_dictionary(&mut c).unwrap();
+        let expected: Dict = [
+            (
+                "name".to_owned(),
+                DictEntry::Lit(Lit::Str("sayan".into()).into()),
+            ),
+            (
+                "verified".to_owned(),
+                DictEntry::Lit(Lit::Bool(true).into()),
+            ),
+            ("burgers".to_owned(), DictEntry::Lit(Lit::Num(152).into())),
+        ]
+        .into_iter()
+        .collect();
+        assert_eq!(dict, expected);
+    }
+}

From e7e799aca77c6e5110e69cdab1ef7c32af7aae6f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 16 Sep 2022 12:11:24 +0530
Subject: [PATCH 004/310] Re-implement dictionary folding algorithm

---
 server/src/engine/ql/ast.rs    |  14 +-
 server/src/engine/ql/lexer.rs  |  24 ++--
 server/src/engine/ql/mod.rs    |  10 +-
 server/src/engine/ql/schema.rs | 223 ++++++++++++++---------------
 server/src/engine/ql/tests.rs  | 250 +++++++++++++++++++++++++--------
 5 files changed, 320 insertions(+), 201 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 9d31246d..d00c1240 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -24,16 +24,17 @@
  *
 */
 
-use core::slice;
-
 use {
     super::{
-        dptr,
         lexer::{Kw, Lexer, Stmt, Token, Ty},
         schema, LangError, LangResult, RawSlice,
     },
     crate::util::{compiler, Life},
-    core::{marker::PhantomData, mem::transmute},
+    core::{
+        marker::PhantomData,
+        mem::{discriminant, transmute},
+        slice,
+    },
 };
 
 /*
@@ -238,13 +239,14 @@ impl<'a> Compiler<'a> {
     }
     #[inline(always)]
     pub(super) fn remaining(&self) -> usize {
-        dptr(self.c, self.e)
+        unsafe { self.e.offset_from(self.c) as usize }
     }
     pub(super) unsafe fn deref_cursor(&self) -> &Token {
         &*self.c
     }
     pub(super) fn peek_eq_and_forward(&mut self, t: &Token) -> bool {
-        let did_fw = self.not_exhausted() && unsafe { self.deref_cursor() == t };
+        let did_fw =
+            self.not_exhausted() && unsafe { discriminant(self.deref_cursor()) == discriminant(t) };
         unsafe {
             self.incr_cursor_if(did_fw);
         }
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index e40cc949..43bfd6d1 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -27,7 +27,7 @@
 use crate::util::Life;
 
 use {
-    super::{dptr, LangError, LangResult, RawSlice},
+    super::{LangError, LangResult, RawSlice},
     crate::util::compiler,
     core::{marker::PhantomData, slice, str},
 };
@@ -109,14 +109,15 @@ pub enum Stmt {
 #[repr(u8)]
 pub enum Kw {
     Stmt(Stmt),
-    Type(Ty),
+    TypeId(Ty),
     Space,
     Model,
+    Type,
 }
 
 impl From<Ty> for Kw {
     fn from(ty: Ty) -> Self {
-        Self::Type(ty)
+        Self::TypeId(ty)
     }
 }
 
@@ -131,10 +132,11 @@ impl Kw {
             b"alter" => Self::Stmt(Stmt::Alter).into(),
             b"space" => Self::Space.into(),
             b"model" => Self::Model.into(),
-            b"string" => Self::Type(Ty::String).into(),
-            b"binary" => Self::Type(Ty::Binary).into(),
-            b"list" => Self::Type(Ty::Ls).into(),
+            b"string" => Self::TypeId(Ty::String).into(),
+            b"binary" => Self::TypeId(Ty::Binary).into(),
+            b"list" => Self::TypeId(Ty::Ls).into(),
             b"true" => Token::Lit(Lit::Bool(true)),
+            b"type" => Kw::Type.into(),
             b"false" => Token::Lit(Lit::Bool(false)),
             _ => return None,
         };
@@ -196,7 +198,7 @@ impl<'a> Lexer<'a> {
     }
     #[inline(always)]
     fn remaining(&self) -> usize {
-        dptr(self.c, self.e)
+        unsafe { self.e.offset_from(self.c) as usize }
     }
     unsafe fn deref_cursor(&self) -> u8 {
         *self.cursor()
@@ -263,7 +265,7 @@ impl<'a> Lexer<'a> {
             while self.peek_is(|b| b.is_ascii_alphanumeric() || b == b'_') {
                 self.incr_cursor();
             }
-            RawSlice::new(s, dptr(s, self.cursor()))
+            RawSlice::new(s, self.cursor().offset_from(s) as usize)
         }
     }
     fn scan_ident_or_keyword(&mut self) {
@@ -288,7 +290,11 @@ impl<'a> Lexer<'a> {
             */
             static TERMINAL_CHAR: [u8; 6] = [b';', b'}', b',', b' ', b'\n', b'\t'];
             let wseof = self.peek_is(|b| TERMINAL_CHAR.contains(&b));
-            match str::from_utf8_unchecked(slice::from_raw_parts(s, dptr(s, self.cursor()))).parse()
+            match str::from_utf8_unchecked(slice::from_raw_parts(
+                s,
+                self.cursor().offset_from(s) as usize,
+            ))
+            .parse()
             {
                 Ok(num) if compiler::likely(wseof) => self.tokens.push(Token::Lit(Lit::Num(num))),
                 _ => self.last_error = Some(LangError::InvalidNumericLiteral),
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 87a0ba2c..81dfa732 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -30,7 +30,9 @@ pub(super) mod schema;
 #[cfg(test)]
 mod tests;
 
-use core::{fmt, mem, ops::Deref, slice};
+#[cfg(debug_assertions)]
+use core::{fmt, ops::Deref};
+use core::{mem, slice};
 
 /*
     Lang errors
@@ -49,6 +51,7 @@ pub enum LangError {
     UnexpectedEndofStatement,
     UnexpectedToken,
     InvalidDictionaryExpression,
+    InvalidTypeDefinition,
 }
 
 /*
@@ -117,8 +120,3 @@ impl From<&'static str> for RawSlice {
         unsafe { Self::new(st.as_bytes().as_ptr(), st.as_bytes().len()) }
     }
 }
-
-#[inline(always)]
-fn dptr<T>(s: *const T, e: *const T) -> usize {
-    e as usize - s as usize
-}
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 3f561f5b..6f385977 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -1,5 +1,5 @@
 /*
- * Created on Tue Sep 13 2022
+ * Created on Fri Sep 16 2022
  *
  * This file is a part of Skytable
  * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@@ -23,183 +23,168 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.
  *
 */
-
 /*
     Most grammar tools are pretty much "off the shelf" which makes some things incredible hard to achieve (such
     as custom error injection logic). To make things icier, Rust's integration with these tools (like lex) is not
     very "refined." Hence, it is best for us to implement our own parsers. In the future, I plan to optimize our
     rule checkers but that's not a concern at the moment.
 
-    -- Sayan (Sept. 14, 2022)
+    This module makes use of DFAs with additional flags, accepting a token stream as input to generate appropriate
+    structures, and have provable correctness. Hence, the unsafe code used here is correct, because the states are
+    only transitioned to if the input is accepted. If you do find otherwise, please file a bug report. The
+    transitions are currently very inefficient but can be made much faster.
+
+    TODO: The SMs can be reduced greatly, enocded to fixed-sized structures even, so do that
+    FIXME: For now, try and reduce reliance on additional flags (encoded into state?)
+
+    --
+    Sayan (@ohsayan)
+    Sept. 15, 2022
 */
 
 use {
     super::{
         ast::{Compiler, Statement},
-        lexer::{Lit, Token, Ty},
-        LangError, LangResult, RawSlice,
+        lexer::{Lit, Token},
+        LangResult, RawSlice,
+    },
+    std::{
+        collections::HashMap,
+        mem::{transmute, MaybeUninit},
     },
-    std::{collections::HashMap, mem::MaybeUninit, str},
 };
 
-macro_rules! boxed {
-    ([] $ty:ty) => {
-        ::std::boxed::Box::<[$ty]>
-    };
-}
-
 /*
     Meta
 */
 
-#[derive(Debug)]
-struct FieldMeta {
-    field_name: Option<RawSlice>,
-    unprocessed_layers: boxed![[] TypeConfig],
-}
-
-#[derive(Debug)]
-struct TypeConfig {
-    ty: Ty,
-    dict: Dict,
-}
-
-/*
-    Dictionary
-*/
-
-pub type Dict = HashMap<String, DictEntry>;
+const HIBIT: u64 = 1 << 63;
+const TRAIL_COMMA: bool = true;
 
 #[derive(Debug, PartialEq)]
 pub enum DictEntry {
     Lit(Lit),
-    Map(HashMap<String, Self>),
+    Map(HashMap<String, DictEntry>),
 }
 
-type NxFlag = u8;
-const EXP_COLON: NxFlag = 0x00;
-const EXP_LIT_OR_OBRC: NxFlag = 0x01;
-const EXP_COMMA_OR_CBRC: NxFlag = 0x02;
-const EXP_IDENT_OR_CBRC: NxFlag = 0x03;
-const EXP_EOF: NxFlag = 0x04;
-const EXP_START: NxFlag = 0x05;
-const HIBIT: u64 = 1 << 63;
-
-pub(super) fn parse_dictionary(c: &mut Compiler) -> LangResult<Dict> {
-    let mut dict = Dict::new();
-    let r = self::fold_dict(EXP_START, c.remslice(), &mut dict);
-    unsafe {
-        // IFEVERBROKEN: When the silicon guys decide to build a new chip with > 48 AS, lmk
-        c.incr_cursor_by((r & !HIBIT) as _);
+impl From<Lit> for DictEntry {
+    fn from(l: Lit) -> Self {
+        Self::Lit(l)
     }
-    if r & HIBIT == HIBIT {
-        Ok(dict)
-    } else {
-        Err(LangError::InvalidDictionaryExpression)
+}
+
+impl From<Dict> for DictEntry {
+    fn from(m: Dict) -> Self {
+        Self::Map(m)
     }
 }
 
-fn fold_dict<'a>(mut next: NxFlag, src: &'a [Token], dict: &mut Dict) -> u64 {
+pub type Dict = HashMap<String, DictEntry>;
+
+/*
+    Non-contextual dict
+*/
+
+type DictFoldState = u8;
+const DICT_STATE_FINAL: DictFoldState = 0xFF;
+const DICT_STATE_ACCEPT_OB: DictFoldState = 0x00;
+const DICT_STATE_ACCEPT_IDENT_OR_CB: DictFoldState = 0x01;
+const DICT_STATE_ACCEPT_COLON: DictFoldState = 0x02;
+const DICT_STATE_ACCEPT_LIT_OR_OB: DictFoldState = 0x03;
+const DICT_STATE_ACCEPT_COMMA_OR_CB: DictFoldState = 0x04;
+const DICT_STATE_ACCEPT_IDENT: DictFoldState = 0x05;
+
+fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dict) -> u64 {
     /*
-        NOTE: Assume respective <lit> validity and other character set rules
+        NOTE: Assume appropriate rule definitions wherever applicable
 
         <lbrace> ::= "{"
         <rbrace> ::= "}"
         <colon> ::= ":"
-        <ident> ::= (<sym_us> | <alpha>) (<alphanum> | <sym_us>)*
-        <dict> ::= <lbrace> (<ident> <colon> (<lit> | <dict> ) <comma>)* <rbrace>
+        <comma> ::= ","
+        <dict> ::= <lbrace> (<ident> <colon> (<lit> | <dict>) <comma> )* <comma>0*1 <rbace>
     */
     let mut i = 0;
+    let l = tok.len();
     let mut okay = true;
-    let mut tmp = MaybeUninit::uninit();
-    while i < src.len() && okay {
-        match (&src[i], next) {
-            // as a future optimization, note that this is just a single call
-            (Token::OpenBrace, EXP_START) => {
-                next = EXP_IDENT_OR_CBRC;
+    let mut tmp = MaybeUninit::<&str>::uninit();
+    while i < l && okay {
+        match (&tok[i], state) {
+            (Token::OpenBrace, DICT_STATE_ACCEPT_OB) => {
+                i += 1;
+                // next state is literal
+                state = DICT_STATE_ACCEPT_IDENT_OR_CB;
+            }
+            (Token::CloseBrace, DICT_STATE_ACCEPT_IDENT_OR_CB | DICT_STATE_ACCEPT_COMMA_OR_CB) => {
+                i += 1;
+                // since someone closed the brace, we're done processing this type
+                state = DICT_STATE_FINAL;
+                break;
             }
-            (Token::Ident(id), EXP_IDENT_OR_CBRC) => {
-                next = EXP_COLON;
-                tmp = MaybeUninit::new(unsafe {
-                    // UNSAFE(@ohsayan): If the token is an ident, the lexer guarantees that is valid unicode
-                    str::from_utf8_unchecked(id.as_slice())
-                });
+            (Token::Ident(key), DICT_STATE_ACCEPT_IDENT_OR_CB | DICT_STATE_ACCEPT_IDENT) => {
+                i += 1;
+                tmp = MaybeUninit::new(unsafe { transmute(key.as_slice()) });
+                state = DICT_STATE_ACCEPT_COLON;
+            }
+            (Token::Colon, DICT_STATE_ACCEPT_COLON) => {
+                i += 1;
+                state = DICT_STATE_ACCEPT_LIT_OR_OB;
             }
-            (Token::Colon, EXP_COLON) => next = EXP_LIT_OR_OBRC,
-            (Token::Lit(lit), EXP_LIT_OR_OBRC) => {
+            (Token::Lit(l), DICT_STATE_ACCEPT_LIT_OR_OB) => {
+                i += 1;
+                // insert this key/value pair
                 okay &= dict
                     .insert(
-                        unsafe {
-                            // UNSAFE(@ohsayan): This is completely safe because the transition and correctness
-                            // of this function makes it a consequence
-                            tmp.assume_init_ref()
-                        }
-                        .to_string(),
-                        DictEntry::Lit(lit.clone()),
+                        unsafe { tmp.assume_init_ref() }.to_string(),
+                        l.clone().into(),
                     )
                     .is_none();
-                next = EXP_COMMA_OR_CBRC;
+                state = DICT_STATE_ACCEPT_COMMA_OR_CB;
             }
-            (Token::OpenBrace, EXP_LIT_OR_OBRC) => {
-                // fold tokens
+            (Token::Comma, DICT_STATE_ACCEPT_COMMA_OR_CB) => {
+                i += 1; // since there is a comma, expect an ident
+                if TRAIL_COMMA {
+                    state = DICT_STATE_ACCEPT_IDENT_OR_CB;
+                } else {
+                    state = DICT_STATE_ACCEPT_IDENT;
+                }
+            }
+            (Token::OpenBrace, DICT_STATE_ACCEPT_LIT_OR_OB) => {
+                i += 1;
+                // there is another dictionary in here. let's parse it
                 let mut this_dict = Dict::new();
-                let read = self::fold_dict(EXP_IDENT_OR_CBRC, &src[i..], &mut this_dict);
-                i += (read & !HIBIT) as usize;
-                okay &= (read & HIBIT) == HIBIT;
+                let r = rfold_dict(DICT_STATE_ACCEPT_IDENT_OR_CB, &tok[i..], &mut this_dict);
                 okay &= dict
                     .insert(
-                        unsafe {
-                            // UNSAFE(@ohsayan): See above comment for context to know why this is safe
-                            tmp.assume_init_ref()
-                        }
-                        .to_string(),
+                        unsafe { tmp.assume_init_ref() }.to_string(),
                         DictEntry::Map(this_dict),
                     )
                     .is_none();
-                next = EXP_COMMA_OR_CBRC;
-            }
-            (Token::Comma, EXP_COMMA_OR_CBRC) => {
-                next = EXP_IDENT_OR_CBRC;
-            }
-            (Token::CloseBrace, EXP_COMMA_OR_CBRC | EXP_IDENT_OR_CBRC) => {
-                // graceful exit
-                next = EXP_EOF;
-                i += 1;
-                break;
+                okay &= r & HIBIT == HIBIT;
+                i += (r & !HIBIT) as usize;
+                // at the end of a dictionary, we expect a comma or brace close
+                state = DICT_STATE_ACCEPT_COMMA_OR_CB;
             }
             _ => {
                 okay = false;
                 break;
             }
         }
-        i += 1;
     }
-    okay &= next == EXP_EOF;
-    ((okay as u64) << 63) | i as u64
+    okay &= state == DICT_STATE_FINAL;
+    i as u64 | ((okay as u64) << 63)
 }
 
-fn parse_type_definition(_c: &mut Compiler) -> LangResult<boxed![[] TypeConfig]> {
-    /*
-        NOTE: Assume correct rules in context
-
-        <lbrace> ::= "{"
-        <rbrace> ::= "}"
-        <langle> ::= "<"
-        <rangle> ::= ">"
-        <colon> ::= ":"
-        <comma> ::= ","
-        <tydef_simple> ::= <type>
-        <tydef_nest> ::= <type> (<langle> <tydef_nest> <rangle>)*
-        <tydef_dict> ::= <type> <lbrace> ( ((<ident> <colon> <lit>) | <tydef_dict>) <comma>)* <rbrace>
-        <tydef> ::= <tydef_simple> | <tydef_nest> | <tydef_dict>
-    */
-    todo!()
-}
-
-fn parse_field(_c: &mut Compiler) -> LangResult<FieldMeta> {
-    todo!()
+pub fn fold_dict(tok: &[Token]) -> Option<Dict> {
+    let mut dict = Dict::new();
+    let r = rfold_dict(DICT_STATE_ACCEPT_OB, tok, &mut dict);
+    if r & HIBIT == HIBIT {
+        Some(dict)
+    } else {
+        None
+    }
 }
 
-pub(super) fn parse_schema(_c: &mut Compiler, _model: RawSlice) -> LangResult<Statement> {
+pub(crate) fn parse_schema(_c: &mut Compiler, _m: RawSlice) -> LangResult<Statement> {
     todo!()
 }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 13ee1b98..2f41c526 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -24,9 +24,42 @@
  *
 */
 
+use {
+    super::{
+        lexer::{Lexer, Token},
+        LangResult,
+    },
+    crate::util::Life,
+};
+
+#[cfg(test)]
+macro_rules! dict {
+    () => {
+        <::std::collections::HashMap<_, _> as ::core::default::Default>::default()
+    };
+    ($($key:expr => $value:expr),* $(,)?) => {{
+        let mut hm: ::std::collections::HashMap<_, _> = ::core::default::Default::default();
+        $(hm.insert($key.into(), $value.into());)*
+        hm
+    }};
+}
+
+macro_rules! multi_assert_eq {
+    ($($lhs:expr),* => $rhs:expr) => {
+        $(assert_eq!($lhs, $rhs);)*
+    };
+}
+
+fn lex(src: &[u8]) -> LangResult<Life<Vec<Token>>> {
+    Lexer::lex(src)
+}
+
 mod lexer_tests {
+    use super::{
+        super::lexer::{Lit, Token},
+        lex,
+    };
     use crate::engine::ql::LangError;
-    use super::super::lexer::{Lexer, Lit, Token};
 
     macro_rules! v(
         ($e:literal) => {{
@@ -40,37 +73,31 @@ mod lexer_tests {
     #[test]
     fn lex_ident() {
         let src = v!("hello");
-        assert_eq!(
-            Lexer::lex(&src).unwrap(),
-            vec![Token::Ident("hello".into())]
-        );
+        assert_eq!(lex(&src).unwrap(), vec![Token::Ident("hello".into())]);
     }
 
     // literals
     #[test]
     fn lex_number() {
         let number = v!("123456");
-        assert_eq!(
-            Lexer::lex(&number).unwrap(),
-            vec![Token::Lit(Lit::Num(123456))]
-        );
+        assert_eq!(lex(&number).unwrap(), vec![Token::Lit(Lit::Num(123456))]);
     }
     #[test]
     fn lex_bool() {
         let (t, f) = v!("true", "false");
-        assert_eq!(Lexer::lex(&t).unwrap(), vec![Token::Lit(Lit::Bool(true))]);
-        assert_eq!(Lexer::lex(&f).unwrap(), vec![Token::Lit(Lit::Bool(false))]);
+        assert_eq!(lex(&t).unwrap(), vec![Token::Lit(Lit::Bool(true))]);
+        assert_eq!(lex(&f).unwrap(), vec![Token::Lit(Lit::Bool(false))]);
     }
     #[test]
     fn lex_string() {
         let s = br#" "hello, world" "#;
         assert_eq!(
-            Lexer::lex(s).unwrap(),
+            lex(s).unwrap(),
             vec![Token::Lit(Lit::Str("hello, world".into()))]
         );
         let s = br#" 'hello, world' "#;
         assert_eq!(
-            Lexer::lex(s).unwrap(),
+            lex(s).unwrap(),
             vec![Token::Lit(Lit::Str("hello, world".into()))]
         );
     }
@@ -78,12 +105,12 @@ mod lexer_tests {
     fn lex_string_test_escape_quote() {
         let s = br#" "\"hello world\"" "#; // == "hello world"
         assert_eq!(
-            Lexer::lex(s).unwrap(),
+            lex(s).unwrap(),
             vec![Token::Lit(Lit::Str("\"hello world\"".into()))]
         );
         let s = br#" '\'hello world\'' "#; // == 'hello world'
         assert_eq!(
-            Lexer::lex(s).unwrap(),
+            lex(s).unwrap(),
             vec![Token::Lit(Lit::Str("'hello world'".into()))]
         );
     }
@@ -91,12 +118,12 @@ mod lexer_tests {
     fn lex_string_use_different_quote_style() {
         let s = br#" "he's on it" "#;
         assert_eq!(
-            Lexer::lex(s).unwrap(),
+            lex(s).unwrap(),
             vec![Token::Lit(Lit::Str("he's on it".into()))]
         );
         let s = br#" 'he thinks that "that girl" fixed it' "#;
         assert_eq!(
-            Lexer::lex(s).unwrap(),
+            lex(s).unwrap(),
             vec![Token::Lit(Lit::Str(
                 "he thinks that \"that girl\" fixed it".into()
             ))]
@@ -106,17 +133,17 @@ mod lexer_tests {
     fn lex_string_escape_bs() {
         let s = v!(r#" "windows has c:\\" "#);
         assert_eq!(
-            Lexer::lex(&s).unwrap(),
+            lex(&s).unwrap(),
             vec![Token::Lit(Lit::Str("windows has c:\\".into()))]
         );
         let s = v!(r#" 'windows has c:\\' "#);
         assert_eq!(
-            Lexer::lex(&s).unwrap(),
+            lex(&s).unwrap(),
             vec![Token::Lit(Lit::Str("windows has c:\\".into()))]
         );
         let lol = v!(r#"'\\\\\\\\\\'"#);
         assert_eq!(
-            Lexer::lex(&lol).unwrap(),
+            lex(&lol).unwrap(),
             vec![Token::Lit(Lit::Str("\\".repeat(5)))],
             "lol"
         )
@@ -124,59 +151,160 @@ mod lexer_tests {
     #[test]
     fn lex_string_bad_escape() {
         let wth = br#" '\a should be an alert on windows apparently' "#;
-        assert_eq!(
-            Lexer::lex(wth).unwrap_err(),
-            LangError::InvalidStringLiteral
-        );
+        assert_eq!(lex(wth).unwrap_err(), LangError::InvalidStringLiteral);
     }
     #[test]
     fn lex_string_unclosed() {
         let wth = br#" 'omg where did the end go "#;
-        assert_eq!(
-            Lexer::lex(wth).unwrap_err(),
-            LangError::InvalidStringLiteral
-        );
+        assert_eq!(lex(wth).unwrap_err(), LangError::InvalidStringLiteral);
         let wth = br#" 'see, we escaped the end\' "#;
-        assert_eq!(
-            Lexer::lex(wth).unwrap_err(),
-            LangError::InvalidStringLiteral
-        );
+        assert_eq!(lex(wth).unwrap_err(), LangError::InvalidStringLiteral);
     }
 }
 
 mod schema_tests {
-    use super::super::{
-        ast::Compiler,
-        lexer::{Lexer, Lit},
-        schema::{parse_dictionary, Dict, DictEntry},
-    };
+    use super::super::{lexer::Lit, schema};
+
+    macro_rules! fold_dict {
+        ($($input:expr),* $(,)?) => {
+            ($({schema::fold_dict(&super::lex($input).unwrap()).unwrap()}),*)
+        }
+    }
+
+    #[test]
+    fn dict_read_mini() {
+        let (d1, d2) = fold_dict! {
+            br#"{name: "sayan"}"#,
+            br#"{name: "sayan",}"#,
+        };
+        let r = dict!("name" => Lit::Str("sayan".into()));
+        multi_assert_eq!(d1, d2 => r);
+    }
     #[test]
     fn dict_read() {
-        let d = Lexer::lex(
+        let (d1, d2) = fold_dict! {
+            br#"
+                {
+                    name: "sayan",
+                    verified: true,
+                    burgers: 152
+                }
+            "#,
+            br#"
+                {
+                    name: "sayan",
+                    verified: true,
+                    burgers: 152,
+                }
+            "#,
+        };
+        let r = dict! (
+            "name" => Lit::Str("sayan".into()),
+            "verified" => Lit::Bool(true),
+            "burgers" => Lit::Num(152),
+        );
+        multi_assert_eq!(d1, d2 => r);
+    }
+    #[test]
+    fn dict_read_pro() {
+        let (d1, d2, d3) = fold_dict! {
+            br#"
+                {
+                    name: "sayan",
+                    notes: {
+                        burgers: "all the time, extra mayo",
+                        taco: true,
+                        pretzels: 1
+                    }
+                }
+            "#,
+            br#"
+                {
+                    name: "sayan",
+                    notes: {
+                        burgers: "all the time, extra mayo",
+                        taco: true,
+                        pretzels: 1,
+                    }
+                }
+            "#,
             br#"
-            {
-                name: "sayan",
-                verified: true,
-                burgers: 152
+                {
+                    name: "sayan",
+                    notes: {
+                        burgers: "all the time, extra mayo",
+                        taco: true,
+                        pretzels: 1,
+                },
+            }"#
+        };
+        multi_assert_eq!(
+            d1, d2, d3 => dict! {
+                "name" => Lit::Str("sayan".into()),
+                "notes" => dict! {
+                    "burgers" => Lit::Str("all the time, extra mayo".into()),
+                    "taco" => Lit::Bool(true),
+                    "pretzels" => Lit::Num(1),
+                }
             }
-        "#,
-        )
-        .unwrap();
-        let mut c = Compiler::new(&d);
-        let dict = parse_dictionary(&mut c).unwrap();
-        let expected: Dict = [
-            (
-                "name".to_owned(),
-                DictEntry::Lit(Lit::Str("sayan".into()).into()),
-            ),
-            (
-                "verified".to_owned(),
-                DictEntry::Lit(Lit::Bool(true).into()),
-            ),
-            ("burgers".to_owned(), DictEntry::Lit(Lit::Num(152).into())),
-        ]
-        .into_iter()
-        .collect();
-        assert_eq!(dict, expected);
+        );
+    }
+
+    #[test]
+    fn dict_read_pro_max() {
+        let (d1, d2, d3) = fold_dict! {
+            br#"
+                {
+                    well: {
+                        now: {
+                            this: {
+                                is: {
+                                    ridiculous: true
+                                }
+                            }
+                        }
+                    }
+                }
+            "#,
+            br#"
+                {
+                    well: {
+                        now: {
+                            this: {
+                                is: {
+                                    ridiculous: true,
+                                }
+                            }
+                        }
+                    }
+                }
+            "#,
+            br#"
+                {
+                    well: {
+                        now: {
+                            this: {
+                                is: {
+                                    ridiculous: true,
+                                },
+                            },
+                        },
+                    },
+                }
+            }"#
+        };
+        multi_assert_eq!(
+            d1, d2, d3 => dict! {
+                "well" => dict! {
+                    "now" => dict! {
+                        "this" => dict! {
+                            "is" => dict! {
+                                "ridiculous" => Lit::Bool(true),
+                            }
+                        }
+                    }
+                }
+            }
+        );
     }
 }

From 707f5c4d7eea20553b08e627d1b8f900ea5cbb15 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 16 Sep 2022 12:30:58 +0530
Subject: [PATCH 005/310] Add type metadata parsing

---
 server/src/engine/ql/schema.rs | 152 ++++++++++++++++++++++++++++++++-
 server/src/engine/ql/tests.rs  |  21 ++++-
 2 files changed, 171 insertions(+), 2 deletions(-)

diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 6f385977..2109903c 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -45,7 +45,7 @@
 use {
     super::{
         ast::{Compiler, Statement},
-        lexer::{Lit, Token},
+        lexer::{Kw, Lit, Token},
         LangResult, RawSlice,
     },
     std::{
@@ -185,6 +185,156 @@ pub fn fold_dict(tok: &[Token]) -> Option<Dict> {
     }
 }
 
+/*
+    Type metadata
+*/
+
+#[derive(Debug, PartialEq)]
+pub struct TypeMetaFoldResult {
+    c: usize,
+    m: [bool; 2],
+}
+
+impl TypeMetaFoldResult {
+    #[inline(always)]
+    const fn new() -> Self {
+        Self {
+            c: 0,
+            m: [true, false],
+        }
+    }
+    #[inline(always)]
+    fn incr(&mut self) {
+        self.incr_by(1);
+    }
+    #[inline(always)]
+    fn set_has_more(&mut self) {
+        self.m[1] = true;
+    }
+    #[inline(always)]
+    fn set_fail(&mut self) {
+        self.m[0] = false;
+    }
+    #[inline(always)]
+    pub fn pos(&self) -> usize {
+        self.c
+    }
+    #[inline(always)]
+    pub fn is_okay(&self) -> bool {
+        self.m[0]
+    }
+    #[inline(always)]
+    pub fn has_more(&self) -> bool {
+        self.m[1]
+    }
+    #[inline(always)]
+    fn record(&mut self, cond: bool) {
+        self.m[0] &= cond;
+    }
+    #[inline(always)]
+    fn incr_by(&mut self, pos: usize) {
+        self.c += pos;
+    }
+}
+
+type TypeMetaFoldState = u8;
+const TYMETA_STATE_FINAL: TypeMetaFoldState = 0xFF;
+const TYMETA_STATE_ACCEPT_IDENT: TypeMetaFoldState = 0x00;
+const TYMETA_STATE_ACCEPT_IDENT_OR_CB: TypeMetaFoldState = 0x01;
+const TYMETA_STATE_ACCEPT_COLON: TypeMetaFoldState = 0x02;
+const TYMETA_STATE_ACCEPT_LIT_OR_OB: TypeMetaFoldState = 0x03;
+const TYMETA_STATE_ACCEPT_COMMA_OR_CB: TypeMetaFoldState = 0x04;
+const TYMETA_STATE_ACCEPT_CB_OR_COMMA: TypeMetaFoldState = 0x05;
+
+pub(super) fn rfold_tymeta(
+    mut state: TypeMetaFoldState,
+    tok: &[Token],
+    dict: &mut Dict,
+) -> TypeMetaFoldResult {
+    let mut r = TypeMetaFoldResult::new();
+    let l = tok.len();
+    let mut tmp = MaybeUninit::<&str>::uninit();
+    while r.pos() < l && r.is_okay() {
+        match (&tok[r.pos()], state) {
+            (Token::Ident(id), TYMETA_STATE_ACCEPT_IDENT | TYMETA_STATE_ACCEPT_IDENT_OR_CB) => {
+                r.incr();
+                state = TYMETA_STATE_ACCEPT_COLON;
+                tmp = MaybeUninit::new(unsafe { transmute(id.as_slice()) });
+            }
+            (Token::Colon, TYMETA_STATE_ACCEPT_COLON) => {
+                r.incr();
+                state = TYMETA_STATE_ACCEPT_LIT_OR_OB;
+            }
+            (Token::Comma, TYMETA_STATE_ACCEPT_CB_OR_COMMA) => {
+                // we got a comma, so this should have more entries
+                r.incr();
+                state = TYMETA_STATE_ACCEPT_IDENT_OR_CB;
+            }
+            (Token::Lit(l), TYMETA_STATE_ACCEPT_LIT_OR_OB | TYMETA_STATE_ACCEPT_IDENT_OR_CB) => {
+                r.incr();
+                r.record(
+                    dict.insert(
+                        unsafe { tmp.assume_init_ref() }.to_string(),
+                        l.clone().into(),
+                    )
+                    .is_none(),
+                );
+                state = TYMETA_STATE_ACCEPT_COMMA_OR_CB;
+            }
+            (Token::OpenBrace, TYMETA_STATE_ACCEPT_LIT_OR_OB) => {
+                // found a nested dict. fold it
+                r.incr();
+                let this_ret = rfold_tymeta(TYMETA_STATE_ACCEPT_IDENT_OR_CB, &tok[r.pos()..], dict);
+                r.incr_by(this_ret.pos());
+                r.record(this_ret.is_okay());
+                if r.has_more() {
+                    // that's broken because L2 can NEVER have a typdef
+                    r.set_fail();
+                    break;
+                }
+                state = TYMETA_STATE_ACCEPT_COMMA_OR_CB;
+            }
+            (
+                Token::Keyword(Kw::Type),
+                TYMETA_STATE_ACCEPT_IDENT | TYMETA_STATE_ACCEPT_IDENT_OR_CB,
+            ) => {
+                // we found the type keyword inplace of a colon! increase depth
+                r.incr();
+                r.set_has_more();
+                state = TYMETA_STATE_FINAL;
+                break;
+            }
+            (
+                Token::CloseBrace,
+                TYMETA_STATE_ACCEPT_COMMA_OR_CB
+                | TYMETA_STATE_ACCEPT_IDENT_OR_CB
+                | TYMETA_STATE_ACCEPT_CB_OR_COMMA,
+            ) => {
+                r.incr();
+                // brace closed, so it's time to exit
+                state = TYMETA_STATE_FINAL;
+                break;
+            }
+            (Token::Comma, TYMETA_STATE_ACCEPT_COMMA_OR_CB | TYMETA_STATE_ACCEPT_IDENT_OR_CB) => {
+                r.incr();
+                if TRAIL_COMMA {
+                    state = TYMETA_STATE_ACCEPT_IDENT_OR_CB;
+                } else {
+                    // comma, so expect something ahead
+                    state = TYMETA_STATE_ACCEPT_IDENT;
+                }
+            }
+            _ => {
+                // in any other case, that is broken
+                r.set_fail();
+                break;
+            }
+        }
+    }
+    r.record(state == TYMETA_STATE_FINAL);
+    r
+}
+
 pub(crate) fn parse_schema(_c: &mut Compiler, _m: RawSlice) -> LangResult<Statement> {
     todo!()
 }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 2f41c526..ce1a9d4d 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -163,7 +163,10 @@ mod lexer_tests {
 }
 
 mod schema_tests {
-    use super::super::{lexer::Lit, schema};
+    use super::{
+        super::{lexer::Lit, schema},
+        lex,
+    };
 
     macro_rules! fold_dict {
         ($($input:expr),* $(,)?) => {
@@ -307,4 +310,20 @@ mod schema_tests {
             }
         );
     }
+    #[test]
+    fn rfold_tymeta_t() {
+        // list { maxlen: 100, type string }
+        //        [0]  [1][2][3][4]
+        let tok = lex(b"maxlen: 100, type string }").unwrap();
+        let mut d = dict! {};
+        let r = schema::rfold_tymeta(0x00, &tok, &mut d);
+        assert_eq!(r.pos(), 5);
+        assert!(r.is_okay());
+        assert_eq!(
+            d,
+            dict! {
+                "maxlen" => Lit::Num(100),
+            }
+        );
+    }
 }

From 103553ef2012d7ddcfd675af2748cd3de86cf5aa Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 16 Sep 2022 12:50:21 +0530
Subject: [PATCH 006/310] Add type folding algorithm

---
 server/src/engine/ql/macros.rs |  42 +++++++++
 server/src/engine/ql/mod.rs    |   2 +
 server/src/engine/ql/schema.rs | 150 +++++++++++++++++++++++++++------
 server/src/engine/ql/tests.rs  |  20 +----
 4 files changed, 169 insertions(+), 45 deletions(-)
 create mode 100644 server/src/engine/ql/macros.rs

diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
new file mode 100644
index 00000000..253059ce
--- /dev/null
+++ b/server/src/engine/ql/macros.rs
@@ -0,0 +1,42 @@
+/*
+ * Created on Fri Sep 16 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+macro_rules! dict {
+    () => {
+        <::std::collections::HashMap<_, _> as ::core::default::Default>::default()
+    };
+    ($($key:expr => $value:expr),* $(,)?) => {{
+        let mut hm: ::std::collections::HashMap<_, _> = ::core::default::Default::default();
+        $(hm.insert($key.into(), $value.into());)*
+        hm
+    }};
+}
+
+macro_rules! multi_assert_eq {
+    ($($lhs:expr),* => $rhs:expr) => {
+        $(assert_eq!($lhs, $rhs);)*
+    };
+}
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 81dfa732..99cac4aa 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -24,6 +24,8 @@
  *
 */
 
+#[macro_use]
+mod macros;
 pub(super) mod ast;
 pub(super) mod lexer;
 pub(super) mod schema;
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 2109903c..37412e8f 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -23,6 +23,7 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.
  *
 */
+
 /*
     Most grammar tools are pretty much "off the shelf" which makes some things incredible hard to achieve (such
     as custom error injection logic). To make things icier, Rust's integration with these tools (like lex) is not
@@ -45,7 +46,7 @@
 use {
     super::{
         ast::{Compiler, Statement},
-        lexer::{Kw, Lit, Token},
+        lexer::{Kw, Lit, Token, Ty},
         LangResult, RawSlice,
     },
     std::{
@@ -86,13 +87,14 @@ pub type Dict = HashMap<String, DictEntry>;
 */
 
 type DictFoldState = u8;
-const DICT_STATE_FINAL: DictFoldState = 0xFF;
-const DICT_STATE_ACCEPT_OB: DictFoldState = 0x00;
-const DICT_STATE_ACCEPT_IDENT_OR_CB: DictFoldState = 0x01;
-const DICT_STATE_ACCEPT_COLON: DictFoldState = 0x02;
-const DICT_STATE_ACCEPT_LIT_OR_OB: DictFoldState = 0x03;
-const DICT_STATE_ACCEPT_COMMA_OR_CB: DictFoldState = 0x04;
-const DICT_STATE_ACCEPT_IDENT: DictFoldState = 0x05;
+/// Start with this stat when you have already read an OB
+const DICTFOLD_STATE_INIT_IDENT_OR_CB: DictFoldState = 0x00;
+const DICTFOLD_STATE_FINAL: DictFoldState = 0xFF;
+const DICTFOLD_STATE_ACCEPT_OB: DictFoldState = 0x01;
+const DICTFOLD_STATE_ACCEPT_COLON: DictFoldState = 0x02;
+const DICTFOLD_STATE_ACCEPT_LIT_OR_OB: DictFoldState = 0x03;
+const DICTFOLD_STATE_ACCEPT_COMMA_OR_CB: DictFoldState = 0x04;
+const DICTFOLD_STATE_ACCEPT_IDENT: DictFoldState = 0x05;
 
 fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dict) -> u64 {
     /*
@@ -110,27 +112,27 @@ fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dict) -> u64 {
     let mut tmp = MaybeUninit::<&str>::uninit();
     while i < l && okay {
         match (&tok[i], state) {
-            (Token::OpenBrace, DICT_STATE_ACCEPT_OB) => {
+            (Token::OpenBrace, DICTFOLD_STATE_ACCEPT_OB) => {
                 i += 1;
                 // next state is literal
-                state = DICT_STATE_ACCEPT_IDENT_OR_CB;
+                state = DICTFOLD_STATE_INIT_IDENT_OR_CB;
             }
-            (Token::CloseBrace, DICT_STATE_ACCEPT_IDENT_OR_CB | DICT_STATE_ACCEPT_COMMA_OR_CB) => {
+            (Token::CloseBrace, DICTFOLD_STATE_INIT_IDENT_OR_CB) => {
                 i += 1;
                 // since someone closed the brace, we're done processing this type
-                state = DICT_STATE_FINAL;
+                state = DICTFOLD_STATE_FINAL;
                 break;
             }
-            (Token::Ident(key), DICT_STATE_ACCEPT_IDENT_OR_CB | DICT_STATE_ACCEPT_IDENT) => {
+            (Token::Ident(key), DICTFOLD_STATE_INIT_IDENT_OR_CB | DICTFOLD_STATE_ACCEPT_IDENT) => {
                 i += 1;
                 tmp = MaybeUninit::new(unsafe { transmute(key.as_slice()) });
-                state = DICT_STATE_ACCEPT_COLON;
+                state = DICTFOLD_STATE_ACCEPT_COLON;
             }
-            (Token::Colon, DICT_STATE_ACCEPT_COLON) => {
+            (Token::Colon, DICTFOLD_STATE_ACCEPT_COLON) => {
                 i += 1;
-                state = DICT_STATE_ACCEPT_LIT_OR_OB;
+                state = DICTFOLD_STATE_ACCEPT_LIT_OR_OB;
             }
-            (Token::Lit(l), DICT_STATE_ACCEPT_LIT_OR_OB) => {
+            (Token::Lit(l), DICTFOLD_STATE_ACCEPT_LIT_OR_OB) => {
                 i += 1;
                 // insert this key/value pair
                 okay &= dict
@@ -139,21 +141,21 @@ fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dict) -> u64 {
                         l.clone().into(),
                     )
                     .is_none();
-                state = DICT_STATE_ACCEPT_COMMA_OR_CB;
+                state = DICTFOLD_STATE_ACCEPT_COMMA_OR_CB;
             }
-            (Token::Comma, DICT_STATE_ACCEPT_COMMA_OR_CB) => {
+            (Token::Comma, DICTFOLD_STATE_ACCEPT_COMMA_OR_CB) => {
                 i += 1; // since there is a comma, expect an ident
                 if TRAIL_COMMA {
-                    state = DICT_STATE_ACCEPT_IDENT_OR_CB;
+                    state = DICTFOLD_STATE_INIT_IDENT_OR_CB;
                 } else {
-                    state = DICT_STATE_ACCEPT_IDENT;
+                    state = DICTFOLD_STATE_ACCEPT_IDENT;
                 }
             }
-            (Token::OpenBrace, DICT_STATE_ACCEPT_LIT_OR_OB) => {
+            (Token::OpenBrace, DICTFOLD_STATE_ACCEPT_LIT_OR_OB) => {
                 i += 1;
                 // there is another dictionary in here. let's parse it
                 let mut this_dict = Dict::new();
-                let r = rfold_dict(DICT_STATE_ACCEPT_IDENT_OR_CB, &tok[i..], &mut this_dict);
+                let r = rfold_dict(DICTFOLD_STATE_INIT_IDENT_OR_CB, &tok[i..], &mut this_dict);
                 okay &= dict
                     .insert(
                         unsafe { tmp.assume_init_ref() }.to_string(),
@@ -163,7 +165,7 @@ fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dict) -> u64 {
                 okay &= r & HIBIT == HIBIT;
                 i += (r & !HIBIT) as usize;
                 // at the end of a dictionary, we expect a comma or brace close
-                state = DICT_STATE_ACCEPT_COMMA_OR_CB;
+                state = DICTFOLD_STATE_ACCEPT_COMMA_OR_CB;
             }
             _ => {
                 okay = false;
@@ -171,13 +173,13 @@ fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dict) -> u64 {
             }
         }
     }
-    okay &= state == DICT_STATE_FINAL;
+    okay &= state == DICTFOLD_STATE_FINAL;
     i as u64 | ((okay as u64) << 63)
 }
 
 pub fn fold_dict(tok: &[Token]) -> Option<Dict> {
     let mut dict = Dict::new();
-    let r = rfold_dict(DICT_STATE_ACCEPT_OB, tok, &mut dict);
+    let r = rfold_dict(DICTFOLD_STATE_ACCEPT_OB, tok, &mut dict);
     if r & HIBIT == HIBIT {
         Some(dict)
     } else {
@@ -335,6 +337,102 @@ pub(super) fn rfold_tymeta(
     r
 }
 
+/*
+    Layer
+*/
+
+#[derive(Debug, PartialEq)]
+pub struct Layer {
+    ty: Ty,
+    props: Dict,
+}
+
+type LayerFoldState = u8;
+const LAYERFOLD_STATE_FINAL: LayerFoldState = 0xFF;
+const LAYERFOLD_STATE_ACCEPT_TYPE: LayerFoldState = 0x00;
+const LAYERFOLD_STATE_ACCEPT_OB_OR_END_ANY: LayerFoldState = 0x01;
+
+fn rfold_layers(tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
+    let mut i = 0;
+    let l = tok.len();
+    let mut okay = true;
+    let mut state = LAYERFOLD_STATE_ACCEPT_TYPE;
+    let mut meta = Dict::new();
+    let mut tmp = MaybeUninit::uninit();
+    while i < l && okay {
+        match (&tok[i], state) {
+            (Token::Keyword(Kw::TypeId(t)), LAYERFOLD_STATE_ACCEPT_TYPE) => {
+                i += 1;
+                tmp = MaybeUninit::new(*t);
+                state = LAYERFOLD_STATE_ACCEPT_OB_OR_END_ANY;
+            }
+            (Token::OpenBrace, LAYERFOLD_STATE_ACCEPT_OB_OR_END_ANY) => {
+                i += 1;
+                // get ty meta
+                let r = rfold_tymeta(TYMETA_STATE_ACCEPT_IDENT_OR_CB, &tok[i..], &mut meta);
+                okay &= r.is_okay();
+                i += r.pos();
+                if r.has_more() {
+                    // mmm, more layers
+                    let r = rfold_layers(&tok[i..], layers);
+                    okay &= r & HIBIT == HIBIT;
+                    i += (r & !HIBIT) as usize;
+                    // fold remaining meta (if this has a closebrace great; if not it *has* to have a comma to provide other meta)
+                    let ret = rfold_tymeta(TYMETA_STATE_ACCEPT_CB_OR_COMMA, &tok[i..], &mut meta);
+                    okay &= ret.is_okay();
+                    okay &= !ret.has_more(); // can't have two kinds
+                    i += ret.pos();
+                }
+                // since we ended a dictionary parse, the exact valid token after this could be anything
+                // since this is CFG, we don't care
+                state = LAYERFOLD_STATE_FINAL;
+                // push in this layer
+                layers.push(Layer {
+                    ty: unsafe { tmp.assume_init() },
+                    props: meta,
+                });
+                break;
+            }
+            (_, LAYERFOLD_STATE_ACCEPT_OB_OR_END_ANY) => {
+                // we don't care what token is here. we want this to end. also, DO NOT incr pos since we haven't
+                // seen this token
+                layers.push(Layer {
+                    ty: unsafe { tmp.assume_init() },
+                    props: meta,
+                });
+                state = LAYERFOLD_STATE_FINAL;
+                break;
+            }
+            _ => {
+                // in any other case, that is broken
+                okay = false;
+                break;
+            }
+        }
+    }
+    if state == LAYERFOLD_STATE_ACCEPT_OB_OR_END_ANY {
+        // if we've exited at this state, there was only one possible exit
+        layers.push(Layer {
+            ty: unsafe { tmp.assume_init() },
+            props: dict!(),
+        });
+        // safe exit
+    } else {
+        okay &= state == LAYERFOLD_STATE_FINAL;
+    }
+    (i as u64) | ((okay as u64) << 63)
+}
+
+pub fn fold_layers(tok: &[Token]) -> Option<Vec<Layer>> {
+    let mut l = Vec::new();
+    let r = rfold_layers(tok, &mut l);
+    if r & HIBIT == HIBIT {
+        Some(l)
+    } else {
+        None
+    }
+}
+
 pub(crate) fn parse_schema(_c: &mut Compiler, _m: RawSlice) -> LangResult<Statement> {
     todo!()
 }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index ce1a9d4d..3649f759 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -32,24 +32,6 @@ use {
     crate::util::Life,
 };
 
-#[cfg(test)]
-macro_rules! dict {
-    () => {
-        <::std::collections::HashMap<_, _> as ::core::default::Default>::default()
-    };
-    ($($key:expr => $value:expr),* $(,)?) => {{
-        let mut hm: ::std::collections::HashMap<_, _> = ::core::default::Default::default();
-        $(hm.insert($key.into(), $value.into());)*
-        hm
-    }};
-}
-
-macro_rules! multi_assert_eq {
-    ($($lhs:expr),* => $rhs:expr) => {
-        $(assert_eq!($lhs, $rhs);)*
-    };
-}
-
 fn lex(src: &[u8]) -> LangResult<Life<Vec<Token>>> {
     Lexer::lex(src)
 }
@@ -311,7 +293,7 @@ mod schema_tests {
         );
     }
     #[test]
-    fn rfold_tymeta_t() {
+    fn rfold_tymeta_stop_at_type() {
         // list { maxlen: 100, type string }
         //        [0]  [1][2][3][4]
         let tok = lex(b"maxlen: 100, type string }").unwrap();

From 8ffe6360df0b2c6135efca2be3bec15e26fb5fca Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 17 Sep 2022 09:33:14 +0530
Subject: [PATCH 007/310] Simplify states

---
 server/src/engine/ql/schema.rs | 434 ++++++++++++++++-----------------
 server/src/engine/ql/tests.rs  |  21 +-
 2 files changed, 212 insertions(+), 243 deletions(-)

diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 37412e8f..cf600ae8 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -59,13 +59,30 @@ use {
     Meta
 */
 
+/// This macro constructs states for our machine
+///
+/// **DO NOT** construct states manually
+macro_rules! states {
+    ($(#[$attr:meta])+$vis:vis struct $stateid:ident: $statebase:ty {$($(#[$tyattr:meta])* $state:ident = $statexp:expr),* $(,)?}) => {
+        #[::core::prelude::v1::derive(::core::cmp::PartialEq, ::core::cmp::Eq, ::core::clone::Clone, ::core::marker::Copy)]
+        $(#[$attr])+$vis struct $stateid {__base: $statebase}
+        impl $stateid {$($(#[$tyattr])* const $state:Self=$stateid{__base: $statexp,};)*}
+        impl ::core::fmt::Debug for $stateid {
+            fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
+                let r = match self.__base {$($statexp => ::core::stringify!($state),)* _ => panic!("invalid state"),};
+                ::core::write!(f, "{}::{}", ::core::stringify!($stateid), r)
+            }
+        }
+    }
+}
+
 const HIBIT: u64 = 1 << 63;
 const TRAIL_COMMA: bool = true;
 
 #[derive(Debug, PartialEq)]
 pub enum DictEntry {
     Lit(Lit),
-    Map(HashMap<String, DictEntry>),
+    Map(Dict),
 }
 
 impl From<Lit> for DictEntry {
@@ -75,97 +92,93 @@ impl From<Lit> for DictEntry {
 }
 
 impl From<Dict> for DictEntry {
-    fn from(m: Dict) -> Self {
-        Self::Map(m)
+    fn from(d: Dict) -> Self {
+        Self::Map(d)
     }
 }
 
 pub type Dict = HashMap<String, DictEntry>;
 
-/*
-    Non-contextual dict
-*/
-
-type DictFoldState = u8;
-/// Start with this stat when you have already read an OB
-const DICTFOLD_STATE_INIT_IDENT_OR_CB: DictFoldState = 0x00;
-const DICTFOLD_STATE_FINAL: DictFoldState = 0xFF;
-const DICTFOLD_STATE_ACCEPT_OB: DictFoldState = 0x01;
-const DICTFOLD_STATE_ACCEPT_COLON: DictFoldState = 0x02;
-const DICTFOLD_STATE_ACCEPT_LIT_OR_OB: DictFoldState = 0x03;
-const DICTFOLD_STATE_ACCEPT_COMMA_OR_CB: DictFoldState = 0x04;
-const DICTFOLD_STATE_ACCEPT_IDENT: DictFoldState = 0x05;
+#[derive(Debug, PartialEq)]
+pub struct Layer {
+    ty: Ty,
+    props: Dict,
+}
 
-fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dict) -> u64 {
-    /*
-        NOTE: Assume appropriate rule definitions wherever applicable
+states! {
+    /// The dict fold state
+    pub struct DictFoldState: u8 {
+        FINAL = 0xFF,
+        OB = 0x00,
+        CB_OR_IDENT = 0x01,
+        COLON = 0x02,
+        LIT_OR_OB = 0x03,
+        COMMA_OR_CB = 0x04,
+    }
+}
 
-        <lbrace> ::= "{"
-        <rbrace> ::= "}"
-        <colon> ::= ":"
-        <comma> ::= ","
-        <dict> ::= <lbrace> (<ident> <colon> (<lit> | <dict>) <comma> )* <comma>0*1 <rbace>
-    */
-    let mut i = 0;
+pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dict) -> u64 {
     let l = tok.len();
+    let mut i = 0;
     let mut okay = true;
     let mut tmp = MaybeUninit::<&str>::uninit();
-    while i < l && okay {
+
+    while i < l {
         match (&tok[i], state) {
-            (Token::OpenBrace, DICTFOLD_STATE_ACCEPT_OB) => {
+            (Token::OpenBrace, DictFoldState::OB) => {
                 i += 1;
-                // next state is literal
-                state = DICTFOLD_STATE_INIT_IDENT_OR_CB;
+                // we found a brace, expect a close brace or an ident
+                state = DictFoldState::CB_OR_IDENT;
             }
-            (Token::CloseBrace, DICTFOLD_STATE_INIT_IDENT_OR_CB) => {
+            (Token::CloseBrace, DictFoldState::CB_OR_IDENT | DictFoldState::COMMA_OR_CB) => {
+                // end of stream
                 i += 1;
-                // since someone closed the brace, we're done processing this type
-                state = DICTFOLD_STATE_FINAL;
+                state = DictFoldState::FINAL;
                 break;
             }
-            (Token::Ident(key), DICTFOLD_STATE_INIT_IDENT_OR_CB | DICTFOLD_STATE_ACCEPT_IDENT) => {
+            (Token::Ident(id), DictFoldState::CB_OR_IDENT) => {
+                // found ident, so expect colon
                 i += 1;
-                tmp = MaybeUninit::new(unsafe { transmute(key.as_slice()) });
-                state = DICTFOLD_STATE_ACCEPT_COLON;
+                tmp = MaybeUninit::new(unsafe { transmute(id.as_slice()) });
+                state = DictFoldState::COLON;
             }
-            (Token::Colon, DICTFOLD_STATE_ACCEPT_COLON) => {
+            (Token::Colon, DictFoldState::COLON) => {
+                // found colon, expect literal or openbrace
                 i += 1;
-                state = DICTFOLD_STATE_ACCEPT_LIT_OR_OB;
+                state = DictFoldState::LIT_OR_OB;
             }
-            (Token::Lit(l), DICTFOLD_STATE_ACCEPT_LIT_OR_OB) => {
+            (Token::Lit(l), DictFoldState::LIT_OR_OB) => {
                 i += 1;
-                // insert this key/value pair
+                // found literal; so push in k/v pair and then expect a comma or close brace
                 okay &= dict
                     .insert(
                         unsafe { tmp.assume_init_ref() }.to_string(),
                         l.clone().into(),
                     )
                     .is_none();
-                state = DICTFOLD_STATE_ACCEPT_COMMA_OR_CB;
+                state = DictFoldState::COMMA_OR_CB;
             }
-            (Token::Comma, DICTFOLD_STATE_ACCEPT_COMMA_OR_CB) => {
-                i += 1; // since there is a comma, expect an ident
-                if TRAIL_COMMA {
-                    state = DICTFOLD_STATE_INIT_IDENT_OR_CB;
-                } else {
-                    state = DICTFOLD_STATE_ACCEPT_IDENT;
-                }
+            // ONLY COMMA CAPTURE
+            (Token::Comma, DictFoldState::COMMA_OR_CB) => {
+                i += 1;
+                // we found a comma, expect a *strict* brace close or ident
+                state = DictFoldState::CB_OR_IDENT;
             }
-            (Token::OpenBrace, DICTFOLD_STATE_ACCEPT_LIT_OR_OB) => {
+            (Token::OpenBrace, DictFoldState::LIT_OR_OB) => {
                 i += 1;
-                // there is another dictionary in here. let's parse it
-                let mut this_dict = Dict::new();
-                let r = rfold_dict(DICTFOLD_STATE_INIT_IDENT_OR_CB, &tok[i..], &mut this_dict);
+                // we found an open brace, so this is a dict
+                let mut new_dict = Dict::new();
+                let ret = rfold_dict(DictFoldState::CB_OR_IDENT, &tok[i..], &mut new_dict);
+                okay &= ret & HIBIT == HIBIT;
+                i += (ret & !HIBIT) as usize;
                 okay &= dict
                     .insert(
                         unsafe { tmp.assume_init_ref() }.to_string(),
-                        DictEntry::Map(this_dict),
+                        new_dict.into(),
                     )
                     .is_none();
-                okay &= r & HIBIT == HIBIT;
-                i += (r & !HIBIT) as usize;
-                // at the end of a dictionary, we expect a comma or brace close
-                state = DICTFOLD_STATE_ACCEPT_COMMA_OR_CB;
+                // at the end of a dict we either expect a comma or close brace
+                state = DictFoldState::COMMA_OR_CB;
             }
             _ => {
                 okay = false;
@@ -173,264 +186,239 @@ fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dict) -> u64 {
             }
         }
     }
-    okay &= state == DICTFOLD_STATE_FINAL;
+    okay &= state == DictFoldState::FINAL;
     i as u64 | ((okay as u64) << 63)
 }
 
 pub fn fold_dict(tok: &[Token]) -> Option<Dict> {
-    let mut dict = Dict::new();
-    let r = rfold_dict(DICTFOLD_STATE_ACCEPT_OB, tok, &mut dict);
+    let mut d = Dict::new();
+    let r = rfold_dict(DictFoldState::OB, tok, &mut d);
     if r & HIBIT == HIBIT {
-        Some(dict)
+        Some(d)
     } else {
         None
     }
 }
 
-/*
-    Type metadata
-*/
+states! {
+    /// Type metadata fold state
+    pub struct TyMetaFoldState: u8 {
+        IDENT_OR_CB = 0x00,
+        COLON = 0x01,
+        LIT_OR_OB = 0x02,
+        COMMA_OR_CB = 0x03,
+        FINAL = 0xFF,
+    }
+}
 
-#[derive(Debug, PartialEq)]
-pub struct TypeMetaFoldResult {
+pub struct TyMetaFoldResult {
     c: usize,
-    m: [bool; 2],
+    b: [bool; 2],
 }
 
-impl TypeMetaFoldResult {
-    #[inline(always)]
+impl TyMetaFoldResult {
     const fn new() -> Self {
         Self {
             c: 0,
-            m: [true, false],
+            b: [true, false],
         }
     }
-    #[inline(always)]
     fn incr(&mut self) {
-        self.incr_by(1);
+        self.incr_by(1)
     }
-    #[inline(always)]
-    fn set_has_more(&mut self) {
-        self.m[1] = true;
+    fn incr_by(&mut self, by: usize) {
+        self.c += by;
     }
-    #[inline(always)]
     fn set_fail(&mut self) {
-        self.m[0] = false;
+        self.b[0] = false;
+    }
+    fn set_has_more(&mut self) {
+        self.b[1] = true;
     }
-    #[inline(always)]
     pub fn pos(&self) -> usize {
         self.c
     }
-    #[inline(always)]
-    pub fn is_okay(&self) -> bool {
-        self.m[0]
-    }
-    #[inline(always)]
     pub fn has_more(&self) -> bool {
-        self.m[1]
+        self.b[1]
     }
-    #[inline(always)]
-    fn record(&mut self, cond: bool) {
-        self.m[0] &= cond;
+    pub fn is_okay(&self) -> bool {
+        self.b[0]
     }
-    #[inline(always)]
-    fn incr_by(&mut self, pos: usize) {
-        self.c += pos;
+    fn record(&mut self, c: bool) {
+        self.b[0] &= c;
     }
 }
 
-type TypeMetaFoldState = u8;
-const TYMETA_STATE_FINAL: TypeMetaFoldState = 0xFF;
-const TYMETA_STATE_ACCEPT_IDENT: TypeMetaFoldState = 0x00;
-const TYMETA_STATE_ACCEPT_IDENT_OR_CB: TypeMetaFoldState = 0x01;
-const TYMETA_STATE_ACCEPT_COLON: TypeMetaFoldState = 0x02;
-const TYMETA_STATE_ACCEPT_LIT_OR_OB: TypeMetaFoldState = 0x03;
-const TYMETA_STATE_ACCEPT_COMMA_OR_CB: TypeMetaFoldState = 0x04;
-const TYMETA_STATE_ACCEPT_CB_OR_COMMA: TypeMetaFoldState = 0x05;
-
 pub(super) fn rfold_tymeta(
-    mut state: TypeMetaFoldState,
+    mut state: TyMetaFoldState,
     tok: &[Token],
     dict: &mut Dict,
-) -> TypeMetaFoldResult {
-    let mut r = TypeMetaFoldResult::new();
+) -> TyMetaFoldResult {
     let l = tok.len();
+    let mut r = TyMetaFoldResult::new();
     let mut tmp = MaybeUninit::<&str>::uninit();
     while r.pos() < l && r.is_okay() {
         match (&tok[r.pos()], state) {
-            (Token::Ident(id), TYMETA_STATE_ACCEPT_IDENT | TYMETA_STATE_ACCEPT_IDENT_OR_CB) => {
+            (Token::Keyword(Kw::Type), TyMetaFoldState::IDENT_OR_CB) => {
+                // we were expecting an ident but found the type keyword! increase depth
                 r.incr();
-                state = TYMETA_STATE_ACCEPT_COLON;
-                tmp = MaybeUninit::new(unsafe { transmute(id.as_slice()) });
+                r.set_has_more();
+                state = TyMetaFoldState::FINAL;
+                break;
+            }
+            (Token::CloseBrace, TyMetaFoldState::IDENT_OR_CB | TyMetaFoldState::COMMA_OR_CB) => {
+                r.incr();
+                // found close brace. end of stream
+                state = TyMetaFoldState::FINAL;
+                break;
             }
-            (Token::Colon, TYMETA_STATE_ACCEPT_COLON) => {
+            (Token::Ident(ident), TyMetaFoldState::IDENT_OR_CB) => {
                 r.incr();
-                state = TYMETA_STATE_ACCEPT_LIT_OR_OB;
+                tmp = MaybeUninit::new(unsafe { transmute(ident.as_slice()) });
+                // we just saw an ident, so we expect to see a colon
+                state = TyMetaFoldState::COLON;
             }
-            (Token::Comma, TYMETA_STATE_ACCEPT_CB_OR_COMMA) => {
-                // we got a comma, so this should have more entries
+            (Token::Colon, TyMetaFoldState::COLON) => {
                 r.incr();
-                state = TYMETA_STATE_ACCEPT_IDENT_OR_CB;
+                // we just saw a colon. now we want a literal or openbrace
+                state = TyMetaFoldState::LIT_OR_OB;
             }
-            (Token::Lit(l), TYMETA_STATE_ACCEPT_LIT_OR_OB | TYMETA_STATE_ACCEPT_IDENT_OR_CB) => {
+            (Token::Lit(lit), TyMetaFoldState::LIT_OR_OB) => {
                 r.incr();
                 r.record(
                     dict.insert(
                         unsafe { tmp.assume_init_ref() }.to_string(),
-                        l.clone().into(),
+                        lit.clone().into(),
                     )
                     .is_none(),
                 );
-                state = TYMETA_STATE_ACCEPT_COMMA_OR_CB;
+                // saw a literal. next is either comma or close brace
+                state = TyMetaFoldState::COMMA_OR_CB;
             }
-            (Token::OpenBrace, TYMETA_STATE_ACCEPT_LIT_OR_OB) => {
-                // found a nested dict. fold it
+            (Token::Comma, TyMetaFoldState::COMMA_OR_CB) => {
                 r.incr();
-                let this_ret = rfold_tymeta(TYMETA_STATE_ACCEPT_IDENT_OR_CB, &tok[r.pos()..], dict);
-                r.incr_by(this_ret.pos());
-                r.record(this_ret.is_okay());
-                if r.has_more() {
-                    // that's broken because L2 can NEVER have a typdef
-                    r.set_fail();
-                    break;
-                }
-                state = TYMETA_STATE_ACCEPT_COMMA_OR_CB;
+                // next is strictly a close brace or ident
+                state = TyMetaFoldState::IDENT_OR_CB;
             }
-            (
-                Token::Keyword(Kw::Type),
-                TYMETA_STATE_ACCEPT_IDENT | TYMETA_STATE_ACCEPT_IDENT_OR_CB,
-            ) => {
-                // we found the type keyword inplace of a colon! increase depth
+            (Token::OpenBrace, TyMetaFoldState::LIT_OR_OB) => {
                 r.incr();
-                r.set_has_more();
-                state = TYMETA_STATE_FINAL;
-                break;
-            }
-            (
-                Token::CloseBrace,
-                TYMETA_STATE_ACCEPT_COMMA_OR_CB
-                | TYMETA_STATE_ACCEPT_IDENT_OR_CB
-                | TYMETA_STATE_ACCEPT_CB_OR_COMMA,
-            ) => {
-                r.incr();
-                // brace closed, so it's time to exit
-                state = TYMETA_STATE_FINAL;
-                break;
+                // another dict in here
+                let mut d = Dict::new();
+                let ret = rfold_tymeta(TyMetaFoldState::IDENT_OR_CB, &tok[r.pos()..], &mut d);
+                r.incr_by(ret.pos());
+                r.record(ret.is_okay());
+                r.record(!ret.has_more()); // L2 cannot have type definitions
+                                           // end of definition or comma followed by something
+                r.record(
+                    dict.insert(unsafe { tmp.assume_init_ref() }.to_string(), d.into())
+                        .is_none(),
+                );
+                state = TyMetaFoldState::COMMA_OR_CB;
             }
-            (Token::Comma, TYMETA_STATE_ACCEPT_COMMA_OR_CB | TYMETA_STATE_ACCEPT_IDENT_OR_CB) => {
-                r.incr();
-                if TRAIL_COMMA {
-                    state = TYMETA_STATE_ACCEPT_IDENT_OR_CB;
-                } else {
-                    // comma, so expect something ahead
-                    state = TYMETA_STATE_ACCEPT_IDENT;
+            (found, exp) => {
+                if cfg!(debug_assertions) {
+                    panic!("found = `{:?}`, expected = `{:?}`", found, exp);
                 }
-            }
-            _ => {
-                // in any other case, that is broken
                 r.set_fail();
                 break;
             }
         }
     }
-    r.record(state == TYMETA_STATE_FINAL);
+    r.record(state == TyMetaFoldState::FINAL);
     r
 }
 
-/*
-    Layer
-*/
-
-#[derive(Debug, PartialEq)]
-pub struct Layer {
-    ty: Ty,
-    props: Dict,
+states! {
+    /// Layer fold state
+    pub struct LayerFoldState: u8 {
+        TY = 0x00,
+        END_OR_OB = 0x01,
+        FOLD_DICT_INCOMPLETE = 0x02,
+        FOLD_COMPLETED = 0xFF
+    }
 }
 
-type LayerFoldState = u8;
-const LAYERFOLD_STATE_FINAL: LayerFoldState = 0xFF;
-const LAYERFOLD_STATE_ACCEPT_TYPE: LayerFoldState = 0x00;
-const LAYERFOLD_STATE_ACCEPT_OB_OR_END_ANY: LayerFoldState = 0x01;
-
-fn rfold_layers(tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
-    let mut i = 0;
+pub(super) fn rfold_layers(tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
     let l = tok.len();
+    let mut i = 0;
     let mut okay = true;
-    let mut state = LAYERFOLD_STATE_ACCEPT_TYPE;
-    let mut meta = Dict::new();
+    let mut state = LayerFoldState::TY;
     let mut tmp = MaybeUninit::uninit();
+    let mut dict = Dict::new();
     while i < l && okay {
         match (&tok[i], state) {
-            (Token::Keyword(Kw::TypeId(t)), LAYERFOLD_STATE_ACCEPT_TYPE) => {
+            (Token::Keyword(Kw::TypeId(ty)), LayerFoldState::TY) => {
+                i += 1;
+                // expecting type, and found type. next is either end or an open brace or some arbitrary token
+                tmp = MaybeUninit::new(ty);
+                state = LayerFoldState::END_OR_OB;
+            }
+            (Token::OpenBrace, LayerFoldState::END_OR_OB) => {
                 i += 1;
-                tmp = MaybeUninit::new(*t);
-                state = LAYERFOLD_STATE_ACCEPT_OB_OR_END_ANY;
+                // since we found an open brace, this type has some meta
+                let ret = rfold_tymeta(TyMetaFoldState::IDENT_OR_CB, &tok[i..], &mut dict);
+                i += ret.pos();
+                okay &= ret.is_okay();
+                if ret.has_more() {
+                    // more layers
+                    let ret = rfold_layers(&tok[i..], layers);
+                    okay &= ret & HIBIT == HIBIT;
+                    i += (ret & !HIBIT) as usize;
+                    state = LayerFoldState::FOLD_DICT_INCOMPLETE;
+                } else if okay {
+                    // done folding dictionary. nothing more expected. break
+                    state = LayerFoldState::FOLD_COMPLETED;
+                    layers.push(Layer {
+                        ty: unsafe { tmp.assume_init() }.clone(),
+                        props: dict,
+                    });
+                    break;
+                }
             }
-            (Token::OpenBrace, LAYERFOLD_STATE_ACCEPT_OB_OR_END_ANY) => {
+            (Token::Comma, LayerFoldState::FOLD_DICT_INCOMPLETE) => {
+                // there is a comma at the end of this
                 i += 1;
-                // get ty meta
-                let r = rfold_tymeta(TYMETA_STATE_ACCEPT_IDENT_OR_CB, &tok[i..], &mut meta);
-                okay &= r.is_okay();
-                i += r.pos();
-                if r.has_more() {
-                    // mmm, more layers
-                    let r = rfold_layers(&tok[i..], layers);
-                    okay &= r & HIBIT == HIBIT;
-                    i += (r & !HIBIT) as usize;
-                    // fold remaining meta (if this has a closebrace great; if not it *has* to have a comma to provide other meta)
-                    let ret = rfold_tymeta(TYMETA_STATE_ACCEPT_CB_OR_COMMA, &tok[i..], &mut meta);
-                    okay &= ret.is_okay();
-                    okay &= !ret.has_more(); // can't have two kinds
-                    i += ret.pos();
+                let ret = rfold_tymeta(TyMetaFoldState::IDENT_OR_CB, &tok[i..], &mut dict);
+                i += ret.pos();
+                okay &= ret.is_okay();
+                okay &= !ret.has_more(); // not more than one type depth
+                if okay {
+                    // done folding dict successfully. nothing more expected. break.
+                    state = LayerFoldState::FOLD_COMPLETED;
+                    layers.push(Layer {
+                        ty: unsafe { tmp.assume_init() }.clone(),
+                        props: dict,
+                    });
+                    break;
                 }
-                // since we ended a dictionary parse, the exact valid token after this could be anything
-                // since this is CFG, we don't care
-                state = LAYERFOLD_STATE_FINAL;
-                // push in this layer
+            }
+            (Token::CloseBrace, LayerFoldState::FOLD_DICT_INCOMPLETE) => {
+                // end of stream
+                i += 1;
+                state = LayerFoldState::FOLD_COMPLETED;
                 layers.push(Layer {
-                    ty: unsafe { tmp.assume_init() },
-                    props: meta,
+                    ty: unsafe { tmp.assume_init() }.clone(),
+                    props: dict,
                 });
                 break;
             }
-            (_, LAYERFOLD_STATE_ACCEPT_OB_OR_END_ANY) => {
-                // we don't care what token is here. we want this to end. also, DO NOT incr pos since we haven't
-                // seen this token
+            (_, LayerFoldState::END_OR_OB) => {
+                // random arbitrary byte. finish append
+                state = LayerFoldState::FOLD_COMPLETED;
                 layers.push(Layer {
-                    ty: unsafe { tmp.assume_init() },
-                    props: meta,
+                    ty: unsafe { tmp.assume_init() }.clone(),
+                    props: dict,
                 });
-                state = LAYERFOLD_STATE_FINAL;
                 break;
             }
             _ => {
-                // in any other case, that is broken
                 okay = false;
                 break;
             }
         }
     }
-    if state == LAYERFOLD_STATE_ACCEPT_OB_OR_END_ANY {
-        // if we've exited at this state, there was only one possible exit
-        layers.push(Layer {
-            ty: unsafe { tmp.assume_init() },
-            props: dict!(),
-        });
-        // safe exit
-    } else {
-        okay &= state == LAYERFOLD_STATE_FINAL;
-    }
-    (i as u64) | ((okay as u64) << 63)
-}
-
-pub fn fold_layers(tok: &[Token]) -> Option<Vec<Layer>> {
-    let mut l = Vec::new();
-    let r = rfold_layers(tok, &mut l);
-    if r & HIBIT == HIBIT {
-        Some(l)
-    } else {
-        None
-    }
+    okay &= state == LayerFoldState::FOLD_COMPLETED;
+    i as u64 | ((okay as u64) << 63)
 }
 
 pub(crate) fn parse_schema(_c: &mut Compiler, _m: RawSlice) -> LangResult<Statement> {
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 3649f759..f4905a93 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -145,10 +145,7 @@ mod lexer_tests {
 }
 
 mod schema_tests {
-    use super::{
-        super::{lexer::Lit, schema},
-        lex,
-    };
+    use super::super::{lexer::Lit, schema};
 
     macro_rules! fold_dict {
         ($($input:expr),* $(,)?) => {
@@ -292,20 +289,4 @@ mod schema_tests {
             }
         );
     }
-    #[test]
-    fn rfold_tymeta_stop_at_type() {
-        // list { maxlen: 100, type string }
-        //        [0]  [1][2][3][4]
-        let tok = lex(b"maxlen: 100, type string }").unwrap();
-        let mut d = dict! {};
-        let r = schema::rfold_tymeta(0x00, &tok, &mut d);
-        assert_eq!(r.pos(), 5);
-        assert!(r.is_okay());
-        assert_eq!(
-            d,
-            dict! {
-                "maxlen" => Lit::Num(100),
-            }
-        );
-    }
 }

From a48da8b0caa9392e10cb31d6cafa1e6c1d766bb7 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 17 Sep 2022 11:30:51 +0530
Subject: [PATCH 008/310] Add tymeta tests

---
 server/src/engine/ql/schema.rs |  15 ++-
 server/src/engine/ql/tests.rs  | 209 ++++++++++++++++++++++++---------
 2 files changed, 161 insertions(+), 63 deletions(-)

diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index cf600ae8..7a462f83 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -63,10 +63,10 @@ use {
 ///
 /// **DO NOT** construct states manually
 macro_rules! states {
-    ($(#[$attr:meta])+$vis:vis struct $stateid:ident: $statebase:ty {$($(#[$tyattr:meta])* $state:ident = $statexp:expr),* $(,)?}) => {
+    ($(#[$attr:meta])+$vis:vis struct $stateid:ident: $statebase:ty {$($(#[$tyattr:meta])*$v:vis$state:ident = $statexp:expr),* $(,)?}) => {
         #[::core::prelude::v1::derive(::core::cmp::PartialEq, ::core::cmp::Eq, ::core::clone::Clone, ::core::marker::Copy)]
         $(#[$attr])+$vis struct $stateid {__base: $statebase}
-        impl $stateid {$($(#[$tyattr])* const $state:Self=$stateid{__base: $statexp,};)*}
+        impl $stateid {$($(#[$tyattr])*$v const $state:Self=$stateid{__base: $statexp,};)*}
         impl ::core::fmt::Debug for $stateid {
             fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
                 let r = match self.__base {$($statexp => ::core::stringify!($state),)* _ => panic!("invalid state"),};
@@ -315,10 +315,7 @@ pub(super) fn rfold_tymeta(
                 );
                 state = TyMetaFoldState::COMMA_OR_CB;
             }
-            (found, exp) => {
-                if cfg!(debug_assertions) {
-                    panic!("found = `{:?}`, expected = `{:?}`", found, exp);
-                }
+            _ => {
                 r.set_fail();
                 break;
             }
@@ -328,6 +325,12 @@ pub(super) fn rfold_tymeta(
     r
 }
 
+pub(super) fn fold_tymeta(tok: &[Token]) -> (TyMetaFoldResult, Dict) {
+    let mut d = Dict::new();
+    let r = rfold_tymeta(TyMetaFoldState::IDENT_OR_CB, tok, &mut d);
+    (r, d)
+}
+
 states! {
     /// Layer fold state
     pub struct LayerFoldState: u8 {
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index f4905a93..54b098e8 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -145,52 +145,57 @@ mod lexer_tests {
 }
 
 mod schema_tests {
-    use super::super::{lexer::Lit, schema};
+    use super::{
+        super::{lexer::Lit, schema},
+        lex,
+    };
+    mod dict {
+        use super::*;
 
-    macro_rules! fold_dict {
+        macro_rules! fold_dict {
         ($($input:expr),* $(,)?) => {
             ($({schema::fold_dict(&super::lex($input).unwrap()).unwrap()}),*)
         }
     }
 
-    #[test]
-    fn dict_read_mini() {
-        let (d1, d2) = fold_dict! {
-            br#"{name: "sayan"}"#,
-            br#"{name: "sayan",}"#,
-        };
-        let r = dict!("name" => Lit::Str("sayan".into()));
-        multi_assert_eq!(d1, d2 => r);
-    }
-    #[test]
-    fn dict_read() {
-        let (d1, d2) = fold_dict! {
-            br#"
+        #[test]
+        fn dict_read_mini() {
+            let (d1, d2) = fold_dict! {
+                br#"{name: "sayan"}"#,
+                br#"{name: "sayan",}"#,
+            };
+            let r = dict!("name" => Lit::Str("sayan".into()));
+            multi_assert_eq!(d1, d2 => r);
+        }
+        #[test]
+        fn dict_read() {
+            let (d1, d2) = fold_dict! {
+                br#"
                 {
                     name: "sayan",
                     verified: true,
                     burgers: 152
                 }
             "#,
-            br#"
+                br#"
                 {
                     name: "sayan",
                     verified: true,
                     burgers: 152,
                 }
             "#,
-        };
-        let r = dict! (
-            "name" => Lit::Str("sayan".into()),
-            "verified" => Lit::Bool(true),
-            "burgers" => Lit::Num(152),
-        );
-        multi_assert_eq!(d1, d2 => r);
-    }
-    #[test]
-    fn dict_read_pro() {
-        let (d1, d2, d3) = fold_dict! {
-            br#"
+            };
+            let r = dict! (
+                "name" => Lit::Str("sayan".into()),
+                "verified" => Lit::Bool(true),
+                "burgers" => Lit::Num(152),
+            );
+            multi_assert_eq!(d1, d2 => r);
+        }
+        #[test]
+        fn dict_read_pro() {
+            let (d1, d2, d3) = fold_dict! {
+                br#"
                 {
                     name: "sayan",
                     notes: {
@@ -200,7 +205,7 @@ mod schema_tests {
                     }
                 }
             "#,
-            br#"
+                br#"
                 {
                     name: "sayan",
                     notes: {
@@ -210,7 +215,7 @@ mod schema_tests {
                     }
                 }
             "#,
-            br#"
+                br#"
                 {
                     name: "sayan",
                     notes: {
@@ -219,23 +224,23 @@ mod schema_tests {
                         pretzels: 1,
                 },
             }"#
-        };
-        multi_assert_eq!(
-            d1, d2, d3 => dict! {
-                "name" => Lit::Str("sayan".into()),
-                "notes" => dict! {
-                    "burgers" => Lit::Str("all the time, extra mayo".into()),
-                    "taco" => Lit::Bool(true),
-                    "pretzels" => Lit::Num(1),
+            };
+            multi_assert_eq!(
+                d1, d2, d3 => dict! {
+                    "name" => Lit::Str("sayan".into()),
+                    "notes" => dict! {
+                        "burgers" => Lit::Str("all the time, extra mayo".into()),
+                        "taco" => Lit::Bool(true),
+                        "pretzels" => Lit::Num(1),
+                    }
                 }
-            }
-        );
-    }
+            );
+        }
 
-    #[test]
-    fn dict_read_pro_max() {
-        let (d1, d2, d3) = fold_dict! {
-            br#"
+        #[test]
+        fn dict_read_pro_max() {
+            let (d1, d2, d3) = fold_dict! {
+                br#"
                 {
                     well: {
                         now: {
@@ -248,7 +253,7 @@ mod schema_tests {
                     }
                 }
             "#,
-            br#"
+                br#"
                 {
                     well: {
                         now: {
@@ -261,7 +266,7 @@ mod schema_tests {
                     }
                 }
             "#,
-            br#"
+                br#"
                 {
                     well: {
                         now: {
@@ -274,19 +279,109 @@ mod schema_tests {
                     },
                 }
             }"#
-        };
-        multi_assert_eq!(
-            d1, d2, d3 => dict! {
-                "well" => dict! {
-                    "now" => dict! {
-                        "this" => dict! {
-                            "is" => dict! {
-                                "ridiculous" => Lit::Bool(true),
+            };
+            multi_assert_eq!(
+                d1, d2, d3 => dict! {
+                    "well" => dict! {
+                        "now" => dict! {
+                            "this" => dict! {
+                                "is" => dict! {
+                                    "ridiculous" => Lit::Bool(true),
+                                }
                             }
                         }
                     }
                 }
-            }
-        );
+            );
+        }
+    }
+    mod tymeta {
+        use super::*;
+        #[test]
+        fn tymeta_mini() {
+            let tok = lex(b"}").unwrap();
+            let (res, ret) = schema::fold_tymeta(&tok);
+            assert!(res.is_okay());
+            assert!(!res.has_more());
+            assert_eq!(res.pos(), 1);
+            assert_eq!(ret, dict!());
+        }
+        #[test]
+        fn tymeta_mini_fail() {
+            let tok = lex(b",}").unwrap();
+            let (res, ret) = schema::fold_tymeta(&tok);
+            assert!(!res.is_okay());
+            assert!(!res.has_more());
+            assert_eq!(res.pos(), 0);
+            assert_eq!(ret, dict!());
+        }
+        #[test]
+        fn tymeta() {
+            let tok = lex(br#"hello: "world", loading: true, size: 100 }"#).unwrap();
+            let (res, ret) = schema::fold_tymeta(&tok);
+            assert!(res.is_okay());
+            assert!(!res.has_more());
+            assert_eq!(res.pos(), tok.len());
+            assert_eq!(
+                ret,
+                dict! {
+                    "hello" => Lit::Str("world".into()),
+                    "loading" => Lit::Bool(true),
+                    "size" => Lit::Num(100)
+                }
+            );
+        }
+        #[test]
+        fn tymeta_pro() {
+            // list { maxlen: 100, type string, unique: true }
+            //        ^^^^^^^^^^^^^^^^^^ cursor should be at string
+            let tok = lex(br#"maxlen: 100, type string, unique: true }"#).unwrap();
+            let (res1, ret1) = schema::fold_tymeta(&tok);
+            assert!(res1.is_okay());
+            assert!(res1.has_more());
+            assert_eq!(res1.pos(), 5);
+            let remslice = &tok[res1.pos() + 2..];
+            let (res2, ret2) = schema::fold_tymeta(remslice);
+            assert!(res2.is_okay());
+            assert!(!res2.has_more());
+            assert_eq!(res2.pos() + res1.pos() + 2, tok.len());
+            let mut final_ret = ret1;
+            final_ret.extend(ret2);
+            assert_eq!(
+                final_ret,
+                dict! {
+                    "maxlen" => Lit::Num(100),
+                    "unique" => Lit::Bool(true)
+                }
+            )
+        }
+        #[test]
+        fn tymeta_pro_max() {
+            // list { maxlen: 100, this: { is: "cool" }, type string, unique: true }
+            //        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cursor should be at string
+            let tok =
+                lex(br#"maxlen: 100, this: { is: "cool" }, type string, unique: true }"#).unwrap();
+            let (res1, ret1) = schema::fold_tymeta(&tok);
+            assert!(res1.is_okay());
+            assert!(res1.has_more());
+            assert_eq!(res1.pos(), 13);
+            let remslice = &tok[res1.pos() + 2..];
+            let (res2, ret2) = schema::fold_tymeta(remslice);
+            assert!(res2.is_okay());
+            assert!(!res2.has_more());
+            assert_eq!(res2.pos() + res1.pos() + 2, tok.len());
+            let mut final_ret = ret1;
+            final_ret.extend(ret2);
+            assert_eq!(
+                final_ret,
+                dict! {
+                    "maxlen" => Lit::Num(100),
+                    "unique" => Lit::Bool(true),
+                    "this" => dict! {
+                        "is" => Lit::Str("cool".into())
+                    }
+                }
+            )
+        }
     }
 }

From 659ed8f6d96903eb2de989994f702d7307905cf7 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 17 Sep 2022 11:50:59 +0530
Subject: [PATCH 009/310] Add tests for layers

---
 server/src/engine/ql/schema.rs | 24 +++++++++
 server/src/engine/ql/tests.rs  | 91 ++++++++++++++++++++++++++++++++++
 2 files changed, 115 insertions(+)

diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 7a462f83..9815ecd1 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -105,6 +105,16 @@ pub struct Layer {
     props: Dict,
 }
 
+impl Layer {
+    pub(super) const fn new(ty: Ty, props: Dict) -> Self {
+        Self { ty, props }
+    }
+}
+
+/*
+    Context-free dict
+*/
+
 states! {
     /// The dict fold state
     pub struct DictFoldState: u8 {
@@ -200,6 +210,10 @@ pub fn fold_dict(tok: &[Token]) -> Option<Dict> {
     }
 }
 
+/*
+    Contextual dict (tymeta)
+*/
+
 states! {
     /// Type metadata fold state
     pub struct TyMetaFoldState: u8 {
@@ -331,6 +345,10 @@ pub(super) fn fold_tymeta(tok: &[Token]) -> (TyMetaFoldResult, Dict) {
     (r, d)
 }
 
+/*
+    Layer
+*/
+
 states! {
     /// Layer fold state
     pub struct LayerFoldState: u8 {
@@ -424,6 +442,12 @@ pub(super) fn rfold_layers(tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
     i as u64 | ((okay as u64) << 63)
 }
 
+pub(super) fn fold_layers(tok: &[Token]) -> (Vec<Layer>, usize, bool) {
+    let mut l = Vec::new();
+    let r = rfold_layers(tok, &mut l);
+    (l, (r & !HIBIT) as _, r & HIBIT == HIBIT)
+}
+
 pub(crate) fn parse_schema(_c: &mut Compiler, _m: RawSlice) -> LangResult<Statement> {
     todo!()
 }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 54b098e8..948a7383 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -384,4 +384,95 @@ mod schema_tests {
             )
         }
     }
+    mod layer {
+        use super::*;
+        use crate::engine::ql::{lexer::Ty, schema::Layer};
+        #[test]
+        fn layer_mini() {
+            let tok = lex(b"string)").unwrap();
+            let (layers, c, okay) = schema::fold_layers(&tok);
+            assert_eq!(c, tok.len() - 1);
+            assert!(okay);
+            assert_eq!(layers, vec![Layer::new(Ty::String, dict! {})]);
+        }
+        #[test]
+        fn layer() {
+            let tok = lex(b"string { maxlen: 100 }").unwrap();
+            let (layers, c, okay) = schema::fold_layers(&tok);
+            assert_eq!(c, tok.len());
+            assert!(okay);
+            assert_eq!(
+                layers,
+                vec![Layer::new(
+                    Ty::String,
+                    dict! {
+                        "maxlen" => Lit::Num(100)
+                    }
+                )]
+            );
+        }
+        #[test]
+        fn layer_plus() {
+            let tok = lex(b"list { type string }").unwrap();
+            let (layers, c, okay) = schema::fold_layers(&tok);
+            assert_eq!(c, tok.len());
+            assert!(okay);
+            assert_eq!(
+                layers,
+                vec![
+                    Layer::new(Ty::String, dict! {}),
+                    Layer::new(Ty::Ls, dict! {})
+                ]
+            );
+        }
+        #[test]
+        fn layer_pro() {
+            let tok = lex(b"list { unique: true, type string, maxlen: 10 }").unwrap();
+            let (layers, c, okay) = schema::fold_layers(&tok);
+            assert_eq!(c, tok.len());
+            assert!(okay);
+            assert_eq!(
+                layers,
+                vec![
+                    Layer::new(Ty::String, dict! {}),
+                    Layer::new(
+                        Ty::Ls,
+                        dict! {
+                            "unique" => Lit::Bool(true),
+                            "maxlen" => Lit::Num(10),
+                        }
+                    )
+                ]
+            );
+        }
+        #[test]
+        fn layer_pro_max() {
+            let tok = lex(
+                b"list { unique: true, type string { ascii_only: true, maxlen: 255 }, maxlen: 10 }",
+            )
+            .unwrap();
+            let (layers, c, okay) = schema::fold_layers(&tok);
+            assert_eq!(c, tok.len());
+            assert!(okay);
+            assert_eq!(
+                layers,
+                vec![
+                    Layer::new(
+                        Ty::String,
+                        dict! {
+                            "ascii_only" => Lit::Bool(true),
+                            "maxlen" => Lit::Num(255)
+                        }
+                    ),
+                    Layer::new(
+                        Ty::Ls,
+                        dict! {
+                            "unique" => Lit::Bool(true),
+                            "maxlen" => Lit::Num(10),
+                        }
+                    )
+                ]
+            );
+        }
+    }
 }

From 3b5bcdde4b9acf43666775277e7dfc96cd958c74 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 17 Sep 2022 11:53:17 +0530
Subject: [PATCH 010/310] Fix litnum parsing

---
 server/src/engine/ql/lexer.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 43bfd6d1..9aaaf6b6 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -289,7 +289,7 @@ impl<'a> Lexer<'a> {
                 1234a // invalid
             */
             static TERMINAL_CHAR: [u8; 6] = [b';', b'}', b',', b' ', b'\n', b'\t'];
-            let wseof = self.peek_is(|b| TERMINAL_CHAR.contains(&b));
+            let wseof = self.peek_is(|b| TERMINAL_CHAR.contains(&b)) || self.exhausted();
             match str::from_utf8_unchecked(slice::from_raw_parts(
                 s,
                 self.cursor().offset_from(s) as usize,

From e7170b1a5433aed578dbccc740e215ca5da34e98 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 17 Sep 2022 20:37:43 +0530
Subject: [PATCH 011/310] Add fuzzing for tokens

---
 server/src/engine/ql/lexer.rs  | 25 +++++----
 server/src/engine/ql/mod.rs    |  5 ++
 server/src/engine/ql/schema.rs |  2 +-
 server/src/engine/ql/tests.rs  | 98 ++++++++++++++++++++++++++++++++--
 server/src/util/mod.rs         |  4 +-
 server/src/util/test_utils.rs  | 38 +++++++++++++
 6 files changed, 157 insertions(+), 15 deletions(-)
 create mode 100644 server/src/util/test_utils.rs

diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 9aaaf6b6..e20ffb52 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -37,18 +37,21 @@ use {
 */
 
 #[derive(Debug)]
-#[cfg_attr(debug_assertions, derive(PartialEq))]
+#[cfg_attr(debug_assertions, derive(PartialEq, Clone))]
 #[repr(u8)]
 pub enum Token {
-    OpenParen,       // (
-    CloseParen,      // )
-    OpenAngular,     // <
-    CloseAngular,    // >
-    OpenSqBracket,   // [
-    CloseSqBracket,  // ]
-    OpenBrace,       // {
-    CloseBrace,      // }
-    Comma,           // ,
+    OpenParen,      // (
+    CloseParen,     // )
+    OpenAngular,    // <
+    CloseAngular,   // >
+    OpenSqBracket,  // [
+    CloseSqBracket, // ]
+    OpenBrace,      // {
+    CloseBrace,     // }
+    Comma,          // ,
+    #[cfg(test)]
+    /// A comma that can be ignored (used for fuzzing)
+    IgnorableComma,
     Colon,           // :
     Period,          // .
     Ident(RawSlice), // ident
@@ -353,6 +356,8 @@ impl<'a> Lexer<'a> {
             b'.' => Token::Period,
             b'=' => Token::OperatorEq,
             b'+' => Token::OperatorAdd,
+            #[cfg(test)]
+            b'\r' => Token::IgnorableComma,
             _ => {
                 self.last_error = Some(LangError::UnexpectedChar);
                 return;
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 99cac4aa..7bfec028 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -62,11 +62,16 @@ pub enum LangError {
 
 /// An unsafe, C-like slice that holds a ptr and length. Construction and usage is at the risk of the user
 #[cfg_attr(not(debug_assertions), derive(Debug))]
+#[cfg_attr(debug_assertions, derive(Clone))]
 pub struct RawSlice {
     ptr: *const u8,
     len: usize,
 }
 
+// again, caller's responsibility
+unsafe impl Send for RawSlice {}
+unsafe impl Sync for RawSlice {}
+
 impl RawSlice {
     const _EALIGN: () = assert!(mem::align_of::<Self>() == mem::align_of::<&[u8]>());
     const unsafe fn new(ptr: *const u8, len: usize) -> Self {
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 9815ecd1..28bc7d48 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -25,7 +25,7 @@
 */
 
 /*
-    Most grammar tools are pretty much "off the shelf" which makes some things incredible hard to achieve (such
+    Most grammar tools are pretty much "off the shelf" which makes some things incredibly hard to achieve (such
     as custom error injection logic). To make things icier, Rust's integration with these tools (like lex) is not
     very "refined." Hence, it is best for us to implement our own parsers. In the future, I plan to optimize our
     rule checkers but that's not a concern at the moment.
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 948a7383..69e2fb45 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -145,10 +145,57 @@ mod lexer_tests {
 }
 
 mod schema_tests {
-    use super::{
-        super::{lexer::Lit, schema},
-        lex,
+    use {
+        super::{
+            super::{
+                lexer::{Lit, Token},
+                schema,
+            },
+            lex,
+        },
+        crate::util::test_utils,
+        rand::{self, Rng},
     };
+
+    /// A very "basic" fuzzer that will randomly inject tokens wherever applicable
+    fn fuzz_tokens(src: &[Token], fuzzwith: impl Fn(bool, &[Token])) {
+        static FUZZ_TARGETS: [Token; 2] = [Token::Comma, Token::IgnorableComma];
+        let mut rng = rand::thread_rng();
+        #[inline(always)]
+        fn inject(new_src: &mut Vec<Token>, rng: &mut impl Rng) -> usize {
+            let start = new_src.len();
+            (0..test_utils::random_number(0, 5, rng)).for_each(|_| new_src.push(Token::Comma));
+            new_src.len() - start
+        }
+        let fuzz_amount = src.iter().filter(|tok| FUZZ_TARGETS.contains(tok)).count();
+        for _ in 0..(fuzz_amount.pow(2)) {
+            let mut new_src = Vec::with_capacity(src.len());
+            let mut should_pass = true;
+            src.iter().for_each(|tok| match tok {
+                Token::IgnorableComma => {
+                    let should_add = test_utils::random_bool(&mut rng);
+                    if should_add {
+                        new_src.push(Token::Comma);
+                    }
+                    let added = inject(&mut new_src, &mut rng);
+                    should_pass &= added <= !should_add as usize;
+                }
+                Token::Comma => {
+                    let should_add = test_utils::random_bool(&mut rng);
+                    if should_add {
+                        new_src.push(Token::Comma);
+                    } else {
+                        should_pass = false;
+                    }
+                    let added = inject(&mut new_src, &mut rng);
+                    should_pass &= added == !should_add as usize;
+                }
+                tok => new_src.push(tok.clone()),
+            });
+            fuzzwith(should_pass, &new_src);
+        }
+    }
+
     mod dict {
         use super::*;
 
@@ -294,6 +341,51 @@ mod schema_tests {
                 }
             );
         }
+
+        #[test]
+        fn fuzz_dict() {
+            let ret = lex(b"
+                {
+                    the_tradition_is: \"hello, world\",
+                    could_have_been: {
+                        this: true,
+                        or_maybe_this: 100,
+                        even_this: \"hello, universe!\"\r
+                    },
+                    but_oh_well: \"it continues to be the 'annoying' phrase\",
+                    lorem: {
+                        ipsum: {
+                            dolor: \"sit amet\"\r
+                        }\r
+                    }\r
+                }
+            ")
+            .unwrap();
+            let ret_dict = dict! {
+                "the_tradition_is" => Lit::Str("hello, world".into()),
+                "could_have_been" => dict! {
+                    "this" => Lit::Bool(true),
+                    "or_maybe_this" => Lit::Num(100),
+                    "even_this" => Lit::Str("hello, universe!".into()),
+                },
+                "but_oh_well" => Lit::Str("it continues to be the 'annoying' phrase".into()),
+                "lorem" => dict! {
+                    "ipsum" => dict! {
+                        "dolor" => Lit::Str("sit amet".into())
+                    }
+                }
+            };
+            fuzz_tokens(&ret, |should_pass, new_src| {
+                let r = schema::fold_dict(&new_src);
+                if should_pass {
+                    assert_eq!(r.unwrap(), ret_dict)
+                } else {
+                    if !r.is_none() {
+                        panic!("failure: {:?}", new_src);
+                    }
+                }
+            });
+        }
     }
     mod tymeta {
         use super::*;
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index ee7b139f..6cf7650a 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -28,6 +28,8 @@
 mod macros;
 pub mod compiler;
 pub mod error;
+#[cfg(test)]
+pub mod test_utils;
 pub mod os;
 use {
     crate::{
@@ -164,7 +166,7 @@ pub struct Life<'a, T> {
 impl<'a, T> Life<'a, T> {
     /// Ensure compile-time alignment (this is just a sanity check)
     const _ENSURE_COMPILETIME_ALIGN: () =
-        assert!(std::mem::align_of::<Life<Vec<u8>>>() == std::mem::align_of::<Vec<u8>>());
+        assert!(std::mem::align_of::<Life<T>>() == std::mem::align_of::<T>());
 
     #[inline(always)]
     pub const fn new(v: T) -> Self {
diff --git a/server/src/util/test_utils.rs b/server/src/util/test_utils.rs
new file mode 100644
index 00000000..62801faf
--- /dev/null
+++ b/server/src/util/test_utils.rs
@@ -0,0 +1,38 @@
+/*
+ * Created on Sat Sep 17 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use rand::{distributions::uniform::SampleUniform, Rng};
+
+// TODO(@ohsayan): Use my own PRNG algo here. Maybe my quadratic one?
+
+/// Generates a random boolean based on Bernoulli distributions
+pub fn random_bool(rng: &mut impl Rng) -> bool {
+    rng.gen_bool(0.5)
+}
+/// Generate a random number within the given range
+pub fn random_number<T: SampleUniform + PartialOrd>(max: T, min: T, rng: &mut impl Rng) -> T {
+    rng.gen_range(max..min)
+}

From 1d69751ccfafac063a54bd55323aa802cca1941b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 17 Sep 2022 20:59:39 +0530
Subject: [PATCH 012/310] Add fuzzing for tymeta

---
 server/src/engine/ql/tests.rs | 90 +++++++++++++++++++++++++++++++----
 1 file changed, 80 insertions(+), 10 deletions(-)

diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 69e2fb45..a7bb5104 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -173,22 +173,22 @@ mod schema_tests {
             let mut should_pass = true;
             src.iter().for_each(|tok| match tok {
                 Token::IgnorableComma => {
-                    let should_add = test_utils::random_bool(&mut rng);
-                    if should_add {
+                    let did_add = test_utils::random_bool(&mut rng);
+                    if did_add {
                         new_src.push(Token::Comma);
                     }
                     let added = inject(&mut new_src, &mut rng);
-                    should_pass &= added <= !should_add as usize;
+                    should_pass &= added <= !did_add as usize;
                 }
                 Token::Comma => {
-                    let should_add = test_utils::random_bool(&mut rng);
-                    if should_add {
+                    let did_add = test_utils::random_bool(&mut rng);
+                    if did_add {
                         new_src.push(Token::Comma);
                     } else {
                         should_pass = false;
                     }
                     let added = inject(&mut new_src, &mut rng);
-                    should_pass &= added == !should_add as usize;
+                    should_pass &= added == !did_add as usize;
                 }
                 tok => new_src.push(tok.clone()),
             });
@@ -379,16 +379,18 @@ mod schema_tests {
                 let r = schema::fold_dict(&new_src);
                 if should_pass {
                     assert_eq!(r.unwrap(), ret_dict)
-                } else {
-                    if !r.is_none() {
-                        panic!("failure: {:?}", new_src);
-                    }
+                } else if r.is_some() {
+                    panic!(
+                        "expected failure, but passed for token stream: `{:?}`",
+                        new_src
+                    );
                 }
             });
         }
     }
     mod tymeta {
         use super::*;
+        use crate::engine::ql::lexer::{Kw, Ty};
         #[test]
         fn tymeta_mini() {
             let tok = lex(b"}").unwrap();
@@ -475,6 +477,74 @@ mod schema_tests {
                 }
             )
         }
+        #[test]
+        fn fuzz_tymeta_normal() {
+            // { maxlen: 10, unique: true, user: "sayan" }
+            //   ^start
+            let tok = lex(b"
+                    maxlen: 10,
+                    unique: true,
+                    auth: {
+                        maybe: true\r
+                    },
+                    user: \"sayan\"\r
+                }
+            ")
+            .unwrap();
+            let expected = dict! {
+                "maxlen" => Lit::Num(10),
+                "unique" => Lit::Bool(true),
+                "auth" => dict! {
+                    "maybe" => Lit::Bool(true),
+                },
+                "user" => Lit::Str("sayan".into())
+            };
+            fuzz_tokens(&tok, |should_pass, new_src| {
+                let (ret, dict) = schema::fold_tymeta(&tok);
+                if should_pass {
+                    assert!(ret.is_okay());
+                    assert!(!ret.has_more());
+                    assert_eq!(ret.pos(), new_src.len());
+                    assert_eq!(dict, expected);
+                } else if ret.is_okay() {
+                    panic!("Expected failure but passed for token stream: `{:?}`", tok);
+                }
+            });
+        }
+        #[test]
+        fn fuzz_tymeta_with_ty() {
+            // list { maxlen: 10, unique: true, type string, user: "sayan" }
+            //   ^start
+            let tok = lex(b"
+                    maxlen: 10,
+                    unique: true,
+                    auth: {
+                        maybe: true\r
+                    },
+                    type string,
+                    user: \"sayan\"\r
+                }
+            ")
+            .unwrap();
+            let expected = dict! {
+                "maxlen" => Lit::Num(10),
+                "unique" => Lit::Bool(true),
+                "auth" => dict! {
+                    "maybe" => Lit::Bool(true),
+                },
+            };
+            fuzz_tokens(&tok, |should_pass, new_src| {
+                let (ret, dict) = schema::fold_tymeta(&tok);
+                if should_pass {
+                    assert!(ret.is_okay());
+                    assert!(ret.has_more());
+                    assert!(new_src[ret.pos()] == Token::Keyword(Kw::TypeId(Ty::String)));
+                    assert_eq!(dict, expected);
+                } else if ret.is_okay() {
+                    panic!("Expected failure but passed for token stream: `{:?}`", tok);
+                }
+            });
+        }
     }
     mod layer {
         use super::*;

From d0a548a59add36ad65c22e44ce7040598aa055c2 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 17 Sep 2022 21:11:41 +0530
Subject: [PATCH 013/310] Add fuzzing for layers

---
 server/src/engine/ql/tests.rs | 42 ++++++++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index a7bb5104..d8ef9d11 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -507,7 +507,10 @@ mod schema_tests {
                     assert_eq!(ret.pos(), new_src.len());
                     assert_eq!(dict, expected);
                 } else if ret.is_okay() {
-                    panic!("Expected failure but passed for token stream: `{:?}`", tok);
+                    panic!(
+                        "Expected failure but passed for token stream: `{:?}`",
+                        new_src
+                    );
                 }
             });
         }
@@ -636,5 +639,42 @@ mod schema_tests {
                 ]
             );
         }
+
+        #[test]
+        fn fuzz_layer() {
+            let tok = lex(b"
+            list {
+                type list {
+                    maxlen: 100,
+                    type string\r
+                },
+                unique: true\r
+            }
+        ")
+            .unwrap();
+            let expected = vec![
+                Layer::new(Ty::String, dict!()),
+                Layer::new(
+                    Ty::Ls,
+                    dict! {
+                        "maxlen" => Lit::Num(100),
+                    },
+                ),
+                Layer::new(Ty::Ls, dict!("unique" => Lit::Bool(true))),
+            ];
+            fuzz_tokens(&tok, |should_pass, new_tok| {
+                let (layers, c, okay) = schema::fold_layers(&new_tok);
+                if should_pass {
+                    assert!(okay);
+                    assert_eq!(c, new_tok.len());
+                    assert_eq!(layers, expected);
+                } else if okay {
+                    panic!(
+                        "expected failure but passed for token stream: `{:?}`",
+                        new_tok
+                    );
+                }
+            });
+        }
     }
 }

From e8aaccdb3260114832de1fb8be468c2bd519188e Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 19 Sep 2022 20:58:18 +0530
Subject: [PATCH 014/310] Improve kw/sym matching algorithms

---
 server/src/engine/ql/ast.rs    |  79 ++-----
 server/src/engine/ql/lexer.rs  | 386 ++++++++++++++++++++++++---------
 server/src/engine/ql/macros.rs |  10 +
 server/src/engine/ql/schema.rs |  43 ++--
 server/src/engine/ql/tests.rs  |  43 ++--
 5 files changed, 360 insertions(+), 201 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index d00c1240..381b4fc2 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -26,66 +26,17 @@
 
 use {
     super::{
-        lexer::{Kw, Lexer, Stmt, Token, Ty},
+        lexer::{DdlKeyword, Keyword, Lexer, Symbol, Token},
         schema, LangError, LangResult, RawSlice,
     },
-    crate::util::{compiler, Life},
-    core::{
-        marker::PhantomData,
-        mem::{discriminant, transmute},
-        slice,
-    },
+    crate::util::Life,
+    core::{marker::PhantomData, mem::discriminant, slice},
 };
 
 /*
     AST
 */
 
-#[derive(Debug, PartialEq)]
-pub struct TypeExpr(Vec<Ty>);
-
-#[repr(u8)]
-#[derive(Debug)]
-enum FTy {
-    String = 0,
-    Binary = 1,
-}
-
-#[derive(Debug)]
-struct TypeDefintion {
-    d: usize,
-    b: Ty,
-    f: FTy,
-}
-
-impl TypeDefintion {
-    const PRIM: usize = 1;
-    pub fn eval(s: TypeExpr) -> LangResult<Self> {
-        let TypeExpr(ex) = s;
-        let l = ex.len();
-        #[inline(always)]
-        fn ls(t: &Ty) -> bool {
-            *t == Ty::Ls
-        }
-        let d = ex.iter().map(|v| ls(v) as usize).sum::<usize>();
-        let v = (l == 1 && ex[0] != Ty::Ls) || (l > 1 && (d == l - 1) && ex[l - 1] != Ty::Ls);
-        if compiler::likely(v) {
-            unsafe {
-                Ok(Self {
-                    d: d + 1,
-                    b: ex[0],
-                    f: transmute(ex[l - 1]),
-                })
-            }
-        } else {
-            compiler::cold_err(Err(LangError::InvalidTypeExpression))
-        }
-    }
-    pub const fn is_prim(&self) -> bool {
-        self.d == Self::PRIM
-    }
-}
-
 pub enum Entity {
     Current(RawSlice),
     Partial(RawSlice),
@@ -98,7 +49,11 @@ impl Entity {
         let b = cm.nxtok_nofw_opt();
         let c = cm.nxtok_nofw_opt();
         match (a, b, c) {
-            (Some(Token::Ident(ks)), Some(Token::Period), Some(Token::Ident(tbl))) => unsafe {
+            (
+                Some(Token::Ident(ks)),
+                Some(Token::Symbol(Symbol::SymPeriod)),
+                Some(Token::Ident(tbl)),
+            ) => unsafe {
                 let r = Ok(Entity::Full(ks.raw_clone(), tbl.raw_clone()));
                 cm.incr_cursor_by(3);
                 r
@@ -108,7 +63,7 @@ impl Entity {
                 cm.incr_cursor();
                 r
             },
-            (Some(Token::Colon), Some(Token::Ident(tbl)), _) => unsafe {
+            (Some(Token::Symbol(Symbol::SymColon)), Some(Token::Ident(tbl)), _) => unsafe {
                 let r = Ok(Entity::Partial(tbl.raw_clone()));
                 cm.incr_cursor_by(2);
                 r
@@ -152,21 +107,19 @@ impl<'a> Compiler<'a> {
     #[inline(always)]
     fn stage0(&mut self) -> Result<Statement, LangError> {
         match self.nxtok_opt() {
-            Some(Token::Keyword(Kw::Stmt(stmt))) => match stmt {
-                Stmt::Create => self.create0(),
-                Stmt::Drop => self.drop0(),
-                Stmt::Alter => self.alter0(),
-                Stmt::Inspect => self.inspect0(),
-                Stmt::Use => self.use0(),
-            },
+            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Create))) => self.create0(),
+            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Drop))) => self.drop0(),
+            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Alter))) => self.alter0(),
+            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Inspect))) => self.inspect0(),
+            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Use))) => self.use0(),
             _ => Err(LangError::ExpectedStatement),
         }
     }
     #[inline(always)]
     fn create0(&mut self) -> Result<Statement, LangError> {
         match self.nxtok_opt() {
-            Some(Token::Keyword(Kw::Model)) => self.c_model0(),
-            Some(Token::Keyword(Kw::Space)) => self.c_space0(),
+            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Model))) => self.c_model0(),
+            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Space))) => self.c_space0(),
             _ => Err(LangError::UnexpectedEndofStatement),
         }
     }
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index e20ffb52..d7172c89 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -24,12 +24,14 @@
  *
 */
 
-use crate::util::Life;
-
 use {
     super::{LangError, LangResult, RawSlice},
-    crate::util::compiler,
-    core::{marker::PhantomData, slice, str},
+    crate::util::{compiler, Life},
+    core::{
+        marker::PhantomData,
+        mem::{size_of, transmute},
+        slice, str,
+    },
 };
 
 /*
@@ -38,121 +40,315 @@ use {
 
 #[derive(Debug)]
 #[cfg_attr(debug_assertions, derive(PartialEq, Clone))]
-#[repr(u8)]
 pub enum Token {
-    OpenParen,      // (
-    CloseParen,     // )
-    OpenAngular,    // <
-    CloseAngular,   // >
-    OpenSqBracket,  // [
-    CloseSqBracket, // ]
-    OpenBrace,      // {
-    CloseBrace,     // }
-    Comma,          // ,
+    Symbol(Symbol),
+    Keyword(Keyword),
+    Ident(RawSlice),
     #[cfg(test)]
     /// A comma that can be ignored (used for fuzzing)
     IgnorableComma,
-    Colon,           // :
-    Period,          // .
-    Ident(RawSlice), // ident
-    Keyword(Kw),     // kw
-    OperatorEq,      // =
-    OperatorAdd,     // +
-    Lit(Lit),        // literal
+    Lit(Lit), // literal
+}
+
+assertions! {
+    size_of::<Token>() == 32, // FIXME(@ohsayan): Damn, what?
+    size_of::<Symbol>() == 1,
+    size_of::<Keyword>() == 2,
+    size_of::<Lit>() == 24, // FIXME(@ohsayan): Ouch
 }
 
-impl From<Kw> for Token {
-    fn from(kw: Kw) -> Self {
-        Self::Keyword(kw)
+enum_impls! {
+    Token => {
+        Keyword as Keyword,
+        Symbol as Symbol,
+        Lit as Lit,
     }
 }
 
 #[derive(Debug, PartialEq, Clone)]
+#[repr(u8)]
 pub enum Lit {
-    Str(String),
+    Str(Box<str>),
     Bool(bool),
     Num(u64),
 }
 
-impl From<Lit> for Token {
-    fn from(l: Lit) -> Self {
-        Self::Lit(l)
+enum_impls! {
+    Lit => {
+        Box<str> as Str,
+        bool as Bool,
+        u64 as Num,
     }
 }
 
-impl From<u64> for Lit {
-    fn from(n: u64) -> Self {
-        Self::Num(n)
-    }
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum Symbol {
+    OpArithmeticAdd,  // +
+    OpArithmeticSub,  // -
+    OpArithmeticMul,  // *
+    OpArithmeticDiv,  // /
+    OpLogicalNot,     // !
+    OpLogicalAnd,     // &
+    OpLogicalXor,     // ^
+    OpLogicalOr,      // |
+    OpAssign,         // =
+    TtOpenParen,      // (
+    TtCloseParen,     // )
+    TtOpenSqBracket,  // [
+    TtCloseSqBracket, // ]
+    TtOpenBrace,      // {
+    TtCloseBrace,     // }
+    OpComparatorLt,   // <
+    OpComparatorGt,   // >
+    QuoteS,           // '
+    QuoteD,           // "
+    SymAt,            // @
+    SymHash,          // #
+    SymDollar,        // $
+    SymPercent,       // %
+    SymUnderscore,    // _
+    SymBackslash,     // \
+    SymColon,         // :
+    SymSemicolon,     // ;
+    SymComma,         // ,
+    SymPeriod,        // .
+    SymQuestion,      // ?
+    SymTilde,         // ~
+    SymAccent,        // `
 }
 
-impl From<String> for Lit {
-    fn from(s: String) -> Self {
-        Self::Str(s)
-    }
+#[derive(Debug, Copy, Clone, PartialEq)]
+#[repr(u8)]
+pub enum Keyword {
+    Ddl(DdlKeyword),
+    DdlMisc(DdlMiscKeyword),
+    Dml(DmlKeyword),
+    DmlMisc(DmlMiscKeyword),
+    TypeId(Type),
 }
 
-impl From<bool> for Lit {
-    fn from(b: bool) -> Self {
-        Self::Bool(b)
+enum_impls! {
+    Keyword => {
+        DdlKeyword as Ddl,
+        DdlMiscKeyword as DdlMisc,
+        DmlKeyword as Dml,
+        DmlMiscKeyword as DmlMisc,
+        Type as TypeId,
     }
 }
 
-#[derive(Debug, PartialEq, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u8)]
+pub enum DmlMiscKeyword {
+    Limit,
+    From,
+    Into,
+    Where,
+    If,
+    And,
+    As,
+    By,
+    Asc,
+    Desc,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
 #[repr(u8)]
-pub enum Stmt {
+pub enum DmlKeyword {
+    Insert,
+    Select,
+    Update,
+    Delete,
+    Exists,
+    Truncate,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u8)]
+pub enum DdlMiscKeyword {
+    With,
+    Add,
+    Remove,
+    Sort,
+    Type,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u8)]
+pub enum DdlKeyword {
     Use,
     Create,
+    Alter,
     Drop,
     Inspect,
-    Alter,
+    Model,
+    Space,
 }
 
-#[derive(Debug, PartialEq, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
 #[repr(u8)]
-pub enum Kw {
-    Stmt(Stmt),
-    TypeId(Ty),
-    Space,
-    Model,
-    Type,
+pub enum Type {
+    String,
+    Binary,
+    List,
+    Map,
+    Bool,
+    Int,
+    Double,
+    Float,
+}
+
+/*
+    This section implements DAGs, as described by Czech et al in their paper. I wrote these pretty much by brute-force using
+    a byte-level multiplicative function (inside a script). This unfortunately implies that every time we *do* need to add a
+    new keyword, I will need to recompute and rewrite the vertices. I don't plan to use any codegen, so I think this is good
+    as-is. The real challenge here is to keep the graph small, and I couldn't do that for the symbols table even with multiple
+    trials. Please see if you can improve them.
+
+    Also the functions are unique to every graph, and every input set, so BE WARNED!
+
+    -- Sayan (@ohsayan)
+    Sept. 18, 2022
+*/
+
+const SYM_MAGIC_A: u8 = b'w';
+const SYM_MAGIC_B: u8 = b'E';
+
+static SYM_GRAPH: [u8; 69] = [
+    0, 0, 25, 0, 3, 0, 21, 0, 6, 13, 0, 0, 0, 0, 8, 0, 0, 0, 17, 0, 0, 30, 0, 28, 0, 20, 19, 12, 0,
+    0, 2, 0, 0, 15, 0, 0, 0, 5, 0, 31, 14, 0, 1, 0, 18, 29, 24, 0, 0, 10, 0, 0, 26, 0, 0, 0, 22, 0,
+    23, 7, 0, 27, 0, 4, 16, 11, 0, 0, 9,
+];
+
+static SYM_DATA: [(u8, Symbol); 32] = [
+    (b'+', Symbol::OpArithmeticAdd),
+    (b'-', Symbol::OpArithmeticSub),
+    (b'*', Symbol::OpArithmeticMul),
+    (b'/', Symbol::OpArithmeticDiv),
+    (b'!', Symbol::OpLogicalNot),
+    (b'&', Symbol::OpLogicalAnd),
+    (b'^', Symbol::OpLogicalXor),
+    (b'|', Symbol::OpLogicalOr),
+    (b'=', Symbol::OpAssign),
+    (b'(', Symbol::TtOpenParen),
+    (b')', Symbol::TtCloseParen),
+    (b'[', Symbol::TtOpenSqBracket),
+    (b']', Symbol::TtCloseSqBracket),
+    (b'{', Symbol::TtOpenBrace),
+    (b'}', Symbol::TtCloseBrace),
+    (b'<', Symbol::OpComparatorLt),
+    (b'>', Symbol::OpComparatorGt),
+    (b'\'', Symbol::QuoteS),
+    (b'"', Symbol::QuoteD),
+    (b'@', Symbol::SymAt),
+    (b'#', Symbol::SymHash),
+    (b'$', Symbol::SymDollar),
+    (b'%', Symbol::SymPercent),
+    (b'_', Symbol::SymUnderscore),
+    (b'\\', Symbol::SymBackslash),
+    (b':', Symbol::SymColon),
+    (b';', Symbol::SymSemicolon),
+    (b',', Symbol::SymComma),
+    (b'.', Symbol::SymPeriod),
+    (b'?', Symbol::SymQuestion),
+    (b'~', Symbol::SymTilde),
+    (b'`', Symbol::SymAccent),
+];
+
+#[inline(always)]
+fn symfh(k: u8, magic: u8) -> u16 {
+    (magic as u16 * k as u16) % SYM_GRAPH.len() as u16
+}
+
+#[inline(always)]
+fn symph(k: u8) -> u8 {
+    (SYM_GRAPH[symfh(k, SYM_MAGIC_A) as usize] + SYM_GRAPH[symfh(k, SYM_MAGIC_B) as usize])
+        % SYM_GRAPH.len() as u8
 }
 
-impl From<Ty> for Kw {
-    fn from(ty: Ty) -> Self {
-        Self::TypeId(ty)
+#[inline(always)]
+fn symof(sym: u8) -> Option<Symbol> {
+    let hf = symph(sym);
+    if hf < SYM_DATA.len() as u8 && SYM_DATA[hf as usize].0 == sym {
+        Some(SYM_DATA[hf as usize].1)
+    } else {
+        None
     }
 }
 
-impl Kw {
-    // FIXME(@ohsayan): Use our pf hack
-    pub fn try_from_slice(s: &[u8]) -> Option<Token> {
-        let r = match s.to_ascii_lowercase().as_slice() {
-            b"use" => Self::Stmt(Stmt::Use).into(),
-            b"create" => Self::Stmt(Stmt::Create).into(),
-            b"drop" => Self::Stmt(Stmt::Drop).into(),
-            b"inspect" => Self::Stmt(Stmt::Inspect).into(),
-            b"alter" => Self::Stmt(Stmt::Alter).into(),
-            b"space" => Self::Space.into(),
-            b"model" => Self::Model.into(),
-            b"string" => Self::TypeId(Ty::String).into(),
-            b"binary" => Self::TypeId(Ty::Binary).into(),
-            b"list" => Self::TypeId(Ty::Ls).into(),
-            b"true" => Token::Lit(Lit::Bool(true)),
-            b"type" => Kw::Type.into(),
-            b"false" => Token::Lit(Lit::Bool(false)),
-            _ => return None,
-        };
-        return Some(r);
+static KW_GRAPH: [u8; 38] = [
+    0, 29, 0, 16, 14, 3, 6, 11, 35, 14, 13, 30, 4, 4, 18, 17, 29, 11, 27, 10, 22, 37, 36, 30, 15,
+    27, 10, 3, 10, 13, 16, 30, 16, 15, 29, 9, 10, 25,
+];
+
+static KW_DATA: [(&str, Keyword); 36] = [
+    ("use", Keyword::Ddl(DdlKeyword::Use)),
+    ("create", Keyword::Ddl(DdlKeyword::Create)),
+    ("alter", Keyword::Ddl(DdlKeyword::Alter)),
+    ("drop", Keyword::Ddl(DdlKeyword::Drop)),
+    ("inspect", Keyword::Ddl(DdlKeyword::Inspect)),
+    ("model", Keyword::Ddl(DdlKeyword::Model)),
+    ("space", Keyword::Ddl(DdlKeyword::Space)),
+    ("with", Keyword::DdlMisc(DdlMiscKeyword::With)),
+    ("add", Keyword::DdlMisc(DdlMiscKeyword::Add)),
+    ("remove", Keyword::DdlMisc(DdlMiscKeyword::Remove)),
+    ("sort", Keyword::DdlMisc(DdlMiscKeyword::Sort)),
+    ("type", Keyword::DdlMisc(DdlMiscKeyword::Type)),
+    ("insert", Keyword::Dml(DmlKeyword::Insert)),
+    ("select", Keyword::Dml(DmlKeyword::Select)),
+    ("update", Keyword::Dml(DmlKeyword::Update)),
+    ("delete", Keyword::Dml(DmlKeyword::Delete)),
+    ("exists", Keyword::Dml(DmlKeyword::Exists)),
+    ("truncate", Keyword::Dml(DmlKeyword::Truncate)),
+    ("limit", Keyword::DmlMisc(DmlMiscKeyword::Limit)),
+    ("from", Keyword::DmlMisc(DmlMiscKeyword::From)),
+    ("into", Keyword::DmlMisc(DmlMiscKeyword::Into)),
+    ("where", Keyword::DmlMisc(DmlMiscKeyword::Where)),
+    ("if", Keyword::DmlMisc(DmlMiscKeyword::If)),
+    ("and", Keyword::DmlMisc(DmlMiscKeyword::And)),
+    ("as", Keyword::DmlMisc(DmlMiscKeyword::As)),
+    ("by", Keyword::DmlMisc(DmlMiscKeyword::By)),
+    ("asc", Keyword::DmlMisc(DmlMiscKeyword::Asc)),
+    ("desc", Keyword::DmlMisc(DmlMiscKeyword::Desc)),
+    ("string", Keyword::TypeId(Type::String)),
+    ("binary", Keyword::TypeId(Type::Binary)),
+    ("list", Keyword::TypeId(Type::List)),
+    ("map", Keyword::TypeId(Type::Map)),
+    ("bool", Keyword::TypeId(Type::Bool)),
+    ("int", Keyword::TypeId(Type::Int)),
+    ("double", Keyword::TypeId(Type::Double)),
+    ("float", Keyword::TypeId(Type::Float)),
+];
+
+const KW_MAGIC_A: &[u8] = b"8DWJbcla";
+const KW_MAGIC_B: &[u8] = b"u7uclIZx";
+const KW_MODULUS: usize = 8;
+
+#[inline(always)]
+fn kwfh(k: &[u8], magic: &[u8]) -> u32 {
+    let mut i = 0;
+    let mut s = 0;
+    while i < k.len() {
+        s += magic[(i % KW_MODULUS) as usize] as u32 * k[i] as u32;
+        i += 1;
     }
+    s % KW_GRAPH.len() as u32
 }
 
-#[derive(Debug, PartialEq, Clone, Copy)]
-#[repr(u8)]
-pub enum Ty {
-    String = 0_u8,
-    Binary = 1_u8,
-    Ls = 2_u8,
+#[inline(always)]
+fn kwph(k: &[u8]) -> u8 {
+    (KW_GRAPH[kwfh(k, KW_MAGIC_A) as usize] + KW_GRAPH[kwfh(k, KW_MAGIC_B) as usize])
+        % KW_GRAPH.len() as u8
+}
+
+#[inline(always)]
+fn kwof(key: &str) -> Option<Keyword> {
+    let ph = kwph(key.as_bytes());
+    if ph < KW_DATA.len() as u8 && KW_DATA[ph as usize].0 == key {
+        Some(KW_DATA[ph as usize].1)
+    } else {
+        None
+    }
 }
 
 /*
@@ -220,8 +416,8 @@ impl<'a> Lexer<'a> {
         self.incr_cursor_by(iff as usize)
     }
     #[inline(always)]
-    fn push_token(&mut self, token: Token) {
-        self.tokens.push(token)
+    fn push_token(&mut self, token: impl Into<Token>) {
+        self.tokens.push(token.into())
     }
     #[inline(always)]
     fn peek_is(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
@@ -273,8 +469,11 @@ impl<'a> Lexer<'a> {
     }
     fn scan_ident_or_keyword(&mut self) {
         let s = self.scan_ident();
-        match Kw::try_from_slice(unsafe { s.as_slice() }) {
-            Some(kw) => self.tokens.push(kw),
+        let st = unsafe { transmute(s.as_slice()) };
+        match kwof(st) {
+            Some(kw) => self.tokens.push(kw.into()),
+            // FIXME(@ohsayan): Uh, mind fixing this? The only advantage is that I can keep the graph *memory* footprint small
+            None if st == "true" || st == "false" => self.push_token(Lit::Bool(st == "true")),
             None => self.tokens.push(Token::Ident(s)),
         }
     }
@@ -336,37 +535,24 @@ impl<'a> Lexer<'a> {
             }
             let terminated = self.peek_eq_and_forward(quote_style);
             match String::from_utf8(buf) {
-                Ok(st) if terminated => self.tokens.push(Token::Lit(st.into())),
+                Ok(st) if terminated => self.tokens.push(Token::Lit(st.into_boxed_str().into())),
                 _ => self.last_error = Some(LangError::InvalidStringLiteral),
             }
         }
     }
     fn scan_byte(&mut self, byte: u8) {
-        let b = match byte {
-            b':' => Token::Colon,
-            b'(' => Token::OpenParen,
-            b')' => Token::CloseParen,
-            b'<' => Token::OpenAngular,
-            b'>' => Token::CloseAngular,
-            b'[' => Token::OpenSqBracket,
-            b']' => Token::CloseSqBracket,
-            b'{' => Token::OpenBrace,
-            b'}' => Token::CloseBrace,
-            b',' => Token::Comma,
-            b'.' => Token::Period,
-            b'=' => Token::OperatorEq,
-            b'+' => Token::OperatorAdd,
+        match symof(byte) {
+            Some(tok) => self.push_token(tok),
             #[cfg(test)]
-            b'\r' => Token::IgnorableComma,
+            None if byte == b'\r' => self.push_token(Token::IgnorableComma),
             _ => {
                 self.last_error = Some(LangError::UnexpectedChar);
                 return;
             }
-        };
+        }
         unsafe {
             self.incr_cursor();
         }
-        self.tokens.push(b)
     }
 
     fn _lex(&mut self) {
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 253059ce..ada4feef 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -40,3 +40,13 @@ macro_rules! multi_assert_eq {
         $(assert_eq!($lhs, $rhs);)*
     };
 }
+
+macro_rules! enum_impls {
+    ($for:ty => {$($other:ty as $me:ident),*$(,)?}) => {
+        $(impl ::core::convert::From<$other> for $for {fn from(v: $other) -> Self {Self::$me(v)}})*
+    }
+}
+
+macro_rules! assertions {
+    ($($assert:expr),*$(,)?) => {$(const _:()=::core::assert!($assert);)*}
+}
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 28bc7d48..ae46de13 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -46,7 +46,7 @@
 use {
     super::{
         ast::{Compiler, Statement},
-        lexer::{Kw, Lit, Token, Ty},
+        lexer::{DdlMiscKeyword, Keyword, Lit, Symbol, Token, Type},
         LangResult, RawSlice,
     },
     std::{
@@ -101,12 +101,12 @@ pub type Dict = HashMap<String, DictEntry>;
 
 #[derive(Debug, PartialEq)]
 pub struct Layer {
-    ty: Ty,
+    ty: Type,
     props: Dict,
 }
 
 impl Layer {
-    pub(super) const fn new(ty: Ty, props: Dict) -> Self {
+    pub(super) const fn new(ty: Type, props: Dict) -> Self {
         Self { ty, props }
     }
 }
@@ -135,12 +135,15 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
 
     while i < l {
         match (&tok[i], state) {
-            (Token::OpenBrace, DictFoldState::OB) => {
+            (Token::Symbol(Symbol::TtOpenBrace), DictFoldState::OB) => {
                 i += 1;
                 // we found a brace, expect a close brace or an ident
                 state = DictFoldState::CB_OR_IDENT;
             }
-            (Token::CloseBrace, DictFoldState::CB_OR_IDENT | DictFoldState::COMMA_OR_CB) => {
+            (
+                Token::Symbol(Symbol::TtCloseBrace),
+                DictFoldState::CB_OR_IDENT | DictFoldState::COMMA_OR_CB,
+            ) => {
                 // end of stream
                 i += 1;
                 state = DictFoldState::FINAL;
@@ -152,7 +155,7 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
                 tmp = MaybeUninit::new(unsafe { transmute(id.as_slice()) });
                 state = DictFoldState::COLON;
             }
-            (Token::Colon, DictFoldState::COLON) => {
+            (Token::Symbol(Symbol::SymColon), DictFoldState::COLON) => {
                 // found colon, expect literal or openbrace
                 i += 1;
                 state = DictFoldState::LIT_OR_OB;
@@ -169,12 +172,12 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
                 state = DictFoldState::COMMA_OR_CB;
             }
             // ONLY COMMA CAPTURE
-            (Token::Comma, DictFoldState::COMMA_OR_CB) => {
+            (Token::Symbol(Symbol::SymComma), DictFoldState::COMMA_OR_CB) => {
                 i += 1;
                 // we found a comma, expect a *strict* brace close or ident
                 state = DictFoldState::CB_OR_IDENT;
             }
-            (Token::OpenBrace, DictFoldState::LIT_OR_OB) => {
+            (Token::Symbol(Symbol::TtOpenBrace), DictFoldState::LIT_OR_OB) => {
                 i += 1;
                 // we found an open brace, so this is a dict
                 let mut new_dict = Dict::new();
@@ -273,14 +276,20 @@ pub(super) fn rfold_tymeta(
     let mut tmp = MaybeUninit::<&str>::uninit();
     while r.pos() < l && r.is_okay() {
         match (&tok[r.pos()], state) {
-            (Token::Keyword(Kw::Type), TyMetaFoldState::IDENT_OR_CB) => {
+            (
+                Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::Type)),
+                TyMetaFoldState::IDENT_OR_CB,
+            ) => {
                 // we were expecting an ident but found the type keyword! increase depth
                 r.incr();
                 r.set_has_more();
                 state = TyMetaFoldState::FINAL;
                 break;
             }
-            (Token::CloseBrace, TyMetaFoldState::IDENT_OR_CB | TyMetaFoldState::COMMA_OR_CB) => {
+            (
+                Token::Symbol(Symbol::TtCloseBrace),
+                TyMetaFoldState::IDENT_OR_CB | TyMetaFoldState::COMMA_OR_CB,
+            ) => {
                 r.incr();
                 // found close brace. end of stream
                 state = TyMetaFoldState::FINAL;
@@ -292,7 +301,7 @@ pub(super) fn rfold_tymeta(
                 // we just saw an ident, so we expect to see a colon
                 state = TyMetaFoldState::COLON;
             }
-            (Token::Colon, TyMetaFoldState::COLON) => {
+            (Token::Symbol(Symbol::SymColon), TyMetaFoldState::COLON) => {
                 r.incr();
                 // we just saw a colon. now we want a literal or openbrace
                 state = TyMetaFoldState::LIT_OR_OB;
@@ -309,12 +318,12 @@ pub(super) fn rfold_tymeta(
                 // saw a literal. next is either comma or close brace
                 state = TyMetaFoldState::COMMA_OR_CB;
             }
-            (Token::Comma, TyMetaFoldState::COMMA_OR_CB) => {
+            (Token::Symbol(Symbol::SymComma), TyMetaFoldState::COMMA_OR_CB) => {
                 r.incr();
                 // next is strictly a close brace or ident
                 state = TyMetaFoldState::IDENT_OR_CB;
             }
-            (Token::OpenBrace, TyMetaFoldState::LIT_OR_OB) => {
+            (Token::Symbol(Symbol::TtOpenBrace), TyMetaFoldState::LIT_OR_OB) => {
                 r.incr();
                 // another dict in here
                 let mut d = Dict::new();
@@ -368,13 +377,13 @@ pub(super) fn rfold_layers(tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
     let mut dict = Dict::new();
     while i < l && okay {
         match (&tok[i], state) {
-            (Token::Keyword(Kw::TypeId(ty)), LayerFoldState::TY) => {
+            (Token::Keyword(Keyword::TypeId(ty)), LayerFoldState::TY) => {
                 i += 1;
                 // expecting type, and found type. next is either end or an open brace or some arbitrary token
                 tmp = MaybeUninit::new(ty);
                 state = LayerFoldState::END_OR_OB;
             }
-            (Token::OpenBrace, LayerFoldState::END_OR_OB) => {
+            (Token::Symbol(Symbol::TtOpenBrace), LayerFoldState::END_OR_OB) => {
                 i += 1;
                 // since we found an open brace, this type has some meta
                 let ret = rfold_tymeta(TyMetaFoldState::IDENT_OR_CB, &tok[i..], &mut dict);
@@ -396,7 +405,7 @@ pub(super) fn rfold_layers(tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
                     break;
                 }
             }
-            (Token::Comma, LayerFoldState::FOLD_DICT_INCOMPLETE) => {
+            (Token::Symbol(Symbol::SymComma), LayerFoldState::FOLD_DICT_INCOMPLETE) => {
                 // there is a comma at the end of this
                 i += 1;
                 let ret = rfold_tymeta(TyMetaFoldState::IDENT_OR_CB, &tok[i..], &mut dict);
@@ -413,7 +422,7 @@ pub(super) fn rfold_layers(tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
                     break;
                 }
             }
-            (Token::CloseBrace, LayerFoldState::FOLD_DICT_INCOMPLETE) => {
+            (Token::Symbol(Symbol::TtCloseBrace), LayerFoldState::FOLD_DICT_INCOMPLETE) => {
                 // end of stream
                 i += 1;
                 state = LayerFoldState::FOLD_COMPLETED;
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index d8ef9d11..53ea4eb7 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -126,7 +126,7 @@ mod lexer_tests {
         let lol = v!(r#"'\\\\\\\\\\'"#);
         assert_eq!(
             lex(&lol).unwrap(),
-            vec![Token::Lit(Lit::Str("\\".repeat(5)))],
+            vec![Token::Lit(Lit::Str("\\".repeat(5).into_boxed_str()))],
             "lol"
         )
     }
@@ -148,7 +148,7 @@ mod schema_tests {
     use {
         super::{
             super::{
-                lexer::{Lit, Token},
+                lexer::{Lit, Symbol, Token},
                 schema,
             },
             lex,
@@ -159,12 +159,13 @@ mod schema_tests {
 
     /// A very "basic" fuzzer that will randomly inject tokens wherever applicable
     fn fuzz_tokens(src: &[Token], fuzzwith: impl Fn(bool, &[Token])) {
-        static FUZZ_TARGETS: [Token; 2] = [Token::Comma, Token::IgnorableComma];
+        static FUZZ_TARGETS: [Token; 2] = [Token::Symbol(Symbol::SymComma), Token::IgnorableComma];
         let mut rng = rand::thread_rng();
         #[inline(always)]
         fn inject(new_src: &mut Vec<Token>, rng: &mut impl Rng) -> usize {
             let start = new_src.len();
-            (0..test_utils::random_number(0, 5, rng)).for_each(|_| new_src.push(Token::Comma));
+            (0..test_utils::random_number(0, 5, rng))
+                .for_each(|_| new_src.push(Token::Symbol(Symbol::SymComma)));
             new_src.len() - start
         }
         let fuzz_amount = src.iter().filter(|tok| FUZZ_TARGETS.contains(tok)).count();
@@ -175,15 +176,15 @@ mod schema_tests {
                 Token::IgnorableComma => {
                     let did_add = test_utils::random_bool(&mut rng);
                     if did_add {
-                        new_src.push(Token::Comma);
+                        new_src.push(Token::Symbol(Symbol::SymComma));
                     }
                     let added = inject(&mut new_src, &mut rng);
                     should_pass &= added <= !did_add as usize;
                 }
-                Token::Comma => {
+                Token::Symbol(Symbol::SymComma) => {
                     let did_add = test_utils::random_bool(&mut rng);
                     if did_add {
-                        new_src.push(Token::Comma);
+                        new_src.push(Token::Symbol(Symbol::SymComma));
                     } else {
                         should_pass = false;
                     }
@@ -390,7 +391,7 @@ mod schema_tests {
     }
     mod tymeta {
         use super::*;
-        use crate::engine::ql::lexer::{Kw, Ty};
+        use crate::engine::ql::lexer::{Keyword, Type};
         #[test]
         fn tymeta_mini() {
             let tok = lex(b"}").unwrap();
@@ -541,7 +542,7 @@ mod schema_tests {
                 if should_pass {
                     assert!(ret.is_okay());
                     assert!(ret.has_more());
-                    assert!(new_src[ret.pos()] == Token::Keyword(Kw::TypeId(Ty::String)));
+                    assert!(new_src[ret.pos()] == Token::Keyword(Keyword::TypeId(Type::String)));
                     assert_eq!(dict, expected);
                 } else if ret.is_okay() {
                     panic!("Expected failure but passed for token stream: `{:?}`", tok);
@@ -551,14 +552,14 @@ mod schema_tests {
     }
     mod layer {
         use super::*;
-        use crate::engine::ql::{lexer::Ty, schema::Layer};
+        use crate::engine::ql::{lexer::Type, schema::Layer};
         #[test]
         fn layer_mini() {
             let tok = lex(b"string)").unwrap();
             let (layers, c, okay) = schema::fold_layers(&tok);
             assert_eq!(c, tok.len() - 1);
             assert!(okay);
-            assert_eq!(layers, vec![Layer::new(Ty::String, dict! {})]);
+            assert_eq!(layers, vec![Layer::new(Type::String, dict! {})]);
         }
         #[test]
         fn layer() {
@@ -569,7 +570,7 @@ mod schema_tests {
             assert_eq!(
                 layers,
                 vec![Layer::new(
-                    Ty::String,
+                    Type::String,
                     dict! {
                         "maxlen" => Lit::Num(100)
                     }
@@ -585,8 +586,8 @@ mod schema_tests {
             assert_eq!(
                 layers,
                 vec![
-                    Layer::new(Ty::String, dict! {}),
-                    Layer::new(Ty::Ls, dict! {})
+                    Layer::new(Type::String, dict! {}),
+                    Layer::new(Type::List, dict! {})
                 ]
             );
         }
@@ -599,9 +600,9 @@ mod schema_tests {
             assert_eq!(
                 layers,
                 vec![
-                    Layer::new(Ty::String, dict! {}),
+                    Layer::new(Type::String, dict! {}),
                     Layer::new(
-                        Ty::Ls,
+                        Type::List,
                         dict! {
                             "unique" => Lit::Bool(true),
                             "maxlen" => Lit::Num(10),
@@ -623,14 +624,14 @@ mod schema_tests {
                 layers,
                 vec![
                     Layer::new(
-                        Ty::String,
+                        Type::String,
                         dict! {
                             "ascii_only" => Lit::Bool(true),
                             "maxlen" => Lit::Num(255)
                         }
                     ),
                     Layer::new(
-                        Ty::Ls,
+                        Type::List,
                         dict! {
                             "unique" => Lit::Bool(true),
                             "maxlen" => Lit::Num(10),
@@ -653,14 +654,14 @@ mod schema_tests {
         ")
             .unwrap();
             let expected = vec![
-                Layer::new(Ty::String, dict!()),
+                Layer::new(Type::String, dict!()),
                 Layer::new(
-                    Ty::Ls,
+                    Type::List,
                     dict! {
                         "maxlen" => Lit::Num(100),
                     },
                 ),
-                Layer::new(Ty::Ls, dict!("unique" => Lit::Bool(true))),
+                Layer::new(Type::List, dict!("unique" => Lit::Bool(true))),
             ];
             fuzz_tokens(&tok, |should_pass, new_tok| {
                 let (layers, c, okay) = schema::fold_layers(&new_tok);

From 71a9ceb09380a89acbc321c4ae805170ee5c0096 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 19 Sep 2022 21:19:11 +0530
Subject: [PATCH 015/310] Add `NULL` and `PRIMARY` keywords

---
 server/src/engine/ql/lexer.rs | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index d7172c89..1fb526c4 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -125,6 +125,12 @@ pub enum Keyword {
     Dml(DmlKeyword),
     DmlMisc(DmlMiscKeyword),
     TypeId(Type),
+    Misc(MiscKeyword),
+}
+
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub enum MiscKeyword {
+    Null,
 }
 
 enum_impls! {
@@ -134,6 +140,7 @@ enum_impls! {
         DmlKeyword as Dml,
         DmlMiscKeyword as DmlMisc,
         Type as TypeId,
+        MiscKeyword as Misc,
     }
 }
 
@@ -183,6 +190,7 @@ pub enum DdlKeyword {
     Inspect,
     Model,
     Space,
+    Primary,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -276,12 +284,12 @@ fn symof(sym: u8) -> Option<Symbol> {
     }
 }
 
-static KW_GRAPH: [u8; 38] = [
-    0, 29, 0, 16, 14, 3, 6, 11, 35, 14, 13, 30, 4, 4, 18, 17, 29, 11, 27, 10, 22, 37, 36, 30, 15,
-    27, 10, 3, 10, 13, 16, 30, 16, 15, 29, 9, 10, 25,
+static KW_GRAPH: [u8; 40] = [
+    0, 2, 32, 18, 4, 37, 11, 27, 34, 35, 26, 33, 0, 0, 10, 2, 22, 8, 5, 7, 16, 9, 8, 39, 21, 5, 0,
+    22, 14, 19, 22, 31, 28, 38, 26, 21, 30, 24, 10, 18,
 ];
 
-static KW_DATA: [(&str, Keyword); 36] = [
+static KW_DATA: [(&str, Keyword); 38] = [
     ("use", Keyword::Ddl(DdlKeyword::Use)),
     ("create", Keyword::Ddl(DdlKeyword::Create)),
     ("alter", Keyword::Ddl(DdlKeyword::Alter)),
@@ -289,6 +297,7 @@ static KW_DATA: [(&str, Keyword); 36] = [
     ("inspect", Keyword::Ddl(DdlKeyword::Inspect)),
     ("model", Keyword::Ddl(DdlKeyword::Model)),
     ("space", Keyword::Ddl(DdlKeyword::Space)),
+    ("primary", Keyword::Ddl(DdlKeyword::Primary)),
     ("with", Keyword::DdlMisc(DdlMiscKeyword::With)),
     ("add", Keyword::DdlMisc(DdlMiscKeyword::Add)),
     ("remove", Keyword::DdlMisc(DdlMiscKeyword::Remove)),
@@ -318,10 +327,11 @@ static KW_DATA: [(&str, Keyword); 36] = [
     ("int", Keyword::TypeId(Type::Int)),
     ("double", Keyword::TypeId(Type::Double)),
     ("float", Keyword::TypeId(Type::Float)),
+    ("null", Keyword::Misc(MiscKeyword::Null)),
 ];
 
-const KW_MAGIC_A: &[u8] = b"8DWJbcla";
-const KW_MAGIC_B: &[u8] = b"u7uclIZx";
+const KW_MAGIC_A: &[u8] = b"GSggb8qI";
+const KW_MAGIC_B: &[u8] = b"ZaljIeOx";
 const KW_MODULUS: usize = 8;
 
 #[inline(always)]

From 185df7a82904d1ea706be552d90b153562a408f2 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 19 Sep 2022 22:34:07 +0530
Subject: [PATCH 016/310] Add field property parsing

---
 server/src/engine/ql/schema.rs | 46 ++++++++++++++++++++++++++++++++--
 server/src/engine/ql/tests.rs  | 26 +++++++++++++++++++
 2 files changed, 70 insertions(+), 2 deletions(-)

diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index ae46de13..7104b925 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -46,11 +46,11 @@
 use {
     super::{
         ast::{Compiler, Statement},
-        lexer::{DdlMiscKeyword, Keyword, Lit, Symbol, Token, Type},
+        lexer::{DdlKeyword, DdlMiscKeyword, Keyword, Lit, MiscKeyword, Symbol, Token, Type},
         LangResult, RawSlice,
     },
     std::{
-        collections::HashMap,
+        collections::{HashMap, HashSet},
         mem::{transmute, MaybeUninit},
     },
 };
@@ -76,6 +76,8 @@ macro_rules! states {
     }
 }
 
+type StaticStr = &'static str;
+
 const HIBIT: u64 = 1 << 63;
 const TRAIL_COMMA: bool = true;
 
@@ -111,6 +113,21 @@ impl Layer {
     }
 }
 
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct FieldProperties {
+    pub(super) propeties: HashSet<StaticStr>,
+}
+
+impl FieldProperties {
+    const NULL: StaticStr = "null";
+    const PRIMARY: StaticStr = "primary";
+    pub fn new() -> Self {
+        Self {
+            propeties: HashSet::new(),
+        }
+    }
+}
+
 /*
     Context-free dict
 */
@@ -457,6 +474,31 @@ pub(super) fn fold_layers(tok: &[Token]) -> (Vec<Layer>, usize, bool) {
     (l, (r & !HIBIT) as _, r & HIBIT == HIBIT)
 }
 
+pub(super) fn collect_field_properties(tok: &[Token]) -> (FieldProperties, usize, bool) {
+    let mut props = FieldProperties::default();
+    let mut i = 0;
+    let mut okay = true;
+    while i < tok.len() {
+        match &tok[i] {
+            Token::Keyword(Keyword::Ddl(DdlKeyword::Primary)) => {
+                okay &= props.propeties.insert(FieldProperties::PRIMARY)
+            }
+            Token::Keyword(Keyword::Misc(MiscKeyword::Null)) => {
+                okay &= props.propeties.insert(FieldProperties::NULL)
+            }
+            Token::Ident(_) => break,
+            _ => {
+                // we could pass this over to the caller, but it's better if we do it since we're doing
+                // a linear scan anyways
+                okay = false;
+                break;
+            }
+        }
+        i += 1;
+    }
+    (props, i, okay)
+}
+
 pub(crate) fn parse_schema(_c: &mut Compiler, _m: RawSlice) -> LangResult<Statement> {
     todo!()
 }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 53ea4eb7..adbac2ef 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -678,4 +678,30 @@ mod schema_tests {
             });
         }
     }
+    mod field_properties {
+        use {super::*, crate::engine::ql::schema::FieldProperties};
+
+        #[test]
+        fn field_properties_empty() {
+            let tok = lex(b"myfield:").unwrap();
+            let (props, c, okay) = schema::collect_field_properties(&tok);
+            assert!(okay);
+            assert_eq!(c, 0);
+            assert_eq!(props, FieldProperties::default());
+        }
+        #[test]
+        fn field_properties_full() {
+            let tok = lex(b"primary null myfield:").unwrap();
+            let (props, c, okay) = schema::collect_field_properties(&tok);
+            assert_eq!(c, 2);
+            assert_eq!(tok[c], Token::Ident("myfield".into()));
+            assert!(okay);
+            assert_eq!(
+                props,
+                FieldProperties {
+                    propeties: ["primary", "null"].into_iter().collect()
+                }
+            )
+        }
+    }
 }

From a75df3a1d7a8eec02c5cb0d94483821d0235acb9 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 19 Sep 2022 23:23:26 +0530
Subject: [PATCH 017/310] Add field parsing

---
 server/src/engine/ql/macros.rs |  11 ++++
 server/src/engine/ql/schema.rs |  74 +++++++++++++++++++--
 server/src/engine/ql/tests.rs  | 114 ++++++++++++++++++++++++++++++++-
 3 files changed, 189 insertions(+), 10 deletions(-)

diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index ada4feef..8b89a2ec 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -35,6 +35,17 @@ macro_rules! dict {
     }};
 }
 
+macro_rules! set {
+    () => {
+        <::std::collections::HashSet<_> as ::core::default::Default>::default()
+    };
+    ($($key:expr),* $(,)?) => {{
+        let mut hs: ::std::collections::HashSet<_> = ::core::default::Default::default();
+        $(hs.insert($key.into());)*
+        hs
+    }};
+}
+
 macro_rules! multi_assert_eq {
     ($($lhs:expr),* => $rhs:expr) => {
         $(assert_eq!($lhs, $rhs);)*
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 7104b925..09533f26 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -37,6 +37,7 @@
 
     TODO: The SMs can be reduced greatly, enocded to fixed-sized structures even, so do that
     FIXME: For now, try and reduce reliance on additional flags (encoded into state?)
+    FIXME: The returns are awfully large right now. Do something about it
 
     --
     Sayan (@ohsayan)
@@ -47,7 +48,7 @@ use {
     super::{
         ast::{Compiler, Statement},
         lexer::{DdlKeyword, DdlMiscKeyword, Keyword, Lit, MiscKeyword, Symbol, Token, Type},
-        LangResult, RawSlice,
+        LangError, LangResult, RawSlice,
     },
     std::{
         collections::{HashMap, HashSet},
@@ -115,7 +116,7 @@ impl Layer {
 
 #[derive(Debug, Default, PartialEq, Eq)]
 pub struct FieldProperties {
-    pub(super) propeties: HashSet<StaticStr>,
+    pub(super) properties: HashSet<StaticStr>,
 }
 
 impl FieldProperties {
@@ -123,11 +124,19 @@ impl FieldProperties {
     const PRIMARY: StaticStr = "primary";
     pub fn new() -> Self {
         Self {
-            propeties: HashSet::new(),
+            properties: HashSet::new(),
         }
     }
 }
 
+#[derive(Debug, PartialEq)]
+/// A field definition
+pub struct Field {
+    pub(super) field_name: Box<str>,
+    pub(super) layers: Vec<Layer>,
+    pub(super) props: HashSet<StaticStr>,
+}
+
 /*
     Context-free dict
 */
@@ -220,6 +229,7 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
     i as u64 | ((okay as u64) << 63)
 }
 
+#[cfg(test)]
 pub fn fold_dict(tok: &[Token]) -> Option<Dict> {
     let mut d = Dict::new();
     let r = rfold_dict(DictFoldState::OB, tok, &mut d);
@@ -365,6 +375,7 @@ pub(super) fn rfold_tymeta(
     r
 }
 
+#[cfg(test)]
 pub(super) fn fold_tymeta(tok: &[Token]) -> (TyMetaFoldResult, Dict) {
     let mut d = Dict::new();
     let r = rfold_tymeta(TyMetaFoldState::IDENT_OR_CB, tok, &mut d);
@@ -468,23 +479,24 @@ pub(super) fn rfold_layers(tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
     i as u64 | ((okay as u64) << 63)
 }
 
+#[cfg(test)]
 pub(super) fn fold_layers(tok: &[Token]) -> (Vec<Layer>, usize, bool) {
     let mut l = Vec::new();
     let r = rfold_layers(tok, &mut l);
     (l, (r & !HIBIT) as _, r & HIBIT == HIBIT)
 }
 
-pub(super) fn collect_field_properties(tok: &[Token]) -> (FieldProperties, usize, bool) {
+pub(super) fn collect_field_properties(tok: &[Token]) -> (FieldProperties, u64) {
     let mut props = FieldProperties::default();
     let mut i = 0;
     let mut okay = true;
     while i < tok.len() {
         match &tok[i] {
             Token::Keyword(Keyword::Ddl(DdlKeyword::Primary)) => {
-                okay &= props.propeties.insert(FieldProperties::PRIMARY)
+                okay &= props.properties.insert(FieldProperties::PRIMARY)
             }
             Token::Keyword(Keyword::Misc(MiscKeyword::Null)) => {
-                okay &= props.propeties.insert(FieldProperties::NULL)
+                okay &= props.properties.insert(FieldProperties::NULL)
             }
             Token::Ident(_) => break,
             _ => {
@@ -496,7 +508,55 @@ pub(super) fn collect_field_properties(tok: &[Token]) -> (FieldProperties, usize
         }
         i += 1;
     }
-    (props, i, okay)
+    (props, i as u64 | ((okay as u64) << 63))
+}
+
+#[cfg(test)]
+pub(super) fn parse_field_properties(tok: &[Token]) -> (FieldProperties, usize, bool) {
+    let (p, r) = collect_field_properties(tok);
+    (p, (r & !HIBIT) as _, r & HIBIT == HIBIT)
+}
+
+pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
+    let l = tok.len();
+    let mut i = 0;
+    let mut okay = true;
+    // parse field properties
+    let (props, r) = collect_field_properties(tok);
+    okay &= r & HIBIT == HIBIT;
+    i += (r & !HIBIT) as usize;
+    // if exhauted or broken, simply return
+    if i == l || !okay || (l - i) == 1 {
+        return Err(LangError::UnexpectedEndofStatement);
+    }
+
+    // field name
+    let field_name = match (&tok[i], &tok[i + 1]) {
+        (Token::Ident(id), Token::Symbol(Symbol::SymColon)) => {
+            unsafe { transmute::<&[u8], &str>(id.as_slice()) }.into()
+        }
+        _ => return Err(LangError::UnexpectedToken),
+    };
+    i += 2;
+
+    // layers
+    let mut layers = Vec::new();
+    let r = rfold_layers(&tok[i..], &mut layers);
+    okay &= r & HIBIT == HIBIT;
+    i += (r & !HIBIT) as usize;
+
+    if okay {
+        Ok((
+            i,
+            Field {
+                field_name,
+                layers,
+                props: props.properties,
+            },
+        ))
+    } else {
+        Err(LangError::UnexpectedToken)
+    }
 }
 
 pub(crate) fn parse_schema(_c: &mut Compiler, _m: RawSlice) -> LangResult<Statement> {
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index adbac2ef..23dc8feb 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -684,7 +684,7 @@ mod schema_tests {
         #[test]
         fn field_properties_empty() {
             let tok = lex(b"myfield:").unwrap();
-            let (props, c, okay) = schema::collect_field_properties(&tok);
+            let (props, c, okay) = schema::parse_field_properties(&tok);
             assert!(okay);
             assert_eq!(c, 0);
             assert_eq!(props, FieldProperties::default());
@@ -692,14 +692,122 @@ mod schema_tests {
         #[test]
         fn field_properties_full() {
             let tok = lex(b"primary null myfield:").unwrap();
-            let (props, c, okay) = schema::collect_field_properties(&tok);
+            let (props, c, okay) = schema::parse_field_properties(&tok);
             assert_eq!(c, 2);
             assert_eq!(tok[c], Token::Ident("myfield".into()));
             assert!(okay);
             assert_eq!(
                 props,
                 FieldProperties {
-                    propeties: ["primary", "null"].into_iter().collect()
+                    properties: set!["primary", "null"],
+                }
+            )
+        }
+    }
+    mod fields {
+        use {
+            super::*,
+            crate::engine::ql::{
+                lexer::Type,
+                schema::{Field, Layer},
+            },
+        };
+        #[test]
+        fn field_mini() {
+            let tok = lex(b"
+                username: string,
+            ")
+            .unwrap();
+            let (c, f) = schema::parse_field(&tok).unwrap();
+            assert_eq!(c, tok.len() - 1);
+            assert_eq!(
+                f,
+                Field {
+                    field_name: "username".into(),
+                    layers: [Layer::new(Type::String, dict! {})].into(),
+                    props: set![],
+                }
+            )
+        }
+        #[test]
+        fn field() {
+            let tok = lex(b"
+                primary username: string,    
+            ")
+            .unwrap();
+            let (c, f) = schema::parse_field(&tok).unwrap();
+            assert_eq!(c, tok.len() - 1);
+            assert_eq!(
+                f,
+                Field {
+                    field_name: "username".into(),
+                    layers: [Layer::new(Type::String, dict! {})].into(),
+                    props: set!["primary"],
+                }
+            )
+        }
+        #[test]
+        fn field_pro() {
+            let tok = lex(b"
+                primary username: string {
+                    maxlen: 10,
+                    ascii_only: true,
+                }
+            ")
+            .unwrap();
+            let (c, f) = schema::parse_field(&tok).unwrap();
+            assert_eq!(c, tok.len());
+            assert_eq!(
+                f,
+                Field {
+                    field_name: "username".into(),
+                    layers: [Layer::new(
+                        Type::String,
+                        dict! {
+                            "maxlen" => Lit::Num(10),
+                            "ascii_only" => Lit::Bool(true),
+                        }
+                    )]
+                    .into(),
+                    props: set!["primary"],
+                }
+            )
+        }
+        #[test]
+        fn field_pro_max() {
+            let tok = lex(b"
+                null notes: list {
+                    type string {
+                        maxlen: 255,
+                        ascii_only: true,
+                    },
+                    unique: true,
+                }
+            ")
+            .unwrap();
+            let (c, f) = schema::parse_field(&tok).unwrap();
+            assert_eq!(c, tok.len());
+            assert_eq!(
+                f,
+                Field {
+                    field_name: "notes".into(),
+                    layers: [
+                        Layer::new(
+                            Type::String,
+                            dict! {
+                                "maxlen" => Lit::Num(255),
+                                "ascii_only" => Lit::Bool(true),
+                            }
+                        ),
+                        Layer::new(
+                            Type::List,
+                            dict! {
+                                "unique" => Lit::Bool(true)
+                            }
+                        ),
+                    ]
+                    .into(),
+                    props: set!["null"],
                 }
             )
         }

From 779fbb77856ae054992718fc06f251d5f2092550 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 20 Sep 2022 11:08:07 +0530
Subject: [PATCH 018/310] Add parsing of schemas

---
 server/src/engine/ql/ast.rs    |   8 +-
 server/src/engine/ql/lexer.rs  |   8 +-
 server/src/engine/ql/mod.rs    |   5 +-
 server/src/engine/ql/schema.rs | 138 ++++++++++++++++++--
 server/src/engine/ql/tests.rs  | 224 +++++++++++++++++++++++++++++++++
 5 files changed, 363 insertions(+), 20 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 381b4fc2..43c8cdea 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -73,7 +73,10 @@ impl Entity {
     }
 }
 
-pub enum Statement {}
+#[derive(Debug, PartialEq)]
+pub enum Statement {
+    CreateModel(schema::Model),
+}
 
 pub struct Compiler<'a> {
     c: *const Token,
@@ -145,7 +148,8 @@ impl<'a> Compiler<'a> {
             Some(Token::Ident(model)) => unsafe { model.raw_clone() },
             _ => return Err(LangError::UnexpectedEndofStatement),
         };
-        schema::parse_schema(self, model_name)
+        let model = schema::parse_schema(self, model_name)?;
+        Ok(Statement::CreateModel(model))
     }
     #[inline(always)]
     fn c_space0(&mut self) -> Result<Statement, LangError> {
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 1fb526c4..f322a27a 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -27,11 +27,7 @@
 use {
     super::{LangError, LangResult, RawSlice},
     crate::util::{compiler, Life},
-    core::{
-        marker::PhantomData,
-        mem::{size_of, transmute},
-        slice, str,
-    },
+    core::{marker::PhantomData, mem::size_of, slice, str},
 };
 
 /*
@@ -479,7 +475,7 @@ impl<'a> Lexer<'a> {
     }
     fn scan_ident_or_keyword(&mut self) {
         let s = self.scan_ident();
-        let st = unsafe { transmute(s.as_slice()) };
+        let st = unsafe { s.as_str() };
         match kwof(st) {
             Some(kw) => self.tokens.push(kw.into()),
             // FIXME(@ohsayan): Uh, mind fixing this? The only advantage is that I can keep the graph *memory* footprint small
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 7bfec028..0b7116ed 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -34,7 +34,7 @@ mod tests;
 
 #[cfg(debug_assertions)]
 use core::{fmt, ops::Deref};
-use core::{mem, slice};
+use core::{mem, slice, str};
 
 /*
     Lang errors
@@ -80,6 +80,9 @@ impl RawSlice {
     unsafe fn as_slice(&self) -> &[u8] {
         slice::from_raw_parts(self.ptr, self.len)
     }
+    unsafe fn as_str(&self) -> &str {
+        str::from_utf8_unchecked(self.as_slice())
+    }
     unsafe fn raw_clone(&self) -> Self {
         Self::new(self.ptr, self.len)
     }
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 09533f26..df4e95e3 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -46,13 +46,13 @@
 
 use {
     super::{
-        ast::{Compiler, Statement},
+        ast::Compiler,
         lexer::{DdlKeyword, DdlMiscKeyword, Keyword, Lit, MiscKeyword, Symbol, Token, Type},
         LangError, LangResult, RawSlice,
     },
     std::{
         collections::{HashMap, HashSet},
-        mem::{transmute, MaybeUninit},
+        mem::MaybeUninit,
     },
 };
 
@@ -137,6 +137,14 @@ pub struct Field {
     pub(super) props: HashSet<StaticStr>,
 }
 
+#[derive(Debug, PartialEq)]
+/// A model definition
+pub struct Model {
+    pub(super) model_name: Box<str>,
+    pub(super) fields: Vec<Field>,
+    pub(super) props: Dict,
+}
+
 /*
     Context-free dict
 */
@@ -157,7 +165,7 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
     let l = tok.len();
     let mut i = 0;
     let mut okay = true;
-    let mut tmp = MaybeUninit::<&str>::uninit();
+    let mut tmp = MaybeUninit::uninit();
 
     while i < l {
         match (&tok[i], state) {
@@ -178,7 +186,7 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
             (Token::Ident(id), DictFoldState::CB_OR_IDENT) => {
                 // found ident, so expect colon
                 i += 1;
-                tmp = MaybeUninit::new(unsafe { transmute(id.as_slice()) });
+                tmp = MaybeUninit::new(unsafe { id.as_str() });
                 state = DictFoldState::COLON;
             }
             (Token::Symbol(Symbol::SymColon), DictFoldState::COLON) => {
@@ -300,7 +308,7 @@ pub(super) fn rfold_tymeta(
 ) -> TyMetaFoldResult {
     let l = tok.len();
     let mut r = TyMetaFoldResult::new();
-    let mut tmp = MaybeUninit::<&str>::uninit();
+    let mut tmp = MaybeUninit::uninit();
     while r.pos() < l && r.is_okay() {
         match (&tok[r.pos()], state) {
             (
@@ -324,7 +332,7 @@ pub(super) fn rfold_tymeta(
             }
             (Token::Ident(ident), TyMetaFoldState::IDENT_OR_CB) => {
                 r.incr();
-                tmp = MaybeUninit::new(unsafe { transmute(ident.as_slice()) });
+                tmp = MaybeUninit::new(unsafe { ident.as_str() });
                 // we just saw an ident, so we expect to see a colon
                 state = TyMetaFoldState::COLON;
             }
@@ -532,9 +540,7 @@ pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
 
     // field name
     let field_name = match (&tok[i], &tok[i + 1]) {
-        (Token::Ident(id), Token::Symbol(Symbol::SymColon)) => {
-            unsafe { transmute::<&[u8], &str>(id.as_slice()) }.into()
-        }
+        (Token::Ident(id), Token::Symbol(Symbol::SymColon)) => unsafe { id.as_str() }.into(),
         _ => return Err(LangError::UnexpectedToken),
     };
     i += 2;
@@ -559,6 +565,116 @@ pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
     }
 }
 
-pub(crate) fn parse_schema(_c: &mut Compiler, _m: RawSlice) -> LangResult<Statement> {
-    todo!()
+/*
+    create model name(..) with { .. }
+                     ^^^^
+*/
+
+states! {
+    ///
+    pub struct SchemaParseState: u8 {
+        OPEN_PAREN = 0x00,
+        FIELD = 0x01,
+        COMMA_OR_END = 0x02,
+        END_OR_FIELD = 0x03,
+    }
+}
+
+pub(super) fn parse_schema_from_tokens(
+    tok: &[Token],
+    model_name: RawSlice,
+) -> LangResult<(Model, usize)> {
+    let model_name = unsafe { model_name.as_str() }.into();
+    // parse fields
+    let l = tok.len();
+    let mut i = 0;
+    let mut state = SchemaParseState::OPEN_PAREN;
+    let mut okay = true;
+    let mut fields = Vec::with_capacity(2);
+
+    while i < l && okay {
+        match (&tok[i], state) {
+            (Token::Symbol(Symbol::TtOpenParen), SchemaParseState::OPEN_PAREN) => {
+                i += 1;
+                state = SchemaParseState::FIELD;
+            }
+            (
+                Token::Keyword(Keyword::Ddl(DdlKeyword::Primary))
+                | Token::Keyword(Keyword::Misc(MiscKeyword::Null))
+                | Token::Ident(_),
+                SchemaParseState::FIELD | SchemaParseState::END_OR_FIELD,
+            ) => {
+                // fine, we found a field. let's see what we've got
+                let (c, f) = self::parse_field(&tok[i..])?;
+                fields.push(f);
+                i += c;
+                state = SchemaParseState::COMMA_OR_END;
+            }
+            (Token::Symbol(Symbol::SymComma), SchemaParseState::COMMA_OR_END) => {
+                i += 1;
+                // expect a field or close paren
+                state = SchemaParseState::END_OR_FIELD;
+            }
+            (
+                Token::Symbol(Symbol::TtCloseParen),
+                SchemaParseState::COMMA_OR_END | SchemaParseState::END_OR_FIELD,
+            ) => {
+                i += 1;
+                // end of stream
+                break;
+            }
+            _ => {
+                okay = false;
+                break;
+            }
+        }
+    }
+
+    // model properties
+    if i == l && okay {
+        // we've reached end of stream, so there's nothing more to parse
+        return Ok((
+            Model {
+                model_name,
+                props: dict! {},
+                fields,
+            },
+            i,
+        ));
+    } else if !okay || tok[i] != Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With)) {
+        return Err(LangError::UnexpectedToken);
+    }
+
+    // we have some more input, and it should be a dict of properties
+    i += 1; // +WITH
+
+    // great, parse the dict
+    let mut dict = Dict::new();
+    let r = self::rfold_dict(DictFoldState::OB, &tok[i..], &mut dict);
+    i += (r & !HIBIT) as usize;
+
+    if r & HIBIT == HIBIT {
+        // sweet, so we got our dict
+        Ok((
+            Model {
+                model_name,
+                props: dict,
+                fields,
+            },
+            i,
+        ))
+    } else {
+        Err(LangError::UnexpectedToken)
+    }
+}
+
+pub(super) fn parse_schema(c: &mut Compiler, m: RawSlice) -> LangResult<Model> {
+    self::parse_schema_from_tokens(c.remslice(), m).map(|(m, i)| {
+        unsafe {
+            // UNSAFE(@ohsayan): All our steps return the correct cursor increments, so this is completely fine (else the tests in
+            // tests::schema would have failed)
+            c.incr_cursor_by(i);
+        }
+        m
+    })
 }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 23dc8feb..5ee34b22 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -812,4 +812,228 @@ mod schema_tests {
             )
         }
     }
+    mod schemas {
+        use crate::engine::ql::{
+            lexer::Type,
+            schema::{Field, Layer, Model},
+        };
+
+        use super::*;
+        #[test]
+        fn schema_mini() {
+            let tok = lex(b"
+                create model mymodel(
+                    primary username: string,
+                    password: binary,
+                )
+            ")
+            .unwrap();
+            let schema_name = match tok[2] {
+                Token::Ident(ref id) => unsafe { id.raw_clone() },
+                _ => panic!("expected ident"),
+            };
+            let tok = &tok[3..];
+
+            // parse model
+            let (model, c) = schema::parse_schema_from_tokens(tok, schema_name).unwrap();
+            assert_eq!(c, tok.len());
+            assert_eq!(
+                model,
+                Model {
+                    model_name: "mymodel".into(),
+                    fields: vec![
+                        Field {
+                            field_name: "username".into(),
+                            layers: vec![Layer::new(Type::String, dict! {})],
+                            props: set!["primary"]
+                        },
+                        Field {
+                            field_name: "password".into(),
+                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            props: set![]
+                        }
+                    ],
+                    props: dict! {}
+                }
+            )
+        }
+        #[test]
+        fn schema() {
+            let tok = lex(b"
+                create model mymodel(
+                    primary username: string,
+                    password: binary,
+                    null profile_pic: binary,
+                )
+            ")
+            .unwrap();
+            let schema_name = match tok[2] {
+                Token::Ident(ref id) => unsafe { id.raw_clone() },
+                _ => panic!("expected ident"),
+            };
+            let tok = &tok[3..];
+
+            // parse model
+            let (model, c) = schema::parse_schema_from_tokens(tok, schema_name).unwrap();
+            assert_eq!(c, tok.len());
+            assert_eq!(
+                model,
+                Model {
+                    model_name: "mymodel".into(),
+                    fields: vec![
+                        Field {
+                            field_name: "username".into(),
+                            layers: vec![Layer::new(Type::String, dict! {})],
+                            props: set!["primary"]
+                        },
+                        Field {
+                            field_name: "password".into(),
+                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            props: set![]
+                        },
+                        Field {
+                            field_name: "profile_pic".into(),
+                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            props: set!["null"]
+                        }
+                    ],
+                    props: dict! {}
+                }
+            )
+        }
+
+        #[test]
+        fn schema_pro() {
+            let tok = lex(b"
+                create model mymodel(
+                    primary username: string,
+                    password: binary,
+                    null profile_pic: binary,
+                    null notes: list {
+                        type string,
+                        unique: true,
+                    },
+                )
+            ")
+            .unwrap();
+            let schema_name = match tok[2] {
+                Token::Ident(ref id) => unsafe { id.raw_clone() },
+                _ => panic!("expected ident"),
+            };
+            let tok = &tok[3..];
+
+            // parse model
+            let (model, c) = schema::parse_schema_from_tokens(tok, schema_name).unwrap();
+            assert_eq!(c, tok.len());
+            assert_eq!(
+                model,
+                Model {
+                    model_name: "mymodel".into(),
+                    fields: vec![
+                        Field {
+                            field_name: "username".into(),
+                            layers: vec![Layer::new(Type::String, dict! {})],
+                            props: set!["primary"]
+                        },
+                        Field {
+                            field_name: "password".into(),
+                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            props: set![]
+                        },
+                        Field {
+                            field_name: "profile_pic".into(),
+                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            props: set!["null"]
+                        },
+                        Field {
+                            field_name: "notes".into(),
+                            layers: vec![
+                                Layer::new(Type::String, dict! {}),
+                                Layer::new(
+                                    Type::List,
+                                    dict! {
+                                        "unique" => Lit::Bool(true)
+                                    }
+                                )
+                            ],
+                            props: set!["null"]
+                        }
+                    ],
+                    props: dict! {}
+                }
+            )
+        }
+
+        #[test]
+        fn schema_pro_max() {
+            let tok = lex(b"
+                create model mymodel(
+                    primary username: string,
+                    password: binary,
+                    null profile_pic: binary,
+                    null notes: list {
+                        type string,
+                        unique: true,
+                    },
+                ) with {
+                    env: {
+                        free_user_limit: 100,
+                    },
+                    storage_driver: \"skyheap\"
+                }
+            ")
+            .unwrap();
+            let schema_name = match tok[2] {
+                Token::Ident(ref id) => unsafe { id.raw_clone() },
+                _ => panic!("expected ident"),
+            };
+            let tok = &tok[3..];
+
+            // parse model
+            let (model, c) = schema::parse_schema_from_tokens(tok, schema_name).unwrap();
+            assert_eq!(c, tok.len());
+            assert_eq!(
+                model,
+                Model {
+                    model_name: "mymodel".into(),
+                    fields: vec![
+                        Field {
+                            field_name: "username".into(),
+                            layers: vec![Layer::new(Type::String, dict! {})],
+                            props: set!["primary"]
+                        },
+                        Field {
+                            field_name: "password".into(),
+                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            props: set![]
+                        },
+                        Field {
+                            field_name: "profile_pic".into(),
+                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            props: set!["null"]
+                        },
+                        Field {
+                            field_name: "notes".into(),
+                            layers: vec![
+                                Layer::new(Type::String, dict! {}),
+                                Layer::new(
+                                    Type::List,
+                                    dict! {
+                                        "unique" => Lit::Bool(true)
+                                    }
+                                )
+                            ],
+                            props: set!["null"]
+                        }
+                    ],
+                    props: dict! {
+                        "env" => dict! {
+                            "free_user_limit" => Lit::Num(100),
+                        },
+                        "storage_driver" => Lit::Str("skyheap".into()),
+                    }
+                }
+            )
+        }
+    }
 }

From 36a2b0162f383e371057d95de8052f44b242f27d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 22 Sep 2022 20:25:38 +0530
Subject: [PATCH 019/310] Enable creating spaces

---
 server/src/engine/ql/ast.rs    |  10 +++-
 server/src/engine/ql/schema.rs | 102 ++++++++++++++++++++++++++-------
 2 files changed, 88 insertions(+), 24 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 43c8cdea..ae63fb54 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -76,6 +76,7 @@ impl Entity {
 #[derive(Debug, PartialEq)]
 pub enum Statement {
     CreateModel(schema::Model),
+    CreateSpace(schema::Space),
 }
 
 pub struct Compiler<'a> {
@@ -146,14 +147,19 @@ impl<'a> Compiler<'a> {
     fn c_model0(&mut self) -> Result<Statement, LangError> {
         let model_name = match self.nxtok_opt() {
             Some(Token::Ident(model)) => unsafe { model.raw_clone() },
-            _ => return Err(LangError::UnexpectedEndofStatement),
+            _ => return Err(LangError::UnexpectedToken),
         };
         let model = schema::parse_schema(self, model_name)?;
         Ok(Statement::CreateModel(model))
     }
     #[inline(always)]
     fn c_space0(&mut self) -> Result<Statement, LangError> {
-        todo!()
+        let space_name = match self.nxtok_opt() {
+            Some(Token::Ident(space_name)) => unsafe { space_name.raw_clone() },
+            _ => return Err(LangError::UnexpectedToken),
+        };
+        let space = schema::parse_space(self, space_name)?;
+        Ok(Statement::CreateSpace(space))
     }
 }
 
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index df4e95e3..dfba8fb0 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -145,6 +145,12 @@ pub struct Model {
     pub(super) props: Dict,
 }
 
+#[derive(Debug, PartialEq)]
+pub struct Space {
+    pub(super) space_name: Box<str>,
+    pub(super) props: Dict,
+}
+
 /*
     Context-free dict
 */
@@ -269,33 +275,42 @@ pub struct TyMetaFoldResult {
 }
 
 impl TyMetaFoldResult {
+    #[inline(always)]
     const fn new() -> Self {
         Self {
             c: 0,
             b: [true, false],
         }
     }
+    #[inline(always)]
     fn incr(&mut self) {
         self.incr_by(1)
     }
+    #[inline(always)]
     fn incr_by(&mut self, by: usize) {
         self.c += by;
     }
+    #[inline(always)]
     fn set_fail(&mut self) {
         self.b[0] = false;
     }
+    #[inline(always)]
     fn set_has_more(&mut self) {
         self.b[1] = true;
     }
+    #[inline(always)]
     pub fn pos(&self) -> usize {
         self.c
     }
+    #[inline(always)]
     pub fn has_more(&self) -> bool {
         self.b[1]
     }
+    #[inline(always)]
     pub fn is_okay(&self) -> bool {
         self.b[0]
     }
+    #[inline(always)]
     fn record(&mut self, c: bool) {
         self.b[0] &= c;
     }
@@ -631,40 +646,42 @@ pub(super) fn parse_schema_from_tokens(
     }
 
     // model properties
-    if i == l && okay {
-        // we've reached end of stream, so there's nothing more to parse
-        return Ok((
-            Model {
-                model_name,
-                props: dict! {},
-                fields,
-            },
-            i,
-        ));
-    } else if !okay || tok[i] != Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With)) {
+    if !okay {
         return Err(LangError::UnexpectedToken);
     }
 
-    // we have some more input, and it should be a dict of properties
-    i += 1; // +WITH
+    if l > i && tok[i] == Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With)) {
+        // we have some more input, and it should be a dict of properties
+        i += 1; // +WITH
 
-    // great, parse the dict
-    let mut dict = Dict::new();
-    let r = self::rfold_dict(DictFoldState::OB, &tok[i..], &mut dict);
-    i += (r & !HIBIT) as usize;
+        // great, parse the dict
+        let mut dict = Dict::new();
+        let r = self::rfold_dict(DictFoldState::OB, &tok[i..], &mut dict);
+        i += (r & !HIBIT) as usize;
 
-    if r & HIBIT == HIBIT {
-        // sweet, so we got our dict
+        if r & HIBIT == HIBIT {
+            // sweet, so we got our dict
+            Ok((
+                Model {
+                    model_name,
+                    props: dict,
+                    fields,
+                },
+                i,
+            ))
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    } else {
+        // we've reached end of stream, so there's nothing more to parse
         Ok((
             Model {
                 model_name,
-                props: dict,
+                props: dict! {},
                 fields,
             },
             i,
         ))
-    } else {
-        Err(LangError::UnexpectedToken)
     }
 }
 
@@ -678,3 +695,44 @@ pub(super) fn parse_schema(c: &mut Compiler, m: RawSlice) -> LangResult<Model> {
         m
     })
 }
+
+pub(super) fn parse_space_from_tokens(tok: &[Token], s: RawSlice) -> LangResult<(Space, usize)> {
+    let space_name = unsafe { s.as_str() }.into();
+
+    // let's see if the cursor is at `with`. ignore other tokens because that's fine
+    if !tok.is_empty() && tok[0] == Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With)) {
+        // we have a dict
+        let mut d = Dict::new();
+        let ret = self::rfold_dict(DictFoldState::OB, &tok[1..], &mut d);
+        if ret & HIBIT == HIBIT {
+            Ok((
+                Space {
+                    space_name,
+                    props: d,
+                },
+                (ret & !HIBIT) as _,
+            ))
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    } else {
+        Ok((
+            Space {
+                space_name,
+                props: dict! {},
+            },
+            0,
+        ))
+    }
+}
+
+pub(super) fn parse_space(c: &mut Compiler, s: RawSlice) -> LangResult<Space> {
+    self::parse_space_from_tokens(c.remslice(), s).map(|(m, i)| {
+        unsafe {
+            // UNSAFE(@ohsayan): The rfolds are very well tested to return the correct cursor position,
+            // so this should be completely safe
+            c.incr_cursor_by(i);
+        }
+        m
+    })
+}

From 9d5f85374bfe482486d17584546c413c12fa5a18 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 22 Sep 2022 21:03:42 +0530
Subject: [PATCH 020/310] Fix equality checks

---
 server/src/engine/ql/ast.rs    | 16 ++++++++++------
 server/src/engine/ql/lexer.rs  | 18 +++++++++++++++++-
 server/src/engine/ql/schema.rs |  4 ++--
 3 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index ae63fb54..e3f7a65c 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -30,13 +30,14 @@ use {
         schema, LangError, LangResult, RawSlice,
     },
     crate::util::Life,
-    core::{marker::PhantomData, mem::discriminant, slice},
+    core::{marker::PhantomData, slice},
 };
 
 /*
     AST
 */
 
+#[cfg_attr(debug_assertions, derive(Debug, PartialEq))]
 pub enum Entity {
     Current(RawSlice),
     Partial(RawSlice),
@@ -73,10 +74,12 @@ impl Entity {
     }
 }
 
-#[derive(Debug, PartialEq)]
+#[cfg_attr(debug_assertions, derive(Debug, PartialEq))]
 pub enum Statement {
     CreateModel(schema::Model),
     CreateSpace(schema::Space),
+    Use(Entity),
+    Inspect(Entity),
 }
 
 pub struct Compiler<'a> {
@@ -137,11 +140,13 @@ impl<'a> Compiler<'a> {
     }
     #[inline(always)]
     fn inspect0(&mut self) -> Result<Statement, LangError> {
-        todo!()
+        let entity = Entity::parse(self)?;
+        Ok(Statement::Inspect(entity))
     }
     #[inline(always)]
     fn use0(&mut self) -> Result<Statement, LangError> {
-        todo!()
+        let entity = Entity::parse(self)?;
+        Ok(Statement::Use(entity))
     }
     #[inline(always)]
     fn c_model0(&mut self) -> Result<Statement, LangError> {
@@ -208,8 +213,7 @@ impl<'a> Compiler<'a> {
         &*self.c
     }
     pub(super) fn peek_eq_and_forward(&mut self, t: &Token) -> bool {
-        let did_fw =
-            self.not_exhausted() && unsafe { discriminant(self.deref_cursor()) == discriminant(t) };
+        let did_fw = self.not_exhausted() && unsafe { self.deref_cursor().deq(t) };
         unsafe {
             self.incr_cursor_if(did_fw);
         }
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index f322a27a..d123eccf 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -27,7 +27,11 @@
 use {
     super::{LangError, LangResult, RawSlice},
     crate::util::{compiler, Life},
-    core::{marker::PhantomData, mem::size_of, slice, str},
+    core::{
+        marker::PhantomData,
+        mem::{discriminant, size_of},
+        slice, str,
+    },
 };
 
 /*
@@ -582,3 +586,15 @@ impl<'a> Lexer<'a> {
         }
     }
 }
+impl Token {
+    #[inline(always)]
+    pub(crate) fn deq(&self, with: impl AsRef<Self>) -> bool {
+        discriminant(self) == discriminant(with.as_ref())
+    }
+}
+
+impl AsRef<Token> for Token {
+    fn as_ref(&self) -> &Token {
+        self
+    }
+}
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index dfba8fb0..456abd47 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -650,7 +650,7 @@ pub(super) fn parse_schema_from_tokens(
         return Err(LangError::UnexpectedToken);
     }
 
-    if l > i && tok[i] == Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With)) {
+    if l > i && tok[i].deq(Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With))) {
         // we have some more input, and it should be a dict of properties
         i += 1; // +WITH
 
@@ -700,7 +700,7 @@ pub(super) fn parse_space_from_tokens(tok: &[Token], s: RawSlice) -> LangResult<
     let space_name = unsafe { s.as_str() }.into();
 
     // let's see if the cursor is at `with`. ignore other tokens because that's fine
-    if !tok.is_empty() && tok[0] == Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With)) {
+    if !tok.is_empty() && tok[0].deq(Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With))) {
         // we have a dict
         let mut d = Dict::new();
         let ret = self::rfold_dict(DictFoldState::OB, &tok[1..], &mut d);

From efcc89d98408194accee5bfd86e859218285e772 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 24 Sep 2022 10:02:58 +0530
Subject: [PATCH 021/310] Enable altering of spaces

---
 server/src/engine/ql/ast.rs    | 33 +++++++++++++++--
 server/src/engine/ql/schema.rs | 65 +++++++++++++++++++++++-----------
 2 files changed, 75 insertions(+), 23 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index e3f7a65c..a99b611f 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -80,6 +80,7 @@ pub enum Statement {
     CreateSpace(schema::Space),
     Use(Entity),
     Inspect(Entity),
+    AlterSpace(schema::Alter),
 }
 
 pub struct Compiler<'a> {
@@ -136,9 +137,31 @@ impl<'a> Compiler<'a> {
     }
     #[inline(always)]
     fn alter0(&mut self) -> Result<Statement, LangError> {
+        match self.nxtok_opt() {
+            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Model))) => self.alter_model(),
+            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Space))) => self.alter_space(),
+            Some(_) => Err(LangError::ExpectedStatement),
+            None => Err(LangError::UnexpectedEndofStatement),
+        }
+    }
+    #[inline(always)]
+    fn alter_model(&mut self) -> Result<Statement, LangError> {
         todo!()
     }
     #[inline(always)]
+    fn alter_space(&mut self) -> Result<Statement, LangError> {
+        let space_name = match self.nxtok_opt() {
+            Some(Token::Ident(id)) => unsafe { id.raw_clone() },
+            Some(_) => return Err(LangError::UnexpectedToken),
+            None => return Err(LangError::UnexpectedEndofStatement),
+        };
+        let (alter, i) = schema::parse_alter_space_from_tokens(self.remslice(), space_name)?;
+        unsafe {
+            self.incr_cursor_by(i);
+        }
+        Ok(Statement::AlterSpace(alter))
+    }
+    #[inline(always)]
     fn inspect0(&mut self) -> Result<Statement, LangError> {
         let entity = Entity::parse(self)?;
         Ok(Statement::Inspect(entity))
@@ -154,7 +177,10 @@ impl<'a> Compiler<'a> {
             Some(Token::Ident(model)) => unsafe { model.raw_clone() },
             _ => return Err(LangError::UnexpectedToken),
         };
-        let model = schema::parse_schema(self, model_name)?;
+        let (model, i) = schema::parse_schema_from_tokens(self.remslice(), model_name)?;
+        unsafe {
+            self.incr_cursor_by(i);
+        }
         Ok(Statement::CreateModel(model))
     }
     #[inline(always)]
@@ -163,7 +189,10 @@ impl<'a> Compiler<'a> {
             Some(Token::Ident(space_name)) => unsafe { space_name.raw_clone() },
             _ => return Err(LangError::UnexpectedToken),
         };
-        let space = schema::parse_space(self, space_name)?;
+        let (space, i) = schema::parse_space_from_tokens(self.remslice(), space_name)?;
+        unsafe {
+            self.incr_cursor_by(i);
+        }
         Ok(Statement::CreateSpace(space))
     }
 }
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 456abd47..f99819f0 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -46,7 +46,6 @@
 
 use {
     super::{
-        ast::Compiler,
         lexer::{DdlKeyword, DdlMiscKeyword, Keyword, Lit, MiscKeyword, Symbol, Token, Type},
         LangError, LangResult, RawSlice,
     },
@@ -151,6 +150,12 @@ pub struct Space {
     pub(super) props: Dict,
 }
 
+#[derive(Debug, PartialEq)]
+pub struct Alter {
+    pub(super) space_name: Box<str>,
+    pub(super) updated_props: Dict,
+}
+
 /*
     Context-free dict
 */
@@ -503,12 +508,14 @@ pub(super) fn rfold_layers(tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
 }
 
 #[cfg(test)]
+#[inline(always)]
 pub(super) fn fold_layers(tok: &[Token]) -> (Vec<Layer>, usize, bool) {
     let mut l = Vec::new();
     let r = rfold_layers(tok, &mut l);
     (l, (r & !HIBIT) as _, r & HIBIT == HIBIT)
 }
 
+#[inline(always)]
 pub(super) fn collect_field_properties(tok: &[Token]) -> (FieldProperties, u64) {
     let mut props = FieldProperties::default();
     let mut i = 0;
@@ -535,11 +542,13 @@ pub(super) fn collect_field_properties(tok: &[Token]) -> (FieldProperties, u64)
 }
 
 #[cfg(test)]
+#[inline(always)]
 pub(super) fn parse_field_properties(tok: &[Token]) -> (FieldProperties, usize, bool) {
     let (p, r) = collect_field_properties(tok);
     (p, (r & !HIBIT) as _, r & HIBIT == HIBIT)
 }
 
+#[inline(always)]
 pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
     let l = tok.len();
     let mut i = 0;
@@ -595,6 +604,7 @@ states! {
     }
 }
 
+#[inline(always)]
 pub(super) fn parse_schema_from_tokens(
     tok: &[Token],
     model_name: RawSlice,
@@ -685,17 +695,7 @@ pub(super) fn parse_schema_from_tokens(
     }
 }
 
-pub(super) fn parse_schema(c: &mut Compiler, m: RawSlice) -> LangResult<Model> {
-    self::parse_schema_from_tokens(c.remslice(), m).map(|(m, i)| {
-        unsafe {
-            // UNSAFE(@ohsayan): All our steps return the correct cursor increments, so this is completely fine (else the tests in
-            // tests::schema would have failed)
-            c.incr_cursor_by(i);
-        }
-        m
-    })
-}
-
+#[inline(always)]
 pub(super) fn parse_space_from_tokens(tok: &[Token], s: RawSlice) -> LangResult<(Space, usize)> {
     let space_name = unsafe { s.as_str() }.into();
 
@@ -726,13 +726,36 @@ pub(super) fn parse_space_from_tokens(tok: &[Token], s: RawSlice) -> LangResult<
     }
 }
 
-pub(super) fn parse_space(c: &mut Compiler, s: RawSlice) -> LangResult<Space> {
-    self::parse_space_from_tokens(c.remslice(), s).map(|(m, i)| {
-        unsafe {
-            // UNSAFE(@ohsayan): The rfolds are very well tested to return the correct cursor position,
-            // so this should be completely safe
-            c.incr_cursor_by(i);
-        }
-        m
-    })
+pub(super) fn parse_alter_space_from_tokens(
+    tok: &[Token],
+    space_name: RawSlice,
+) -> LangResult<(Alter, usize)> {
+    let mut i = 0;
+    let l = tok.len();
+
+    let invalid = l < 3
+        || !(tok[i].deq(Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With)))
+            && tok[i + 1].deq(Token::Symbol(Symbol::TtOpenBrace)));
+
+    if invalid {
+        return Err(LangError::UnexpectedToken);
+    }
+
+    i += 2;
+
+    let mut d = Dict::new();
+    let ret = rfold_dict(DictFoldState::CB_OR_IDENT, &tok[i..], &mut d);
+    i += (ret & !HIBIT) as usize;
+
+    if ret & HIBIT == HIBIT {
+        Ok((
+            Alter {
+                space_name: unsafe { space_name.as_str() }.into(),
+                updated_props: d,
+            },
+            i,
+        ))
+    } else {
+        Err(LangError::UnexpectedToken)
+    }
 }

From 500bfc353d7283a2bf850a4598e2b1ae8b7a7c71 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 30 Sep 2022 23:52:07 +0530
Subject: [PATCH 022/310] Add field-dict expression syntax

---
 server/src/engine/ql/lexer.rs  |   8 +++
 server/src/engine/ql/schema.rs |  98 +++++++++++++++++++++++++--
 server/src/engine/ql/tests.rs  | 118 +++++++++++++++++++++++++++++++++
 3 files changed, 218 insertions(+), 6 deletions(-)

diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index d123eccf..a61064a4 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -591,6 +591,14 @@ impl Token {
     pub(crate) fn deq(&self, with: impl AsRef<Self>) -> bool {
         discriminant(self) == discriminant(with.as_ref())
     }
+    #[inline(always)]
+    pub(crate) fn is_ident(&self) -> bool {
+        matches!(self, Token::Ident(_))
+    }
+    #[inline(always)]
+    pub(crate) fn is_typeid(&self) -> bool {
+        matches!(self, Token::Keyword(Keyword::TypeId(_)))
+    }
 }
 
 impl AsRef<Token> for Token {
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index f99819f0..b6be47d3 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -63,7 +63,7 @@ use {
 ///
 /// **DO NOT** construct states manually
 macro_rules! states {
-    ($(#[$attr:meta])+$vis:vis struct $stateid:ident: $statebase:ty {$($(#[$tyattr:meta])*$v:vis$state:ident = $statexp:expr),* $(,)?}) => {
+    ($(#[$attr:meta])+$vis:vis struct $stateid:ident: $statebase:ty {$($(#[$tyattr:meta])*$v:vis$state:ident = $statexp:expr),+ $(,)?}) => {
         #[::core::prelude::v1::derive(::core::cmp::PartialEq, ::core::cmp::Eq, ::core::clone::Clone, ::core::marker::Copy)]
         $(#[$attr])+$vis struct $stateid {__base: $statebase}
         impl $stateid {$($(#[$tyattr])*$v const $state:Self=$stateid{__base: $statexp,};)*}
@@ -424,11 +424,11 @@ states! {
     }
 }
 
-pub(super) fn rfold_layers(tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
+pub(super) fn rfold_layers(start: LayerFoldState, tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
     let l = tok.len();
     let mut i = 0;
     let mut okay = true;
-    let mut state = LayerFoldState::TY;
+    let mut state = start;
     let mut tmp = MaybeUninit::uninit();
     let mut dict = Dict::new();
     while i < l && okay {
@@ -447,7 +447,7 @@ pub(super) fn rfold_layers(tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
                 okay &= ret.is_okay();
                 if ret.has_more() {
                     // more layers
-                    let ret = rfold_layers(&tok[i..], layers);
+                    let ret = rfold_layers(LayerFoldState::TY, &tok[i..], layers);
                     okay &= ret & HIBIT == HIBIT;
                     i += (ret & !HIBIT) as usize;
                     state = LayerFoldState::FOLD_DICT_INCOMPLETE;
@@ -511,7 +511,7 @@ pub(super) fn rfold_layers(tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
 #[inline(always)]
 pub(super) fn fold_layers(tok: &[Token]) -> (Vec<Layer>, usize, bool) {
     let mut l = Vec::new();
-    let r = rfold_layers(tok, &mut l);
+    let r = rfold_layers(LayerFoldState::TY, tok, &mut l);
     (l, (r & !HIBIT) as _, r & HIBIT == HIBIT)
 }
 
@@ -571,7 +571,7 @@ pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
 
     // layers
     let mut layers = Vec::new();
-    let r = rfold_layers(&tok[i..], &mut layers);
+    let r = rfold_layers(LayerFoldState::TY, &tok[i..], &mut layers);
     okay &= r & HIBIT == HIBIT;
     i += (r & !HIBIT) as usize;
 
@@ -759,3 +759,89 @@ pub(super) fn parse_alter_space_from_tokens(
         Err(LangError::UnexpectedToken)
     }
 }
+
+states! {
+    /// The field syntax parse state
+    pub struct FieldSyntaxParseState: u8 {
+        IDENT = 0x00,
+        OB = 0x01,
+        FOLD_DICT_INCOMPLETE = 0x02,
+        COMPLETED = 0xFF,
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub(super) struct ExpandedField {
+    pub(super) field_name: Box<str>,
+    pub(super) props: Dict,
+    pub(super) layers: Vec<Layer>,
+}
+
+pub(super) fn parse_field_syntax(tok: &[Token]) -> LangResult<(ExpandedField, usize)> {
+    let l = tok.len();
+    let mut i = 0_usize;
+    let mut state = FieldSyntaxParseState::IDENT;
+    let mut okay = true;
+    let mut tmp = MaybeUninit::uninit();
+    let mut props = Dict::new();
+    let mut layers = vec![];
+    while i < l && okay {
+        match (&tok[i], state) {
+            (Token::Ident(field), FieldSyntaxParseState::IDENT) => {
+                i += 1;
+                tmp = MaybeUninit::new(field);
+                // expect open brace
+                state = FieldSyntaxParseState::OB;
+            }
+            (Token::Symbol(Symbol::TtOpenBrace), FieldSyntaxParseState::OB) => {
+                i += 1;
+                let r = self::rfold_tymeta(TyMetaFoldState::IDENT_OR_CB, &tok[i..], &mut props);
+                okay &= r.is_okay();
+                i += r.pos();
+                if r.has_more() && i < l {
+                    // now parse layers
+                    let r = self::rfold_layers(LayerFoldState::TY, &tok[i..], &mut layers);
+                    okay &= r & HIBIT == HIBIT;
+                    i += (r & !HIBIT) as usize;
+                    state = FieldSyntaxParseState::FOLD_DICT_INCOMPLETE;
+                } else {
+                    okay = false;
+                    break;
+                }
+            }
+            (Token::Symbol(Symbol::SymComma), FieldSyntaxParseState::FOLD_DICT_INCOMPLETE) => {
+                i += 1;
+                let r = self::rfold_dict(DictFoldState::CB_OR_IDENT, &tok[i..], &mut props);
+                okay &= r & HIBIT == HIBIT;
+                i += (r & !HIBIT) as usize;
+                if okay {
+                    state = FieldSyntaxParseState::COMPLETED;
+                    break;
+                }
+            }
+            (Token::Symbol(Symbol::TtCloseBrace), FieldSyntaxParseState::FOLD_DICT_INCOMPLETE) => {
+                i += 1;
+                // great, were done
+                state = FieldSyntaxParseState::COMPLETED;
+                break;
+            }
+            _ => {
+                okay = false;
+                break;
+            }
+        }
+    }
+    okay &= state == FieldSyntaxParseState::COMPLETED;
+    if okay {
+        Ok((
+            ExpandedField {
+                field_name: unsafe { tmp.assume_init().as_str() }.into(),
+                layers,
+                props,
+            },
+            i,
+        ))
+    } else {
+        Err(LangError::UnexpectedToken)
+    }
+}
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 5ee34b22..652285f8 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -1036,4 +1036,122 @@ mod schema_tests {
             )
         }
     }
+    mod dict_field_syntax {
+        use super::*;
+        use crate::engine::ql::{
+            lexer::Type,
+            schema::{ExpandedField, Layer},
+        };
+        #[test]
+        fn field_syn_mini() {
+            let tok = lex(b"username { type string }").unwrap();
+            let (ef, i) = schema::parse_field_syntax(&tok).unwrap();
+            assert_eq!(i, tok.len());
+            assert_eq!(
+                ef,
+                ExpandedField {
+                    field_name: "username".into(),
+                    layers: vec![Layer::new(Type::String, dict! {})],
+                    props: dict! {}
+                }
+            )
+        }
+        #[test]
+        fn field_syn() {
+            let tok = lex(b"
+                username {
+                    nullable: false,
+                    type string,
+                }
+            ")
+            .unwrap();
+            let (ef, i) = schema::parse_field_syntax(&tok).unwrap();
+            assert_eq!(i, tok.len());
+            assert_eq!(
+                ef,
+                ExpandedField {
+                    field_name: "username".into(),
+                    props: dict! {
+                        "nullable" => Lit::Bool(false),
+                    },
+                    layers: vec![Layer::new(Type::String, dict! {})]
+                }
+            );
+        }
+        #[test]
+        fn field_syn_pro() {
+            let tok = lex(b"
+                username {
+                    nullable: false,
+                    type string {
+                        minlen: 6,
+                        maxlen: 255,
+                    },
+                    jingle_bells: \"snow\"
+                }
+            ")
+            .unwrap();
+            let (ef, i) = schema::parse_field_syntax(&tok).unwrap();
+            assert_eq!(i, tok.len());
+            assert_eq!(
+                ef,
+                ExpandedField {
+                    field_name: "username".into(),
+                    props: dict! {
+                        "nullable" => Lit::Bool(false),
+                        "jingle_bells" => Lit::Str("snow".into()),
+                    },
+                    layers: vec![Layer::new(
+                        Type::String,
+                        dict! {
+                            "minlen" => Lit::Num(6),
+                            "maxlen" => Lit::Num(255),
+                        }
+                    )]
+                }
+            );
+        }
+        #[test]
+        fn field_syn_pro_max() {
+            let tok = lex(b"
+                notes {
+                    nullable: true,
+                    type list {
+                        type string {
+                            ascii_only: true,
+                        },
+                        unique: true,
+                    },
+                    jingle_bells: \"snow\"
+                }
+            ")
+            .unwrap();
+            let (ef, i) = schema::parse_field_syntax(&tok).unwrap();
+            assert_eq!(i, tok.len());
+            assert_eq!(
+                ef,
+                ExpandedField {
+                    field_name: "notes".into(),
+                    props: dict! {
+                        "nullable" => Lit::Bool(true),
+                        "jingle_bells" => Lit::Str("snow".into()),
+                    },
+                    layers: vec![
+                        Layer::new(
+                            Type::String,
+                            dict! {
+                                "ascii_only" => Lit::Bool(true),
+                            }
+                        ),
+                        Layer::new(
+                            Type::List,
+                            dict! {
+                                "unique" => Lit::Bool(true),
+                            }
+                        )
+                    ]
+                }
+            );
+        }
+    }
 }

From 70754cbb4d0114ed347827afefea3610ba8d5373 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 2 Oct 2022 22:04:30 +0530
Subject: [PATCH 023/310] Support field removal in alter model

---
 server/src/engine/ql/ast.rs    |  20 ++---
 server/src/engine/ql/lexer.rs  |  13 +--
 server/src/engine/ql/mod.rs    |  20 ++++-
 server/src/engine/ql/schema.rs | 149 +++++++++++++++++++++++++++++----
 server/src/engine/ql/tests.rs  |  43 +++++++++-
 5 files changed, 201 insertions(+), 44 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index a99b611f..97fcf458 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -37,7 +37,7 @@ use {
     AST
 */
 
-#[cfg_attr(debug_assertions, derive(Debug, PartialEq))]
+#[derive(Debug, PartialEq)]
 pub enum Entity {
     Current(RawSlice),
     Partial(RawSlice),
@@ -55,17 +55,17 @@ impl Entity {
                 Some(Token::Symbol(Symbol::SymPeriod)),
                 Some(Token::Ident(tbl)),
             ) => unsafe {
-                let r = Ok(Entity::Full(ks.raw_clone(), tbl.raw_clone()));
+                let r = Ok(Entity::Full(ks.clone(), tbl.clone()));
                 cm.incr_cursor_by(3);
                 r
             },
             (Some(Token::Ident(ident)), _, _) => unsafe {
-                let r = Ok(Entity::Current(ident.raw_clone()));
+                let r = Ok(Entity::Current(ident.clone()));
                 cm.incr_cursor();
                 r
             },
             (Some(Token::Symbol(Symbol::SymColon)), Some(Token::Ident(tbl)), _) => unsafe {
-                let r = Ok(Entity::Partial(tbl.raw_clone()));
+                let r = Ok(Entity::Partial(tbl.clone()));
                 cm.incr_cursor_by(2);
                 r
             },
@@ -80,7 +80,7 @@ pub enum Statement {
     CreateSpace(schema::Space),
     Use(Entity),
     Inspect(Entity),
-    AlterSpace(schema::Alter),
+    AlterSpace(schema::AlterSpace),
 }
 
 pub struct Compiler<'a> {
@@ -151,7 +151,7 @@ impl<'a> Compiler<'a> {
     #[inline(always)]
     fn alter_space(&mut self) -> Result<Statement, LangError> {
         let space_name = match self.nxtok_opt() {
-            Some(Token::Ident(id)) => unsafe { id.raw_clone() },
+            Some(Token::Ident(id)) => id.clone(),
             Some(_) => return Err(LangError::UnexpectedToken),
             None => return Err(LangError::UnexpectedEndofStatement),
         };
@@ -174,7 +174,7 @@ impl<'a> Compiler<'a> {
     #[inline(always)]
     fn c_model0(&mut self) -> Result<Statement, LangError> {
         let model_name = match self.nxtok_opt() {
-            Some(Token::Ident(model)) => unsafe { model.raw_clone() },
+            Some(Token::Ident(model)) => model.clone(),
             _ => return Err(LangError::UnexpectedToken),
         };
         let (model, i) = schema::parse_schema_from_tokens(self.remslice(), model_name)?;
@@ -186,7 +186,7 @@ impl<'a> Compiler<'a> {
     #[inline(always)]
     fn c_space0(&mut self) -> Result<Statement, LangError> {
         let space_name = match self.nxtok_opt() {
-            Some(Token::Ident(space_name)) => unsafe { space_name.raw_clone() },
+            Some(Token::Ident(space_name)) => space_name.clone(),
             _ => return Err(LangError::UnexpectedToken),
         };
         let (space, i) = schema::parse_space_from_tokens(self.remslice(), space_name)?;
@@ -241,8 +241,8 @@ impl<'a> Compiler<'a> {
     pub(super) unsafe fn deref_cursor(&self) -> &Token {
         &*self.c
     }
-    pub(super) fn peek_eq_and_forward(&mut self, t: &Token) -> bool {
-        let did_fw = self.not_exhausted() && unsafe { self.deref_cursor().deq(t) };
+    pub(super) fn peek_eq_and_forward(&mut self, t: Token) -> bool {
+        let did_fw = self.not_exhausted() && unsafe { self.deref_cursor() == &t };
         unsafe {
             self.incr_cursor_if(did_fw);
         }
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index a61064a4..6e1909bb 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -27,19 +27,14 @@
 use {
     super::{LangError, LangResult, RawSlice},
     crate::util::{compiler, Life},
-    core::{
-        marker::PhantomData,
-        mem::{discriminant, size_of},
-        slice, str,
-    },
+    core::{marker::PhantomData, mem::size_of, slice, str},
 };
 
 /*
     Lex meta
 */
 
-#[derive(Debug)]
-#[cfg_attr(debug_assertions, derive(PartialEq, Clone))]
+#[derive(Debug, PartialEq, Clone)]
 pub enum Token {
     Symbol(Symbol),
     Keyword(Keyword),
@@ -587,10 +582,6 @@ impl<'a> Lexer<'a> {
     }
 }
 impl Token {
-    #[inline(always)]
-    pub(crate) fn deq(&self, with: impl AsRef<Self>) -> bool {
-        discriminant(self) == discriminant(with.as_ref())
-    }
     #[inline(always)]
     pub(crate) fn is_ident(&self) -> bool {
         matches!(self, Token::Ident(_))
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 0b7116ed..39bc66b9 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -61,8 +61,16 @@ pub enum LangError {
 */
 
 /// An unsafe, C-like slice that holds a ptr and length. Construction and usage is at the risk of the user
+///
+/// Notes:
+/// - [`Clone`] is implemented for [`RawSlice`] because it is a simple bitwise copy of the fat ptr
+/// - [`fmt::Debug`] is implemented in different ways
+///     - With debug assertions enabled, it will output a slice
+///     - In release mode, it will output the fat ptr meta
+/// - [`PartialEq`] is implemented in debug mode with slice comparison, but is **NOT implemented for release mode in the
+///   way you'd expect it to**. In release mode, a comparison will simply panic.
 #[cfg_attr(not(debug_assertions), derive(Debug))]
-#[cfg_attr(debug_assertions, derive(Clone))]
+#[derive(Clone)]
 pub struct RawSlice {
     ptr: *const u8,
     len: usize,
@@ -83,9 +91,6 @@ impl RawSlice {
     unsafe fn as_str(&self) -> &str {
         str::from_utf8_unchecked(self.as_slice())
     }
-    unsafe fn raw_clone(&self) -> Self {
-        Self::new(self.ptr, self.len)
-    }
 }
 
 #[cfg(debug_assertions)]
@@ -112,6 +117,13 @@ impl PartialEq for RawSlice {
     }
 }
 
+#[cfg(not(debug_assertions))]
+impl PartialEq for RawSlice {
+    fn eq(&self, _other: &Self) -> bool {
+        panic!("Called partialeq on rawslice in release mode");
+    }
+}
+
 #[cfg(debug_assertions)]
 impl<U> PartialEq<U> for RawSlice
 where
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index b6be47d3..39846f83 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -44,6 +44,8 @@
     Sept. 15, 2022
 */
 
+use super::lexer::DmlKeyword;
+
 use {
     super::{
         lexer::{DdlKeyword, DdlMiscKeyword, Keyword, Lit, MiscKeyword, Symbol, Token, Type},
@@ -151,7 +153,7 @@ pub struct Space {
 }
 
 #[derive(Debug, PartialEq)]
-pub struct Alter {
+pub struct AlterSpace {
     pub(super) space_name: Box<str>,
     pub(super) updated_props: Dict,
 }
@@ -173,6 +175,15 @@ states! {
 }
 
 pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dict) -> u64 {
+    /*
+        NOTE: Assume rules wherever applicable
+
+        <openbrace> ::= "{"
+        <closebrace> ::= "}"
+        <comma> ::= ","
+        <colon> ::= ":"
+        <dict> ::= <openbrace> (<ident> <colon> (<lit> | <dict>) <comma>)* <comma>* <closebrace>
+    */
     let l = tok.len();
     let mut i = 0;
     let mut okay = true;
@@ -425,6 +436,19 @@ states! {
 }
 
 pub(super) fn rfold_layers(start: LayerFoldState, tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
+    /*
+        NOTE: Assume rules wherever applicable
+
+        <openbrace> ::= "{"
+        <closebrace> ::= "}"
+        <comma> ::= ","
+        <colon> ::= ":"
+        <kw_type> ::= "type"
+        <layer> ::= <openbrace>
+            (<kw_type> <kw_typeid> <layer> <comma>)*1
+            (<ident> <colon> (<lit> | <dict>) <comma>)*
+            <comma>* <closebrace>
+    */
     let l = tok.len();
     let mut i = 0;
     let mut okay = true;
@@ -564,7 +588,7 @@ pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
 
     // field name
     let field_name = match (&tok[i], &tok[i + 1]) {
-        (Token::Ident(id), Token::Symbol(Symbol::SymColon)) => unsafe { id.as_str() }.into(),
+        (Token::Ident(id), Token::Symbol(Symbol::SymColon)) => unsafe { id.as_str() },
         _ => return Err(LangError::UnexpectedToken),
     };
     i += 2;
@@ -579,7 +603,7 @@ pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
         Ok((
             i,
             Field {
-                field_name,
+                field_name: field_name.into(),
                 layers,
                 props: props.properties,
             },
@@ -609,7 +633,7 @@ pub(super) fn parse_schema_from_tokens(
     tok: &[Token],
     model_name: RawSlice,
 ) -> LangResult<(Model, usize)> {
-    let model_name = unsafe { model_name.as_str() }.into();
+    let model_name = unsafe { model_name.as_str() };
     // parse fields
     let l = tok.len();
     let mut i = 0;
@@ -660,7 +684,7 @@ pub(super) fn parse_schema_from_tokens(
         return Err(LangError::UnexpectedToken);
     }
 
-    if l > i && tok[i].deq(Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With))) {
+    if l > i && tok[i] == (Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With))) {
         // we have some more input, and it should be a dict of properties
         i += 1; // +WITH
 
@@ -673,7 +697,7 @@ pub(super) fn parse_schema_from_tokens(
             // sweet, so we got our dict
             Ok((
                 Model {
-                    model_name,
+                    model_name: model_name.into(),
                     props: dict,
                     fields,
                 },
@@ -686,7 +710,7 @@ pub(super) fn parse_schema_from_tokens(
         // we've reached end of stream, so there's nothing more to parse
         Ok((
             Model {
-                model_name,
+                model_name: model_name.into(),
                 props: dict! {},
                 fields,
             },
@@ -697,17 +721,17 @@ pub(super) fn parse_schema_from_tokens(
 
 #[inline(always)]
 pub(super) fn parse_space_from_tokens(tok: &[Token], s: RawSlice) -> LangResult<(Space, usize)> {
-    let space_name = unsafe { s.as_str() }.into();
+    let space_name = unsafe { s.as_str() };
 
     // let's see if the cursor is at `with`. ignore other tokens because that's fine
-    if !tok.is_empty() && tok[0].deq(Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With))) {
+    if !tok.is_empty() && tok[0] == (Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With))) {
         // we have a dict
         let mut d = Dict::new();
         let ret = self::rfold_dict(DictFoldState::OB, &tok[1..], &mut d);
         if ret & HIBIT == HIBIT {
             Ok((
                 Space {
-                    space_name,
+                    space_name: space_name.into(),
                     props: d,
                 },
                 (ret & !HIBIT) as _,
@@ -718,7 +742,7 @@ pub(super) fn parse_space_from_tokens(tok: &[Token], s: RawSlice) -> LangResult<
     } else {
         Ok((
             Space {
-                space_name,
+                space_name: space_name.into(),
                 props: dict! {},
             },
             0,
@@ -729,13 +753,13 @@ pub(super) fn parse_space_from_tokens(tok: &[Token], s: RawSlice) -> LangResult<
 pub(super) fn parse_alter_space_from_tokens(
     tok: &[Token],
     space_name: RawSlice,
-) -> LangResult<(Alter, usize)> {
+) -> LangResult<(AlterSpace, usize)> {
     let mut i = 0;
     let l = tok.len();
 
     let invalid = l < 3
-        || !(tok[i].deq(Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With)))
-            && tok[i + 1].deq(Token::Symbol(Symbol::TtOpenBrace)));
+        || !(tok[i] == (Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With)))
+            && tok[i + 1] == (Token::Symbol(Symbol::TtOpenBrace)));
 
     if invalid {
         return Err(LangError::UnexpectedToken);
@@ -749,7 +773,7 @@ pub(super) fn parse_alter_space_from_tokens(
 
     if ret & HIBIT == HIBIT {
         Ok((
-            Alter {
+            AlterSpace {
                 space_name: unsafe { space_name.as_str() }.into(),
                 updated_props: d,
             },
@@ -845,3 +869,98 @@ pub(super) fn parse_field_syntax(tok: &[Token]) -> LangResult<(ExpandedField, us
         Err(LangError::UnexpectedToken)
     }
 }
+
+#[derive(Debug)]
+#[cfg_attr(debug_assertions, derive(PartialEq))]
+pub(super) enum AlterKind {
+    Add(Field),
+    Remove(Box<[RawSlice]>),
+    Update(ExpandedField),
+}
+
+#[inline(always)]
+pub(super) fn parse_alter_kind_from_tokens(
+    tok: &[Token],
+    current: &mut usize,
+) -> LangResult<AlterKind> {
+    let l = tok.len();
+    let mut i = 0;
+    if l < 2 {
+        return Err(LangError::UnexpectedEndofStatement);
+    }
+    *current += 1;
+    let r = match tok[i] {
+        Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::Add)) => {
+            AlterKind::Add(alter_add(&tok[1..], &mut i))
+        }
+        Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::Remove)) => {
+            AlterKind::Remove(alter_remove(&tok[1..], &mut i)?)
+        }
+        Token::Keyword(Keyword::Dml(DmlKeyword::Update)) => {
+            AlterKind::Update(alter_update(&tok[1..], &mut i))
+        }
+        _ => return Err(LangError::ExpectedStatement),
+    };
+    *current += i;
+    Ok(r)
+}
+
+#[inline(always)]
+pub(super) fn alter_add(_tok: &[Token], _current: &mut usize) -> Field {
+    todo!()
+}
+
+#[inline(always)]
+pub(super) fn alter_remove(tok: &[Token], current: &mut usize) -> LangResult<Box<[RawSlice]>> {
+    const DEFAULT_REMOVE_COL_CNT: usize = 4;
+    /*
+        WARNING: No trailing commas allowed
+        <remove> ::= <ident> | <openparen> (<ident> <comma>)*<closeparen>
+    */
+    if tok.is_empty() {
+        return Err(LangError::UnexpectedEndofStatement);
+    }
+
+    let r = match &tok[0] {
+        Token::Ident(id) => {
+            *current += 1;
+            Box::new([id.clone()])
+        }
+        Token::Symbol(Symbol::TtOpenParen) => {
+            let l = tok.len();
+            let mut i = 1_usize;
+            let mut okay = true;
+            let mut stop = false;
+            let mut cols = Vec::with_capacity(DEFAULT_REMOVE_COL_CNT);
+            while i < tok.len() && okay && !stop {
+                match tok[i] {
+                    Token::Ident(ref ident) => {
+                        cols.push(ident.clone());
+                        i += 1;
+                        let nx_comma = i < l && tok[i] == (Token::Symbol(Symbol::SymComma));
+                        let nx_close = i < l && tok[i] == (Token::Symbol(Symbol::TtCloseParen));
+                        okay &= nx_comma | nx_close;
+                        stop = nx_close;
+                        i += (nx_comma | nx_close) as usize;
+                    }
+                    _ => {
+                        okay = false;
+                        break;
+                    }
+                }
+            }
+            if okay && stop {
+                cols.into_boxed_slice()
+            } else {
+                return Err(LangError::UnexpectedToken);
+            }
+        }
+        _ => return Err(LangError::ExpectedStatement),
+    };
+    Ok(r)
+}
+
+#[inline(always)]
+pub(super) fn alter_update(_tok: &[Token], _current: &mut usize) -> ExpandedField {
+    todo!()
+}
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 652285f8..9bede639 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -829,7 +829,7 @@ mod schema_tests {
             ")
             .unwrap();
             let schema_name = match tok[2] {
-                Token::Ident(ref id) => unsafe { id.raw_clone() },
+                Token::Ident(ref id) => id.clone(),
                 _ => panic!("expected ident"),
             };
             let tok = &tok[3..];
@@ -868,7 +868,7 @@ mod schema_tests {
             ")
             .unwrap();
             let schema_name = match tok[2] {
-                Token::Ident(ref id) => unsafe { id.raw_clone() },
+                Token::Ident(ref id) => id.clone(),
                 _ => panic!("expected ident"),
             };
             let tok = &tok[3..];
@@ -917,7 +917,7 @@ mod schema_tests {
             ")
             .unwrap();
             let schema_name = match tok[2] {
-                Token::Ident(ref id) => unsafe { id.raw_clone() },
+                Token::Ident(ref id) => id.clone(),
                 _ => panic!("expected ident"),
             };
             let tok = &tok[3..];
@@ -984,7 +984,7 @@ mod schema_tests {
             ")
             .unwrap();
             let schema_name = match tok[2] {
-                Token::Ident(ref id) => unsafe { id.raw_clone() },
+                Token::Ident(ref id) => id.clone(),
                 _ => panic!("expected ident"),
             };
             let tok = &tok[3..];
@@ -1154,4 +1154,39 @@ mod schema_tests {
             );
         }
     }
+    mod alter_model {
+        use super::*;
+        use crate::engine::ql::RawSlice;
+        #[test]
+        fn alter_mini() {
+            let tok = lex(b"alter model mymodel remove myfield").unwrap();
+            let mut i = 4;
+            let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+            assert_eq!(remove, [RawSlice::from("myfield")].into());
+        }
+        #[test]
+        fn alter_mini_2() {
+            let tok = lex(b"alter model mymodel remove (myfield)").unwrap();
+            let mut i = 4;
+            let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+            assert_eq!(remove, [RawSlice::from("myfield")].into());
+        }
+        #[test]
+        fn alter() {
+            let tok = lex(b"alter model mymodel remove (myfield1, myfield2, myfield3, myfield4)")
+                .unwrap();
+            let mut i = 4;
+            let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+            assert_eq!(
+                remove,
+                [
+                    RawSlice::from("myfield1"),
+                    RawSlice::from("myfield2"),
+                    RawSlice::from("myfield3"),
+                    RawSlice::from("myfield4")
+                ]
+                .into()
+            );
+        }
+    }
 }

From a84bb28a6a85ed570f81ed50bb625e779a5abce8 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 2 Oct 2022 22:10:31 +0530
Subject: [PATCH 024/310] Avoid pointless allocations for DDL operations

---
 server/src/engine/ql/schema.rs | 31 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 39846f83..67599e81 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -133,7 +133,7 @@ impl FieldProperties {
 #[derive(Debug, PartialEq)]
 /// A field definition
 pub struct Field {
-    pub(super) field_name: Box<str>,
+    pub(super) field_name: RawSlice,
     pub(super) layers: Vec<Layer>,
     pub(super) props: HashSet<StaticStr>,
 }
@@ -141,20 +141,20 @@ pub struct Field {
 #[derive(Debug, PartialEq)]
 /// A model definition
 pub struct Model {
-    pub(super) model_name: Box<str>,
+    pub(super) model_name: RawSlice,
     pub(super) fields: Vec<Field>,
     pub(super) props: Dict,
 }
 
 #[derive(Debug, PartialEq)]
 pub struct Space {
-    pub(super) space_name: Box<str>,
+    pub(super) space_name: RawSlice,
     pub(super) props: Dict,
 }
 
 #[derive(Debug, PartialEq)]
 pub struct AlterSpace {
-    pub(super) space_name: Box<str>,
+    pub(super) space_name: RawSlice,
     pub(super) updated_props: Dict,
 }
 
@@ -588,7 +588,7 @@ pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
 
     // field name
     let field_name = match (&tok[i], &tok[i + 1]) {
-        (Token::Ident(id), Token::Symbol(Symbol::SymColon)) => unsafe { id.as_str() },
+        (Token::Ident(id), Token::Symbol(Symbol::SymColon)) => id,
         _ => return Err(LangError::UnexpectedToken),
     };
     i += 2;
@@ -603,7 +603,7 @@ pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
         Ok((
             i,
             Field {
-                field_name: field_name.into(),
+                field_name: field_name.clone(),
                 layers,
                 props: props.properties,
             },
@@ -633,7 +633,6 @@ pub(super) fn parse_schema_from_tokens(
     tok: &[Token],
     model_name: RawSlice,
 ) -> LangResult<(Model, usize)> {
-    let model_name = unsafe { model_name.as_str() };
     // parse fields
     let l = tok.len();
     let mut i = 0;
@@ -697,7 +696,7 @@ pub(super) fn parse_schema_from_tokens(
             // sweet, so we got our dict
             Ok((
                 Model {
-                    model_name: model_name.into(),
+                    model_name,
                     props: dict,
                     fields,
                 },
@@ -710,7 +709,7 @@ pub(super) fn parse_schema_from_tokens(
         // we've reached end of stream, so there's nothing more to parse
         Ok((
             Model {
-                model_name: model_name.into(),
+                model_name,
                 props: dict! {},
                 fields,
             },
@@ -721,8 +720,6 @@ pub(super) fn parse_schema_from_tokens(
 
 #[inline(always)]
 pub(super) fn parse_space_from_tokens(tok: &[Token], s: RawSlice) -> LangResult<(Space, usize)> {
-    let space_name = unsafe { s.as_str() };
-
     // let's see if the cursor is at `with`. ignore other tokens because that's fine
     if !tok.is_empty() && tok[0] == (Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With))) {
         // we have a dict
@@ -731,7 +728,7 @@ pub(super) fn parse_space_from_tokens(tok: &[Token], s: RawSlice) -> LangResult<
         if ret & HIBIT == HIBIT {
             Ok((
                 Space {
-                    space_name: space_name.into(),
+                    space_name: s,
                     props: d,
                 },
                 (ret & !HIBIT) as _,
@@ -742,7 +739,7 @@ pub(super) fn parse_space_from_tokens(tok: &[Token], s: RawSlice) -> LangResult<
     } else {
         Ok((
             Space {
-                space_name: space_name.into(),
+                space_name: s,
                 props: dict! {},
             },
             0,
@@ -774,7 +771,7 @@ pub(super) fn parse_alter_space_from_tokens(
     if ret & HIBIT == HIBIT {
         Ok((
             AlterSpace {
-                space_name: unsafe { space_name.as_str() }.into(),
+                space_name,
                 updated_props: d,
             },
             i,
@@ -796,7 +793,7 @@ states! {
 
 #[derive(Debug, PartialEq)]
 pub(super) struct ExpandedField {
-    pub(super) field_name: Box<str>,
+    pub(super) field_name: RawSlice,
     pub(super) props: Dict,
     pub(super) layers: Vec<Layer>,
 }
@@ -813,7 +810,7 @@ pub(super) fn parse_field_syntax(tok: &[Token]) -> LangResult<(ExpandedField, us
         match (&tok[i], state) {
             (Token::Ident(field), FieldSyntaxParseState::IDENT) => {
                 i += 1;
-                tmp = MaybeUninit::new(field);
+                tmp = MaybeUninit::new(field.clone());
                 // expect open brace
                 state = FieldSyntaxParseState::OB;
             }
@@ -859,7 +856,7 @@ pub(super) fn parse_field_syntax(tok: &[Token]) -> LangResult<(ExpandedField, us
     if okay {
         Ok((
             ExpandedField {
-                field_name: unsafe { tmp.assume_init().as_str() }.into(),
+                field_name: unsafe { tmp.assume_init() },
                 layers,
                 props,
             },

From 700204913a200a6e9342c11225098ec24c31a036 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 3 Oct 2022 09:08:33 +0530
Subject: [PATCH 025/310] Enable adding fields using `alter model`

---
 server/src/engine/ql/schema.rs |  67 ++++++++++++++---
 server/src/engine/ql/tests.rs  | 127 ++++++++++++++++++++++++++++++++-
 2 files changed, 184 insertions(+), 10 deletions(-)

diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 67599e81..42736550 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -870,7 +870,7 @@ pub(super) fn parse_field_syntax(tok: &[Token]) -> LangResult<(ExpandedField, us
 #[derive(Debug)]
 #[cfg_attr(debug_assertions, derive(PartialEq))]
 pub(super) enum AlterKind {
-    Add(Field),
+    Add(Box<[ExpandedField]>),
     Remove(Box<[RawSlice]>),
     Update(ExpandedField),
 }
@@ -881,30 +881,78 @@ pub(super) fn parse_alter_kind_from_tokens(
     current: &mut usize,
 ) -> LangResult<AlterKind> {
     let l = tok.len();
-    let mut i = 0;
     if l < 2 {
         return Err(LangError::UnexpectedEndofStatement);
     }
     *current += 1;
-    let r = match tok[i] {
+    let r = match tok[0] {
         Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::Add)) => {
-            AlterKind::Add(alter_add(&tok[1..], &mut i))
+            AlterKind::Add(alter_add(&tok[1..], current)?)
         }
         Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::Remove)) => {
-            AlterKind::Remove(alter_remove(&tok[1..], &mut i)?)
+            AlterKind::Remove(alter_remove(&tok[1..], current)?)
         }
         Token::Keyword(Keyword::Dml(DmlKeyword::Update)) => {
-            AlterKind::Update(alter_update(&tok[1..], &mut i))
+            AlterKind::Update(alter_update(&tok[1..], current))
         }
         _ => return Err(LangError::ExpectedStatement),
     };
-    *current += i;
     Ok(r)
 }
 
 #[inline(always)]
-pub(super) fn alter_add(_tok: &[Token], _current: &mut usize) -> Field {
-    todo!()
+pub(super) fn alter_add(tok: &[Token], current: &mut usize) -> LangResult<Box<[ExpandedField]>> {
+    const DEFAULT_ADD_COL_CNT: usize = 4;
+    /*
+        WARNING: No trailing commas allowed
+
+        <add> ::= (<field_syntax> <comma>)*
+
+        Smallest length:
+        alter model add myfield { type string };
+    */
+    let l = tok.len();
+    if l < 5 {
+        return Err(LangError::UnexpectedEndofStatement);
+    }
+    match tok[0] {
+        Token::Ident(_) => {
+            let (r, i) = parse_field_syntax(&tok)?;
+            *current += i;
+            Ok([r].into())
+        }
+        Token::Symbol(Symbol::TtOpenParen) => {
+            let mut i = 1;
+            let mut okay = true;
+            let mut stop = false;
+            let mut cols = Vec::with_capacity(DEFAULT_ADD_COL_CNT);
+            while i < l && okay && !stop {
+                match tok[i] {
+                    Token::Ident(_) => {
+                        let (r, cnt) = parse_field_syntax(&tok[i..])?;
+                        i += cnt;
+                        cols.push(r);
+                        let nx_comma = i < l && tok[i] == Token::Symbol(Symbol::SymComma);
+                        let nx_close = i < l && tok[i] == Token::Symbol(Symbol::TtCloseParen);
+                        stop = nx_close;
+                        okay &= nx_comma | nx_close;
+                        i += (nx_comma | nx_close) as usize;
+                    }
+                    _ => {
+                        okay = false;
+                        break;
+                    }
+                }
+            }
+            *current += i;
+            if okay && stop {
+                Ok(cols.into_boxed_slice())
+            } else {
+                Err(LangError::UnexpectedToken)
+            }
+        }
+        _ => Err(LangError::ExpectedStatement),
+    }
 }
 
 #[inline(always)]
@@ -946,6 +994,7 @@ pub(super) fn alter_remove(tok: &[Token], current: &mut usize) -> LangResult<Box
                     }
                 }
             }
+            *current += i;
             if okay && stop {
                 cols.into_boxed_slice()
             } else {
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 9bede639..67539bcb 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -1154,7 +1154,7 @@ mod schema_tests {
             );
         }
     }
-    mod alter_model {
+    mod alter_model_remove {
         use super::*;
         use crate::engine::ql::RawSlice;
         #[test]
@@ -1162,6 +1162,7 @@ mod schema_tests {
             let tok = lex(b"alter model mymodel remove myfield").unwrap();
             let mut i = 4;
             let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+            assert_eq!(i, tok.len());
             assert_eq!(remove, [RawSlice::from("myfield")].into());
         }
         #[test]
@@ -1169,6 +1170,7 @@ mod schema_tests {
             let tok = lex(b"alter model mymodel remove (myfield)").unwrap();
             let mut i = 4;
             let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+            assert_eq!(i, tok.len());
             assert_eq!(remove, [RawSlice::from("myfield")].into());
         }
         #[test]
@@ -1177,6 +1179,7 @@ mod schema_tests {
                 .unwrap();
             let mut i = 4;
             let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+            assert_eq!(i, tok.len());
             assert_eq!(
                 remove,
                 [
@@ -1189,4 +1192,126 @@ mod schema_tests {
             );
         }
     }
+    mod alter_model_add {
+        use super::*;
+        use crate::engine::ql::{
+            lexer::Type,
+            schema::{ExpandedField, Layer},
+        };
+        #[test]
+        fn add_mini() {
+            let tok = lex(b"
+                alter model mymodel add myfield { type string }
+            ")
+            .unwrap();
+            let mut i = 4;
+            let r = schema::alter_add(&tok[i..], &mut i).unwrap();
+            assert_eq!(i, tok.len());
+            assert_eq!(
+                r.as_ref(),
+                [ExpandedField {
+                    field_name: "myfield".into(),
+                    props: dict! {},
+                    layers: [Layer::new(Type::String, dict! {})].into()
+                }]
+            );
+        }
+        #[test]
+        fn add() {
+            let tok = lex(b"
+                alter model mymodel add myfield { type string, nullable: true }
+            ")
+            .unwrap();
+            let mut i = 4;
+            let r = schema::alter_add(&tok[i..], &mut i).unwrap();
+            assert_eq!(i, tok.len());
+            assert_eq!(
+                r.as_ref(),
+                [ExpandedField {
+                    field_name: "myfield".into(),
+                    props: dict! {
+                        "nullable" => Lit::Bool(true)
+                    },
+                    layers: [Layer::new(Type::String, dict! {})].into()
+                }]
+            );
+        }
+        #[test]
+        fn add_pro() {
+            let tok = lex(b"
+                alter model mymodel add (myfield { type string, nullable: true })
+            ")
+            .unwrap();
+            let mut i = 4;
+            let r = schema::alter_add(&tok[i..], &mut i).unwrap();
+            assert_eq!(i, tok.len());
+            assert_eq!(
+                r.as_ref(),
+                [ExpandedField {
+                    field_name: "myfield".into(),
+                    props: dict! {
+                        "nullable" => Lit::Bool(true)
+                    },
+                    layers: [Layer::new(Type::String, dict! {})].into()
+                }]
+            );
+        }
+        #[test]
+        fn add_pro_max() {
+            let tok = lex(b"
+                alter model mymodel add (
+                    myfield {
+                        type string,
+                        nullable: true
+                    },
+                    another {
+                        type list {
+                            type string {
+                                maxlen: 255
+                            },
+                            unique: true
+                        },
+                        nullable: false
+                    }
+                )
+            ")
+            .unwrap();
+            let mut i = 4;
+            let r = schema::alter_add(&tok[i..], &mut i).unwrap();
+            assert_eq!(i, tok.len());
+            assert_eq!(
+                r.as_ref(),
+                [
+                    ExpandedField {
+                        field_name: "myfield".into(),
+                        props: dict! {
+                            "nullable" => Lit::Bool(true)
+                        },
+                        layers: [Layer::new(Type::String, dict! {})].into()
+                    },
+                    ExpandedField {
+                        field_name: "another".into(),
+                        props: dict! {
+                            "nullable" => Lit::Bool(false)
+                        },
+                        layers: [
+                            Layer::new(
+                                Type::String,
+                                dict! {
+                                    "maxlen" => Lit::Num(255)
+                                }
+                            ),
+                            Layer::new(
+                                Type::List,
+                                dict! {
+                                   "unique" => Lit::Bool(true)
+                                }
+                            )
+                        ]
+                        .into()
+                    }
+                ]
+            );
+        }
+    }
 }

From 7a5b95840285aeea65d3789b71766c7f6d6d9064 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 3 Oct 2022 14:38:24 +0530
Subject: [PATCH 026/310] Support reset syntax

---
 server/src/engine/ql/mod.rs    |  12 +-
 server/src/engine/ql/schema.rs | 186 +++++++++++++++++-----
 server/src/engine/ql/tests.rs  | 271 ++++++++++++++++++++++++++-------
 3 files changed, 375 insertions(+), 94 deletions(-)

diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 39bc66b9..180292ee 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -96,14 +96,10 @@ impl RawSlice {
 #[cfg(debug_assertions)]
 impl fmt::Debug for RawSlice {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_list()
-            .entries(unsafe {
-                // UNSAFE(@ohsayan): Note, the caller is responsible for ensuring validity as long the
-                // slice is used. also note, the Debug impl only exists for Debug builds so we never use
-                // this in release builds
-                self.as_slice()
-            })
-            .finish()
+        f.write_str(unsafe {
+            // UNSAFE(@ohsayan): Only implemented in debug
+            self.as_str()
+        })
     }
 }
 
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 42736550..de60d973 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -78,12 +78,17 @@ macro_rules! states {
     }
 }
 
+/// A static string slice
 type StaticStr = &'static str;
 
 const HIBIT: u64 = 1 << 63;
-const TRAIL_COMMA: bool = true;
+/// Flag for disallowing the `..` syntax
+const DISALLOW_RESET_SYNTAX: bool = false;
+/// Flag for allowing the `..` syntax
+const ALLOW_RESET_SYNTAX: bool = true;
 
 #[derive(Debug, PartialEq)]
+/// A dictionary entry type. Either a literal or another dictionary
 pub enum DictEntry {
     Lit(Lit),
     Map(Dict),
@@ -101,21 +106,34 @@ impl From<Dict> for DictEntry {
     }
 }
 
+/// A metadata dictionary
 pub type Dict = HashMap<String, DictEntry>;
 
 #[derive(Debug, PartialEq)]
+/// A layer contains a type and corresponding metadata
 pub struct Layer {
     ty: Type,
     props: Dict,
+    reset: bool,
 }
 
 impl Layer {
-    pub(super) const fn new(ty: Type, props: Dict) -> Self {
-        Self { ty, props }
+    //// Create a new layer
+    pub(super) const fn new(ty: Type, props: Dict, reset: bool) -> Self {
+        Self { ty, props, reset }
+    }
+    /// Create a new layer that doesn't have any reset
+    pub(super) const fn new_noreset(ty: Type, props: Dict) -> Self {
+        Self::new(ty, props, false)
+    }
+    /// Create a new layer that adds a reset
+    pub(super) const fn new_reset(ty: Type, props: Dict) -> Self {
+        Self::new(ty, props, true)
     }
 }
 
 #[derive(Debug, Default, PartialEq, Eq)]
+/// Field properties
 pub struct FieldProperties {
     pub(super) properties: HashSet<StaticStr>,
 }
@@ -133,26 +151,36 @@ impl FieldProperties {
 #[derive(Debug, PartialEq)]
 /// A field definition
 pub struct Field {
+    /// the field name
     pub(super) field_name: RawSlice,
+    /// layers
     pub(super) layers: Vec<Layer>,
+    /// properties
     pub(super) props: HashSet<StaticStr>,
 }
 
 #[derive(Debug, PartialEq)]
 /// A model definition
 pub struct Model {
+    /// the model name
     pub(super) model_name: RawSlice,
+    /// the fields
     pub(super) fields: Vec<Field>,
+    /// properties
     pub(super) props: Dict,
 }
 
 #[derive(Debug, PartialEq)]
+/// A space
 pub struct Space {
+    /// the space name
     pub(super) space_name: RawSlice,
+    /// properties
     pub(super) props: Dict,
 }
 
 #[derive(Debug, PartialEq)]
+/// An alter space query with corresponding data
 pub struct AlterSpace {
     pub(super) space_name: RawSlice,
     pub(super) updated_props: Dict,
@@ -174,6 +202,7 @@ states! {
     }
 }
 
+/// Fold a dictionary
 pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dict) -> u64 {
     /*
         NOTE: Assume rules wherever applicable
@@ -260,6 +289,7 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
 }
 
 #[cfg(test)]
+/// Fold a dictionary (**test-only**)
 pub fn fold_dict(tok: &[Token]) -> Option<Dict> {
     let mut d = Dict::new();
     let r = rfold_dict(DictFoldState::OB, tok, &mut d);
@@ -281,58 +311,84 @@ states! {
         COLON = 0x01,
         LIT_OR_OB = 0x02,
         COMMA_OR_CB = 0x03,
+        CB = 0x04,
         FINAL = 0xFF,
     }
 }
 
+#[derive(Debug, PartialEq)]
+/// The result of a type metadata fold
 pub struct TyMetaFoldResult {
     c: usize,
-    b: [bool; 2],
+    b: [bool; 3],
 }
 
 impl TyMetaFoldResult {
     #[inline(always)]
+    /// Create a new [`TyMetaFoldResult`] with the default settings:
+    /// - reset: false
+    /// - more: false
+    /// - okay: true
     const fn new() -> Self {
         Self {
             c: 0,
-            b: [true, false],
+            b: [true, false, false],
         }
     }
     #[inline(always)]
+    /// Increment the position
     fn incr(&mut self) {
         self.incr_by(1)
     }
     #[inline(always)]
+    /// Increment the position by `by`
     fn incr_by(&mut self, by: usize) {
         self.c += by;
     }
     #[inline(always)]
+    /// Set fail
     fn set_fail(&mut self) {
         self.b[0] = false;
     }
     #[inline(always)]
+    /// Set has more
     fn set_has_more(&mut self) {
         self.b[1] = true;
     }
     #[inline(always)]
+    /// Set reset
+    fn set_reset(&mut self) {
+        self.b[2] = true;
+    }
+    #[inline(always)]
+    /// Should the meta be reset?
+    pub fn should_reset(&self) -> bool {
+        self.b[2]
+    }
+    #[inline(always)]
+    /// Returns the cursor
     pub fn pos(&self) -> usize {
         self.c
     }
     #[inline(always)]
+    /// Returns if more layers are expected
     pub fn has_more(&self) -> bool {
         self.b[1]
     }
     #[inline(always)]
+    /// Returns if the internal state is okay
     pub fn is_okay(&self) -> bool {
         self.b[0]
     }
     #[inline(always)]
+    /// Records an expression
     fn record(&mut self, c: bool) {
         self.b[0] &= c;
     }
 }
 
-pub(super) fn rfold_tymeta(
+/// Fold type metadata (flag setup dependent on caller)
+pub(super) fn rfold_tymeta<const ALLOW_RESET: bool>(
     mut state: TyMetaFoldState,
     tok: &[Token],
     dict: &mut Dict,
@@ -352,9 +408,17 @@ pub(super) fn rfold_tymeta(
                 state = TyMetaFoldState::FINAL;
                 break;
             }
+            (Token::Symbol(Symbol::SymPeriod), TyMetaFoldState::IDENT_OR_CB) if ALLOW_RESET => {
+                r.incr();
+                let reset = r.pos() < l && tok[r.pos()] == Token::Symbol(Symbol::SymPeriod);
+                r.incr_by(reset as _);
+                r.record(reset);
+                r.set_reset();
+                state = TyMetaFoldState::CB;
+            }
             (
                 Token::Symbol(Symbol::TtCloseBrace),
-                TyMetaFoldState::IDENT_OR_CB | TyMetaFoldState::COMMA_OR_CB,
+                TyMetaFoldState::IDENT_OR_CB | TyMetaFoldState::COMMA_OR_CB | TyMetaFoldState::CB,
             ) => {
                 r.incr();
                 // found close brace. end of stream
@@ -393,7 +457,11 @@ pub(super) fn rfold_tymeta(
                 r.incr();
                 // another dict in here
                 let mut d = Dict::new();
-                let ret = rfold_tymeta(TyMetaFoldState::IDENT_OR_CB, &tok[r.pos()..], &mut d);
+                let ret = rfold_tymeta::<ALLOW_RESET>(
+                    TyMetaFoldState::IDENT_OR_CB,
+                    &tok[r.pos()..],
+                    &mut d,
+                );
                 r.incr_by(ret.pos());
                 r.record(ret.is_okay());
                 r.record(!ret.has_more()); // L2 cannot have type definitions
@@ -415,9 +483,10 @@ pub(super) fn rfold_tymeta(
 }
 
 #[cfg(test)]
+/// (**test-only**) fold type metadata
 pub(super) fn fold_tymeta(tok: &[Token]) -> (TyMetaFoldResult, Dict) {
     let mut d = Dict::new();
-    let r = rfold_tymeta(TyMetaFoldState::IDENT_OR_CB, tok, &mut d);
+    let r = rfold_tymeta::<DISALLOW_RESET_SYNTAX>(TyMetaFoldState::IDENT_OR_CB, tok, &mut d);
     (r, d)
 }
 
@@ -435,7 +504,12 @@ states! {
     }
 }
 
-pub(super) fn rfold_layers(start: LayerFoldState, tok: &[Token], layers: &mut Vec<Layer>) -> u64 {
+/// Fold layers
+pub(super) fn rfold_layers<const ALLOW_RESET: bool>(
+    start: LayerFoldState,
+    tok: &[Token],
+    layers: &mut Vec<Layer>,
+) -> u64 {
     /*
         NOTE: Assume rules wherever applicable
 
@@ -466,12 +540,13 @@ pub(super) fn rfold_layers(start: LayerFoldState, tok: &[Token], layers: &mut Ve
             (Token::Symbol(Symbol::TtOpenBrace), LayerFoldState::END_OR_OB) => {
                 i += 1;
                 // since we found an open brace, this type has some meta
-                let ret = rfold_tymeta(TyMetaFoldState::IDENT_OR_CB, &tok[i..], &mut dict);
+                let ret =
+                    rfold_tymeta::<ALLOW_RESET>(TyMetaFoldState::IDENT_OR_CB, &tok[i..], &mut dict);
                 i += ret.pos();
                 okay &= ret.is_okay();
                 if ret.has_more() {
                     // more layers
-                    let ret = rfold_layers(LayerFoldState::TY, &tok[i..], layers);
+                    let ret = rfold_layers::<ALLOW_RESET>(LayerFoldState::TY, &tok[i..], layers);
                     okay &= ret & HIBIT == HIBIT;
                     i += (ret & !HIBIT) as usize;
                     state = LayerFoldState::FOLD_DICT_INCOMPLETE;
@@ -481,6 +556,7 @@ pub(super) fn rfold_layers(start: LayerFoldState, tok: &[Token], layers: &mut Ve
                     layers.push(Layer {
                         ty: unsafe { tmp.assume_init() }.clone(),
                         props: dict,
+                        reset: ret.should_reset(),
                     });
                     break;
                 }
@@ -488,7 +564,8 @@ pub(super) fn rfold_layers(start: LayerFoldState, tok: &[Token], layers: &mut Ve
             (Token::Symbol(Symbol::SymComma), LayerFoldState::FOLD_DICT_INCOMPLETE) => {
                 // there is a comma at the end of this
                 i += 1;
-                let ret = rfold_tymeta(TyMetaFoldState::IDENT_OR_CB, &tok[i..], &mut dict);
+                let ret =
+                    rfold_tymeta::<ALLOW_RESET>(TyMetaFoldState::IDENT_OR_CB, &tok[i..], &mut dict);
                 i += ret.pos();
                 okay &= ret.is_okay();
                 okay &= !ret.has_more(); // not more than one type depth
@@ -498,6 +575,7 @@ pub(super) fn rfold_layers(start: LayerFoldState, tok: &[Token], layers: &mut Ve
                     layers.push(Layer {
                         ty: unsafe { tmp.assume_init() }.clone(),
                         props: dict,
+                        reset: ret.should_reset(),
                     });
                     break;
                 }
@@ -509,6 +587,7 @@ pub(super) fn rfold_layers(start: LayerFoldState, tok: &[Token], layers: &mut Ve
                 layers.push(Layer {
                     ty: unsafe { tmp.assume_init() }.clone(),
                     props: dict,
+                    reset: false,
                 });
                 break;
             }
@@ -518,6 +597,7 @@ pub(super) fn rfold_layers(start: LayerFoldState, tok: &[Token], layers: &mut Ve
                 layers.push(Layer {
                     ty: unsafe { tmp.assume_init() }.clone(),
                     props: dict,
+                    reset: false,
                 });
                 break;
             }
@@ -533,13 +613,15 @@ pub(super) fn rfold_layers(start: LayerFoldState, tok: &[Token], layers: &mut Ve
 
 #[cfg(test)]
 #[inline(always)]
+/// (**test-only**) fold layers
 pub(super) fn fold_layers(tok: &[Token]) -> (Vec<Layer>, usize, bool) {
     let mut l = Vec::new();
-    let r = rfold_layers(LayerFoldState::TY, tok, &mut l);
+    let r = rfold_layers::<DISALLOW_RESET_SYNTAX>(LayerFoldState::TY, tok, &mut l);
     (l, (r & !HIBIT) as _, r & HIBIT == HIBIT)
 }
 
 #[inline(always)]
+/// Collect field properties
 pub(super) fn collect_field_properties(tok: &[Token]) -> (FieldProperties, u64) {
     let mut props = FieldProperties::default();
     let mut i = 0;
@@ -567,12 +649,14 @@ pub(super) fn collect_field_properties(tok: &[Token]) -> (FieldProperties, u64)
 
 #[cfg(test)]
 #[inline(always)]
+/// (**test-only**) parse field properties
 pub(super) fn parse_field_properties(tok: &[Token]) -> (FieldProperties, usize, bool) {
     let (p, r) = collect_field_properties(tok);
     (p, (r & !HIBIT) as _, r & HIBIT == HIBIT)
 }
 
 #[inline(always)]
+/// Parse a field using the declaration-syntax (not field syntax)
 pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
     let l = tok.len();
     let mut i = 0;
@@ -595,7 +679,7 @@ pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
 
     // layers
     let mut layers = Vec::new();
-    let r = rfold_layers(LayerFoldState::TY, &tok[i..], &mut layers);
+    let r = rfold_layers::<DISALLOW_RESET_SYNTAX>(LayerFoldState::TY, &tok[i..], &mut layers);
     okay &= r & HIBIT == HIBIT;
     i += (r & !HIBIT) as usize;
 
@@ -613,13 +697,8 @@ pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
     }
 }
 
-/*
-    create model name(..) with { .. }
-                     ^^^^
-*/
-
 states! {
-    ///
+    /// Accept state for a schema parse
     pub struct SchemaParseState: u8 {
         OPEN_PAREN = 0x00,
         FIELD = 0x01,
@@ -629,6 +708,7 @@ states! {
 }
 
 #[inline(always)]
+/// Parse a fresh schema with declaration-syntax fields
 pub(super) fn parse_schema_from_tokens(
     tok: &[Token],
     model_name: RawSlice,
@@ -719,6 +799,7 @@ pub(super) fn parse_schema_from_tokens(
 }
 
 #[inline(always)]
+/// Parse space data from the given tokens
 pub(super) fn parse_space_from_tokens(tok: &[Token], s: RawSlice) -> LangResult<(Space, usize)> {
     // let's see if the cursor is at `with`. ignore other tokens because that's fine
     if !tok.is_empty() && tok[0] == (Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With))) {
@@ -747,6 +828,8 @@ pub(super) fn parse_space_from_tokens(tok: &[Token], s: RawSlice) -> LangResult<
     }
 }
 
+#[inline(always)]
+/// Parse alter space from tokens
 pub(super) fn parse_alter_space_from_tokens(
     tok: &[Token],
     space_name: RawSlice,
@@ -792,13 +875,19 @@ states! {
 }
 
 #[derive(Debug, PartialEq)]
+/// An [`ExpandedField`] is a full field definition with advanced metadata
 pub(super) struct ExpandedField {
     pub(super) field_name: RawSlice,
     pub(super) props: Dict,
     pub(super) layers: Vec<Layer>,
+    pub(super) reset: bool,
 }
 
-pub(super) fn parse_field_syntax(tok: &[Token]) -> LangResult<(ExpandedField, usize)> {
+#[inline(always)]
+/// Parse a field declared using the field syntax
+pub(super) fn parse_field_syntax<const ALLOW_RESET: bool>(
+    tok: &[Token],
+) -> LangResult<(ExpandedField, usize)> {
     let l = tok.len();
     let mut i = 0_usize;
     let mut state = FieldSyntaxParseState::IDENT;
@@ -806,6 +895,7 @@ pub(super) fn parse_field_syntax(tok: &[Token]) -> LangResult<(ExpandedField, us
     let mut tmp = MaybeUninit::uninit();
     let mut props = Dict::new();
     let mut layers = vec![];
+    let mut reset = false;
     while i < l && okay {
         match (&tok[i], state) {
             (Token::Ident(field), FieldSyntaxParseState::IDENT) => {
@@ -816,12 +906,20 @@ pub(super) fn parse_field_syntax(tok: &[Token]) -> LangResult<(ExpandedField, us
             }
             (Token::Symbol(Symbol::TtOpenBrace), FieldSyntaxParseState::OB) => {
                 i += 1;
-                let r = self::rfold_tymeta(TyMetaFoldState::IDENT_OR_CB, &tok[i..], &mut props);
+                let r = self::rfold_tymeta::<ALLOW_RESET>(
+                    TyMetaFoldState::IDENT_OR_CB,
+                    &tok[i..],
+                    &mut props,
+                );
                 okay &= r.is_okay();
                 i += r.pos();
                 if r.has_more() && i < l {
                     // now parse layers
-                    let r = self::rfold_layers(LayerFoldState::TY, &tok[i..], &mut layers);
+                    let r = self::rfold_layers::<ALLOW_RESET>(
+                        LayerFoldState::TY,
+                        &tok[i..],
+                        &mut layers,
+                    );
                     okay &= r & HIBIT == HIBIT;
                     i += (r & !HIBIT) as usize;
                     state = FieldSyntaxParseState::FOLD_DICT_INCOMPLETE;
@@ -832,9 +930,14 @@ pub(super) fn parse_field_syntax(tok: &[Token]) -> LangResult<(ExpandedField, us
             }
             (Token::Symbol(Symbol::SymComma), FieldSyntaxParseState::FOLD_DICT_INCOMPLETE) => {
                 i += 1;
-                let r = self::rfold_dict(DictFoldState::CB_OR_IDENT, &tok[i..], &mut props);
-                okay &= r & HIBIT == HIBIT;
-                i += (r & !HIBIT) as usize;
+                let r = self::rfold_tymeta::<ALLOW_RESET>(
+                    TyMetaFoldState::IDENT_OR_CB,
+                    &tok[i..],
+                    &mut props,
+                );
+                okay &= r.is_okay() && !r.has_more();
+                i += r.pos();
+                reset = ALLOW_RESET && r.should_reset();
                 if okay {
                     state = FieldSyntaxParseState::COMPLETED;
                     break;
@@ -859,6 +962,7 @@ pub(super) fn parse_field_syntax(tok: &[Token]) -> LangResult<(ExpandedField, us
                 field_name: unsafe { tmp.assume_init() },
                 layers,
                 props,
+                reset,
             },
             i,
         ))
@@ -869,13 +973,15 @@ pub(super) fn parse_field_syntax(tok: &[Token]) -> LangResult<(ExpandedField, us
 
 #[derive(Debug)]
 #[cfg_attr(debug_assertions, derive(PartialEq))]
+/// The alter operation kind
 pub(super) enum AlterKind {
     Add(Box<[ExpandedField]>),
     Remove(Box<[RawSlice]>),
-    Update(ExpandedField),
+    Update(Box<[ExpandedField]>),
 }
 
 #[inline(always)]
+/// Parse an [`AlterKind`] from the given token stream
 pub(super) fn parse_alter_kind_from_tokens(
     tok: &[Token],
     current: &mut usize,
@@ -893,7 +999,7 @@ pub(super) fn parse_alter_kind_from_tokens(
             AlterKind::Remove(alter_remove(&tok[1..], current)?)
         }
         Token::Keyword(Keyword::Dml(DmlKeyword::Update)) => {
-            AlterKind::Update(alter_update(&tok[1..], current))
+            AlterKind::Update(alter_update(&tok[1..], current)?)
         }
         _ => return Err(LangError::ExpectedStatement),
     };
@@ -901,7 +1007,11 @@ pub(super) fn parse_alter_kind_from_tokens(
 }
 
 #[inline(always)]
-pub(super) fn alter_add(tok: &[Token], current: &mut usize) -> LangResult<Box<[ExpandedField]>> {
+/// Parse multiple fields declared using the field syntax. Flag setting allows or disallows reset syntax
+pub(super) fn parse_multiple_field_syntax<const ALLOW_RESET: bool>(
+    tok: &[Token],
+    current: &mut usize,
+) -> LangResult<Box<[ExpandedField]>> {
     const DEFAULT_ADD_COL_CNT: usize = 4;
     /*
         WARNING: No trailing commas allowed
@@ -917,7 +1027,7 @@ pub(super) fn alter_add(tok: &[Token], current: &mut usize) -> LangResult<Box<[E
     }
     match tok[0] {
         Token::Ident(_) => {
-            let (r, i) = parse_field_syntax(&tok)?;
+            let (r, i) = parse_field_syntax::<ALLOW_RESET>(&tok)?;
             *current += i;
             Ok([r].into())
         }
@@ -929,7 +1039,7 @@ pub(super) fn alter_add(tok: &[Token], current: &mut usize) -> LangResult<Box<[E
             while i < l && okay && !stop {
                 match tok[i] {
                     Token::Ident(_) => {
-                        let (r, cnt) = parse_field_syntax(&tok[i..])?;
+                        let (r, cnt) = parse_field_syntax::<ALLOW_RESET>(&tok[i..])?;
                         i += cnt;
                         cols.push(r);
                         let nx_comma = i < l && tok[i] == Token::Symbol(Symbol::SymComma);
@@ -956,6 +1066,13 @@ pub(super) fn alter_add(tok: &[Token], current: &mut usize) -> LangResult<Box<[E
 }
 
 #[inline(always)]
+/// Parse the expression for `alter model <> add (..)`
+pub(super) fn alter_add(tok: &[Token], current: &mut usize) -> LangResult<Box<[ExpandedField]>> {
+    self::parse_multiple_field_syntax::<DISALLOW_RESET_SYNTAX>(tok, current)
+}
+
+#[inline(always)]
+/// Parse the expression for `alter model <> remove (..)`
 pub(super) fn alter_remove(tok: &[Token], current: &mut usize) -> LangResult<Box<[RawSlice]>> {
     const DEFAULT_REMOVE_COL_CNT: usize = 4;
     /*
@@ -1007,6 +1124,7 @@ pub(super) fn alter_remove(tok: &[Token], current: &mut usize) -> LangResult<Box
 }
 
 #[inline(always)]
-pub(super) fn alter_update(_tok: &[Token], _current: &mut usize) -> ExpandedField {
-    todo!()
+/// Parse the expression for `alter model <> update (..)`
+pub(super) fn alter_update(tok: &[Token], current: &mut usize) -> LangResult<Box<[ExpandedField]>> {
+    self::parse_multiple_field_syntax::<ALLOW_RESET_SYNTAX>(tok, current)
 }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 67539bcb..18d247e0 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -559,7 +559,7 @@ mod schema_tests {
             let (layers, c, okay) = schema::fold_layers(&tok);
             assert_eq!(c, tok.len() - 1);
             assert!(okay);
-            assert_eq!(layers, vec![Layer::new(Type::String, dict! {})]);
+            assert_eq!(layers, vec![Layer::new_noreset(Type::String, dict! {})]);
         }
         #[test]
         fn layer() {
@@ -569,7 +569,7 @@ mod schema_tests {
             assert!(okay);
             assert_eq!(
                 layers,
-                vec![Layer::new(
+                vec![Layer::new_noreset(
                     Type::String,
                     dict! {
                         "maxlen" => Lit::Num(100)
@@ -586,8 +586,8 @@ mod schema_tests {
             assert_eq!(
                 layers,
                 vec![
-                    Layer::new(Type::String, dict! {}),
-                    Layer::new(Type::List, dict! {})
+                    Layer::new_noreset(Type::String, dict! {}),
+                    Layer::new_noreset(Type::List, dict! {})
                 ]
             );
         }
@@ -600,8 +600,8 @@ mod schema_tests {
             assert_eq!(
                 layers,
                 vec![
-                    Layer::new(Type::String, dict! {}),
-                    Layer::new(
+                    Layer::new_noreset(Type::String, dict! {}),
+                    Layer::new_noreset(
                         Type::List,
                         dict! {
                             "unique" => Lit::Bool(true),
@@ -623,14 +623,14 @@ mod schema_tests {
             assert_eq!(
                 layers,
                 vec![
-                    Layer::new(
+                    Layer::new_noreset(
                         Type::String,
                         dict! {
                             "ascii_only" => Lit::Bool(true),
                             "maxlen" => Lit::Num(255)
                         }
                     ),
-                    Layer::new(
+                    Layer::new_noreset(
                         Type::List,
                         dict! {
                             "unique" => Lit::Bool(true),
@@ -654,14 +654,14 @@ mod schema_tests {
         ")
             .unwrap();
             let expected = vec![
-                Layer::new(Type::String, dict!()),
-                Layer::new(
+                Layer::new_noreset(Type::String, dict!()),
+                Layer::new_noreset(
                     Type::List,
                     dict! {
                         "maxlen" => Lit::Num(100),
                     },
                 ),
-                Layer::new(Type::List, dict!("unique" => Lit::Bool(true))),
+                Layer::new_noreset(Type::List, dict!("unique" => Lit::Bool(true))),
             ];
             fuzz_tokens(&tok, |should_pass, new_tok| {
                 let (layers, c, okay) = schema::fold_layers(&new_tok);
@@ -724,7 +724,7 @@ mod schema_tests {
                 f,
                 Field {
                     field_name: "username".into(),
-                    layers: [Layer::new(Type::String, dict! {})].into(),
+                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
                     props: set![],
                 }
             )
@@ -741,7 +741,7 @@ mod schema_tests {
                 f,
                 Field {
                     field_name: "username".into(),
-                    layers: [Layer::new(Type::String, dict! {})].into(),
+                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
                     props: set!["primary"],
                 }
             )
@@ -761,7 +761,7 @@ mod schema_tests {
                 f,
                 Field {
                     field_name: "username".into(),
-                    layers: [Layer::new(
+                    layers: [Layer::new_noreset(
                         Type::String,
                         dict! {
                             "maxlen" => Lit::Num(10),
@@ -792,14 +792,14 @@ mod schema_tests {
                 Field {
                     field_name: "notes".into(),
                     layers: [
-                        Layer::new(
+                        Layer::new_noreset(
                             Type::String,
                             dict! {
                                 "maxlen" => Lit::Num(255),
                                 "ascii_only" => Lit::Bool(true),
                             }
                         ),
-                        Layer::new(
+                        Layer::new_noreset(
                             Type::List,
                             dict! {
                                 "unique" => Lit::Bool(true)
@@ -844,12 +844,12 @@ mod schema_tests {
                     fields: vec![
                         Field {
                             field_name: "username".into(),
-                            layers: vec![Layer::new(Type::String, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::String, dict! {})],
                             props: set!["primary"]
                         },
                         Field {
                             field_name: "password".into(),
-                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
                             props: set![]
                         }
                     ],
@@ -883,17 +883,17 @@ mod schema_tests {
                     fields: vec![
                         Field {
                             field_name: "username".into(),
-                            layers: vec![Layer::new(Type::String, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::String, dict! {})],
                             props: set!["primary"]
                         },
                         Field {
                             field_name: "password".into(),
-                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
                             props: set![]
                         },
                         Field {
                             field_name: "profile_pic".into(),
-                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
                             props: set!["null"]
                         }
                     ],
@@ -932,24 +932,24 @@ mod schema_tests {
                     fields: vec![
                         Field {
                             field_name: "username".into(),
-                            layers: vec![Layer::new(Type::String, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::String, dict! {})],
                             props: set!["primary"]
                         },
                         Field {
                             field_name: "password".into(),
-                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
                             props: set![]
                         },
                         Field {
                             field_name: "profile_pic".into(),
-                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
                             props: set!["null"]
                         },
                         Field {
                             field_name: "notes".into(),
                             layers: vec![
-                                Layer::new(Type::String, dict! {}),
-                                Layer::new(
+                                Layer::new_noreset(Type::String, dict! {}),
+                                Layer::new_noreset(
                                     Type::List,
                                     dict! {
                                         "unique" => Lit::Bool(true)
@@ -999,24 +999,24 @@ mod schema_tests {
                     fields: vec![
                         Field {
                             field_name: "username".into(),
-                            layers: vec![Layer::new(Type::String, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::String, dict! {})],
                             props: set!["primary"]
                         },
                         Field {
                             field_name: "password".into(),
-                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
                             props: set![]
                         },
                         Field {
                             field_name: "profile_pic".into(),
-                            layers: vec![Layer::new(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
                             props: set!["null"]
                         },
                         Field {
                             field_name: "notes".into(),
                             layers: vec![
-                                Layer::new(Type::String, dict! {}),
-                                Layer::new(
+                                Layer::new_noreset(Type::String, dict! {}),
+                                Layer::new_noreset(
                                     Type::List,
                                     dict! {
                                         "unique" => Lit::Bool(true)
@@ -1045,14 +1045,15 @@ mod schema_tests {
         #[test]
         fn field_syn_mini() {
             let tok = lex(b"username { type string }").unwrap();
-            let (ef, i) = schema::parse_field_syntax(&tok).unwrap();
+            let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
             assert_eq!(i, tok.len());
             assert_eq!(
                 ef,
                 ExpandedField {
                     field_name: "username".into(),
-                    layers: vec![Layer::new(Type::String, dict! {})],
-                    props: dict! {}
+                    layers: vec![Layer::new_noreset(Type::String, dict! {})],
+                    props: dict! {},
+                    reset: false
                 }
             )
         }
@@ -1065,7 +1066,7 @@ mod schema_tests {
                 }
             ")
             .unwrap();
-            let (ef, i) = schema::parse_field_syntax(&tok).unwrap();
+            let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
             assert_eq!(i, tok.len());
             assert_eq!(
                 ef,
@@ -1074,7 +1075,8 @@ mod schema_tests {
                     props: dict! {
                         "nullable" => Lit::Bool(false),
                     },
-                    layers: vec![Layer::new(Type::String, dict! {})]
+                    layers: vec![Layer::new_noreset(Type::String, dict! {})],
+                    reset: false
                 }
             );
         }
@@ -1091,7 +1093,7 @@ mod schema_tests {
                 }
             ")
             .unwrap();
-            let (ef, i) = schema::parse_field_syntax(&tok).unwrap();
+            let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
             assert_eq!(i, tok.len());
             assert_eq!(
                 ef,
@@ -1101,13 +1103,14 @@ mod schema_tests {
                         "nullable" => Lit::Bool(false),
                         "jingle_bells" => Lit::Str("snow".into()),
                     },
-                    layers: vec![Layer::new(
+                    layers: vec![Layer::new_noreset(
                         Type::String,
                         dict! {
                             "minlen" => Lit::Num(6),
                             "maxlen" => Lit::Num(255),
                         }
-                    )]
+                    )],
+                    reset: false
                 }
             );
         }
@@ -1126,7 +1129,7 @@ mod schema_tests {
                 }
             ")
             .unwrap();
-            let (ef, i) = schema::parse_field_syntax(&tok).unwrap();
+            let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
             assert_eq!(i, tok.len());
             assert_eq!(
                 ef,
@@ -1137,19 +1140,20 @@ mod schema_tests {
                         "jingle_bells" => Lit::Str("snow".into()),
                     },
                     layers: vec![
-                        Layer::new(
+                        Layer::new_noreset(
                             Type::String,
                             dict! {
                                 "ascii_only" => Lit::Bool(true),
                             }
                         ),
-                        Layer::new(
+                        Layer::new_noreset(
                             Type::List,
                             dict! {
                                 "unique" => Lit::Bool(true),
                             }
                         )
-                    ]
+                    ],
+                    reset: false
                 }
             );
         }
@@ -1212,7 +1216,8 @@ mod schema_tests {
                 [ExpandedField {
                     field_name: "myfield".into(),
                     props: dict! {},
-                    layers: [Layer::new(Type::String, dict! {})].into()
+                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    reset: false
                 }]
             );
         }
@@ -1232,7 +1237,8 @@ mod schema_tests {
                     props: dict! {
                         "nullable" => Lit::Bool(true)
                     },
-                    layers: [Layer::new(Type::String, dict! {})].into()
+                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    reset: false
                 }]
             );
         }
@@ -1252,7 +1258,8 @@ mod schema_tests {
                     props: dict! {
                         "nullable" => Lit::Bool(true)
                     },
-                    layers: [Layer::new(Type::String, dict! {})].into()
+                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    reset: false
                 }]
             );
         }
@@ -1271,7 +1278,7 @@ mod schema_tests {
                             },
                             unique: true
                         },
-                        nullable: false
+                        nullable: false,
                     }
                 )
             ")
@@ -1287,7 +1294,8 @@ mod schema_tests {
                         props: dict! {
                             "nullable" => Lit::Bool(true)
                         },
-                        layers: [Layer::new(Type::String, dict! {})].into()
+                        layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                        reset: false
                     },
                     ExpandedField {
                         field_name: "another".into(),
@@ -1295,20 +1303,179 @@ mod schema_tests {
                             "nullable" => Lit::Bool(false)
                         },
                         layers: [
-                            Layer::new(
+                            Layer::new_noreset(
                                 Type::String,
                                 dict! {
                                     "maxlen" => Lit::Num(255)
                                 }
                             ),
-                            Layer::new(
+                            Layer::new_noreset(
                                 Type::List,
                                 dict! {
                                    "unique" => Lit::Bool(true)
-                                }
+                                },
                             )
                         ]
-                        .into()
+                        .into(),
+                        reset: false
+                    }
+                ]
+            );
+        }
+    }
+    mod alter_model_update {
+        use crate::engine::ql::{
+            lexer::Type,
+            schema::{ExpandedField, Layer},
+        };
+
+        use super::*;
+        #[test]
+        fn alter_mini() {
+            let tok = lex(b"
+                alter model mymodel update myfield { type string, .. }
+            ")
+            .unwrap();
+            let mut i = 4;
+            let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+            assert_eq!(i, tok.len());
+            assert_eq!(
+                r.as_ref(),
+                [ExpandedField {
+                    field_name: "myfield".into(),
+                    props: dict! {},
+                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    reset: true
+                }]
+            );
+        }
+        #[test]
+        fn alter_mini_2() {
+            let tok = lex(b"
+                alter model mymodel update (myfield { type string, .. })
+            ")
+            .unwrap();
+            let mut i = 4;
+            let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+            assert_eq!(i, tok.len());
+            assert_eq!(
+                r.as_ref(),
+                [ExpandedField {
+                    field_name: "myfield".into(),
+                    props: dict! {},
+                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    reset: true
+                }]
+            );
+        }
+        #[test]
+        fn alter() {
+            let tok = lex(b"
+                alter model mymodel update (
+                    myfield {
+                        type string,
+                        nullable: true,
+                        ..
+                    }
+                )
+            ")
+            .unwrap();
+            let mut i = 4;
+            let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+            assert_eq!(i, tok.len());
+            assert_eq!(
+                r.as_ref(),
+                [ExpandedField {
+                    field_name: "myfield".into(),
+                    props: dict! {
+                        "nullable" => Lit::Bool(true)
+                    },
+                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    reset: true
+                }]
+            );
+        }
+        #[test]
+        fn alter_pro() {
+            let tok = lex(b"
+                alter model mymodel update (
+                    myfield {
+                        type string,
+                        nullable: true,
+                        ..
+                    },
+                    myfield2 {
+                        type string,
+                        ..
+                    }
+                )
+            ")
+            .unwrap();
+            let mut i = 4;
+            let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+            assert_eq!(i, tok.len());
+            assert_eq!(
+                r.as_ref(),
+                [
+                    ExpandedField {
+                        field_name: "myfield".into(),
+                        props: dict! {
+                            "nullable" => Lit::Bool(true)
+                        },
+                        layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                        reset: true
+                    },
+                    ExpandedField {
+                        field_name: "myfield2".into(),
+                        props: dict! {},
+                        layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                        reset: true
+                    }
+                ]
+            );
+        }
+        #[test]
+        fn alter_pro_max() {
+            let tok = lex(b"
+                alter model mymodel update (
+                    myfield {
+                        type string {..},
+                        nullable: true,
+                        ..
+                    },
+                    myfield2 {
+                        type string {
+                            maxlen: 255,
+                            ..
+                        },
+                        ..
+                    }
+                )
+            ")
+            .unwrap();
+            let mut i = 4;
+            let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+            assert_eq!(i, tok.len());
+            assert_eq!(
+                r.as_ref(),
+                [
+                    ExpandedField {
+                        field_name: "myfield".into(),
+                        props: dict! {
+                            "nullable" => Lit::Bool(true)
+                        },
+                        layers: [Layer::new_reset(Type::String, dict! {})].into(),
+                        reset: true
+                    },
+                    ExpandedField {
+                        field_name: "myfield2".into(),
+                        props: dict! {},
+                        layers: [Layer::new_reset(
+                            Type::String,
+                            dict! {"maxlen" => Lit::Num(255)}
+                        )]
+                        .into(),
+                        reset: true
                     }
                 ]
             );

From deed0136485054e88a9c0160a61ab8ad3aeff272 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 3 Oct 2022 17:59:54 +0530
Subject: [PATCH 027/310] Fix entity parsing

---
 server/src/engine/ql/ast.rs    | 43 ++++++++++++++++------------------
 server/src/engine/ql/schema.rs |  4 ++--
 server/src/engine/ql/tests.rs  | 26 ++++++++++++++++++++
 3 files changed, 48 insertions(+), 25 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 97fcf458..04e44cce 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -45,28 +45,27 @@ pub enum Entity {
 }
 
 impl Entity {
-    fn parse(cm: &mut Compiler) -> LangResult<Self> {
-        let a = cm.nxtok_nofw_opt();
-        let b = cm.nxtok_nofw_opt();
-        let c = cm.nxtok_nofw_opt();
+    pub(super) fn parse(cm: &mut Compiler) -> LangResult<Self> {
+        let a = cm.nxtok_opt();
+        let b = cm.nxtok_opt();
+        let c = cm.nxtok_opt();
         match (a, b, c) {
             (
                 Some(Token::Ident(ks)),
                 Some(Token::Symbol(Symbol::SymPeriod)),
                 Some(Token::Ident(tbl)),
-            ) => unsafe {
+            ) => {
                 let r = Ok(Entity::Full(ks.clone(), tbl.clone()));
-                cm.incr_cursor_by(3);
                 r
-            },
+            }
             (Some(Token::Ident(ident)), _, _) => unsafe {
                 let r = Ok(Entity::Current(ident.clone()));
-                cm.incr_cursor();
+                cm.decr_cursor_by(2);
                 r
             },
             (Some(Token::Symbol(Symbol::SymColon)), Some(Token::Ident(tbl)), _) => unsafe {
                 let r = Ok(Entity::Partial(tbl.clone()));
-                cm.incr_cursor_by(2);
+                cm.decr_cursor_by(1);
                 r
             },
             _ => Err(LangError::UnexpectedToken),
@@ -81,6 +80,7 @@ pub enum Statement {
     Use(Entity),
     Inspect(Entity),
     AlterSpace(schema::AlterSpace),
+    AlterModel(RawSlice, schema::AlterKind),
 }
 
 pub struct Compiler<'a> {
@@ -146,7 +146,13 @@ impl<'a> Compiler<'a> {
     }
     #[inline(always)]
     fn alter_model(&mut self) -> Result<Statement, LangError> {
-        todo!()
+        let model_name = match self.nxtok_opt() {
+            Some(Token::Ident(md)) => md.clone(),
+            _ => return Err(LangError::ExpectedStatement),
+        };
+        let mut c = 0;
+        schema::parse_alter_kind_from_tokens(self.remslice(), &mut c)
+            .map(|ak| Statement::AlterModel(model_name.clone(), ak))
     }
     #[inline(always)]
     fn alter_space(&mut self) -> Result<Statement, LangError> {
@@ -199,7 +205,10 @@ impl<'a> Compiler<'a> {
 
 impl<'a> Compiler<'a> {
     #[inline(always)]
-    pub(super) fn nxtok_opt(&mut self) -> Option<&Token> {
+    pub(super) fn nxtok_opt<'b>(&mut self) -> Option<&'b Token>
+    where
+        'a: 'b,
+    {
         if self.not_exhausted() {
             unsafe {
                 let r = Some(&*self.c);
@@ -215,14 +224,6 @@ impl<'a> Compiler<'a> {
         self.c
     }
     #[inline(always)]
-    pub(super) fn nxtok_nofw_opt(&self) -> Option<&Token> {
-        if self.not_exhausted() {
-            unsafe { Some(&*self.c) }
-        } else {
-            None
-        }
-    }
-    #[inline(always)]
     pub(super) fn remslice(&'a self) -> &'a [Token] {
         unsafe { slice::from_raw_parts(self.c, self.remaining()) }
     }
@@ -262,10 +263,6 @@ impl<'a> Compiler<'a> {
     }
     #[inline(always)]
     pub(super) unsafe fn decr_cursor_by(&mut self, by: usize) {
-        debug_assert!(
-            self.remaining().checked_sub(by).is_some(),
-            "cursor crossed e"
-        );
         self.c = self.c.sub(by);
     }
 }
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index de60d973..c33a2f04 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -876,7 +876,7 @@ states! {
 
 #[derive(Debug, PartialEq)]
 /// An [`ExpandedField`] is a full field definition with advanced metadata
-pub(super) struct ExpandedField {
+pub struct ExpandedField {
     pub(super) field_name: RawSlice,
     pub(super) props: Dict,
     pub(super) layers: Vec<Layer>,
@@ -974,7 +974,7 @@ pub(super) fn parse_field_syntax<const ALLOW_RESET: bool>(
 #[derive(Debug)]
 #[cfg_attr(debug_assertions, derive(PartialEq))]
 /// The alter operation kind
-pub(super) enum AlterKind {
+pub enum AlterKind {
     Add(Box<[ExpandedField]>),
     Remove(Box<[RawSlice]>),
     Update(Box<[ExpandedField]>),
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 18d247e0..1412365c 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -144,6 +144,32 @@ mod lexer_tests {
     }
 }
 
+mod entity {
+    use super::*;
+    use crate::engine::ql::ast::{Compiler, Entity};
+    #[test]
+    fn entity_current() {
+        let t = lex(b"hello").unwrap();
+        let mut c = Compiler::new(&t);
+        let r = Entity::parse(&mut c).unwrap();
+        assert_eq!(r, Entity::Current("hello".into()))
+    }
+    #[test]
+    fn entity_partial() {
+        let t = lex(b":hello").unwrap();
+        let mut c = Compiler::new(&t);
+        let r = Entity::parse(&mut c).unwrap();
+        assert_eq!(r, Entity::Partial("hello".into()))
+    }
+    #[test]
+    fn entity_full() {
+        let t = lex(b"hello.world").unwrap();
+        let mut c = Compiler::new(&t);
+        let r = Entity::parse(&mut c).unwrap();
+        assert_eq!(r, Entity::Full("hello".into(), "world".into()))
+    }
+}
+
 mod schema_tests {
     use {
         super::{

From 7ecec7ffa67337270b835d2765a1f3688dd13c7e Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 3 Oct 2022 18:25:40 +0530
Subject: [PATCH 028/310] Add inspect statements

---
 server/src/engine/ql/ast.rs   | 67 ++++++++++++++++++++++++++++++++---
 server/src/engine/ql/lexer.rs | 11 ++++++
 2 files changed, 74 insertions(+), 4 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 04e44cce..cecfaab1 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -78,9 +78,13 @@ pub enum Statement {
     CreateModel(schema::Model),
     CreateSpace(schema::Space),
     Use(Entity),
-    Inspect(Entity),
     AlterSpace(schema::AlterSpace),
     AlterModel(RawSlice, schema::AlterKind),
+    DropModel(RawSlice, bool),
+    DropSpace(RawSlice, bool),
+    InspectSpace(RawSlice),
+    InspectModel(Entity),
+    InspectSpaces,
 }
 
 pub struct Compiler<'a> {
@@ -133,7 +137,31 @@ impl<'a> Compiler<'a> {
     }
     #[inline(always)]
     fn drop0(&mut self) -> Result<Statement, LangError> {
-        todo!()
+        if self.remaining() < 2 {
+            return Err(LangError::ExpectedStatement);
+        }
+        let rs = self.remslice();
+        let ident = match rs[1] {
+            Token::Ident(ref id) => id,
+            _ => return Err(LangError::ExpectedStatement),
+        };
+        let should_force = self.remaining() > 2 && rs[2].as_ident_eq_ignore_case(b"force");
+        let r = match rs[0] {
+            Token::Keyword(Keyword::Ddl(DdlKeyword::Model)) => {
+                // dropping a model
+                Ok(Statement::DropModel(ident.clone(), should_force))
+            }
+            Token::Keyword(Keyword::Ddl(DdlKeyword::Space)) => {
+                // dropping a space
+                Ok(Statement::DropSpace(ident.clone(), should_force))
+            }
+            _ => Err(LangError::UnexpectedToken),
+        };
+        unsafe {
+            self.incr_cursor_by(2);
+            self.incr_cursor_if(should_force);
+        }
+        r
     }
     #[inline(always)]
     fn alter0(&mut self) -> Result<Statement, LangError> {
@@ -169,8 +197,28 @@ impl<'a> Compiler<'a> {
     }
     #[inline(always)]
     fn inspect0(&mut self) -> Result<Statement, LangError> {
-        let entity = Entity::parse(self)?;
-        Ok(Statement::Inspect(entity))
+        if self.remaining() == 0 {
+            return Err(LangError::UnexpectedEndofStatement);
+        }
+        match self.nxtok_opt() {
+            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Space))) => {
+                let space_name = match self.nxtok_opt() {
+                    Some(Token::Ident(id)) => id.clone(),
+                    _ => return Err(LangError::UnexpectedToken),
+                };
+                Ok(Statement::InspectSpace(space_name))
+            }
+            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Model))) => {
+                let entity = Entity::parse(self)?;
+                Ok(Statement::InspectModel(entity))
+            }
+            Some(Token::Ident(id))
+                if unsafe { id.as_slice() }.eq_ignore_ascii_case(b"keyspaces") =>
+            {
+                Ok(Statement::InspectSpaces)
+            }
+            _ => Err(LangError::ExpectedStatement),
+        }
     }
     #[inline(always)]
     fn use0(&mut self) -> Result<Statement, LangError> {
@@ -265,4 +313,15 @@ impl<'a> Compiler<'a> {
     pub(super) unsafe fn decr_cursor_by(&mut self, by: usize) {
         self.c = self.c.sub(by);
     }
+    fn try_read_index<'b>(&'a self, index: usize) -> Option<&'b Token>
+    where
+        'a: 'b,
+    {
+        let sl = self.remslice();
+        if sl.len() > index {
+            Some(&sl[index])
+        } else {
+            None
+        }
+    }
 }
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 6e1909bb..3c2ec418 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -590,6 +590,17 @@ impl Token {
     pub(crate) fn is_typeid(&self) -> bool {
         matches!(self, Token::Keyword(Keyword::TypeId(_)))
     }
+    #[inline(always)]
+    pub(crate) fn as_ident_eq_ignore_case(&self, arg: &[u8]) -> bool {
+        self.is_ident()
+            && unsafe {
+                if let Self::Ident(id) = self {
+                    id.as_slice().eq_ignore_ascii_case(arg)
+                } else {
+                    impossible!()
+                }
+            }
+    }
 }
 
 impl AsRef<Token> for Token {

From 68ed434c964fd077233dcd3c3aa61e428a82037e Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 14 Oct 2022 21:16:24 +0530
Subject: [PATCH 029/310] Add complete list support

---
 server/src/engine/macros.rs     | 51 ++++++++++++++++++
 server/src/engine/memory/mod.rs | 62 ++++++++++++++++++++++
 server/src/engine/mod.rs        |  3 ++
 server/src/engine/ql/ast.rs     | 64 ++++++++++-------------
 server/src/engine/ql/dml.rs     | 91 +++++++++++++++++++++++++++++++++
 server/src/engine/ql/lexer.rs   | 38 +++++++++++++-
 server/src/engine/ql/macros.rs  | 16 +-----
 server/src/engine/ql/mod.rs     |  1 +
 server/src/engine/ql/schema.rs  |  6 +--
 server/src/engine/ql/tests.rs   | 74 +++++++++++++++++++++++++++
 10 files changed, 351 insertions(+), 55 deletions(-)
 create mode 100644 server/src/engine/macros.rs
 create mode 100644 server/src/engine/memory/mod.rs
 create mode 100644 server/src/engine/ql/dml.rs

diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
new file mode 100644
index 00000000..3c68cab4
--- /dev/null
+++ b/server/src/engine/macros.rs
@@ -0,0 +1,51 @@
+/*
+ * Created on Wed Oct 12 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+macro_rules! extract {
+    ($src:expr, $what:pat => $ret:expr) => {
+        if let $what = $src {
+            $ret
+        } else {
+            $crate::impossible!()
+        }
+    };
+}
+
+macro_rules! multi_assert_eq {
+    ($($lhs:expr),* => $rhs:expr) => {
+        $(assert_eq!($lhs, $rhs);)*
+    };
+}
+
+macro_rules! enum_impls {
+    ($for:ty => {$($other:ty as $me:ident),*$(,)?}) => {
+        $(impl ::core::convert::From<$other> for $for {fn from(v: $other) -> Self {Self::$me(v.into())}})*
+    }
+}
+
+macro_rules! assertions {
+    ($($assert:expr),*$(,)?) => {$(const _:()=::core::assert!($assert);)*}
+}
diff --git a/server/src/engine/memory/mod.rs b/server/src/engine/memory/mod.rs
new file mode 100644
index 00000000..570e353f
--- /dev/null
+++ b/server/src/engine/memory/mod.rs
@@ -0,0 +1,62 @@
+/*
+ * Created on Wed Oct 12 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+// TODO(@ohsayan): Change the underlying structures, there are just rudimentary ones used during integration with the QL
+
+/// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
+/// be of one type.
+#[derive(Debug, PartialEq)]
+pub enum DataType {
+    /// An UTF-8 string
+    String(String),
+    /// Bytes
+    Binary(Vec<u8>),
+    /// An integer
+    Number(u64),
+    /// A boolean
+    Boolean(bool),
+    /// A single-type list. Note, you **need** to keep up the invariant that the [`DataType`] disc. remains the same for all
+    /// elements to ensure correctness in this specific context
+    /// FIXME(@ohsayan): Try enforcing this somehow
+    List(Vec<Self>),
+}
+
+enum_impls! {
+    DataType => {
+        String as String,
+        Vec<u8> as Binary,
+        u64 as Number,
+        bool as Boolean,
+        Vec<Self> as List,
+        &'static str as String,
+    }
+}
+
+impl<const N: usize> From<[DataType; N]> for DataType {
+    fn from(f: [DataType; N]) -> Self {
+        Self::List(f.into())
+    }
+}
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 6edec42a..b1e0940f 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -27,4 +27,7 @@
 #![allow(dead_code)]
 #![allow(unused_macros)]
 
+#[macro_use]
+mod macros;
+mod memory;
 mod ql;
diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index cecfaab1..c9b918a5 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -46,30 +46,37 @@ pub enum Entity {
 
 impl Entity {
     pub(super) fn parse(cm: &mut Compiler) -> LangResult<Self> {
-        let a = cm.nxtok_opt();
-        let b = cm.nxtok_opt();
-        let c = cm.nxtok_opt();
-        match (a, b, c) {
-            (
-                Some(Token::Ident(ks)),
-                Some(Token::Symbol(Symbol::SymPeriod)),
-                Some(Token::Ident(tbl)),
-            ) => {
-                let r = Ok(Entity::Full(ks.clone(), tbl.clone()));
-                r
-            }
-            (Some(Token::Ident(ident)), _, _) => unsafe {
-                let r = Ok(Entity::Current(ident.clone()));
-                cm.decr_cursor_by(2);
-                r
+        let sl = cm.remslice();
+        let is_partial =
+            sl.len() > 1 && sl[0] == Token::Symbol(Symbol::SymColon) && sl[1].is_ident();
+        let is_current = !sl.is_empty() && sl[0].is_ident();
+        let is_full = sl.len() > 2
+            && sl[0].is_ident()
+            && sl[1] == Token::Symbol(Symbol::SymPeriod)
+            && sl[2].is_ident();
+        let c;
+        let r = match () {
+            _ if is_full => unsafe {
+                c = 3;
+                Entity::Full(
+                    extract!(&sl[0], Token::Ident(sl) => sl.clone()),
+                    extract!(&sl[2], Token::Ident(sl) => sl.clone()),
+                )
             },
-            (Some(Token::Symbol(Symbol::SymColon)), Some(Token::Ident(tbl)), _) => unsafe {
-                let r = Ok(Entity::Partial(tbl.clone()));
-                cm.decr_cursor_by(1);
-                r
+            _ if is_current => unsafe {
+                c = 1;
+                Entity::Current(extract!(&sl[0], Token::Ident(sl) => sl.clone()))
             },
-            _ => Err(LangError::UnexpectedToken),
+            _ if is_partial => unsafe {
+                c = 2;
+                Entity::Partial(extract!(&sl[1], Token::Ident(sl) => sl.clone()))
+            },
+            _ => return Err(LangError::UnexpectedToken),
+        };
+        unsafe {
+            cm.incr_cursor_by(c);
         }
+        Ok(r)
     }
 }
 
@@ -309,19 +316,4 @@ impl<'a> Compiler<'a> {
         debug_assert!(self.remaining() >= by);
         self.c = self.c.add(by);
     }
-    #[inline(always)]
-    pub(super) unsafe fn decr_cursor_by(&mut self, by: usize) {
-        self.c = self.c.sub(by);
-    }
-    fn try_read_index<'b>(&'a self, index: usize) -> Option<&'b Token>
-    where
-        'a: 'b,
-    {
-        let sl = self.remslice();
-        if sl.len() > index {
-            Some(&sl[index])
-        } else {
-            None
-        }
-    }
 }
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
new file mode 100644
index 00000000..cf852ff0
--- /dev/null
+++ b/server/src/engine/ql/dml.rs
@@ -0,0 +1,91 @@
+/*
+ * Created on Fri Oct 14 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use std::mem::{discriminant, Discriminant};
+
+use super::lexer::{Lit, Symbol};
+
+use {super::lexer::Token, crate::engine::memory::DataType};
+
+pub(super) fn parse_list(
+    tok: &[Token],
+    list: &mut Vec<DataType>,
+) -> (Option<Discriminant<DataType>>, usize, bool) {
+    let l = tok.len();
+    let mut okay = l != 0;
+    let mut stop = okay && tok[0] == Symbol::TtCloseSqBracket;
+    let mut i = stop as usize;
+    let mut overall_dscr = None;
+    let mut prev_nlist_dscr = None;
+    while i < l && okay && !stop {
+        let d = match &tok[i] {
+            Token::Lit(Lit::Str(s)) => DataType::String(s.to_string()),
+            Token::Lit(Lit::Num(n)) => DataType::Number(*n),
+            Token::Lit(Lit::Bool(b)) => DataType::Boolean(*b),
+            Token::Symbol(Symbol::TtOpenSqBracket) => {
+                // a nested list
+                let mut nested_list = Vec::new();
+                let (nlist_dscr, nlist_i, nlist_okay) = parse_list(&tok[i + 1..], &mut nested_list);
+                okay &= nlist_okay;
+                i += nlist_i;
+                // check type return
+                okay &= {
+                    prev_nlist_dscr.is_none()
+                        || nlist_dscr.is_none()
+                        || prev_nlist_dscr == nlist_dscr
+                };
+                if prev_nlist_dscr.is_none() && nlist_dscr.is_some() {
+                    prev_nlist_dscr = nlist_dscr;
+                }
+                DataType::List(nested_list)
+            }
+            _ => {
+                okay = false;
+                break;
+            }
+        };
+        i += 1;
+        okay &= list.is_empty() || discriminant(&d) == discriminant(&list[0]);
+        overall_dscr = Some(discriminant(&d));
+        list.push(d);
+        let nx_comma = i < l && tok[i] == Symbol::SymComma;
+        let nx_csqrb = i < l && tok[i] == Symbol::TtCloseSqBracket;
+        okay &= nx_comma | nx_csqrb;
+        i += okay as usize;
+        stop = nx_csqrb;
+    }
+    (overall_dscr, i, okay && stop)
+}
+
+#[cfg(test)]
+pub(super) fn parse_list_full(tok: &[Token]) -> Option<Vec<DataType>> {
+    let mut l = Vec::new();
+    if let (_, _, true) = parse_list(tok, &mut l) {
+        Some(l)
+    } else {
+        None
+    }
+}
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 3c2ec418..f0b9d871 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -45,6 +45,15 @@ pub enum Token {
     Lit(Lit), // literal
 }
 
+impl PartialEq<Symbol> for Token {
+    fn eq(&self, other: &Symbol) -> bool {
+        match self {
+            Self::Symbol(s) => s == other,
+            _ => false,
+        }
+    }
+}
+
 assertions! {
     size_of::<Token>() == 32, // FIXME(@ohsayan): Damn, what?
     size_of::<Symbol>() == 1,
@@ -495,7 +504,7 @@ impl<'a> Lexer<'a> {
                 1234, // valid
                 1234a // invalid
             */
-            static TERMINAL_CHAR: [u8; 6] = [b';', b'}', b',', b' ', b'\n', b'\t'];
+            static TERMINAL_CHAR: [u8; 8] = [b';', b'}', b',', b' ', b'\n', b'\t', b',', b']'];
             let wseof = self.peek_is(|b| TERMINAL_CHAR.contains(&b)) || self.exhausted();
             match str::from_utf8_unchecked(slice::from_raw_parts(
                 s,
@@ -549,7 +558,20 @@ impl<'a> Lexer<'a> {
         match symof(byte) {
             Some(tok) => self.push_token(tok),
             #[cfg(test)]
-            None if byte == b'\r' => self.push_token(Token::IgnorableComma),
+            None if byte == b'\r'
+                && self.remaining() > 1
+                && !(unsafe {
+                    // UNSAFE(@ohsayan): The previous condition ensures that this doesn't segfault
+                    *self.cursor().add(1)
+                })
+                .is_ascii_digit() =>
+            {
+                /*
+                    NOTE(@ohsayan): The above guard might look a little messy but is necessary to support raw
+                    literals which will use the carriage return
+                */
+                self.push_token(Token::IgnorableComma)
+            }
             _ => {
                 self.last_error = Some(LangError::UnexpectedChar);
                 return;
@@ -601,6 +623,18 @@ impl Token {
                 }
             }
     }
+    #[inline(always)]
+    pub(super) unsafe fn ident_unchecked(&self) -> RawSlice {
+        if let Self::Ident(id) = self {
+            id.clone()
+        } else {
+            impossible!()
+        }
+    }
+    #[inline(always)]
+    pub(super) fn is_lit(&self) -> bool {
+        matches!(self, Self::Lit(_))
+    }
 }
 
 impl AsRef<Token> for Token {
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 8b89a2ec..2678d4c7 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -46,18 +46,6 @@ macro_rules! set {
     }};
 }
 
-macro_rules! multi_assert_eq {
-    ($($lhs:expr),* => $rhs:expr) => {
-        $(assert_eq!($lhs, $rhs);)*
-    };
-}
-
-macro_rules! enum_impls {
-    ($for:ty => {$($other:ty as $me:ident),*$(,)?}) => {
-        $(impl ::core::convert::From<$other> for $for {fn from(v: $other) -> Self {Self::$me(v)}})*
-    }
-}
-
-macro_rules! assertions {
-    ($($assert:expr),*$(,)?) => {$(const _:()=::core::assert!($assert);)*}
+macro_rules! into_array {
+    ($($e:expr),* $(,)?) => { [$($e.into()),*] };
 }
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 180292ee..9c621317 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -27,6 +27,7 @@
 #[macro_use]
 mod macros;
 pub(super) mod ast;
+pub(super) mod dml;
 pub(super) mod lexer;
 pub(super) mod schema;
 #[cfg(test)]
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index c33a2f04..007ae945 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -44,11 +44,11 @@
     Sept. 15, 2022
 */
 
-use super::lexer::DmlKeyword;
-
 use {
     super::{
-        lexer::{DdlKeyword, DdlMiscKeyword, Keyword, Lit, MiscKeyword, Symbol, Token, Type},
+        lexer::{
+            DdlKeyword, DdlMiscKeyword, DmlKeyword, Keyword, Lit, MiscKeyword, Symbol, Token, Type,
+        },
         LangError, LangResult, RawSlice,
     },
     std::{
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 1412365c..581bc230 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -1508,3 +1508,77 @@ mod schema_tests {
         }
     }
 }
+
+mod dml_tests {
+    use super::*;
+    mod list_parse {
+        use super::*;
+        use crate::engine::ql::dml::parse_list_full;
+
+        #[test]
+        fn list_mini() {
+            let tok = lex(b"
+                []
+            ")
+            .unwrap();
+            let r = parse_list_full(&tok[1..]).unwrap();
+            assert_eq!(r, vec![])
+        }
+
+        #[test]
+        fn list() {
+            let tok = lex(b"
+                [1, 2, 3, 4]
+            ")
+            .unwrap();
+            let r = parse_list_full(&tok[1..]).unwrap();
+            assert_eq!(r.as_slice(), into_array![1, 2, 3, 4])
+        }
+
+        #[test]
+        fn list_pro() {
+            let tok = lex(b"
+                [
+                    [1, 2],
+                    [3, 4],
+                    [5, 6],
+                    [7, 8]
+                ]
+            ")
+            .unwrap();
+            let r = parse_list_full(&tok[1..]).unwrap();
+            assert_eq!(
+                r.as_slice(),
+                into_array![
+                    into_array![1, 2],
+                    into_array![3, 4],
+                    into_array![5, 6],
+                    into_array![7, 8]
+                ]
+            )
+        }
+
+        #[test]
+        fn list_pro_max() {
+            let tok = lex(b"
+                [
+                    [[1, 1], [2, 2]],
+                    [[3, 3], [4, 4]],
+                    [[5, 5], [6, 6]],
+                    [[7, 7], [8, 8]]
+                ]
+            ")
+            .unwrap();
+            let r = parse_list_full(&tok[1..]).unwrap();
+            assert_eq!(
+                r.as_slice(),
+                into_array![
+                    into_array![into_array![1, 1], into_array![2, 2]],
+                    into_array![into_array![3, 3], into_array![4, 4]],
+                    into_array![into_array![5, 5], into_array![6, 6]],
+                    into_array![into_array![7, 7], into_array![8, 8]],
+                ]
+            )
+        }
+    }
+}

From 132633c54272d89dc4920fba7053cba95bf6968d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 14 Oct 2022 22:19:24 +0530
Subject: [PATCH 030/310] Add tuple syntax parsing

---
 server/src/engine/ql/dml.rs   | 69 +++++++++++++++++++++++---
 server/src/engine/ql/tests.rs | 93 ++++++++++++++++++++++++++++++++---
 2 files changed, 150 insertions(+), 12 deletions(-)

diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index cf852ff0..d8aff3d2 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -24,12 +24,15 @@
  *
 */
 
-use std::mem::{discriminant, Discriminant};
-
-use super::lexer::{Lit, Symbol};
-
-use {super::lexer::Token, crate::engine::memory::DataType};
+use {
+    super::lexer::{Lit, Symbol, Token},
+    crate::engine::memory::DataType,
+    std::mem::{discriminant, Discriminant},
+};
 
+/// Parse a list
+///
+/// **NOTE:** This function will error if the `[` token is passed. Make sure this is forwarded by the caller
 pub(super) fn parse_list(
     tok: &[Token],
     list: &mut Vec<DataType>,
@@ -83,9 +86,63 @@ pub(super) fn parse_list(
 #[cfg(test)]
 pub(super) fn parse_list_full(tok: &[Token]) -> Option<Vec<DataType>> {
     let mut l = Vec::new();
-    if let (_, _, true) = parse_list(tok, &mut l) {
+    if matches!(parse_list(tok, &mut l), (_, i, true) if i == tok.len()) {
         Some(l)
     } else {
         None
     }
 }
+
+#[cfg(test)]
+/// Parse the tuple data passed in with an insert query.
+///
+/// **Note:** Make sure you pass the `(` token
+pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<DataType>, usize, bool) {
+    let l = tok.len();
+    let mut okay = l != 0;
+    let mut stop = okay && tok[0] == Token::Symbol(Symbol::TtCloseParen);
+    let mut i = stop as usize;
+    let mut data = Vec::new();
+    while i < l && okay && !stop {
+        match &tok[i] {
+            Token::Lit(Lit::Str(s)) => {
+                data.push(s.to_string().into());
+            }
+            Token::Lit(Lit::Num(n)) => {
+                data.push((*n).into());
+            }
+            Token::Lit(Lit::Bool(b)) => {
+                data.push((*b).into());
+            }
+            Token::Symbol(Symbol::TtOpenSqBracket) => {
+                // ah, a list
+                let mut l = Vec::new();
+                let (_, lst_i, lst_okay) = parse_list(&tok[i + 1..], &mut l);
+                data.push(l.into());
+                i += lst_i;
+                okay &= lst_okay;
+            }
+            _ => {
+                okay = false;
+                break;
+            }
+        }
+        i += 1;
+        let nx_comma = i < l && tok[i] == Symbol::SymComma;
+        let nx_csprn = i < l && tok[i] == Symbol::TtCloseParen;
+        okay &= nx_comma | nx_csprn;
+        i += okay as usize;
+        stop = nx_csprn;
+    }
+    (data, i, okay && stop)
+}
+
+#[cfg(test)]
+pub(super) fn parse_data_tuple_syntax_full(tok: &[Token]) -> Option<Vec<DataType>> {
+    let (ret, cnt, okay) = parse_data_tuple_syntax(tok);
+    if cnt == tok.len() && okay {
+        Some(ret)
+    } else {
+        None
+    }
+}
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 581bc230..772f50c1 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -1542,7 +1542,7 @@ mod dml_tests {
                     [1, 2],
                     [3, 4],
                     [5, 6],
-                    [7, 8]
+                    []
                 ]
             ")
             .unwrap();
@@ -1553,7 +1553,7 @@ mod dml_tests {
                     into_array![1, 2],
                     into_array![3, 4],
                     into_array![5, 6],
-                    into_array![7, 8]
+                    into_array![]
                 ]
             )
         }
@@ -1563,9 +1563,9 @@ mod dml_tests {
             let tok = lex(b"
                 [
                     [[1, 1], [2, 2]],
-                    [[3, 3], [4, 4]],
+                    [[], [4, 4]],
                     [[5, 5], [6, 6]],
-                    [[7, 7], [8, 8]]
+                    [[7, 7], []]
                 ]
             ")
             .unwrap();
@@ -1574,11 +1574,92 @@ mod dml_tests {
                 r.as_slice(),
                 into_array![
                     into_array![into_array![1, 1], into_array![2, 2]],
-                    into_array![into_array![3, 3], into_array![4, 4]],
+                    into_array![into_array![], into_array![4, 4]],
                     into_array![into_array![5, 5], into_array![6, 6]],
-                    into_array![into_array![7, 7], into_array![8, 8]],
+                    into_array![into_array![7, 7], into_array![]],
                 ]
             )
         }
     }
+    mod tuple_syntax {
+        use super::*;
+        use crate::engine::ql::dml::parse_data_tuple_syntax_full;
+
+        #[test]
+        fn tuple_mini() {
+            let tok = lex(b"()").unwrap();
+            let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
+            assert_eq!(r, vec![]);
+        }
+
+        #[test]
+        fn tuple() {
+            let tok = lex(br#"
+                (1234, "email@example.com", true)
+            "#)
+            .unwrap();
+            let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
+            assert_eq!(r.as_slice(), into_array![1234, "email@example.com", true]);
+        }
+
+        #[test]
+        fn tuple_pro() {
+            let tok = lex(br#"
+                (
+                    1234,
+                    "email@example.com",
+                    true,
+                    ["hello", "world", "and", "the", "universe"]
+                )
+            "#)
+            .unwrap();
+            let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
+            assert_eq!(
+                r.as_slice(),
+                into_array![
+                    1234,
+                    "email@example.com",
+                    true,
+                    into_array!["hello", "world", "and", "the", "universe"]
+                ]
+            );
+        }
+
+        #[test]
+        fn tuple_pro_max() {
+            let tok = lex(br#"
+                (
+                    1234,
+                    "email@example.com",
+                    true,
+                    [
+                        ["h", "hello"],
+                        ["w", "world"],
+                        ["a", "and"],
+                        ["the"],
+                        ["universe"],
+                        []
+                    ]
+                )
+            "#)
+            .unwrap();
+            let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
+            assert_eq!(
+                r.as_slice(),
+                into_array![
+                    1234,
+                    "email@example.com",
+                    true,
+                    into_array![
+                        into_array!["h", "hello"],
+                        into_array!["w", "world"],
+                        into_array!["a", "and"],
+                        into_array!["the"],
+                        into_array!["universe"],
+                        into_array![],
+                    ]
+                ]
+            );
+        }
+    }
 }

From e580f2b8d889b1162e18af6497f9a47728290c1b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 14 Oct 2022 23:25:02 +0530
Subject: [PATCH 031/310] Add full support for map-based insert syntax

---
 server/src/engine/ql/dml.rs   | 65 +++++++++++++++++++++++-
 server/src/engine/ql/tests.rs | 96 +++++++++++++++++++++++++++++++++++
 2 files changed, 160 insertions(+), 1 deletion(-)

diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index d8aff3d2..9668aa30 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -27,7 +27,10 @@
 use {
     super::lexer::{Lit, Symbol, Token},
     crate::engine::memory::DataType,
-    std::mem::{discriminant, Discriminant},
+    std::{
+        collections::HashMap,
+        mem::{discriminant, Discriminant},
+    },
 };
 
 /// Parse a list
@@ -146,3 +149,63 @@ pub(super) fn parse_data_tuple_syntax_full(tok: &[Token]) -> Option<Vec<DataType
         None
     }
 }
+
+pub(super) fn parse_data_map_syntax(tok: &[Token]) -> (HashMap<Box<str>, DataType>, usize, bool) {
+    let l = tok.len();
+    let mut okay = l != 0;
+    let mut stop = okay && tok[0] == Token::Symbol(Symbol::TtCloseBrace);
+    let mut i = stop as usize;
+    let mut data = HashMap::new();
+    while i + 3 < l && okay && !stop {
+        let (field, colon, expression) = (&tok[i], &tok[i + 1], &tok[i + 2]);
+        okay &= colon == &Symbol::SymColon;
+        match (field, expression) {
+            (Token::Ident(id), Token::Lit(Lit::Str(s))) => {
+                okay &= data
+                    .insert(unsafe { id.as_str() }.into(), s.to_string().into())
+                    .is_none();
+            }
+            (Token::Ident(id), Token::Lit(Lit::Num(n))) => {
+                okay &= data
+                    .insert(unsafe { id.as_str() }.into(), (*n).into())
+                    .is_none();
+            }
+            (Token::Ident(id), Token::Lit(Lit::Bool(b))) => {
+                okay &= data
+                    .insert(unsafe { id.as_str() }.into(), (*b).into())
+                    .is_none();
+            }
+            (Token::Ident(id), Token::Symbol(Symbol::TtOpenSqBracket)) => {
+                // ooh a list
+                let mut l = Vec::new();
+                let (_, lst_i, lst_ok) = parse_list(&tok[i + 3..], &mut l);
+                okay &= lst_ok;
+                i += lst_i;
+                okay &= data
+                    .insert(unsafe { id.as_str() }.into(), l.into())
+                    .is_none();
+            }
+            _ => {
+                okay = false;
+                break;
+            }
+        }
+        i += 3;
+        let nx_comma = i < l && tok[i] == Symbol::SymComma;
+        let nx_csprn = i < l && tok[i] == Symbol::TtCloseBrace;
+        okay &= nx_comma | nx_csprn;
+        i += okay as usize;
+        stop = nx_csprn;
+    }
+    (data, i, okay && stop)
+}
+
+#[cfg(test)]
+pub(super) fn parse_data_map_syntax_full(tok: &[Token]) -> Option<HashMap<Box<str>, DataType>> {
+    let (dat, i, ok) = parse_data_map_syntax(tok);
+    if i == tok.len() && ok {
+        Some(dat)
+    } else {
+        None
+    }
+}
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 772f50c1..92b1fd04 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -1662,4 +1662,100 @@ mod dml_tests {
             );
         }
     }
+    mod map_syntax {
+        use super::*;
+        use crate::engine::ql::dml::parse_data_map_syntax_full;
+
+        #[test]
+        fn map_mini() {
+            let tok = lex(b"{}").unwrap();
+            let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
+            assert_eq!(r, dict! {})
+        }
+
+        #[test]
+        fn map() {
+            let tok = lex(br#"
+                {
+                    name: "John Appletree",
+                    email: "john@example.com",
+                    verified: false,
+                    followers: 12345
+                }
+            "#)
+            .unwrap();
+            let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
+            assert_eq!(
+                r,
+                dict! {
+                    "name" => "John Appletree",
+                    "email" => "john@example.com",
+                    "verified" => false,
+                    "followers" => 12345,
+                }
+            )
+        }
+
+        #[test]
+        fn map_pro() {
+            let tok = lex(br#"
+                {
+                    name: "John Appletree",
+                    email: "john@example.com",
+                    verified: false,
+                    followers: 12345,
+                    tweets_by_day: []
+                }
+            "#)
+            .unwrap();
+            let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
+            assert_eq!(
+                r,
+                dict! {
+                    "name" => "John Appletree",
+                    "email" => "john@example.com",
+                    "verified" => false,
+                    "followers" => 12345,
+                    "tweets_by_day" => []
+                }
+            )
+        }
+
+        #[test]
+        fn map_pro_max() {
+            let tok = lex(br#"
+                {
+                    name: "John Appletree",
+                    email: "john@example.com",
+                    verified: false,
+                    followers: 12345,
+                    tweets_by_day: [
+                        ["it's a fresh monday", "monday was tiring"],
+                        ["already bored with tuesday", "nope. gotta change stuff, life's getting boring"],
+                        ["sunday, going to bed"]
+                    ]
+                }
+            "#)
+            .unwrap();
+            let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
+            assert_eq!(
+                r,
+                dict! {
+                    "name" => "John Appletree",
+                    "email" => "john@example.com",
+                    "verified" => false,
+                    "followers" => 12345,
+                    "tweets_by_day" => into_array![
+                        into_array![
+                            "it's a fresh monday", "monday was tiring"
+                        ],
+                        into_array![
+                            "already bored with tuesday", "nope. gotta change stuff, life's getting boring"
+                        ],
+                        into_array!["sunday, going to bed"]
+                    ]
+                }
+            )
+        }
+    }
 }

From 9c28c978307e0366da9193eb4019cc2e4c291017 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 6 Nov 2022 11:23:29 +0530
Subject: [PATCH 032/310] Support unsafe literals

---
 server/src/engine/ql/lexer.rs | 80 +++++++++++++++++++++++++++--------
 server/src/engine/ql/mod.rs   |  1 +
 server/src/engine/ql/tests.rs | 18 ++++++++
 3 files changed, 82 insertions(+), 17 deletions(-)

diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index f0b9d871..bca8678e 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -38,6 +38,7 @@ use {
 pub enum Token {
     Symbol(Symbol),
     Keyword(Keyword),
+    UnsafeLit(RawSlice),
     Ident(RawSlice),
     #[cfg(test)]
     /// A comma that can be ignored (used for fuzzing)
@@ -55,7 +56,7 @@ impl PartialEq<Symbol> for Token {
 }
 
 assertions! {
-    size_of::<Token>() == 32, // FIXME(@ohsayan): Damn, what?
+    size_of::<Token>() == 24, // FIXME(@ohsayan): Damn, what?
     size_of::<Symbol>() == 1,
     size_of::<Keyword>() == 2,
     size_of::<Lit>() == 24, // FIXME(@ohsayan): Ouch
@@ -481,6 +482,7 @@ impl<'a> Lexer<'a> {
             RawSlice::new(s, self.cursor().offset_from(s) as usize)
         }
     }
+
     fn scan_ident_or_keyword(&mut self) {
         let s = self.scan_ident();
         let st = unsafe { s.as_str() };
@@ -491,6 +493,7 @@ impl<'a> Lexer<'a> {
             None => self.tokens.push(Token::Ident(s)),
         }
     }
+
     fn scan_number(&mut self) {
         let s = self.cursor();
         unsafe {
@@ -517,6 +520,7 @@ impl<'a> Lexer<'a> {
             }
         }
     }
+
     fn scan_quoted_string(&mut self, quote_style: u8) {
         debug_assert!(
             unsafe { self.deref_cursor() } == quote_style,
@@ -557,22 +561,7 @@ impl<'a> Lexer<'a> {
     fn scan_byte(&mut self, byte: u8) {
         match symof(byte) {
             Some(tok) => self.push_token(tok),
-            #[cfg(test)]
-            None if byte == b'\r'
-                && self.remaining() > 1
-                && !(unsafe {
-                    // UNSAFE(@ohsayan): The previous condition ensures that this doesn't segfault
-                    *self.cursor().add(1)
-                })
-                .is_ascii_digit() =>
-            {
-                /*
-                    NOTE(@ohsayan): The above guard might look a little messy but is necessary to support raw
-                    literals which will use the carriage return
-                */
-                self.push_token(Token::IgnorableComma)
-            }
-            _ => {
+            None => {
                 self.last_error = Some(LangError::UnexpectedChar);
                 return;
             }
@@ -582,10 +571,66 @@ impl<'a> Lexer<'a> {
         }
     }
 
+    fn scan_unsafe_literal(&mut self) {
+        unsafe {
+            self.incr_cursor();
+        }
+        let mut size = 0usize;
+        let mut okay = true;
+        while self.not_exhausted() && unsafe { self.deref_cursor() != b'\n' } && okay {
+            /*
+                Don't ask me how stupid this is. Like, I was probably in some "mood" when I wrote this
+                and it works duh, but isn't the most elegant of things (could I have just used a parse?
+                nah, I'm just a hardcore numeric normie)
+                -- Sayan
+            */
+            let byte = unsafe { self.deref_cursor() };
+            okay &= byte.is_ascii_digit();
+            let (prod, of_flag) = size.overflowing_mul(10);
+            okay &= !of_flag;
+            let (sum, of_flag) = prod.overflowing_add((byte & 0x0F) as _);
+            size = sum;
+            okay &= !of_flag;
+            unsafe {
+                self.incr_cursor();
+            }
+        }
+        okay &= self.peek_eq_and_forward(b'\n');
+        okay &= self.remaining() >= size;
+        if compiler::likely(okay) {
+            unsafe {
+                self.push_token(Token::UnsafeLit(RawSlice::new(self.cursor(), size)));
+                self.incr_cursor_by(size);
+            }
+        } else {
+            self.last_error = Some(LangError::InvalidUnsafeLiteral);
+        }
+    }
+
     fn _lex(&mut self) {
         while self.not_exhausted() && self.last_error.is_none() {
             match unsafe { self.deref_cursor() } {
                 byte if byte.is_ascii_alphabetic() => self.scan_ident_or_keyword(),
+                #[cfg(test)]
+                byte if byte == b'\r'
+                    && self.remaining() > 1
+                    && !(unsafe {
+                        // UNSAFE(@ohsayan): The previous condition ensures that this doesn't segfault
+                        *self.cursor().add(1)
+                    })
+                    .is_ascii_digit() =>
+                {
+                    /*
+                        NOTE(@ohsayan): The above guard might look a little messy but is necessary to support raw
+                        literals which will use the carriage return
+                    */
+                    self.push_token(Token::IgnorableComma);
+                    unsafe {
+                        // UNSAFE(@ohsayan): All good here. Already read the token
+                        self.incr_cursor();
+                    }
+                }
+                b'\r' => self.scan_unsafe_literal(),
                 byte if byte.is_ascii_digit() => self.scan_number(),
                 qs @ (b'\'' | b'"') => self.scan_quoted_string(qs),
                 b' ' | b'\n' | b'\t' => self.trim_ahead(),
@@ -603,6 +648,7 @@ impl<'a> Lexer<'a> {
         }
     }
 }
+
 impl Token {
     #[inline(always)]
     pub(crate) fn is_ident(&self) -> bool {
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 9c621317..69954b98 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -55,6 +55,7 @@ pub enum LangError {
     UnexpectedToken,
     InvalidDictionaryExpression,
     InvalidTypeDefinition,
+    InvalidUnsafeLiteral,
 }
 
 /*
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 92b1fd04..8f80e82d 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -142,6 +142,24 @@ mod lexer_tests {
         let wth = br#" 'see, we escaped the end\' "#;
         assert_eq!(lex(wth).unwrap_err(), LangError::InvalidStringLiteral);
     }
+    #[test]
+    fn lex_unsafe_literal_mini() {
+        let usl = lex("\r0\n".as_bytes()).unwrap();
+        assert_eq!(usl.len(), 1);
+        assert_eq!(Token::UnsafeLit("".into()), usl[0]);
+    }
+    #[test]
+    fn lex_unsafe_literal() {
+        let usl = lex("\r9\nabcdefghi".as_bytes()).unwrap();
+        assert_eq!(usl.len(), 1);
+        assert_eq!(Token::UnsafeLit("abcdefghi".into()), usl[0]);
+    }
+    #[test]
+    fn lex_unsafe_literal_pro() {
+        let usl = lex("\r18\nabcdefghi123456789".as_bytes()).unwrap();
+        assert_eq!(usl.len(), 1);
+        assert_eq!(Token::UnsafeLit("abcdefghi123456789".into()), usl[0]);
+    }
 }
 
 mod entity {

From 2d1fdd3417600c0e72a619e47ba82b68b1174e99 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 6 Nov 2022 21:18:11 +0530
Subject: [PATCH 033/310] Keep `UnsafeLit` as a `Lit` variant

Per our language specification, unsafe literals are by their inherent
name literals, hence keep them under this variant. Also, it simplifies
a bunch of implementations. Although I'm sure we can do something for
performance to simply provide an error case when we encounter a
disallowed literal.
---
 server/src/engine/ql/dml.rs   | 4 ++++
 server/src/engine/ql/lexer.rs | 4 ++--
 server/src/engine/ql/tests.rs | 9 ++++++---
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 9668aa30..5cda0c8a 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -33,6 +33,10 @@ use {
     },
 };
 
+/*
+    Impls for insert
+*/
+
 /// Parse a list
 ///
 /// **NOTE:** This function will error if the `[` token is passed. Make sure this is forwarded by the caller
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index bca8678e..89868f66 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -38,7 +38,6 @@ use {
 pub enum Token {
     Symbol(Symbol),
     Keyword(Keyword),
-    UnsafeLit(RawSlice),
     Ident(RawSlice),
     #[cfg(test)]
     /// A comma that can be ignored (used for fuzzing)
@@ -76,6 +75,7 @@ pub enum Lit {
     Str(Box<str>),
     Bool(bool),
     Num(u64),
+    UnsafeLit(RawSlice),
 }
 
 enum_impls! {
@@ -599,7 +599,7 @@ impl<'a> Lexer<'a> {
         okay &= self.remaining() >= size;
         if compiler::likely(okay) {
             unsafe {
-                self.push_token(Token::UnsafeLit(RawSlice::new(self.cursor(), size)));
+                self.push_token(Lit::UnsafeLit(RawSlice::new(self.cursor(), size)));
                 self.incr_cursor_by(size);
             }
         } else {
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 8f80e82d..ab212fbc 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -146,19 +146,22 @@ mod lexer_tests {
     fn lex_unsafe_literal_mini() {
         let usl = lex("\r0\n".as_bytes()).unwrap();
         assert_eq!(usl.len(), 1);
-        assert_eq!(Token::UnsafeLit("".into()), usl[0]);
+        assert_eq!(Token::Lit(Lit::UnsafeLit("".into())), usl[0]);
     }
     #[test]
     fn lex_unsafe_literal() {
         let usl = lex("\r9\nabcdefghi".as_bytes()).unwrap();
         assert_eq!(usl.len(), 1);
-        assert_eq!(Token::UnsafeLit("abcdefghi".into()), usl[0]);
+        assert_eq!(Token::Lit(Lit::UnsafeLit("abcdefghi".into())), usl[0]);
     }
     #[test]
     fn lex_unsafe_literal_pro() {
         let usl = lex("\r18\nabcdefghi123456789".as_bytes()).unwrap();
         assert_eq!(usl.len(), 1);
-        assert_eq!(Token::UnsafeLit("abcdefghi123456789".into()), usl[0]);
+        assert_eq!(
+            Token::Lit(Lit::UnsafeLit("abcdefghi123456789".into())),
+            usl[0]
+        );
     }
 }
 

From f69ec8835464c113a4d58f38b5a3effbff0482b8 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 7 Nov 2022 13:02:57 +0530
Subject: [PATCH 034/310] Use `Token!` for token shorthand

---
 server/src/engine/ql/ast.rs    |  36 +++--
 server/src/engine/ql/macros.rs | 240 +++++++++++++++++++++++++++++++++
 2 files changed, 256 insertions(+), 20 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index c9b918a5..24abe98c 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -26,7 +26,7 @@
 
 use {
     super::{
-        lexer::{DdlKeyword, Keyword, Lexer, Symbol, Token},
+        lexer::{Lexer, Token},
         schema, LangError, LangResult, RawSlice,
     },
     crate::util::Life,
@@ -47,13 +47,9 @@ pub enum Entity {
 impl Entity {
     pub(super) fn parse(cm: &mut Compiler) -> LangResult<Self> {
         let sl = cm.remslice();
-        let is_partial =
-            sl.len() > 1 && sl[0] == Token::Symbol(Symbol::SymColon) && sl[1].is_ident();
+        let is_partial = sl.len() > 1 && sl[0] == Token![:] && sl[1].is_ident();
         let is_current = !sl.is_empty() && sl[0].is_ident();
-        let is_full = sl.len() > 2
-            && sl[0].is_ident()
-            && sl[1] == Token::Symbol(Symbol::SymPeriod)
-            && sl[2].is_ident();
+        let is_full = sl.len() > 2 && sl[0].is_ident() && sl[1] == Token![.] && sl[2].is_ident();
         let c;
         let r = match () {
             _ if is_full => unsafe {
@@ -126,19 +122,19 @@ impl<'a> Compiler<'a> {
     #[inline(always)]
     fn stage0(&mut self) -> Result<Statement, LangError> {
         match self.nxtok_opt() {
-            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Create))) => self.create0(),
-            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Drop))) => self.drop0(),
-            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Alter))) => self.alter0(),
-            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Inspect))) => self.inspect0(),
-            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Use))) => self.use0(),
+            Some(Token![create]) => self.create0(),
+            Some(Token![drop]) => self.drop0(),
+            Some(Token![alter]) => self.alter0(),
+            Some(Token![inspect]) => self.inspect0(),
+            Some(Token![use]) => self.use0(),
             _ => Err(LangError::ExpectedStatement),
         }
     }
     #[inline(always)]
     fn create0(&mut self) -> Result<Statement, LangError> {
         match self.nxtok_opt() {
-            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Model))) => self.c_model0(),
-            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Space))) => self.c_space0(),
+            Some(Token![model]) => self.c_model0(),
+            Some(Token![space]) => self.c_space0(),
             _ => Err(LangError::UnexpectedEndofStatement),
         }
     }
@@ -154,11 +150,11 @@ impl<'a> Compiler<'a> {
         };
         let should_force = self.remaining() > 2 && rs[2].as_ident_eq_ignore_case(b"force");
         let r = match rs[0] {
-            Token::Keyword(Keyword::Ddl(DdlKeyword::Model)) => {
+            Token![model] => {
                 // dropping a model
                 Ok(Statement::DropModel(ident.clone(), should_force))
             }
-            Token::Keyword(Keyword::Ddl(DdlKeyword::Space)) => {
+            Token![space] => {
                 // dropping a space
                 Ok(Statement::DropSpace(ident.clone(), should_force))
             }
@@ -173,8 +169,8 @@ impl<'a> Compiler<'a> {
     #[inline(always)]
     fn alter0(&mut self) -> Result<Statement, LangError> {
         match self.nxtok_opt() {
-            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Model))) => self.alter_model(),
-            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Space))) => self.alter_space(),
+            Some(Token![model]) => self.alter_model(),
+            Some(Token![space]) => self.alter_space(),
             Some(_) => Err(LangError::ExpectedStatement),
             None => Err(LangError::UnexpectedEndofStatement),
         }
@@ -208,14 +204,14 @@ impl<'a> Compiler<'a> {
             return Err(LangError::UnexpectedEndofStatement);
         }
         match self.nxtok_opt() {
-            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Space))) => {
+            Some(Token![space]) => {
                 let space_name = match self.nxtok_opt() {
                     Some(Token::Ident(id)) => id.clone(),
                     _ => return Err(LangError::UnexpectedToken),
                 };
                 Ok(Statement::InspectSpace(space_name))
             }
-            Some(Token::Keyword(Keyword::Ddl(DdlKeyword::Model))) => {
+            Some(Token![model]) => {
                 let entity = Entity::parse(self)?;
                 Ok(Statement::InspectModel(entity))
             }
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 2678d4c7..e01d23d5 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -24,6 +24,246 @@
  *
 */
 
+macro_rules! __sym_token {
+    ($ident:ident) => {
+        $crate::engine::ql::lexer::Token::Symbol($crate::engine::ql::lexer::Symbol::$ident)
+    };
+}
+
+macro_rules! __ddl_token {
+    ($ident:ident) => {
+        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::Ddl(
+            $crate::engine::ql::lexer::DdlKeyword::$ident,
+        ))
+    };
+}
+
+macro_rules! __ddl_misc_token {
+    ($ident:ident) => {
+        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::DdlMisc(
+            $crate::engine::ql::lexer::DdlMiscKeyword::$ident,
+        ))
+    };
+}
+
+macro_rules! __dml_token {
+    ($ident:ident) => {
+        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::Dml(
+            $crate::engine::ql::lexer::DmlKeyword::$ident,
+        ))
+    };
+}
+
+macro_rules! __dml_misc_token {
+    ($ident:ident) => {
+        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::DmlMisc(
+            $crate::engine::ql::lexer::DmlMiscKeyword::$ident,
+        ))
+    };
+}
+
+macro_rules! __type_token {
+    ($ident:ident) => {
+        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::TypeId(
+            $crate::engine::ql::lexer::Type::$ident,
+        ))
+    };
+}
+
+/*
+    Frankly, this is just for lazy people like me. Do not judge
+    -- Sayan (@ohsayan)
+*/
+macro_rules! Token {
+    // misc
+    (@) => {
+        __sym_token!(SymAt)
+    };
+    (#) => {
+        __sym_token!(SymHash)
+    };
+    ($) => {
+        __sym_token!(SymDollar)
+    };
+    (%) => {
+        __sym_token!(SymPercent)
+    };
+    (.) => {
+        __sym_token!(SymPeriod)
+    };
+    (,) => {
+        __sym_token!(SymComma)
+    };
+    (_) => {
+        __sym_token!(SymUnderscore)
+    };
+    (?) => {
+        __sym_token!(SymQuestion)
+    };
+    (:) => {
+        __sym_token!(SymColon)
+    };
+    (;) => {
+        __sym_token!(SymSemicolon)
+    };
+    (~) => {
+        __sym_token!(SymTilde)
+    };
+    // logical
+    (!) => {
+        __sym_token!(OpLogicalNot)
+    };
+    (^) => {
+        __sym_token!(OpLogicalXor)
+    };
+    (&) => {
+        __sym_token!(OpLogicalAnd)
+    };
+    (|) => {
+        __sym_token!(OpLogicalOr)
+    };
+    // operator misc.
+    (=) => {
+        __sym_token!(OpAssign)
+    };
+    // arithmetic
+    (+) => {
+        __sym_token!(OpArithmeticAdd)
+    };
+    (-) => {
+        __sym_token!(OpArithmeticSub)
+    };
+    (*) => {
+        __sym_token!(OpArithmeticMul)
+    };
+    (/) => {
+        __sym_token!(OpArithmeticDiv)
+    };
+    // relational
+    (>) => {
+        __sym_token!(OpComparatorGt)
+    };
+    (<) => {
+        __sym_token!(OpComparatorLt)
+    };
+    // ddl keywords
+    (use) => {
+        __ddl_token!(Use)
+    };
+    (create) => {
+        __ddl_token!(Create)
+    };
+    (alter) => {
+        __ddl_token!(Alter)
+    };
+    (drop) => {
+        __ddl_token!(Drop)
+    };
+    (inspect) => {
+        __ddl_token!(Inspect)
+    };
+    (model) => {
+        __ddl_token!(Model)
+    };
+    (space) => {
+        __ddl_token!(Space)
+    };
+    (primary) => {
+        __ddl_token!(Primary)
+    };
+    // ddl misc
+    (with) => {
+        __ddl_misc_token!(With)
+    };
+    (add) => {
+        __ddl_misc_token!(Add)
+    };
+    (remove) => {
+        __ddl_misc_token!(Remove)
+    };
+    (sort) => {
+        __ddl_misc_token!(Sort)
+    };
+    (type) => {
+        __ddl_misc_token!(Type)
+    };
+    // dml
+    (insert) => {
+        __dml_token!(Insert)
+    };
+    (select) => {
+        __dml_token!(Select)
+    };
+    (update) => {
+        __dml_token!(Update)
+    };
+    (delete) => {
+        __dml_token!(Delete)
+    };
+    (exists) => {
+        __dml_token!(Exists)
+    };
+    (truncate) => {
+        __dml_token!(Truncate)
+    };
+    // dml misc
+    (limit) => {
+        __dml_misc_token!(Limit)
+    };
+    (from) => {
+        __dml_misc_token!(From)
+    };
+    (into) => {
+        __dml_misc_token!(Into)
+    };
+    (where) => {
+        __dml_misc_token!(Where)
+    };
+    (if) => {
+        __dml_misc_token!(If)
+    };
+    (and) => {
+        __dml_misc_token!(And)
+    };
+    (as) => {
+        __dml_misc_token!(As)
+    };
+    (by) => {
+        __dml_misc_token!(By)
+    };
+    (asc) => {
+        __dml_misc_token!(Asc)
+    };
+    (desc) => {
+        __dml_misc_token!(Desc)
+    };
+    // types
+    (string) => {
+        __type_token!(String)
+    };
+    (binary) => {
+        __type_token!(Binary)
+    };
+    (list) => {
+        __type_token!(List)
+    };
+    (map) => {
+        __type_token!(Map)
+    };
+    (bool) => {
+        __type_token!(Bool)
+    };
+    (int) => {
+        __type_token!(Int)
+    };
+    (double) => {
+        __type_token!(Double)
+    };
+    (float) => {
+        __type_token!(Float)
+    };
+}
+
 macro_rules! dict {
     () => {
         <::std::collections::HashMap<_, _> as ::core::default::Default>::default()

From f69505182625c84f7646b16b935b6a6ed65e0325 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 9 Nov 2022 21:35:38 +0530
Subject: [PATCH 035/310] Add parsing for insert statements

---
 server/src/engine/memory/mod.rs |   1 +
 server/src/engine/ql/ast.rs     |  44 +++++--
 server/src/engine/ql/dml.rs     | 163 ++++++++++++++++++++++---
 server/src/engine/ql/lexer.rs   |   1 +
 server/src/engine/ql/macros.rs  |  48 +++++++-
 server/src/engine/ql/tests.rs   | 205 ++++++++++++++++++++++++++++++--
 server/src/util/macros.rs       |  10 +-
 7 files changed, 428 insertions(+), 44 deletions(-)

diff --git a/server/src/engine/memory/mod.rs b/server/src/engine/memory/mod.rs
index 570e353f..ab3a74e2 100644
--- a/server/src/engine/memory/mod.rs
+++ b/server/src/engine/memory/mod.rs
@@ -29,6 +29,7 @@
 /// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
 /// be of one type.
 #[derive(Debug, PartialEq)]
+#[cfg_attr(debug_assertions, derive(Clone))]
 pub enum DataType {
     /// An UTF-8 string
     String(String),
diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 24abe98c..8f9f96c0 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -39,33 +39,57 @@ use {
 
 #[derive(Debug, PartialEq)]
 pub enum Entity {
-    Current(RawSlice),
     Partial(RawSlice),
+    Single(RawSlice),
     Full(RawSlice, RawSlice),
 }
 
 impl Entity {
+    #[inline(always)]
+    pub(super) unsafe fn full_entity_from_slice(sl: &[Token]) -> Self {
+        Entity::Full(
+            extract!(&sl[0], Token::Ident(sl) => sl.clone()),
+            extract!(&sl[2], Token::Ident(sl) => sl.clone()),
+        )
+    }
+    #[inline(always)]
+    pub(super) unsafe fn single_entity_from_slice(sl: &[Token]) -> Self {
+        Entity::Single(extract!(&sl[0], Token::Ident(sl) => sl.clone()))
+    }
+    #[inline(always)]
+    pub(super) unsafe fn partial_entity_from_slice(sl: &[Token]) -> Self {
+        Entity::Partial(extract!(&sl[1], Token::Ident(sl) => sl.clone()))
+    }
+    #[inline(always)]
+    pub(super) fn tokens_with_partial(sl: &[Token]) -> bool {
+        sl.len() > 1 && sl[0] == Token![:] && sl[1].is_ident()
+    }
+    #[inline(always)]
+    pub(super) fn tokens_with_single(sl: &[Token]) -> bool {
+        !sl.is_empty() && sl[0].is_ident()
+    }
+    #[inline(always)]
+    pub(super) fn tokens_with_full(sl: &[Token]) -> bool {
+        sl.len() > 2 && sl[0].is_ident() && sl[1] == Token![.] && sl[2].is_ident()
+    }
     pub(super) fn parse(cm: &mut Compiler) -> LangResult<Self> {
         let sl = cm.remslice();
-        let is_partial = sl.len() > 1 && sl[0] == Token![:] && sl[1].is_ident();
-        let is_current = !sl.is_empty() && sl[0].is_ident();
-        let is_full = sl.len() > 2 && sl[0].is_ident() && sl[1] == Token![.] && sl[2].is_ident();
+        let is_partial = Self::tokens_with_partial(sl);
+        let is_current = Self::tokens_with_single(sl);
+        let is_full = Self::tokens_with_full(sl);
         let c;
         let r = match () {
             _ if is_full => unsafe {
                 c = 3;
-                Entity::Full(
-                    extract!(&sl[0], Token::Ident(sl) => sl.clone()),
-                    extract!(&sl[2], Token::Ident(sl) => sl.clone()),
-                )
+                Self::full_entity_from_slice(sl)
             },
             _ if is_current => unsafe {
                 c = 1;
-                Entity::Current(extract!(&sl[0], Token::Ident(sl) => sl.clone()))
+                Self::single_entity_from_slice(sl)
             },
             _ if is_partial => unsafe {
                 c = 2;
-                Entity::Partial(extract!(&sl[1], Token::Ident(sl) => sl.clone()))
+                Self::partial_entity_from_slice(sl)
             },
             _ => return Err(LangError::UnexpectedToken),
         };
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 5cda0c8a..feb4f06d 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -24,8 +24,14 @@
  *
 */
 
+use std::mem::MaybeUninit;
+
 use {
-    super::lexer::{Lit, Symbol, Token},
+    super::{
+        ast::Entity,
+        lexer::{Lit, Symbol, Token},
+        LangError, LangResult,
+    },
     crate::engine::memory::DataType,
     std::{
         collections::HashMap,
@@ -100,11 +106,10 @@ pub(super) fn parse_list_full(tok: &[Token]) -> Option<Vec<DataType>> {
     }
 }
 
-#[cfg(test)]
 /// Parse the tuple data passed in with an insert query.
 ///
 /// **Note:** Make sure you pass the `(` token
-pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<DataType>, usize, bool) {
+pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<Option<DataType>>, usize, bool) {
     let l = tok.len();
     let mut okay = l != 0;
     let mut stop = okay && tok[0] == Token::Symbol(Symbol::TtCloseParen);
@@ -113,22 +118,25 @@ pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<DataType>, usize, b
     while i < l && okay && !stop {
         match &tok[i] {
             Token::Lit(Lit::Str(s)) => {
-                data.push(s.to_string().into());
+                data.push(Some(s.to_string().into()));
             }
             Token::Lit(Lit::Num(n)) => {
-                data.push((*n).into());
+                data.push(Some((*n).into()));
             }
             Token::Lit(Lit::Bool(b)) => {
-                data.push((*b).into());
+                data.push(Some((*b).into()));
             }
             Token::Symbol(Symbol::TtOpenSqBracket) => {
                 // ah, a list
                 let mut l = Vec::new();
                 let (_, lst_i, lst_okay) = parse_list(&tok[i + 1..], &mut l);
-                data.push(l.into());
+                data.push(Some(l.into()));
                 i += lst_i;
                 okay &= lst_okay;
             }
+            Token![null] => {
+                data.push(None);
+            }
             _ => {
                 okay = false;
                 break;
@@ -145,7 +153,7 @@ pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<DataType>, usize, b
 }
 
 #[cfg(test)]
-pub(super) fn parse_data_tuple_syntax_full(tok: &[Token]) -> Option<Vec<DataType>> {
+pub(super) fn parse_data_tuple_syntax_full(tok: &[Token]) -> Option<Vec<Option<DataType>>> {
     let (ret, cnt, okay) = parse_data_tuple_syntax(tok);
     if cnt == tok.len() && okay {
         Some(ret)
@@ -154,7 +162,9 @@ pub(super) fn parse_data_tuple_syntax_full(tok: &[Token]) -> Option<Vec<DataType
     }
 }
 
-pub(super) fn parse_data_map_syntax(tok: &[Token]) -> (HashMap<Box<str>, DataType>, usize, bool) {
+pub(super) fn parse_data_map_syntax<'a>(
+    tok: &'a [Token],
+) -> (HashMap<&'a [u8], Option<DataType>>, usize, bool) {
     let l = tok.len();
     let mut okay = l != 0;
     let mut stop = okay && tok[0] == Token::Symbol(Symbol::TtCloseBrace);
@@ -166,17 +176,17 @@ pub(super) fn parse_data_map_syntax(tok: &[Token]) -> (HashMap<Box<str>, DataTyp
         match (field, expression) {
             (Token::Ident(id), Token::Lit(Lit::Str(s))) => {
                 okay &= data
-                    .insert(unsafe { id.as_str() }.into(), s.to_string().into())
+                    .insert(unsafe { id.as_slice() }, Some(s.to_string().into()))
                     .is_none();
             }
             (Token::Ident(id), Token::Lit(Lit::Num(n))) => {
                 okay &= data
-                    .insert(unsafe { id.as_str() }.into(), (*n).into())
+                    .insert(unsafe { id.as_slice() }, Some((*n).into()))
                     .is_none();
             }
             (Token::Ident(id), Token::Lit(Lit::Bool(b))) => {
                 okay &= data
-                    .insert(unsafe { id.as_str() }.into(), (*b).into())
+                    .insert(unsafe { id.as_slice() }, Some((*b).into()))
                     .is_none();
             }
             (Token::Ident(id), Token::Symbol(Symbol::TtOpenSqBracket)) => {
@@ -186,9 +196,12 @@ pub(super) fn parse_data_map_syntax(tok: &[Token]) -> (HashMap<Box<str>, DataTyp
                 okay &= lst_ok;
                 i += lst_i;
                 okay &= data
-                    .insert(unsafe { id.as_str() }.into(), l.into())
+                    .insert(unsafe { id.as_slice() }, Some(l.into()))
                     .is_none();
             }
+            (Token::Ident(id), Token![null]) => {
+                okay &= data.insert(unsafe { id.as_slice() }, None).is_none();
+            }
             _ => {
                 okay = false;
                 break;
@@ -196,19 +209,133 @@ pub(super) fn parse_data_map_syntax(tok: &[Token]) -> (HashMap<Box<str>, DataTyp
         }
         i += 3;
         let nx_comma = i < l && tok[i] == Symbol::SymComma;
-        let nx_csprn = i < l && tok[i] == Symbol::TtCloseBrace;
-        okay &= nx_comma | nx_csprn;
+        let nx_csbrc = i < l && tok[i] == Symbol::TtCloseBrace;
+        okay &= nx_comma | nx_csbrc;
         i += okay as usize;
-        stop = nx_csprn;
+        stop = nx_csbrc;
     }
     (data, i, okay && stop)
 }
 
 #[cfg(test)]
-pub(super) fn parse_data_map_syntax_full(tok: &[Token]) -> Option<HashMap<Box<str>, DataType>> {
+pub(super) fn parse_data_map_syntax_full(
+    tok: &[Token],
+) -> Option<HashMap<Box<str>, Option<DataType>>> {
     let (dat, i, ok) = parse_data_map_syntax(tok);
     if i == tok.len() && ok {
-        Some(dat)
+        Some(
+            dat.into_iter()
+                .map(|(ident, val)| {
+                    (
+                        String::from_utf8_lossy(ident).to_string().into_boxed_str(),
+                        val,
+                    )
+                })
+                .collect(),
+        )
+    } else {
+        None
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum InsertData<'a> {
+    Ordered(Vec<Option<DataType>>),
+    Map(HashMap<&'a [u8], Option<DataType>>),
+}
+
+impl<'a> From<Vec<Option<DataType>>> for InsertData<'a> {
+    fn from(v: Vec<Option<DataType>>) -> Self {
+        Self::Ordered(v)
+    }
+}
+
+impl<'a> From<HashMap<&'static [u8], Option<DataType>>> for InsertData<'a> {
+    fn from(m: HashMap<&'static [u8], Option<DataType>>) -> Self {
+        Self::Map(m)
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub struct InsertStatement<'a> {
+    pub(super) primary_key: &'a Lit,
+    pub(super) entity: Entity,
+    pub(super) data: InsertData<'a>,
+}
+
+pub(super) fn parse_insert<'a>(
+    src: &'a [Token],
+    counter: &mut usize,
+) -> LangResult<InsertStatement<'a>> {
+    /*
+        smallest:
+        insert space:primary_key ()
+        ^1     ^2   ^3^4         ^^5,6
+    */
+    let l = src.len();
+    let is_full = Entity::tokens_with_full(src);
+    let is_half = Entity::tokens_with_single(src);
+
+    let mut okay = is_full | is_half;
+    let mut i = 0;
+    let mut entity = MaybeUninit::uninit();
+
+    if is_full {
+        i += 3;
+        entity = MaybeUninit::new(unsafe { Entity::full_entity_from_slice(src) });
+    } else if is_half {
+        i += 1;
+        entity = MaybeUninit::new(unsafe { Entity::single_entity_from_slice(src) });
+    }
+
+    // primary key is a lit; atleast lit + (<oparen><cparen>) | (<obrace><cbrace>)
+    okay &= l >= (i + 4);
+    // colon, lit
+    okay &= src[i] == Token![:] && src[i + 1].is_lit();
+    // check data
+    let is_map = okay && src[i + 2] == Token![open {}];
+    let is_tuple = okay && src[i + 2] == Token![() open];
+    okay &= is_map | is_tuple;
+
+    if !okay {
+        return Err(LangError::UnexpectedToken);
+    }
+
+    let primary_key = unsafe { extract!(&src[i+1], Token::Lit(l) => l) };
+    i += 3; // skip col, lit + op/ob
+
+    let data;
+    if is_tuple {
+        let (ord, cnt, ok) = parse_data_tuple_syntax(&src[i..]);
+        okay &= ok;
+        i += cnt;
+        data = InsertData::Ordered(ord);
+    } else {
+        let (map, cnt, ok) = parse_data_map_syntax(&src[i..]);
+        okay &= ok;
+        i += cnt;
+        data = InsertData::Map(map);
+    }
+
+    *counter += i;
+
+    if okay {
+        Ok(InsertStatement {
+            primary_key,
+            entity: unsafe { entity.assume_init() },
+            data,
+        })
+    } else {
+        Err(LangError::UnexpectedToken)
+    }
+}
+
+#[cfg(test)]
+pub(super) fn parse_insert_full<'a>(tok: &'a [Token]) -> Option<InsertStatement<'a>> {
+    let mut z = 0;
+    let s = self::parse_insert(tok, &mut z);
+    if z == tok.len() {
+        s.ok()
     } else {
         None
     }
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 89868f66..68c2f174 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -81,6 +81,7 @@ pub enum Lit {
 enum_impls! {
     Lit => {
         Box<str> as Str,
+        String as Str,
         bool as Bool,
         u64 as Num,
     }
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index e01d23d5..18a3e56c 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -70,12 +70,20 @@ macro_rules! __type_token {
     };
 }
 
+macro_rules! __misc_token {
+    ($ident:ident) => {
+        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::Misc(
+            $crate::engine::ql::lexer::MiscKeyword::$ident,
+        ))
+    };
+}
+
 /*
     Frankly, this is just for lazy people like me. Do not judge
     -- Sayan (@ohsayan)
 */
 macro_rules! Token {
-    // misc
+    // misc symbol
     (@) => {
         __sym_token!(SymAt)
     };
@@ -262,6 +270,29 @@ macro_rules! Token {
     (float) => {
         __type_token!(Float)
     };
+    // tt
+    (open {}) => {
+        __sym_token!(TtOpenBrace)
+    };
+    (close {}) => {
+        __sym_token!(TtCloseBrace)
+    };
+    (() open) => {
+        __sym_token!(TtOpenParen)
+    };
+    (() close) => {
+        __sym_token!(TtCloseParen)
+    };
+    (open []) => {
+        __sym_token!(TtOpenSqBracket)
+    };
+    (close []) => {
+        __sym_token!(TtCloseSqBracket)
+    };
+    // misc
+    (null) => {
+        __misc_token!(Null)
+    };
 }
 
 macro_rules! dict {
@@ -275,6 +306,17 @@ macro_rules! dict {
     }};
 }
 
+macro_rules! dict_nullable {
+    () => {
+        <::std::collections::HashMap<_, _> as ::core::default::Default>::default()
+    };
+    ($($key:expr => $value:expr),* $(,)?) => {{
+        let mut hm: ::std::collections::HashMap<_, _> = ::core::default::Default::default();
+        $(hm.insert($key.into(), $crate::engine::ql::tests::nullable_datatype($value));)*
+        hm
+    }};
+}
+
 macro_rules! set {
     () => {
         <::std::collections::HashSet<_> as ::core::default::Default>::default()
@@ -289,3 +331,7 @@ macro_rules! set {
 macro_rules! into_array {
     ($($e:expr),* $(,)?) => { [$($e.into()),*] };
 }
+
+macro_rules! into_array_nullable {
+    ($($e:expr),* $(,)?) => { [$($crate::engine::ql::tests::nullable_datatype($e)),*] };
+}
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index ab212fbc..376eed7e 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -29,19 +29,46 @@ use {
         lexer::{Lexer, Token},
         LangResult,
     },
-    crate::util::Life,
+    crate::{engine::memory::DataType, util::Life},
 };
 
 fn lex(src: &[u8]) -> LangResult<Life<Vec<Token>>> {
     Lexer::lex(src)
 }
 
+pub trait NullableData<T> {
+    fn data(self) -> Option<T>;
+}
+
+impl<T> NullableData<DataType> for T
+where
+    T: Into<DataType>,
+{
+    fn data(self) -> Option<DataType> {
+        Some(self.into())
+    }
+}
+
+struct Null;
+
+impl NullableData<DataType> for Null {
+    fn data(self) -> Option<DataType> {
+        None
+    }
+}
+
+fn nullable_datatype(v: impl NullableData<DataType>) -> Option<DataType> {
+    v.data()
+}
+
 mod lexer_tests {
-    use super::{
-        super::lexer::{Lit, Token},
-        lex,
+    use {
+        super::{
+            super::lexer::{Lit, Token},
+            lex,
+        },
+        crate::engine::ql::LangError,
     };
-    use crate::engine::ql::LangError;
 
     macro_rules! v(
         ($e:literal) => {{
@@ -173,7 +200,7 @@ mod entity {
         let t = lex(b"hello").unwrap();
         let mut c = Compiler::new(&t);
         let r = Entity::parse(&mut c).unwrap();
-        assert_eq!(r, Entity::Current("hello".into()))
+        assert_eq!(r, Entity::Single("hello".into()))
     }
     #[test]
     fn entity_partial() {
@@ -1620,7 +1647,10 @@ mod dml_tests {
             "#)
             .unwrap();
             let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
-            assert_eq!(r.as_slice(), into_array![1234, "email@example.com", true]);
+            assert_eq!(
+                r.as_slice(),
+                into_array_nullable![1234, "email@example.com", true]
+            );
         }
 
         #[test]
@@ -1637,7 +1667,7 @@ mod dml_tests {
             let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
             assert_eq!(
                 r.as_slice(),
-                into_array![
+                into_array_nullable![
                     1234,
                     "email@example.com",
                     true,
@@ -1667,7 +1697,7 @@ mod dml_tests {
             let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
             assert_eq!(
                 r.as_slice(),
-                into_array![
+                into_array_nullable![
                     1234,
                     "email@example.com",
                     true,
@@ -1708,7 +1738,7 @@ mod dml_tests {
             let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
             assert_eq!(
                 r,
-                dict! {
+                dict_nullable! {
                     "name" => "John Appletree",
                     "email" => "john@example.com",
                     "verified" => false,
@@ -1732,7 +1762,7 @@ mod dml_tests {
             let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
             assert_eq!(
                 r,
-                dict! {
+                dict_nullable! {
                     "name" => "John Appletree",
                     "email" => "john@example.com",
                     "verified" => false,
@@ -1761,7 +1791,7 @@ mod dml_tests {
             let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
             assert_eq!(
                 r,
-                dict! {
+                dict_nullable! {
                     "name" => "John Appletree",
                     "email" => "john@example.com",
                     "verified" => false,
@@ -1779,4 +1809,155 @@ mod dml_tests {
             )
         }
     }
+    mod stmt_insert {
+        use {
+            super::*,
+            crate::engine::ql::{
+                ast::Entity,
+                dml::{self, InsertStatement},
+            },
+        };
+
+        #[test]
+        fn insert_tuple_mini() {
+            let x = lex(br#"
+                insert twitter.user:"sayan" ()
+            "#)
+            .unwrap();
+            let r = dml::parse_insert_full(&x[1..]).unwrap();
+            let e = InsertStatement {
+                primary_key: &("sayan".to_string().into()),
+                entity: Entity::Full("twitter".into(), "user".into()),
+                data: vec![].into(),
+            };
+            assert_eq!(e, r);
+        }
+        #[test]
+        fn insert_tuple() {
+            let x = lex(br#"
+                insert twitter.users:"sayan" (
+                    "Sayan",
+                    "sayan@example.com",
+                    true,
+                    12345,
+                    67890
+                )
+            "#)
+            .unwrap();
+            let r = dml::parse_insert_full(&x[1..]).unwrap();
+            let e = InsertStatement {
+                primary_key: &("sayan".to_string().into()),
+                entity: Entity::Full("twitter".into(), "users".into()),
+                data: into_array_nullable!["Sayan", "sayan@example.com", true, 12345, 67890]
+                    .to_vec()
+                    .into(),
+            };
+            assert_eq!(e, r);
+        }
+        #[test]
+        fn insert_tuple_pro() {
+            let x = lex(br#"
+                insert twitter.users:"sayan" (
+                    "Sayan",
+                    "sayan@example.com",
+                    true,
+                    12345,
+                    67890,
+                    null,
+                    12345,
+                    null
+                )
+            "#)
+            .unwrap();
+            let r = dml::parse_insert_full(&x[1..]).unwrap();
+            let e = InsertStatement {
+                primary_key: &("sayan".to_string().into()),
+                entity: Entity::Full("twitter".into(), "users".into()),
+                data: into_array_nullable![
+                    "Sayan",
+                    "sayan@example.com",
+                    true,
+                    12345,
+                    67890,
+                    Null,
+                    12345,
+                    Null
+                ]
+                .to_vec()
+                .into(),
+            };
+            assert_eq!(e, r);
+        }
+        #[test]
+        fn insert_map_mini() {
+            let tok = lex(br#"insert jotsy.app:"sayan" {}"#).unwrap();
+            let r = dml::parse_insert_full(&tok[1..]).unwrap();
+            let e = InsertStatement {
+                primary_key: &("sayan".to_string().into()),
+                entity: Entity::Full("jotsy".into(), "app".into()),
+                data: dict! {}.into(),
+            };
+            assert_eq!(e, r);
+        }
+        #[test]
+        fn insert_map() {
+            let tok = lex(br#"
+                insert jotsy.app:"sayan" {
+                    name: "Sayan",
+                    email: "sayan@example.com",
+                    verified: true,
+                    following: 12345,
+                    followers: 67890
+                }
+            "#)
+            .unwrap();
+            let r = dml::parse_insert_full(&tok[1..]).unwrap();
+            let e = InsertStatement {
+                primary_key: &("sayan".to_string().into()),
+                entity: Entity::Full("jotsy".into(), "app".into()),
+                data: dict_nullable! {
+                    "name".as_bytes() => "Sayan",
+                    "email".as_bytes() => "sayan@example.com",
+                    "verified".as_bytes() => true,
+                    "following".as_bytes() => 12345,
+                    "followers".as_bytes() => 67890
+                }
+                .into(),
+            };
+            assert_eq!(e, r);
+        }
+        #[test]
+        fn insert_map_pro() {
+            let tok = lex(br#"
+                insert jotsy.app:"sayan" {
+                    password: "pass123",
+                    email: "sayan@example.com",
+                    verified: true,
+                    following: 12345,
+                    followers: 67890,
+                    linked_smart_devices: null,
+                    bookmarks: 12345,
+                    other_linked_accounts: null
+                }
+            "#)
+            .unwrap();
+            let r = dml::parse_insert_full(&tok[1..]).unwrap();
+            let e = InsertStatement {
+                primary_key: &("sayan".to_string()).into(),
+                entity: Entity::Full("jotsy".into(), "app".into()),
+                data: dict_nullable! {
+                    "password".as_bytes() => "pass123",
+                    "email".as_bytes() => "sayan@example.com",
+                    "verified".as_bytes() => true,
+                    "following".as_bytes() => 12345,
+                    "followers".as_bytes() => 67890,
+                    "linked_smart_devices".as_bytes() => Null,
+                    "bookmarks".as_bytes() => 12345,
+                    "other_linked_accounts".as_bytes() => Null
+                }
+                .into(),
+            };
+            assert_eq!(r, e);
+        }
+    }
 }
diff --git a/server/src/util/macros.rs b/server/src/util/macros.rs
index 80acde3e..c82f80d9 100644
--- a/server/src/util/macros.rs
+++ b/server/src/util/macros.rs
@@ -26,9 +26,13 @@
 
 #[macro_export]
 macro_rules! impossible {
-    () => {
-        core::hint::unreachable_unchecked()
-    };
+    () => {{
+        if cfg!(debug_assertions) {
+            panic!("called unreachable code at: {}:{}", ::core::file!(), ::core::line!());
+        } else {
+            core::hint::unreachable_unchecked()
+        }
+    }};
 }
 
 #[macro_export]

From fe3e20ffd7d8e129d764b7592c59aabf6bf1fe64 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 9 Nov 2022 22:51:38 +0530
Subject: [PATCH 036/310] Fix fuzz tests and use custom uninit structure

---
 server/src/engine/ql/dml.rs    |  8 ++--
 server/src/engine/ql/lexer.rs  | 13 +-----
 server/src/engine/ql/schema.rs | 52 ++++++++++++-----------
 server/src/engine/ql/tests.rs  | 46 +++++++++-----------
 server/src/util/mod.rs         | 76 +++++++++++++++++++++++++++++++++-
 5 files changed, 126 insertions(+), 69 deletions(-)

diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index feb4f06d..bce853fd 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -24,7 +24,7 @@
  *
 */
 
-use std::mem::MaybeUninit;
+use crate::util::MaybeInit;
 
 use {
     super::{
@@ -278,14 +278,14 @@ pub(super) fn parse_insert<'a>(
 
     let mut okay = is_full | is_half;
     let mut i = 0;
-    let mut entity = MaybeUninit::uninit();
+    let mut entity = MaybeInit::uninit();
 
     if is_full {
         i += 3;
-        entity = MaybeUninit::new(unsafe { Entity::full_entity_from_slice(src) });
+        entity = MaybeInit::new(unsafe { Entity::full_entity_from_slice(src) });
     } else if is_half {
         i += 1;
-        entity = MaybeUninit::new(unsafe { Entity::single_entity_from_slice(src) });
+        entity = MaybeInit::new(unsafe { Entity::single_entity_from_slice(src) });
     }
 
     // primary key is a lit; atleast lit + (<oparen><cparen>) | (<obrace><cbrace>)
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 68c2f174..35bc582c 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -613,18 +613,7 @@ impl<'a> Lexer<'a> {
             match unsafe { self.deref_cursor() } {
                 byte if byte.is_ascii_alphabetic() => self.scan_ident_or_keyword(),
                 #[cfg(test)]
-                byte if byte == b'\r'
-                    && self.remaining() > 1
-                    && !(unsafe {
-                        // UNSAFE(@ohsayan): The previous condition ensures that this doesn't segfault
-                        *self.cursor().add(1)
-                    })
-                    .is_ascii_digit() =>
-                {
-                    /*
-                        NOTE(@ohsayan): The above guard might look a little messy but is necessary to support raw
-                        literals which will use the carriage return
-                    */
+                byte if byte == b'\x01' => {
                     self.push_token(Token::IgnorableComma);
                     unsafe {
                         // UNSAFE(@ohsayan): All good here. Already read the token
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 007ae945..e5e741fb 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -51,10 +51,8 @@ use {
         },
         LangError, LangResult, RawSlice,
     },
-    std::{
-        collections::{HashMap, HashSet},
-        mem::MaybeUninit,
-    },
+    crate::util::MaybeInit,
+    std::collections::{HashMap, HashSet},
 };
 
 /*
@@ -216,7 +214,7 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
     let l = tok.len();
     let mut i = 0;
     let mut okay = true;
-    let mut tmp = MaybeUninit::uninit();
+    let mut tmp = MaybeInit::uninit();
 
     while i < l {
         match (&tok[i], state) {
@@ -237,7 +235,7 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
             (Token::Ident(id), DictFoldState::CB_OR_IDENT) => {
                 // found ident, so expect colon
                 i += 1;
-                tmp = MaybeUninit::new(unsafe { id.as_str() });
+                tmp = MaybeInit::new(unsafe { id.as_str() });
                 state = DictFoldState::COLON;
             }
             (Token::Symbol(Symbol::SymColon), DictFoldState::COLON) => {
@@ -319,8 +317,10 @@ states! {
 #[derive(Debug, PartialEq)]
 /// The result of a type metadata fold
 pub struct TyMetaFoldResult {
-    c: usize,
-    b: [bool; 3],
+    cnt: usize,
+    reset: bool,
+    more: bool,
+    okay: bool,
 }
 
 impl TyMetaFoldResult {
@@ -331,8 +331,10 @@ impl TyMetaFoldResult {
     /// - okay: true
     const fn new() -> Self {
         Self {
-            c: 0,
-            b: [true, false, false],
+            cnt: 0,
+            reset: false,
+            more: false,
+            okay: true,
         }
     }
     #[inline(always)]
@@ -343,47 +345,47 @@ impl TyMetaFoldResult {
     #[inline(always)]
     /// Increment the position by `by`
     fn incr_by(&mut self, by: usize) {
-        self.c += by;
+        self.cnt += by;
     }
     #[inline(always)]
     /// Set fail
     fn set_fail(&mut self) {
-        self.b[0] = false;
+        self.okay = false;
     }
     #[inline(always)]
     /// Set has more
     fn set_has_more(&mut self) {
-        self.b[1] = true;
+        self.more = true;
     }
     #[inline(always)]
     /// Set reset
     fn set_reset(&mut self) {
-        self.b[2] = true;
+        self.reset = true;
     }
     #[inline(always)]
     /// Should the meta be reset?
     pub fn should_reset(&self) -> bool {
-        self.b[2]
+        self.reset
     }
     #[inline(always)]
     /// Returns the cursor
     pub fn pos(&self) -> usize {
-        self.c
+        self.cnt
     }
     #[inline(always)]
     /// Returns if more layers are expected
     pub fn has_more(&self) -> bool {
-        self.b[1]
+        self.more
     }
     #[inline(always)]
     /// Returns if the internal state is okay
     pub fn is_okay(&self) -> bool {
-        self.b[0]
+        self.okay
     }
     #[inline(always)]
     /// Records an expression
     fn record(&mut self, c: bool) {
-        self.b[0] &= c;
+        self.okay &= c;
     }
 }
 
@@ -395,7 +397,7 @@ pub(super) fn rfold_tymeta<const ALLOW_RESET: bool>(
 ) -> TyMetaFoldResult {
     let l = tok.len();
     let mut r = TyMetaFoldResult::new();
-    let mut tmp = MaybeUninit::uninit();
+    let mut tmp = MaybeInit::uninit();
     while r.pos() < l && r.is_okay() {
         match (&tok[r.pos()], state) {
             (
@@ -427,7 +429,7 @@ pub(super) fn rfold_tymeta<const ALLOW_RESET: bool>(
             }
             (Token::Ident(ident), TyMetaFoldState::IDENT_OR_CB) => {
                 r.incr();
-                tmp = MaybeUninit::new(unsafe { ident.as_str() });
+                tmp = MaybeInit::new(unsafe { ident.as_str() });
                 // we just saw an ident, so we expect to see a colon
                 state = TyMetaFoldState::COLON;
             }
@@ -527,14 +529,14 @@ pub(super) fn rfold_layers<const ALLOW_RESET: bool>(
     let mut i = 0;
     let mut okay = true;
     let mut state = start;
-    let mut tmp = MaybeUninit::uninit();
+    let mut tmp = MaybeInit::uninit();
     let mut dict = Dict::new();
     while i < l && okay {
         match (&tok[i], state) {
             (Token::Keyword(Keyword::TypeId(ty)), LayerFoldState::TY) => {
                 i += 1;
                 // expecting type, and found type. next is either end or an open brace or some arbitrary token
-                tmp = MaybeUninit::new(ty);
+                tmp = MaybeInit::new(ty);
                 state = LayerFoldState::END_OR_OB;
             }
             (Token::Symbol(Symbol::TtOpenBrace), LayerFoldState::END_OR_OB) => {
@@ -892,7 +894,7 @@ pub(super) fn parse_field_syntax<const ALLOW_RESET: bool>(
     let mut i = 0_usize;
     let mut state = FieldSyntaxParseState::IDENT;
     let mut okay = true;
-    let mut tmp = MaybeUninit::uninit();
+    let mut tmp = MaybeInit::uninit();
     let mut props = Dict::new();
     let mut layers = vec![];
     let mut reset = false;
@@ -900,7 +902,7 @@ pub(super) fn parse_field_syntax<const ALLOW_RESET: bool>(
         match (&tok[i], state) {
             (Token::Ident(field), FieldSyntaxParseState::IDENT) => {
                 i += 1;
-                tmp = MaybeUninit::new(field.clone());
+                tmp = MaybeInit::new(field.clone());
                 // expect open brace
                 state = FieldSyntaxParseState::OB;
             }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 376eed7e..fe93f994 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -32,7 +32,7 @@ use {
     crate::{engine::memory::DataType, util::Life},
 };
 
-fn lex(src: &[u8]) -> LangResult<Life<Vec<Token>>> {
+pub(super) fn lex(src: &[u8]) -> LangResult<Life<Vec<Token>>> {
     Lexer::lex(src)
 }
 
@@ -248,25 +248,19 @@ mod schema_tests {
             let mut should_pass = true;
             src.iter().for_each(|tok| match tok {
                 Token::IgnorableComma => {
-                    let did_add = test_utils::random_bool(&mut rng);
-                    if did_add {
-                        new_src.push(Token::Symbol(Symbol::SymComma));
-                    }
                     let added = inject(&mut new_src, &mut rng);
-                    should_pass &= added <= !did_add as usize;
+                    should_pass &= added <= 1;
                 }
                 Token::Symbol(Symbol::SymComma) => {
-                    let did_add = test_utils::random_bool(&mut rng);
-                    if did_add {
-                        new_src.push(Token::Symbol(Symbol::SymComma));
-                    } else {
-                        should_pass = false;
-                    }
                     let added = inject(&mut new_src, &mut rng);
-                    should_pass &= added == !did_add as usize;
+                    should_pass &= added == 1;
                 }
                 tok => new_src.push(tok.clone()),
             });
+            assert!(
+                new_src.iter().all(|tok| tok != &Token::IgnorableComma),
+                "found ignorable comma in rectified source"
+            );
             fuzzwith(should_pass, &new_src);
         }
     }
@@ -425,14 +419,14 @@ mod schema_tests {
                     could_have_been: {
                         this: true,
                         or_maybe_this: 100,
-                        even_this: \"hello, universe!\"\r
+                        even_this: \"hello, universe!\"\x01
                     },
                     but_oh_well: \"it continues to be the 'annoying' phrase\",
                     lorem: {
                         ipsum: {
-                            dolor: \"sit amet\"\r
-                        }\r
-                    }\r
+                            dolor: \"sit amet\"\x01
+                        }\x01
+                    }\x01
                 }
             ")
             .unwrap();
@@ -560,9 +554,9 @@ mod schema_tests {
                     maxlen: 10,
                     unique: true,
                     auth: {
-                        maybe: true\r
+                        maybe: true\x01
                     },
-                    user: \"sayan\"\r
+                    user: \"sayan\"\x01
                 }
             ")
             .unwrap();
@@ -575,9 +569,9 @@ mod schema_tests {
                 "user" => Lit::Str("sayan".into())
             };
             fuzz_tokens(&tok, |should_pass, new_src| {
-                let (ret, dict) = schema::fold_tymeta(&tok);
+                let (ret, dict) = schema::fold_tymeta(&new_src);
                 if should_pass {
-                    assert!(ret.is_okay());
+                    assert!(ret.is_okay(), "{:?}", &new_src);
                     assert!(!ret.has_more());
                     assert_eq!(ret.pos(), new_src.len());
                     assert_eq!(dict, expected);
@@ -597,10 +591,10 @@ mod schema_tests {
                     maxlen: 10,
                     unique: true,
                     auth: {
-                        maybe: true\r
+                        maybe: true\x01
                     },
                     type string,
-                    user: \"sayan\"\r
+                    user: \"sayan\"\x01
                 }
             ")
             .unwrap();
@@ -612,7 +606,7 @@ mod schema_tests {
                 },
             };
             fuzz_tokens(&tok, |should_pass, new_src| {
-                let (ret, dict) = schema::fold_tymeta(&tok);
+                let (ret, dict) = schema::fold_tymeta(&new_src);
                 if should_pass {
                     assert!(ret.is_okay());
                     assert!(ret.has_more());
@@ -721,9 +715,9 @@ mod schema_tests {
             list {
                 type list {
                     maxlen: 100,
-                    type string\r
+                    type string\x01
                 },
-                unique: true\r
+                unique: true\x01
             }
         ")
             .unwrap();
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index 6cf7650a..c76d30c3 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -28,15 +28,20 @@
 mod macros;
 pub mod compiler;
 pub mod error;
+pub mod os;
 #[cfg(test)]
 pub mod test_utils;
-pub mod os;
 use {
     crate::{
         actions::{ActionError, ActionResult},
         protocol::interface::ProtocolSpec,
     },
-    core::{fmt::Debug, marker::PhantomData, ops::Deref},
+    core::{
+        fmt::{self, Debug},
+        marker::PhantomData,
+        mem::MaybeUninit,
+        ops::Deref,
+    },
     std::process,
 };
 
@@ -213,3 +218,70 @@ impl<'a, T: PartialEq> PartialEq<T> for Life<'a, T> {
 
 unsafe impl<'a, T: Send> Send for Life<'a, T> {}
 unsafe impl<'a, T: Sync> Sync for Life<'a, T> {}
+
+pub struct MaybeInit<T> {
+    #[cfg(test)]
+    is_init: bool,
+    base: MaybeUninit<T>,
+}
+
+impl<T> MaybeInit<T> {
+    #[cfg(not(test))]
+    const _SZ_REL: () =
+        assert!(core::mem::size_of::<Self>() == core::mem::size_of::<MaybeUninit<T>>());
+    pub const fn uninit() -> Self {
+        Self {
+            #[cfg(test)]
+            is_init: false,
+            base: MaybeUninit::uninit(),
+        }
+    }
+    pub const fn new(val: T) -> Self {
+        Self {
+            #[cfg(test)]
+            is_init: true,
+            base: MaybeUninit::new(val),
+        }
+    }
+    pub const unsafe fn assume_init(self) -> T {
+        #[cfg(test)]
+        {
+            if !self.is_init {
+                panic!("Tried to `assume_init` on uninitialized data");
+            }
+        }
+        self.base.assume_init()
+    }
+    pub const unsafe fn assume_init_ref(&self) -> &T {
+        #[cfg(test)]
+        {
+            if !self.is_init {
+                panic!("Tried to `assume_init_ref` on uninitialized data");
+            }
+        }
+        self.base.assume_init_ref()
+    }
+}
+
+#[cfg(test)]
+impl<T: fmt::Debug> fmt::Debug for MaybeInit<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let dat_fmt = if self.is_init {
+            unsafe { format!("{:?}", self.base.assume_init_ref()) }
+        } else {
+            format!("MaybeUninit {{...}}")
+        };
+        f.debug_struct("MaybeInit")
+            .field("is_init", &self.is_init)
+            .field("base", &dat_fmt)
+            .finish()
+    }
+}
+#[cfg(not(test))]
+impl<T: fmt::Debug> fmt::Debug for MaybeInit<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("MaybeInit")
+            .field("base", &self.base)
+            .finish()
+    }
+}

From ee7196d499d51cac72be75f6e3937ee6023457f0 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 11 Nov 2022 23:43:41 +0530
Subject: [PATCH 037/310] Add parsing for select queries

---
 server/src/engine/ql/ast.rs   |  12 ++--
 server/src/engine/ql/dml.rs   | 122 +++++++++++++++++++++++++++++-----
 server/src/engine/ql/tests.rs |  71 ++++++++++++++++++++
 server/src/util/macros.rs     |   8 ++-
 server/src/util/mod.rs        |  19 +++++-
 5 files changed, 206 insertions(+), 26 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 8f9f96c0..36f07920 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -61,16 +61,16 @@ impl Entity {
         Entity::Partial(extract!(&sl[1], Token::Ident(sl) => sl.clone()))
     }
     #[inline(always)]
-    pub(super) fn tokens_with_partial(sl: &[Token]) -> bool {
-        sl.len() > 1 && sl[0] == Token![:] && sl[1].is_ident()
+    pub(super) fn tokens_with_partial(tok: &[Token]) -> bool {
+        tok.len() > 1 && tok[0] == Token![:] && tok[1].is_ident()
     }
     #[inline(always)]
-    pub(super) fn tokens_with_single(sl: &[Token]) -> bool {
-        !sl.is_empty() && sl[0].is_ident()
+    pub(super) fn tokens_with_single(tok: &[Token]) -> bool {
+        !tok.is_empty() && tok[0].is_ident()
     }
     #[inline(always)]
-    pub(super) fn tokens_with_full(sl: &[Token]) -> bool {
-        sl.len() > 2 && sl[0].is_ident() && sl[1] == Token![.] && sl[2].is_ident()
+    pub(super) fn tokens_with_full(tok: &[Token]) -> bool {
+        tok.len() > 2 && tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident()
     }
     pub(super) fn parse(cm: &mut Compiler) -> LangResult<Self> {
         let sl = cm.remslice();
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index bce853fd..9ed22f72 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -24,21 +24,42 @@
  *
 */
 
-use crate::util::MaybeInit;
+/*
+    TODO(@ohsayan): For now we've settled for an imprecise error site reporting for simplicity, which we
+    should augment in future revisions of the QL engine
+*/
 
 use {
     super::{
         ast::Entity,
         lexer::{Lit, Symbol, Token},
-        LangError, LangResult,
+        LangError, LangResult, RawSlice,
     },
-    crate::engine::memory::DataType,
+    crate::{engine::memory::DataType, util::MaybeInit},
     std::{
         collections::HashMap,
         mem::{discriminant, Discriminant},
     },
 };
 
+/*
+    Misc
+*/
+
+#[inline(always)]
+fn process_entity(tok: &[Token], d: &mut MaybeInit<Entity>, i: &mut usize) -> bool {
+    let is_full = Entity::tokens_with_full(tok);
+    let is_single = Entity::tokens_with_single(tok);
+    if is_full {
+        *i += 3;
+        *d = MaybeInit::new(unsafe { Entity::full_entity_from_slice(tok) })
+    } else if is_single {
+        *i += 1;
+        *d = MaybeInit::new(unsafe { Entity::single_entity_from_slice(tok) });
+    }
+    is_full | is_single
+}
+
 /*
     Impls for insert
 */
@@ -155,7 +176,8 @@ pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<Option<DataType>>,
 #[cfg(test)]
 pub(super) fn parse_data_tuple_syntax_full(tok: &[Token]) -> Option<Vec<Option<DataType>>> {
     let (ret, cnt, okay) = parse_data_tuple_syntax(tok);
-    if cnt == tok.len() && okay {
+    assert!(cnt == tok.len(), "didn't use full length");
+    if okay {
         Some(ret)
     } else {
         None
@@ -222,7 +244,8 @@ pub(super) fn parse_data_map_syntax_full(
     tok: &[Token],
 ) -> Option<HashMap<Box<str>, Option<DataType>>> {
     let (dat, i, ok) = parse_data_map_syntax(tok);
-    if i == tok.len() && ok {
+    assert!(i == tok.len(), "didn't use full length");
+    if ok {
         Some(
             dat.into_iter()
                 .map(|(ident, val)| {
@@ -280,13 +303,7 @@ pub(super) fn parse_insert<'a>(
     let mut i = 0;
     let mut entity = MaybeInit::uninit();
 
-    if is_full {
-        i += 3;
-        entity = MaybeInit::new(unsafe { Entity::full_entity_from_slice(src) });
-    } else if is_half {
-        i += 1;
-        entity = MaybeInit::new(unsafe { Entity::single_entity_from_slice(src) });
-    }
+    okay &= process_entity(&src[i..], &mut entity, &mut i);
 
     // primary key is a lit; atleast lit + (<oparen><cparen>) | (<obrace><cbrace>)
     okay &= l >= (i + 4);
@@ -334,9 +351,84 @@ pub(super) fn parse_insert<'a>(
 pub(super) fn parse_insert_full<'a>(tok: &'a [Token]) -> Option<InsertStatement<'a>> {
     let mut z = 0;
     let s = self::parse_insert(tok, &mut z);
-    if z == tok.len() {
-        s.ok()
+    assert!(z == tok.len(), "didn't use full length");
+    s.ok()
+}
+
+/*
+    Impls for select
+*/
+
+#[derive(Debug, PartialEq)]
+pub(super) struct SelectStatement<'a> {
+    /// the primary key
+    pub(super) primary_key: &'a Lit,
+    /// the entity
+    pub(super) entity: Entity,
+    /// fields in order of querying. will be zero when wildcard is set
+    pub(super) fields: Vec<RawSlice>,
+    /// whether a wildcard was passed
+    pub(super) wildcard: bool,
+}
+
+/// Parse a `select` query. The cursor should have already passed the `select` token when this
+/// function is called.
+pub(super) fn parse_select<'a>(
+    tok: &'a [Token],
+    counter: &mut usize,
+) -> LangResult<SelectStatement<'a>> {
+    let l = tok.len();
+
+    let mut i = 0_usize;
+    let mut okay = l > 4;
+    let mut fields = Vec::new();
+    let is_wildcard = i < l && tok[i] == Token![*];
+    i += is_wildcard as usize;
+
+    while okay && i < l && tok[i].is_ident() && !is_wildcard {
+        unsafe {
+            fields.push(extract!(&tok[i], Token::Ident(id) => id.clone()));
+        }
+        i += 1;
+        // skip comma
+        let nx_comma = i < l && tok[i] == Token![,];
+        let nx_from = i < l && tok[i] == Token![from];
+        okay &= nx_comma | nx_from;
+        i += nx_comma as usize;
+    }
+
+    okay &= i < l && tok[i] == Token![from];
+    i += okay as usize;
+
+    // parsed upto select a, b, c from ...; now parse entity and select
+    let mut entity = MaybeInit::uninit();
+    okay &= process_entity(&tok[i..], &mut entity, &mut i);
+
+    // now primary key
+    okay &= i < l && tok[i] == Token![:];
+    i += okay as usize;
+    okay &= i < l && tok[i].is_lit();
+
+    *counter += i + okay as usize;
+
+    if okay {
+        let primary_key = unsafe { extract!(tok[i], Token::Lit(ref l) => l) };
+        Ok(SelectStatement {
+            primary_key,
+            entity: unsafe { entity.assume_init() },
+            fields,
+            wildcard: is_wildcard,
+        })
     } else {
-        None
+        Err(LangError::UnexpectedToken)
     }
 }
+
+#[cfg(test)]
+/// **test-mode only** parse for a `select` where the full token stream is exhausted
+pub(super) fn parse_select_full<'a>(tok: &'a [Token]) -> Option<SelectStatement<'a>> {
+    let mut i = 0;
+    let r = self::parse_select(tok, &mut i);
+    assert!(i == tok.len(), "didn't use full length");
+    r.ok()
+}
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index fe93f994..5348591a 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -1954,4 +1954,75 @@ mod dml_tests {
             assert_eq!(r, e);
         }
     }
+
+    mod stmt_select {
+        use {
+            super::*,
+            crate::engine::ql::{
+                ast::Entity,
+                dml::{self, SelectStatement},
+                lexer::Lit,
+            },
+        };
+        #[test]
+        fn select_mini() {
+            let tok = lex(br#"
+                select * from user:"sayan"
+            "#)
+            .unwrap();
+            let r = dml::parse_select_full(&tok[1..]).unwrap();
+            let e = SelectStatement {
+                primary_key: &Lit::Str("sayan".into()),
+                entity: Entity::Single("user".into()),
+                fields: [].to_vec(),
+                wildcard: true,
+            };
+            assert_eq!(r, e);
+        }
+        #[test]
+        fn select() {
+            let tok = lex(br#"
+                select field1 from user:"sayan"
+            "#)
+            .unwrap();
+            let r = dml::parse_select_full(&tok[1..]).unwrap();
+            let e = SelectStatement {
+                primary_key: &Lit::Str("sayan".into()),
+                entity: Entity::Single("user".into()),
+                fields: ["field1".into()].to_vec(),
+                wildcard: false,
+            };
+            assert_eq!(r, e);
+        }
+        #[test]
+        fn select_pro() {
+            let tok = lex(br#"
+                select field1 from twitter.user:"sayan"
+            "#)
+            .unwrap();
+            let r = dml::parse_select_full(&tok[1..]).unwrap();
+            let e = SelectStatement {
+                primary_key: &Lit::Str("sayan".into()),
+                entity: Entity::Full("twitter".into(), "user".into()),
+                fields: ["field1".into()].to_vec(),
+                wildcard: false,
+            };
+            assert_eq!(r, e);
+        }
+        #[test]
+        fn select_pro_max() {
+            let tok = lex(br#"
+                select field1, field2 from twitter.user:"sayan"
+            "#)
+            .unwrap();
+            let r = dml::parse_select_full(&tok[1..]).unwrap();
+            let e = SelectStatement {
+                primary_key: &Lit::Str("sayan".into()),
+                entity: Entity::Full("twitter".into(), "user".into()),
+                fields: ["field1".into(), "field2".into()].to_vec(),
+                wildcard: false,
+            };
+            assert_eq!(r, e);
+        }
+    }
 }
diff --git a/server/src/util/macros.rs b/server/src/util/macros.rs
index c82f80d9..3c37cc8c 100644
--- a/server/src/util/macros.rs
+++ b/server/src/util/macros.rs
@@ -28,9 +28,13 @@
 macro_rules! impossible {
     () => {{
         if cfg!(debug_assertions) {
-            panic!("called unreachable code at: {}:{}", ::core::file!(), ::core::line!());
+            panic!(
+                "reached unreachable case at: {}:{}",
+                ::core::file!(),
+                ::core::line!()
+            );
         } else {
-            core::hint::unreachable_unchecked()
+            ::core::hint::unreachable_unchecked()
         }
     }};
 }
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index c76d30c3..95f6a5d1 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -219,6 +219,9 @@ impl<'a, T: PartialEq> PartialEq<T> for Life<'a, T> {
 unsafe impl<'a, T: Send> Send for Life<'a, T> {}
 unsafe impl<'a, T: Sync> Sync for Life<'a, T> {}
 
+/// [`MaybeInit`] is a structure that is like an [`Option`] in debug mode and like
+/// [`MaybeUninit`] in release mode. This means that provided there are good enough test cases, most
+/// incorrect `assume_init` calls should be detected in the test phase.
 pub struct MaybeInit<T> {
     #[cfg(test)]
     is_init: bool,
@@ -226,9 +229,7 @@ pub struct MaybeInit<T> {
 }
 
 impl<T> MaybeInit<T> {
-    #[cfg(not(test))]
-    const _SZ_REL: () =
-        assert!(core::mem::size_of::<Self>() == core::mem::size_of::<MaybeUninit<T>>());
+    /// Initialize a new uninitialized variant
     pub const fn uninit() -> Self {
         Self {
             #[cfg(test)]
@@ -236,6 +237,7 @@ impl<T> MaybeInit<T> {
             base: MaybeUninit::uninit(),
         }
     }
+    /// Initialize with a value
     pub const fn new(val: T) -> Self {
         Self {
             #[cfg(test)]
@@ -243,6 +245,11 @@ impl<T> MaybeInit<T> {
             base: MaybeUninit::new(val),
         }
     }
+    /// Assume that `self` is initialized and return the inner value
+    ///
+    /// ## Safety
+    ///
+    /// Caller needs to ensure that the data is actually initialized
     pub const unsafe fn assume_init(self) -> T {
         #[cfg(test)]
         {
@@ -252,6 +259,11 @@ impl<T> MaybeInit<T> {
         }
         self.base.assume_init()
     }
+    /// Assume that `self` is initialized and return a reference
+    ///
+    /// ## Safety
+    ///
+    /// Caller needs to ensure that the data is actually initialized
     pub const unsafe fn assume_init_ref(&self) -> &T {
         #[cfg(test)]
         {
@@ -277,6 +289,7 @@ impl<T: fmt::Debug> fmt::Debug for MaybeInit<T> {
             .finish()
     }
 }
+
 #[cfg(not(test))]
 impl<T: fmt::Debug> fmt::Debug for MaybeInit<T> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

From a1c85fb3059861a9e9e657108bdff46ef56ec9d7 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 15 Nov 2022 11:31:32 +0530
Subject: [PATCH 038/310] Add parsing for update queries

---
 server/src/corestore/buffers.rs |   2 +-
 server/src/engine/ql/ast.rs     |   6 ++
 server/src/engine/ql/dml.rs     | 174 ++++++++++++++++++++++++++++++++
 server/src/engine/ql/lexer.rs   |   6 ++
 server/src/engine/ql/tests.rs   | 121 ++++++++++++++++++++++
 5 files changed, 308 insertions(+), 1 deletion(-)

diff --git a/server/src/corestore/buffers.rs b/server/src/corestore/buffers.rs
index 7987de75..63c1e605 100644
--- a/server/src/corestore/buffers.rs
+++ b/server/src/corestore/buffers.rs
@@ -41,7 +41,7 @@ macro_rules! lut {
     };
 }
 
-const PAIR_MAP_LUT: [u8; 200] = [
+static PAIR_MAP_LUT: [u8; 200] = [
     0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, 0x30, 0x36, 0x30, 0x37,
     0x30, 0x38, 0x30, 0x39, // 0x30
     0x31, 0x30, 0x31, 0x31, 0x31, 0x32, 0x31, 0x33, 0x31, 0x34, 0x31, 0x35, 0x31, 0x36, 0x31, 0x37,
diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 36f07920..635e827a 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -44,6 +44,12 @@ pub enum Entity {
     Full(RawSlice, RawSlice),
 }
 
+impl<T: Into<RawSlice>, U: Into<RawSlice>> From<(T, U)> for Entity {
+    fn from((space, model): (T, U)) -> Self {
+        Self::Full(space.into(), model.into())
+    }
+}
+
 impl Entity {
     #[inline(always)]
     pub(super) unsafe fn full_entity_from_slice(sl: &[Token]) -> Self {
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 9ed22f72..e3e6f9a2 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -432,3 +432,177 @@ pub(super) fn parse_select_full<'a>(tok: &'a [Token]) -> Option<SelectStatement<
     assert!(i == tok.len(), "didn't use full length");
     r.ok()
 }
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+/// TODO(@ohsayan): This only helps with the parser test for now. Replace this with actual operator expressions
+pub(super) enum Operator {
+    Assign,
+    AddAssign,
+    SubAssign,
+    MulAssign,
+    DivAssign,
+}
+
+static OPERATOR: [Operator; 6] = [
+    Operator::Assign,
+    Operator::Assign,
+    Operator::AddAssign,
+    Operator::SubAssign,
+    Operator::MulAssign,
+    Operator::DivAssign,
+];
+
+#[derive(Debug, PartialEq)]
+pub struct AssignmentExpression<'a> {
+    /// the LHS ident
+    pub(super) lhs: RawSlice,
+    /// the RHS lit
+    pub(super) rhs: &'a Lit,
+    /// operator
+    pub(super) operator_fn: Operator,
+}
+
+impl<'a> AssignmentExpression<'a> {
+    pub(super) fn new(lhs: RawSlice, rhs: &'a Lit, operator_fn: Operator) -> Self {
+        Self {
+            lhs,
+            rhs,
+            operator_fn,
+        }
+    }
+    /// Attempt to parse an expression and then append it to the given vector of expressions. This will return `true`
+    /// if the expression was parsed correctly, otherwise `false` is returned
+    #[inline(always)]
+    fn parse_and_append_expression(
+        tok: &'a [Token],
+        expressions: &mut Vec<Self>,
+        counter: &mut usize,
+    ) -> bool {
+        /*
+            smallest expression:
+            <ident> <operator> <lit>
+        */
+        let l = tok.len();
+        let mut i = 0;
+        let mut okay = tok.len() > 2 && tok[0].is_ident();
+        i += okay as usize;
+
+        let op_assign = (i < l && tok[i] == Token![=]) as usize * 1;
+        let op_add = (i < l && tok[i] == Token![+]) as usize * 2;
+        let op_sub = (i < l && tok[i] == Token![-]) as usize * 3;
+        let op_mul = (i < l && tok[i] == Token![*]) as usize * 4;
+        let op_div = (i < l && tok[i] == Token![/]) as usize * 5;
+
+        let operator_code = op_assign + op_add + op_sub + op_mul + op_div;
+        unsafe {
+            // UNSAFE(@ohsayan): Inherently obvious, just a hint
+            if operator_code > 5 {
+                impossible!()
+            }
+        }
+        okay &= operator_code != 0;
+        i += okay as usize;
+
+        let has_double_assign = i < l && tok[i] == Token![=];
+        let double_assign_okay = operator_code != 1 && has_double_assign;
+        let single_assign_okay = operator_code == 1 && !double_assign_okay;
+        okay &= single_assign_okay | double_assign_okay;
+        i += double_assign_okay as usize; // skip on <op>assign
+
+        let has_rhs = i < l && tok[i].is_lit();
+        okay &= has_rhs;
+        *counter += i + has_rhs as usize;
+
+        if okay {
+            let expression = unsafe {
+                AssignmentExpression {
+                    lhs: extract!(tok[0], Token::Ident(ref r) => r.clone()),
+                    rhs: extract!(tok[i], Token::Lit(ref l) => l),
+                    operator_fn: OPERATOR[operator_code as usize],
+                }
+            };
+            expressions.push(expression);
+        }
+
+        okay
+    }
+}
+
+#[cfg(test)]
+pub(super) fn parse_expression_full<'a>(tok: &'a [Token]) -> Option<AssignmentExpression<'a>> {
+    let mut i = 0;
+    let mut exprs = Vec::new();
+    if AssignmentExpression::parse_and_append_expression(tok, &mut exprs, &mut i) {
+        assert_eq!(i, tok.len(), "full token stream not utilized");
+        Some(exprs.remove(0))
+    } else {
+        None
+    }
+}
+
+/*
+    Impls for update
+*/
+
+#[derive(Debug, PartialEq)]
+pub struct UpdateStatement<'a> {
+    pub(super) primary_key: &'a Lit,
+    pub(super) entity: Entity,
+    pub(super) expressions: Vec<AssignmentExpression<'a>>,
+}
+
+impl<'a> UpdateStatement<'a> {
+    pub(super) fn parse_update(tok: &'a [Token], counter: &mut usize) -> LangResult<Self> {
+        let l = tok.len();
+        // TODO(@ohsayan): This would become 8 when we add `SET`. It isn't exactly needed but is for purely aesthetic purposes
+        let mut okay = l > 7;
+        let mut i = 0_usize;
+
+        // parse entity
+        let mut entity = MaybeInit::uninit();
+        okay &= process_entity(tok, &mut entity, &mut i);
+
+        // check if we have our primary key
+        okay &= i < l && tok[i] == Token![:];
+        i += okay as usize;
+        okay &= i < l && tok[i].is_lit();
+        let primary_key_location = i;
+        i += okay as usize;
+
+        // now parse expressions that we have to update
+        let mut expressions = Vec::new();
+        while i < l && okay {
+            okay &= AssignmentExpression::parse_and_append_expression(
+                &tok[i..],
+                &mut expressions,
+                &mut i,
+            );
+            let nx_comma = i < l && tok[i] == Token![,];
+            // TODO(@ohsayan): Define the need for a semicolon; remember, no SQL unsafety!
+            let nx_over = i == l;
+            okay &= nx_comma | nx_over;
+            i += nx_comma as usize;
+        }
+        *counter += i;
+
+        if okay {
+            let primary_key =
+                unsafe { extract!(tok[primary_key_location], Token::Lit(ref pk) => pk) };
+            Ok(Self {
+                primary_key,
+                entity: unsafe { entity.assume_init() },
+                expressions,
+            })
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    }
+}
+
+#[cfg(test)]
+pub(super) fn parse_update_full<'a>(tok: &'a [Token]) -> LangResult<UpdateStatement<'a>> {
+    let mut i = 0;
+    let r = UpdateStatement::parse_update(tok, &mut i);
+    assert_eq!(i, tok.len(), "full token stream not utilized");
+    r
+}
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 35bc582c..869b3681 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -78,6 +78,12 @@ pub enum Lit {
     UnsafeLit(RawSlice),
 }
 
+impl From<&'static str> for Lit {
+    fn from(s: &'static str) -> Self {
+        Self::Str(s.into())
+    }
+}
+
 enum_impls! {
     Lit => {
         Box<str> as Str,
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 5348591a..796534b6 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -2025,4 +2025,125 @@ mod dml_tests {
             assert_eq!(r, e);
         }
     }
+    mod expression_tests {
+        use {
+            super::*,
+            crate::engine::ql::{
+                dml::{self, AssignmentExpression, Operator},
+                lexer::Lit,
+            },
+        };
+        #[test]
+        fn expr_assign() {
+            let src = lex(b"username = 'sayan'").unwrap();
+            let r = dml::parse_expression_full(&src).unwrap();
+            assert_eq!(
+                r,
+                AssignmentExpression {
+                    lhs: "username".into(),
+                    rhs: &Lit::Str("sayan".into()),
+                    operator_fn: Operator::Assign
+                }
+            );
+        }
+        #[test]
+        fn expr_add_assign() {
+            let src = lex(b"followers += 100").unwrap();
+            let r = dml::parse_expression_full(&src).unwrap();
+            assert_eq!(
+                r,
+                AssignmentExpression {
+                    lhs: "followers".into(),
+                    rhs: &(100.into()),
+                    operator_fn: Operator::AddAssign
+                }
+            );
+        }
+        #[test]
+        fn expr_sub_assign() {
+            let src = lex(b"following -= 150").unwrap();
+            let r = dml::parse_expression_full(&src).unwrap();
+            assert_eq!(
+                r,
+                AssignmentExpression {
+                    lhs: "following".into(),
+                    rhs: &(150.into()),
+                    operator_fn: Operator::SubAssign
+                }
+            );
+        }
+        #[test]
+        fn expr_mul_assign() {
+            let src = lex(b"product_qty *= 2").unwrap();
+            let r = dml::parse_expression_full(&src).unwrap();
+            assert_eq!(
+                r,
+                AssignmentExpression {
+                    lhs: "product_qty".into(),
+                    rhs: &(2.into()),
+                    operator_fn: Operator::MulAssign
+                }
+            );
+        }
+        #[test]
+        fn expr_div_assign() {
+            let src = lex(b"image_crop_factor /= 2").unwrap();
+            let r = dml::parse_expression_full(&src).unwrap();
+            assert_eq!(
+                r,
+                AssignmentExpression {
+                    lhs: "image_crop_factor".into(),
+                    rhs: &(2.into()),
+                    operator_fn: Operator::DivAssign
+                }
+            );
+        }
+    }
+    mod update_statement {
+        use {
+            super::*,
+            crate::engine::ql::dml::{self, AssignmentExpression, Operator, UpdateStatement},
+        };
+        #[test]
+        fn update_mini() {
+            let tok = lex(br#"
+                update jotsy.app:"sayan" notes += "this is my new note"
+            "#)
+            .unwrap();
+            let note = "this is my new note".to_string().into();
+            let r = dml::parse_update_full(&tok[1..]).unwrap();
+            let e = UpdateStatement {
+                primary_key: &("sayan".to_owned().into()),
+                entity: ("jotsy", "app").into(),
+                expressions: vec![AssignmentExpression {
+                    lhs: "notes".into(),
+                    rhs: &note,
+                    operator_fn: Operator::AddAssign,
+                }],
+            };
+            assert_eq!(r, e);
+        }
+        #[test]
+        fn update() {
+            let tok = lex(br#"
+                update jotsy.app:"sayan" notes += "this is my new note", email = "sayan@example.com"
+            "#)
+            .unwrap();
+            let r = dml::parse_update_full(&tok[1..]).unwrap();
+
+            let field_note = "this is my new note".into();
+            let field_email = "sayan@example.com".into();
+            let primary_key = "sayan".into();
+            let e = UpdateStatement {
+                primary_key: &primary_key,
+                entity: ("jotsy", "app").into(),
+                expressions: vec![
+                    AssignmentExpression::new("notes".into(), &field_note, Operator::AddAssign),
+                    AssignmentExpression::new("email".into(), &field_email, Operator::Assign),
+                ],
+            };
+
+            assert_eq!(r, e);
+        }
+    }
 }

From 943125116bfa07152a23f68a90f586c1d29e1429 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 15 Nov 2022 12:20:07 +0530
Subject: [PATCH 039/310] Add parsing for delete queries

---
 server/src/engine/ql/dml.rs   | 60 +++++++++++++++++++++++++++++++++++
 server/src/engine/ql/tests.rs | 32 +++++++++++++++++++
 2 files changed, 92 insertions(+)

diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index e3e6f9a2..6d7d56f6 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -606,3 +606,63 @@ pub(super) fn parse_update_full<'a>(tok: &'a [Token]) -> LangResult<UpdateStatem
     assert_eq!(i, tok.len(), "full token stream not utilized");
     r
 }
+
+/*
+    Impls for delete
+    ---
+    Smallest statement:
+    delete model:primary_key
+*/
+
+#[derive(Debug, PartialEq)]
+pub(super) struct DeleteStatement<'a> {
+    pub(super) primary_key: &'a Lit,
+    pub(super) entity: Entity,
+}
+
+impl<'a> DeleteStatement<'a> {
+    #[inline(always)]
+    pub(super) fn new(primary_key: &'a Lit, entity: Entity) -> Self {
+        Self {
+            primary_key,
+            entity,
+        }
+    }
+    pub(super) fn parse_delete(tok: &'a [Token], counter: &mut usize) -> LangResult<Self> {
+        let l = tok.len();
+        let mut okay = l > 2;
+        let mut i = 0_usize;
+
+        // parse entity
+        let mut entity = MaybeInit::uninit();
+        okay &= process_entity(tok, &mut entity, &mut i);
+
+        // find primary key
+        okay &= i < l && tok[i] == Token![:];
+        i += okay as usize;
+        okay &= i < l && tok[i].is_lit();
+        let primary_key_idx = i;
+        i += okay as usize;
+
+        *counter += i;
+
+        if okay {
+            unsafe {
+                Ok(Self {
+                    primary_key: extract!(tok[primary_key_idx], Token::Lit(ref l) => l),
+                    entity: entity.assume_init(),
+                })
+            }
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    }
+}
+
+#[cfg(test)]
+pub(super) fn parse_delete_full<'a>(tok: &'a [Token]) -> LangResult<DeleteStatement<'a>> {
+    let mut i = 0_usize;
+    let r = DeleteStatement::parse_delete(tok, &mut i);
+    assert_eq!(i, tok.len());
+    r
+}
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 796534b6..ad7995ff 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -2146,4 +2146,36 @@ mod dml_tests {
             assert_eq!(r, e);
         }
     }
+    mod delete_stmt {
+        use {
+            super::*,
+            crate::engine::ql::{
+                ast::Entity,
+                dml::{self, DeleteStatement},
+            },
+        };
+
+        #[test]
+        fn delete_mini() {
+            let tok = lex(br#"
+                delete user:"sayan"
+            "#)
+            .unwrap();
+            let primary_key = "sayan".into();
+            let e = DeleteStatement::new(&primary_key, Entity::Single("user".into()));
+            let r = dml::parse_delete_full(&tok[1..]).unwrap();
+            assert_eq!(r, e);
+        }
+        #[test]
+        fn delete() {
+            let tok = lex(br#"
+                delete twitter.user:"sayan"
+            "#)
+            .unwrap();
+            let primary_key = "sayan".into();
+            let e = DeleteStatement::new(&primary_key, ("twitter", "user").into());
+            let r = dml::parse_delete_full(&tok[1..]).unwrap();
+            assert_eq!(r, e);
+        }
+    }
 }

From db35f8a31b7fc3cd1ba446d51cb96d40846d7155 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 15 Nov 2022 12:25:33 +0530
Subject: [PATCH 040/310] Fix update to allow single entities to be passed

---
 server/src/engine/ql/dml.rs   | 2 +-
 server/src/engine/ql/tests.rs | 9 ++++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 6d7d56f6..503e2f47 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -555,7 +555,7 @@ impl<'a> UpdateStatement<'a> {
     pub(super) fn parse_update(tok: &'a [Token], counter: &mut usize) -> LangResult<Self> {
         let l = tok.len();
         // TODO(@ohsayan): This would become 8 when we add `SET`. It isn't exactly needed but is for purely aesthetic purposes
-        let mut okay = l > 7;
+        let mut okay = l > 6;
         let mut i = 0_usize;
 
         // parse entity
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index ad7995ff..81427242 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -2102,19 +2102,22 @@ mod dml_tests {
     mod update_statement {
         use {
             super::*,
-            crate::engine::ql::dml::{self, AssignmentExpression, Operator, UpdateStatement},
+            crate::engine::ql::{
+                ast::Entity,
+                dml::{self, AssignmentExpression, Operator, UpdateStatement},
+            },
         };
         #[test]
         fn update_mini() {
             let tok = lex(br#"
-                update jotsy.app:"sayan" notes += "this is my new note"
+                update app:"sayan" notes += "this is my new note"
             "#)
             .unwrap();
             let note = "this is my new note".to_string().into();
             let r = dml::parse_update_full(&tok[1..]).unwrap();
             let e = UpdateStatement {
                 primary_key: &("sayan".to_owned().into()),
-                entity: ("jotsy", "app").into(),
+                entity: Entity::Single("app".into()),
                 expressions: vec![AssignmentExpression {
                     lhs: "notes".into(),
                     rhs: &note,

From 05b8fe81c0db734bfa4782379cd4a7272e3df1ac Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 16 Nov 2022 19:25:42 +0530
Subject: [PATCH 041/310] Add basic benches for `ql`

The benches for the protocol were fixed as well.
---
 server/src/engine/memory/mod.rs   |   2 +-
 server/src/engine/ql/ast.rs       |  34 +++++----
 server/src/engine/ql/benches.rs   | 116 ++++++++++++++++++++++++++++++
 server/src/engine/ql/mod.rs       |  19 ++---
 server/src/engine/ql/schema.rs    |  24 ++++++-
 server/src/protocol/v1/benches.rs |   5 +-
 server/src/protocol/v1/mod.rs     |   1 +
 server/src/protocol/v2/benches.rs |   4 +-
 server/src/protocol/v2/mod.rs     |   1 +
 9 files changed, 178 insertions(+), 28 deletions(-)
 create mode 100644 server/src/engine/ql/benches.rs

diff --git a/server/src/engine/memory/mod.rs b/server/src/engine/memory/mod.rs
index ab3a74e2..4c3147ce 100644
--- a/server/src/engine/memory/mod.rs
+++ b/server/src/engine/memory/mod.rs
@@ -29,7 +29,7 @@
 /// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
 /// be of one type.
 #[derive(Debug, PartialEq)]
-#[cfg_attr(debug_assertions, derive(Clone))]
+#[cfg_attr(test, derive(Clone))]
 pub enum DataType {
     /// An UTF-8 string
     String(String),
diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 635e827a..0535ed52 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -78,35 +78,41 @@ impl Entity {
     pub(super) fn tokens_with_full(tok: &[Token]) -> bool {
         tok.len() > 2 && tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident()
     }
-    pub(super) fn parse(cm: &mut Compiler) -> LangResult<Self> {
-        let sl = cm.remslice();
-        let is_partial = Self::tokens_with_partial(sl);
-        let is_current = Self::tokens_with_single(sl);
-        let is_full = Self::tokens_with_full(sl);
-        let c;
+    #[inline(always)]
+    pub(super) fn parse_from_tokens(tok: &[Token], c: &mut usize) -> LangResult<Self> {
+        let is_partial = Self::tokens_with_partial(tok);
+        let is_current = Self::tokens_with_single(tok);
+        let is_full = Self::tokens_with_full(tok);
         let r = match () {
             _ if is_full => unsafe {
-                c = 3;
-                Self::full_entity_from_slice(sl)
+                *c = 3;
+                Self::full_entity_from_slice(tok)
             },
             _ if is_current => unsafe {
-                c = 1;
-                Self::single_entity_from_slice(sl)
+                *c = 1;
+                Self::single_entity_from_slice(tok)
             },
             _ if is_partial => unsafe {
-                c = 2;
-                Self::partial_entity_from_slice(sl)
+                *c = 2;
+                Self::partial_entity_from_slice(tok)
             },
             _ => return Err(LangError::UnexpectedToken),
         };
+        Ok(r)
+    }
+    #[inline(always)]
+    pub(super) fn parse(cm: &mut Compiler) -> LangResult<Self> {
+        let sl = cm.remslice();
+        let mut c = 0;
+        let r = Self::parse_from_tokens(sl, &mut c);
         unsafe {
             cm.incr_cursor_by(c);
         }
-        Ok(r)
+        r
     }
 }
 
-#[cfg_attr(debug_assertions, derive(Debug, PartialEq))]
+#[cfg_attr(test, derive(Debug, PartialEq))]
 pub enum Statement {
     CreateModel(schema::Model),
     CreateSpace(schema::Space),
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
new file mode 100644
index 00000000..6d2a7cbf
--- /dev/null
+++ b/server/src/engine/ql/benches.rs
@@ -0,0 +1,116 @@
+/*
+ * Created on Wed Nov 16 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/*
+    All benches should be aggregate costs of full execution. This means that when for example
+    you're writing a benchmark for something like parsing a `select` statement, you should calculate
+    the total time of execution including lexing, parsing and allocating. Hopefully in the future we can
+    implement a testing framework that enables us to find the total tiered cost of execution for each stage
+    and hence enable us to iterate on the weakness and fix it. Maybe even visualize it? That'd be amazing
+    and maybe would be something I'll work on around 0.9.
+
+    -- Sayan (@ohsayan)
+*/
+
+extern crate test;
+
+use {crate::engine::ql::tests::lex, test::Bencher};
+
+mod lexer {
+    use {
+        super::*,
+        crate::engine::ql::{
+            lexer::{Lit, Token},
+            RawSlice,
+        },
+    };
+    #[bench]
+    fn lex_number(b: &mut Bencher) {
+        let src = b"1234567890";
+        let expected = vec![Token::Lit(1234567890.into())];
+        b.iter(|| assert_eq!(lex(src).unwrap(), expected));
+    }
+    #[bench]
+    fn lex_bool(b: &mut Bencher) {
+        let s = b"true";
+        let e = vec![Token::Lit(true.into())];
+        b.iter(|| assert_eq!(lex(s).unwrap(), e));
+    }
+    #[bench]
+    fn lex_string_noescapes(b: &mut Bencher) {
+        let s = br#"'hello, world!'"#;
+        let e = vec![Token::Lit("hello, world!".into())];
+        b.iter(|| assert_eq!(lex(s).unwrap(), e));
+    }
+    #[bench]
+    fn lex_string_with_escapes(b: &mut Bencher) {
+        let s = br#"'hello, world! this is within a \'quote\''"#;
+        let e = vec![Token::Lit("hello, world! this is within a 'quote'".into())];
+        b.iter(|| assert_eq!(lex(s).unwrap(), e));
+    }
+    #[bench]
+    fn lex_raw_literal(b: &mut Bencher) {
+        let src = b"\r44\ne69b10ffcc250ae5091dec6f299072e23b0b41d6a739";
+        let expected = vec![Token::Lit(Lit::UnsafeLit(RawSlice::from(
+            "e69b10ffcc250ae5091dec6f299072e23b0b41d6a739",
+        )))];
+        b.iter(|| assert_eq!(lex(src).unwrap(), expected));
+    }
+}
+
+mod ast {
+    use {super::*, crate::engine::ql::ast::Entity};
+    #[bench]
+    fn parse_entity_single(b: &mut Bencher) {
+        let e = Entity::Single("tweeter".into());
+        b.iter(|| {
+            let src = lex(b"tweeter").unwrap();
+            let mut i = 0;
+            assert_eq!(Entity::parse_from_tokens(&src, &mut i).unwrap(), e);
+            assert_eq!(i, src.len());
+        });
+    }
+    #[bench]
+    fn parse_entity_double(b: &mut Bencher) {
+        let e = ("tweeter", "user").into();
+        b.iter(|| {
+            let src = lex(b"tweeter.user").unwrap();
+            let mut i = 0;
+            assert_eq!(Entity::parse_from_tokens(&src, &mut i).unwrap(), e);
+            assert_eq!(i, src.len());
+        });
+    }
+    #[bench]
+    fn parse_entity_partial(b: &mut Bencher) {
+        let e = Entity::Partial("user".into());
+        b.iter(|| {
+            let src = lex(b":user").unwrap();
+            let mut i = 0;
+            assert_eq!(Entity::parse_from_tokens(&src, &mut i).unwrap(), e);
+            assert_eq!(i, src.len());
+        });
+    }
+}
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 69954b98..168907a8 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -27,13 +27,16 @@
 #[macro_use]
 mod macros;
 pub(super) mod ast;
+#[cfg(feature = "nightly")]
+#[cfg(test)]
+mod benches;
 pub(super) mod dml;
 pub(super) mod lexer;
 pub(super) mod schema;
 #[cfg(test)]
 mod tests;
 
-#[cfg(debug_assertions)]
+#[cfg(test)]
 use core::{fmt, ops::Deref};
 use core::{mem, slice, str};
 
@@ -67,11 +70,11 @@ pub enum LangError {
 /// Notes:
 /// - [`Clone`] is implemented for [`RawSlice`] because it is a simple bitwise copy of the fat ptr
 /// - [`fmt::Debug`] is implemented in different ways
-///     - With debug assertions enabled, it will output a slice
+///     - For test builds like test and bench, it will output a slice
 ///     - In release mode, it will output the fat ptr meta
 /// - [`PartialEq`] is implemented in debug mode with slice comparison, but is **NOT implemented for release mode in the
-///   way you'd expect it to**. In release mode, a comparison will simply panic.
-#[cfg_attr(not(debug_assertions), derive(Debug))]
+///   way you'd expect it to**. In release mode (non-test), a comparison will simply panic.
+#[cfg_attr(not(test), derive(Debug))]
 #[derive(Clone)]
 pub struct RawSlice {
     ptr: *const u8,
@@ -95,7 +98,7 @@ impl RawSlice {
     }
 }
 
-#[cfg(debug_assertions)]
+#[cfg(test)]
 impl fmt::Debug for RawSlice {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str(unsafe {
@@ -105,7 +108,7 @@ impl fmt::Debug for RawSlice {
     }
 }
 
-#[cfg(debug_assertions)]
+#[cfg(test)]
 impl PartialEq for RawSlice {
     fn eq(&self, other: &Self) -> bool {
         unsafe {
@@ -115,14 +118,14 @@ impl PartialEq for RawSlice {
     }
 }
 
-#[cfg(not(debug_assertions))]
+#[cfg(not(test))]
 impl PartialEq for RawSlice {
     fn eq(&self, _other: &Self) -> bool {
         panic!("Called partialeq on rawslice in release mode");
     }
 }
 
-#[cfg(debug_assertions)]
+#[cfg(test)]
 impl<U> PartialEq<U> for RawSlice
 where
     U: Deref<Target = [u8]>,
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index e5e741fb..b064ac2c 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -157,6 +157,17 @@ pub struct Field {
     pub(super) props: HashSet<StaticStr>,
 }
 
+impl Field {
+    #[inline(always)]
+    pub fn new(field_name: RawSlice, layers: Vec<Layer>, props: HashSet<StaticStr>) -> Self {
+        Self {
+            field_name,
+            layers,
+            props,
+        }
+    }
+}
+
 #[derive(Debug, PartialEq)]
 /// A model definition
 pub struct Model {
@@ -168,6 +179,17 @@ pub struct Model {
     pub(super) props: Dict,
 }
 
+impl Model {
+    #[inline(always)]
+    pub fn new(model_name: RawSlice, fields: Vec<Field>, props: Dict) -> Self {
+        Self {
+            model_name,
+            fields,
+            props,
+        }
+    }
+}
+
 #[derive(Debug, PartialEq)]
 /// A space
 pub struct Space {
@@ -974,7 +996,7 @@ pub(super) fn parse_field_syntax<const ALLOW_RESET: bool>(
 }
 
 #[derive(Debug)]
-#[cfg_attr(debug_assertions, derive(PartialEq))]
+#[cfg_attr(test, derive(PartialEq))]
 /// The alter operation kind
 pub enum AlterKind {
     Add(Box<[ExpandedField]>),
diff --git a/server/src/protocol/v1/benches.rs b/server/src/protocol/v1/benches.rs
index ed6d6d56..0d63ac23 100644
--- a/server/src/protocol/v1/benches.rs
+++ b/server/src/protocol/v1/benches.rs
@@ -25,6 +25,7 @@
 */
 
 extern crate test;
+
 use {
     super::{super::Query, Parser},
     test::Bencher,
@@ -45,7 +46,7 @@ fn simple_query(b: &mut Bencher) {
         let ret: Vec<String> = query
             .as_slice()
             .iter()
-            .map(|s| String::from_utf8_lossy(s.as_slice()).to_string())
+            .map(|s| String::from_utf8_lossy(unsafe { s.as_slice() }).to_string())
             .collect();
         assert_eq!(ret, expected)
     });
@@ -73,7 +74,7 @@ fn pipelined_query(b: &mut Bencher) {
                 query
                     .as_slice()
                     .iter()
-                    .map(|v| String::from_utf8_lossy(v.as_slice()).to_string())
+                    .map(|v| String::from_utf8_lossy(unsafe { v.as_slice() }).to_string())
                     .collect()
             })
             .collect();
diff --git a/server/src/protocol/v1/mod.rs b/server/src/protocol/v1/mod.rs
index 8717601e..1cb52cc5 100644
--- a/server/src/protocol/v1/mod.rs
+++ b/server/src/protocol/v1/mod.rs
@@ -38,6 +38,7 @@ use {
 mod interface_impls;
 // test and bench modules
 #[cfg(feature = "nightly")]
+#[cfg(test)]
 mod benches;
 #[cfg(test)]
 mod tests;
diff --git a/server/src/protocol/v2/benches.rs b/server/src/protocol/v2/benches.rs
index ff41a812..4b1c8f64 100644
--- a/server/src/protocol/v2/benches.rs
+++ b/server/src/protocol/v2/benches.rs
@@ -45,7 +45,7 @@ fn simple_query(b: &mut Bencher) {
         let ret: Vec<String> = query
             .as_slice()
             .iter()
-            .map(|s| String::from_utf8_lossy(s.as_slice()).to_string())
+            .map(|s| String::from_utf8_lossy(unsafe { s.as_slice() }).to_string())
             .collect();
         assert_eq!(ret, expected)
     });
@@ -73,7 +73,7 @@ fn pipelined_query(b: &mut Bencher) {
                 query
                     .as_slice()
                     .iter()
-                    .map(|v| String::from_utf8_lossy(v.as_slice()).to_string())
+                    .map(|v| String::from_utf8_lossy(unsafe { v.as_slice() }).to_string())
                     .collect()
             })
             .collect();
diff --git a/server/src/protocol/v2/mod.rs b/server/src/protocol/v2/mod.rs
index 651eaaac..6720e8b5 100644
--- a/server/src/protocol/v2/mod.rs
+++ b/server/src/protocol/v2/mod.rs
@@ -35,6 +35,7 @@ use {
 };
 
 #[cfg(feature = "nightly")]
+#[cfg(test)]
 mod benches;
 #[cfg(test)]
 mod tests;

From 89c603536730b0c9ad96a6b0f0c5b8d3b4b9b12d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 16 Nov 2022 22:32:55 +0530
Subject: [PATCH 042/310] Fix some DDL query implementations

Summary of changes:
- The `drop` queries now use the `DropItem` for drop definitions
- `create model` is now entirely handled by the function in `schema`
- `create space` is now entirely handled by the function in `schema`
- Tests were added for drop (they were never present before)
---
 server/src/engine/ql/ast.rs    | 43 +++++-------------
 server/src/engine/ql/ddl.rs    | 81 ++++++++++++++++++++++++++++++++++
 server/src/engine/ql/mod.rs    |  1 +
 server/src/engine/ql/schema.rs | 58 +++++++++++++-----------
 server/src/engine/ql/tests.rs  | 74 +++++++++++++++++++++----------
 5 files changed, 174 insertions(+), 83 deletions(-)
 create mode 100644 server/src/engine/ql/ddl.rs

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 0535ed52..87a98330 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -26,6 +26,7 @@
 
 use {
     super::{
+        ddl,
         lexer::{Lexer, Token},
         schema, LangError, LangResult, RawSlice,
     },
@@ -119,8 +120,8 @@ pub enum Statement {
     Use(Entity),
     AlterSpace(schema::AlterSpace),
     AlterModel(RawSlice, schema::AlterKind),
-    DropModel(RawSlice, bool),
-    DropSpace(RawSlice, bool),
+    DropModel(ddl::DropItem),
+    DropSpace(ddl::DropItem),
     InspectSpace(RawSlice),
     InspectModel(Entity),
     InspectSpaces,
@@ -176,29 +177,13 @@ impl<'a> Compiler<'a> {
     }
     #[inline(always)]
     fn drop0(&mut self) -> Result<Statement, LangError> {
-        if self.remaining() < 2 {
-            return Err(LangError::ExpectedStatement);
+        unsafe {
+            self.incr_cursor(); // forward stage token
         }
-        let rs = self.remslice();
-        let ident = match rs[1] {
-            Token::Ident(ref id) => id,
-            _ => return Err(LangError::ExpectedStatement),
-        };
-        let should_force = self.remaining() > 2 && rs[2].as_ident_eq_ignore_case(b"force");
-        let r = match rs[0] {
-            Token![model] => {
-                // dropping a model
-                Ok(Statement::DropModel(ident.clone(), should_force))
-            }
-            Token![space] => {
-                // dropping a space
-                Ok(Statement::DropSpace(ident.clone(), should_force))
-            }
-            _ => Err(LangError::UnexpectedToken),
-        };
+        let mut i = 0;
+        let r = ddl::parse_drop(self.remslice(), &mut i);
         unsafe {
-            self.incr_cursor_by(2);
-            self.incr_cursor_if(should_force);
+            self.incr_cursor_by(i);
         }
         r
     }
@@ -266,11 +251,7 @@ impl<'a> Compiler<'a> {
     }
     #[inline(always)]
     fn c_model0(&mut self) -> Result<Statement, LangError> {
-        let model_name = match self.nxtok_opt() {
-            Some(Token::Ident(model)) => model.clone(),
-            _ => return Err(LangError::UnexpectedToken),
-        };
-        let (model, i) = schema::parse_schema_from_tokens(self.remslice(), model_name)?;
+        let (model, i) = schema::parse_schema_from_tokens(self.remslice())?;
         unsafe {
             self.incr_cursor_by(i);
         }
@@ -278,11 +259,7 @@ impl<'a> Compiler<'a> {
     }
     #[inline(always)]
     fn c_space0(&mut self) -> Result<Statement, LangError> {
-        let space_name = match self.nxtok_opt() {
-            Some(Token::Ident(space_name)) => space_name.clone(),
-            _ => return Err(LangError::UnexpectedToken),
-        };
-        let (space, i) = schema::parse_space_from_tokens(self.remslice(), space_name)?;
+        let (space, i) = schema::parse_space_from_tokens(self.remslice())?;
         unsafe {
             self.incr_cursor_by(i);
         }
diff --git a/server/src/engine/ql/ddl.rs b/server/src/engine/ql/ddl.rs
new file mode 100644
index 00000000..a929425a
--- /dev/null
+++ b/server/src/engine/ql/ddl.rs
@@ -0,0 +1,81 @@
+/*
+ * Created on Wed Nov 16 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::{ast::Statement, lexer::Token, LangError, LangResult, RawSlice};
+
+#[derive(Debug, PartialEq)]
+pub struct DropItem(pub RawSlice, pub bool);
+
+impl DropItem {
+    #[inline(always)]
+    pub(super) const fn new(slice: RawSlice, force: bool) -> Self {
+        Self(slice, force)
+    }
+}
+
+// drop (<space> | <model>) <ident> [<force>]
+pub(super) fn parse_drop(tok: &[Token], counter: &mut usize) -> LangResult<Statement> {
+    let l = tok.len();
+    // drop space/model
+    let mut i = 0;
+    let drop_space = i < l && tok[i] == Token![space];
+    let drop_model = i < l && tok[i] == Token![model];
+    let mut okay = drop_space | drop_model;
+    i += okay as usize;
+    // check if we have the target entity name
+    okay &= i < l && tok[i].is_ident();
+    i += okay as usize;
+    // next token is either `force` or end of stream
+    let force_drop = i < l && tok[i] == Token::Ident("force".into());
+    okay &= force_drop | (i == l);
+    i += force_drop as usize;
+
+    if !okay {
+        return Err(LangError::UnexpectedToken);
+    }
+
+    let drop_item = DropItem(
+        unsafe { extract!(tok[1], Token::Ident(ref id) => id.clone()) },
+        force_drop,
+    );
+
+    *counter += i;
+
+    let stmt = if drop_space {
+        Statement::DropSpace(drop_item)
+    } else {
+        Statement::DropModel(drop_item)
+    };
+    Ok(stmt)
+}
+
+#[cfg(test)]
+pub(super) fn parse_drop_full(tok: &[Token]) -> LangResult<Statement> {
+    let mut i = 0;
+    let r = self::parse_drop(tok, &mut i);
+    assert_eq!(i, tok.len(), "full token stream not utilized");
+    r
+}
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 168907a8..7deca32f 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -30,6 +30,7 @@ pub(super) mod ast;
 #[cfg(feature = "nightly")]
 #[cfg(test)]
 mod benches;
+pub(super) mod ddl;
 pub(super) mod dml;
 pub(super) mod lexer;
 pub(super) mod schema;
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index b064ac2c..2b5c5f0d 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -733,16 +733,15 @@ states! {
 
 #[inline(always)]
 /// Parse a fresh schema with declaration-syntax fields
-pub(super) fn parse_schema_from_tokens(
-    tok: &[Token],
-    model_name: RawSlice,
-) -> LangResult<(Model, usize)> {
+pub(super) fn parse_schema_from_tokens(tok: &[Token]) -> LangResult<(Model, usize)> {
     // parse fields
     let l = tok.len();
     let mut i = 0;
-    let mut state = SchemaParseState::OPEN_PAREN;
-    let mut okay = true;
+    // check if we have our model name
+    let mut okay = i < l && tok[i].is_ident();
+    i += okay as usize;
     let mut fields = Vec::with_capacity(2);
+    let mut state = SchemaParseState::OPEN_PAREN;
 
     while i < l && okay {
         match (&tok[i], state) {
@@ -787,6 +786,11 @@ pub(super) fn parse_schema_from_tokens(
         return Err(LangError::UnexpectedToken);
     }
 
+    let model_name = unsafe {
+        // UNSAFE(@ohsayan): Now that we're sure that we have the model name ident, get it
+        extract!(tok[0], Token::Ident(ref model_name) => model_name.clone())
+    };
+
     if l > i && tok[i] == (Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With))) {
         // we have some more input, and it should be a dict of properties
         i += 1; // +WITH
@@ -824,31 +828,33 @@ pub(super) fn parse_schema_from_tokens(
 
 #[inline(always)]
 /// Parse space data from the given tokens
-pub(super) fn parse_space_from_tokens(tok: &[Token], s: RawSlice) -> LangResult<(Space, usize)> {
-    // let's see if the cursor is at `with`. ignore other tokens because that's fine
-    if !tok.is_empty() && tok[0] == (Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With))) {
-        // we have a dict
-        let mut d = Dict::new();
+pub(super) fn parse_space_from_tokens(tok: &[Token]) -> LangResult<(Space, usize)> {
+    let l = tok.len();
+    let mut okay = !tok.is_empty() && tok[0].is_ident();
+    let mut i = 0;
+    i += okay as usize;
+    // either we have `with` or nothing. don't be stupid
+    let has_more_properties = i < l && tok[i] == Token![with];
+    okay &= has_more_properties | (i == l);
+    // properties
+    let mut d = Dict::new();
+
+    if has_more_properties && okay {
         let ret = self::rfold_dict(DictFoldState::OB, &tok[1..], &mut d);
-        if ret & HIBIT == HIBIT {
-            Ok((
-                Space {
-                    space_name: s,
-                    props: d,
-                },
-                (ret & !HIBIT) as _,
-            ))
-        } else {
-            Err(LangError::UnexpectedToken)
-        }
-    } else {
+        i += (ret & !HIBIT) as usize;
+        okay &= ret & HIBIT == HIBIT;
+    }
+
+    if okay {
         Ok((
             Space {
-                space_name: s,
-                props: dict! {},
+                space_name: unsafe { extract!(tok[0], Token::Ident(ref id) => id.clone()) },
+                props: d,
             },
-            0,
+            i,
         ))
+    } else {
+        Err(LangError::UnexpectedToken)
     }
 }
 
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 81427242..84a33b70 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -218,6 +218,48 @@ mod entity {
     }
 }
 
+mod ddl_other_query_tests {
+    use {
+        super::*,
+        crate::engine::ql::{
+            ast::Statement,
+            ddl::{self, DropItem},
+        },
+    };
+    #[test]
+    fn drop_space() {
+        let src = lex(br"drop space myspace").unwrap();
+        assert_eq!(
+            ddl::parse_drop_full(&src[1..]).unwrap(),
+            Statement::DropSpace(DropItem::new("myspace".into(), false))
+        );
+    }
+    #[test]
+    fn drop_space_force() {
+        let src = lex(br"drop space myspace force").unwrap();
+        assert_eq!(
+            ddl::parse_drop_full(&src[1..]).unwrap(),
+            Statement::DropSpace(DropItem::new("myspace".into(), true))
+        );
+    }
+    #[test]
+    fn drop_model() {
+        let src = lex(br"drop model mymodel").unwrap();
+        assert_eq!(
+            ddl::parse_drop_full(&src[1..]).unwrap(),
+            Statement::DropModel(DropItem::new("mymodel".into(), false))
+        );
+    }
+    #[test]
+    fn drop_model_force() {
+        let src = lex(br"drop model mymodel force").unwrap();
+        assert_eq!(
+            ddl::parse_drop_full(&src[1..]).unwrap(),
+            Statement::DropModel(DropItem::new("mymodel".into(), true))
+        );
+    }
+}
+
 mod schema_tests {
     use {
         super::{
@@ -896,14 +938,10 @@ mod schema_tests {
                 )
             ")
             .unwrap();
-            let schema_name = match tok[2] {
-                Token::Ident(ref id) => id.clone(),
-                _ => panic!("expected ident"),
-            };
-            let tok = &tok[3..];
+            let tok = &tok[2..];
 
             // parse model
-            let (model, c) = schema::parse_schema_from_tokens(tok, schema_name).unwrap();
+            let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
             assert_eq!(c, tok.len());
             assert_eq!(
                 model,
@@ -935,14 +973,10 @@ mod schema_tests {
                 )
             ")
             .unwrap();
-            let schema_name = match tok[2] {
-                Token::Ident(ref id) => id.clone(),
-                _ => panic!("expected ident"),
-            };
-            let tok = &tok[3..];
+            let tok = &tok[2..];
 
             // parse model
-            let (model, c) = schema::parse_schema_from_tokens(tok, schema_name).unwrap();
+            let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
             assert_eq!(c, tok.len());
             assert_eq!(
                 model,
@@ -984,14 +1018,10 @@ mod schema_tests {
                 )
             ")
             .unwrap();
-            let schema_name = match tok[2] {
-                Token::Ident(ref id) => id.clone(),
-                _ => panic!("expected ident"),
-            };
-            let tok = &tok[3..];
+            let tok = &tok[2..];
 
             // parse model
-            let (model, c) = schema::parse_schema_from_tokens(tok, schema_name).unwrap();
+            let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
             assert_eq!(c, tok.len());
             assert_eq!(
                 model,
@@ -1051,14 +1081,10 @@ mod schema_tests {
                 }
             ")
             .unwrap();
-            let schema_name = match tok[2] {
-                Token::Ident(ref id) => id.clone(),
-                _ => panic!("expected ident"),
-            };
-            let tok = &tok[3..];
+            let tok = &tok[2..];
 
             // parse model
-            let (model, c) = schema::parse_schema_from_tokens(tok, schema_name).unwrap();
+            let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
             assert_eq!(c, tok.len());
             assert_eq!(
                 model,

From 5711fd1089b50766d2adca5cf84a70107df60bd6 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 17 Nov 2022 20:39:26 +0530
Subject: [PATCH 043/310] Improve DDL query parsing

Summary of changes:
- Remaining query parse methods that directly used the `Compiler` were
modified to use token streams instead
- `Entity::parse_from_tokens` to fix assignment to counter instead of
increments
- Allow entity in `drop model`
- `alter model` and `alter space` directly handle space/model names
instead of depending on passing value via args
- Tests added for `drop` and `inspect`
---
 server/src/engine/ql/ast.rs    | 180 ++++++++++++++++++++++++---------
 server/src/engine/ql/ddl.rs    | 126 ++++++++++++++++-------
 server/src/engine/ql/schema.rs |  70 +++++++++----
 server/src/engine/ql/tests.rs  |  94 +++++++++++++++--
 4 files changed, 360 insertions(+), 110 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 87a98330..bff34a06 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -39,9 +39,31 @@ use {
 */
 
 #[derive(Debug, PartialEq)]
+/// An [`Entity`] represents the location for a specific structure, such as a model
 pub enum Entity {
+    /// A partial entity is used when switching to a model wrt the currently set space (commonly used
+    /// when running `use` queries)
+    ///
+    /// syntax:
+    /// ```sql
+    /// :model
+    /// ```
     Partial(RawSlice),
+    /// A single entity is used when switching to a model wrt the currently set space (commonly used
+    /// when running DML queries)
+    ///
+    /// syntax:
+    /// ```sql
+    /// model
+    /// ```
     Single(RawSlice),
+    /// A full entity is a complete definition to a model wrt to the given space (commonly used with
+    /// DML queries)
+    ///
+    /// syntax:
+    /// ```sql
+    /// space.model
+    /// ```
     Full(RawSlice, RawSlice),
 }
 
@@ -53,6 +75,12 @@ impl<T: Into<RawSlice>, U: Into<RawSlice>> From<(T, U)> for Entity {
 
 impl Entity {
     #[inline(always)]
+    /// Parse a full entity from the given slice
+    ///
+    /// ## Safety
+    ///
+    /// Caller guarantees that the token stream matches the exact stream of tokens
+    /// expected for a full entity
     pub(super) unsafe fn full_entity_from_slice(sl: &[Token]) -> Self {
         Entity::Full(
             extract!(&sl[0], Token::Ident(sl) => sl.clone()),
@@ -60,41 +88,60 @@ impl Entity {
         )
     }
     #[inline(always)]
+    /// Parse a single entity from the given slice
+    ///
+    /// ## Safety
+    ///
+    /// Caller guarantees that the token stream matches the exact stream of tokens
+    /// expected for a single entity
     pub(super) unsafe fn single_entity_from_slice(sl: &[Token]) -> Self {
         Entity::Single(extract!(&sl[0], Token::Ident(sl) => sl.clone()))
     }
     #[inline(always)]
+    /// Parse a partial entity from the given slice
+    ///
+    /// ## Safety
+    ///
+    /// Caller guarantees that the token stream matches the exact stream of tokens
+    /// expected for a partial entity
     pub(super) unsafe fn partial_entity_from_slice(sl: &[Token]) -> Self {
         Entity::Partial(extract!(&sl[1], Token::Ident(sl) => sl.clone()))
     }
     #[inline(always)]
+    /// Returns true if the given token stream matches the signature of partial entity syntax
     pub(super) fn tokens_with_partial(tok: &[Token]) -> bool {
         tok.len() > 1 && tok[0] == Token![:] && tok[1].is_ident()
     }
     #[inline(always)]
+    /// Returns true if the given token stream matches the signature of single entity syntax
+    ///
+    /// ⚠ WARNING: This will pass for full and single
     pub(super) fn tokens_with_single(tok: &[Token]) -> bool {
         !tok.is_empty() && tok[0].is_ident()
     }
     #[inline(always)]
+    /// Returns true if the given token stream matches the signature of full entity syntax
     pub(super) fn tokens_with_full(tok: &[Token]) -> bool {
         tok.len() > 2 && tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident()
     }
     #[inline(always)]
+    /// Attempt to parse an entity using the given token stream. It also accepts a counter
+    /// argument to forward the cursor
     pub(super) fn parse_from_tokens(tok: &[Token], c: &mut usize) -> LangResult<Self> {
         let is_partial = Self::tokens_with_partial(tok);
         let is_current = Self::tokens_with_single(tok);
         let is_full = Self::tokens_with_full(tok);
         let r = match () {
             _ if is_full => unsafe {
-                *c = 3;
+                *c += 3;
                 Self::full_entity_from_slice(tok)
             },
             _ if is_current => unsafe {
-                *c = 1;
+                *c += 1;
                 Self::single_entity_from_slice(tok)
             },
             _ if is_partial => unsafe {
-                *c = 2;
+                *c += 2;
                 Self::partial_entity_from_slice(tok)
             },
             _ => return Err(LangError::UnexpectedToken),
@@ -102,6 +149,9 @@ impl Entity {
         Ok(r)
     }
     #[inline(always)]
+    /// Parse an entity using the given [`Compiler`] instance. Internally this just evalutes it
+    /// using a token stream, finally forwarding the [`Compiler`]'s internal cursor depending on the
+    /// number of bytes consumed
     pub(super) fn parse(cm: &mut Compiler) -> LangResult<Self> {
         let sl = cm.remslice();
         let mut c = 0;
@@ -114,19 +164,41 @@ impl Entity {
 }
 
 #[cfg_attr(test, derive(Debug, PartialEq))]
+/// A [`Statement`] is a fully BlueQL statement that can be executed by the query engine
+// TODO(@ohsayan): Determine whether we actually need this
 pub enum Statement {
+    /// DDL query to create a model
     CreateModel(schema::Model),
+    /// DDL query to create a space
     CreateSpace(schema::Space),
+    /// DDL query to switch between spaces and models
     Use(Entity),
+    /// DDL query to alter a space (properties)
     AlterSpace(schema::AlterSpace),
-    AlterModel(RawSlice, schema::AlterKind),
-    DropModel(ddl::DropItem),
-    DropSpace(ddl::DropItem),
+    /// DDL query to alter a model (properties, field types, etc)
+    AlterModel(schema::Alter),
+    /// DDL query to drop a model
+    ///
+    /// Conditions:
+    /// - Model view is empty
+    /// - Model is not in active use
+    DropModel(ddl::DropModel),
+    /// DDL query to drop a space
+    ///
+    /// Conditions:
+    /// - Space doesn't have any other structures
+    /// - Space is not in active use
+    DropSpace(ddl::DropSpace),
+    /// DDL query to inspect a space (returns a list of models in the space)
     InspectSpace(RawSlice),
+    /// DDL query to inspect a model (returns the model definition)
     InspectModel(Entity),
+    /// DDL query to inspect all spaces (returns a list of spaces in the database)
     InspectSpaces,
 }
 
+/// A [`Compiler`] for BlueQL queries
+// TODO(@ohsayan): Decide whether we need this
 pub struct Compiler<'a> {
     c: *const Token,
     e: *const Token,
@@ -134,11 +206,13 @@ pub struct Compiler<'a> {
 }
 
 impl<'a> Compiler<'a> {
+    /// Compile a BlueQL query
     pub fn compile(src: &'a [u8]) -> LangResult<Life<'a, Statement>> {
         let token_stream = Lexer::lex(src)?;
         Self::new(&token_stream).compile_link_lt()
     }
     #[inline(always)]
+    /// Create a new [`Compiler`] instance
     pub(super) const fn new(token_stream: &[Token]) -> Self {
         unsafe {
             Self {
@@ -149,6 +223,8 @@ impl<'a> Compiler<'a> {
         }
     }
     #[inline(always)]
+    /// Utility method to link a lifetime to the statement since the statement makes use of some
+    /// unsafe lifetime-free code that would otherwise cause the program to crash and burn
     fn compile_link_lt(mut self) -> LangResult<Life<'a, Statement>> {
         match self.stage0() {
             Ok(t) if self.exhausted() => Ok(Life::new(t)),
@@ -157,8 +233,9 @@ impl<'a> Compiler<'a> {
         }
     }
     #[inline(always)]
+    /// Stage 0: what statement
     fn stage0(&mut self) -> Result<Statement, LangError> {
-        match self.nxtok_opt() {
+        match self.nxtok_opt_forward() {
             Some(Token![create]) => self.create0(),
             Some(Token![drop]) => self.drop0(),
             Some(Token![alter]) => self.alter0(),
@@ -168,18 +245,17 @@ impl<'a> Compiler<'a> {
         }
     }
     #[inline(always)]
+    /// Create 0: Create what (model/space)
     fn create0(&mut self) -> Result<Statement, LangError> {
-        match self.nxtok_opt() {
+        match self.nxtok_opt_forward() {
             Some(Token![model]) => self.c_model0(),
             Some(Token![space]) => self.c_space0(),
             _ => Err(LangError::UnexpectedEndofStatement),
         }
     }
     #[inline(always)]
+    /// Drop 0: Drop what (model/space)
     fn drop0(&mut self) -> Result<Statement, LangError> {
-        unsafe {
-            self.incr_cursor(); // forward stage token
-        }
         let mut i = 0;
         let r = ddl::parse_drop(self.remslice(), &mut i);
         unsafe {
@@ -188,8 +264,9 @@ impl<'a> Compiler<'a> {
         r
     }
     #[inline(always)]
+    /// Alter 0: Alter what (model/space)
     fn alter0(&mut self) -> Result<Statement, LangError> {
-        match self.nxtok_opt() {
+        match self.nxtok_opt_forward() {
             Some(Token![model]) => self.alter_model(),
             Some(Token![space]) => self.alter_space(),
             Some(_) => Err(LangError::ExpectedStatement),
@@ -197,59 +274,42 @@ impl<'a> Compiler<'a> {
         }
     }
     #[inline(always)]
+    /// Alter model
     fn alter_model(&mut self) -> Result<Statement, LangError> {
-        let model_name = match self.nxtok_opt() {
-            Some(Token::Ident(md)) => md.clone(),
-            _ => return Err(LangError::ExpectedStatement),
-        };
         let mut c = 0;
-        schema::parse_alter_kind_from_tokens(self.remslice(), &mut c)
-            .map(|ak| Statement::AlterModel(model_name.clone(), ak))
+        let r = schema::parse_alter_kind_from_tokens(self.remslice(), &mut c);
+        unsafe {
+            self.incr_cursor_by(c);
+        }
+        r.map(Statement::AlterModel)
     }
     #[inline(always)]
+    /// Alter space
     fn alter_space(&mut self) -> Result<Statement, LangError> {
-        let space_name = match self.nxtok_opt() {
-            Some(Token::Ident(id)) => id.clone(),
-            Some(_) => return Err(LangError::UnexpectedToken),
-            None => return Err(LangError::UnexpectedEndofStatement),
-        };
-        let (alter, i) = schema::parse_alter_space_from_tokens(self.remslice(), space_name)?;
+        let (alter, i) = schema::parse_alter_space_from_tokens(self.remslice())?;
         unsafe {
             self.incr_cursor_by(i);
         }
         Ok(Statement::AlterSpace(alter))
     }
     #[inline(always)]
+    /// Inspect 0: Inpsect what (model/space/spaces)
     fn inspect0(&mut self) -> Result<Statement, LangError> {
-        if self.remaining() == 0 {
-            return Err(LangError::UnexpectedEndofStatement);
-        }
-        match self.nxtok_opt() {
-            Some(Token![space]) => {
-                let space_name = match self.nxtok_opt() {
-                    Some(Token::Ident(id)) => id.clone(),
-                    _ => return Err(LangError::UnexpectedToken),
-                };
-                Ok(Statement::InspectSpace(space_name))
-            }
-            Some(Token![model]) => {
-                let entity = Entity::parse(self)?;
-                Ok(Statement::InspectModel(entity))
-            }
-            Some(Token::Ident(id))
-                if unsafe { id.as_slice() }.eq_ignore_ascii_case(b"keyspaces") =>
-            {
-                Ok(Statement::InspectSpaces)
-            }
-            _ => Err(LangError::ExpectedStatement),
+        let mut i = 0;
+        let r = ddl::parse_inspect(self.remslice(), &mut i);
+        unsafe {
+            self.incr_cursor_by(i);
         }
+        r
     }
     #[inline(always)]
+    /// Parse an `use` query
     fn use0(&mut self) -> Result<Statement, LangError> {
         let entity = Entity::parse(self)?;
         Ok(Statement::Use(entity))
     }
     #[inline(always)]
+    /// Create model
     fn c_model0(&mut self) -> Result<Statement, LangError> {
         let (model, i) = schema::parse_schema_from_tokens(self.remslice())?;
         unsafe {
@@ -258,6 +318,7 @@ impl<'a> Compiler<'a> {
         Ok(Statement::CreateModel(model))
     }
     #[inline(always)]
+    /// Create space
     fn c_space0(&mut self) -> Result<Statement, LangError> {
         let (space, i) = schema::parse_space_from_tokens(self.remslice())?;
         unsafe {
@@ -269,7 +330,8 @@ impl<'a> Compiler<'a> {
 
 impl<'a> Compiler<'a> {
     #[inline(always)]
-    pub(super) fn nxtok_opt<'b>(&mut self) -> Option<&'b Token>
+    /// Attempt to read the next token and forward the interal cursor if there is a cursor ahead
+    pub(super) fn nxtok_opt_forward<'b>(&mut self) -> Option<&'b Token>
     where
         'a: 'b,
     {
@@ -284,28 +346,39 @@ impl<'a> Compiler<'a> {
         }
     }
     #[inline(always)]
+    /// Returns the cursor
     pub(super) const fn cursor(&self) -> *const Token {
         self.c
     }
     #[inline(always)]
+    /// Returns the remaining buffer as a slice
     pub(super) fn remslice(&'a self) -> &'a [Token] {
         unsafe { slice::from_raw_parts(self.c, self.remaining()) }
     }
     #[inline(always)]
+    /// Check if the buffer is not exhausted
     pub(super) fn not_exhausted(&self) -> bool {
         self.c != self.e
     }
     #[inline(always)]
+    /// Check if the buffer is exhausted
     pub(super) fn exhausted(&self) -> bool {
         self.c == self.e
     }
     #[inline(always)]
+    /// Check the remaining bytes in the buffer
     pub(super) fn remaining(&self) -> usize {
         unsafe { self.e.offset_from(self.c) as usize }
     }
+    /// Deref the cursor
+    ///
+    /// ## Safety
+    ///
+    /// Have to ensure it isn't pointing to garbage i.e beyond EOA
     pub(super) unsafe fn deref_cursor(&self) -> &Token {
         &*self.c
     }
+    /// Increment the cursor if the next token matches the given token
     pub(super) fn peek_eq_and_forward(&mut self, t: Token) -> bool {
         let did_fw = self.not_exhausted() && unsafe { self.deref_cursor() == &t };
         unsafe {
@@ -314,13 +387,28 @@ impl<'a> Compiler<'a> {
         did_fw
     }
     #[inline(always)]
+    /// Increment the cursor
+    ///
+    /// ## Safety
+    ///
+    /// Should be >= EOA
     pub(super) unsafe fn incr_cursor(&mut self) {
         self.incr_cursor_by(1)
     }
+    /// Increment the cursor if the given boolean expr is satisified
+    ///
+    /// ## Safety
+    ///
+    /// Should be >= EOA (if true)
     pub(super) unsafe fn incr_cursor_if(&mut self, did_fw: bool) {
         self.incr_cursor_by(did_fw as _)
     }
     #[inline(always)]
+    /// Increment the cursor by the given count
+    ///
+    /// ## Safety
+    ///
+    /// >= EOA (if nonzero)
     pub(super) unsafe fn incr_cursor_by(&mut self, by: usize) {
         debug_assert!(self.remaining() >= by);
         self.c = self.c.add(by);
diff --git a/server/src/engine/ql/ddl.rs b/server/src/engine/ql/ddl.rs
index a929425a..8bc57d65 100644
--- a/server/src/engine/ql/ddl.rs
+++ b/server/src/engine/ql/ddl.rs
@@ -24,52 +24,72 @@
  *
 */
 
-use super::{ast::Statement, lexer::Token, LangError, LangResult, RawSlice};
+use super::{
+    ast::{Entity, Statement},
+    lexer::Token,
+    LangError, LangResult, RawSlice,
+};
 
 #[derive(Debug, PartialEq)]
-pub struct DropItem(pub RawSlice, pub bool);
+/// A generic representation of `drop` query
+pub struct DropSpace {
+    pub(super) space: RawSlice,
+    pub(super) force: bool,
+}
 
-impl DropItem {
+impl DropSpace {
     #[inline(always)]
-    pub(super) const fn new(slice: RawSlice, force: bool) -> Self {
-        Self(slice, force)
+    /// Instantiate
+    pub(super) const fn new(space: RawSlice, force: bool) -> Self {
+        Self { space, force }
     }
 }
 
-// drop (<space> | <model>) <ident> [<force>]
-pub(super) fn parse_drop(tok: &[Token], counter: &mut usize) -> LangResult<Statement> {
-    let l = tok.len();
-    // drop space/model
-    let mut i = 0;
-    let drop_space = i < l && tok[i] == Token![space];
-    let drop_model = i < l && tok[i] == Token![model];
-    let mut okay = drop_space | drop_model;
-    i += okay as usize;
-    // check if we have the target entity name
-    okay &= i < l && tok[i].is_ident();
-    i += okay as usize;
-    // next token is either `force` or end of stream
-    let force_drop = i < l && tok[i] == Token::Ident("force".into());
-    okay &= force_drop | (i == l);
-    i += force_drop as usize;
+#[derive(Debug, PartialEq)]
+pub struct DropModel {
+    pub(super) entity: Entity,
+    pub(super) force: bool,
+}
 
-    if !okay {
-        return Err(LangError::UnexpectedToken);
+impl DropModel {
+    #[inline(always)]
+    pub fn new(entity: Entity, force: bool) -> Self {
+        Self { entity, force }
     }
+}
 
-    let drop_item = DropItem(
-        unsafe { extract!(tok[1], Token::Ident(ref id) => id.clone()) },
-        force_drop,
-    );
-
-    *counter += i;
-
-    let stmt = if drop_space {
-        Statement::DropSpace(drop_item)
-    } else {
-        Statement::DropModel(drop_item)
-    };
-    Ok(stmt)
+// drop (<space> | <model>) <ident> [<force>]
+pub(super) fn parse_drop(tok: &[Token], counter: &mut usize) -> LangResult<Statement> {
+    match tok.get(0) {
+        Some(Token![model]) => {
+            // we have a model. now parse entity and see if we should force deletion
+            let mut i = 1;
+            let e = Entity::parse_from_tokens(&tok[1..], &mut i)?;
+            let force = i < tok.len() && tok[i] == Token::Ident("force".into());
+            i += force as usize;
+            *counter += i;
+            // if we've exhausted the stream, we're good to go (either `force`, or nothing)
+            if tok.len() == i {
+                return Ok(Statement::DropModel(DropModel::new(e, force)));
+            }
+        }
+        Some(Token![space]) if tok.len() > 1 && tok[1].is_ident() => {
+            let mut i = 2; // (`space` and space name)
+            // should we force drop?
+            let force = i < tok.len() && tok[i] == Token::Ident("force".into());
+            i += force as usize;
+            *counter += i;
+            // either `force` or nothing
+            if tok.len() == i {
+                return Ok(Statement::DropSpace(DropSpace::new(
+                    unsafe { extract!(tok[1], Token::Ident(ref space) => space.clone()) },
+                    force,
+                )));
+            }
+        }
+        _ => {}
+    }
+    Err(LangError::UnexpectedToken)
 }
 
 #[cfg(test)]
@@ -79,3 +99,37 @@ pub(super) fn parse_drop_full(tok: &[Token]) -> LangResult<Statement> {
     assert_eq!(i, tok.len(), "full token stream not utilized");
     r
 }
+
+pub(super) fn parse_inspect(tok: &[Token], c: &mut usize) -> LangResult<Statement> {
+    /*
+        inpsect model <entity>
+        inspect space <entity>
+        inspect spaces
+    */
+
+    let nxt = tok.get(0);
+    *c += nxt.is_some() as usize;
+    match nxt {
+        Some(Token![model]) => Entity::parse_from_tokens(&tok[1..], c).map(Statement::InspectModel),
+        Some(Token![space]) if tok.len() == 2 && tok[1].is_ident() => {
+            *c += 1;
+            Ok(Statement::InspectSpace(unsafe {
+                extract!(tok[1], Token::Ident(ref space) => space.clone())
+            }))
+        }
+        Some(Token::Ident(id))
+            if unsafe { id.as_slice().eq_ignore_ascii_case(b"spaces") } && tok.len() == 1 =>
+        {
+            Ok(Statement::InspectSpaces)
+        }
+        _ => Err(LangError::ExpectedStatement),
+    }
+}
+
+#[cfg(test)]
+pub(super) fn parse_inspect_full(tok: &[Token]) -> LangResult<Statement> {
+    let mut i = 0;
+    let r = self::parse_inspect(tok, &mut i);
+    assert_eq!(i, tok.len(), "full token stream not used");
+    r
+}
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 2b5c5f0d..713131b3 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -860,22 +860,21 @@ pub(super) fn parse_space_from_tokens(tok: &[Token]) -> LangResult<(Space, usize
 
 #[inline(always)]
 /// Parse alter space from tokens
-pub(super) fn parse_alter_space_from_tokens(
-    tok: &[Token],
-    space_name: RawSlice,
-) -> LangResult<(AlterSpace, usize)> {
+pub(super) fn parse_alter_space_from_tokens(tok: &[Token]) -> LangResult<(AlterSpace, usize)> {
     let mut i = 0;
     let l = tok.len();
 
-    let invalid = l < 3
-        || !(tok[i] == (Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With)))
-            && tok[i + 1] == (Token::Symbol(Symbol::TtOpenBrace)));
+    let okay = l > 3 && tok[0].is_ident() && tok[1] == Token![with] && tok[2] == Token![open {}];
 
-    if invalid {
+    if !okay {
         return Err(LangError::UnexpectedToken);
     }
 
-    i += 2;
+    let space_name = unsafe {
+        extract!(tok[0], Token::Ident(ref space) => space.clone())
+    };
+
+    i += 3;
 
     let mut d = Dict::new();
     let ret = rfold_dict(DictFoldState::CB_OR_IDENT, &tok[i..], &mut d);
@@ -894,6 +893,13 @@ pub(super) fn parse_alter_space_from_tokens(
     }
 }
 
+#[cfg(test)]
+pub(super) fn alter_space_full(tok: &[Token]) -> LangResult<AlterSpace> {
+    let (r, i) = self::parse_alter_space_from_tokens(tok)?;
+    assert_eq!(i, tok.len(), "full token stream not used");
+    Ok(r)
+}
+
 states! {
     /// The field syntax parse state
     pub struct FieldSyntaxParseState: u8 {
@@ -1001,6 +1007,23 @@ pub(super) fn parse_field_syntax<const ALLOW_RESET: bool>(
     }
 }
 
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+pub struct Alter {
+    model: RawSlice,
+    kind: AlterKind,
+}
+
+impl Alter {
+    #[inline(always)]
+    pub(super) fn new(model: RawSlice, kind: AlterKind) -> Self {
+        Self {
+            model,
+            kind: kind.into(),
+        }
+    }
+}
+
 #[derive(Debug)]
 #[cfg_attr(test, derive(PartialEq))]
 /// The alter operation kind
@@ -1015,25 +1038,28 @@ pub enum AlterKind {
 pub(super) fn parse_alter_kind_from_tokens(
     tok: &[Token],
     current: &mut usize,
-) -> LangResult<AlterKind> {
+) -> LangResult<Alter> {
     let l = tok.len();
-    if l < 2 {
+    let okay = l > 2 && tok[0].is_ident();
+    if !okay {
         return Err(LangError::UnexpectedEndofStatement);
     }
-    *current += 1;
-    let r = match tok[0] {
-        Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::Add)) => {
-            AlterKind::Add(alter_add(&tok[1..], current)?)
-        }
+    *current += 2;
+    let model_name = unsafe { extract!(tok[0], Token::Ident(ref l) => l.clone()) };
+    match tok[1] {
+        Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::Add)) => alter_add(&tok[1..], current)
+            .map(AlterKind::Add)
+            .map(|kind| Alter::new(model_name, kind)),
         Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::Remove)) => {
-            AlterKind::Remove(alter_remove(&tok[1..], current)?)
-        }
-        Token::Keyword(Keyword::Dml(DmlKeyword::Update)) => {
-            AlterKind::Update(alter_update(&tok[1..], current)?)
+            alter_remove(&tok[1..], current)
+                .map(AlterKind::Remove)
+                .map(|kind| Alter::new(model_name, kind))
         }
+        Token::Keyword(Keyword::Dml(DmlKeyword::Update)) => alter_update(&tok[1..], current)
+            .map(AlterKind::Update)
+            .map(|kind| Alter::new(model_name, kind)),
         _ => return Err(LangError::ExpectedStatement),
-    };
-    Ok(r)
+    }
 }
 
 #[inline(always)]
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 84a33b70..00509494 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -222,8 +222,8 @@ mod ddl_other_query_tests {
     use {
         super::*,
         crate::engine::ql::{
-            ast::Statement,
-            ddl::{self, DropItem},
+            ast::{Entity, Statement},
+            ddl::{self, DropModel, DropSpace},
         },
     };
     #[test]
@@ -231,7 +231,7 @@ mod ddl_other_query_tests {
         let src = lex(br"drop space myspace").unwrap();
         assert_eq!(
             ddl::parse_drop_full(&src[1..]).unwrap(),
-            Statement::DropSpace(DropItem::new("myspace".into(), false))
+            Statement::DropSpace(DropSpace::new("myspace".into(), false))
         );
     }
     #[test]
@@ -239,7 +239,7 @@ mod ddl_other_query_tests {
         let src = lex(br"drop space myspace force").unwrap();
         assert_eq!(
             ddl::parse_drop_full(&src[1..]).unwrap(),
-            Statement::DropSpace(DropItem::new("myspace".into(), true))
+            Statement::DropSpace(DropSpace::new("myspace".into(), true))
         );
     }
     #[test]
@@ -247,7 +247,7 @@ mod ddl_other_query_tests {
         let src = lex(br"drop model mymodel").unwrap();
         assert_eq!(
             ddl::parse_drop_full(&src[1..]).unwrap(),
-            Statement::DropModel(DropItem::new("mymodel".into(), false))
+            Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), false))
         );
     }
     #[test]
@@ -255,7 +255,7 @@ mod ddl_other_query_tests {
         let src = lex(br"drop model mymodel force").unwrap();
         assert_eq!(
             ddl::parse_drop_full(&src[1..]).unwrap(),
-            Statement::DropModel(DropItem::new("mymodel".into(), true))
+            Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), true))
         );
     }
 }
@@ -307,6 +307,88 @@ mod schema_tests {
         }
     }
 
+    mod inspect {
+        use {
+            super::*,
+            crate::engine::ql::{
+                ast::{Entity, Statement},
+                ddl,
+            },
+        };
+        #[test]
+        fn inspect_space() {
+            let tok = lex(b"inspect space myspace").unwrap();
+            assert_eq!(
+                ddl::parse_inspect_full(&tok[1..]).unwrap(),
+                Statement::InspectSpace("myspace".into())
+            );
+        }
+        #[test]
+        fn inspect_model() {
+            let tok = lex(b"inspect model user").unwrap();
+            assert_eq!(
+                ddl::parse_inspect_full(&tok[1..]).unwrap(),
+                Statement::InspectModel(Entity::Single("user".into()))
+            );
+            let tok = lex(b"inspect model tweeter.user").unwrap();
+            assert_eq!(
+                ddl::parse_inspect_full(&tok[1..]).unwrap(),
+                Statement::InspectModel(("tweeter", "user").into())
+            );
+        }
+        #[test]
+        fn inspect_spaces() {
+            let tok = lex(b"inspect spaces").unwrap();
+            assert_eq!(
+                ddl::parse_inspect_full(&tok[1..]).unwrap(),
+                Statement::InspectSpaces
+            );
+        }
+    }
+
+    mod alter_space {
+        use {
+            super::*,
+            crate::engine::ql::{
+                lexer::Lit,
+                schema::{self, AlterSpace},
+            },
+        };
+        #[test]
+        fn alter_space_mini() {
+            let tok = lex(b"alter model mymodel with {}").unwrap();
+            let r = schema::alter_space_full(&tok[2..]).unwrap();
+            assert_eq!(
+                r,
+                AlterSpace {
+                    space_name: "mymodel".into(),
+                    updated_props: dict! {}
+                }
+            );
+        }
+        #[test]
+        fn alter_space() {
+            let tok = lex(br#"
+                alter model mymodel with {
+                    max_entry: 1000,
+                    driver: "ts-0.8"
+                }
+            "#)
+            .unwrap();
+            let r = schema::alter_space_full(&tok[2..]).unwrap();
+            assert_eq!(
+                r,
+                AlterSpace {
+                    space_name: "mymodel".into(),
+                    updated_props: dict! {
+                        "max_entry" => Lit::Num(1000),
+                        "driver" => Lit::Str("ts-0.8".into())
+                    }
+                }
+            );
+        }
+    }
+
     mod dict {
         use super::*;
 

From c36693afd5f990a974016acf0ae054b4b0eae17b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 17 Nov 2022 22:13:29 +0530
Subject: [PATCH 044/310] Add benches for `use` and `inspect`

---
 server/src/engine/ql/ast.rs     |  4 +--
 server/src/engine/ql/benches.rs | 49 +++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index bff34a06..0f31253b 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -167,12 +167,12 @@ impl Entity {
 /// A [`Statement`] is a fully BlueQL statement that can be executed by the query engine
 // TODO(@ohsayan): Determine whether we actually need this
 pub enum Statement {
+    /// DDL query to switch between spaces and models
+    Use(Entity),
     /// DDL query to create a model
     CreateModel(schema::Model),
     /// DDL query to create a space
     CreateSpace(schema::Space),
-    /// DDL query to switch between spaces and models
-    Use(Entity),
     /// DDL query to alter a space (properties)
     AlterSpace(schema::AlterSpace),
     /// DDL query to alter a model (properties, field types, etc)
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index 6d2a7cbf..5c2b89b0 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -114,3 +114,52 @@ mod ast {
         });
     }
 }
+
+mod ddl_queries {
+    use {
+        super::*,
+        crate::engine::ql::ast::{Compiler, Entity, Statement},
+    };
+    mod use_stmt {
+        use super::*;
+        #[bench]
+        fn use_space(b: &mut Bencher) {
+            let src = b"use myspace";
+            let expected = Statement::Use(Entity::Single("myspace".into()));
+            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+        }
+        #[bench]
+        fn use_model(b: &mut Bencher) {
+            let src = b"use myspace.mymodel";
+            let expected = Statement::Use(("myspace", "mymodel").into());
+            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+        }
+    }
+    mod inspect_stmt {
+        use super::*;
+        #[bench]
+        fn inspect_space(b: &mut Bencher) {
+            let src = b"inspect space myspace";
+            let expected = Statement::InspectSpace("myspace".into());
+            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+        }
+        #[bench]
+        fn inspect_model_single_entity(b: &mut Bencher) {
+            let src = b"inspect model mymodel";
+            let expected = Statement::InspectModel(Entity::Single("mymodel".into()));
+            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+        }
+        #[bench]
+        fn inspect_model_full_entity(b: &mut Bencher) {
+            let src = b"inspect model myspace.mymodel";
+            let expected = Statement::InspectModel(("myspace", "mymodel").into());
+            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+        }
+        #[bench]
+        fn inspect_spaces(b: &mut Bencher) {
+            let src = b"inspect spaces";
+            let expected = Statement::InspectSpaces;
+            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+        }
+    }
+}

From 173832dd132f0f6785c8f51769b56200a72e87d6 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 18 Nov 2022 09:05:48 +0530
Subject: [PATCH 045/310] Add benches for `drop`

---
 server/src/engine/ql/benches.rs | 46 +++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index 5c2b89b0..7b6e31ae 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -162,4 +162,50 @@ mod ddl_queries {
             b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
         }
     }
+    mod drop_stmt {
+        use {
+            super::*,
+            crate::engine::ql::ddl::{DropModel, DropSpace},
+        };
+        #[bench]
+        fn drop_space(b: &mut Bencher) {
+            let src = b"drop space myspace";
+            let expected = Statement::DropSpace(DropSpace::new("myspace".into(), false));
+            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+        }
+        #[bench]
+        fn drop_space_force(b: &mut Bencher) {
+            let src = b"drop space myspace force";
+            let expected = Statement::DropSpace(DropSpace::new("myspace".into(), true));
+            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+        }
+        #[bench]
+        fn drop_model_single(b: &mut Bencher) {
+            let src = b"drop model mymodel";
+            let expected =
+                Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), false));
+            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+        }
+        #[bench]
+        fn drop_model_single_force(b: &mut Bencher) {
+            let src = b"drop model mymodel force";
+            let expected =
+                Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), true));
+            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+        }
+        #[bench]
+        fn drop_model_full(b: &mut Bencher) {
+            let src = b"drop model myspace.mymodel";
+            let expected =
+                Statement::DropModel(DropModel::new(("myspace", "mymodel").into(), false));
+            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+        }
+        #[bench]
+        fn drop_model_full_force(b: &mut Bencher) {
+            let src = b"drop model myspace.mymodel force";
+            let expected =
+                Statement::DropModel(DropModel::new(("myspace", "mymodel").into(), true));
+            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+        }
+    }
 }

From 986cb26ebdac212117af1fe771ca76704d4e02a3 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 20 Nov 2022 20:58:52 +0530
Subject: [PATCH 046/310] Support `null` in dict and tymeta

---
 server/src/engine/memory/mod.rs |  21 ++
 server/src/engine/ql/dml.rs     |   6 +-
 server/src/engine/ql/lexer.rs   |  11 +-
 server/src/engine/ql/macros.rs  |  13 ++
 server/src/engine/ql/schema.rs  |  36 +++-
 server/src/engine/ql/tests.rs   | 363 +++++++++++++++++++++-----------
 6 files changed, 308 insertions(+), 142 deletions(-)

diff --git a/server/src/engine/memory/mod.rs b/server/src/engine/memory/mod.rs
index 4c3147ce..de803c64 100644
--- a/server/src/engine/memory/mod.rs
+++ b/server/src/engine/memory/mod.rs
@@ -61,3 +61,24 @@ impl<const N: usize> From<[DataType; N]> for DataType {
         Self::List(f.into())
     }
 }
+
+#[repr(u8, align(1))]
+pub enum DataKind {
+    // primitive: integer unsigned
+    UInt8 = 0,
+    UInt16 = 1,
+    Uint32 = 2,
+    UInt64 = 3,
+    // primitive: integer unsigned
+    SInt8 = 4,
+    SInt16 = 5,
+    SInt32 = 6,
+    SInt64 = 7,
+    // primitive: misc
+    Bool = 8,
+    // compound: flat
+    String = 9,
+    Binary = 10,
+    // compound: recursive
+    List = 11,
+}
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 503e2f47..15b145e9 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -80,7 +80,7 @@ pub(super) fn parse_list(
     while i < l && okay && !stop {
         let d = match &tok[i] {
             Token::Lit(Lit::Str(s)) => DataType::String(s.to_string()),
-            Token::Lit(Lit::Num(n)) => DataType::Number(*n),
+            Token::Lit(Lit::UnsignedInt(n)) => DataType::Number(*n),
             Token::Lit(Lit::Bool(b)) => DataType::Boolean(*b),
             Token::Symbol(Symbol::TtOpenSqBracket) => {
                 // a nested list
@@ -141,7 +141,7 @@ pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<Option<DataType>>,
             Token::Lit(Lit::Str(s)) => {
                 data.push(Some(s.to_string().into()));
             }
-            Token::Lit(Lit::Num(n)) => {
+            Token::Lit(Lit::UnsignedInt(n)) => {
                 data.push(Some((*n).into()));
             }
             Token::Lit(Lit::Bool(b)) => {
@@ -201,7 +201,7 @@ pub(super) fn parse_data_map_syntax<'a>(
                     .insert(unsafe { id.as_slice() }, Some(s.to_string().into()))
                     .is_none();
             }
-            (Token::Ident(id), Token::Lit(Lit::Num(n))) => {
+            (Token::Ident(id), Token::Lit(Lit::UnsignedInt(n))) => {
                 okay &= data
                     .insert(unsafe { id.as_slice() }, Some((*n).into()))
                     .is_none();
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 869b3681..96a0301a 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -74,7 +74,7 @@ enum_impls! {
 pub enum Lit {
     Str(Box<str>),
     Bool(bool),
-    Num(u64),
+    UnsignedInt(u64),
     UnsafeLit(RawSlice),
 }
 
@@ -89,7 +89,7 @@ enum_impls! {
         Box<str> as Str,
         String as Str,
         bool as Bool,
-        u64 as Num,
+        u64 as UnsignedInt,
     }
 }
 
@@ -514,15 +514,16 @@ impl<'a> Lexer<'a> {
                 1234, // valid
                 1234a // invalid
             */
-            static TERMINAL_CHAR: [u8; 8] = [b';', b'}', b',', b' ', b'\n', b'\t', b',', b']'];
-            let wseof = self.peek_is(|b| TERMINAL_CHAR.contains(&b)) || self.exhausted();
+            let wseof = self.peek_is(|char| !char.is_ascii_alphabetic()) || self.exhausted();
             match str::from_utf8_unchecked(slice::from_raw_parts(
                 s,
                 self.cursor().offset_from(s) as usize,
             ))
             .parse()
             {
-                Ok(num) if compiler::likely(wseof) => self.tokens.push(Token::Lit(Lit::Num(num))),
+                Ok(num) if compiler::likely(wseof) => {
+                    self.tokens.push(Token::Lit(Lit::UnsignedInt(num)))
+                }
                 _ => self.last_error = Some(LangError::InvalidNumericLiteral),
             }
         }
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 18a3e56c..197319cb 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -306,6 +306,19 @@ macro_rules! dict {
     }};
 }
 
+macro_rules! nullable_dict {
+    () => {
+        dict! {}
+    };
+    ($($key:expr => $value:expr),* $(,)?) => {
+        dict! {
+            $(
+                $key => $crate::engine::ql::tests::NullableMapEntry::data($value),
+            )*
+        }
+    };
+}
+
 macro_rules! dict_nullable {
     () => {
         <::std::collections::HashMap<_, _> as ::core::default::Default>::default()
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 713131b3..7bac5e11 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -105,7 +105,7 @@ impl From<Dict> for DictEntry {
 }
 
 /// A metadata dictionary
-pub type Dict = HashMap<String, DictEntry>;
+pub type Dict = HashMap<String, Option<DictEntry>>;
 
 #[derive(Debug, PartialEq)]
 /// A layer contains a type and corresponding metadata
@@ -271,11 +271,19 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
                 okay &= dict
                     .insert(
                         unsafe { tmp.assume_init_ref() }.to_string(),
-                        l.clone().into(),
+                        Some(l.clone().into()),
                     )
                     .is_none();
                 state = DictFoldState::COMMA_OR_CB;
             }
+            (Token![null], DictFoldState::LIT_OR_OB) => {
+                // null
+                i += 1;
+                okay &= dict
+                    .insert(unsafe { tmp.assume_init_ref() }.to_string(), None)
+                    .is_none();
+                state = DictFoldState::COMMA_OR_CB;
+            }
             // ONLY COMMA CAPTURE
             (Token::Symbol(Symbol::SymComma), DictFoldState::COMMA_OR_CB) => {
                 i += 1;
@@ -292,7 +300,7 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
                 okay &= dict
                     .insert(
                         unsafe { tmp.assume_init_ref() }.to_string(),
-                        new_dict.into(),
+                        Some(new_dict.into()),
                     )
                     .is_none();
                 // at the end of a dict we either expect a comma or close brace
@@ -465,13 +473,22 @@ pub(super) fn rfold_tymeta<const ALLOW_RESET: bool>(
                 r.record(
                     dict.insert(
                         unsafe { tmp.assume_init_ref() }.to_string(),
-                        lit.clone().into(),
+                        Some(lit.clone().into()),
                     )
                     .is_none(),
                 );
                 // saw a literal. next is either comma or close brace
                 state = TyMetaFoldState::COMMA_OR_CB;
             }
+            (Token![null], TyMetaFoldState::LIT_OR_OB) => {
+                r.incr();
+                r.record(
+                    dict.insert(unsafe { tmp.assume_init_ref() }.to_string(), None)
+                        .is_none(),
+                );
+                // saw null, start parsing another entry
+                state = TyMetaFoldState::COMMA_OR_CB;
+            }
             (Token::Symbol(Symbol::SymComma), TyMetaFoldState::COMMA_OR_CB) => {
                 r.incr();
                 // next is strictly a close brace or ident
@@ -488,10 +505,11 @@ pub(super) fn rfold_tymeta<const ALLOW_RESET: bool>(
                 );
                 r.incr_by(ret.pos());
                 r.record(ret.is_okay());
-                r.record(!ret.has_more()); // L2 cannot have type definitions
-                                           // end of definition or comma followed by something
+                // L2 cannot have type definitions
+                r.record(!ret.has_more());
+                // end of definition or comma followed by something
                 r.record(
-                    dict.insert(unsafe { tmp.assume_init_ref() }.to_string(), d.into())
+                    dict.insert(unsafe { tmp.assume_init_ref() }.to_string(), Some(d.into()))
                         .is_none(),
                 );
                 state = TyMetaFoldState::COMMA_OR_CB;
@@ -870,9 +888,7 @@ pub(super) fn parse_alter_space_from_tokens(tok: &[Token]) -> LangResult<(AlterS
         return Err(LangError::UnexpectedToken);
     }
 
-    let space_name = unsafe {
-        extract!(tok[0], Token::Ident(ref space) => space.clone())
-    };
+    let space_name = unsafe { extract!(tok[0], Token::Ident(ref space) => space.clone()) };
 
     i += 3;
 
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 00509494..5bde3ada 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -61,6 +61,34 @@ fn nullable_datatype(v: impl NullableData<DataType>) -> Option<DataType> {
     v.data()
 }
 
+pub trait NullableMapEntry {
+    fn data(self) -> Option<super::schema::DictEntry>;
+}
+
+impl NullableMapEntry for Null {
+    fn data(self) -> Option<super::schema::DictEntry> {
+        None
+    }
+}
+
+impl NullableMapEntry for super::lexer::Lit {
+    fn data(self) -> Option<super::schema::DictEntry> {
+        Some(super::schema::DictEntry::Lit(self))
+    }
+}
+
+impl NullableMapEntry for super::schema::Dict {
+    fn data(self) -> Option<super::schema::DictEntry> {
+        Some(super::schema::DictEntry::Map(self))
+    }
+}
+
+macro_rules! fold_dict {
+    ($($input:expr),* $(,)?) => {
+        ($({$crate::engine::ql::schema::fold_dict(&super::lex($input).unwrap()).unwrap()}),*)
+    }
+}
+
 mod lexer_tests {
     use {
         super::{
@@ -89,7 +117,10 @@ mod lexer_tests {
     #[test]
     fn lex_number() {
         let number = v!("123456");
-        assert_eq!(lex(&number).unwrap(), vec![Token::Lit(Lit::Num(123456))]);
+        assert_eq!(
+            lex(&number).unwrap(),
+            vec![Token::Lit(Lit::UnsignedInt(123456))]
+        );
     }
     #[test]
     fn lex_bool() {
@@ -362,7 +393,7 @@ mod schema_tests {
                 r,
                 AlterSpace {
                     space_name: "mymodel".into(),
-                    updated_props: dict! {}
+                    updated_props: nullable_dict! {}
                 }
             );
         }
@@ -380,8 +411,8 @@ mod schema_tests {
                 r,
                 AlterSpace {
                     space_name: "mymodel".into(),
-                    updated_props: dict! {
-                        "max_entry" => Lit::Num(1000),
+                    updated_props: nullable_dict! {
+                        "max_entry" => Lit::UnsignedInt(1000),
                         "driver" => Lit::Str("ts-0.8".into())
                     }
                 }
@@ -392,19 +423,13 @@ mod schema_tests {
     mod dict {
         use super::*;
 
-        macro_rules! fold_dict {
-        ($($input:expr),* $(,)?) => {
-            ($({schema::fold_dict(&super::lex($input).unwrap()).unwrap()}),*)
-        }
-    }
-
         #[test]
         fn dict_read_mini() {
             let (d1, d2) = fold_dict! {
                 br#"{name: "sayan"}"#,
                 br#"{name: "sayan",}"#,
             };
-            let r = dict!("name" => Lit::Str("sayan".into()));
+            let r = nullable_dict!("name" => Lit::Str("sayan".into()));
             multi_assert_eq!(d1, d2 => r);
         }
         #[test]
@@ -425,10 +450,10 @@ mod schema_tests {
                 }
             "#,
             };
-            let r = dict! (
+            let r = nullable_dict! (
                 "name" => Lit::Str("sayan".into()),
                 "verified" => Lit::Bool(true),
-                "burgers" => Lit::Num(152),
+                "burgers" => Lit::UnsignedInt(152),
             );
             multi_assert_eq!(d1, d2 => r);
         }
@@ -466,12 +491,12 @@ mod schema_tests {
             }"#
             };
             multi_assert_eq!(
-                d1, d2, d3 => dict! {
+                d1, d2, d3 => nullable_dict! {
                     "name" => Lit::Str("sayan".into()),
-                    "notes" => dict! {
+                    "notes" => nullable_dict! {
                         "burgers" => Lit::Str("all the time, extra mayo".into()),
                         "taco" => Lit::Bool(true),
-                        "pretzels" => Lit::Num(1),
+                        "pretzels" => Lit::UnsignedInt(1),
                     }
                 }
             );
@@ -521,11 +546,11 @@ mod schema_tests {
             }"#
             };
             multi_assert_eq!(
-                d1, d2, d3 => dict! {
-                    "well" => dict! {
-                        "now" => dict! {
-                            "this" => dict! {
-                                "is" => dict! {
+                d1, d2, d3 => nullable_dict! {
+                    "well" => nullable_dict! {
+                        "now" => nullable_dict! {
+                            "this" => nullable_dict! {
+                                "is" => nullable_dict! {
                                     "ridiculous" => Lit::Bool(true),
                                 }
                             }
@@ -554,16 +579,16 @@ mod schema_tests {
                 }
             ")
             .unwrap();
-            let ret_dict = dict! {
+            let ret_dict = nullable_dict! {
                 "the_tradition_is" => Lit::Str("hello, world".into()),
-                "could_have_been" => dict! {
+                "could_have_been" => nullable_dict! {
                     "this" => Lit::Bool(true),
-                    "or_maybe_this" => Lit::Num(100),
+                    "or_maybe_this" => Lit::UnsignedInt(100),
                     "even_this" => Lit::Str("hello, universe!".into()),
                 },
                 "but_oh_well" => Lit::Str("it continues to be the 'annoying' phrase".into()),
-                "lorem" => dict! {
-                    "ipsum" => dict! {
+                "lorem" => nullable_dict! {
+                    "ipsum" => nullable_dict! {
                         "dolor" => Lit::Str("sit amet".into())
                     }
                 }
@@ -591,7 +616,7 @@ mod schema_tests {
             assert!(res.is_okay());
             assert!(!res.has_more());
             assert_eq!(res.pos(), 1);
-            assert_eq!(ret, dict!());
+            assert_eq!(ret, nullable_dict!());
         }
         #[test]
         fn tymeta_mini_fail() {
@@ -600,7 +625,7 @@ mod schema_tests {
             assert!(!res.is_okay());
             assert!(!res.has_more());
             assert_eq!(res.pos(), 0);
-            assert_eq!(ret, dict!());
+            assert_eq!(ret, nullable_dict!());
         }
         #[test]
         fn tymeta() {
@@ -611,10 +636,10 @@ mod schema_tests {
             assert_eq!(res.pos(), tok.len());
             assert_eq!(
                 ret,
-                dict! {
+                nullable_dict! {
                     "hello" => Lit::Str("world".into()),
                     "loading" => Lit::Bool(true),
-                    "size" => Lit::Num(100)
+                    "size" => Lit::UnsignedInt(100)
                 }
             );
         }
@@ -636,8 +661,8 @@ mod schema_tests {
             final_ret.extend(ret2);
             assert_eq!(
                 final_ret,
-                dict! {
-                    "maxlen" => Lit::Num(100),
+                nullable_dict! {
+                    "maxlen" => Lit::UnsignedInt(100),
                     "unique" => Lit::Bool(true)
                 }
             )
@@ -661,10 +686,10 @@ mod schema_tests {
             final_ret.extend(ret2);
             assert_eq!(
                 final_ret,
-                dict! {
-                    "maxlen" => Lit::Num(100),
+                nullable_dict! {
+                    "maxlen" => Lit::UnsignedInt(100),
                     "unique" => Lit::Bool(true),
-                    "this" => dict! {
+                    "this" => nullable_dict! {
                         "is" => Lit::Str("cool".into())
                     }
                 }
@@ -684,10 +709,10 @@ mod schema_tests {
                 }
             ")
             .unwrap();
-            let expected = dict! {
-                "maxlen" => Lit::Num(10),
+            let expected = nullable_dict! {
+                "maxlen" => Lit::UnsignedInt(10),
                 "unique" => Lit::Bool(true),
-                "auth" => dict! {
+                "auth" => nullable_dict! {
                     "maybe" => Lit::Bool(true),
                 },
                 "user" => Lit::Str("sayan".into())
@@ -722,10 +747,10 @@ mod schema_tests {
                 }
             ")
             .unwrap();
-            let expected = dict! {
-                "maxlen" => Lit::Num(10),
+            let expected = nullable_dict! {
+                "maxlen" => Lit::UnsignedInt(10),
                 "unique" => Lit::Bool(true),
-                "auth" => dict! {
+                "auth" => nullable_dict! {
                     "maybe" => Lit::Bool(true),
                 },
             };
@@ -751,7 +776,10 @@ mod schema_tests {
             let (layers, c, okay) = schema::fold_layers(&tok);
             assert_eq!(c, tok.len() - 1);
             assert!(okay);
-            assert_eq!(layers, vec![Layer::new_noreset(Type::String, dict! {})]);
+            assert_eq!(
+                layers,
+                vec![Layer::new_noreset(Type::String, nullable_dict! {})]
+            );
         }
         #[test]
         fn layer() {
@@ -763,8 +791,8 @@ mod schema_tests {
                 layers,
                 vec![Layer::new_noreset(
                     Type::String,
-                    dict! {
-                        "maxlen" => Lit::Num(100)
+                    nullable_dict! {
+                        "maxlen" => Lit::UnsignedInt(100)
                     }
                 )]
             );
@@ -778,8 +806,8 @@ mod schema_tests {
             assert_eq!(
                 layers,
                 vec![
-                    Layer::new_noreset(Type::String, dict! {}),
-                    Layer::new_noreset(Type::List, dict! {})
+                    Layer::new_noreset(Type::String, nullable_dict! {}),
+                    Layer::new_noreset(Type::List, nullable_dict! {})
                 ]
             );
         }
@@ -792,12 +820,12 @@ mod schema_tests {
             assert_eq!(
                 layers,
                 vec![
-                    Layer::new_noreset(Type::String, dict! {}),
+                    Layer::new_noreset(Type::String, nullable_dict! {}),
                     Layer::new_noreset(
                         Type::List,
-                        dict! {
+                        nullable_dict! {
                             "unique" => Lit::Bool(true),
-                            "maxlen" => Lit::Num(10),
+                            "maxlen" => Lit::UnsignedInt(10),
                         }
                     )
                 ]
@@ -817,16 +845,16 @@ mod schema_tests {
                 vec![
                     Layer::new_noreset(
                         Type::String,
-                        dict! {
+                        nullable_dict! {
                             "ascii_only" => Lit::Bool(true),
-                            "maxlen" => Lit::Num(255)
+                            "maxlen" => Lit::UnsignedInt(255)
                         }
                     ),
                     Layer::new_noreset(
                         Type::List,
-                        dict! {
+                        nullable_dict! {
                             "unique" => Lit::Bool(true),
-                            "maxlen" => Lit::Num(10),
+                            "maxlen" => Lit::UnsignedInt(10),
                         }
                     )
                 ]
@@ -846,14 +874,14 @@ mod schema_tests {
         ")
             .unwrap();
             let expected = vec![
-                Layer::new_noreset(Type::String, dict!()),
+                Layer::new_noreset(Type::String, nullable_dict!()),
                 Layer::new_noreset(
                     Type::List,
-                    dict! {
-                        "maxlen" => Lit::Num(100),
+                    nullable_dict! {
+                        "maxlen" => Lit::UnsignedInt(100),
                     },
                 ),
-                Layer::new_noreset(Type::List, dict!("unique" => Lit::Bool(true))),
+                Layer::new_noreset(Type::List, nullable_dict!("unique" => Lit::Bool(true))),
             ];
             fuzz_tokens(&tok, |should_pass, new_tok| {
                 let (layers, c, okay) = schema::fold_layers(&new_tok);
@@ -916,7 +944,7 @@ mod schema_tests {
                 f,
                 Field {
                     field_name: "username".into(),
-                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
                     props: set![],
                 }
             )
@@ -933,7 +961,7 @@ mod schema_tests {
                 f,
                 Field {
                     field_name: "username".into(),
-                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
                     props: set!["primary"],
                 }
             )
@@ -955,8 +983,8 @@ mod schema_tests {
                     field_name: "username".into(),
                     layers: [Layer::new_noreset(
                         Type::String,
-                        dict! {
-                            "maxlen" => Lit::Num(10),
+                        nullable_dict! {
+                            "maxlen" => Lit::UnsignedInt(10),
                             "ascii_only" => Lit::Bool(true),
                         }
                     )]
@@ -986,14 +1014,14 @@ mod schema_tests {
                     layers: [
                         Layer::new_noreset(
                             Type::String,
-                            dict! {
-                                "maxlen" => Lit::Num(255),
+                            nullable_dict! {
+                                "maxlen" => Lit::UnsignedInt(255),
                                 "ascii_only" => Lit::Bool(true),
                             }
                         ),
                         Layer::new_noreset(
                             Type::List,
-                            dict! {
+                            nullable_dict! {
                                 "unique" => Lit::Bool(true)
                             }
                         ),
@@ -1032,16 +1060,16 @@ mod schema_tests {
                     fields: vec![
                         Field {
                             field_name: "username".into(),
-                            layers: vec![Layer::new_noreset(Type::String, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::String, nullable_dict! {})],
                             props: set!["primary"]
                         },
                         Field {
                             field_name: "password".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
                             props: set![]
                         }
                     ],
-                    props: dict! {}
+                    props: nullable_dict! {}
                 }
             )
         }
@@ -1067,21 +1095,21 @@ mod schema_tests {
                     fields: vec![
                         Field {
                             field_name: "username".into(),
-                            layers: vec![Layer::new_noreset(Type::String, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::String, nullable_dict! {})],
                             props: set!["primary"]
                         },
                         Field {
                             field_name: "password".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
                             props: set![]
                         },
                         Field {
                             field_name: "profile_pic".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
                             props: set!["null"]
                         }
                     ],
-                    props: dict! {}
+                    props: nullable_dict! {}
                 }
             )
         }
@@ -1112,26 +1140,26 @@ mod schema_tests {
                     fields: vec![
                         Field {
                             field_name: "username".into(),
-                            layers: vec![Layer::new_noreset(Type::String, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::String, nullable_dict! {})],
                             props: set!["primary"]
                         },
                         Field {
                             field_name: "password".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
                             props: set![]
                         },
                         Field {
                             field_name: "profile_pic".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
                             props: set!["null"]
                         },
                         Field {
                             field_name: "notes".into(),
                             layers: vec![
-                                Layer::new_noreset(Type::String, dict! {}),
+                                Layer::new_noreset(Type::String, nullable_dict! {}),
                                 Layer::new_noreset(
                                     Type::List,
-                                    dict! {
+                                    nullable_dict! {
                                         "unique" => Lit::Bool(true)
                                     }
                                 )
@@ -1139,7 +1167,7 @@ mod schema_tests {
                             props: set!["null"]
                         }
                     ],
-                    props: dict! {}
+                    props: nullable_dict! {}
                 }
             )
         }
@@ -1175,26 +1203,26 @@ mod schema_tests {
                     fields: vec![
                         Field {
                             field_name: "username".into(),
-                            layers: vec![Layer::new_noreset(Type::String, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::String, nullable_dict! {})],
                             props: set!["primary"]
                         },
                         Field {
                             field_name: "password".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
                             props: set![]
                         },
                         Field {
                             field_name: "profile_pic".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, dict! {})],
+                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
                             props: set!["null"]
                         },
                         Field {
                             field_name: "notes".into(),
                             layers: vec![
-                                Layer::new_noreset(Type::String, dict! {}),
+                                Layer::new_noreset(Type::String, nullable_dict! {}),
                                 Layer::new_noreset(
                                     Type::List,
-                                    dict! {
+                                    nullable_dict! {
                                         "unique" => Lit::Bool(true)
                                     }
                                 )
@@ -1202,9 +1230,9 @@ mod schema_tests {
                             props: set!["null"]
                         }
                     ],
-                    props: dict! {
-                        "env" => dict! {
-                            "free_user_limit" => Lit::Num(100),
+                    props: nullable_dict! {
+                        "env" => nullable_dict! {
+                            "free_user_limit" => Lit::UnsignedInt(100),
                         },
                         "storage_driver" => Lit::Str("skyheap".into()),
                     }
@@ -1227,8 +1255,8 @@ mod schema_tests {
                 ef,
                 ExpandedField {
                     field_name: "username".into(),
-                    layers: vec![Layer::new_noreset(Type::String, dict! {})],
-                    props: dict! {},
+                    layers: vec![Layer::new_noreset(Type::String, nullable_dict! {})],
+                    props: nullable_dict! {},
                     reset: false
                 }
             )
@@ -1248,10 +1276,10 @@ mod schema_tests {
                 ef,
                 ExpandedField {
                     field_name: "username".into(),
-                    props: dict! {
+                    props: nullable_dict! {
                         "nullable" => Lit::Bool(false),
                     },
-                    layers: vec![Layer::new_noreset(Type::String, dict! {})],
+                    layers: vec![Layer::new_noreset(Type::String, nullable_dict! {})],
                     reset: false
                 }
             );
@@ -1275,15 +1303,15 @@ mod schema_tests {
                 ef,
                 ExpandedField {
                     field_name: "username".into(),
-                    props: dict! {
+                    props: nullable_dict! {
                         "nullable" => Lit::Bool(false),
                         "jingle_bells" => Lit::Str("snow".into()),
                     },
                     layers: vec![Layer::new_noreset(
                         Type::String,
-                        dict! {
-                            "minlen" => Lit::Num(6),
-                            "maxlen" => Lit::Num(255),
+                        nullable_dict! {
+                            "minlen" => Lit::UnsignedInt(6),
+                            "maxlen" => Lit::UnsignedInt(255),
                         }
                     )],
                     reset: false
@@ -1311,20 +1339,20 @@ mod schema_tests {
                 ef,
                 ExpandedField {
                     field_name: "notes".into(),
-                    props: dict! {
+                    props: nullable_dict! {
                         "nullable" => Lit::Bool(true),
                         "jingle_bells" => Lit::Str("snow".into()),
                     },
                     layers: vec![
                         Layer::new_noreset(
                             Type::String,
-                            dict! {
+                            nullable_dict! {
                                 "ascii_only" => Lit::Bool(true),
                             }
                         ),
                         Layer::new_noreset(
                             Type::List,
-                            dict! {
+                            nullable_dict! {
                                 "unique" => Lit::Bool(true),
                             }
                         )
@@ -1391,8 +1419,8 @@ mod schema_tests {
                 r.as_ref(),
                 [ExpandedField {
                     field_name: "myfield".into(),
-                    props: dict! {},
-                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    props: nullable_dict! {},
+                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
                     reset: false
                 }]
             );
@@ -1410,10 +1438,10 @@ mod schema_tests {
                 r.as_ref(),
                 [ExpandedField {
                     field_name: "myfield".into(),
-                    props: dict! {
+                    props: nullable_dict! {
                         "nullable" => Lit::Bool(true)
                     },
-                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
                     reset: false
                 }]
             );
@@ -1431,10 +1459,10 @@ mod schema_tests {
                 r.as_ref(),
                 [ExpandedField {
                     field_name: "myfield".into(),
-                    props: dict! {
+                    props: nullable_dict! {
                         "nullable" => Lit::Bool(true)
                     },
-                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
                     reset: false
                 }]
             );
@@ -1467,27 +1495,27 @@ mod schema_tests {
                 [
                     ExpandedField {
                         field_name: "myfield".into(),
-                        props: dict! {
+                        props: nullable_dict! {
                             "nullable" => Lit::Bool(true)
                         },
-                        layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                        layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
                         reset: false
                     },
                     ExpandedField {
                         field_name: "another".into(),
-                        props: dict! {
+                        props: nullable_dict! {
                             "nullable" => Lit::Bool(false)
                         },
                         layers: [
                             Layer::new_noreset(
                                 Type::String,
-                                dict! {
-                                    "maxlen" => Lit::Num(255)
+                                nullable_dict! {
+                                    "maxlen" => Lit::UnsignedInt(255)
                                 }
                             ),
                             Layer::new_noreset(
                                 Type::List,
-                                dict! {
+                                nullable_dict! {
                                    "unique" => Lit::Bool(true)
                                 },
                             )
@@ -1519,8 +1547,8 @@ mod schema_tests {
                 r.as_ref(),
                 [ExpandedField {
                     field_name: "myfield".into(),
-                    props: dict! {},
-                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    props: nullable_dict! {},
+                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
                     reset: true
                 }]
             );
@@ -1538,8 +1566,8 @@ mod schema_tests {
                 r.as_ref(),
                 [ExpandedField {
                     field_name: "myfield".into(),
-                    props: dict! {},
-                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    props: nullable_dict! {},
+                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
                     reset: true
                 }]
             );
@@ -1563,10 +1591,10 @@ mod schema_tests {
                 r.as_ref(),
                 [ExpandedField {
                     field_name: "myfield".into(),
-                    props: dict! {
+                    props: nullable_dict! {
                         "nullable" => Lit::Bool(true)
                     },
-                    layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
                     reset: true
                 }]
             );
@@ -1595,16 +1623,16 @@ mod schema_tests {
                 [
                     ExpandedField {
                         field_name: "myfield".into(),
-                        props: dict! {
+                        props: nullable_dict! {
                             "nullable" => Lit::Bool(true)
                         },
-                        layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                        layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
                         reset: true
                     },
                     ExpandedField {
                         field_name: "myfield2".into(),
-                        props: dict! {},
-                        layers: [Layer::new_noreset(Type::String, dict! {})].into(),
+                        props: nullable_dict! {},
+                        layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
                         reset: true
                     }
                 ]
@@ -1637,18 +1665,18 @@ mod schema_tests {
                 [
                     ExpandedField {
                         field_name: "myfield".into(),
-                        props: dict! {
+                        props: nullable_dict! {
                             "nullable" => Lit::Bool(true)
                         },
-                        layers: [Layer::new_reset(Type::String, dict! {})].into(),
+                        layers: [Layer::new_reset(Type::String, nullable_dict! {})].into(),
                         reset: true
                     },
                     ExpandedField {
                         field_name: "myfield2".into(),
-                        props: dict! {},
+                        props: nullable_dict! {},
                         layers: [Layer::new_reset(
                             Type::String,
-                            dict! {"maxlen" => Lit::Num(255)}
+                            nullable_dict! {"maxlen" => Lit::UnsignedInt(255)}
                         )]
                         .into(),
                         reset: true
@@ -1823,7 +1851,7 @@ mod dml_tests {
         fn map_mini() {
             let tok = lex(b"{}").unwrap();
             let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
-            assert_eq!(r, dict! {})
+            assert_eq!(r, nullable_dict! {})
         }
 
         #[test]
@@ -1997,7 +2025,7 @@ mod dml_tests {
             let e = InsertStatement {
                 primary_key: &("sayan".to_string().into()),
                 entity: Entity::Full("jotsy".into(), "app".into()),
-                data: dict! {}.into(),
+                data: nullable_dict! {}.into(),
             };
             assert_eq!(e, r);
         }
@@ -2290,3 +2318,90 @@ mod dml_tests {
         }
     }
 }
+
+mod nullable_dict_tests {
+    use super::*;
+    mod dict {
+        use {super::*, crate::engine::ql::lexer::Lit};
+
+        #[test]
+        fn null_mini() {
+            let d = fold_dict!(br"{ x: null }");
+            assert_eq!(
+                d,
+                nullable_dict! {
+                    "x" => Null,
+                }
+            );
+        }
+        #[test]
+        fn null() {
+            let d = fold_dict! {
+                br#"
+                    {
+                        this_is_non_null: "hello",
+                        but_this_is_null: null,
+                    }
+                "#
+            };
+            assert_eq!(
+                d,
+                nullable_dict! {
+                    "this_is_non_null" => Lit::Str("hello".into()),
+                    "but_this_is_null" => Null,
+                }
+            )
+        }
+        #[test]
+        fn null_pro() {
+            let d = fold_dict! {
+                br#"
+                    {
+                        a_string: "this is a string",
+                        num: 1234,
+                        a_dict: {
+                            a_null: null,
+                        }
+                    }
+                "#
+            };
+            assert_eq!(
+                d,
+                nullable_dict! {
+                    "a_string" => Lit::Str("this is a string".into()),
+                    "num" => Lit::UnsignedInt(1234),
+                    "a_dict" => nullable_dict! {
+                        "a_null" => Null,
+                    }
+                }
+            )
+        }
+        #[test]
+        fn null_pro_max() {
+            let d = fold_dict! {
+                br#"
+                    {
+                        a_string: "this is a string",
+                        num: 1234,
+                        a_dict: {
+                            a_null: null,
+                        },
+                        another_null: null,
+                    }
+                "#
+            };
+            assert_eq!(
+                d,
+                nullable_dict! {
+                    "a_string" => Lit::Str("this is a string".into()),
+                    "num" => Lit::UnsignedInt(1234),
+                    "a_dict" => nullable_dict! {
+                        "a_null" => Null,
+                    },
+                    "another_null" => Null,
+                }
+            )
+        }
+    }
+    // TODO(@ohsayan): Add null tests
+}

From 9d51dc70fe15a6da0dc8ba704b5cf2f6004638d4 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 20 Nov 2022 22:10:57 +0530
Subject: [PATCH 047/310] Make sure we accept all literal types in queries

---
 server/src/engine/memory/mod.rs |  4 +++
 server/src/engine/ql/dml.rs     | 52 ++++++++++++++++-----------------
 2 files changed, 30 insertions(+), 26 deletions(-)

diff --git a/server/src/engine/memory/mod.rs b/server/src/engine/memory/mod.rs
index de803c64..e3f8b28a 100644
--- a/server/src/engine/memory/mod.rs
+++ b/server/src/engine/memory/mod.rs
@@ -26,6 +26,8 @@
 
 // TODO(@ohsayan): Change the underlying structures, there are just rudimentary ones used during integration with the QL
 
+use super::ql::RawSlice;
+
 /// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
 /// be of one type.
 #[derive(Debug, PartialEq)]
@@ -43,6 +45,8 @@ pub enum DataType {
     /// elements to ensure correctness in this specific context
     /// FIXME(@ohsayan): Try enforcing this somehow
     List(Vec<Self>),
+    /// Not an actual data type but MUST be translated into an actual data type
+    AnonymousTypeNeedsEval(RawSlice),
 }
 
 enum_impls! {
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 15b145e9..1c38e23c 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -79,9 +79,12 @@ pub(super) fn parse_list(
     let mut prev_nlist_dscr = None;
     while i < l && okay && !stop {
         let d = match &tok[i] {
-            Token::Lit(Lit::Str(s)) => DataType::String(s.to_string()),
-            Token::Lit(Lit::UnsignedInt(n)) => DataType::Number(*n),
-            Token::Lit(Lit::Bool(b)) => DataType::Boolean(*b),
+            Token::Lit(l) => match l {
+                Lit::Str(s) => DataType::String(s.to_string()),
+                Lit::UnsignedInt(n) => DataType::Number(*n),
+                Lit::Bool(b) => DataType::Boolean(*b),
+                Lit::UnsafeLit(l) => DataType::AnonymousTypeNeedsEval(l.clone()),
+            },
             Token::Symbol(Symbol::TtOpenSqBracket) => {
                 // a nested list
                 let mut nested_list = Vec::new();
@@ -138,15 +141,18 @@ pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<Option<DataType>>,
     let mut data = Vec::new();
     while i < l && okay && !stop {
         match &tok[i] {
-            Token::Lit(Lit::Str(s)) => {
-                data.push(Some(s.to_string().into()));
-            }
-            Token::Lit(Lit::UnsignedInt(n)) => {
-                data.push(Some((*n).into()));
-            }
-            Token::Lit(Lit::Bool(b)) => {
-                data.push(Some((*b).into()));
-            }
+            Token::Lit(l) => match l {
+                Lit::Str(s) => {
+                    data.push(Some(s.to_string().into()));
+                }
+                Lit::UnsignedInt(n) => {
+                    data.push(Some((*n).into()));
+                }
+                Lit::Bool(b) => {
+                    data.push(Some((*b).into()));
+                }
+                Lit::UnsafeLit(r) => data.push(Some(DataType::AnonymousTypeNeedsEval(r.clone()))),
+            },
             Token::Symbol(Symbol::TtOpenSqBracket) => {
                 // ah, a list
                 let mut l = Vec::new();
@@ -196,20 +202,14 @@ pub(super) fn parse_data_map_syntax<'a>(
         let (field, colon, expression) = (&tok[i], &tok[i + 1], &tok[i + 2]);
         okay &= colon == &Symbol::SymColon;
         match (field, expression) {
-            (Token::Ident(id), Token::Lit(Lit::Str(s))) => {
-                okay &= data
-                    .insert(unsafe { id.as_slice() }, Some(s.to_string().into()))
-                    .is_none();
-            }
-            (Token::Ident(id), Token::Lit(Lit::UnsignedInt(n))) => {
-                okay &= data
-                    .insert(unsafe { id.as_slice() }, Some((*n).into()))
-                    .is_none();
-            }
-            (Token::Ident(id), Token::Lit(Lit::Bool(b))) => {
-                okay &= data
-                    .insert(unsafe { id.as_slice() }, Some((*b).into()))
-                    .is_none();
+            (Token::Ident(id), Token::Lit(l)) => {
+                let dt = match l {
+                    Lit::Str(s) => s.to_string().into(),
+                    Lit::Bool(b) => (*b).into(),
+                    Lit::UnsignedInt(s) => (*s).into(),
+                    Lit::UnsafeLit(l) => DataType::AnonymousTypeNeedsEval(l.clone()),
+                };
+                okay &= data.insert(unsafe { id.as_slice() }, Some(dt)).is_none();
             }
             (Token::Ident(id), Token::Symbol(Symbol::TtOpenSqBracket)) => {
                 // ooh a list

From d1cba5d8b47579df7ed718ab8f90368987c8f15b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 21 Nov 2022 09:54:07 +0530
Subject: [PATCH 048/310] Support signed integers

---
 server/src/engine/memory/mod.rs | 18 +++++++++---
 server/src/engine/ql/dml.rs     |  5 +++-
 server/src/engine/ql/lexer.rs   | 50 ++++++++++++++++++++++++++++-----
 server/src/engine/ql/tests.rs   | 10 ++++++-
 4 files changed, 70 insertions(+), 13 deletions(-)

diff --git a/server/src/engine/memory/mod.rs b/server/src/engine/memory/mod.rs
index e3f8b28a..a487b7b0 100644
--- a/server/src/engine/memory/mod.rs
+++ b/server/src/engine/memory/mod.rs
@@ -37,15 +37,25 @@ pub enum DataType {
     String(String),
     /// Bytes
     Binary(Vec<u8>),
-    /// An integer
-    Number(u64),
+    /// An unsigned integer
+    ///
+    /// **NOTE:** This is the default evaluated type for unsigned integers by the query processor. It is the
+    /// responsibility of the executor to ensure integrity checks depending on actual type width in the declared
+    /// schema (if any)
+    UnsignedInt(u64),
+    /// A signed integer
+    ///
+    /// **NOTE:** This is the default evaluated type for signed integers by the query processor. It is the
+    /// responsibility of the executor to ensure integrity checks depending on actual type width in the declared
+    /// schema (if any)
+    SignedInt(i64),
     /// A boolean
     Boolean(bool),
     /// A single-type list. Note, you **need** to keep up the invariant that the [`DataType`] disc. remains the same for all
     /// elements to ensure correctness in this specific context
     /// FIXME(@ohsayan): Try enforcing this somehow
     List(Vec<Self>),
-    /// Not an actual data type but MUST be translated into an actual data type
+    /// **☢ WARNING:** Not an actual data type but MUST be translated into an actual data type
     AnonymousTypeNeedsEval(RawSlice),
 }
 
@@ -53,7 +63,7 @@ enum_impls! {
     DataType => {
         String as String,
         Vec<u8> as Binary,
-        u64 as Number,
+        u64 as UnsignedInt,
         bool as Boolean,
         Vec<Self> as List,
         &'static str as String,
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 1c38e23c..af9292d5 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -81,9 +81,10 @@ pub(super) fn parse_list(
         let d = match &tok[i] {
             Token::Lit(l) => match l {
                 Lit::Str(s) => DataType::String(s.to_string()),
-                Lit::UnsignedInt(n) => DataType::Number(*n),
+                Lit::UnsignedInt(n) => DataType::UnsignedInt(*n),
                 Lit::Bool(b) => DataType::Boolean(*b),
                 Lit::UnsafeLit(l) => DataType::AnonymousTypeNeedsEval(l.clone()),
+                Lit::SignedInt(uint) => DataType::SignedInt(*uint),
             },
             Token::Symbol(Symbol::TtOpenSqBracket) => {
                 // a nested list
@@ -152,6 +153,7 @@ pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<Option<DataType>>,
                     data.push(Some((*b).into()));
                 }
                 Lit::UnsafeLit(r) => data.push(Some(DataType::AnonymousTypeNeedsEval(r.clone()))),
+                Lit::SignedInt(int) => data.push(Some(DataType::SignedInt(*int))),
             },
             Token::Symbol(Symbol::TtOpenSqBracket) => {
                 // ah, a list
@@ -208,6 +210,7 @@ pub(super) fn parse_data_map_syntax<'a>(
                     Lit::Bool(b) => (*b).into(),
                     Lit::UnsignedInt(s) => (*s).into(),
                     Lit::UnsafeLit(l) => DataType::AnonymousTypeNeedsEval(l.clone()),
+                    Lit::SignedInt(int) => DataType::SignedInt(*int),
                 };
                 okay &= data.insert(unsafe { id.as_slice() }, Some(dt)).is_none();
             }
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 96a0301a..5ce5ff6e 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -75,6 +75,7 @@ pub enum Lit {
     Str(Box<str>),
     Bool(bool),
     UnsignedInt(u64),
+    SignedInt(i64),
     UnsafeLit(RawSlice),
 }
 
@@ -219,11 +220,11 @@ pub enum Type {
 }
 
 /*
-    This section implements DAGs, as described by Czech et al in their paper. I wrote these pretty much by brute-force using
-    a byte-level multiplicative function (inside a script). This unfortunately implies that every time we *do* need to add a
-    new keyword, I will need to recompute and rewrite the vertices. I don't plan to use any codegen, so I think this is good
-    as-is. The real challenge here is to keep the graph small, and I couldn't do that for the symbols table even with multiple
-    trials. Please see if you can improve them.
+    This section implements LUTs constructed using DAGs, as described by Czech et al in their paper. I wrote these pretty much by
+    brute-force using a byte-level multiplicative function (inside a script). This unfortunately implies that every time we *do*
+    need to add a new keyword, I will need to recompute and rewrite the vertices. I don't plan to use any codegen, so I think
+    this is good as-is. The real challenge here is to keep the graph small, and I couldn't do that for the symbols table even with
+    multiple trials. Please see if you can improve them.
 
     Also the functions are unique to every graph, and every input set, so BE WARNED!
 
@@ -501,7 +502,7 @@ impl<'a> Lexer<'a> {
         }
     }
 
-    fn scan_number(&mut self) {
+    fn scan_unsigned_integer(&mut self) {
         let s = self.cursor();
         unsafe {
             while self.peek_is(|b| b.is_ascii_digit()) {
@@ -615,6 +616,40 @@ impl<'a> Lexer<'a> {
         }
     }
 
+    #[inline(always)]
+    fn scan_signed_integer(&mut self) {
+        unsafe {
+            self.incr_cursor();
+        }
+        if self.peek_is(|b| b.is_ascii_digit()) {
+            // we have some digits
+            let start = unsafe {
+                // UNSAFE(@ohsayan): Take the (-) into the parse
+                // TODO(@ohsayan): we can maybe look at a more efficient way later
+                self.cursor().sub(1)
+            };
+            while self.peek_is_and_forward(|b| b.is_ascii_digit()) {}
+            let wseof = self.peek_is(|char| !char.is_ascii_alphabetic()) || self.exhausted();
+            match unsafe {
+                str::from_utf8_unchecked(slice::from_raw_parts(
+                    start,
+                    self.cursor().offset_from(start) as usize,
+                ))
+            }
+            .parse::<i64>()
+            {
+                Ok(num) if compiler::likely(wseof) => {
+                    self.push_token(Lit::SignedInt(num));
+                }
+                _ => {
+                    compiler::cold_err(self.last_error = Some(LangError::InvalidNumericLiteral));
+                }
+            }
+        } else {
+            self.push_token(Token![-]);
+        }
+    }
+
     fn _lex(&mut self) {
         while self.not_exhausted() && self.last_error.is_none() {
             match unsafe { self.deref_cursor() } {
@@ -628,7 +663,8 @@ impl<'a> Lexer<'a> {
                     }
                 }
                 b'\r' => self.scan_unsafe_literal(),
-                byte if byte.is_ascii_digit() => self.scan_number(),
+                byte if byte.is_ascii_digit() => self.scan_unsigned_integer(),
+                b'-' => self.scan_signed_integer(),
                 qs @ (b'\'' | b'"') => self.scan_quoted_string(qs),
                 b' ' | b'\n' | b'\t' => self.trim_ahead(),
                 b => self.scan_byte(b),
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 5bde3ada..e9c0206f 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -115,7 +115,7 @@ mod lexer_tests {
 
     // literals
     #[test]
-    fn lex_number() {
+    fn lex_unsigned_int() {
         let number = v!("123456");
         assert_eq!(
             lex(&number).unwrap(),
@@ -123,6 +123,14 @@ mod lexer_tests {
         );
     }
     #[test]
+    fn lex_signed_int() {
+        let number = v!("-123456");
+        assert_eq!(
+            lex(&number).unwrap(),
+            vec![Token::Lit(Lit::SignedInt(-123456))]
+        );
+    }
+    #[test]
     fn lex_bool() {
         let (t, f) = v!("true", "false");
         assert_eq!(lex(&t).unwrap(), vec![Token::Lit(Lit::Bool(true))]);

From ddba886a0fc3a6d544f14be7e0ccdae650f2c6d6 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 21 Nov 2022 13:16:21 +0530
Subject: [PATCH 049/310] Add operating modes

---
 server/src/engine/ql/ast.rs   |  4 ++--
 server/src/engine/ql/lexer.rs | 26 +++++++++++++++++++-------
 server/src/engine/ql/tests.rs |  4 ++--
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 0f31253b..217dad97 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -27,7 +27,7 @@
 use {
     super::{
         ddl,
-        lexer::{Lexer, Token},
+        lexer::{InsecureLexer, Token},
         schema, LangError, LangResult, RawSlice,
     },
     crate::util::Life,
@@ -208,7 +208,7 @@ pub struct Compiler<'a> {
 impl<'a> Compiler<'a> {
     /// Compile a BlueQL query
     pub fn compile(src: &'a [u8]) -> LangResult<Life<'a, Statement>> {
-        let token_stream = Lexer::lex(src)?;
+        let token_stream = InsecureLexer::lex(src)?;
         Self::new(&token_stream).compile_link_lt()
     }
     #[inline(always)]
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 5ce5ff6e..376e5c6f 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -378,7 +378,12 @@ fn kwof(key: &str) -> Option<Keyword> {
     Lexer impl
 */
 
-pub struct Lexer<'a> {
+const LEXER_MODE_INSECURE: u8 = 0;
+const LEXER_MODE_SECURE: u8 = 1;
+
+pub type InsecureLexer<'a> = Lexer<'a, LEXER_MODE_INSECURE>;
+
+pub struct Lexer<'a, const OPERATING_MODE: u8> {
     c: *const u8,
     e: *const u8,
     last_error: Option<LangError>,
@@ -386,7 +391,7 @@ pub struct Lexer<'a> {
     _lt: PhantomData<&'a [u8]>,
 }
 
-impl<'a> Lexer<'a> {
+impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
     pub const fn new(src: &'a [u8]) -> Self {
         unsafe {
             Self {
@@ -401,7 +406,7 @@ impl<'a> Lexer<'a> {
 }
 
 // meta
-impl<'a> Lexer<'a> {
+impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
     #[inline(always)]
     const fn cursor(&self) -> *const u8 {
         self.c
@@ -480,7 +485,7 @@ impl<'a> Lexer<'a> {
     }
 }
 
-impl<'a> Lexer<'a> {
+impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
     fn scan_ident(&mut self) -> RawSlice {
         let s = self.cursor();
         unsafe {
@@ -653,6 +658,7 @@ impl<'a> Lexer<'a> {
     fn _lex(&mut self) {
         while self.not_exhausted() && self.last_error.is_none() {
             match unsafe { self.deref_cursor() } {
+                // secure features
                 byte if byte.is_ascii_alphabetic() => self.scan_ident_or_keyword(),
                 #[cfg(test)]
                 byte if byte == b'\x01' => {
@@ -663,9 +669,15 @@ impl<'a> Lexer<'a> {
                     }
                 }
                 b'\r' => self.scan_unsafe_literal(),
-                byte if byte.is_ascii_digit() => self.scan_unsigned_integer(),
-                b'-' => self.scan_signed_integer(),
-                qs @ (b'\'' | b'"') => self.scan_quoted_string(qs),
+                // insecure features
+                byte if byte.is_ascii_digit() && OPERATING_MODE == LEXER_MODE_INSECURE => {
+                    self.scan_unsigned_integer()
+                }
+                b'-' if OPERATING_MODE == LEXER_MODE_INSECURE => self.scan_signed_integer(),
+                qs @ (b'\'' | b'"') if OPERATING_MODE == LEXER_MODE_INSECURE => {
+                    self.scan_quoted_string(qs)
+                }
+                // blank space or an arbitrary byte
                 b' ' | b'\n' | b'\t' => self.trim_ahead(),
                 b => self.scan_byte(b),
             }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index e9c0206f..7605aa4e 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -26,14 +26,14 @@
 
 use {
     super::{
-        lexer::{Lexer, Token},
+        lexer::{InsecureLexer, Token},
         LangResult,
     },
     crate::{engine::memory::DataType, util::Life},
 };
 
 pub(super) fn lex(src: &[u8]) -> LangResult<Life<Vec<Token>>> {
-    Lexer::lex(src)
+    InsecureLexer::lex(src)
 }
 
 pub trait NullableData<T> {

From 7e5e2838cc907bc5b8db54e5a4920018c68d042c Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 30 Nov 2022 23:47:09 +0530
Subject: [PATCH 050/310] Add support for relational expressions

---
 server/src/engine/ql/ddl.rs    |  4 +-
 server/src/engine/ql/dml.rs    | 87 ++++++++++++++++++++++++++++++++--
 server/src/engine/ql/lexer.rs  |  6 +--
 server/src/engine/ql/macros.rs |  7 +++
 server/src/engine/ql/mod.rs    |  4 ++
 server/src/engine/ql/tests.rs  | 85 +++++++++++++++++++++++++++++++++
 6 files changed, 184 insertions(+), 9 deletions(-)

diff --git a/server/src/engine/ql/ddl.rs b/server/src/engine/ql/ddl.rs
index 8bc57d65..daad8546 100644
--- a/server/src/engine/ql/ddl.rs
+++ b/server/src/engine/ql/ddl.rs
@@ -96,7 +96,7 @@ pub(super) fn parse_drop(tok: &[Token], counter: &mut usize) -> LangResult<State
 pub(super) fn parse_drop_full(tok: &[Token]) -> LangResult<Statement> {
     let mut i = 0;
     let r = self::parse_drop(tok, &mut i);
-    assert_eq!(i, tok.len(), "full token stream not utilized");
+    full_tt!(i, tok.len());
     r
 }
 
@@ -130,6 +130,6 @@ pub(super) fn parse_inspect(tok: &[Token], c: &mut usize) -> LangResult<Statemen
 pub(super) fn parse_inspect_full(tok: &[Token]) -> LangResult<Statement> {
     let mut i = 0;
     let r = self::parse_inspect(tok, &mut i);
-    assert_eq!(i, tok.len(), "full token stream not used");
+    full_tt!(i, tok.len());
     r
 }
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index af9292d5..0cecc948 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -35,7 +35,10 @@ use {
         lexer::{Lit, Symbol, Token},
         LangError, LangResult, RawSlice,
     },
-    crate::{engine::memory::DataType, util::MaybeInit},
+    crate::{
+        engine::memory::DataType,
+        util::{compiler, MaybeInit},
+    },
     std::{
         collections::HashMap,
         mem::{discriminant, Discriminant},
@@ -536,7 +539,7 @@ pub(super) fn parse_expression_full<'a>(tok: &'a [Token]) -> Option<AssignmentEx
     let mut i = 0;
     let mut exprs = Vec::new();
     if AssignmentExpression::parse_and_append_expression(tok, &mut exprs, &mut i) {
-        assert_eq!(i, tok.len(), "full token stream not utilized");
+        full_tt!(i, tok.len());
         Some(exprs.remove(0))
     } else {
         None
@@ -606,7 +609,7 @@ impl<'a> UpdateStatement<'a> {
 pub(super) fn parse_update_full<'a>(tok: &'a [Token]) -> LangResult<UpdateStatement<'a>> {
     let mut i = 0;
     let r = UpdateStatement::parse_update(tok, &mut i);
-    assert_eq!(i, tok.len(), "full token stream not utilized");
+    full_tt!(i, tok.len());
     r
 }
 
@@ -666,6 +669,82 @@ impl<'a> DeleteStatement<'a> {
 pub(super) fn parse_delete_full<'a>(tok: &'a [Token]) -> LangResult<DeleteStatement<'a>> {
     let mut i = 0_usize;
     let r = DeleteStatement::parse_delete(tok, &mut i);
-    assert_eq!(i, tok.len());
+    full_tt!(i, tok.len());
     r
 }
+
+#[derive(Debug, PartialEq)]
+pub(super) struct RelationalExpr<'a> {
+    pub(super) lhs: RawSlice,
+    pub(super) rhs: &'a Lit,
+    pub(super) opc: u8,
+}
+
+impl<'a> RelationalExpr<'a> {
+    pub(super) const OP_EQ: u8 = 1;
+    pub(super) const OP_NE: u8 = 2;
+    pub(super) const OP_GT: u8 = 3;
+    pub(super) const OP_GE: u8 = 4;
+    pub(super) const OP_LT: u8 = 5;
+    pub(super) const OP_LE: u8 = 6;
+    #[inline(always)]
+    fn parse_operator(tok: &[Token], i: &mut usize, okay: &mut bool) -> u8 {
+        /*
+            FIXME(@ohsayan): This is relatively messy right now, but does the job. Will
+            re-implement later.
+        */
+        #[inline(always)]
+        fn u(b: bool) -> u8 {
+            b as _
+        }
+        let op_eq = u(tok[0] == Token![=]) * Self::OP_EQ;
+        let op_ne = u(tok[0] == Token![!] && tok[1] == Token![=]) * Self::OP_NE;
+        let op_ge = u(tok[0] == Token![>] && tok[1] == Token![=]) * Self::OP_GE;
+        let op_gt = u(tok[0] == Token![>] && op_ge == 0) * Self::OP_GT;
+        let op_le = u(tok[0] == Token![<] && tok[1] == Token![=]) * Self::OP_LE;
+        let op_lt = u(tok[0] == Token![<] && op_le == 0) * Self::OP_LT;
+        let opc = op_eq + op_ne + op_ge + op_gt + op_le + op_lt;
+        *okay = opc != 0;
+        *i += 1 + (opc & 1 == 0) as usize;
+        opc
+    }
+    #[inline(always)]
+    fn try_parse(tok: &'a [Token], cnt: &mut usize) -> Option<Self> {
+        /*
+            Minimum length of an expression:
+            [lhs] [operator] [rhs]
+        */
+        let mut okay = tok.len() >= 3;
+        let mut i = 0_usize;
+        if compiler::unlikely(!okay) {
+            return None;
+        }
+        okay &= tok[0].is_ident();
+        i += 1;
+        // let's get ourselves the operator
+        let operator = Self::parse_operator(&tok[1..], &mut i, &mut okay);
+        okay &= i < tok.len();
+        okay &= tok[tok.len() - 1].is_lit(); // LOL, I really like saving cycles
+        *cnt += i + okay as usize;
+        if compiler::likely(okay) {
+            Some(unsafe {
+                Self {
+                    lhs: extract!(tok[0], Token::Ident(ref id) => id.clone()),
+                    rhs: extract!(tok[tok.len() - 1], Token::Lit(ref l) => l),
+                    opc: operator,
+                }
+            })
+        } else {
+            compiler::cold_err(None)
+        }
+    }
+}
+
+#[cfg(test)]
+#[inline(always)]
+pub(super) fn parse_relexpr_full<'a>(tok: &'a [Token]) -> Option<RelationalExpr<'a>> {
+    let mut i = 0;
+    let okay = RelationalExpr::try_parse(tok, &mut i);
+    full_tt!(tok.len(), i);
+    okay
+}
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 376e5c6f..3bc160c8 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -696,11 +696,11 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
 
 impl Token {
     #[inline(always)]
-    pub(crate) fn is_ident(&self) -> bool {
+    pub(crate) const fn is_ident(&self) -> bool {
         matches!(self, Token::Ident(_))
     }
     #[inline(always)]
-    pub(crate) fn is_typeid(&self) -> bool {
+    pub(crate) const fn is_typeid(&self) -> bool {
         matches!(self, Token::Keyword(Keyword::TypeId(_)))
     }
     #[inline(always)]
@@ -723,7 +723,7 @@ impl Token {
         }
     }
     #[inline(always)]
-    pub(super) fn is_lit(&self) -> bool {
+    pub(super) const fn is_lit(&self) -> bool {
         matches!(self, Self::Lit(_))
     }
 }
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 197319cb..c24805ef 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -24,6 +24,13 @@
  *
 */
 
+#[cfg(test)]
+macro_rules! full_tt {
+    ($a:expr, $b:expr) => {
+        assert_eq!($a, $b, "full token stream not utilized")
+    };
+}
+
 macro_rules! __sym_token {
     ($ident:ident) => {
         $crate::engine::ql::lexer::Token::Symbol($crate::engine::ql::lexer::Symbol::$ident)
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 7deca32f..688ec990 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -88,9 +88,13 @@ unsafe impl Sync for RawSlice {}
 
 impl RawSlice {
     const _EALIGN: () = assert!(mem::align_of::<Self>() == mem::align_of::<&[u8]>());
+    const FAKE_SLICE: Self = unsafe { Self::new_from_str("") };
     const unsafe fn new(ptr: *const u8, len: usize) -> Self {
         Self { ptr, len }
     }
+    const unsafe fn new_from_str(s: &str) -> Self {
+        Self::new(s.as_bytes().as_ptr(), s.as_bytes().len())
+    }
     unsafe fn as_slice(&self) -> &[u8] {
         slice::from_raw_parts(self.ptr, self.len)
     }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 7605aa4e..34b61553 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -2325,6 +2325,91 @@ mod dml_tests {
             assert_eq!(r, e);
         }
     }
+    mod relational_expr {
+        use {
+            super::*,
+            crate::engine::ql::dml::{self, RelationalExpr},
+        };
+
+        #[test]
+        fn expr_eq() {
+            let expr = lex(b"primary_key = 10").unwrap();
+            let r = dml::parse_relexpr_full(&expr).unwrap();
+            assert_eq!(
+                r,
+                RelationalExpr {
+                    rhs: &(10.into()),
+                    lhs: "primary_key".into(),
+                    opc: RelationalExpr::OP_EQ
+                }
+            );
+        }
+        #[test]
+        fn expr_ne() {
+            let expr = lex(b"primary_key != 10").unwrap();
+            let r = dml::parse_relexpr_full(&expr).unwrap();
+            assert_eq!(
+                r,
+                RelationalExpr {
+                    rhs: &(10.into()),
+                    lhs: "primary_key".into(),
+                    opc: RelationalExpr::OP_NE
+                }
+            );
+        }
+        #[test]
+        fn expr_gt() {
+            let expr = lex(b"primary_key > 10").unwrap();
+            let r = dml::parse_relexpr_full(&expr).unwrap();
+            assert_eq!(
+                r,
+                RelationalExpr {
+                    rhs: &(10.into()),
+                    lhs: "primary_key".into(),
+                    opc: RelationalExpr::OP_GT
+                }
+            );
+        }
+        #[test]
+        fn expr_ge() {
+            let expr = lex(b"primary_key >= 10").unwrap();
+            let r = dml::parse_relexpr_full(&expr).unwrap();
+            assert_eq!(
+                r,
+                RelationalExpr {
+                    rhs: &(10.into()),
+                    lhs: "primary_key".into(),
+                    opc: RelationalExpr::OP_GE
+                }
+            );
+        }
+        #[test]
+        fn expr_lt() {
+            let expr = lex(b"primary_key < 10").unwrap();
+            let r = dml::parse_relexpr_full(&expr).unwrap();
+            assert_eq!(
+                r,
+                RelationalExpr {
+                    rhs: &(10.into()),
+                    lhs: "primary_key".into(),
+                    opc: RelationalExpr::OP_LT
+                }
+            );
+        }
+        #[test]
+        fn expr_le() {
+            let expr = lex(b"primary_key <= 10").unwrap();
+            let r = dml::parse_relexpr_full(&expr).unwrap();
+            assert_eq!(
+                r,
+                RelationalExpr {
+                    rhs: &(10.into()),
+                    lhs: "primary_key".into(),
+                    opc: RelationalExpr::OP_LE
+                }
+            );
+        }
+    }
 }
 
 mod nullable_dict_tests {

From cac7bd4860879034eef50db969b2b2f33f439e2d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 2 Dec 2022 10:44:36 +0530
Subject: [PATCH 051/310] Add new keywords

---
 server/src/engine/ql/ast.rs    |   2 +-
 server/src/engine/ql/dml.rs    | 156 ++++++++++---------
 server/src/engine/ql/lexer.rs  | 276 ++++++++++++++++-----------------
 server/src/engine/ql/macros.rs | 124 +++++----------
 server/src/engine/ql/schema.rs |  47 ++----
 server/src/engine/ql/tests.rs  | 176 ++++++++++-----------
 6 files changed, 354 insertions(+), 427 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 217dad97..410516b9 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -239,7 +239,7 @@ impl<'a> Compiler<'a> {
             Some(Token![create]) => self.create0(),
             Some(Token![drop]) => self.drop0(),
             Some(Token![alter]) => self.alter0(),
-            Some(Token![inspect]) => self.inspect0(),
+            Some(Token![describe]) => self.inspect0(),
             Some(Token![use]) => self.use0(),
             _ => Err(LangError::ExpectedStatement),
         }
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 0cecc948..7e579233 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -63,6 +63,86 @@ fn process_entity(tok: &[Token], d: &mut MaybeInit<Entity>, i: &mut usize) -> bo
     is_full | is_single
 }
 
+/*
+    Contexts
+*/
+
+#[derive(Debug, PartialEq)]
+pub(super) struct RelationalExpr<'a> {
+    pub(super) lhs: RawSlice,
+    pub(super) rhs: &'a Lit,
+    pub(super) opc: u8,
+}
+
+impl<'a> RelationalExpr<'a> {
+    pub(super) const OP_EQ: u8 = 1;
+    pub(super) const OP_NE: u8 = 2;
+    pub(super) const OP_GT: u8 = 3;
+    pub(super) const OP_GE: u8 = 4;
+    pub(super) const OP_LT: u8 = 5;
+    pub(super) const OP_LE: u8 = 6;
+    #[inline(always)]
+    fn parse_operator(tok: &[Token], i: &mut usize, okay: &mut bool) -> u8 {
+        /*
+            FIXME(@ohsayan): This is relatively messy right now, but does the job. Will
+            re-implement later.
+        */
+        #[inline(always)]
+        fn u(b: bool) -> u8 {
+            b as _
+        }
+        let op_eq = u(tok[0] == Token![=]) * Self::OP_EQ;
+        let op_ne = u(tok[0] == Token![!] && tok[1] == Token![=]) * Self::OP_NE;
+        let op_ge = u(tok[0] == Token![>] && tok[1] == Token![=]) * Self::OP_GE;
+        let op_gt = u(tok[0] == Token![>] && op_ge == 0) * Self::OP_GT;
+        let op_le = u(tok[0] == Token![<] && tok[1] == Token![=]) * Self::OP_LE;
+        let op_lt = u(tok[0] == Token![<] && op_le == 0) * Self::OP_LT;
+        let opc = op_eq + op_ne + op_ge + op_gt + op_le + op_lt;
+        *okay = opc != 0;
+        *i += 1 + (opc & 1 == 0) as usize;
+        opc
+    }
+    #[inline(always)]
+    fn try_parse(tok: &'a [Token], cnt: &mut usize) -> Option<Self> {
+        /*
+            Minimum length of an expression:
+            [lhs] [operator] [rhs]
+        */
+        let mut okay = tok.len() >= 3;
+        let mut i = 0_usize;
+        if compiler::unlikely(!okay) {
+            return None;
+        }
+        okay &= tok[0].is_ident();
+        i += 1;
+        // let's get ourselves the operator
+        let operator = Self::parse_operator(&tok[1..], &mut i, &mut okay);
+        okay &= i < tok.len();
+        okay &= tok[tok.len() - 1].is_lit(); // LOL, I really like saving cycles
+        *cnt += i + okay as usize;
+        if compiler::likely(okay) {
+            Some(unsafe {
+                Self {
+                    lhs: extract!(tok[0], Token::Ident(ref id) => id.clone()),
+                    rhs: extract!(tok[tok.len() - 1], Token::Lit(ref l) => l),
+                    opc: operator,
+                }
+            })
+        } else {
+            compiler::cold_err(None)
+        }
+    }
+}
+
+#[cfg(test)]
+#[inline(always)]
+pub(super) fn parse_relexpr_full<'a>(tok: &'a [Token]) -> Option<RelationalExpr<'a>> {
+    let mut i = 0;
+    let okay = RelationalExpr::try_parse(tok, &mut i);
+    full_tt!(tok.len(), i);
+    okay
+}
+
 /*
     Impls for insert
 */
@@ -672,79 +752,3 @@ pub(super) fn parse_delete_full<'a>(tok: &'a [Token]) -> LangResult<DeleteStatem
     full_tt!(i, tok.len());
     r
 }
-
-#[derive(Debug, PartialEq)]
-pub(super) struct RelationalExpr<'a> {
-    pub(super) lhs: RawSlice,
-    pub(super) rhs: &'a Lit,
-    pub(super) opc: u8,
-}
-
-impl<'a> RelationalExpr<'a> {
-    pub(super) const OP_EQ: u8 = 1;
-    pub(super) const OP_NE: u8 = 2;
-    pub(super) const OP_GT: u8 = 3;
-    pub(super) const OP_GE: u8 = 4;
-    pub(super) const OP_LT: u8 = 5;
-    pub(super) const OP_LE: u8 = 6;
-    #[inline(always)]
-    fn parse_operator(tok: &[Token], i: &mut usize, okay: &mut bool) -> u8 {
-        /*
-            FIXME(@ohsayan): This is relatively messy right now, but does the job. Will
-            re-implement later.
-        */
-        #[inline(always)]
-        fn u(b: bool) -> u8 {
-            b as _
-        }
-        let op_eq = u(tok[0] == Token![=]) * Self::OP_EQ;
-        let op_ne = u(tok[0] == Token![!] && tok[1] == Token![=]) * Self::OP_NE;
-        let op_ge = u(tok[0] == Token![>] && tok[1] == Token![=]) * Self::OP_GE;
-        let op_gt = u(tok[0] == Token![>] && op_ge == 0) * Self::OP_GT;
-        let op_le = u(tok[0] == Token![<] && tok[1] == Token![=]) * Self::OP_LE;
-        let op_lt = u(tok[0] == Token![<] && op_le == 0) * Self::OP_LT;
-        let opc = op_eq + op_ne + op_ge + op_gt + op_le + op_lt;
-        *okay = opc != 0;
-        *i += 1 + (opc & 1 == 0) as usize;
-        opc
-    }
-    #[inline(always)]
-    fn try_parse(tok: &'a [Token], cnt: &mut usize) -> Option<Self> {
-        /*
-            Minimum length of an expression:
-            [lhs] [operator] [rhs]
-        */
-        let mut okay = tok.len() >= 3;
-        let mut i = 0_usize;
-        if compiler::unlikely(!okay) {
-            return None;
-        }
-        okay &= tok[0].is_ident();
-        i += 1;
-        // let's get ourselves the operator
-        let operator = Self::parse_operator(&tok[1..], &mut i, &mut okay);
-        okay &= i < tok.len();
-        okay &= tok[tok.len() - 1].is_lit(); // LOL, I really like saving cycles
-        *cnt += i + okay as usize;
-        if compiler::likely(okay) {
-            Some(unsafe {
-                Self {
-                    lhs: extract!(tok[0], Token::Ident(ref id) => id.clone()),
-                    rhs: extract!(tok[tok.len() - 1], Token::Lit(ref l) => l),
-                    opc: operator,
-                }
-            })
-        } else {
-            compiler::cold_err(None)
-        }
-    }
-}
-
-#[cfg(test)]
-#[inline(always)]
-pub(super) fn parse_relexpr_full<'a>(tok: &'a [Token]) -> Option<RelationalExpr<'a>> {
-    let mut i = 0;
-    let okay = RelationalExpr::try_parse(tok, &mut i);
-    full_tt!(tok.len(), i);
-    okay
-}
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 3bc160c8..32a6c64a 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -57,7 +57,7 @@ impl PartialEq<Symbol> for Token {
 assertions! {
     size_of::<Token>() == 24, // FIXME(@ohsayan): Damn, what?
     size_of::<Symbol>() == 1,
-    size_of::<Keyword>() == 2,
+    size_of::<Keyword>() == 1,
     size_of::<Lit>() == 24, // FIXME(@ohsayan): Ouch
 }
 
@@ -130,93 +130,69 @@ pub enum Symbol {
     SymAccent,        // `
 }
 
-#[derive(Debug, Copy, Clone, PartialEq)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
 #[repr(u8)]
 pub enum Keyword {
-    Ddl(DdlKeyword),
-    DdlMisc(DdlMiscKeyword),
-    Dml(DmlKeyword),
-    DmlMisc(DmlMiscKeyword),
-    TypeId(Type),
-    Misc(MiscKeyword),
-}
-
-#[derive(Debug, Copy, Clone, PartialEq)]
-pub enum MiscKeyword {
-    Null,
-}
-
-enum_impls! {
-    Keyword => {
-        DdlKeyword as Ddl,
-        DdlMiscKeyword as DdlMisc,
-        DmlKeyword as Dml,
-        DmlMiscKeyword as DmlMisc,
-        Type as TypeId,
-        MiscKeyword as Misc,
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-#[repr(u8)]
-pub enum DmlMiscKeyword {
-    Limit,
-    From,
-    Into,
-    Where,
-    If,
-    And,
-    As,
-    By,
-    Asc,
-    Desc,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-#[repr(u8)]
-pub enum DmlKeyword {
-    Insert,
-    Select,
-    Update,
-    Delete,
-    Exists,
-    Truncate,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-#[repr(u8)]
-pub enum DdlMiscKeyword {
-    With,
-    Add,
-    Remove,
-    Sort,
+    Table,
+    Model,
+    Space,
+    Index,
     Type,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-#[repr(u8)]
-pub enum DdlKeyword {
+    Function,
     Use,
     Create,
     Alter,
     Drop,
-    Inspect,
-    Model,
-    Space,
+    Describe,
+    Truncate,
+    Rename,
+    Add,
+    Remove,
+    Transform,
+    Order,
+    By,
     Primary,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-#[repr(u8)]
-pub enum Type {
-    String,
-    Binary,
-    List,
-    Map,
-    Bool,
-    Int,
-    Double,
-    Float,
+    Key,
+    Value,
+    With,
+    On,
+    Lock,
+    All,
+    Insert,
+    Select,
+    Exists,
+    Update,
+    Delere,
+    Into,
+    From,
+    As,
+    Return,
+    Sort,
+    Group,
+    Limit,
+    Asc,
+    Desc,
+    To,
+    Set,
+    Auto,
+    Default,
+    In,
+    Of,
+    Transaction,
+    Batch,
+    Read,
+    Write,
+    Begin,
+    End,
+    Where,
+    If,
+    And,
+    Or,
+    Not,
+    User,
+    Revoke,
+    Null,
+    Infinity,
 }
 
 /*
@@ -241,7 +217,7 @@ static SYM_GRAPH: [u8; 69] = [
     23, 7, 0, 27, 0, 4, 16, 11, 0, 0, 9,
 ];
 
-static SYM_DATA: [(u8, Symbol); 32] = [
+static SYM_LUT: [(u8, Symbol); 32] = [
     (b'+', Symbol::OpArithmeticAdd),
     (b'-', Symbol::OpArithmeticSub),
     (b'*', Symbol::OpArithmeticMul),
@@ -290,85 +266,107 @@ fn symph(k: u8) -> u8 {
 #[inline(always)]
 fn symof(sym: u8) -> Option<Symbol> {
     let hf = symph(sym);
-    if hf < SYM_DATA.len() as u8 && SYM_DATA[hf as usize].0 == sym {
-        Some(SYM_DATA[hf as usize].1)
+    if hf < SYM_LUT.len() as u8 && SYM_LUT[hf as usize].0 == sym {
+        Some(SYM_LUT[hf as usize].1)
     } else {
         None
     }
 }
 
-static KW_GRAPH: [u8; 40] = [
-    0, 2, 32, 18, 4, 37, 11, 27, 34, 35, 26, 33, 0, 0, 10, 2, 22, 8, 5, 7, 16, 9, 8, 39, 21, 5, 0,
-    22, 14, 19, 22, 31, 28, 38, 26, 21, 30, 24, 10, 18,
+static KW_LUT: [(&[u8], Keyword); 60] = [
+    (b"table", Keyword::Table),
+    (b"model", Keyword::Model),
+    (b"space", Keyword::Space),
+    (b"index", Keyword::Index),
+    (b"type", Keyword::Type),
+    (b"function", Keyword::Function),
+    (b"use", Keyword::Use),
+    (b"create", Keyword::Create),
+    (b"alter", Keyword::Alter),
+    (b"drop", Keyword::Drop),
+    (b"describe", Keyword::Describe),
+    (b"truncate", Keyword::Truncate),
+    (b"rename", Keyword::Rename),
+    (b"add", Keyword::Add),
+    (b"remove", Keyword::Remove),
+    (b"transform", Keyword::Transform),
+    (b"order", Keyword::Order),
+    (b"by", Keyword::By),
+    (b"primary", Keyword::Primary),
+    (b"key", Keyword::Key),
+    (b"value", Keyword::Value),
+    (b"with", Keyword::With),
+    (b"on", Keyword::On),
+    (b"lock", Keyword::Lock),
+    (b"all", Keyword::All),
+    (b"insert", Keyword::Insert),
+    (b"select", Keyword::Select),
+    (b"exists", Keyword::Exists),
+    (b"update", Keyword::Update),
+    (b"delere", Keyword::Delere),
+    (b"into", Keyword::Into),
+    (b"from", Keyword::From),
+    (b"as", Keyword::As),
+    (b"return", Keyword::Return),
+    (b"sort", Keyword::Sort),
+    (b"group", Keyword::Group),
+    (b"limit", Keyword::Limit),
+    (b"asc", Keyword::Asc),
+    (b"desc", Keyword::Desc),
+    (b"to", Keyword::To),
+    (b"set", Keyword::Set),
+    (b"auto", Keyword::Auto),
+    (b"default", Keyword::Default),
+    (b"in", Keyword::In),
+    (b"of", Keyword::Of),
+    (b"transaction", Keyword::Transaction),
+    (b"batch", Keyword::Batch),
+    (b"read", Keyword::Read),
+    (b"write", Keyword::Write),
+    (b"begin", Keyword::Begin),
+    (b"end", Keyword::End),
+    (b"where", Keyword::Where),
+    (b"if", Keyword::If),
+    (b"and", Keyword::And),
+    (b"or", Keyword::Or),
+    (b"not", Keyword::Not),
+    (b"user", Keyword::User),
+    (b"revoke", Keyword::Revoke),
+    (b"null", Keyword::Null),
+    (b"infinity", Keyword::Infinity),
 ];
 
-static KW_DATA: [(&str, Keyword); 38] = [
-    ("use", Keyword::Ddl(DdlKeyword::Use)),
-    ("create", Keyword::Ddl(DdlKeyword::Create)),
-    ("alter", Keyword::Ddl(DdlKeyword::Alter)),
-    ("drop", Keyword::Ddl(DdlKeyword::Drop)),
-    ("inspect", Keyword::Ddl(DdlKeyword::Inspect)),
-    ("model", Keyword::Ddl(DdlKeyword::Model)),
-    ("space", Keyword::Ddl(DdlKeyword::Space)),
-    ("primary", Keyword::Ddl(DdlKeyword::Primary)),
-    ("with", Keyword::DdlMisc(DdlMiscKeyword::With)),
-    ("add", Keyword::DdlMisc(DdlMiscKeyword::Add)),
-    ("remove", Keyword::DdlMisc(DdlMiscKeyword::Remove)),
-    ("sort", Keyword::DdlMisc(DdlMiscKeyword::Sort)),
-    ("type", Keyword::DdlMisc(DdlMiscKeyword::Type)),
-    ("insert", Keyword::Dml(DmlKeyword::Insert)),
-    ("select", Keyword::Dml(DmlKeyword::Select)),
-    ("update", Keyword::Dml(DmlKeyword::Update)),
-    ("delete", Keyword::Dml(DmlKeyword::Delete)),
-    ("exists", Keyword::Dml(DmlKeyword::Exists)),
-    ("truncate", Keyword::Dml(DmlKeyword::Truncate)),
-    ("limit", Keyword::DmlMisc(DmlMiscKeyword::Limit)),
-    ("from", Keyword::DmlMisc(DmlMiscKeyword::From)),
-    ("into", Keyword::DmlMisc(DmlMiscKeyword::Into)),
-    ("where", Keyword::DmlMisc(DmlMiscKeyword::Where)),
-    ("if", Keyword::DmlMisc(DmlMiscKeyword::If)),
-    ("and", Keyword::DmlMisc(DmlMiscKeyword::And)),
-    ("as", Keyword::DmlMisc(DmlMiscKeyword::As)),
-    ("by", Keyword::DmlMisc(DmlMiscKeyword::By)),
-    ("asc", Keyword::DmlMisc(DmlMiscKeyword::Asc)),
-    ("desc", Keyword::DmlMisc(DmlMiscKeyword::Desc)),
-    ("string", Keyword::TypeId(Type::String)),
-    ("binary", Keyword::TypeId(Type::Binary)),
-    ("list", Keyword::TypeId(Type::List)),
-    ("map", Keyword::TypeId(Type::Map)),
-    ("bool", Keyword::TypeId(Type::Bool)),
-    ("int", Keyword::TypeId(Type::Int)),
-    ("double", Keyword::TypeId(Type::Double)),
-    ("float", Keyword::TypeId(Type::Float)),
-    ("null", Keyword::Misc(MiscKeyword::Null)),
+static KWG: [u8; 64] = [
+    0, 55, 32, 25, 4, 21, 51, 43, 28, 59, 34, 1, 9, 39, 5, 49, 0, 16, 29, 0, 48, 0, 17, 60, 19, 21,
+    26, 18, 0, 41, 55, 10, 48, 62, 55, 35, 56, 18, 29, 41, 5, 46, 25, 52, 32, 26, 27, 17, 61, 60,
+    61, 59, 24, 12, 17, 30, 53, 4, 17, 0, 6, 2, 45, 56,
 ];
 
-const KW_MAGIC_A: &[u8] = b"GSggb8qI";
-const KW_MAGIC_B: &[u8] = b"ZaljIeOx";
-const KW_MODULUS: usize = 8;
+const KWMG_1: [u8; 11] = *b"nJEcjrLflKX";
+const KWMG_2: [u8; 11] = *b"KWHPUPK3Fh3";
+const KWMG_S: usize = KWMG_1.len();
 
-#[inline(always)]
-fn kwfh(k: &[u8], magic: &[u8]) -> u32 {
+fn kwhf(k: &[u8], mg: &[u8]) -> u32 {
     let mut i = 0;
     let mut s = 0;
     while i < k.len() {
-        s += magic[(i % KW_MODULUS) as usize] as u32 * k[i] as u32;
+        s += mg[(i % KWMG_S) as usize] as u32 * k[i] as u32;
         i += 1;
     }
-    s % KW_GRAPH.len() as u32
+    s % KWG.len() as u32
 }
 
 #[inline(always)]
 fn kwph(k: &[u8]) -> u8 {
-    (KW_GRAPH[kwfh(k, KW_MAGIC_A) as usize] + KW_GRAPH[kwfh(k, KW_MAGIC_B) as usize])
-        % KW_GRAPH.len() as u8
+    (KWG[kwhf(k, &KWMG_1) as usize] + KWG[kwhf(k, &KWMG_2) as usize]) % KWG.len() as u8
 }
 
 #[inline(always)]
 fn kwof(key: &str) -> Option<Keyword> {
-    let ph = kwph(key.as_bytes());
-    if ph < KW_DATA.len() as u8 && KW_DATA[ph as usize].0 == key {
-        Some(KW_DATA[ph as usize].1)
+    let key = key.as_bytes();
+    let ph = kwph(key);
+    if ph < KW_LUT.len() as u8 && KW_LUT[ph as usize].0 == key {
+        Some(KW_LUT[ph as usize].1)
     } else {
         None
     }
@@ -700,10 +698,6 @@ impl Token {
         matches!(self, Token::Ident(_))
     }
     #[inline(always)]
-    pub(crate) const fn is_typeid(&self) -> bool {
-        matches!(self, Token::Keyword(Keyword::TypeId(_)))
-    }
-    #[inline(always)]
     pub(crate) fn as_ident_eq_ignore_case(&self, arg: &[u8]) -> bool {
         self.is_ident()
             && unsafe {
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index c24805ef..f55b9dd8 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -37,51 +37,9 @@ macro_rules! __sym_token {
     };
 }
 
-macro_rules! __ddl_token {
+macro_rules! __kw {
     ($ident:ident) => {
-        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::Ddl(
-            $crate::engine::ql::lexer::DdlKeyword::$ident,
-        ))
-    };
-}
-
-macro_rules! __ddl_misc_token {
-    ($ident:ident) => {
-        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::DdlMisc(
-            $crate::engine::ql::lexer::DdlMiscKeyword::$ident,
-        ))
-    };
-}
-
-macro_rules! __dml_token {
-    ($ident:ident) => {
-        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::Dml(
-            $crate::engine::ql::lexer::DmlKeyword::$ident,
-        ))
-    };
-}
-
-macro_rules! __dml_misc_token {
-    ($ident:ident) => {
-        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::DmlMisc(
-            $crate::engine::ql::lexer::DmlMiscKeyword::$ident,
-        ))
-    };
-}
-
-macro_rules! __type_token {
-    ($ident:ident) => {
-        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::TypeId(
-            $crate::engine::ql::lexer::Type::$ident,
-        ))
-    };
-}
-
-macro_rules! __misc_token {
-    ($ident:ident) => {
-        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::Misc(
-            $crate::engine::ql::lexer::MiscKeyword::$ident,
-        ))
+        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::$ident)
     };
 }
 
@@ -163,119 +121,119 @@ macro_rules! Token {
     };
     // ddl keywords
     (use) => {
-        __ddl_token!(Use)
+        __kw!(Use)
     };
     (create) => {
-        __ddl_token!(Create)
+        __kw!(Create)
     };
     (alter) => {
-        __ddl_token!(Alter)
+        __kw!(Alter)
     };
     (drop) => {
-        __ddl_token!(Drop)
+        __kw!(Drop)
     };
-    (inspect) => {
-        __ddl_token!(Inspect)
+    (describe) => {
+        __kw!(Describe)
     };
     (model) => {
-        __ddl_token!(Model)
+        __kw!(Model)
     };
     (space) => {
-        __ddl_token!(Space)
+        __kw!(Space)
     };
     (primary) => {
-        __ddl_token!(Primary)
+        __kw!(Primary)
     };
     // ddl misc
     (with) => {
-        __ddl_misc_token!(With)
+        __kw!(With)
     };
     (add) => {
-        __ddl_misc_token!(Add)
+        __kw!(Add)
     };
     (remove) => {
-        __ddl_misc_token!(Remove)
+        __kw!(Remove)
     };
     (sort) => {
-        __ddl_misc_token!(Sort)
+        __kw!(Sort)
     };
     (type) => {
-        __ddl_misc_token!(Type)
+        __kw!(Type)
     };
     // dml
     (insert) => {
-        __dml_token!(Insert)
+        __kw!(Insert)
     };
     (select) => {
-        __dml_token!(Select)
+        __kw!(Select)
     };
     (update) => {
-        __dml_token!(Update)
+        __kw!(Update)
     };
     (delete) => {
-        __dml_token!(Delete)
+        __kw!(Delete)
     };
     (exists) => {
-        __dml_token!(Exists)
+        __kw!(Exists)
     };
     (truncate) => {
-        __dml_token!(Truncate)
+        __kw!(Truncate)
     };
     // dml misc
     (limit) => {
-        __dml_misc_token!(Limit)
+        __kw!(Limit)
     };
     (from) => {
-        __dml_misc_token!(From)
+        __kw!(From)
     };
     (into) => {
-        __dml_misc_token!(Into)
+        __kw!(Into)
     };
     (where) => {
-        __dml_misc_token!(Where)
+        __kw!(Where)
     };
     (if) => {
-        __dml_misc_token!(If)
+        __kw!(If)
     };
     (and) => {
-        __dml_misc_token!(And)
+        __kw!(And)
     };
     (as) => {
-        __dml_misc_token!(As)
+        __kw!(As)
     };
     (by) => {
-        __dml_misc_token!(By)
+        __kw!(By)
     };
     (asc) => {
-        __dml_misc_token!(Asc)
+        __kw!(Asc)
     };
     (desc) => {
-        __dml_misc_token!(Desc)
+        __kw!(Desc)
     };
     // types
     (string) => {
-        __type_token!(String)
+        __kw!(String)
     };
     (binary) => {
-        __type_token!(Binary)
+        __kw!(Binary)
     };
     (list) => {
-        __type_token!(List)
+        __kw!(List)
     };
     (map) => {
-        __type_token!(Map)
+        __kw!(Map)
     };
     (bool) => {
-        __type_token!(Bool)
+        __kw!(Bool)
     };
     (int) => {
-        __type_token!(Int)
+        __kw!(Int)
     };
     (double) => {
-        __type_token!(Double)
+        __kw!(Double)
     };
     (float) => {
-        __type_token!(Float)
+        __kw!(Float)
     };
     // tt
     (open {}) => {
@@ -298,7 +256,7 @@ macro_rules! Token {
     };
     // misc
     (null) => {
-        __misc_token!(Null)
+        __kw!(Null)
     };
 }
 
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 7bac5e11..4c62e4a0 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -46,9 +46,7 @@
 
 use {
     super::{
-        lexer::{
-            DdlKeyword, DdlMiscKeyword, DmlKeyword, Keyword, Lit, MiscKeyword, Symbol, Token, Type,
-        },
+        lexer::{Lit, Symbol, Token},
         LangError, LangResult, RawSlice,
     },
     crate::util::MaybeInit,
@@ -110,22 +108,22 @@ pub type Dict = HashMap<String, Option<DictEntry>>;
 #[derive(Debug, PartialEq)]
 /// A layer contains a type and corresponding metadata
 pub struct Layer {
-    ty: Type,
+    ty: RawSlice,
     props: Dict,
     reset: bool,
 }
 
 impl Layer {
     //// Create a new layer
-    pub(super) const fn new(ty: Type, props: Dict, reset: bool) -> Self {
+    pub(super) const fn new(ty: RawSlice, props: Dict, reset: bool) -> Self {
         Self { ty, props, reset }
     }
     /// Create a new layer that doesn't have any reset
-    pub(super) const fn new_noreset(ty: Type, props: Dict) -> Self {
+    pub(super) const fn new_noreset(ty: RawSlice, props: Dict) -> Self {
         Self::new(ty, props, false)
     }
     /// Create a new layer that adds a reset
-    pub(super) const fn new_reset(ty: Type, props: Dict) -> Self {
+    pub(super) const fn new_reset(ty: RawSlice, props: Dict) -> Self {
         Self::new(ty, props, true)
     }
 }
@@ -430,10 +428,7 @@ pub(super) fn rfold_tymeta<const ALLOW_RESET: bool>(
     let mut tmp = MaybeInit::uninit();
     while r.pos() < l && r.is_okay() {
         match (&tok[r.pos()], state) {
-            (
-                Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::Type)),
-                TyMetaFoldState::IDENT_OR_CB,
-            ) => {
+            (Token![type], TyMetaFoldState::IDENT_OR_CB) => {
                 // we were expecting an ident but found the type keyword! increase depth
                 r.incr();
                 r.set_has_more();
@@ -573,10 +568,10 @@ pub(super) fn rfold_layers<const ALLOW_RESET: bool>(
     let mut dict = Dict::new();
     while i < l && okay {
         match (&tok[i], state) {
-            (Token::Keyword(Keyword::TypeId(ty)), LayerFoldState::TY) => {
+            (Token::Ident(ty), LayerFoldState::TY) => {
                 i += 1;
                 // expecting type, and found type. next is either end or an open brace or some arbitrary token
-                tmp = MaybeInit::new(ty);
+                tmp = MaybeInit::new(ty.clone());
                 state = LayerFoldState::END_OR_OB;
             }
             (Token::Symbol(Symbol::TtOpenBrace), LayerFoldState::END_OR_OB) => {
@@ -670,12 +665,8 @@ pub(super) fn collect_field_properties(tok: &[Token]) -> (FieldProperties, u64)
     let mut okay = true;
     while i < tok.len() {
         match &tok[i] {
-            Token::Keyword(Keyword::Ddl(DdlKeyword::Primary)) => {
-                okay &= props.properties.insert(FieldProperties::PRIMARY)
-            }
-            Token::Keyword(Keyword::Misc(MiscKeyword::Null)) => {
-                okay &= props.properties.insert(FieldProperties::NULL)
-            }
+            Token![primary] => okay &= props.properties.insert(FieldProperties::PRIMARY),
+            Token![null] => okay &= props.properties.insert(FieldProperties::NULL),
             Token::Ident(_) => break,
             _ => {
                 // we could pass this over to the caller, but it's better if we do it since we're doing
@@ -768,9 +759,7 @@ pub(super) fn parse_schema_from_tokens(tok: &[Token]) -> LangResult<(Model, usiz
                 state = SchemaParseState::FIELD;
             }
             (
-                Token::Keyword(Keyword::Ddl(DdlKeyword::Primary))
-                | Token::Keyword(Keyword::Misc(MiscKeyword::Null))
-                | Token::Ident(_),
+                Token![primary] | Token![null] | Token::Ident(_),
                 SchemaParseState::FIELD | SchemaParseState::END_OR_FIELD,
             ) => {
                 // fine, we found a field. let's see what we've got
@@ -809,7 +798,7 @@ pub(super) fn parse_schema_from_tokens(tok: &[Token]) -> LangResult<(Model, usiz
         extract!(tok[0], Token::Ident(ref model_name) => model_name.clone())
     };
 
-    if l > i && tok[i] == (Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::With))) {
+    if l > i && tok[i] == (Token![with]) {
         // we have some more input, and it should be a dict of properties
         i += 1; // +WITH
 
@@ -1063,15 +1052,13 @@ pub(super) fn parse_alter_kind_from_tokens(
     *current += 2;
     let model_name = unsafe { extract!(tok[0], Token::Ident(ref l) => l.clone()) };
     match tok[1] {
-        Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::Add)) => alter_add(&tok[1..], current)
+        Token![add] => alter_add(&tok[1..], current)
             .map(AlterKind::Add)
             .map(|kind| Alter::new(model_name, kind)),
-        Token::Keyword(Keyword::DdlMisc(DdlMiscKeyword::Remove)) => {
-            alter_remove(&tok[1..], current)
-                .map(AlterKind::Remove)
-                .map(|kind| Alter::new(model_name, kind))
-        }
-        Token::Keyword(Keyword::Dml(DmlKeyword::Update)) => alter_update(&tok[1..], current)
+        Token![remove] => alter_remove(&tok[1..], current)
+            .map(AlterKind::Remove)
+            .map(|kind| Alter::new(model_name, kind)),
+        Token![update] => alter_update(&tok[1..], current)
             .map(AlterKind::Update)
             .map(|kind| Alter::new(model_name, kind)),
         _ => return Err(LangError::ExpectedStatement),
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 34b61553..0f937760 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -364,15 +364,15 @@ mod schema_tests {
         }
         #[test]
         fn inspect_model() {
-            let tok = lex(b"inspect model user").unwrap();
+            let tok = lex(b"inspect model users").unwrap();
             assert_eq!(
                 ddl::parse_inspect_full(&tok[1..]).unwrap(),
-                Statement::InspectModel(Entity::Single("user".into()))
+                Statement::InspectModel(Entity::Single("users".into()))
             );
-            let tok = lex(b"inspect model tweeter.user").unwrap();
+            let tok = lex(b"inspect model tweeter.users").unwrap();
             assert_eq!(
                 ddl::parse_inspect_full(&tok[1..]).unwrap(),
-                Statement::InspectModel(("tweeter", "user").into())
+                Statement::InspectModel(("tweeter", "users").into())
             );
         }
         #[test]
@@ -616,7 +616,6 @@ mod schema_tests {
     }
     mod tymeta {
         use super::*;
-        use crate::engine::ql::lexer::{Keyword, Type};
         #[test]
         fn tymeta_mini() {
             let tok = lex(b"}").unwrap();
@@ -705,7 +704,7 @@ mod schema_tests {
         }
         #[test]
         fn fuzz_tymeta_normal() {
-            // { maxlen: 10, unique: true, user: "sayan" }
+            // { maxlen: 10, unique: true, users: "sayan" }
             //   ^start
             let tok = lex(b"
                     maxlen: 10,
@@ -713,7 +712,7 @@ mod schema_tests {
                     auth: {
                         maybe: true\x01
                     },
-                    user: \"sayan\"\x01
+                    users: \"sayan\"\x01
                 }
             ")
             .unwrap();
@@ -723,7 +722,7 @@ mod schema_tests {
                 "auth" => nullable_dict! {
                     "maybe" => Lit::Bool(true),
                 },
-                "user" => Lit::Str("sayan".into())
+                "users" => Lit::Str("sayan".into())
             };
             fuzz_tokens(&tok, |should_pass, new_src| {
                 let (ret, dict) = schema::fold_tymeta(&new_src);
@@ -742,7 +741,7 @@ mod schema_tests {
         }
         #[test]
         fn fuzz_tymeta_with_ty() {
-            // list { maxlen: 10, unique: true, type string, user: "sayan" }
+            // list { maxlen: 10, unique: true, type string, users: "sayan" }
             //   ^start
             let tok = lex(b"
                     maxlen: 10,
@@ -751,7 +750,7 @@ mod schema_tests {
                         maybe: true\x01
                     },
                     type string,
-                    user: \"sayan\"\x01
+                    users: \"sayan\"\x01
                 }
             ")
             .unwrap();
@@ -767,7 +766,7 @@ mod schema_tests {
                 if should_pass {
                     assert!(ret.is_okay());
                     assert!(ret.has_more());
-                    assert!(new_src[ret.pos()] == Token::Keyword(Keyword::TypeId(Type::String)));
+                    assert!(new_src[ret.pos()] == Token::Ident("string".into()));
                     assert_eq!(dict, expected);
                 } else if ret.is_okay() {
                     panic!("Expected failure but passed for token stream: `{:?}`", tok);
@@ -777,7 +776,7 @@ mod schema_tests {
     }
     mod layer {
         use super::*;
-        use crate::engine::ql::{lexer::Type, schema::Layer};
+        use crate::engine::ql::schema::Layer;
         #[test]
         fn layer_mini() {
             let tok = lex(b"string)").unwrap();
@@ -786,7 +785,7 @@ mod schema_tests {
             assert!(okay);
             assert_eq!(
                 layers,
-                vec![Layer::new_noreset(Type::String, nullable_dict! {})]
+                vec![Layer::new_noreset("string".into(), nullable_dict! {})]
             );
         }
         #[test]
@@ -798,7 +797,7 @@ mod schema_tests {
             assert_eq!(
                 layers,
                 vec![Layer::new_noreset(
-                    Type::String,
+                    "string".into(),
                     nullable_dict! {
                         "maxlen" => Lit::UnsignedInt(100)
                     }
@@ -814,8 +813,8 @@ mod schema_tests {
             assert_eq!(
                 layers,
                 vec![
-                    Layer::new_noreset(Type::String, nullable_dict! {}),
-                    Layer::new_noreset(Type::List, nullable_dict! {})
+                    Layer::new_noreset("string".into(), nullable_dict! {}),
+                    Layer::new_noreset("list".into(), nullable_dict! {})
                 ]
             );
         }
@@ -828,9 +827,9 @@ mod schema_tests {
             assert_eq!(
                 layers,
                 vec![
-                    Layer::new_noreset(Type::String, nullable_dict! {}),
+                    Layer::new_noreset("string".into(), nullable_dict! {}),
                     Layer::new_noreset(
-                        Type::List,
+                        "list".into(),
                         nullable_dict! {
                             "unique" => Lit::Bool(true),
                             "maxlen" => Lit::UnsignedInt(10),
@@ -852,14 +851,14 @@ mod schema_tests {
                 layers,
                 vec![
                     Layer::new_noreset(
-                        Type::String,
+                        "string".into(),
                         nullable_dict! {
                             "ascii_only" => Lit::Bool(true),
                             "maxlen" => Lit::UnsignedInt(255)
                         }
                     ),
                     Layer::new_noreset(
-                        Type::List,
+                        "list".into(),
                         nullable_dict! {
                             "unique" => Lit::Bool(true),
                             "maxlen" => Lit::UnsignedInt(10),
@@ -882,14 +881,14 @@ mod schema_tests {
         ")
             .unwrap();
             let expected = vec![
-                Layer::new_noreset(Type::String, nullable_dict!()),
+                Layer::new_noreset("string".into(), nullable_dict!()),
                 Layer::new_noreset(
-                    Type::List,
+                    "list".into(),
                     nullable_dict! {
                         "maxlen" => Lit::UnsignedInt(100),
                     },
                 ),
-                Layer::new_noreset(Type::List, nullable_dict!("unique" => Lit::Bool(true))),
+                Layer::new_noreset("list".into(), nullable_dict!("unique" => Lit::Bool(true))),
             ];
             fuzz_tokens(&tok, |should_pass, new_tok| {
                 let (layers, c, okay) = schema::fold_layers(&new_tok);
@@ -935,10 +934,7 @@ mod schema_tests {
     mod fields {
         use {
             super::*,
-            crate::engine::ql::{
-                lexer::Type,
-                schema::{Field, Layer},
-            },
+            crate::engine::ql::schema::{Field, Layer},
         };
         #[test]
         fn field_mini() {
@@ -952,7 +948,7 @@ mod schema_tests {
                 f,
                 Field {
                     field_name: "username".into(),
-                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
+                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
                     props: set![],
                 }
             )
@@ -969,7 +965,7 @@ mod schema_tests {
                 f,
                 Field {
                     field_name: "username".into(),
-                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
+                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
                     props: set!["primary"],
                 }
             )
@@ -990,7 +986,7 @@ mod schema_tests {
                 Field {
                     field_name: "username".into(),
                     layers: [Layer::new_noreset(
-                        Type::String,
+                        "string".into(),
                         nullable_dict! {
                             "maxlen" => Lit::UnsignedInt(10),
                             "ascii_only" => Lit::Bool(true),
@@ -1021,14 +1017,14 @@ mod schema_tests {
                     field_name: "notes".into(),
                     layers: [
                         Layer::new_noreset(
-                            Type::String,
+                            "string".into(),
                             nullable_dict! {
                                 "maxlen" => Lit::UnsignedInt(255),
                                 "ascii_only" => Lit::Bool(true),
                             }
                         ),
                         Layer::new_noreset(
-                            Type::List,
+                            "list".into(),
                             nullable_dict! {
                                 "unique" => Lit::Bool(true)
                             }
@@ -1041,10 +1037,7 @@ mod schema_tests {
         }
     }
     mod schemas {
-        use crate::engine::ql::{
-            lexer::Type,
-            schema::{Field, Layer, Model},
-        };
+        use crate::engine::ql::schema::{Field, Layer, Model};
 
         use super::*;
         #[test]
@@ -1068,12 +1061,12 @@ mod schema_tests {
                     fields: vec![
                         Field {
                             field_name: "username".into(),
-                            layers: vec![Layer::new_noreset(Type::String, nullable_dict! {})],
+                            layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
                             props: set!["primary"]
                         },
                         Field {
                             field_name: "password".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
+                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
                             props: set![]
                         }
                     ],
@@ -1103,17 +1096,17 @@ mod schema_tests {
                     fields: vec![
                         Field {
                             field_name: "username".into(),
-                            layers: vec![Layer::new_noreset(Type::String, nullable_dict! {})],
+                            layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
                             props: set!["primary"]
                         },
                         Field {
                             field_name: "password".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
+                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
                             props: set![]
                         },
                         Field {
                             field_name: "profile_pic".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
+                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
                             props: set!["null"]
                         }
                     ],
@@ -1148,25 +1141,25 @@ mod schema_tests {
                     fields: vec![
                         Field {
                             field_name: "username".into(),
-                            layers: vec![Layer::new_noreset(Type::String, nullable_dict! {})],
+                            layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
                             props: set!["primary"]
                         },
                         Field {
                             field_name: "password".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
+                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
                             props: set![]
                         },
                         Field {
                             field_name: "profile_pic".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
+                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
                             props: set!["null"]
                         },
                         Field {
                             field_name: "notes".into(),
                             layers: vec![
-                                Layer::new_noreset(Type::String, nullable_dict! {}),
+                                Layer::new_noreset("string".into(), nullable_dict! {}),
                                 Layer::new_noreset(
-                                    Type::List,
+                                    "list".into(),
                                     nullable_dict! {
                                         "unique" => Lit::Bool(true)
                                     }
@@ -1211,25 +1204,25 @@ mod schema_tests {
                     fields: vec![
                         Field {
                             field_name: "username".into(),
-                            layers: vec![Layer::new_noreset(Type::String, nullable_dict! {})],
+                            layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
                             props: set!["primary"]
                         },
                         Field {
                             field_name: "password".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
+                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
                             props: set![]
                         },
                         Field {
                             field_name: "profile_pic".into(),
-                            layers: vec![Layer::new_noreset(Type::Binary, nullable_dict! {})],
+                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
                             props: set!["null"]
                         },
                         Field {
                             field_name: "notes".into(),
                             layers: vec![
-                                Layer::new_noreset(Type::String, nullable_dict! {}),
+                                Layer::new_noreset("string".into(), nullable_dict! {}),
                                 Layer::new_noreset(
-                                    Type::List,
+                                    "list".into(),
                                     nullable_dict! {
                                         "unique" => Lit::Bool(true)
                                     }
@@ -1250,10 +1243,7 @@ mod schema_tests {
     }
     mod dict_field_syntax {
         use super::*;
-        use crate::engine::ql::{
-            lexer::Type,
-            schema::{ExpandedField, Layer},
-        };
+        use crate::engine::ql::schema::{ExpandedField, Layer};
         #[test]
         fn field_syn_mini() {
             let tok = lex(b"username { type string }").unwrap();
@@ -1263,7 +1253,7 @@ mod schema_tests {
                 ef,
                 ExpandedField {
                     field_name: "username".into(),
-                    layers: vec![Layer::new_noreset(Type::String, nullable_dict! {})],
+                    layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
                     props: nullable_dict! {},
                     reset: false
                 }
@@ -1287,7 +1277,7 @@ mod schema_tests {
                     props: nullable_dict! {
                         "nullable" => Lit::Bool(false),
                     },
-                    layers: vec![Layer::new_noreset(Type::String, nullable_dict! {})],
+                    layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
                     reset: false
                 }
             );
@@ -1316,7 +1306,7 @@ mod schema_tests {
                         "jingle_bells" => Lit::Str("snow".into()),
                     },
                     layers: vec![Layer::new_noreset(
-                        Type::String,
+                        "string".into(),
                         nullable_dict! {
                             "minlen" => Lit::UnsignedInt(6),
                             "maxlen" => Lit::UnsignedInt(255),
@@ -1353,13 +1343,13 @@ mod schema_tests {
                     },
                     layers: vec![
                         Layer::new_noreset(
-                            Type::String,
+                            "string".into(),
                             nullable_dict! {
                                 "ascii_only" => Lit::Bool(true),
                             }
                         ),
                         Layer::new_noreset(
-                            Type::List,
+                            "list".into(),
                             nullable_dict! {
                                 "unique" => Lit::Bool(true),
                             }
@@ -1410,10 +1400,7 @@ mod schema_tests {
     }
     mod alter_model_add {
         use super::*;
-        use crate::engine::ql::{
-            lexer::Type,
-            schema::{ExpandedField, Layer},
-        };
+        use crate::engine::ql::schema::{ExpandedField, Layer};
         #[test]
         fn add_mini() {
             let tok = lex(b"
@@ -1428,7 +1415,7 @@ mod schema_tests {
                 [ExpandedField {
                     field_name: "myfield".into(),
                     props: nullable_dict! {},
-                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
+                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
                     reset: false
                 }]
             );
@@ -1449,7 +1436,7 @@ mod schema_tests {
                     props: nullable_dict! {
                         "nullable" => Lit::Bool(true)
                     },
-                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
+                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
                     reset: false
                 }]
             );
@@ -1470,7 +1457,7 @@ mod schema_tests {
                     props: nullable_dict! {
                         "nullable" => Lit::Bool(true)
                     },
-                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
+                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
                     reset: false
                 }]
             );
@@ -1506,7 +1493,7 @@ mod schema_tests {
                         props: nullable_dict! {
                             "nullable" => Lit::Bool(true)
                         },
-                        layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
+                        layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
                         reset: false
                     },
                     ExpandedField {
@@ -1516,13 +1503,13 @@ mod schema_tests {
                         },
                         layers: [
                             Layer::new_noreset(
-                                Type::String,
+                                "string".into(),
                                 nullable_dict! {
                                     "maxlen" => Lit::UnsignedInt(255)
                                 }
                             ),
                             Layer::new_noreset(
-                                Type::List,
+                                "list".into(),
                                 nullable_dict! {
                                    "unique" => Lit::Bool(true)
                                 },
@@ -1536,12 +1523,9 @@ mod schema_tests {
         }
     }
     mod alter_model_update {
-        use crate::engine::ql::{
-            lexer::Type,
-            schema::{ExpandedField, Layer},
-        };
-
         use super::*;
+        use crate::engine::ql::schema::{ExpandedField, Layer};
+
         #[test]
         fn alter_mini() {
             let tok = lex(b"
@@ -1556,7 +1540,7 @@ mod schema_tests {
                 [ExpandedField {
                     field_name: "myfield".into(),
                     props: nullable_dict! {},
-                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
+                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
                     reset: true
                 }]
             );
@@ -1575,7 +1559,7 @@ mod schema_tests {
                 [ExpandedField {
                     field_name: "myfield".into(),
                     props: nullable_dict! {},
-                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
+                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
                     reset: true
                 }]
             );
@@ -1602,7 +1586,7 @@ mod schema_tests {
                     props: nullable_dict! {
                         "nullable" => Lit::Bool(true)
                     },
-                    layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
+                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
                     reset: true
                 }]
             );
@@ -1634,13 +1618,13 @@ mod schema_tests {
                         props: nullable_dict! {
                             "nullable" => Lit::Bool(true)
                         },
-                        layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
+                        layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
                         reset: true
                     },
                     ExpandedField {
                         field_name: "myfield2".into(),
                         props: nullable_dict! {},
-                        layers: [Layer::new_noreset(Type::String, nullable_dict! {})].into(),
+                        layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
                         reset: true
                     }
                 ]
@@ -1676,14 +1660,14 @@ mod schema_tests {
                         props: nullable_dict! {
                             "nullable" => Lit::Bool(true)
                         },
-                        layers: [Layer::new_reset(Type::String, nullable_dict! {})].into(),
+                        layers: [Layer::new_reset("string".into(), nullable_dict! {})].into(),
                         reset: true
                     },
                     ExpandedField {
                         field_name: "myfield2".into(),
                         props: nullable_dict! {},
                         layers: [Layer::new_reset(
-                            Type::String,
+                            "string".into(),
                             nullable_dict! {"maxlen" => Lit::UnsignedInt(255)}
                         )]
                         .into(),
@@ -1959,13 +1943,13 @@ mod dml_tests {
         #[test]
         fn insert_tuple_mini() {
             let x = lex(br#"
-                insert twitter.user:"sayan" ()
+                insert twitter.users:"sayan" ()
             "#)
             .unwrap();
             let r = dml::parse_insert_full(&x[1..]).unwrap();
             let e = InsertStatement {
                 primary_key: &("sayan".to_string().into()),
-                entity: Entity::Full("twitter".into(), "user".into()),
+                entity: Entity::Full("twitter".into(), "users".into()),
                 data: vec![].into(),
             };
             assert_eq!(e, r);
@@ -2111,13 +2095,13 @@ mod dml_tests {
         #[test]
         fn select_mini() {
             let tok = lex(br#"
-                select * from user:"sayan"
+                select * from users:"sayan"
             "#)
             .unwrap();
             let r = dml::parse_select_full(&tok[1..]).unwrap();
             let e = SelectStatement {
                 primary_key: &Lit::Str("sayan".into()),
-                entity: Entity::Single("user".into()),
+                entity: Entity::Single("users".into()),
                 fields: [].to_vec(),
                 wildcard: true,
             };
@@ -2126,13 +2110,13 @@ mod dml_tests {
         #[test]
         fn select() {
             let tok = lex(br#"
-                select field1 from user:"sayan"
+                select field1 from users:"sayan"
             "#)
             .unwrap();
             let r = dml::parse_select_full(&tok[1..]).unwrap();
             let e = SelectStatement {
                 primary_key: &Lit::Str("sayan".into()),
-                entity: Entity::Single("user".into()),
+                entity: Entity::Single("users".into()),
                 fields: ["field1".into()].to_vec(),
                 wildcard: false,
             };
@@ -2141,13 +2125,13 @@ mod dml_tests {
         #[test]
         fn select_pro() {
             let tok = lex(br#"
-                select field1 from twitter.user:"sayan"
+                select field1 from twitter.users:"sayan"
             "#)
             .unwrap();
             let r = dml::parse_select_full(&tok[1..]).unwrap();
             let e = SelectStatement {
                 primary_key: &Lit::Str("sayan".into()),
-                entity: Entity::Full("twitter".into(), "user".into()),
+                entity: Entity::Full("twitter".into(), "users".into()),
                 fields: ["field1".into()].to_vec(),
                 wildcard: false,
             };
@@ -2156,13 +2140,13 @@ mod dml_tests {
         #[test]
         fn select_pro_max() {
             let tok = lex(br#"
-                select field1, field2 from twitter.user:"sayan"
+                select field1, field2 from twitter.users:"sayan"
             "#)
             .unwrap();
             let r = dml::parse_select_full(&tok[1..]).unwrap();
             let e = SelectStatement {
                 primary_key: &Lit::Str("sayan".into()),
-                entity: Entity::Full("twitter".into(), "user".into()),
+                entity: Entity::Full("twitter".into(), "users".into()),
                 fields: ["field1".into(), "field2".into()].to_vec(),
                 wildcard: false,
             };
@@ -2305,22 +2289,22 @@ mod dml_tests {
         #[test]
         fn delete_mini() {
             let tok = lex(br#"
-                delete user:"sayan"
+                delete users:"sayan"
             "#)
             .unwrap();
             let primary_key = "sayan".into();
-            let e = DeleteStatement::new(&primary_key, Entity::Single("user".into()));
+            let e = DeleteStatement::new(&primary_key, Entity::Single("users".into()));
             let r = dml::parse_delete_full(&tok[1..]).unwrap();
             assert_eq!(r, e);
         }
         #[test]
         fn delete() {
             let tok = lex(br#"
-                delete twitter.user:"sayan"
+                delete twitter.users:"sayan"
             "#)
             .unwrap();
             let primary_key = "sayan".into();
-            let e = DeleteStatement::new(&primary_key, ("twitter", "user").into());
+            let e = DeleteStatement::new(&primary_key, ("twitter", "users").into());
             let r = dml::parse_delete_full(&tok[1..]).unwrap();
             assert_eq!(r, e);
         }

From 2dec28d9890722256697b84cd22d4888791b75f6 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 2 Dec 2022 21:54:39 +0530
Subject: [PATCH 052/310] Add `where` clause parsing

---
 server/src/engine/ql/dml.rs   |  54 +++-
 server/src/engine/ql/lexer.rs |   2 +
 server/src/engine/ql/tests.rs | 478 +++++++++++++++++++---------------
 3 files changed, 314 insertions(+), 220 deletions(-)

diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 7e579233..c99d580e 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -40,6 +40,7 @@ use {
         util::{compiler, MaybeInit},
     },
     std::{
+        cmp,
         collections::HashMap,
         mem::{discriminant, Discriminant},
     },
@@ -69,7 +70,7 @@ fn process_entity(tok: &[Token], d: &mut MaybeInit<Entity>, i: &mut usize) -> bo
 
 #[derive(Debug, PartialEq)]
 pub(super) struct RelationalExpr<'a> {
-    pub(super) lhs: RawSlice,
+    pub(super) lhs: &'a [u8],
     pub(super) rhs: &'a Lit,
     pub(super) opc: u8,
 }
@@ -81,6 +82,9 @@ impl<'a> RelationalExpr<'a> {
     pub(super) const OP_GE: u8 = 4;
     pub(super) const OP_LT: u8 = 5;
     pub(super) const OP_LE: u8 = 6;
+    fn filter_hint_none(&self) -> bool {
+        self.opc == Self::OP_EQ
+    }
     #[inline(always)]
     fn parse_operator(tok: &[Token], i: &mut usize, okay: &mut bool) -> u8 {
         /*
@@ -116,15 +120,16 @@ impl<'a> RelationalExpr<'a> {
         okay &= tok[0].is_ident();
         i += 1;
         // let's get ourselves the operator
-        let operator = Self::parse_operator(&tok[1..], &mut i, &mut okay);
+        let operator = Self::parse_operator(&tok[i..], &mut i, &mut okay);
         okay &= i < tok.len();
-        okay &= tok[tok.len() - 1].is_lit(); // LOL, I really like saving cycles
+        let lit_idx = cmp::min(i, tok.len() - 1);
+        okay &= tok[lit_idx].is_lit(); // LOL, I really like saving cycles
         *cnt += i + okay as usize;
         if compiler::likely(okay) {
             Some(unsafe {
                 Self {
-                    lhs: extract!(tok[0], Token::Ident(ref id) => id.clone()),
-                    rhs: extract!(tok[tok.len() - 1], Token::Lit(ref l) => l),
+                    lhs: extract!(tok[0], Token::Ident(ref id) => id.as_slice()),
+                    rhs: extract!(tok[lit_idx], Token::Lit(ref l) => l),
                     opc: operator,
                 }
             })
@@ -134,6 +139,45 @@ impl<'a> RelationalExpr<'a> {
     }
 }
 
+#[derive(Debug, PartialEq)]
+pub(super) struct WhereClause<'a> {
+    c: HashMap<&'a [u8], RelationalExpr<'a>>,
+}
+
+impl<'a> WhereClause<'a> {
+    #[inline(always)]
+    pub(super) fn new(c: HashMap<&'a [u8], RelationalExpr<'a>>) -> Self {
+        Self { c }
+    }
+    #[inline(always)]
+    pub(super) fn parse_where(tok: &'a [Token], flag: &mut bool, cnt: &mut usize) -> Self {
+        let l = tok.len();
+        let mut okay = true;
+        let mut i = 0;
+        let mut c = HashMap::with_capacity(2);
+        let mut has_more = true;
+        while okay && i < l && has_more {
+            okay &= RelationalExpr::try_parse(&tok[i..], &mut i)
+                .map(|clause| c.insert(clause.lhs, clause).is_none())
+                .unwrap_or(false);
+            has_more = tok[cmp::min(i, l - 1)] == Token![and] && i < l;
+            i += has_more as usize;
+        }
+        *flag &= okay;
+        *cnt += i;
+        Self { c }
+    }
+}
+
+#[cfg(test)]
+pub(super) fn parse_where_clause_full<'a>(tok: &'a [Token]) -> Option<WhereClause<'a>> {
+    let mut flag = true;
+    let mut i = 0;
+    let ret = WhereClause::parse_where(tok, &mut flag, &mut i);
+    full_tt!(tok.len(), i);
+    flag.then_some(ret)
+}
+
 #[cfg(test)]
 #[inline(always)]
 pub(super) fn parse_relexpr_full<'a>(tok: &'a [Token]) -> Option<RelationalExpr<'a>> {
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 32a6c64a..0feb0096 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -346,6 +346,7 @@ const KWMG_1: [u8; 11] = *b"nJEcjrLflKX";
 const KWMG_2: [u8; 11] = *b"KWHPUPK3Fh3";
 const KWMG_S: usize = KWMG_1.len();
 
+#[inline(always)]
 fn kwhf(k: &[u8], mg: &[u8]) -> u32 {
     let mut i = 0;
     let mut s = 0;
@@ -380,6 +381,7 @@ const LEXER_MODE_INSECURE: u8 = 0;
 const LEXER_MODE_SECURE: u8 = 1;
 
 pub type InsecureLexer<'a> = Lexer<'a, LEXER_MODE_INSECURE>;
+pub type SecureLexer<'a> = Lexer<'a, LEXER_MODE_SECURE>;
 
 pub struct Lexer<'a, const OPERATING_MODE: u8> {
     c: *const u8,
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 0f937760..99919475 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -26,10 +26,14 @@
 
 use {
     super::{
-        lexer::{InsecureLexer, Token},
+        lexer::{InsecureLexer, Symbol, Token},
         LangResult,
     },
-    crate::{engine::memory::DataType, util::Life},
+    crate::{
+        engine::memory::DataType,
+        util::{test_utils, Life},
+    },
+    rand::{self, Rng},
 };
 
 pub(super) fn lex(src: &[u8]) -> LangResult<Life<Vec<Token>>> {
@@ -89,6 +93,40 @@ macro_rules! fold_dict {
     }
 }
 
+/// A very "basic" fuzzer that will randomly inject tokens wherever applicable
+fn fuzz_tokens(src: &[Token], fuzzwith: impl Fn(bool, &[Token])) {
+    static FUZZ_TARGETS: [Token; 2] = [Token::Symbol(Symbol::SymComma), Token::IgnorableComma];
+    let mut rng = rand::thread_rng();
+    #[inline(always)]
+    fn inject(new_src: &mut Vec<Token>, rng: &mut impl Rng) -> usize {
+        let start = new_src.len();
+        (0..test_utils::random_number(0, 5, rng))
+            .for_each(|_| new_src.push(Token::Symbol(Symbol::SymComma)));
+        new_src.len() - start
+    }
+    let fuzz_amount = src.iter().filter(|tok| FUZZ_TARGETS.contains(tok)).count();
+    for _ in 0..(fuzz_amount.pow(2)) {
+        let mut new_src = Vec::with_capacity(src.len());
+        let mut should_pass = true;
+        src.iter().for_each(|tok| match tok {
+            Token::IgnorableComma => {
+                let added = inject(&mut new_src, &mut rng);
+                should_pass &= added <= 1;
+            }
+            Token::Symbol(Symbol::SymComma) => {
+                let added = inject(&mut new_src, &mut rng);
+                should_pass &= added == 1;
+            }
+            tok => new_src.push(tok.clone()),
+        });
+        assert!(
+            new_src.iter().all(|tok| tok != &Token::IgnorableComma),
+            "found ignorable comma in rectified source"
+        );
+        fuzzwith(should_pass, &new_src);
+    }
+}
+
 mod lexer_tests {
     use {
         super::{
@@ -299,135 +337,11 @@ mod ddl_other_query_tests {
     }
 }
 
-mod schema_tests {
+mod dict_tests {
     use {
-        super::{
-            super::{
-                lexer::{Lit, Symbol, Token},
-                schema,
-            },
-            lex,
-        },
-        crate::util::test_utils,
-        rand::{self, Rng},
+        super::*,
+        crate::engine::ql::{lexer::Lit, schema},
     };
-
-    /// A very "basic" fuzzer that will randomly inject tokens wherever applicable
-    fn fuzz_tokens(src: &[Token], fuzzwith: impl Fn(bool, &[Token])) {
-        static FUZZ_TARGETS: [Token; 2] = [Token::Symbol(Symbol::SymComma), Token::IgnorableComma];
-        let mut rng = rand::thread_rng();
-        #[inline(always)]
-        fn inject(new_src: &mut Vec<Token>, rng: &mut impl Rng) -> usize {
-            let start = new_src.len();
-            (0..test_utils::random_number(0, 5, rng))
-                .for_each(|_| new_src.push(Token::Symbol(Symbol::SymComma)));
-            new_src.len() - start
-        }
-        let fuzz_amount = src.iter().filter(|tok| FUZZ_TARGETS.contains(tok)).count();
-        for _ in 0..(fuzz_amount.pow(2)) {
-            let mut new_src = Vec::with_capacity(src.len());
-            let mut should_pass = true;
-            src.iter().for_each(|tok| match tok {
-                Token::IgnorableComma => {
-                    let added = inject(&mut new_src, &mut rng);
-                    should_pass &= added <= 1;
-                }
-                Token::Symbol(Symbol::SymComma) => {
-                    let added = inject(&mut new_src, &mut rng);
-                    should_pass &= added == 1;
-                }
-                tok => new_src.push(tok.clone()),
-            });
-            assert!(
-                new_src.iter().all(|tok| tok != &Token::IgnorableComma),
-                "found ignorable comma in rectified source"
-            );
-            fuzzwith(should_pass, &new_src);
-        }
-    }
-
-    mod inspect {
-        use {
-            super::*,
-            crate::engine::ql::{
-                ast::{Entity, Statement},
-                ddl,
-            },
-        };
-        #[test]
-        fn inspect_space() {
-            let tok = lex(b"inspect space myspace").unwrap();
-            assert_eq!(
-                ddl::parse_inspect_full(&tok[1..]).unwrap(),
-                Statement::InspectSpace("myspace".into())
-            );
-        }
-        #[test]
-        fn inspect_model() {
-            let tok = lex(b"inspect model users").unwrap();
-            assert_eq!(
-                ddl::parse_inspect_full(&tok[1..]).unwrap(),
-                Statement::InspectModel(Entity::Single("users".into()))
-            );
-            let tok = lex(b"inspect model tweeter.users").unwrap();
-            assert_eq!(
-                ddl::parse_inspect_full(&tok[1..]).unwrap(),
-                Statement::InspectModel(("tweeter", "users").into())
-            );
-        }
-        #[test]
-        fn inspect_spaces() {
-            let tok = lex(b"inspect spaces").unwrap();
-            assert_eq!(
-                ddl::parse_inspect_full(&tok[1..]).unwrap(),
-                Statement::InspectSpaces
-            );
-        }
-    }
-
-    mod alter_space {
-        use {
-            super::*,
-            crate::engine::ql::{
-                lexer::Lit,
-                schema::{self, AlterSpace},
-            },
-        };
-        #[test]
-        fn alter_space_mini() {
-            let tok = lex(b"alter model mymodel with {}").unwrap();
-            let r = schema::alter_space_full(&tok[2..]).unwrap();
-            assert_eq!(
-                r,
-                AlterSpace {
-                    space_name: "mymodel".into(),
-                    updated_props: nullable_dict! {}
-                }
-            );
-        }
-        #[test]
-        fn alter_space() {
-            let tok = lex(br#"
-                alter model mymodel with {
-                    max_entry: 1000,
-                    driver: "ts-0.8"
-                }
-            "#)
-            .unwrap();
-            let r = schema::alter_space_full(&tok[2..]).unwrap();
-            assert_eq!(
-                r,
-                AlterSpace {
-                    space_name: "mymodel".into(),
-                    updated_props: nullable_dict! {
-                        "max_entry" => Lit::UnsignedInt(1000),
-                        "driver" => Lit::Str("ts-0.8".into())
-                    }
-                }
-            );
-        }
-    }
-
     mod dict {
         use super::*;
 
@@ -614,6 +528,183 @@ mod schema_tests {
             });
         }
     }
+    mod nullable_dict_tests {
+        use super::*;
+        mod dict {
+            use {super::*, crate::engine::ql::lexer::Lit};
+
+            #[test]
+            fn null_mini() {
+                let d = fold_dict!(br"{ x: null }");
+                assert_eq!(
+                    d,
+                    nullable_dict! {
+                        "x" => Null,
+                    }
+                );
+            }
+            #[test]
+            fn null() {
+                let d = fold_dict! {
+                    br#"
+                        {
+                            this_is_non_null: "hello",
+                            but_this_is_null: null,
+                        }
+                    "#
+                };
+                assert_eq!(
+                    d,
+                    nullable_dict! {
+                        "this_is_non_null" => Lit::Str("hello".into()),
+                        "but_this_is_null" => Null,
+                    }
+                )
+            }
+            #[test]
+            fn null_pro() {
+                let d = fold_dict! {
+                    br#"
+                        {
+                            a_string: "this is a string",
+                            num: 1234,
+                            a_dict: {
+                                a_null: null,
+                            }
+                        }
+                    "#
+                };
+                assert_eq!(
+                    d,
+                    nullable_dict! {
+                        "a_string" => Lit::Str("this is a string".into()),
+                        "num" => Lit::UnsignedInt(1234),
+                        "a_dict" => nullable_dict! {
+                            "a_null" => Null,
+                        }
+                    }
+                )
+            }
+            #[test]
+            fn null_pro_max() {
+                let d = fold_dict! {
+                    br#"
+                        {
+                            a_string: "this is a string",
+                            num: 1234,
+                            a_dict: {
+                                a_null: null,
+                            },
+                            another_null: null,
+                        }
+                    "#
+                };
+                assert_eq!(
+                    d,
+                    nullable_dict! {
+                        "a_string" => Lit::Str("this is a string".into()),
+                        "num" => Lit::UnsignedInt(1234),
+                        "a_dict" => nullable_dict! {
+                            "a_null" => Null,
+                        },
+                        "another_null" => Null,
+                    }
+                )
+            }
+        }
+        // TODO(@ohsayan): Add null tests
+    }
+}
+
+mod schema_tests {
+    use super::{
+        super::{
+            lexer::{Lit, Token},
+            schema,
+        },
+        lex, *,
+    };
+    mod inspect {
+        use {
+            super::*,
+            crate::engine::ql::{
+                ast::{Entity, Statement},
+                ddl,
+            },
+        };
+        #[test]
+        fn inspect_space() {
+            let tok = lex(b"inspect space myspace").unwrap();
+            assert_eq!(
+                ddl::parse_inspect_full(&tok[1..]).unwrap(),
+                Statement::InspectSpace("myspace".into())
+            );
+        }
+        #[test]
+        fn inspect_model() {
+            let tok = lex(b"inspect model users").unwrap();
+            assert_eq!(
+                ddl::parse_inspect_full(&tok[1..]).unwrap(),
+                Statement::InspectModel(Entity::Single("users".into()))
+            );
+            let tok = lex(b"inspect model tweeter.users").unwrap();
+            assert_eq!(
+                ddl::parse_inspect_full(&tok[1..]).unwrap(),
+                Statement::InspectModel(("tweeter", "users").into())
+            );
+        }
+        #[test]
+        fn inspect_spaces() {
+            let tok = lex(b"inspect spaces").unwrap();
+            assert_eq!(
+                ddl::parse_inspect_full(&tok[1..]).unwrap(),
+                Statement::InspectSpaces
+            );
+        }
+    }
+
+    mod alter_space {
+        use {
+            super::*,
+            crate::engine::ql::{
+                lexer::Lit,
+                schema::{self, AlterSpace},
+            },
+        };
+        #[test]
+        fn alter_space_mini() {
+            let tok = lex(b"alter model mymodel with {}").unwrap();
+            let r = schema::alter_space_full(&tok[2..]).unwrap();
+            assert_eq!(
+                r,
+                AlterSpace {
+                    space_name: "mymodel".into(),
+                    updated_props: nullable_dict! {}
+                }
+            );
+        }
+        #[test]
+        fn alter_space() {
+            let tok = lex(br#"
+                alter model mymodel with {
+                    max_entry: 1000,
+                    driver: "ts-0.8"
+                }
+            "#)
+            .unwrap();
+            let r = schema::alter_space_full(&tok[2..]).unwrap();
+            assert_eq!(
+                r,
+                AlterSpace {
+                    space_name: "mymodel".into(),
+                    updated_props: nullable_dict! {
+                        "max_entry" => Lit::UnsignedInt(1000),
+                        "driver" => Lit::Str("ts-0.8".into())
+                    }
+                }
+            );
+        }
+    }
     mod tymeta {
         use super::*;
         #[test]
@@ -2323,7 +2414,7 @@ mod dml_tests {
                 r,
                 RelationalExpr {
                     rhs: &(10.into()),
-                    lhs: "primary_key".into(),
+                    lhs: "primary_key".as_bytes(),
                     opc: RelationalExpr::OP_EQ
                 }
             );
@@ -2336,7 +2427,7 @@ mod dml_tests {
                 r,
                 RelationalExpr {
                     rhs: &(10.into()),
-                    lhs: "primary_key".into(),
+                    lhs: "primary_key".as_bytes(),
                     opc: RelationalExpr::OP_NE
                 }
             );
@@ -2349,7 +2440,7 @@ mod dml_tests {
                 r,
                 RelationalExpr {
                     rhs: &(10.into()),
-                    lhs: "primary_key".into(),
+                    lhs: "primary_key".as_bytes(),
                     opc: RelationalExpr::OP_GT
                 }
             );
@@ -2362,7 +2453,7 @@ mod dml_tests {
                 r,
                 RelationalExpr {
                     rhs: &(10.into()),
-                    lhs: "primary_key".into(),
+                    lhs: "primary_key".as_bytes(),
                     opc: RelationalExpr::OP_GE
                 }
             );
@@ -2375,7 +2466,7 @@ mod dml_tests {
                 r,
                 RelationalExpr {
                     rhs: &(10.into()),
-                    lhs: "primary_key".into(),
+                    lhs: "primary_key".as_bytes(),
                     opc: RelationalExpr::OP_LT
                 }
             );
@@ -2388,97 +2479,54 @@ mod dml_tests {
                 r,
                 RelationalExpr {
                     rhs: &(10.into()),
-                    lhs: "primary_key".into(),
+                    lhs: "primary_key".as_bytes(),
                     opc: RelationalExpr::OP_LE
                 }
             );
         }
     }
-}
-
-mod nullable_dict_tests {
-    use super::*;
-    mod dict {
-        use {super::*, crate::engine::ql::lexer::Lit};
-
-        #[test]
-        fn null_mini() {
-            let d = fold_dict!(br"{ x: null }");
-            assert_eq!(
-                d,
-                nullable_dict! {
-                    "x" => Null,
-                }
-            );
-        }
-        #[test]
-        fn null() {
-            let d = fold_dict! {
-                br#"
-                    {
-                        this_is_non_null: "hello",
-                        but_this_is_null: null,
-                    }
-                "#
-            };
-            assert_eq!(
-                d,
-                nullable_dict! {
-                    "this_is_non_null" => Lit::Str("hello".into()),
-                    "but_this_is_null" => Null,
-                }
-            )
-        }
+    mod where_clause {
+        use {
+            super::*,
+            crate::engine::ql::dml::{self, RelationalExpr, WhereClause},
+        };
         #[test]
-        fn null_pro() {
-            let d = fold_dict! {
-                br#"
-                    {
-                        a_string: "this is a string",
-                        num: 1234,
-                        a_dict: {
-                            a_null: null,
-                        }
-                    }
-                "#
-            };
-            assert_eq!(
-                d,
-                nullable_dict! {
-                    "a_string" => Lit::Str("this is a string".into()),
-                    "num" => Lit::UnsignedInt(1234),
-                    "a_dict" => nullable_dict! {
-                        "a_null" => Null,
-                    }
+        fn where_single() {
+            let tok = lex(br#"
+                x = 100
+            "#)
+            .unwrap();
+            let rhs_hundred = 100.into();
+            let expected = WhereClause::new(dict! {
+                "x".as_bytes() => RelationalExpr {
+                    rhs: &rhs_hundred,
+                    lhs: "x".as_bytes(),
+                    opc: RelationalExpr::OP_EQ
                 }
-            )
+            });
+            assert_eq!(expected, dml::parse_where_clause_full(&tok).unwrap());
         }
         #[test]
-        fn null_pro_max() {
-            let d = fold_dict! {
-                br#"
-                    {
-                        a_string: "this is a string",
-                        num: 1234,
-                        a_dict: {
-                            a_null: null,
-                        },
-                        another_null: null,
-                    }
-                "#
-            };
-            assert_eq!(
-                d,
-                nullable_dict! {
-                    "a_string" => Lit::Str("this is a string".into()),
-                    "num" => Lit::UnsignedInt(1234),
-                    "a_dict" => nullable_dict! {
-                        "a_null" => Null,
-                    },
-                    "another_null" => Null,
+        fn where_double() {
+            let tok = lex(br#"
+                userid = 100 and pass = "password"
+            "#)
+            .unwrap();
+            let rhs_hundred = 100.into();
+            let rhs_password = "password".into();
+            let expected = WhereClause::new(dict! {
+                "userid".as_bytes() => RelationalExpr {
+                    rhs: &rhs_hundred,
+                    lhs: "userid".as_bytes(),
+                    opc: RelationalExpr::OP_EQ
+                },
+                "pass".as_bytes() => RelationalExpr {
+                    rhs: &rhs_password,
+                    lhs: "pass".as_bytes(),
+                    opc: RelationalExpr::OP_EQ
                 }
-            )
+            });
+            assert_eq!(expected, dml::parse_where_clause_full(&tok).unwrap());
         }
     }
-    // TODO(@ohsayan): Add null tests
 }

From bd0c06652cc5cadfa765632e81d5c039de5c2843 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 4 Dec 2022 11:27:25 +0530
Subject: [PATCH 053/310] Use `where` clauses by default for DML queries

Along with this, here's a summary of other changes:
- `RawSlice` now uses `NonNull` to help the compiler with size opts
- The lexer now allows case insensitive keyword usage (I still DO NOT
particularly approve of it, because enforcing is important)
---
 server/src/engine/ql/ddl.rs    |  17 +-
 server/src/engine/ql/dml.rs    | 438 ++++++++++++++++++++++-----------
 server/src/engine/ql/lexer.rs  |   9 +-
 server/src/engine/ql/macros.rs |   5 +-
 server/src/engine/ql/mod.rs    |  11 +-
 server/src/engine/ql/tests.rs  | 198 ++++++++++-----
 server/src/util/mod.rs         |   4 +
 7 files changed, 467 insertions(+), 215 deletions(-)

diff --git a/server/src/engine/ql/ddl.rs b/server/src/engine/ql/ddl.rs
index daad8546..396dbb7e 100644
--- a/server/src/engine/ql/ddl.rs
+++ b/server/src/engine/ql/ddl.rs
@@ -75,14 +75,17 @@ pub(super) fn parse_drop(tok: &[Token], counter: &mut usize) -> LangResult<State
         }
         Some(Token![space]) if tok.len() > 1 && tok[1].is_ident() => {
             let mut i = 2; // (`space` and space name)
-            // should we force drop?
+                           // should we force drop?
             let force = i < tok.len() && tok[i] == Token::Ident("force".into());
             i += force as usize;
             *counter += i;
             // either `force` or nothing
             if tok.len() == i {
                 return Ok(Statement::DropSpace(DropSpace::new(
-                    unsafe { extract!(tok[1], Token::Ident(ref space) => space.clone()) },
+                    unsafe {
+                        // UNSAFE(@ohsayan): Safe because the match predicate ensures that tok[1] is indeed an ident
+                        extract!(tok[1], Token::Ident(ref space) => space.clone())
+                    },
                     force,
                 )));
             }
@@ -96,7 +99,7 @@ pub(super) fn parse_drop(tok: &[Token], counter: &mut usize) -> LangResult<State
 pub(super) fn parse_drop_full(tok: &[Token]) -> LangResult<Statement> {
     let mut i = 0;
     let r = self::parse_drop(tok, &mut i);
-    full_tt!(i, tok.len());
+    assert_full_tt!(i, tok.len());
     r
 }
 
@@ -114,11 +117,15 @@ pub(super) fn parse_inspect(tok: &[Token], c: &mut usize) -> LangResult<Statemen
         Some(Token![space]) if tok.len() == 2 && tok[1].is_ident() => {
             *c += 1;
             Ok(Statement::InspectSpace(unsafe {
+                // UNSAFE(@ohsayan): Safe because of the match predicate
                 extract!(tok[1], Token::Ident(ref space) => space.clone())
             }))
         }
         Some(Token::Ident(id))
-            if unsafe { id.as_slice().eq_ignore_ascii_case(b"spaces") } && tok.len() == 1 =>
+            if unsafe {
+                // UNSAFE(@ohsayan): Token lifetime ensures validity of slice
+                id.as_slice().eq_ignore_ascii_case(b"spaces")
+            } && tok.len() == 1 =>
         {
             Ok(Statement::InspectSpaces)
         }
@@ -130,6 +137,6 @@ pub(super) fn parse_inspect(tok: &[Token], c: &mut usize) -> LangResult<Statemen
 pub(super) fn parse_inspect_full(tok: &[Token]) -> LangResult<Statement> {
     let mut i = 0;
     let r = self::parse_inspect(tok, &mut i);
-    full_tt!(i, tok.len());
+    assert_full_tt!(i, tok.len());
     r
 }
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index c99d580e..24ca94c4 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -56,10 +56,16 @@ fn process_entity(tok: &[Token], d: &mut MaybeInit<Entity>, i: &mut usize) -> bo
     let is_single = Entity::tokens_with_single(tok);
     if is_full {
         *i += 3;
-        *d = MaybeInit::new(unsafe { Entity::full_entity_from_slice(tok) })
+        *d = MaybeInit::new(unsafe {
+            // UNSAFE(@ohsayan): Predicate ensures validity
+            Entity::full_entity_from_slice(tok)
+        })
     } else if is_single {
         *i += 1;
-        *d = MaybeInit::new(unsafe { Entity::single_entity_from_slice(tok) });
+        *d = MaybeInit::new(unsafe {
+            // UNSAFE(@ohsayan): Predicate ensures validity
+            Entity::single_entity_from_slice(tok)
+        });
     }
     is_full | is_single
 }
@@ -69,13 +75,17 @@ fn process_entity(tok: &[Token], d: &mut MaybeInit<Entity>, i: &mut usize) -> bo
 */
 
 #[derive(Debug, PartialEq)]
-pub(super) struct RelationalExpr<'a> {
+pub struct RelationalExpr<'a> {
     pub(super) lhs: &'a [u8],
     pub(super) rhs: &'a Lit,
     pub(super) opc: u8,
 }
 
 impl<'a> RelationalExpr<'a> {
+    #[inline(always)]
+    pub(super) fn new(lhs: &'a [u8], rhs: &'a Lit, opc: u8) -> Self {
+        Self { lhs, rhs, opc }
+    }
     pub(super) const OP_EQ: u8 = 1;
     pub(super) const OP_NE: u8 = 2;
     pub(super) const OP_GT: u8 = 3;
@@ -102,7 +112,7 @@ impl<'a> RelationalExpr<'a> {
         let op_le = u(tok[0] == Token![<] && tok[1] == Token![=]) * Self::OP_LE;
         let op_lt = u(tok[0] == Token![<] && op_le == 0) * Self::OP_LT;
         let opc = op_eq + op_ne + op_ge + op_gt + op_le + op_lt;
-        *okay = opc != 0;
+        *okay &= opc != 0;
         *i += 1 + (opc & 1 == 0) as usize;
         opc
     }
@@ -127,6 +137,7 @@ impl<'a> RelationalExpr<'a> {
         *cnt += i + okay as usize;
         if compiler::likely(okay) {
             Some(unsafe {
+                // UNSAFE(@ohsayan): tok[0] is checked for being an ident, tok[lit_idx] also checked to be a lit
                 Self {
                     lhs: extract!(tok[0], Token::Ident(ref id) => id.as_slice()),
                     rhs: extract!(tok[lit_idx], Token::Lit(ref l) => l),
@@ -140,7 +151,7 @@ impl<'a> RelationalExpr<'a> {
 }
 
 #[derive(Debug, PartialEq)]
-pub(super) struct WhereClause<'a> {
+pub struct WhereClause<'a> {
     c: HashMap<&'a [u8], RelationalExpr<'a>>,
 }
 
@@ -150,11 +161,19 @@ impl<'a> WhereClause<'a> {
         Self { c }
     }
     #[inline(always)]
-    pub(super) fn parse_where(tok: &'a [Token], flag: &mut bool, cnt: &mut usize) -> Self {
+    /// Parse the expressions in a `where` context, appending it to the given map
+    ///
+    /// Notes:
+    /// - Deny duplicate clauses
+    /// - No enforcement on minimum number of clauses
+    fn parse_where_and_append_to(
+        tok: &'a [Token],
+        cnt: &mut usize,
+        c: &mut HashMap<&'a [u8], RelationalExpr<'a>>,
+    ) -> bool {
         let l = tok.len();
         let mut okay = true;
         let mut i = 0;
-        let mut c = HashMap::with_capacity(2);
         let mut has_more = true;
         while okay && i < l && has_more {
             okay &= RelationalExpr::try_parse(&tok[i..], &mut i)
@@ -163,8 +182,18 @@ impl<'a> WhereClause<'a> {
             has_more = tok[cmp::min(i, l - 1)] == Token![and] && i < l;
             i += has_more as usize;
         }
-        *flag &= okay;
         *cnt += i;
+        okay
+    }
+    #[inline(always)]
+    /// Parse a where context
+    ///
+    /// Notes:
+    /// - Enforce a minimum of 1 clause
+    pub(super) fn parse_where(tok: &'a [Token], flag: &mut bool, cnt: &mut usize) -> Self {
+        let mut c = HashMap::with_capacity(2);
+        *flag &= Self::parse_where_and_append_to(tok, cnt, &mut c);
+        *flag &= !c.is_empty();
         Self { c }
     }
 }
@@ -174,7 +203,7 @@ pub(super) fn parse_where_clause_full<'a>(tok: &'a [Token]) -> Option<WhereClaus
     let mut flag = true;
     let mut i = 0;
     let ret = WhereClause::parse_where(tok, &mut flag, &mut i);
-    full_tt!(tok.len(), i);
+    assert_full_tt!(tok.len(), i);
     flag.then_some(ret)
 }
 
@@ -183,7 +212,7 @@ pub(super) fn parse_where_clause_full<'a>(tok: &'a [Token]) -> Option<WhereClaus
 pub(super) fn parse_relexpr_full<'a>(tok: &'a [Token]) -> Option<RelationalExpr<'a>> {
     let mut i = 0;
     let okay = RelationalExpr::try_parse(tok, &mut i);
-    full_tt!(tok.len(), i);
+    assert_full_tt!(tok.len(), i);
     okay
 }
 
@@ -339,7 +368,15 @@ pub(super) fn parse_data_map_syntax<'a>(
                     Lit::UnsafeLit(l) => DataType::AnonymousTypeNeedsEval(l.clone()),
                     Lit::SignedInt(int) => DataType::SignedInt(*int),
                 };
-                okay &= data.insert(unsafe { id.as_slice() }, Some(dt)).is_none();
+                okay &= data
+                    .insert(
+                        unsafe {
+                            // UNSAFE(@ohsayan): Token lifetime ensures slice validity
+                            id.as_slice()
+                        },
+                        Some(dt),
+                    )
+                    .is_none();
             }
             (Token::Ident(id), Token::Symbol(Symbol::TtOpenSqBracket)) => {
                 // ooh a list
@@ -348,11 +385,25 @@ pub(super) fn parse_data_map_syntax<'a>(
                 okay &= lst_ok;
                 i += lst_i;
                 okay &= data
-                    .insert(unsafe { id.as_slice() }, Some(l.into()))
+                    .insert(
+                        unsafe {
+                            // UNSAFE(@ohsayan): Token lifetime ensures validity
+                            id.as_slice()
+                        },
+                        Some(l.into()),
+                    )
                     .is_none();
             }
             (Token::Ident(id), Token![null]) => {
-                okay &= data.insert(unsafe { id.as_slice() }, None).is_none();
+                okay &= data
+                    .insert(
+                        unsafe {
+                            // UNSAFE(@ohsayan): Token lifetime ensures validity
+                            id.as_slice()
+                        },
+                        None,
+                    )
+                    .is_none();
             }
             _ => {
                 okay = false;
@@ -411,65 +462,77 @@ impl<'a> From<HashMap<&'static [u8], Option<DataType>>> for InsertData<'a> {
 
 #[derive(Debug, PartialEq)]
 pub struct InsertStatement<'a> {
-    pub(super) primary_key: &'a Lit,
     pub(super) entity: Entity,
     pub(super) data: InsertData<'a>,
 }
 
+#[inline(always)]
+fn parse_entity(tok: &[Token], entity: &mut MaybeInit<Entity>, i: &mut usize) -> bool {
+    let is_full = tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident();
+    let is_half = tok[0].is_ident();
+    unsafe {
+        // UNSAFE(@ohsayan): The branch predicates assert their correctness
+        if is_full {
+            *i += 3;
+            *entity = MaybeInit::new(Entity::full_entity_from_slice(&tok));
+        } else if is_half {
+            *i += 1;
+            *entity = MaybeInit::new(Entity::single_entity_from_slice(&tok));
+        }
+    }
+    is_full | is_half
+}
+
 pub(super) fn parse_insert<'a>(
-    src: &'a [Token],
+    tok: &'a [Token],
     counter: &mut usize,
 ) -> LangResult<InsertStatement<'a>> {
     /*
         smallest:
-        insert space:primary_key ()
-        ^1     ^2   ^3^4         ^^5,6
+        insert into model (primarykey)
+               ^1    ^2   ^3      ^4 ^5
     */
-    let l = src.len();
-    let is_full = Entity::tokens_with_full(src);
-    let is_half = Entity::tokens_with_single(src);
-
-    let mut okay = is_full | is_half;
-    let mut i = 0;
+    let l = tok.len();
+    if compiler::unlikely(l < 5) {
+        return compiler::cold_err(Err(LangError::UnexpectedEndofStatement));
+    }
+    let mut okay = tok[0] == Token![into];
+    let mut i = okay as usize;
     let mut entity = MaybeInit::uninit();
-
-    okay &= process_entity(&src[i..], &mut entity, &mut i);
-
-    // primary key is a lit; atleast lit + (<oparen><cparen>) | (<obrace><cbrace>)
-    okay &= l >= (i + 4);
-    // colon, lit
-    okay &= src[i] == Token![:] && src[i + 1].is_lit();
-    // check data
-    let is_map = okay && src[i + 2] == Token![open {}];
-    let is_tuple = okay && src[i + 2] == Token![() open];
-    okay &= is_map | is_tuple;
-
-    if !okay {
-        return Err(LangError::UnexpectedToken);
+    okay &= parse_entity(&tok[i..], &mut entity, &mut i);
+    let mut data = None;
+    if !(i < l) {
+        unsafe {
+            // UNSAFE(@ohsayan): ALWAYS true because 1 + 3 for entity; early exit if smaller
+            impossible!();
+        }
     }
-
-    let primary_key = unsafe { extract!(&src[i+1], Token::Lit(l) => l) };
-    i += 3; // skip col, lit + op/ob
-
-    let data;
-    if is_tuple {
-        let (ord, cnt, ok) = parse_data_tuple_syntax(&src[i..]);
-        okay &= ok;
-        i += cnt;
-        data = InsertData::Ordered(ord);
-    } else {
-        let (map, cnt, ok) = parse_data_map_syntax(&src[i..]);
-        okay &= ok;
-        i += cnt;
-        data = InsertData::Map(map);
+    match tok[i] {
+        Token![() open] => {
+            let (this_data, incr, ok) = parse_data_tuple_syntax(&tok[i + 1..]);
+            okay &= ok;
+            i += incr + 1;
+            data = Some(InsertData::Ordered(this_data));
+        }
+        Token![open {}] => {
+            let (this_data, incr, ok) = parse_data_map_syntax(&tok[i + 1..]);
+            okay &= ok;
+            i += incr + 1;
+            data = Some(InsertData::Map(this_data));
+        }
+        _ => okay = false,
     }
-
     *counter += i;
-
     if okay {
+        let data = unsafe {
+            // UNSAFE(@ohsayan): Will be safe because of `okay` since it ensures that entity has been initialized
+            data.unwrap_unchecked()
+        };
         Ok(InsertStatement {
-            primary_key,
-            entity: unsafe { entity.assume_init() },
+            entity: unsafe {
+                // UNSAFE(@ohsayan): Will be safe because of `okay` since it ensures that entity has been initialized
+                entity.assume_init()
+            },
             data,
         })
     } else {
@@ -491,14 +554,39 @@ pub(super) fn parse_insert_full<'a>(tok: &'a [Token]) -> Option<InsertStatement<
 
 #[derive(Debug, PartialEq)]
 pub(super) struct SelectStatement<'a> {
-    /// the primary key
-    pub(super) primary_key: &'a Lit,
     /// the entity
     pub(super) entity: Entity,
     /// fields in order of querying. will be zero when wildcard is set
     pub(super) fields: Vec<RawSlice>,
     /// whether a wildcard was passed
     pub(super) wildcard: bool,
+    /// where clause
+    pub(super) clause: WhereClause<'a>,
+}
+impl<'a> SelectStatement<'a> {
+    #[inline(always)]
+    pub(crate) fn new_test(
+        entity: Entity,
+        fields: Vec<RawSlice>,
+        wildcard: bool,
+        clauses: HashMap<&'a [u8], RelationalExpr<'a>>,
+    ) -> SelectStatement<'a> {
+        Self::new(entity, fields, wildcard, clauses)
+    }
+    #[inline(always)]
+    fn new(
+        entity: Entity,
+        fields: Vec<RawSlice>,
+        wildcard: bool,
+        clauses: HashMap<&'a [u8], RelationalExpr<'a>>,
+    ) -> SelectStatement<'a> {
+        Self {
+            entity,
+            fields,
+            wildcard,
+            clause: WhereClause::new(clauses),
+        }
+    }
 }
 
 /// Parse a `select` query. The cursor should have already passed the `select` token when this
@@ -507,47 +595,62 @@ pub(super) fn parse_select<'a>(
     tok: &'a [Token],
     counter: &mut usize,
 ) -> LangResult<SelectStatement<'a>> {
+    /*
+        Smallest query:
+        select * from model
+               ^ ^    ^
+               1 2    3
+    */
     let l = tok.len();
-
-    let mut i = 0_usize;
-    let mut okay = l > 4;
-    let mut fields = Vec::new();
-    let is_wildcard = i < l && tok[i] == Token![*];
+    if compiler::unlikely(l < 3) {
+        return compiler::cold_err(Err(LangError::UnexpectedEndofStatement));
+    }
+    let mut i = 0;
+    let mut okay = true;
+    let mut select_fields = Vec::new();
+    let is_wildcard = tok[0] == Token![*];
     i += is_wildcard as usize;
-
-    while okay && i < l && tok[i].is_ident() && !is_wildcard {
-        unsafe {
-            fields.push(extract!(&tok[i], Token::Ident(id) => id.clone()));
+    while i < l && okay && !is_wildcard {
+        match tok[i] {
+            Token::Ident(ref id) => select_fields.push(id.clone()),
+            _ => {
+                break;
+            }
         }
         i += 1;
-        // skip comma
-        let nx_comma = i < l && tok[i] == Token![,];
-        let nx_from = i < l && tok[i] == Token![from];
+        let nx_idx = cmp::min(i, l);
+        let nx_comma = tok[nx_idx] == Token![,] && i < l;
+        let nx_from = tok[nx_idx] == Token![from];
         okay &= nx_comma | nx_from;
         i += nx_comma as usize;
     }
-
-    okay &= i < l && tok[i] == Token![from];
+    okay &= is_wildcard | !select_fields.is_empty();
+    okay &= (i + 2) <= l;
+    if compiler::unlikely(!okay) {
+        return compiler::cold_err(Err(LangError::UnexpectedToken));
+    }
+    okay &= tok[i] == Token![from];
     i += okay as usize;
-
-    // parsed upto select a, b, c from ...; now parse entity and select
+    // now process entity
     let mut entity = MaybeInit::uninit();
     okay &= process_entity(&tok[i..], &mut entity, &mut i);
-
-    // now primary key
-    okay &= i < l && tok[i] == Token![:];
-    i += okay as usize;
-    okay &= i < l && tok[i].is_lit();
-
-    *counter += i + okay as usize;
-
+    let has_where = tok[cmp::min(i, l)] == Token![where];
+    i += has_where as usize;
+    let mut clauses = <_ as Default>::default();
+    if has_where {
+        okay &= WhereClause::parse_where_and_append_to(&tok[i..], &mut i, &mut clauses);
+        okay &= !clauses.is_empty(); // append doesn't enforce clause arity
+    }
+    *counter += i;
     if okay {
-        let primary_key = unsafe { extract!(tok[i], Token::Lit(ref l) => l) };
         Ok(SelectStatement {
-            primary_key,
-            entity: unsafe { entity.assume_init() },
-            fields,
+            entity: unsafe {
+                // UNSAFE(@ohsayan): `process_entity` and `okay` assert correctness
+                entity.assume_init()
+            },
+            fields: select_fields,
             wildcard: is_wildcard,
+            clause: WhereClause::new(clauses),
         })
     } else {
         Err(LangError::UnexpectedToken)
@@ -559,7 +662,7 @@ pub(super) fn parse_select<'a>(
 pub(super) fn parse_select_full<'a>(tok: &'a [Token]) -> Option<SelectStatement<'a>> {
     let mut i = 0;
     let r = self::parse_select(tok, &mut i);
-    assert!(i == tok.len(), "didn't use full length");
+    assert_full_tt!(i, tok.len());
     r.ok()
 }
 
@@ -645,6 +748,10 @@ impl<'a> AssignmentExpression<'a> {
 
         if okay {
             let expression = unsafe {
+                /*
+                   UNSAFE(@ohsayan): tok[0] is checked for being an ident early on; second, tok[i]
+                   is also checked for being a lit and then `okay` ensures correctness
+                */
                 AssignmentExpression {
                     lhs: extract!(tok[0], Token::Ident(ref r) => r.clone()),
                     rhs: extract!(tok[i], Token::Lit(ref l) => l),
@@ -663,7 +770,7 @@ pub(super) fn parse_expression_full<'a>(tok: &'a [Token]) -> Option<AssignmentEx
     let mut i = 0;
     let mut exprs = Vec::new();
     if AssignmentExpression::parse_and_append_expression(tok, &mut exprs, &mut i) {
-        full_tt!(i, tok.len());
+        assert_full_tt!(i, tok.len());
         Some(exprs.remove(0))
     } else {
         None
@@ -676,52 +783,86 @@ pub(super) fn parse_expression_full<'a>(tok: &'a [Token]) -> Option<AssignmentEx
 
 #[derive(Debug, PartialEq)]
 pub struct UpdateStatement<'a> {
-    pub(super) primary_key: &'a Lit,
     pub(super) entity: Entity,
     pub(super) expressions: Vec<AssignmentExpression<'a>>,
+    pub(super) wc: WhereClause<'a>,
 }
 
 impl<'a> UpdateStatement<'a> {
+    #[inline(always)]
+    #[cfg(test)]
+    pub fn new_test(
+        entity: Entity,
+        expressions: Vec<AssignmentExpression<'a>>,
+        wc: HashMap<&'a [u8], RelationalExpr<'a>>,
+    ) -> Self {
+        Self::new(entity, expressions, WhereClause::new(wc))
+    }
+    #[inline(always)]
+    pub fn new(
+        entity: Entity,
+        expressions: Vec<AssignmentExpression<'a>>,
+        wc: WhereClause<'a>,
+    ) -> Self {
+        Self {
+            entity,
+            expressions,
+            wc,
+        }
+    }
+    #[inline(always)]
     pub(super) fn parse_update(tok: &'a [Token], counter: &mut usize) -> LangResult<Self> {
+        /*
+            TODO(@ohsayan): Allow volcanoes
+            smallest tt:
+            update model SET x  =  1 where x = 1
+                   ^1    ^2  ^3 ^4 ^5^6    ^7^8^9
+        */
         let l = tok.len();
-        // TODO(@ohsayan): This would become 8 when we add `SET`. It isn't exactly needed but is for purely aesthetic purposes
-        let mut okay = l > 6;
-        let mut i = 0_usize;
-
-        // parse entity
+        if compiler::unlikely(l < 9) {
+            return compiler::cold_err(Err(LangError::UnexpectedEndofStatement));
+        }
+        let mut i = 0;
         let mut entity = MaybeInit::uninit();
-        okay &= process_entity(tok, &mut entity, &mut i);
-
-        // check if we have our primary key
-        okay &= i < l && tok[i] == Token![:];
-        i += okay as usize;
-        okay &= i < l && tok[i].is_lit();
-        let primary_key_location = i;
-        i += okay as usize;
-
-        // now parse expressions that we have to update
+        let mut okay = parse_entity(&tok[i..], &mut entity, &mut i);
+        if !((i + 6) <= l) {
+            unsafe {
+                // UNSAFE(@ohsayan): Obvious, just a hint; entity can fw by 3 max
+                impossible!();
+            }
+        }
+        okay &= tok[i] == Token![set];
+        i += 1; // ignore whatever we have here, even if it's broken
+        let mut nx_where = false;
         let mut expressions = Vec::new();
-        while i < l && okay {
+        while i < l && okay && !nx_where {
             okay &= AssignmentExpression::parse_and_append_expression(
                 &tok[i..],
                 &mut expressions,
                 &mut i,
             );
-            let nx_comma = i < l && tok[i] == Token![,];
-            // TODO(@ohsayan): Define the need for a semicolon; remember, no SQL unsafety!
-            let nx_over = i == l;
-            okay &= nx_comma | nx_over;
+            let nx_idx = cmp::min(i, l);
+            let nx_comma = tok[nx_idx] == Token![,] && i < l;
+            // NOTE: volcano
+            nx_where = tok[nx_idx] == Token![where] && i < l;
+            okay &= nx_comma | nx_where; // NOTE: volcano
             i += nx_comma as usize;
         }
+        okay &= nx_where;
+        i += okay as usize;
+        // now process expressions
+        let mut clauses = <_ as Default>::default();
+        okay &= WhereClause::parse_where_and_append_to(&tok[i..], &mut i, &mut clauses);
+        okay &= !clauses.is_empty(); // NOTE: volcano
         *counter += i;
-
         if okay {
-            let primary_key =
-                unsafe { extract!(tok[primary_key_location], Token::Lit(ref pk) => pk) };
             Ok(Self {
-                primary_key,
-                entity: unsafe { entity.assume_init() },
+                entity: unsafe {
+                    // UNSAFE(@ohsayan): This is safe because of `parse_entity` and `okay`
+                    entity.assume_init()
+                },
                 expressions,
+                wc: WhereClause::new(clauses),
             })
         } else {
             Err(LangError::UnexpectedToken)
@@ -733,7 +874,7 @@ impl<'a> UpdateStatement<'a> {
 pub(super) fn parse_update_full<'a>(tok: &'a [Token]) -> LangResult<UpdateStatement<'a>> {
     let mut i = 0;
     let r = UpdateStatement::parse_update(tok, &mut i);
-    full_tt!(i, tok.len());
+    assert_full_tt!(i, tok.len());
     r
 }
 
@@ -746,43 +887,56 @@ pub(super) fn parse_update_full<'a>(tok: &'a [Token]) -> LangResult<UpdateStatem
 
 #[derive(Debug, PartialEq)]
 pub(super) struct DeleteStatement<'a> {
-    pub(super) primary_key: &'a Lit,
     pub(super) entity: Entity,
+    pub(super) wc: WhereClause<'a>,
 }
 
 impl<'a> DeleteStatement<'a> {
     #[inline(always)]
-    pub(super) fn new(primary_key: &'a Lit, entity: Entity) -> Self {
-        Self {
-            primary_key,
-            entity,
-        }
+    pub(super) fn new(entity: Entity, wc: WhereClause<'a>) -> Self {
+        Self { entity, wc }
+    }
+    #[inline(always)]
+    #[cfg(test)]
+    pub(super) fn new_test(entity: Entity, wc: HashMap<&'a [u8], RelationalExpr<'a>>) -> Self {
+        Self::new(entity, WhereClause::new(wc))
     }
     pub(super) fn parse_delete(tok: &'a [Token], counter: &mut usize) -> LangResult<Self> {
+        /*
+            TODO(@ohsayan): Volcano
+            smallest tt:
+            delete from model where x = 1
+                   ^1   ^2    ^3    ^4  ^5
+        */
         let l = tok.len();
-        let mut okay = l > 2;
-        let mut i = 0_usize;
-
-        // parse entity
+        if compiler::unlikely(l < 5) {
+            return compiler::cold_err(Err(LangError::UnexpectedEndofStatement));
+        }
+        let mut i = 0;
+        let mut okay = tok[i] == Token![from];
+        i += 1; // skip even if incorrect
         let mut entity = MaybeInit::uninit();
-        okay &= process_entity(tok, &mut entity, &mut i);
-
-        // find primary key
-        okay &= i < l && tok[i] == Token![:];
-        i += okay as usize;
-        okay &= i < l && tok[i].is_lit();
-        let primary_key_idx = i;
-        i += okay as usize;
-
-        *counter += i;
-
-        if okay {
+        okay &= parse_entity(&tok[i..], &mut entity, &mut i);
+        if !(i < l) {
             unsafe {
-                Ok(Self {
-                    primary_key: extract!(tok[primary_key_idx], Token::Lit(ref l) => l),
-                    entity: entity.assume_init(),
-                })
+                // UNSAFE(@ohsayan): Obvious, we have atleast 5, used max 4
+                impossible!();
             }
+        }
+        okay &= tok[i] == Token![where]; // NOTE: volcano
+        i += 1; // skip even if incorrect
+        let mut clauses = <_ as Default>::default();
+        okay &= WhereClause::parse_where_and_append_to(&tok[i..], &mut i, &mut clauses);
+        okay &= !clauses.is_empty();
+        *counter += i;
+        if okay {
+            Ok(Self {
+                entity: unsafe {
+                    // UNSAFE(@ohsayan): obvious due to `okay` and `parse_entity`
+                    entity.assume_init()
+                },
+                wc: WhereClause::new(clauses),
+            })
         } else {
             Err(LangError::UnexpectedToken)
         }
@@ -793,6 +947,6 @@ impl<'a> DeleteStatement<'a> {
 pub(super) fn parse_delete_full<'a>(tok: &'a [Token]) -> LangResult<DeleteStatement<'a>> {
     let mut i = 0_usize;
     let r = DeleteStatement::parse_delete(tok, &mut i);
-    full_tt!(i, tok.len());
+    assert_full_tt!(i, tok.len());
     r
 }
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 0feb0096..24d78555 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -363,8 +363,7 @@ fn kwph(k: &[u8]) -> u8 {
 }
 
 #[inline(always)]
-fn kwof(key: &str) -> Option<Keyword> {
-    let key = key.as_bytes();
+fn kwof(key: &[u8]) -> Option<Keyword> {
     let ph = kwph(key);
     if ph < KW_LUT.len() as u8 && KW_LUT[ph as usize].0 == key {
         Some(KW_LUT[ph as usize].1)
@@ -498,11 +497,11 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
 
     fn scan_ident_or_keyword(&mut self) {
         let s = self.scan_ident();
-        let st = unsafe { s.as_str() };
-        match kwof(st) {
+        let st = unsafe { s.as_slice() }.to_ascii_lowercase();
+        match kwof(&st) {
             Some(kw) => self.tokens.push(kw.into()),
             // FIXME(@ohsayan): Uh, mind fixing this? The only advantage is that I can keep the graph *memory* footprint small
-            None if st == "true" || st == "false" => self.push_token(Lit::Bool(st == "true")),
+            None if st == b"true" || st == b"false" => self.push_token(Lit::Bool(st == b"true")),
             None => self.tokens.push(Token::Ident(s)),
         }
     }
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index f55b9dd8..8bdd9d53 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -25,7 +25,7 @@
 */
 
 #[cfg(test)]
-macro_rules! full_tt {
+macro_rules! assert_full_tt {
     ($a:expr, $b:expr) => {
         assert_eq!($a, $b, "full token stream not utilized")
     };
@@ -180,6 +180,9 @@ macro_rules! Token {
         __kw!(Truncate)
     };
     // dml misc
+    (set) => {
+        __kw!(Set)
+    };
     (limit) => {
         __kw!(Limit)
     };
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 688ec990..1b728ad9 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -39,7 +39,7 @@ mod tests;
 
 #[cfg(test)]
 use core::{fmt, ops::Deref};
-use core::{mem, slice, str};
+use core::{mem, ptr::NonNull, slice, str};
 
 /*
     Lang errors
@@ -78,7 +78,7 @@ pub enum LangError {
 #[cfg_attr(not(test), derive(Debug))]
 #[derive(Clone)]
 pub struct RawSlice {
-    ptr: *const u8,
+    ptr: NonNull<u8>,
     len: usize,
 }
 
@@ -90,13 +90,16 @@ impl RawSlice {
     const _EALIGN: () = assert!(mem::align_of::<Self>() == mem::align_of::<&[u8]>());
     const FAKE_SLICE: Self = unsafe { Self::new_from_str("") };
     const unsafe fn new(ptr: *const u8, len: usize) -> Self {
-        Self { ptr, len }
+        Self {
+            ptr: NonNull::new_unchecked(ptr.cast_mut()),
+            len,
+        }
     }
     const unsafe fn new_from_str(s: &str) -> Self {
         Self::new(s.as_bytes().as_ptr(), s.as_bytes().len())
     }
     unsafe fn as_slice(&self) -> &[u8] {
-        slice::from_raw_parts(self.ptr, self.len)
+        slice::from_raw_parts(self.ptr.as_ptr(), self.len)
     }
     unsafe fn as_str(&self) -> &str {
         str::from_utf8_unchecked(self.as_slice())
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 99919475..8885d655 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -2034,21 +2034,21 @@ mod dml_tests {
         #[test]
         fn insert_tuple_mini() {
             let x = lex(br#"
-                insert twitter.users:"sayan" ()
+                insert into twitter.users ("sayan")
             "#)
             .unwrap();
             let r = dml::parse_insert_full(&x[1..]).unwrap();
             let e = InsertStatement {
-                primary_key: &("sayan".to_string().into()),
                 entity: Entity::Full("twitter".into(), "users".into()),
-                data: vec![].into(),
+                data: into_array_nullable!["sayan"].to_vec().into(),
             };
             assert_eq!(e, r);
         }
         #[test]
         fn insert_tuple() {
             let x = lex(br#"
-                insert twitter.users:"sayan" (
+                insert into twitter.users (
+                    "sayan",
                     "Sayan",
                     "sayan@example.com",
                     true,
@@ -2059,18 +2059,25 @@ mod dml_tests {
             .unwrap();
             let r = dml::parse_insert_full(&x[1..]).unwrap();
             let e = InsertStatement {
-                primary_key: &("sayan".to_string().into()),
                 entity: Entity::Full("twitter".into(), "users".into()),
-                data: into_array_nullable!["Sayan", "sayan@example.com", true, 12345, 67890]
-                    .to_vec()
-                    .into(),
+                data: into_array_nullable![
+                    "sayan",
+                    "Sayan",
+                    "sayan@example.com",
+                    true,
+                    12345,
+                    67890
+                ]
+                .to_vec()
+                .into(),
             };
             assert_eq!(e, r);
         }
         #[test]
         fn insert_tuple_pro() {
             let x = lex(br#"
-                insert twitter.users:"sayan" (
+                insert into twitter.users (
+                    "sayan",
                     "Sayan",
                     "sayan@example.com",
                     true,
@@ -2084,9 +2091,9 @@ mod dml_tests {
             .unwrap();
             let r = dml::parse_insert_full(&x[1..]).unwrap();
             let e = InsertStatement {
-                primary_key: &("sayan".to_string().into()),
                 entity: Entity::Full("twitter".into(), "users".into()),
                 data: into_array_nullable![
+                    "sayan",
                     "Sayan",
                     "sayan@example.com",
                     true,
@@ -2103,19 +2110,25 @@ mod dml_tests {
         }
         #[test]
         fn insert_map_mini() {
-            let tok = lex(br#"insert jotsy.app:"sayan" {}"#).unwrap();
+            let tok = lex(br#"
+                insert into jotsy.app { username: "sayan" }
+            "#)
+            .unwrap();
             let r = dml::parse_insert_full(&tok[1..]).unwrap();
             let e = InsertStatement {
-                primary_key: &("sayan".to_string().into()),
                 entity: Entity::Full("jotsy".into(), "app".into()),
-                data: nullable_dict! {}.into(),
+                data: dict_nullable! {
+                    "username".as_bytes() => "sayan"
+                }
+                .into(),
             };
             assert_eq!(e, r);
         }
         #[test]
         fn insert_map() {
             let tok = lex(br#"
-                insert jotsy.app:"sayan" {
+                insert into jotsy.app {
+                    username: "sayan",
                     name: "Sayan",
                     email: "sayan@example.com",
                     verified: true,
@@ -2126,9 +2139,9 @@ mod dml_tests {
             .unwrap();
             let r = dml::parse_insert_full(&tok[1..]).unwrap();
             let e = InsertStatement {
-                primary_key: &("sayan".to_string().into()),
                 entity: Entity::Full("jotsy".into(), "app".into()),
                 data: dict_nullable! {
+                    "username".as_bytes() => "sayan",
                     "name".as_bytes() => "Sayan",
                     "email".as_bytes() => "sayan@example.com",
                     "verified".as_bytes() => true,
@@ -2142,7 +2155,8 @@ mod dml_tests {
         #[test]
         fn insert_map_pro() {
             let tok = lex(br#"
-                insert jotsy.app:"sayan" {
+                insert into jotsy.app {
+                    username: "sayan",
                     password: "pass123",
                     email: "sayan@example.com",
                     verified: true,
@@ -2156,9 +2170,9 @@ mod dml_tests {
             .unwrap();
             let r = dml::parse_insert_full(&tok[1..]).unwrap();
             let e = InsertStatement {
-                primary_key: &("sayan".to_string()).into(),
                 entity: Entity::Full("jotsy".into(), "app".into()),
                 data: dict_nullable! {
+                    "username".as_bytes() => "sayan",
                     "password".as_bytes() => "pass123",
                     "email".as_bytes() => "sayan@example.com",
                     "verified".as_bytes() => true,
@@ -2175,72 +2189,93 @@ mod dml_tests {
     }
 
     mod stmt_select {
+        use crate::engine::ql::dml::RelationalExpr;
+
         use {
             super::*,
             crate::engine::ql::{
                 ast::Entity,
                 dml::{self, SelectStatement},
-                lexer::Lit,
             },
         };
         #[test]
         fn select_mini() {
             let tok = lex(br#"
-                select * from users:"sayan"
+                select * from users where username = "sayan"
             "#)
             .unwrap();
             let r = dml::parse_select_full(&tok[1..]).unwrap();
-            let e = SelectStatement {
-                primary_key: &Lit::Str("sayan".into()),
-                entity: Entity::Single("users".into()),
-                fields: [].to_vec(),
-                wildcard: true,
-            };
+            let username_where = "sayan".into();
+            let e = SelectStatement::new_test(
+                Entity::Single("users".into()),
+                [].to_vec(),
+                true,
+                dict! {
+                    "username".as_bytes() => RelationalExpr::new(
+                        "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
+                    ),
+                },
+            );
             assert_eq!(r, e);
         }
         #[test]
         fn select() {
             let tok = lex(br#"
-                select field1 from users:"sayan"
+                select field1 from users where username = "sayan"
             "#)
             .unwrap();
             let r = dml::parse_select_full(&tok[1..]).unwrap();
-            let e = SelectStatement {
-                primary_key: &Lit::Str("sayan".into()),
-                entity: Entity::Single("users".into()),
-                fields: ["field1".into()].to_vec(),
-                wildcard: false,
-            };
+            let username_where = "sayan".into();
+            let e = SelectStatement::new_test(
+                Entity::Single("users".into()),
+                ["field1".into()].to_vec(),
+                false,
+                dict! {
+                    "username".as_bytes() => RelationalExpr::new(
+                        "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
+                    ),
+                },
+            );
             assert_eq!(r, e);
         }
         #[test]
         fn select_pro() {
             let tok = lex(br#"
-                select field1 from twitter.users:"sayan"
+                select field1 from twitter.users where username = "sayan"
             "#)
             .unwrap();
             let r = dml::parse_select_full(&tok[1..]).unwrap();
-            let e = SelectStatement {
-                primary_key: &Lit::Str("sayan".into()),
-                entity: Entity::Full("twitter".into(), "users".into()),
-                fields: ["field1".into()].to_vec(),
-                wildcard: false,
-            };
+            let username_where = "sayan".into();
+            let e = SelectStatement::new_test(
+                Entity::Full("twitter".into(), "users".into()),
+                ["field1".into()].to_vec(),
+                false,
+                dict! {
+                    "username".as_bytes() => RelationalExpr::new(
+                        "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
+                    ),
+                },
+            );
             assert_eq!(r, e);
         }
         #[test]
         fn select_pro_max() {
             let tok = lex(br#"
-                select field1, field2 from twitter.users:"sayan"
+                select field1, field2 from twitter.users where username = "sayan"
             "#)
             .unwrap();
             let r = dml::parse_select_full(&tok[1..]).unwrap();
-            let e = SelectStatement {
-                primary_key: &Lit::Str("sayan".into()),
-                entity: Entity::Full("twitter".into(), "users".into()),
-                fields: ["field1".into(), "field2".into()].to_vec(),
-                wildcard: false,
-            };
+            let username_where = "sayan".into();
+            let e = SelectStatement::new_test(
+                Entity::Full("twitter".into(), "users".into()),
+                ["field1".into(), "field2".into()].to_vec(),
+                false,
+                dict! {
+                    "username".as_bytes() => RelationalExpr::new(
+                        "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
+                    ),
+                },
+            );
             assert_eq!(r, e);
         }
     }
@@ -2323,46 +2358,67 @@ mod dml_tests {
             super::*,
             crate::engine::ql::{
                 ast::Entity,
-                dml::{self, AssignmentExpression, Operator, UpdateStatement},
+                dml::{
+                    self, AssignmentExpression, Operator, RelationalExpr, UpdateStatement,
+                    WhereClause,
+                },
             },
         };
         #[test]
         fn update_mini() {
             let tok = lex(br#"
-                update app:"sayan" notes += "this is my new note"
+                update app SET notes += "this is my new note" where username = "sayan"
             "#)
             .unwrap();
+            let where_username = "sayan".into();
             let note = "this is my new note".to_string().into();
             let r = dml::parse_update_full(&tok[1..]).unwrap();
             let e = UpdateStatement {
-                primary_key: &("sayan".to_owned().into()),
                 entity: Entity::Single("app".into()),
                 expressions: vec![AssignmentExpression {
                     lhs: "notes".into(),
                     rhs: &note,
                     operator_fn: Operator::AddAssign,
                 }],
+                wc: WhereClause::new(dict! {
+                    "username".as_bytes() => RelationalExpr::new(
+                        "username".as_bytes(),
+                        &where_username,
+                        RelationalExpr::OP_EQ
+                    )
+                }),
             };
             assert_eq!(r, e);
         }
         #[test]
         fn update() {
             let tok = lex(br#"
-                update jotsy.app:"sayan" notes += "this is my new note", email = "sayan@example.com"
+                update
+                    jotsy.app
+                SET
+                    notes += "this is my new note",
+                    email = "sayan@example.com"
+                WHERE
+                    username = "sayan"
             "#)
             .unwrap();
             let r = dml::parse_update_full(&tok[1..]).unwrap();
-
+            let where_username = "sayan".into();
             let field_note = "this is my new note".into();
             let field_email = "sayan@example.com".into();
-            let primary_key = "sayan".into();
             let e = UpdateStatement {
-                primary_key: &primary_key,
                 entity: ("jotsy", "app").into(),
                 expressions: vec![
                     AssignmentExpression::new("notes".into(), &field_note, Operator::AddAssign),
                     AssignmentExpression::new("email".into(), &field_email, Operator::Assign),
                 ],
+                wc: WhereClause::new(dict! {
+                    "username".as_bytes() => RelationalExpr::new(
+                        "username".as_bytes(),
+                        &where_username,
+                        RelationalExpr::OP_EQ
+                    )
+                }),
             };
 
             assert_eq!(r, e);
@@ -2373,29 +2429,47 @@ mod dml_tests {
             super::*,
             crate::engine::ql::{
                 ast::Entity,
-                dml::{self, DeleteStatement},
+                dml::{self, DeleteStatement, RelationalExpr},
             },
         };
 
         #[test]
         fn delete_mini() {
             let tok = lex(br#"
-                delete users:"sayan"
+                delete from users where username = "sayan"
             "#)
             .unwrap();
             let primary_key = "sayan".into();
-            let e = DeleteStatement::new(&primary_key, Entity::Single("users".into()));
+            let e = DeleteStatement::new_test(
+                Entity::Single("users".into()),
+                dict! {
+                    "username".as_bytes() => RelationalExpr::new(
+                        "username".as_bytes(),
+                        &primary_key,
+                        RelationalExpr::OP_EQ
+                    )
+                },
+            );
             let r = dml::parse_delete_full(&tok[1..]).unwrap();
             assert_eq!(r, e);
         }
         #[test]
         fn delete() {
             let tok = lex(br#"
-                delete twitter.users:"sayan"
+                delete from twitter.users where username = "sayan"
             "#)
             .unwrap();
             let primary_key = "sayan".into();
-            let e = DeleteStatement::new(&primary_key, ("twitter", "users").into());
+            let e = DeleteStatement::new_test(
+                ("twitter", "users").into(),
+                dict! {
+                    "username".as_bytes() => RelationalExpr::new(
+                        "username".as_bytes(),
+                        &primary_key,
+                        RelationalExpr::OP_EQ
+                    )
+                },
+            );
             let r = dml::parse_delete_full(&tok[1..]).unwrap();
             assert_eq!(r, e);
         }
@@ -2528,5 +2602,13 @@ mod dml_tests {
             });
             assert_eq!(expected, dml::parse_where_clause_full(&tok).unwrap());
         }
+        #[test]
+        fn where_duplicate_condition() {
+            let tok = lex(br#"
+                userid = 100 and userid > 200
+            "#)
+            .unwrap();
+            assert!(dml::parse_where_clause_full(&tok).is_none());
+        }
     }
 }
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index 95f6a5d1..252d4f53 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -230,6 +230,7 @@ pub struct MaybeInit<T> {
 
 impl<T> MaybeInit<T> {
     /// Initialize a new uninitialized variant
+    #[inline(always)]
     pub const fn uninit() -> Self {
         Self {
             #[cfg(test)]
@@ -238,6 +239,7 @@ impl<T> MaybeInit<T> {
         }
     }
     /// Initialize with a value
+    #[inline(always)]
     pub const fn new(val: T) -> Self {
         Self {
             #[cfg(test)]
@@ -250,6 +252,7 @@ impl<T> MaybeInit<T> {
     /// ## Safety
     ///
     /// Caller needs to ensure that the data is actually initialized
+    #[inline(always)]
     pub const unsafe fn assume_init(self) -> T {
         #[cfg(test)]
         {
@@ -264,6 +267,7 @@ impl<T> MaybeInit<T> {
     /// ## Safety
     ///
     /// Caller needs to ensure that the data is actually initialized
+    #[inline(always)]
     pub const unsafe fn assume_init_ref(&self) -> &T {
         #[cfg(test)]
         {

From 79050e5fff2b210274705c7f076ab1e2b105f7e8 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 14 Dec 2022 10:31:50 +0530
Subject: [PATCH 054/310] Rename `UnsafeLit` to `SafeLit`

This is more logical due to the inherent meaning of `unsafe`
---
 server/src/actions/get.rs            |  2 +-
 server/src/actions/strong/sdel.rs    |  2 +-
 server/src/actions/strong/sset.rs    |  2 +-
 server/src/actions/strong/supdate.rs |  2 +-
 server/src/engine/memory/mod.rs      | 20 +++++--
 server/src/engine/ql/benches.rs      |  2 +-
 server/src/engine/ql/dml.rs          | 59 ++++++-------------
 server/src/engine/ql/lexer.rs        | 86 ++++++++++++++--------------
 server/src/engine/ql/macros.rs       |  8 +++
 server/src/engine/ql/mod.rs          |  8 ++-
 server/src/engine/ql/tests.rs        |  6 +-
 server/src/util/compiler.rs          |  8 ++-
 12 files changed, 106 insertions(+), 99 deletions(-)

diff --git a/server/src/actions/get.rs b/server/src/actions/get.rs
index 868e39fb..5240ce12 100644
--- a/server/src/actions/get.rs
+++ b/server/src/actions/get.rs
@@ -44,7 +44,7 @@ action!(
                     con.write_mono_length_prefixed_with_tsymbol(&val, kve.get_value_tsymbol())
                         .await?
                 }
-                Err(_) => compiler::cold_err(con._write_raw(P::RCODE_ENCODING_ERROR)).await?,
+                Err(_) => compiler::cold_val(con._write_raw(P::RCODE_ENCODING_ERROR)).await?,
                 Ok(_) => con._write_raw(P::RCODE_NIL).await?,
             }
         }
diff --git a/server/src/actions/strong/sdel.rs b/server/src/actions/strong/sdel.rs
index 27ae991a..baff6aef 100644
--- a/server/src/actions/strong/sdel.rs
+++ b/server/src/actions/strong/sdel.rs
@@ -109,7 +109,7 @@ pub(super) fn snapshot_and_del<'a, T: 'a + DerefUnsafeSlice>(
         do_sleep!(10 s);
     });
     if compiler::unlikely(err_enc) {
-        return compiler::cold_err(StrongActionResult::EncodingError);
+        return compiler::cold_val(StrongActionResult::EncodingError);
     }
     if registry::state_okay() {
         // guarantee upholded: consistency
diff --git a/server/src/actions/strong/sset.rs b/server/src/actions/strong/sset.rs
index f8157e5b..dca70564 100644
--- a/server/src/actions/strong/sset.rs
+++ b/server/src/actions/strong/sset.rs
@@ -99,7 +99,7 @@ pub(super) fn snapshot_and_insert<'a, T: 'a + DerefUnsafeSlice>(
         do_sleep!(10 s);
     });
     if compiler::unlikely(enc_err) {
-        return compiler::cold_err(StrongActionResult::EncodingError);
+        return compiler::cold_val(StrongActionResult::EncodingError);
     }
     if registry::state_okay() {
         if key_iter_stat_ok {
diff --git a/server/src/actions/strong/supdate.rs b/server/src/actions/strong/supdate.rs
index d6f7dd2d..836b25c6 100644
--- a/server/src/actions/strong/supdate.rs
+++ b/server/src/actions/strong/supdate.rs
@@ -108,7 +108,7 @@ pub(super) fn snapshot_and_update<'a, T: 'a + DerefUnsafeSlice>(
         do_sleep!(10 s);
     });
     if compiler::unlikely(enc_err) {
-        return compiler::cold_err(StrongActionResult::EncodingError);
+        return compiler::cold_val(StrongActionResult::EncodingError);
     }
     if registry::state_okay() {
         // uphold consistency
diff --git a/server/src/engine/memory/mod.rs b/server/src/engine/memory/mod.rs
index a487b7b0..f638cc5d 100644
--- a/server/src/engine/memory/mod.rs
+++ b/server/src/engine/memory/mod.rs
@@ -26,15 +26,14 @@
 
 // TODO(@ohsayan): Change the underlying structures, there are just rudimentary ones used during integration with the QL
 
-use super::ql::RawSlice;
+use super::ql::{lexer::Lit, RawSlice};
 
 /// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
 /// be of one type.
-#[derive(Debug, PartialEq)]
-#[cfg_attr(test, derive(Clone))]
+#[derive(Debug, PartialEq, Clone)]
 pub enum DataType {
     /// An UTF-8 string
-    String(String),
+    String(Box<str>),
     /// Bytes
     Binary(Vec<u8>),
     /// An unsigned integer
@@ -70,6 +69,19 @@ enum_impls! {
     }
 }
 
+impl DataType {
+    #[inline(always)]
+    pub(super) fn clone_from_lit(lit: &Lit) -> Self {
+        match lit {
+            Lit::Str(s) => DataType::String(s.clone()),
+            Lit::Bool(b) => DataType::Boolean(*b),
+            Lit::UnsignedInt(u) => DataType::UnsignedInt(*u),
+            Lit::SignedInt(i) => DataType::SignedInt(*i),
+            Lit::SafeLit(l) => DataType::AnonymousTypeNeedsEval(l.clone()),
+        }
+    }
+}
+
 impl<const N: usize> From<[DataType; N]> for DataType {
     fn from(f: [DataType; N]) -> Self {
         Self::List(f.into())
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index 7b6e31ae..0252ce78 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -74,7 +74,7 @@ mod lexer {
     #[bench]
     fn lex_raw_literal(b: &mut Bencher) {
         let src = b"\r44\ne69b10ffcc250ae5091dec6f299072e23b0b41d6a739";
-        let expected = vec![Token::Lit(Lit::UnsafeLit(RawSlice::from(
+        let expected = vec![Token::Lit(Lit::SafeLit(RawSlice::from(
             "e69b10ffcc250ae5091dec6f299072e23b0b41d6a739",
         )))];
         b.iter(|| assert_eq!(lex(src).unwrap(), expected));
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 24ca94c4..9f053fe8 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -145,19 +145,21 @@ impl<'a> RelationalExpr<'a> {
                 }
             })
         } else {
-            compiler::cold_err(None)
+            compiler::cold_val(None)
         }
     }
 }
 
 #[derive(Debug, PartialEq)]
 pub struct WhereClause<'a> {
-    c: HashMap<&'a [u8], RelationalExpr<'a>>,
+    c: WhereClauseCollection<'a>,
 }
 
+type WhereClauseCollection<'a> = HashMap<&'a [u8], RelationalExpr<'a>>;
+
 impl<'a> WhereClause<'a> {
     #[inline(always)]
-    pub(super) fn new(c: HashMap<&'a [u8], RelationalExpr<'a>>) -> Self {
+    pub(super) fn new(c: WhereClauseCollection<'a>) -> Self {
         Self { c }
     }
     #[inline(always)]
@@ -169,7 +171,7 @@ impl<'a> WhereClause<'a> {
     fn parse_where_and_append_to(
         tok: &'a [Token],
         cnt: &mut usize,
-        c: &mut HashMap<&'a [u8], RelationalExpr<'a>>,
+        c: &mut WhereClauseCollection<'a>,
     ) -> bool {
         let l = tok.len();
         let mut okay = true;
@@ -235,13 +237,7 @@ pub(super) fn parse_list(
     let mut prev_nlist_dscr = None;
     while i < l && okay && !stop {
         let d = match &tok[i] {
-            Token::Lit(l) => match l {
-                Lit::Str(s) => DataType::String(s.to_string()),
-                Lit::UnsignedInt(n) => DataType::UnsignedInt(*n),
-                Lit::Bool(b) => DataType::Boolean(*b),
-                Lit::UnsafeLit(l) => DataType::AnonymousTypeNeedsEval(l.clone()),
-                Lit::SignedInt(uint) => DataType::SignedInt(*uint),
-            },
+            Token::Lit(l) => DataType::clone_from_lit(l),
             Token::Symbol(Symbol::TtOpenSqBracket) => {
                 // a nested list
                 let mut nested_list = Vec::new();
@@ -298,19 +294,7 @@ pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<Option<DataType>>,
     let mut data = Vec::new();
     while i < l && okay && !stop {
         match &tok[i] {
-            Token::Lit(l) => match l {
-                Lit::Str(s) => {
-                    data.push(Some(s.to_string().into()));
-                }
-                Lit::UnsignedInt(n) => {
-                    data.push(Some((*n).into()));
-                }
-                Lit::Bool(b) => {
-                    data.push(Some((*b).into()));
-                }
-                Lit::UnsafeLit(r) => data.push(Some(DataType::AnonymousTypeNeedsEval(r.clone()))),
-                Lit::SignedInt(int) => data.push(Some(DataType::SignedInt(*int))),
-            },
+            Token::Lit(l) => data.push(Some(DataType::clone_from_lit(l))),
             Token::Symbol(Symbol::TtOpenSqBracket) => {
                 // ah, a list
                 let mut l = Vec::new();
@@ -361,20 +345,13 @@ pub(super) fn parse_data_map_syntax<'a>(
         okay &= colon == &Symbol::SymColon;
         match (field, expression) {
             (Token::Ident(id), Token::Lit(l)) => {
-                let dt = match l {
-                    Lit::Str(s) => s.to_string().into(),
-                    Lit::Bool(b) => (*b).into(),
-                    Lit::UnsignedInt(s) => (*s).into(),
-                    Lit::UnsafeLit(l) => DataType::AnonymousTypeNeedsEval(l.clone()),
-                    Lit::SignedInt(int) => DataType::SignedInt(*int),
-                };
                 okay &= data
                     .insert(
                         unsafe {
                             // UNSAFE(@ohsayan): Token lifetime ensures slice validity
                             id.as_slice()
                         },
-                        Some(dt),
+                        Some(DataType::clone_from_lit(l)),
                     )
                     .is_none();
             }
@@ -494,7 +471,7 @@ pub(super) fn parse_insert<'a>(
     */
     let l = tok.len();
     if compiler::unlikely(l < 5) {
-        return compiler::cold_err(Err(LangError::UnexpectedEndofStatement));
+        return compiler::cold_val(Err(LangError::UnexpectedEndofStatement));
     }
     let mut okay = tok[0] == Token![into];
     let mut i = okay as usize;
@@ -569,7 +546,7 @@ impl<'a> SelectStatement<'a> {
         entity: Entity,
         fields: Vec<RawSlice>,
         wildcard: bool,
-        clauses: HashMap<&'a [u8], RelationalExpr<'a>>,
+        clauses: WhereClauseCollection<'a>,
     ) -> SelectStatement<'a> {
         Self::new(entity, fields, wildcard, clauses)
     }
@@ -578,7 +555,7 @@ impl<'a> SelectStatement<'a> {
         entity: Entity,
         fields: Vec<RawSlice>,
         wildcard: bool,
-        clauses: HashMap<&'a [u8], RelationalExpr<'a>>,
+        clauses: WhereClauseCollection<'a>,
     ) -> SelectStatement<'a> {
         Self {
             entity,
@@ -603,7 +580,7 @@ pub(super) fn parse_select<'a>(
     */
     let l = tok.len();
     if compiler::unlikely(l < 3) {
-        return compiler::cold_err(Err(LangError::UnexpectedEndofStatement));
+        return compiler::cold_val(Err(LangError::UnexpectedEndofStatement));
     }
     let mut i = 0;
     let mut okay = true;
@@ -627,7 +604,7 @@ pub(super) fn parse_select<'a>(
     okay &= is_wildcard | !select_fields.is_empty();
     okay &= (i + 2) <= l;
     if compiler::unlikely(!okay) {
-        return compiler::cold_err(Err(LangError::UnexpectedToken));
+        return compiler::cold_val(Err(LangError::UnexpectedToken));
     }
     okay &= tok[i] == Token![from];
     i += okay as usize;
@@ -794,7 +771,7 @@ impl<'a> UpdateStatement<'a> {
     pub fn new_test(
         entity: Entity,
         expressions: Vec<AssignmentExpression<'a>>,
-        wc: HashMap<&'a [u8], RelationalExpr<'a>>,
+        wc: WhereClauseCollection<'a>,
     ) -> Self {
         Self::new(entity, expressions, WhereClause::new(wc))
     }
@@ -820,7 +797,7 @@ impl<'a> UpdateStatement<'a> {
         */
         let l = tok.len();
         if compiler::unlikely(l < 9) {
-            return compiler::cold_err(Err(LangError::UnexpectedEndofStatement));
+            return compiler::cold_val(Err(LangError::UnexpectedEndofStatement));
         }
         let mut i = 0;
         let mut entity = MaybeInit::uninit();
@@ -898,7 +875,7 @@ impl<'a> DeleteStatement<'a> {
     }
     #[inline(always)]
     #[cfg(test)]
-    pub(super) fn new_test(entity: Entity, wc: HashMap<&'a [u8], RelationalExpr<'a>>) -> Self {
+    pub(super) fn new_test(entity: Entity, wc: WhereClauseCollection<'a>) -> Self {
         Self::new(entity, WhereClause::new(wc))
     }
     pub(super) fn parse_delete(tok: &'a [Token], counter: &mut usize) -> LangResult<Self> {
@@ -910,7 +887,7 @@ impl<'a> DeleteStatement<'a> {
         */
         let l = tok.len();
         if compiler::unlikely(l < 5) {
-            return compiler::cold_err(Err(LangError::UnexpectedEndofStatement));
+            return compiler::cold_val(Err(LangError::UnexpectedEndofStatement));
         }
         let mut i = 0;
         let mut okay = tok[i] == Token![from];
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 24d78555..26551dff 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -76,7 +76,7 @@ pub enum Lit {
     Bool(bool),
     UnsignedInt(u64),
     SignedInt(i64),
-    UnsafeLit(RawSlice),
+    SafeLit(RawSlice),
 }
 
 impl From<&'static str> for Lit {
@@ -376,11 +376,12 @@ fn kwof(key: &[u8]) -> Option<Keyword> {
     Lexer impl
 */
 
-const LEXER_MODE_INSECURE: u8 = 0;
-const LEXER_MODE_SECURE: u8 = 1;
+pub(super) const LANG_MODE_INSECURE: u8 = 0;
+pub(super) const LANG_MODE_SECURE: u8 = 1;
 
-pub type InsecureLexer<'a> = Lexer<'a, LEXER_MODE_INSECURE>;
-pub type SecureLexer<'a> = Lexer<'a, LEXER_MODE_SECURE>;
+pub type OperatingMode = u8;
+pub type InsecureLexer<'a> = Lexer<'a, LANG_MODE_INSECURE>;
+pub type SecureLexer<'a> = Lexer<'a, LANG_MODE_SECURE>;
 
 pub struct Lexer<'a, const OPERATING_MODE: u8> {
     c: *const u8,
@@ -391,6 +392,7 @@ pub struct Lexer<'a, const OPERATING_MODE: u8> {
 }
 
 impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
+    #[inline(always)]
     pub const fn new(src: &'a [u8]) -> Self {
         unsafe {
             Self {
@@ -426,6 +428,7 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
     fn remaining(&self) -> usize {
         unsafe { self.e.offset_from(self.c) as usize }
     }
+    #[inline(always)]
     unsafe fn deref_cursor(&self) -> u8 {
         *self.cursor()
     }
@@ -482,9 +485,18 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
     fn trim_ahead(&mut self) {
         while self.peek_is_and_forward(|b| b == b' ' || b == b'\t' || b == b'\n') {}
     }
+    #[inline(always)]
+    fn set_error(&mut self, e: LangError) {
+        self.last_error = Some(e);
+    }
+    #[inline(always)]
+    fn no_error(&self) -> bool {
+        self.last_error.is_none()
+    }
 }
 
 impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
+    #[inline(always)]
     fn scan_ident(&mut self) -> RawSlice {
         let s = self.cursor();
         unsafe {
@@ -494,7 +506,7 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
             RawSlice::new(s, self.cursor().offset_from(s) as usize)
         }
     }
-
+    #[inline(always)]
     fn scan_ident_or_keyword(&mut self) {
         let s = self.scan_ident();
         let st = unsafe { s.as_slice() }.to_ascii_lowercase();
@@ -505,7 +517,7 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
             None => self.tokens.push(Token::Ident(s)),
         }
     }
-
+    #[inline(always)]
     fn scan_unsigned_integer(&mut self) {
         let s = self.cursor();
         unsafe {
@@ -529,11 +541,11 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
                 Ok(num) if compiler::likely(wseof) => {
                     self.tokens.push(Token::Lit(Lit::UnsignedInt(num)))
                 }
-                _ => self.last_error = Some(LangError::InvalidNumericLiteral),
+                _ => self.set_error(LangError::InvalidNumericLiteral),
             }
         }
     }
-
+    #[inline(always)]
     fn scan_quoted_string(&mut self, quote_style: u8) {
         debug_assert!(
             unsafe { self.deref_cursor() } == quote_style,
@@ -567,23 +579,21 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
             let terminated = self.peek_eq_and_forward(quote_style);
             match String::from_utf8(buf) {
                 Ok(st) if terminated => self.tokens.push(Token::Lit(st.into_boxed_str().into())),
-                _ => self.last_error = Some(LangError::InvalidStringLiteral),
+                _ => self.set_error(LangError::InvalidStringLiteral),
             }
         }
     }
+    #[inline(always)]
     fn scan_byte(&mut self, byte: u8) {
         match symof(byte) {
             Some(tok) => self.push_token(tok),
-            None => {
-                self.last_error = Some(LangError::UnexpectedChar);
-                return;
-            }
+            None => return self.set_error(LangError::UnexpectedChar),
         }
         unsafe {
             self.incr_cursor();
         }
     }
-
+    #[inline(always)]
     fn scan_unsafe_literal(&mut self) {
         unsafe {
             self.incr_cursor();
@@ -612,14 +622,13 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
         okay &= self.remaining() >= size;
         if compiler::likely(okay) {
             unsafe {
-                self.push_token(Lit::UnsafeLit(RawSlice::new(self.cursor(), size)));
+                self.push_token(Lit::SafeLit(RawSlice::new(self.cursor(), size)));
                 self.incr_cursor_by(size);
             }
         } else {
-            self.last_error = Some(LangError::InvalidUnsafeLiteral);
+            self.set_error(LangError::InvalidSafeLiteral);
         }
     }
-
     #[inline(always)]
     fn scan_signed_integer(&mut self) {
         unsafe {
@@ -646,16 +655,16 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
                     self.push_token(Lit::SignedInt(num));
                 }
                 _ => {
-                    compiler::cold_err(self.last_error = Some(LangError::InvalidNumericLiteral));
+                    compiler::cold_val(self.set_error(LangError::InvalidNumericLiteral));
                 }
             }
         } else {
             self.push_token(Token![-]);
         }
     }
-
+    #[inline(always)]
     fn _lex(&mut self) {
-        while self.not_exhausted() && self.last_error.is_none() {
+        while self.not_exhausted() && self.no_error() {
             match unsafe { self.deref_cursor() } {
                 // secure features
                 byte if byte.is_ascii_alphabetic() => self.scan_ident_or_keyword(),
@@ -669,11 +678,11 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
                 }
                 b'\r' => self.scan_unsafe_literal(),
                 // insecure features
-                byte if byte.is_ascii_digit() && OPERATING_MODE == LEXER_MODE_INSECURE => {
+                byte if byte.is_ascii_digit() && OPERATING_MODE == LANG_MODE_INSECURE => {
                     self.scan_unsigned_integer()
                 }
-                b'-' if OPERATING_MODE == LEXER_MODE_INSECURE => self.scan_signed_integer(),
-                qs @ (b'\'' | b'"') if OPERATING_MODE == LEXER_MODE_INSECURE => {
+                b'-' if OPERATING_MODE == LANG_MODE_INSECURE => self.scan_signed_integer(),
+                qs @ (b'\'' | b'"') if OPERATING_MODE == LANG_MODE_INSECURE => {
                     self.scan_quoted_string(qs)
                 }
                 // blank space or an arbitrary byte
@@ -682,7 +691,7 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
             }
         }
     }
-
+    #[inline(always)]
     pub fn lex(src: &'a [u8]) -> LangResult<Life<'a, Vec<Token>>> {
         let mut slf = Self::new(src);
         slf._lex();
@@ -699,31 +708,20 @@ impl Token {
         matches!(self, Token::Ident(_))
     }
     #[inline(always)]
-    pub(crate) fn as_ident_eq_ignore_case(&self, arg: &[u8]) -> bool {
-        self.is_ident()
-            && unsafe {
-                if let Self::Ident(id) = self {
-                    id.as_slice().eq_ignore_ascii_case(arg)
-                } else {
-                    impossible!()
-                }
-            }
-    }
-    #[inline(always)]
-    pub(super) unsafe fn ident_unchecked(&self) -> RawSlice {
-        if let Self::Ident(id) = self {
-            id.clone()
-        } else {
-            impossible!()
-        }
-    }
-    #[inline(always)]
     pub(super) const fn is_lit(&self) -> bool {
         matches!(self, Self::Lit(_))
     }
+    #[inline(always)]
+    /// Returns true if the token *could* be a positive number. since safe literals do not
+    /// provide any type information at this moment, we lookahead to see if it's a litnum or
+    /// an unsafe lit
+    pub(super) const fn is_like_positive_num(&self) -> bool {
+        matches!(self, Self::Lit(Lit::UnsignedInt(_) | Lit::SafeLit(_)))
+    }
 }
 
 impl AsRef<Token> for Token {
+    #[inline(always)]
     fn as_ref(&self) -> &Token {
         self
     }
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 8bdd9d53..1e4d48f1 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -316,3 +316,11 @@ macro_rules! into_array {
 macro_rules! into_array_nullable {
     ($($e:expr),* $(,)?) => { [$($crate::engine::ql::tests::nullable_datatype($e)),*] };
 }
+
+macro_rules! statictbl {
+    ($name:ident: $kind:ty => [$($expr:expr),+]) => {{
+        const LEN: usize = {let mut i = 0;$(let _ = $expr; i += 1;)*i};
+        static $name: [$kind; LEN] = [$($expr),*];
+        $name
+    }};
+}
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 1b728ad9..abfb1695 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -59,7 +59,7 @@ pub enum LangError {
     UnexpectedToken,
     InvalidDictionaryExpression,
     InvalidTypeDefinition,
-    InvalidUnsafeLiteral,
+    InvalidSafeLiteral,
 }
 
 /*
@@ -104,6 +104,12 @@ impl RawSlice {
     unsafe fn as_str(&self) -> &str {
         str::from_utf8_unchecked(self.as_slice())
     }
+    pub fn ptr(&self) -> NonNull<u8> {
+        self.ptr
+    }
+    pub fn len(&self) -> usize {
+        self.len
+    }
 }
 
 #[cfg(test)]
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 8885d655..b05717df 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -250,20 +250,20 @@ mod lexer_tests {
     fn lex_unsafe_literal_mini() {
         let usl = lex("\r0\n".as_bytes()).unwrap();
         assert_eq!(usl.len(), 1);
-        assert_eq!(Token::Lit(Lit::UnsafeLit("".into())), usl[0]);
+        assert_eq!(Token::Lit(Lit::SafeLit("".into())), usl[0]);
     }
     #[test]
     fn lex_unsafe_literal() {
         let usl = lex("\r9\nabcdefghi".as_bytes()).unwrap();
         assert_eq!(usl.len(), 1);
-        assert_eq!(Token::Lit(Lit::UnsafeLit("abcdefghi".into())), usl[0]);
+        assert_eq!(Token::Lit(Lit::SafeLit("abcdefghi".into())), usl[0]);
     }
     #[test]
     fn lex_unsafe_literal_pro() {
         let usl = lex("\r18\nabcdefghi123456789".as_bytes()).unwrap();
         assert_eq!(usl.len(), 1);
         assert_eq!(
-            Token::Lit(Lit::UnsafeLit("abcdefghi123456789".into())),
+            Token::Lit(Lit::SafeLit("abcdefghi123456789".into())),
             usl[0]
         );
     }
diff --git a/server/src/util/compiler.rs b/server/src/util/compiler.rs
index 56d042ed..c865dfbf 100644
--- a/server/src/util/compiler.rs
+++ b/server/src/util/compiler.rs
@@ -49,7 +49,7 @@ pub const fn unlikely(b: bool) -> bool {
 
 #[cold]
 #[inline(never)]
-pub const fn cold_err<T>(v: T) -> T {
+pub const fn cold_val<T>(v: T) -> T {
     v
 }
 #[inline(always)]
@@ -72,3 +72,9 @@ pub const unsafe fn extend_lifetime<'a, 'b, T>(inp: &'a T) -> &'b T {
 pub unsafe fn extend_lifetime_mut<'a, 'b, T>(inp: &'a mut T) -> &'b mut T {
     mem::transmute(inp)
 }
+
+#[cold]
+#[inline(never)]
+pub fn cold_rerr<T, E>(e: E) -> Result<T, E> {
+    Err(e)
+}

From 5a7145fa40eb67d3fbdaf4e98395c56ab9fe9a4a Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 18 Dec 2022 19:26:33 +0530
Subject: [PATCH 055/310] Implement parameterization and remove unsafe literals

Also refactored tests for clarity
---
 server/src/engine/memory/mod.rs             |    2 +-
 server/src/engine/ql/benches.rs             |    2 +-
 server/src/engine/ql/lexer.rs               |  511 +++-
 server/src/engine/ql/macros.rs              |    2 +-
 server/src/engine/ql/mod.rs                 |    1 +
 server/src/engine/ql/tests.rs               | 2504 +------------------
 server/src/engine/ql/tests/dml_tests.rs     |  866 +++++++
 server/src/engine/ql/tests/entity.rs        |   49 +
 server/src/engine/ql/tests/lexer_tests.rs   |  393 +++
 server/src/engine/ql/tests/schema_tests.rs  | 1219 +++++++++
 server/src/engine/ql/tests/structure_syn.rs |  302 +++
 11 files changed, 3336 insertions(+), 2515 deletions(-)
 create mode 100644 server/src/engine/ql/tests/dml_tests.rs
 create mode 100644 server/src/engine/ql/tests/entity.rs
 create mode 100644 server/src/engine/ql/tests/lexer_tests.rs
 create mode 100644 server/src/engine/ql/tests/schema_tests.rs
 create mode 100644 server/src/engine/ql/tests/structure_syn.rs

diff --git a/server/src/engine/memory/mod.rs b/server/src/engine/memory/mod.rs
index f638cc5d..5449dff3 100644
--- a/server/src/engine/memory/mod.rs
+++ b/server/src/engine/memory/mod.rs
@@ -77,7 +77,7 @@ impl DataType {
             Lit::Bool(b) => DataType::Boolean(*b),
             Lit::UnsignedInt(u) => DataType::UnsignedInt(*u),
             Lit::SignedInt(i) => DataType::SignedInt(*i),
-            Lit::SafeLit(l) => DataType::AnonymousTypeNeedsEval(l.clone()),
+            Lit::Bin(l) => DataType::AnonymousTypeNeedsEval(l.clone()),
         }
     }
 }
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index 0252ce78..0b9d5578 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -74,7 +74,7 @@ mod lexer {
     #[bench]
     fn lex_raw_literal(b: &mut Bencher) {
         let src = b"\r44\ne69b10ffcc250ae5091dec6f299072e23b0b41d6a739";
-        let expected = vec![Token::Lit(Lit::SafeLit(RawSlice::from(
+        let expected = vec![Token::Lit(Lit::Bin(RawSlice::from(
             "e69b10ffcc250ae5091dec6f299072e23b0b41d6a739",
         )))];
         b.iter(|| assert_eq!(lex(src).unwrap(), expected));
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 26551dff..0d7549d6 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -24,10 +24,12 @@
  *
 */
 
+use std::ops::BitOr;
+
 use {
     super::{LangError, LangResult, RawSlice},
     crate::util::{compiler, Life},
-    core::{marker::PhantomData, mem::size_of, slice, str},
+    core::{cmp, fmt, marker::PhantomData, mem::size_of, slice, str},
 };
 
 /*
@@ -76,7 +78,7 @@ pub enum Lit {
     Bool(bool),
     UnsignedInt(u64),
     SignedInt(i64),
-    SafeLit(RawSlice),
+    Bin(RawSlice),
 }
 
 impl From<&'static str> for Lit {
@@ -95,6 +97,7 @@ enum_impls! {
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
+#[repr(u8)]
 pub enum Symbol {
     OpArithmeticAdd,  // +
     OpArithmeticSub,  // -
@@ -394,14 +397,15 @@ pub struct Lexer<'a, const OPERATING_MODE: u8> {
 impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
     #[inline(always)]
     pub const fn new(src: &'a [u8]) -> Self {
-        unsafe {
-            Self {
-                c: src.as_ptr(),
-                e: src.as_ptr().add(src.len()),
-                last_error: None,
-                tokens: Vec::new(),
-                _lt: PhantomData,
-            }
+        Self {
+            c: src.as_ptr(),
+            e: unsafe {
+                // UNSAFE(@ohsayan): Always safe (<= EOA)
+                src.as_ptr().add(src.len())
+            },
+            last_error: None,
+            tokens: Vec::new(),
+            _lt: PhantomData,
         }
     }
 }
@@ -594,7 +598,7 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
         }
     }
     #[inline(always)]
-    fn scan_unsafe_literal(&mut self) {
+    fn scan_binary_literal(&mut self) {
         unsafe {
             self.incr_cursor();
         }
@@ -622,7 +626,7 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
         okay &= self.remaining() >= size;
         if compiler::likely(okay) {
             unsafe {
-                self.push_token(Lit::SafeLit(RawSlice::new(self.cursor(), size)));
+                self.push_token(Lit::Bin(RawSlice::new(self.cursor(), size)));
                 self.incr_cursor_by(size);
             }
         } else {
@@ -676,7 +680,6 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
                         self.incr_cursor();
                     }
                 }
-                b'\r' => self.scan_unsafe_literal(),
                 // insecure features
                 byte if byte.is_ascii_digit() && OPERATING_MODE == LANG_MODE_INSECURE => {
                     self.scan_unsigned_integer()
@@ -711,13 +714,6 @@ impl Token {
     pub(super) const fn is_lit(&self) -> bool {
         matches!(self, Self::Lit(_))
     }
-    #[inline(always)]
-    /// Returns true if the token *could* be a positive number. since safe literals do not
-    /// provide any type information at this moment, we lookahead to see if it's a litnum or
-    /// an unsafe lit
-    pub(super) const fn is_like_positive_num(&self) -> bool {
-        matches!(self, Self::Lit(Lit::UnsignedInt(_) | Lit::SafeLit(_)))
-    }
 }
 
 impl AsRef<Token> for Token {
@@ -726,3 +722,478 @@ impl AsRef<Token> for Token {
         self
     }
 }
+
+#[derive(Debug)]
+pub struct RawLexer<'a> {
+    c: *const u8,
+    e: *const u8,
+    tokens: Vec<Token>,
+    last_error: Option<LangError>,
+    _lt: PhantomData<&'a [u8]>,
+}
+
+// ctor
+impl<'a> RawLexer<'a> {
+    #[inline(always)]
+    pub const fn new(src: &'a [u8]) -> Self {
+        Self {
+            c: src.as_ptr(),
+            e: unsafe {
+                // UNSAFE(@ohsayan): Always safe (<= EOA)
+                src.as_ptr().add(src.len())
+            },
+            last_error: None,
+            tokens: Vec::new(),
+            _lt: PhantomData,
+        }
+    }
+}
+
+// meta
+impl<'a> RawLexer<'a> {
+    #[inline(always)]
+    const fn cursor(&self) -> *const u8 {
+        self.c
+    }
+    #[inline(always)]
+    const fn data_end_ptr(&self) -> *const u8 {
+        self.e
+    }
+    #[inline(always)]
+    fn not_exhausted(&self) -> bool {
+        self.data_end_ptr() > self.cursor()
+    }
+    #[inline(always)]
+    fn exhausted(&self) -> bool {
+        self.cursor() == self.data_end_ptr()
+    }
+    #[inline(always)]
+    fn remaining(&self) -> usize {
+        unsafe { self.e.offset_from(self.c) as usize }
+    }
+    #[inline(always)]
+    unsafe fn deref_cursor(&self) -> u8 {
+        *self.cursor()
+    }
+    #[inline(always)]
+    unsafe fn incr_cursor_by(&mut self, by: usize) {
+        debug_assert!(self.remaining() >= by);
+        self.c = self.cursor().add(by)
+    }
+    #[inline(always)]
+    unsafe fn incr_cursor(&mut self) {
+        self.incr_cursor_by(1)
+    }
+    #[inline(always)]
+    unsafe fn incr_cursor_if(&mut self, iff: bool) {
+        self.incr_cursor_by(iff as usize)
+    }
+    #[inline(always)]
+    fn push_token(&mut self, token: impl Into<Token>) {
+        self.tokens.push(token.into())
+    }
+    #[inline(always)]
+    fn peek_is(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
+        self.not_exhausted() && unsafe { f(self.deref_cursor()) }
+    }
+    #[inline(always)]
+    fn peek_is_and_forward(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
+        let did_fw = self.not_exhausted() && unsafe { f(self.deref_cursor()) };
+        unsafe {
+            self.incr_cursor_if(did_fw);
+        }
+        did_fw
+    }
+    #[inline(always)]
+    fn peek_eq_and_forward_or_eof(&mut self, eq: u8) -> bool {
+        unsafe {
+            let eq = self.not_exhausted() && self.deref_cursor() == eq;
+            self.incr_cursor_if(eq);
+            eq | self.exhausted()
+        }
+    }
+    #[inline(always)]
+    fn peek_neq(&self, b: u8) -> bool {
+        self.not_exhausted() && unsafe { self.deref_cursor() != b }
+    }
+    #[inline(always)]
+    fn peek_eq_and_forward(&mut self, b: u8) -> bool {
+        unsafe {
+            let r = self.not_exhausted() && self.deref_cursor() == b;
+            self.incr_cursor_if(r);
+            r
+        }
+    }
+    #[inline(always)]
+    fn trim_ahead(&mut self) {
+        while self.peek_is_and_forward(|b| b == b' ' || b == b'\t' || b == b'\n') {}
+    }
+    #[inline(always)]
+    fn set_error(&mut self, e: LangError) {
+        self.last_error = Some(e);
+    }
+    #[inline(always)]
+    fn no_error(&self) -> bool {
+        self.last_error.is_none()
+    }
+}
+
+// high level methods
+impl<'a> RawLexer<'a> {
+    #[inline(always)]
+    fn scan_ident(&mut self) -> RawSlice {
+        let s = self.cursor();
+        unsafe {
+            while self.peek_is(|b| b.is_ascii_alphanumeric() || b == b'_') {
+                self.incr_cursor();
+            }
+            RawSlice::new(s, self.cursor().offset_from(s) as usize)
+        }
+    }
+    #[inline(always)]
+    fn scan_ident_or_keyword(&mut self) {
+        let s = self.scan_ident();
+        let st = unsafe { s.as_slice() }.to_ascii_lowercase();
+        match kwof(&st) {
+            Some(kw) => self.tokens.push(kw.into()),
+            // FIXME(@ohsayan): Uh, mind fixing this? The only advantage is that I can keep the graph *memory* footprint small
+            None if st == b"true" || st == b"false" => self.push_token(Lit::Bool(st == b"true")),
+            None => self.tokens.push(Token::Ident(s)),
+        }
+    }
+    #[inline(always)]
+    fn scan_byte(&mut self, byte: u8) {
+        match symof(byte) {
+            Some(tok) => self.push_token(tok),
+            None => return self.set_error(LangError::UnexpectedChar),
+        }
+        unsafe {
+            self.incr_cursor();
+        }
+    }
+}
+
+#[derive(Debug)]
+/// This lexer implements the `opmod-safe` for BlueQL
+pub struct SafeLexer<'a> {
+    base: RawLexer<'a>,
+}
+
+impl<'a> SafeLexer<'a> {
+    #[inline(always)]
+    pub const fn new(src: &'a [u8]) -> Self {
+        Self {
+            base: RawLexer::new(src),
+        }
+    }
+    #[inline(always)]
+    pub fn lex(src: &'a [u8]) -> LangResult<Vec<Token>> {
+        Self::new(src)._lex()
+    }
+    #[inline(always)]
+    fn _lex(self) -> LangResult<Vec<Token>> {
+        let Self { base: mut l } = self;
+        while l.not_exhausted() && l.no_error() {
+            let b = unsafe { l.deref_cursor() };
+            match b {
+                // ident or kw
+                b if b.is_ascii_alphabetic() => l.scan_ident_or_keyword(),
+                // extra terminal chars
+                b'\n' | b'\t' | b' ' => l.trim_ahead(),
+                // arbitrary byte
+                b => l.scan_byte(b),
+            }
+        }
+        let RawLexer {
+            last_error, tokens, ..
+        } = l;
+        match last_error {
+            None => Ok(tokens),
+            Some(e) => Err(e),
+        }
+    }
+}
+
+const ALLOW_UNSIGNED: bool = false;
+const ALLOW_SIGNED: bool = true;
+
+pub trait NumberDefinition: Sized + fmt::Debug + Copy + Clone + BitOr<Self, Output = Self> {
+    const ALLOW_SIGNED: bool;
+    fn overflowing_mul(&self, v: u8) -> (Self, bool);
+    fn overflowing_add(&self, v: u8) -> (Self, bool);
+    fn negate(&mut self);
+    fn qualified_max_length() -> usize;
+    fn zero() -> Self;
+    fn b(self, b: bool) -> Self;
+}
+
+macro_rules! impl_number_def {
+	($(
+        $ty:ty {$supports_signed:ident, $qualified_max_length:expr}),* $(,)?
+    ) => {
+		$(impl NumberDefinition for $ty {
+			const ALLOW_SIGNED: bool = $supports_signed;
+            #[inline(always)] fn zero() -> Self { 0 }
+            #[inline(always)] fn b(self, b: bool) -> Self { b as Self * self }
+			#[inline(always)]
+			fn overflowing_mul(&self, v: u8) -> ($ty, bool) { <$ty>::overflowing_mul(*self, v as $ty) }
+			#[inline(always)]
+			fn overflowing_add(&self, v: u8) -> ($ty, bool) { <$ty>::overflowing_add(*self, v as $ty) }
+			#[inline(always)]
+			fn negate(&mut self) {
+				assert!(Self::ALLOW_SIGNED, "tried to negate an unsigned integer");
+                *self = !(*self - 1);
+			}
+            #[inline(always)] fn qualified_max_length() -> usize { $qualified_max_length }
+		})*
+	}
+}
+
+#[cfg(target_pointer_width = "64")]
+const SZ_USIZE: usize = 20;
+#[cfg(target_pointer_width = "32")]
+const SZ_USIZE: usize = 10;
+#[cfg(target_pointer_width = "64")]
+const SZ_ISIZE: usize = 20;
+#[cfg(target_pointer_width = "32")]
+const SZ_ISIZE: usize = 11;
+
+impl_number_def! {
+    usize {ALLOW_SIGNED, SZ_USIZE},
+    // 255
+    u8 {ALLOW_UNSIGNED, 3},
+    // 65536
+    u16 {ALLOW_UNSIGNED, 5},
+    // 4294967296
+    u32 {ALLOW_UNSIGNED, 10},
+    // 18446744073709551616
+    u64 {ALLOW_UNSIGNED, 20},
+    // signed
+    isize {ALLOW_SIGNED, SZ_ISIZE},
+    // -128
+    i8 {ALLOW_SIGNED, 4},
+    // -32768
+    i16 {ALLOW_SIGNED, 6},
+    // -2147483648
+    i32 {ALLOW_SIGNED, 11},
+    // -9223372036854775808
+    i64 {ALLOW_SIGNED, 20},
+}
+
+#[inline(always)]
+fn decode_num_from_unbounded_payload<N>(src: &[u8], flag: &mut bool, cnt: &mut usize) -> N
+where
+    N: NumberDefinition,
+{
+    let l = src.len();
+    let mut okay = !src.is_empty();
+    let mut i = 0;
+    let mut number = N::zero();
+    let mut nx_stop = false;
+
+    let is_signed = if N::ALLOW_SIGNED {
+        let is_signed = i < l && src[i] == b'-';
+        i += is_signed as usize;
+        okay &= (i + 2) <= l; // [-][digit][LF]
+        is_signed
+    } else {
+        false
+    };
+
+    while i < l && okay && !nx_stop {
+        // potential exit
+        nx_stop = src[i] == b'\n';
+        // potential entry
+        let mut local_ok = src[i].is_ascii_digit();
+        let (p, p_of) = number.overflowing_mul(10);
+        local_ok &= !p_of;
+        let (s, s_of) = p.overflowing_add(src[i] & 0x0f);
+        local_ok &= !s_of;
+        // reassign or assign
+        let reassign = number.b(nx_stop);
+        let assign = s.b(!nx_stop);
+        number = reassign | assign;
+        okay &= local_ok | nx_stop;
+        i += okay as usize;
+    }
+    okay &= nx_stop;
+    *cnt += i;
+    *flag &= okay;
+
+    if N::ALLOW_SIGNED && is_signed {
+        number.negate()
+    }
+    number
+}
+
+#[inline(always)]
+fn decode_num_from_bounded_payload<N>(src: &[u8], flag: &mut bool) -> N
+where
+    N: NumberDefinition,
+{
+    let l = src.len();
+
+    let mut i = 0;
+    let mut number = N::zero();
+    let mut okay = l <= N::qualified_max_length();
+
+    if N::ALLOW_SIGNED {
+        let is_signed = i < l && src[i] == b'-';
+        if is_signed {
+            number.negate();
+        }
+        i += is_signed as usize;
+    }
+
+    while i < l && okay {
+        okay &= src[i].is_ascii_digit();
+        let (product, p_of) = number.overflowing_mul(10);
+        okay &= !p_of;
+        let (sum, s_of) = product.overflowing_add(src[i] & 0x0F);
+        okay &= !s_of;
+        number = sum;
+        i += 1;
+    }
+
+    *flag &= okay;
+    number
+}
+
+#[derive(PartialEq, Debug, Clone, Copy)]
+/// Intermediate literal repr
+pub enum LitIR<'a> {
+    Str(&'a str),
+    Bin(&'a [u8]),
+    UInt(u64),
+    SInt(i64),
+    Bool(bool),
+    Float(f64),
+}
+
+#[derive(Debug, PartialEq)]
+/// Data constructed from `opmode-safe`
+pub struct SafeQueryData<'a> {
+    p: Box<[LitIR<'a>]>,
+    t: Vec<Token>,
+}
+
+impl<'a> SafeQueryData<'a> {
+    #[cfg(test)]
+    pub fn new_test(p: Box<[LitIR<'a>]>, t: Vec<Token>) -> Self {
+        Self { p, t }
+    }
+    #[inline(always)]
+    pub fn parse(qf: &'a [u8], pf: &'a [u8], pf_sz: usize) -> LangResult<Self> {
+        let q = SafeLexer::lex(qf);
+        let p = Self::p_revloop(pf, pf_sz);
+        let (Ok(t), Ok(p)) = (q, p) else {
+            return Err(LangError::UnexpectedChar)
+        };
+        Ok(Self { p, t })
+    }
+    #[inline]
+    pub(super) fn p_revloop(mut src: &'a [u8], size: usize) -> LangResult<Box<[LitIR<'a>]>> {
+        static LITIR_TF: [for<'a> fn(&'a [u8], &mut usize, &mut Vec<LitIR<'a>>) -> bool; 7] = [
+            SafeQueryData::uint,  // tc: 0
+            SafeQueryData::sint,  // tc: 1
+            SafeQueryData::bool,  // tc: 2
+            SafeQueryData::float, // tc: 3
+            SafeQueryData::bin,   // tc: 4
+            SafeQueryData::str,   // tc: 5
+            |_, _, _| false,      // ecc: 6
+        ];
+        let nonpadded_offset = (LITIR_TF.len() - 2) as u8;
+        let ecc_offset = LITIR_TF.len() - 1;
+        let mut okay = true;
+        let mut data = Vec::with_capacity(size);
+        while src.len() >= 3 && okay {
+            let tc = src[0];
+            okay &= tc <= nonpadded_offset;
+            let mx = cmp::min(ecc_offset, tc as usize);
+            let mut i_ = 1;
+            okay &= LITIR_TF[mx](&src[1..], &mut i_, &mut data);
+            src = &src[i_..];
+        }
+        okay &= src.is_empty() && data.len() == size;
+        if compiler::likely(okay) {
+            Ok(data.into_boxed_slice())
+        } else {
+            Err(LangError::BadPframe)
+        }
+    }
+}
+
+// low level methods
+impl<'b> SafeQueryData<'b> {
+    #[inline(always)]
+    fn mxple<'a>(src: &'a [u8], cnt: &mut usize, flag: &mut bool) -> &'a [u8] {
+        // find payload length
+        let mut i = 0;
+        let payload_len = decode_num_from_unbounded_payload::<usize>(src, flag, &mut i);
+        let src = &src[i..];
+        // find payload
+        *flag &= src.len() >= payload_len;
+        let mx_extract = cmp::min(payload_len, src.len());
+        // incr cursor
+        i += mx_extract;
+        *cnt += i;
+        unsafe { slice::from_raw_parts(src.as_ptr(), mx_extract) }
+    }
+    #[inline(always)]
+    pub(super) fn uint<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
+        let mut b = true;
+        let r = decode_num_from_unbounded_payload(src, &mut b, cnt);
+        data.push(LitIR::UInt(r));
+        b
+    }
+    #[inline(always)]
+    pub(super) fn sint<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
+        let mut b = true;
+        let r = decode_num_from_unbounded_payload(src, &mut b, cnt);
+        data.push(LitIR::SInt(r));
+        b
+    }
+    #[inline(always)]
+    pub(super) fn bool<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
+        // `true\n` or `false\n`
+        let mx = cmp::min(6, src.len());
+        let slice = &src[..mx];
+        let v_true = slice.starts_with(b"true\n");
+        let v_false = slice.starts_with(b"false\n");
+        let incr = v_true as usize * 5 + v_false as usize * 6;
+        data.push(LitIR::Bool(v_true));
+        *cnt += incr;
+        v_true | v_false
+    }
+    #[inline(always)]
+    pub(super) fn float<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
+        let mut okay = true;
+        let payload = Self::mxple(src, cnt, &mut okay);
+        match String::from_utf8_lossy(payload).parse() {
+            Ok(p) if okay => {
+                data.push(LitIR::Float(p));
+            }
+            _ => {}
+        }
+        okay
+    }
+    #[inline(always)]
+    pub(super) fn bin<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
+        let mut okay = true;
+        let payload = Self::mxple(src, cnt, &mut okay);
+        data.push(LitIR::Bin(payload));
+        okay
+    }
+    #[inline(always)]
+    pub(super) fn str<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
+        let mut okay = true;
+        let payload = Self::mxple(src, cnt, &mut okay);
+        match str::from_utf8(payload) {
+            Ok(s) if okay => {
+                data.push(LitIR::Str(s));
+                true
+            }
+            _ => false,
+        }
+    }
+}
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 1e4d48f1..f75ccbe6 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -318,7 +318,7 @@ macro_rules! into_array_nullable {
 }
 
 macro_rules! statictbl {
-    ($name:ident: $kind:ty => [$($expr:expr),+]) => {{
+    ($name:ident: $kind:ty => [$($expr:expr),*]) => {{
         const LEN: usize = {let mut i = 0;$(let _ = $expr; i += 1;)*i};
         static $name: [$kind; LEN] = [$($expr),*];
         $name
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index abfb1695..52d22f91 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -60,6 +60,7 @@ pub enum LangError {
     InvalidDictionaryExpression,
     InvalidTypeDefinition,
     InvalidSafeLiteral,
+    BadPframe,
 }
 
 /*
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index b05717df..f124bf6a 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -36,6 +36,18 @@ use {
     rand::{self, Rng},
 };
 
+macro_rules! fold_dict {
+    ($($input:expr),* $(,)?) => {
+        ($({$crate::engine::ql::schema::fold_dict(&super::lex($input).unwrap()).unwrap()}),*)
+    }
+}
+
+mod dml_tests;
+mod entity;
+mod lexer_tests;
+mod schema_tests;
+mod structure_syn;
+
 pub(super) fn lex(src: &[u8]) -> LangResult<Life<Vec<Token>>> {
     InsecureLexer::lex(src)
 }
@@ -87,12 +99,6 @@ impl NullableMapEntry for super::schema::Dict {
     }
 }
 
-macro_rules! fold_dict {
-    ($($input:expr),* $(,)?) => {
-        ($({$crate::engine::ql::schema::fold_dict(&super::lex($input).unwrap()).unwrap()}),*)
-    }
-}
-
 /// A very "basic" fuzzer that will randomly inject tokens wherever applicable
 fn fuzz_tokens(src: &[Token], fuzzwith: impl Fn(bool, &[Token])) {
     static FUZZ_TARGETS: [Token; 2] = [Token::Symbol(Symbol::SymComma), Token::IgnorableComma];
@@ -126,2489 +132,3 @@ fn fuzz_tokens(src: &[Token], fuzzwith: impl Fn(bool, &[Token])) {
         fuzzwith(should_pass, &new_src);
     }
 }
-
-mod lexer_tests {
-    use {
-        super::{
-            super::lexer::{Lit, Token},
-            lex,
-        },
-        crate::engine::ql::LangError,
-    };
-
-    macro_rules! v(
-        ($e:literal) => {{
-            $e.as_bytes().to_vec()
-        }};
-        ($($e:literal),* $(,)?) => {{
-            ($(v!($e)),*)
-        }};
-    );
-
-    #[test]
-    fn lex_ident() {
-        let src = v!("hello");
-        assert_eq!(lex(&src).unwrap(), vec![Token::Ident("hello".into())]);
-    }
-
-    // literals
-    #[test]
-    fn lex_unsigned_int() {
-        let number = v!("123456");
-        assert_eq!(
-            lex(&number).unwrap(),
-            vec![Token::Lit(Lit::UnsignedInt(123456))]
-        );
-    }
-    #[test]
-    fn lex_signed_int() {
-        let number = v!("-123456");
-        assert_eq!(
-            lex(&number).unwrap(),
-            vec![Token::Lit(Lit::SignedInt(-123456))]
-        );
-    }
-    #[test]
-    fn lex_bool() {
-        let (t, f) = v!("true", "false");
-        assert_eq!(lex(&t).unwrap(), vec![Token::Lit(Lit::Bool(true))]);
-        assert_eq!(lex(&f).unwrap(), vec![Token::Lit(Lit::Bool(false))]);
-    }
-    #[test]
-    fn lex_string() {
-        let s = br#" "hello, world" "#;
-        assert_eq!(
-            lex(s).unwrap(),
-            vec![Token::Lit(Lit::Str("hello, world".into()))]
-        );
-        let s = br#" 'hello, world' "#;
-        assert_eq!(
-            lex(s).unwrap(),
-            vec![Token::Lit(Lit::Str("hello, world".into()))]
-        );
-    }
-    #[test]
-    fn lex_string_test_escape_quote() {
-        let s = br#" "\"hello world\"" "#; // == "hello world"
-        assert_eq!(
-            lex(s).unwrap(),
-            vec![Token::Lit(Lit::Str("\"hello world\"".into()))]
-        );
-        let s = br#" '\'hello world\'' "#; // == 'hello world'
-        assert_eq!(
-            lex(s).unwrap(),
-            vec![Token::Lit(Lit::Str("'hello world'".into()))]
-        );
-    }
-    #[test]
-    fn lex_string_use_different_quote_style() {
-        let s = br#" "he's on it" "#;
-        assert_eq!(
-            lex(s).unwrap(),
-            vec![Token::Lit(Lit::Str("he's on it".into()))]
-        );
-        let s = br#" 'he thinks that "that girl" fixed it' "#;
-        assert_eq!(
-            lex(s).unwrap(),
-            vec![Token::Lit(Lit::Str(
-                "he thinks that \"that girl\" fixed it".into()
-            ))]
-        )
-    }
-    #[test]
-    fn lex_string_escape_bs() {
-        let s = v!(r#" "windows has c:\\" "#);
-        assert_eq!(
-            lex(&s).unwrap(),
-            vec![Token::Lit(Lit::Str("windows has c:\\".into()))]
-        );
-        let s = v!(r#" 'windows has c:\\' "#);
-        assert_eq!(
-            lex(&s).unwrap(),
-            vec![Token::Lit(Lit::Str("windows has c:\\".into()))]
-        );
-        let lol = v!(r#"'\\\\\\\\\\'"#);
-        assert_eq!(
-            lex(&lol).unwrap(),
-            vec![Token::Lit(Lit::Str("\\".repeat(5).into_boxed_str()))],
-            "lol"
-        )
-    }
-    #[test]
-    fn lex_string_bad_escape() {
-        let wth = br#" '\a should be an alert on windows apparently' "#;
-        assert_eq!(lex(wth).unwrap_err(), LangError::InvalidStringLiteral);
-    }
-    #[test]
-    fn lex_string_unclosed() {
-        let wth = br#" 'omg where did the end go "#;
-        assert_eq!(lex(wth).unwrap_err(), LangError::InvalidStringLiteral);
-        let wth = br#" 'see, we escaped the end\' "#;
-        assert_eq!(lex(wth).unwrap_err(), LangError::InvalidStringLiteral);
-    }
-    #[test]
-    fn lex_unsafe_literal_mini() {
-        let usl = lex("\r0\n".as_bytes()).unwrap();
-        assert_eq!(usl.len(), 1);
-        assert_eq!(Token::Lit(Lit::SafeLit("".into())), usl[0]);
-    }
-    #[test]
-    fn lex_unsafe_literal() {
-        let usl = lex("\r9\nabcdefghi".as_bytes()).unwrap();
-        assert_eq!(usl.len(), 1);
-        assert_eq!(Token::Lit(Lit::SafeLit("abcdefghi".into())), usl[0]);
-    }
-    #[test]
-    fn lex_unsafe_literal_pro() {
-        let usl = lex("\r18\nabcdefghi123456789".as_bytes()).unwrap();
-        assert_eq!(usl.len(), 1);
-        assert_eq!(
-            Token::Lit(Lit::SafeLit("abcdefghi123456789".into())),
-            usl[0]
-        );
-    }
-}
-
-mod entity {
-    use super::*;
-    use crate::engine::ql::ast::{Compiler, Entity};
-    #[test]
-    fn entity_current() {
-        let t = lex(b"hello").unwrap();
-        let mut c = Compiler::new(&t);
-        let r = Entity::parse(&mut c).unwrap();
-        assert_eq!(r, Entity::Single("hello".into()))
-    }
-    #[test]
-    fn entity_partial() {
-        let t = lex(b":hello").unwrap();
-        let mut c = Compiler::new(&t);
-        let r = Entity::parse(&mut c).unwrap();
-        assert_eq!(r, Entity::Partial("hello".into()))
-    }
-    #[test]
-    fn entity_full() {
-        let t = lex(b"hello.world").unwrap();
-        let mut c = Compiler::new(&t);
-        let r = Entity::parse(&mut c).unwrap();
-        assert_eq!(r, Entity::Full("hello".into(), "world".into()))
-    }
-}
-
-mod ddl_other_query_tests {
-    use {
-        super::*,
-        crate::engine::ql::{
-            ast::{Entity, Statement},
-            ddl::{self, DropModel, DropSpace},
-        },
-    };
-    #[test]
-    fn drop_space() {
-        let src = lex(br"drop space myspace").unwrap();
-        assert_eq!(
-            ddl::parse_drop_full(&src[1..]).unwrap(),
-            Statement::DropSpace(DropSpace::new("myspace".into(), false))
-        );
-    }
-    #[test]
-    fn drop_space_force() {
-        let src = lex(br"drop space myspace force").unwrap();
-        assert_eq!(
-            ddl::parse_drop_full(&src[1..]).unwrap(),
-            Statement::DropSpace(DropSpace::new("myspace".into(), true))
-        );
-    }
-    #[test]
-    fn drop_model() {
-        let src = lex(br"drop model mymodel").unwrap();
-        assert_eq!(
-            ddl::parse_drop_full(&src[1..]).unwrap(),
-            Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), false))
-        );
-    }
-    #[test]
-    fn drop_model_force() {
-        let src = lex(br"drop model mymodel force").unwrap();
-        assert_eq!(
-            ddl::parse_drop_full(&src[1..]).unwrap(),
-            Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), true))
-        );
-    }
-}
-
-mod dict_tests {
-    use {
-        super::*,
-        crate::engine::ql::{lexer::Lit, schema},
-    };
-    mod dict {
-        use super::*;
-
-        #[test]
-        fn dict_read_mini() {
-            let (d1, d2) = fold_dict! {
-                br#"{name: "sayan"}"#,
-                br#"{name: "sayan",}"#,
-            };
-            let r = nullable_dict!("name" => Lit::Str("sayan".into()));
-            multi_assert_eq!(d1, d2 => r);
-        }
-        #[test]
-        fn dict_read() {
-            let (d1, d2) = fold_dict! {
-                br#"
-                {
-                    name: "sayan",
-                    verified: true,
-                    burgers: 152
-                }
-            "#,
-                br#"
-                {
-                    name: "sayan",
-                    verified: true,
-                    burgers: 152,
-                }
-            "#,
-            };
-            let r = nullable_dict! (
-                "name" => Lit::Str("sayan".into()),
-                "verified" => Lit::Bool(true),
-                "burgers" => Lit::UnsignedInt(152),
-            );
-            multi_assert_eq!(d1, d2 => r);
-        }
-        #[test]
-        fn dict_read_pro() {
-            let (d1, d2, d3) = fold_dict! {
-                br#"
-                {
-                    name: "sayan",
-                    notes: {
-                        burgers: "all the time, extra mayo",
-                        taco: true,
-                        pretzels: 1
-                    }
-                }
-            "#,
-                br#"
-                {
-                    name: "sayan",
-                    notes: {
-                        burgers: "all the time, extra mayo",
-                        taco: true,
-                        pretzels: 1,
-                    }
-                }
-            "#,
-                br#"
-                {
-                    name: "sayan",
-                    notes: {
-                        burgers: "all the time, extra mayo",
-                        taco: true,
-                        pretzels: 1,
-                },
-            }"#
-            };
-            multi_assert_eq!(
-                d1, d2, d3 => nullable_dict! {
-                    "name" => Lit::Str("sayan".into()),
-                    "notes" => nullable_dict! {
-                        "burgers" => Lit::Str("all the time, extra mayo".into()),
-                        "taco" => Lit::Bool(true),
-                        "pretzels" => Lit::UnsignedInt(1),
-                    }
-                }
-            );
-        }
-
-        #[test]
-        fn dict_read_pro_max() {
-            let (d1, d2, d3) = fold_dict! {
-                br#"
-                {
-                    well: {
-                        now: {
-                            this: {
-                                is: {
-                                    ridiculous: true
-                                }
-                            }
-                        }
-                    }
-                }
-            "#,
-                br#"
-                {
-                    well: {
-                        now: {
-                            this: {
-                                is: {
-                                    ridiculous: true,
-                                }
-                            }
-                        }
-                    }
-                }
-            "#,
-                br#"
-                {
-                    well: {
-                        now: {
-                            this: {
-                                is: {
-                                    ridiculous: true,
-                                },
-                            },
-                        },
-                    },
-                }
-            }"#
-            };
-            multi_assert_eq!(
-                d1, d2, d3 => nullable_dict! {
-                    "well" => nullable_dict! {
-                        "now" => nullable_dict! {
-                            "this" => nullable_dict! {
-                                "is" => nullable_dict! {
-                                    "ridiculous" => Lit::Bool(true),
-                                }
-                            }
-                        }
-                    }
-                }
-            );
-        }
-
-        #[test]
-        fn fuzz_dict() {
-            let ret = lex(b"
-                {
-                    the_tradition_is: \"hello, world\",
-                    could_have_been: {
-                        this: true,
-                        or_maybe_this: 100,
-                        even_this: \"hello, universe!\"\x01
-                    },
-                    but_oh_well: \"it continues to be the 'annoying' phrase\",
-                    lorem: {
-                        ipsum: {
-                            dolor: \"sit amet\"\x01
-                        }\x01
-                    }\x01
-                }
-            ")
-            .unwrap();
-            let ret_dict = nullable_dict! {
-                "the_tradition_is" => Lit::Str("hello, world".into()),
-                "could_have_been" => nullable_dict! {
-                    "this" => Lit::Bool(true),
-                    "or_maybe_this" => Lit::UnsignedInt(100),
-                    "even_this" => Lit::Str("hello, universe!".into()),
-                },
-                "but_oh_well" => Lit::Str("it continues to be the 'annoying' phrase".into()),
-                "lorem" => nullable_dict! {
-                    "ipsum" => nullable_dict! {
-                        "dolor" => Lit::Str("sit amet".into())
-                    }
-                }
-            };
-            fuzz_tokens(&ret, |should_pass, new_src| {
-                let r = schema::fold_dict(&new_src);
-                if should_pass {
-                    assert_eq!(r.unwrap(), ret_dict)
-                } else if r.is_some() {
-                    panic!(
-                        "expected failure, but passed for token stream: `{:?}`",
-                        new_src
-                    );
-                }
-            });
-        }
-    }
-    mod nullable_dict_tests {
-        use super::*;
-        mod dict {
-            use {super::*, crate::engine::ql::lexer::Lit};
-
-            #[test]
-            fn null_mini() {
-                let d = fold_dict!(br"{ x: null }");
-                assert_eq!(
-                    d,
-                    nullable_dict! {
-                        "x" => Null,
-                    }
-                );
-            }
-            #[test]
-            fn null() {
-                let d = fold_dict! {
-                    br#"
-                        {
-                            this_is_non_null: "hello",
-                            but_this_is_null: null,
-                        }
-                    "#
-                };
-                assert_eq!(
-                    d,
-                    nullable_dict! {
-                        "this_is_non_null" => Lit::Str("hello".into()),
-                        "but_this_is_null" => Null,
-                    }
-                )
-            }
-            #[test]
-            fn null_pro() {
-                let d = fold_dict! {
-                    br#"
-                        {
-                            a_string: "this is a string",
-                            num: 1234,
-                            a_dict: {
-                                a_null: null,
-                            }
-                        }
-                    "#
-                };
-                assert_eq!(
-                    d,
-                    nullable_dict! {
-                        "a_string" => Lit::Str("this is a string".into()),
-                        "num" => Lit::UnsignedInt(1234),
-                        "a_dict" => nullable_dict! {
-                            "a_null" => Null,
-                        }
-                    }
-                )
-            }
-            #[test]
-            fn null_pro_max() {
-                let d = fold_dict! {
-                    br#"
-                        {
-                            a_string: "this is a string",
-                            num: 1234,
-                            a_dict: {
-                                a_null: null,
-                            },
-                            another_null: null,
-                        }
-                    "#
-                };
-                assert_eq!(
-                    d,
-                    nullable_dict! {
-                        "a_string" => Lit::Str("this is a string".into()),
-                        "num" => Lit::UnsignedInt(1234),
-                        "a_dict" => nullable_dict! {
-                            "a_null" => Null,
-                        },
-                        "another_null" => Null,
-                    }
-                )
-            }
-        }
-        // TODO(@ohsayan): Add null tests
-    }
-}
-
-mod schema_tests {
-    use super::{
-        super::{
-            lexer::{Lit, Token},
-            schema,
-        },
-        lex, *,
-    };
-    mod inspect {
-        use {
-            super::*,
-            crate::engine::ql::{
-                ast::{Entity, Statement},
-                ddl,
-            },
-        };
-        #[test]
-        fn inspect_space() {
-            let tok = lex(b"inspect space myspace").unwrap();
-            assert_eq!(
-                ddl::parse_inspect_full(&tok[1..]).unwrap(),
-                Statement::InspectSpace("myspace".into())
-            );
-        }
-        #[test]
-        fn inspect_model() {
-            let tok = lex(b"inspect model users").unwrap();
-            assert_eq!(
-                ddl::parse_inspect_full(&tok[1..]).unwrap(),
-                Statement::InspectModel(Entity::Single("users".into()))
-            );
-            let tok = lex(b"inspect model tweeter.users").unwrap();
-            assert_eq!(
-                ddl::parse_inspect_full(&tok[1..]).unwrap(),
-                Statement::InspectModel(("tweeter", "users").into())
-            );
-        }
-        #[test]
-        fn inspect_spaces() {
-            let tok = lex(b"inspect spaces").unwrap();
-            assert_eq!(
-                ddl::parse_inspect_full(&tok[1..]).unwrap(),
-                Statement::InspectSpaces
-            );
-        }
-    }
-
-    mod alter_space {
-        use {
-            super::*,
-            crate::engine::ql::{
-                lexer::Lit,
-                schema::{self, AlterSpace},
-            },
-        };
-        #[test]
-        fn alter_space_mini() {
-            let tok = lex(b"alter model mymodel with {}").unwrap();
-            let r = schema::alter_space_full(&tok[2..]).unwrap();
-            assert_eq!(
-                r,
-                AlterSpace {
-                    space_name: "mymodel".into(),
-                    updated_props: nullable_dict! {}
-                }
-            );
-        }
-        #[test]
-        fn alter_space() {
-            let tok = lex(br#"
-                alter model mymodel with {
-                    max_entry: 1000,
-                    driver: "ts-0.8"
-                }
-            "#)
-            .unwrap();
-            let r = schema::alter_space_full(&tok[2..]).unwrap();
-            assert_eq!(
-                r,
-                AlterSpace {
-                    space_name: "mymodel".into(),
-                    updated_props: nullable_dict! {
-                        "max_entry" => Lit::UnsignedInt(1000),
-                        "driver" => Lit::Str("ts-0.8".into())
-                    }
-                }
-            );
-        }
-    }
-    mod tymeta {
-        use super::*;
-        #[test]
-        fn tymeta_mini() {
-            let tok = lex(b"}").unwrap();
-            let (res, ret) = schema::fold_tymeta(&tok);
-            assert!(res.is_okay());
-            assert!(!res.has_more());
-            assert_eq!(res.pos(), 1);
-            assert_eq!(ret, nullable_dict!());
-        }
-        #[test]
-        fn tymeta_mini_fail() {
-            let tok = lex(b",}").unwrap();
-            let (res, ret) = schema::fold_tymeta(&tok);
-            assert!(!res.is_okay());
-            assert!(!res.has_more());
-            assert_eq!(res.pos(), 0);
-            assert_eq!(ret, nullable_dict!());
-        }
-        #[test]
-        fn tymeta() {
-            let tok = lex(br#"hello: "world", loading: true, size: 100 }"#).unwrap();
-            let (res, ret) = schema::fold_tymeta(&tok);
-            assert!(res.is_okay());
-            assert!(!res.has_more());
-            assert_eq!(res.pos(), tok.len());
-            assert_eq!(
-                ret,
-                nullable_dict! {
-                    "hello" => Lit::Str("world".into()),
-                    "loading" => Lit::Bool(true),
-                    "size" => Lit::UnsignedInt(100)
-                }
-            );
-        }
-        #[test]
-        fn tymeta_pro() {
-            // list { maxlen: 100, type string, unique: true }
-            //        ^^^^^^^^^^^^^^^^^^ cursor should be at string
-            let tok = lex(br#"maxlen: 100, type string, unique: true }"#).unwrap();
-            let (res1, ret1) = schema::fold_tymeta(&tok);
-            assert!(res1.is_okay());
-            assert!(res1.has_more());
-            assert_eq!(res1.pos(), 5);
-            let remslice = &tok[res1.pos() + 2..];
-            let (res2, ret2) = schema::fold_tymeta(remslice);
-            assert!(res2.is_okay());
-            assert!(!res2.has_more());
-            assert_eq!(res2.pos() + res1.pos() + 2, tok.len());
-            let mut final_ret = ret1;
-            final_ret.extend(ret2);
-            assert_eq!(
-                final_ret,
-                nullable_dict! {
-                    "maxlen" => Lit::UnsignedInt(100),
-                    "unique" => Lit::Bool(true)
-                }
-            )
-        }
-        #[test]
-        fn tymeta_pro_max() {
-            // list { maxlen: 100, this: { is: "cool" }, type string, unique: true }
-            //        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cursor should be at string
-            let tok =
-                lex(br#"maxlen: 100, this: { is: "cool" }, type string, unique: true }"#).unwrap();
-            let (res1, ret1) = schema::fold_tymeta(&tok);
-            assert!(res1.is_okay());
-            assert!(res1.has_more());
-            assert_eq!(res1.pos(), 13);
-            let remslice = &tok[res1.pos() + 2..];
-            let (res2, ret2) = schema::fold_tymeta(remslice);
-            assert!(res2.is_okay());
-            assert!(!res2.has_more());
-            assert_eq!(res2.pos() + res1.pos() + 2, tok.len());
-            let mut final_ret = ret1;
-            final_ret.extend(ret2);
-            assert_eq!(
-                final_ret,
-                nullable_dict! {
-                    "maxlen" => Lit::UnsignedInt(100),
-                    "unique" => Lit::Bool(true),
-                    "this" => nullable_dict! {
-                        "is" => Lit::Str("cool".into())
-                    }
-                }
-            )
-        }
-        #[test]
-        fn fuzz_tymeta_normal() {
-            // { maxlen: 10, unique: true, users: "sayan" }
-            //   ^start
-            let tok = lex(b"
-                    maxlen: 10,
-                    unique: true,
-                    auth: {
-                        maybe: true\x01
-                    },
-                    users: \"sayan\"\x01
-                }
-            ")
-            .unwrap();
-            let expected = nullable_dict! {
-                "maxlen" => Lit::UnsignedInt(10),
-                "unique" => Lit::Bool(true),
-                "auth" => nullable_dict! {
-                    "maybe" => Lit::Bool(true),
-                },
-                "users" => Lit::Str("sayan".into())
-            };
-            fuzz_tokens(&tok, |should_pass, new_src| {
-                let (ret, dict) = schema::fold_tymeta(&new_src);
-                if should_pass {
-                    assert!(ret.is_okay(), "{:?}", &new_src);
-                    assert!(!ret.has_more());
-                    assert_eq!(ret.pos(), new_src.len());
-                    assert_eq!(dict, expected);
-                } else if ret.is_okay() {
-                    panic!(
-                        "Expected failure but passed for token stream: `{:?}`",
-                        new_src
-                    );
-                }
-            });
-        }
-        #[test]
-        fn fuzz_tymeta_with_ty() {
-            // list { maxlen: 10, unique: true, type string, users: "sayan" }
-            //   ^start
-            let tok = lex(b"
-                    maxlen: 10,
-                    unique: true,
-                    auth: {
-                        maybe: true\x01
-                    },
-                    type string,
-                    users: \"sayan\"\x01
-                }
-            ")
-            .unwrap();
-            let expected = nullable_dict! {
-                "maxlen" => Lit::UnsignedInt(10),
-                "unique" => Lit::Bool(true),
-                "auth" => nullable_dict! {
-                    "maybe" => Lit::Bool(true),
-                },
-            };
-            fuzz_tokens(&tok, |should_pass, new_src| {
-                let (ret, dict) = schema::fold_tymeta(&new_src);
-                if should_pass {
-                    assert!(ret.is_okay());
-                    assert!(ret.has_more());
-                    assert!(new_src[ret.pos()] == Token::Ident("string".into()));
-                    assert_eq!(dict, expected);
-                } else if ret.is_okay() {
-                    panic!("Expected failure but passed for token stream: `{:?}`", tok);
-                }
-            });
-        }
-    }
-    mod layer {
-        use super::*;
-        use crate::engine::ql::schema::Layer;
-        #[test]
-        fn layer_mini() {
-            let tok = lex(b"string)").unwrap();
-            let (layers, c, okay) = schema::fold_layers(&tok);
-            assert_eq!(c, tok.len() - 1);
-            assert!(okay);
-            assert_eq!(
-                layers,
-                vec![Layer::new_noreset("string".into(), nullable_dict! {})]
-            );
-        }
-        #[test]
-        fn layer() {
-            let tok = lex(b"string { maxlen: 100 }").unwrap();
-            let (layers, c, okay) = schema::fold_layers(&tok);
-            assert_eq!(c, tok.len());
-            assert!(okay);
-            assert_eq!(
-                layers,
-                vec![Layer::new_noreset(
-                    "string".into(),
-                    nullable_dict! {
-                        "maxlen" => Lit::UnsignedInt(100)
-                    }
-                )]
-            );
-        }
-        #[test]
-        fn layer_plus() {
-            let tok = lex(b"list { type string }").unwrap();
-            let (layers, c, okay) = schema::fold_layers(&tok);
-            assert_eq!(c, tok.len());
-            assert!(okay);
-            assert_eq!(
-                layers,
-                vec![
-                    Layer::new_noreset("string".into(), nullable_dict! {}),
-                    Layer::new_noreset("list".into(), nullable_dict! {})
-                ]
-            );
-        }
-        #[test]
-        fn layer_pro() {
-            let tok = lex(b"list { unique: true, type string, maxlen: 10 }").unwrap();
-            let (layers, c, okay) = schema::fold_layers(&tok);
-            assert_eq!(c, tok.len());
-            assert!(okay);
-            assert_eq!(
-                layers,
-                vec![
-                    Layer::new_noreset("string".into(), nullable_dict! {}),
-                    Layer::new_noreset(
-                        "list".into(),
-                        nullable_dict! {
-                            "unique" => Lit::Bool(true),
-                            "maxlen" => Lit::UnsignedInt(10),
-                        }
-                    )
-                ]
-            );
-        }
-        #[test]
-        fn layer_pro_max() {
-            let tok = lex(
-                b"list { unique: true, type string { ascii_only: true, maxlen: 255 }, maxlen: 10 }",
-            )
-            .unwrap();
-            let (layers, c, okay) = schema::fold_layers(&tok);
-            assert_eq!(c, tok.len());
-            assert!(okay);
-            assert_eq!(
-                layers,
-                vec![
-                    Layer::new_noreset(
-                        "string".into(),
-                        nullable_dict! {
-                            "ascii_only" => Lit::Bool(true),
-                            "maxlen" => Lit::UnsignedInt(255)
-                        }
-                    ),
-                    Layer::new_noreset(
-                        "list".into(),
-                        nullable_dict! {
-                            "unique" => Lit::Bool(true),
-                            "maxlen" => Lit::UnsignedInt(10),
-                        }
-                    )
-                ]
-            );
-        }
-
-        #[test]
-        fn fuzz_layer() {
-            let tok = lex(b"
-            list {
-                type list {
-                    maxlen: 100,
-                    type string\x01
-                },
-                unique: true\x01
-            }
-        ")
-            .unwrap();
-            let expected = vec![
-                Layer::new_noreset("string".into(), nullable_dict!()),
-                Layer::new_noreset(
-                    "list".into(),
-                    nullable_dict! {
-                        "maxlen" => Lit::UnsignedInt(100),
-                    },
-                ),
-                Layer::new_noreset("list".into(), nullable_dict!("unique" => Lit::Bool(true))),
-            ];
-            fuzz_tokens(&tok, |should_pass, new_tok| {
-                let (layers, c, okay) = schema::fold_layers(&new_tok);
-                if should_pass {
-                    assert!(okay);
-                    assert_eq!(c, new_tok.len());
-                    assert_eq!(layers, expected);
-                } else if okay {
-                    panic!(
-                        "expected failure but passed for token stream: `{:?}`",
-                        new_tok
-                    );
-                }
-            });
-        }
-    }
-    mod field_properties {
-        use {super::*, crate::engine::ql::schema::FieldProperties};
-
-        #[test]
-        fn field_properties_empty() {
-            let tok = lex(b"myfield:").unwrap();
-            let (props, c, okay) = schema::parse_field_properties(&tok);
-            assert!(okay);
-            assert_eq!(c, 0);
-            assert_eq!(props, FieldProperties::default());
-        }
-        #[test]
-        fn field_properties_full() {
-            let tok = lex(b"primary null myfield:").unwrap();
-            let (props, c, okay) = schema::parse_field_properties(&tok);
-            assert_eq!(c, 2);
-            assert_eq!(tok[c], Token::Ident("myfield".into()));
-            assert!(okay);
-            assert_eq!(
-                props,
-                FieldProperties {
-                    properties: set!["primary", "null"],
-                }
-            )
-        }
-    }
-    mod fields {
-        use {
-            super::*,
-            crate::engine::ql::schema::{Field, Layer},
-        };
-        #[test]
-        fn field_mini() {
-            let tok = lex(b"
-                username: string,
-            ")
-            .unwrap();
-            let (c, f) = schema::parse_field(&tok).unwrap();
-            assert_eq!(c, tok.len() - 1);
-            assert_eq!(
-                f,
-                Field {
-                    field_name: "username".into(),
-                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
-                    props: set![],
-                }
-            )
-        }
-        #[test]
-        fn field() {
-            let tok = lex(b"
-                primary username: string,    
-            ")
-            .unwrap();
-            let (c, f) = schema::parse_field(&tok).unwrap();
-            assert_eq!(c, tok.len() - 1);
-            assert_eq!(
-                f,
-                Field {
-                    field_name: "username".into(),
-                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
-                    props: set!["primary"],
-                }
-            )
-        }
-        #[test]
-        fn field_pro() {
-            let tok = lex(b"
-                primary username: string {
-                    maxlen: 10,
-                    ascii_only: true,
-                }
-            ")
-            .unwrap();
-            let (c, f) = schema::parse_field(&tok).unwrap();
-            assert_eq!(c, tok.len());
-            assert_eq!(
-                f,
-                Field {
-                    field_name: "username".into(),
-                    layers: [Layer::new_noreset(
-                        "string".into(),
-                        nullable_dict! {
-                            "maxlen" => Lit::UnsignedInt(10),
-                            "ascii_only" => Lit::Bool(true),
-                        }
-                    )]
-                    .into(),
-                    props: set!["primary"],
-                }
-            )
-        }
-        #[test]
-        fn field_pro_max() {
-            let tok = lex(b"
-                null notes: list {
-                    type string {
-                        maxlen: 255,
-                        ascii_only: true,
-                    },
-                    unique: true,
-                }
-            ")
-            .unwrap();
-            let (c, f) = schema::parse_field(&tok).unwrap();
-            assert_eq!(c, tok.len());
-            assert_eq!(
-                f,
-                Field {
-                    field_name: "notes".into(),
-                    layers: [
-                        Layer::new_noreset(
-                            "string".into(),
-                            nullable_dict! {
-                                "maxlen" => Lit::UnsignedInt(255),
-                                "ascii_only" => Lit::Bool(true),
-                            }
-                        ),
-                        Layer::new_noreset(
-                            "list".into(),
-                            nullable_dict! {
-                                "unique" => Lit::Bool(true)
-                            }
-                        ),
-                    ]
-                    .into(),
-                    props: set!["null"],
-                }
-            )
-        }
-    }
-    mod schemas {
-        use crate::engine::ql::schema::{Field, Layer, Model};
-
-        use super::*;
-        #[test]
-        fn schema_mini() {
-            let tok = lex(b"
-                create model mymodel(
-                    primary username: string,
-                    password: binary,
-                )
-            ")
-            .unwrap();
-            let tok = &tok[2..];
-
-            // parse model
-            let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
-            assert_eq!(c, tok.len());
-            assert_eq!(
-                model,
-                Model {
-                    model_name: "mymodel".into(),
-                    fields: vec![
-                        Field {
-                            field_name: "username".into(),
-                            layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
-                            props: set!["primary"]
-                        },
-                        Field {
-                            field_name: "password".into(),
-                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
-                            props: set![]
-                        }
-                    ],
-                    props: nullable_dict! {}
-                }
-            )
-        }
-        #[test]
-        fn schema() {
-            let tok = lex(b"
-                create model mymodel(
-                    primary username: string,
-                    password: binary,
-                    null profile_pic: binary,
-                )
-            ")
-            .unwrap();
-            let tok = &tok[2..];
-
-            // parse model
-            let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
-            assert_eq!(c, tok.len());
-            assert_eq!(
-                model,
-                Model {
-                    model_name: "mymodel".into(),
-                    fields: vec![
-                        Field {
-                            field_name: "username".into(),
-                            layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
-                            props: set!["primary"]
-                        },
-                        Field {
-                            field_name: "password".into(),
-                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
-                            props: set![]
-                        },
-                        Field {
-                            field_name: "profile_pic".into(),
-                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
-                            props: set!["null"]
-                        }
-                    ],
-                    props: nullable_dict! {}
-                }
-            )
-        }
-
-        #[test]
-        fn schema_pro() {
-            let tok = lex(b"
-                create model mymodel(
-                    primary username: string,
-                    password: binary,
-                    null profile_pic: binary,
-                    null notes: list {
-                        type string,
-                        unique: true,
-                    },
-                )
-            ")
-            .unwrap();
-            let tok = &tok[2..];
-
-            // parse model
-            let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
-            assert_eq!(c, tok.len());
-            assert_eq!(
-                model,
-                Model {
-                    model_name: "mymodel".into(),
-                    fields: vec![
-                        Field {
-                            field_name: "username".into(),
-                            layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
-                            props: set!["primary"]
-                        },
-                        Field {
-                            field_name: "password".into(),
-                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
-                            props: set![]
-                        },
-                        Field {
-                            field_name: "profile_pic".into(),
-                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
-                            props: set!["null"]
-                        },
-                        Field {
-                            field_name: "notes".into(),
-                            layers: vec![
-                                Layer::new_noreset("string".into(), nullable_dict! {}),
-                                Layer::new_noreset(
-                                    "list".into(),
-                                    nullable_dict! {
-                                        "unique" => Lit::Bool(true)
-                                    }
-                                )
-                            ],
-                            props: set!["null"]
-                        }
-                    ],
-                    props: nullable_dict! {}
-                }
-            )
-        }
-
-        #[test]
-        fn schema_pro_max() {
-            let tok = lex(b"
-                create model mymodel(
-                    primary username: string,
-                    password: binary,
-                    null profile_pic: binary,
-                    null notes: list {
-                        type string,
-                        unique: true,
-                    },
-                ) with {
-                    env: {
-                        free_user_limit: 100,
-                    },
-                    storage_driver: \"skyheap\"
-                }
-            ")
-            .unwrap();
-            let tok = &tok[2..];
-
-            // parse model
-            let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
-            assert_eq!(c, tok.len());
-            assert_eq!(
-                model,
-                Model {
-                    model_name: "mymodel".into(),
-                    fields: vec![
-                        Field {
-                            field_name: "username".into(),
-                            layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
-                            props: set!["primary"]
-                        },
-                        Field {
-                            field_name: "password".into(),
-                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
-                            props: set![]
-                        },
-                        Field {
-                            field_name: "profile_pic".into(),
-                            layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
-                            props: set!["null"]
-                        },
-                        Field {
-                            field_name: "notes".into(),
-                            layers: vec![
-                                Layer::new_noreset("string".into(), nullable_dict! {}),
-                                Layer::new_noreset(
-                                    "list".into(),
-                                    nullable_dict! {
-                                        "unique" => Lit::Bool(true)
-                                    }
-                                )
-                            ],
-                            props: set!["null"]
-                        }
-                    ],
-                    props: nullable_dict! {
-                        "env" => nullable_dict! {
-                            "free_user_limit" => Lit::UnsignedInt(100),
-                        },
-                        "storage_driver" => Lit::Str("skyheap".into()),
-                    }
-                }
-            )
-        }
-    }
-    mod dict_field_syntax {
-        use super::*;
-        use crate::engine::ql::schema::{ExpandedField, Layer};
-        #[test]
-        fn field_syn_mini() {
-            let tok = lex(b"username { type string }").unwrap();
-            let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                ef,
-                ExpandedField {
-                    field_name: "username".into(),
-                    layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
-                    props: nullable_dict! {},
-                    reset: false
-                }
-            )
-        }
-        #[test]
-        fn field_syn() {
-            let tok = lex(b"
-                username {
-                    nullable: false,
-                    type string,
-                }
-            ")
-            .unwrap();
-            let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                ef,
-                ExpandedField {
-                    field_name: "username".into(),
-                    props: nullable_dict! {
-                        "nullable" => Lit::Bool(false),
-                    },
-                    layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
-                    reset: false
-                }
-            );
-        }
-        #[test]
-        fn field_syn_pro() {
-            let tok = lex(b"
-                username {
-                    nullable: false,
-                    type string {
-                        minlen: 6,
-                        maxlen: 255,
-                    },
-                    jingle_bells: \"snow\"
-                }
-            ")
-            .unwrap();
-            let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                ef,
-                ExpandedField {
-                    field_name: "username".into(),
-                    props: nullable_dict! {
-                        "nullable" => Lit::Bool(false),
-                        "jingle_bells" => Lit::Str("snow".into()),
-                    },
-                    layers: vec![Layer::new_noreset(
-                        "string".into(),
-                        nullable_dict! {
-                            "minlen" => Lit::UnsignedInt(6),
-                            "maxlen" => Lit::UnsignedInt(255),
-                        }
-                    )],
-                    reset: false
-                }
-            );
-        }
-        #[test]
-        fn field_syn_pro_max() {
-            let tok = lex(b"
-                notes {
-                    nullable: true,
-                    type list {
-                        type string {
-                            ascii_only: true,
-                        },
-                        unique: true,
-                    },
-                    jingle_bells: \"snow\"
-                }
-            ")
-            .unwrap();
-            let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                ef,
-                ExpandedField {
-                    field_name: "notes".into(),
-                    props: nullable_dict! {
-                        "nullable" => Lit::Bool(true),
-                        "jingle_bells" => Lit::Str("snow".into()),
-                    },
-                    layers: vec![
-                        Layer::new_noreset(
-                            "string".into(),
-                            nullable_dict! {
-                                "ascii_only" => Lit::Bool(true),
-                            }
-                        ),
-                        Layer::new_noreset(
-                            "list".into(),
-                            nullable_dict! {
-                                "unique" => Lit::Bool(true),
-                            }
-                        )
-                    ],
-                    reset: false
-                }
-            );
-        }
-    }
-    mod alter_model_remove {
-        use super::*;
-        use crate::engine::ql::RawSlice;
-        #[test]
-        fn alter_mini() {
-            let tok = lex(b"alter model mymodel remove myfield").unwrap();
-            let mut i = 4;
-            let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(remove, [RawSlice::from("myfield")].into());
-        }
-        #[test]
-        fn alter_mini_2() {
-            let tok = lex(b"alter model mymodel remove (myfield)").unwrap();
-            let mut i = 4;
-            let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(remove, [RawSlice::from("myfield")].into());
-        }
-        #[test]
-        fn alter() {
-            let tok = lex(b"alter model mymodel remove (myfield1, myfield2, myfield3, myfield4)")
-                .unwrap();
-            let mut i = 4;
-            let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                remove,
-                [
-                    RawSlice::from("myfield1"),
-                    RawSlice::from("myfield2"),
-                    RawSlice::from("myfield3"),
-                    RawSlice::from("myfield4")
-                ]
-                .into()
-            );
-        }
-    }
-    mod alter_model_add {
-        use super::*;
-        use crate::engine::ql::schema::{ExpandedField, Layer};
-        #[test]
-        fn add_mini() {
-            let tok = lex(b"
-                alter model mymodel add myfield { type string }
-            ")
-            .unwrap();
-            let mut i = 4;
-            let r = schema::alter_add(&tok[i..], &mut i).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                r.as_ref(),
-                [ExpandedField {
-                    field_name: "myfield".into(),
-                    props: nullable_dict! {},
-                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
-                    reset: false
-                }]
-            );
-        }
-        #[test]
-        fn add() {
-            let tok = lex(b"
-                alter model mymodel add myfield { type string, nullable: true }
-            ")
-            .unwrap();
-            let mut i = 4;
-            let r = schema::alter_add(&tok[i..], &mut i).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                r.as_ref(),
-                [ExpandedField {
-                    field_name: "myfield".into(),
-                    props: nullable_dict! {
-                        "nullable" => Lit::Bool(true)
-                    },
-                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
-                    reset: false
-                }]
-            );
-        }
-        #[test]
-        fn add_pro() {
-            let tok = lex(b"
-                alter model mymodel add (myfield { type string, nullable: true })
-            ")
-            .unwrap();
-            let mut i = 4;
-            let r = schema::alter_add(&tok[i..], &mut i).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                r.as_ref(),
-                [ExpandedField {
-                    field_name: "myfield".into(),
-                    props: nullable_dict! {
-                        "nullable" => Lit::Bool(true)
-                    },
-                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
-                    reset: false
-                }]
-            );
-        }
-        #[test]
-        fn add_pro_max() {
-            let tok = lex(b"
-                alter model mymodel add (
-                    myfield {
-                        type string,
-                        nullable: true
-                    },
-                    another {
-                        type list {
-                            type string {
-                                maxlen: 255
-                            },
-                            unique: true
-                        },
-                        nullable: false,
-                    }
-                )
-            ")
-            .unwrap();
-            let mut i = 4;
-            let r = schema::alter_add(&tok[i..], &mut i).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                r.as_ref(),
-                [
-                    ExpandedField {
-                        field_name: "myfield".into(),
-                        props: nullable_dict! {
-                            "nullable" => Lit::Bool(true)
-                        },
-                        layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
-                        reset: false
-                    },
-                    ExpandedField {
-                        field_name: "another".into(),
-                        props: nullable_dict! {
-                            "nullable" => Lit::Bool(false)
-                        },
-                        layers: [
-                            Layer::new_noreset(
-                                "string".into(),
-                                nullable_dict! {
-                                    "maxlen" => Lit::UnsignedInt(255)
-                                }
-                            ),
-                            Layer::new_noreset(
-                                "list".into(),
-                                nullable_dict! {
-                                   "unique" => Lit::Bool(true)
-                                },
-                            )
-                        ]
-                        .into(),
-                        reset: false
-                    }
-                ]
-            );
-        }
-    }
-    mod alter_model_update {
-        use super::*;
-        use crate::engine::ql::schema::{ExpandedField, Layer};
-
-        #[test]
-        fn alter_mini() {
-            let tok = lex(b"
-                alter model mymodel update myfield { type string, .. }
-            ")
-            .unwrap();
-            let mut i = 4;
-            let r = schema::alter_update(&tok[i..], &mut i).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                r.as_ref(),
-                [ExpandedField {
-                    field_name: "myfield".into(),
-                    props: nullable_dict! {},
-                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
-                    reset: true
-                }]
-            );
-        }
-        #[test]
-        fn alter_mini_2() {
-            let tok = lex(b"
-                alter model mymodel update (myfield { type string, .. })
-            ")
-            .unwrap();
-            let mut i = 4;
-            let r = schema::alter_update(&tok[i..], &mut i).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                r.as_ref(),
-                [ExpandedField {
-                    field_name: "myfield".into(),
-                    props: nullable_dict! {},
-                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
-                    reset: true
-                }]
-            );
-        }
-        #[test]
-        fn alter() {
-            let tok = lex(b"
-                alter model mymodel update (
-                    myfield {
-                        type string,
-                        nullable: true,
-                        ..
-                    }
-                )
-            ")
-            .unwrap();
-            let mut i = 4;
-            let r = schema::alter_update(&tok[i..], &mut i).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                r.as_ref(),
-                [ExpandedField {
-                    field_name: "myfield".into(),
-                    props: nullable_dict! {
-                        "nullable" => Lit::Bool(true)
-                    },
-                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
-                    reset: true
-                }]
-            );
-        }
-        #[test]
-        fn alter_pro() {
-            let tok = lex(b"
-                alter model mymodel update (
-                    myfield {
-                        type string,
-                        nullable: true,
-                        ..
-                    },
-                    myfield2 {
-                        type string,
-                        ..
-                    }
-                )
-            ")
-            .unwrap();
-            let mut i = 4;
-            let r = schema::alter_update(&tok[i..], &mut i).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                r.as_ref(),
-                [
-                    ExpandedField {
-                        field_name: "myfield".into(),
-                        props: nullable_dict! {
-                            "nullable" => Lit::Bool(true)
-                        },
-                        layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
-                        reset: true
-                    },
-                    ExpandedField {
-                        field_name: "myfield2".into(),
-                        props: nullable_dict! {},
-                        layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
-                        reset: true
-                    }
-                ]
-            );
-        }
-        #[test]
-        fn alter_pro_max() {
-            let tok = lex(b"
-                alter model mymodel update (
-                    myfield {
-                        type string {..},
-                        nullable: true,
-                        ..
-                    },
-                    myfield2 {
-                        type string {
-                            maxlen: 255,
-                            ..
-                        },
-                        ..
-                    }
-                )
-            ")
-            .unwrap();
-            let mut i = 4;
-            let r = schema::alter_update(&tok[i..], &mut i).unwrap();
-            assert_eq!(i, tok.len());
-            assert_eq!(
-                r.as_ref(),
-                [
-                    ExpandedField {
-                        field_name: "myfield".into(),
-                        props: nullable_dict! {
-                            "nullable" => Lit::Bool(true)
-                        },
-                        layers: [Layer::new_reset("string".into(), nullable_dict! {})].into(),
-                        reset: true
-                    },
-                    ExpandedField {
-                        field_name: "myfield2".into(),
-                        props: nullable_dict! {},
-                        layers: [Layer::new_reset(
-                            "string".into(),
-                            nullable_dict! {"maxlen" => Lit::UnsignedInt(255)}
-                        )]
-                        .into(),
-                        reset: true
-                    }
-                ]
-            );
-        }
-    }
-}
-
-mod dml_tests {
-    use super::*;
-    mod list_parse {
-        use super::*;
-        use crate::engine::ql::dml::parse_list_full;
-
-        #[test]
-        fn list_mini() {
-            let tok = lex(b"
-                []
-            ")
-            .unwrap();
-            let r = parse_list_full(&tok[1..]).unwrap();
-            assert_eq!(r, vec![])
-        }
-
-        #[test]
-        fn list() {
-            let tok = lex(b"
-                [1, 2, 3, 4]
-            ")
-            .unwrap();
-            let r = parse_list_full(&tok[1..]).unwrap();
-            assert_eq!(r.as_slice(), into_array![1, 2, 3, 4])
-        }
-
-        #[test]
-        fn list_pro() {
-            let tok = lex(b"
-                [
-                    [1, 2],
-                    [3, 4],
-                    [5, 6],
-                    []
-                ]
-            ")
-            .unwrap();
-            let r = parse_list_full(&tok[1..]).unwrap();
-            assert_eq!(
-                r.as_slice(),
-                into_array![
-                    into_array![1, 2],
-                    into_array![3, 4],
-                    into_array![5, 6],
-                    into_array![]
-                ]
-            )
-        }
-
-        #[test]
-        fn list_pro_max() {
-            let tok = lex(b"
-                [
-                    [[1, 1], [2, 2]],
-                    [[], [4, 4]],
-                    [[5, 5], [6, 6]],
-                    [[7, 7], []]
-                ]
-            ")
-            .unwrap();
-            let r = parse_list_full(&tok[1..]).unwrap();
-            assert_eq!(
-                r.as_slice(),
-                into_array![
-                    into_array![into_array![1, 1], into_array![2, 2]],
-                    into_array![into_array![], into_array![4, 4]],
-                    into_array![into_array![5, 5], into_array![6, 6]],
-                    into_array![into_array![7, 7], into_array![]],
-                ]
-            )
-        }
-    }
-    mod tuple_syntax {
-        use super::*;
-        use crate::engine::ql::dml::parse_data_tuple_syntax_full;
-
-        #[test]
-        fn tuple_mini() {
-            let tok = lex(b"()").unwrap();
-            let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
-            assert_eq!(r, vec![]);
-        }
-
-        #[test]
-        fn tuple() {
-            let tok = lex(br#"
-                (1234, "email@example.com", true)
-            "#)
-            .unwrap();
-            let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
-            assert_eq!(
-                r.as_slice(),
-                into_array_nullable![1234, "email@example.com", true]
-            );
-        }
-
-        #[test]
-        fn tuple_pro() {
-            let tok = lex(br#"
-                (
-                    1234,
-                    "email@example.com",
-                    true,
-                    ["hello", "world", "and", "the", "universe"]
-                )
-            "#)
-            .unwrap();
-            let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
-            assert_eq!(
-                r.as_slice(),
-                into_array_nullable![
-                    1234,
-                    "email@example.com",
-                    true,
-                    into_array!["hello", "world", "and", "the", "universe"]
-                ]
-            );
-        }
-
-        #[test]
-        fn tuple_pro_max() {
-            let tok = lex(br#"
-                (
-                    1234,
-                    "email@example.com",
-                    true,
-                    [
-                        ["h", "hello"],
-                        ["w", "world"],
-                        ["a", "and"],
-                        ["the"],
-                        ["universe"],
-                        []
-                    ]
-                )
-            "#)
-            .unwrap();
-            let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
-            assert_eq!(
-                r.as_slice(),
-                into_array_nullable![
-                    1234,
-                    "email@example.com",
-                    true,
-                    into_array![
-                        into_array!["h", "hello"],
-                        into_array!["w", "world"],
-                        into_array!["a", "and"],
-                        into_array!["the"],
-                        into_array!["universe"],
-                        into_array![],
-                    ]
-                ]
-            );
-        }
-    }
-    mod map_syntax {
-        use super::*;
-        use crate::engine::ql::dml::parse_data_map_syntax_full;
-
-        #[test]
-        fn map_mini() {
-            let tok = lex(b"{}").unwrap();
-            let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
-            assert_eq!(r, nullable_dict! {})
-        }
-
-        #[test]
-        fn map() {
-            let tok = lex(br#"
-                {
-                    name: "John Appletree",
-                    email: "john@example.com",
-                    verified: false,
-                    followers: 12345
-                }
-            "#)
-            .unwrap();
-            let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
-            assert_eq!(
-                r,
-                dict_nullable! {
-                    "name" => "John Appletree",
-                    "email" => "john@example.com",
-                    "verified" => false,
-                    "followers" => 12345,
-                }
-            )
-        }
-
-        #[test]
-        fn map_pro() {
-            let tok = lex(br#"
-                {
-                    name: "John Appletree",
-                    email: "john@example.com",
-                    verified: false,
-                    followers: 12345,
-                    tweets_by_day: []
-                }
-            "#)
-            .unwrap();
-            let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
-            assert_eq!(
-                r,
-                dict_nullable! {
-                    "name" => "John Appletree",
-                    "email" => "john@example.com",
-                    "verified" => false,
-                    "followers" => 12345,
-                    "tweets_by_day" => []
-                }
-            )
-        }
-
-        #[test]
-        fn map_pro_max() {
-            let tok = lex(br#"
-                {
-                    name: "John Appletree",
-                    email: "john@example.com",
-                    verified: false,
-                    followers: 12345,
-                    tweets_by_day: [
-                        ["it's a fresh monday", "monday was tiring"],
-                        ["already bored with tuesday", "nope. gotta change stuff, life's getting boring"],
-                        ["sunday, going to bed"]
-                    ]
-                }
-            "#)
-            .unwrap();
-            let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
-            assert_eq!(
-                r,
-                dict_nullable! {
-                    "name" => "John Appletree",
-                    "email" => "john@example.com",
-                    "verified" => false,
-                    "followers" => 12345,
-                    "tweets_by_day" => into_array![
-                        into_array![
-                            "it's a fresh monday", "monday was tiring"
-                        ],
-                        into_array![
-                            "already bored with tuesday", "nope. gotta change stuff, life's getting boring"
-                        ],
-                        into_array!["sunday, going to bed"]
-                    ]
-                }
-            )
-        }
-    }
-    mod stmt_insert {
-        use {
-            super::*,
-            crate::engine::ql::{
-                ast::Entity,
-                dml::{self, InsertStatement},
-            },
-        };
-
-        #[test]
-        fn insert_tuple_mini() {
-            let x = lex(br#"
-                insert into twitter.users ("sayan")
-            "#)
-            .unwrap();
-            let r = dml::parse_insert_full(&x[1..]).unwrap();
-            let e = InsertStatement {
-                entity: Entity::Full("twitter".into(), "users".into()),
-                data: into_array_nullable!["sayan"].to_vec().into(),
-            };
-            assert_eq!(e, r);
-        }
-        #[test]
-        fn insert_tuple() {
-            let x = lex(br#"
-                insert into twitter.users (
-                    "sayan",
-                    "Sayan",
-                    "sayan@example.com",
-                    true,
-                    12345,
-                    67890
-                )
-            "#)
-            .unwrap();
-            let r = dml::parse_insert_full(&x[1..]).unwrap();
-            let e = InsertStatement {
-                entity: Entity::Full("twitter".into(), "users".into()),
-                data: into_array_nullable![
-                    "sayan",
-                    "Sayan",
-                    "sayan@example.com",
-                    true,
-                    12345,
-                    67890
-                ]
-                .to_vec()
-                .into(),
-            };
-            assert_eq!(e, r);
-        }
-        #[test]
-        fn insert_tuple_pro() {
-            let x = lex(br#"
-                insert into twitter.users (
-                    "sayan",
-                    "Sayan",
-                    "sayan@example.com",
-                    true,
-                    12345,
-                    67890,
-                    null,
-                    12345,
-                    null
-                )
-            "#)
-            .unwrap();
-            let r = dml::parse_insert_full(&x[1..]).unwrap();
-            let e = InsertStatement {
-                entity: Entity::Full("twitter".into(), "users".into()),
-                data: into_array_nullable![
-                    "sayan",
-                    "Sayan",
-                    "sayan@example.com",
-                    true,
-                    12345,
-                    67890,
-                    Null,
-                    12345,
-                    Null
-                ]
-                .to_vec()
-                .into(),
-            };
-            assert_eq!(e, r);
-        }
-        #[test]
-        fn insert_map_mini() {
-            let tok = lex(br#"
-                insert into jotsy.app { username: "sayan" }
-            "#)
-            .unwrap();
-            let r = dml::parse_insert_full(&tok[1..]).unwrap();
-            let e = InsertStatement {
-                entity: Entity::Full("jotsy".into(), "app".into()),
-                data: dict_nullable! {
-                    "username".as_bytes() => "sayan"
-                }
-                .into(),
-            };
-            assert_eq!(e, r);
-        }
-        #[test]
-        fn insert_map() {
-            let tok = lex(br#"
-                insert into jotsy.app {
-                    username: "sayan",
-                    name: "Sayan",
-                    email: "sayan@example.com",
-                    verified: true,
-                    following: 12345,
-                    followers: 67890
-                }
-            "#)
-            .unwrap();
-            let r = dml::parse_insert_full(&tok[1..]).unwrap();
-            let e = InsertStatement {
-                entity: Entity::Full("jotsy".into(), "app".into()),
-                data: dict_nullable! {
-                    "username".as_bytes() => "sayan",
-                    "name".as_bytes() => "Sayan",
-                    "email".as_bytes() => "sayan@example.com",
-                    "verified".as_bytes() => true,
-                    "following".as_bytes() => 12345,
-                    "followers".as_bytes() => 67890
-                }
-                .into(),
-            };
-            assert_eq!(e, r);
-        }
-        #[test]
-        fn insert_map_pro() {
-            let tok = lex(br#"
-                insert into jotsy.app {
-                    username: "sayan",
-                    password: "pass123",
-                    email: "sayan@example.com",
-                    verified: true,
-                    following: 12345,
-                    followers: 67890,
-                    linked_smart_devices: null,
-                    bookmarks: 12345,
-                    other_linked_accounts: null
-                }
-            "#)
-            .unwrap();
-            let r = dml::parse_insert_full(&tok[1..]).unwrap();
-            let e = InsertStatement {
-                entity: Entity::Full("jotsy".into(), "app".into()),
-                data: dict_nullable! {
-                    "username".as_bytes() => "sayan",
-                    "password".as_bytes() => "pass123",
-                    "email".as_bytes() => "sayan@example.com",
-                    "verified".as_bytes() => true,
-                    "following".as_bytes() => 12345,
-                    "followers".as_bytes() => 67890,
-                    "linked_smart_devices".as_bytes() => Null,
-                    "bookmarks".as_bytes() => 12345,
-                    "other_linked_accounts".as_bytes() => Null
-                }
-                .into(),
-            };
-            assert_eq!(r, e);
-        }
-    }
-
-    mod stmt_select {
-        use crate::engine::ql::dml::RelationalExpr;
-
-        use {
-            super::*,
-            crate::engine::ql::{
-                ast::Entity,
-                dml::{self, SelectStatement},
-            },
-        };
-        #[test]
-        fn select_mini() {
-            let tok = lex(br#"
-                select * from users where username = "sayan"
-            "#)
-            .unwrap();
-            let r = dml::parse_select_full(&tok[1..]).unwrap();
-            let username_where = "sayan".into();
-            let e = SelectStatement::new_test(
-                Entity::Single("users".into()),
-                [].to_vec(),
-                true,
-                dict! {
-                    "username".as_bytes() => RelationalExpr::new(
-                        "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
-                    ),
-                },
-            );
-            assert_eq!(r, e);
-        }
-        #[test]
-        fn select() {
-            let tok = lex(br#"
-                select field1 from users where username = "sayan"
-            "#)
-            .unwrap();
-            let r = dml::parse_select_full(&tok[1..]).unwrap();
-            let username_where = "sayan".into();
-            let e = SelectStatement::new_test(
-                Entity::Single("users".into()),
-                ["field1".into()].to_vec(),
-                false,
-                dict! {
-                    "username".as_bytes() => RelationalExpr::new(
-                        "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
-                    ),
-                },
-            );
-            assert_eq!(r, e);
-        }
-        #[test]
-        fn select_pro() {
-            let tok = lex(br#"
-                select field1 from twitter.users where username = "sayan"
-            "#)
-            .unwrap();
-            let r = dml::parse_select_full(&tok[1..]).unwrap();
-            let username_where = "sayan".into();
-            let e = SelectStatement::new_test(
-                Entity::Full("twitter".into(), "users".into()),
-                ["field1".into()].to_vec(),
-                false,
-                dict! {
-                    "username".as_bytes() => RelationalExpr::new(
-                        "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
-                    ),
-                },
-            );
-            assert_eq!(r, e);
-        }
-        #[test]
-        fn select_pro_max() {
-            let tok = lex(br#"
-                select field1, field2 from twitter.users where username = "sayan"
-            "#)
-            .unwrap();
-            let r = dml::parse_select_full(&tok[1..]).unwrap();
-            let username_where = "sayan".into();
-            let e = SelectStatement::new_test(
-                Entity::Full("twitter".into(), "users".into()),
-                ["field1".into(), "field2".into()].to_vec(),
-                false,
-                dict! {
-                    "username".as_bytes() => RelationalExpr::new(
-                        "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
-                    ),
-                },
-            );
-            assert_eq!(r, e);
-        }
-    }
-    mod expression_tests {
-        use {
-            super::*,
-            crate::engine::ql::{
-                dml::{self, AssignmentExpression, Operator},
-                lexer::Lit,
-            },
-        };
-        #[test]
-        fn expr_assign() {
-            let src = lex(b"username = 'sayan'").unwrap();
-            let r = dml::parse_expression_full(&src).unwrap();
-            assert_eq!(
-                r,
-                AssignmentExpression {
-                    lhs: "username".into(),
-                    rhs: &Lit::Str("sayan".into()),
-                    operator_fn: Operator::Assign
-                }
-            );
-        }
-        #[test]
-        fn expr_add_assign() {
-            let src = lex(b"followers += 100").unwrap();
-            let r = dml::parse_expression_full(&src).unwrap();
-            assert_eq!(
-                r,
-                AssignmentExpression {
-                    lhs: "followers".into(),
-                    rhs: &(100.into()),
-                    operator_fn: Operator::AddAssign
-                }
-            );
-        }
-        #[test]
-        fn expr_sub_assign() {
-            let src = lex(b"following -= 150").unwrap();
-            let r = dml::parse_expression_full(&src).unwrap();
-            assert_eq!(
-                r,
-                AssignmentExpression {
-                    lhs: "following".into(),
-                    rhs: &(150.into()),
-                    operator_fn: Operator::SubAssign
-                }
-            );
-        }
-        #[test]
-        fn expr_mul_assign() {
-            let src = lex(b"product_qty *= 2").unwrap();
-            let r = dml::parse_expression_full(&src).unwrap();
-            assert_eq!(
-                r,
-                AssignmentExpression {
-                    lhs: "product_qty".into(),
-                    rhs: &(2.into()),
-                    operator_fn: Operator::MulAssign
-                }
-            );
-        }
-        #[test]
-        fn expr_div_assign() {
-            let src = lex(b"image_crop_factor /= 2").unwrap();
-            let r = dml::parse_expression_full(&src).unwrap();
-            assert_eq!(
-                r,
-                AssignmentExpression {
-                    lhs: "image_crop_factor".into(),
-                    rhs: &(2.into()),
-                    operator_fn: Operator::DivAssign
-                }
-            );
-        }
-    }
-    mod update_statement {
-        use {
-            super::*,
-            crate::engine::ql::{
-                ast::Entity,
-                dml::{
-                    self, AssignmentExpression, Operator, RelationalExpr, UpdateStatement,
-                    WhereClause,
-                },
-            },
-        };
-        #[test]
-        fn update_mini() {
-            let tok = lex(br#"
-                update app SET notes += "this is my new note" where username = "sayan"
-            "#)
-            .unwrap();
-            let where_username = "sayan".into();
-            let note = "this is my new note".to_string().into();
-            let r = dml::parse_update_full(&tok[1..]).unwrap();
-            let e = UpdateStatement {
-                entity: Entity::Single("app".into()),
-                expressions: vec![AssignmentExpression {
-                    lhs: "notes".into(),
-                    rhs: &note,
-                    operator_fn: Operator::AddAssign,
-                }],
-                wc: WhereClause::new(dict! {
-                    "username".as_bytes() => RelationalExpr::new(
-                        "username".as_bytes(),
-                        &where_username,
-                        RelationalExpr::OP_EQ
-                    )
-                }),
-            };
-            assert_eq!(r, e);
-        }
-        #[test]
-        fn update() {
-            let tok = lex(br#"
-                update
-                    jotsy.app
-                SET
-                    notes += "this is my new note",
-                    email = "sayan@example.com"
-                WHERE
-                    username = "sayan"
-            "#)
-            .unwrap();
-            let r = dml::parse_update_full(&tok[1..]).unwrap();
-            let where_username = "sayan".into();
-            let field_note = "this is my new note".into();
-            let field_email = "sayan@example.com".into();
-            let e = UpdateStatement {
-                entity: ("jotsy", "app").into(),
-                expressions: vec![
-                    AssignmentExpression::new("notes".into(), &field_note, Operator::AddAssign),
-                    AssignmentExpression::new("email".into(), &field_email, Operator::Assign),
-                ],
-                wc: WhereClause::new(dict! {
-                    "username".as_bytes() => RelationalExpr::new(
-                        "username".as_bytes(),
-                        &where_username,
-                        RelationalExpr::OP_EQ
-                    )
-                }),
-            };
-
-            assert_eq!(r, e);
-        }
-    }
-    mod delete_stmt {
-        use {
-            super::*,
-            crate::engine::ql::{
-                ast::Entity,
-                dml::{self, DeleteStatement, RelationalExpr},
-            },
-        };
-
-        #[test]
-        fn delete_mini() {
-            let tok = lex(br#"
-                delete from users where username = "sayan"
-            "#)
-            .unwrap();
-            let primary_key = "sayan".into();
-            let e = DeleteStatement::new_test(
-                Entity::Single("users".into()),
-                dict! {
-                    "username".as_bytes() => RelationalExpr::new(
-                        "username".as_bytes(),
-                        &primary_key,
-                        RelationalExpr::OP_EQ
-                    )
-                },
-            );
-            let r = dml::parse_delete_full(&tok[1..]).unwrap();
-            assert_eq!(r, e);
-        }
-        #[test]
-        fn delete() {
-            let tok = lex(br#"
-                delete from twitter.users where username = "sayan"
-            "#)
-            .unwrap();
-            let primary_key = "sayan".into();
-            let e = DeleteStatement::new_test(
-                ("twitter", "users").into(),
-                dict! {
-                    "username".as_bytes() => RelationalExpr::new(
-                        "username".as_bytes(),
-                        &primary_key,
-                        RelationalExpr::OP_EQ
-                    )
-                },
-            );
-            let r = dml::parse_delete_full(&tok[1..]).unwrap();
-            assert_eq!(r, e);
-        }
-    }
-    mod relational_expr {
-        use {
-            super::*,
-            crate::engine::ql::dml::{self, RelationalExpr},
-        };
-
-        #[test]
-        fn expr_eq() {
-            let expr = lex(b"primary_key = 10").unwrap();
-            let r = dml::parse_relexpr_full(&expr).unwrap();
-            assert_eq!(
-                r,
-                RelationalExpr {
-                    rhs: &(10.into()),
-                    lhs: "primary_key".as_bytes(),
-                    opc: RelationalExpr::OP_EQ
-                }
-            );
-        }
-        #[test]
-        fn expr_ne() {
-            let expr = lex(b"primary_key != 10").unwrap();
-            let r = dml::parse_relexpr_full(&expr).unwrap();
-            assert_eq!(
-                r,
-                RelationalExpr {
-                    rhs: &(10.into()),
-                    lhs: "primary_key".as_bytes(),
-                    opc: RelationalExpr::OP_NE
-                }
-            );
-        }
-        #[test]
-        fn expr_gt() {
-            let expr = lex(b"primary_key > 10").unwrap();
-            let r = dml::parse_relexpr_full(&expr).unwrap();
-            assert_eq!(
-                r,
-                RelationalExpr {
-                    rhs: &(10.into()),
-                    lhs: "primary_key".as_bytes(),
-                    opc: RelationalExpr::OP_GT
-                }
-            );
-        }
-        #[test]
-        fn expr_ge() {
-            let expr = lex(b"primary_key >= 10").unwrap();
-            let r = dml::parse_relexpr_full(&expr).unwrap();
-            assert_eq!(
-                r,
-                RelationalExpr {
-                    rhs: &(10.into()),
-                    lhs: "primary_key".as_bytes(),
-                    opc: RelationalExpr::OP_GE
-                }
-            );
-        }
-        #[test]
-        fn expr_lt() {
-            let expr = lex(b"primary_key < 10").unwrap();
-            let r = dml::parse_relexpr_full(&expr).unwrap();
-            assert_eq!(
-                r,
-                RelationalExpr {
-                    rhs: &(10.into()),
-                    lhs: "primary_key".as_bytes(),
-                    opc: RelationalExpr::OP_LT
-                }
-            );
-        }
-        #[test]
-        fn expr_le() {
-            let expr = lex(b"primary_key <= 10").unwrap();
-            let r = dml::parse_relexpr_full(&expr).unwrap();
-            assert_eq!(
-                r,
-                RelationalExpr {
-                    rhs: &(10.into()),
-                    lhs: "primary_key".as_bytes(),
-                    opc: RelationalExpr::OP_LE
-                }
-            );
-        }
-    }
-    mod where_clause {
-        use {
-            super::*,
-            crate::engine::ql::dml::{self, RelationalExpr, WhereClause},
-        };
-        #[test]
-        fn where_single() {
-            let tok = lex(br#"
-                x = 100
-            "#)
-            .unwrap();
-            let rhs_hundred = 100.into();
-            let expected = WhereClause::new(dict! {
-                "x".as_bytes() => RelationalExpr {
-                    rhs: &rhs_hundred,
-                    lhs: "x".as_bytes(),
-                    opc: RelationalExpr::OP_EQ
-                }
-            });
-            assert_eq!(expected, dml::parse_where_clause_full(&tok).unwrap());
-        }
-        #[test]
-        fn where_double() {
-            let tok = lex(br#"
-                userid = 100 and pass = "password"
-            "#)
-            .unwrap();
-            let rhs_hundred = 100.into();
-            let rhs_password = "password".into();
-            let expected = WhereClause::new(dict! {
-                "userid".as_bytes() => RelationalExpr {
-                    rhs: &rhs_hundred,
-                    lhs: "userid".as_bytes(),
-                    opc: RelationalExpr::OP_EQ
-                },
-                "pass".as_bytes() => RelationalExpr {
-                    rhs: &rhs_password,
-                    lhs: "pass".as_bytes(),
-                    opc: RelationalExpr::OP_EQ
-                }
-            });
-            assert_eq!(expected, dml::parse_where_clause_full(&tok).unwrap());
-        }
-        #[test]
-        fn where_duplicate_condition() {
-            let tok = lex(br#"
-                userid = 100 and userid > 200
-            "#)
-            .unwrap();
-            assert!(dml::parse_where_clause_full(&tok).is_none());
-        }
-    }
-}
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
new file mode 100644
index 00000000..2f2b97be
--- /dev/null
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -0,0 +1,866 @@
+/*
+ * Created on Sun Dec 18 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::*;
+mod list_parse {
+    use super::*;
+    use crate::engine::ql::dml::parse_list_full;
+
+    #[test]
+    fn list_mini() {
+        let tok = lex(b"
+                []
+            ")
+        .unwrap();
+        let r = parse_list_full(&tok[1..]).unwrap();
+        assert_eq!(r, vec![])
+    }
+
+    #[test]
+    fn list() {
+        let tok = lex(b"
+                [1, 2, 3, 4]
+            ")
+        .unwrap();
+        let r = parse_list_full(&tok[1..]).unwrap();
+        assert_eq!(r.as_slice(), into_array![1, 2, 3, 4])
+    }
+
+    #[test]
+    fn list_pro() {
+        let tok = lex(b"
+                [
+                    [1, 2],
+                    [3, 4],
+                    [5, 6],
+                    []
+                ]
+            ")
+        .unwrap();
+        let r = parse_list_full(&tok[1..]).unwrap();
+        assert_eq!(
+            r.as_slice(),
+            into_array![
+                into_array![1, 2],
+                into_array![3, 4],
+                into_array![5, 6],
+                into_array![]
+            ]
+        )
+    }
+
+    #[test]
+    fn list_pro_max() {
+        let tok = lex(b"
+                [
+                    [[1, 1], [2, 2]],
+                    [[], [4, 4]],
+                    [[5, 5], [6, 6]],
+                    [[7, 7], []]
+                ]
+            ")
+        .unwrap();
+        let r = parse_list_full(&tok[1..]).unwrap();
+        assert_eq!(
+            r.as_slice(),
+            into_array![
+                into_array![into_array![1, 1], into_array![2, 2]],
+                into_array![into_array![], into_array![4, 4]],
+                into_array![into_array![5, 5], into_array![6, 6]],
+                into_array![into_array![7, 7], into_array![]],
+            ]
+        )
+    }
+}
+mod tuple_syntax {
+    use super::*;
+    use crate::engine::ql::dml::parse_data_tuple_syntax_full;
+
+    #[test]
+    fn tuple_mini() {
+        let tok = lex(b"()").unwrap();
+        let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
+        assert_eq!(r, vec![]);
+    }
+
+    #[test]
+    fn tuple() {
+        let tok = lex(br#"
+                (1234, "email@example.com", true)
+            "#)
+        .unwrap();
+        let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
+        assert_eq!(
+            r.as_slice(),
+            into_array_nullable![1234, "email@example.com", true]
+        );
+    }
+
+    #[test]
+    fn tuple_pro() {
+        let tok = lex(br#"
+                (
+                    1234,
+                    "email@example.com",
+                    true,
+                    ["hello", "world", "and", "the", "universe"]
+                )
+            "#)
+        .unwrap();
+        let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
+        assert_eq!(
+            r.as_slice(),
+            into_array_nullable![
+                1234,
+                "email@example.com",
+                true,
+                into_array!["hello", "world", "and", "the", "universe"]
+            ]
+        );
+    }
+
+    #[test]
+    fn tuple_pro_max() {
+        let tok = lex(br#"
+                (
+                    1234,
+                    "email@example.com",
+                    true,
+                    [
+                        ["h", "hello"],
+                        ["w", "world"],
+                        ["a", "and"],
+                        ["the"],
+                        ["universe"],
+                        []
+                    ]
+                )
+            "#)
+        .unwrap();
+        let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
+        assert_eq!(
+            r.as_slice(),
+            into_array_nullable![
+                1234,
+                "email@example.com",
+                true,
+                into_array![
+                    into_array!["h", "hello"],
+                    into_array!["w", "world"],
+                    into_array!["a", "and"],
+                    into_array!["the"],
+                    into_array!["universe"],
+                    into_array![],
+                ]
+            ]
+        );
+    }
+}
+mod map_syntax {
+    use super::*;
+    use crate::engine::ql::dml::parse_data_map_syntax_full;
+
+    #[test]
+    fn map_mini() {
+        let tok = lex(b"{}").unwrap();
+        let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
+        assert_eq!(r, nullable_dict! {})
+    }
+
+    #[test]
+    fn map() {
+        let tok = lex(br#"
+                {
+                    name: "John Appletree",
+                    email: "john@example.com",
+                    verified: false,
+                    followers: 12345
+                }
+            "#)
+        .unwrap();
+        let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
+        assert_eq!(
+            r,
+            dict_nullable! {
+                "name" => "John Appletree",
+                "email" => "john@example.com",
+                "verified" => false,
+                "followers" => 12345,
+            }
+        )
+    }
+
+    #[test]
+    fn map_pro() {
+        let tok = lex(br#"
+                {
+                    name: "John Appletree",
+                    email: "john@example.com",
+                    verified: false,
+                    followers: 12345,
+                    tweets_by_day: []
+                }
+            "#)
+        .unwrap();
+        let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
+        assert_eq!(
+            r,
+            dict_nullable! {
+                "name" => "John Appletree",
+                "email" => "john@example.com",
+                "verified" => false,
+                "followers" => 12345,
+                "tweets_by_day" => []
+            }
+        )
+    }
+
+    #[test]
+    fn map_pro_max() {
+        let tok = lex(br#"
+                {
+                    name: "John Appletree",
+                    email: "john@example.com",
+                    verified: false,
+                    followers: 12345,
+                    tweets_by_day: [
+                        ["it's a fresh monday", "monday was tiring"],
+                        ["already bored with tuesday", "nope. gotta change stuff, life's getting boring"],
+                        ["sunday, going to bed"]
+                    ]
+                }
+            "#)
+        .unwrap();
+        let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
+        assert_eq!(
+            r,
+            dict_nullable! {
+                "name" => "John Appletree",
+                "email" => "john@example.com",
+                "verified" => false,
+                "followers" => 12345,
+                "tweets_by_day" => into_array![
+                    into_array![
+                        "it's a fresh monday", "monday was tiring"
+                    ],
+                    into_array![
+                        "already bored with tuesday", "nope. gotta change stuff, life's getting boring"
+                    ],
+                    into_array!["sunday, going to bed"]
+                ]
+            }
+        )
+    }
+}
+mod stmt_insert {
+    use {
+        super::*,
+        crate::engine::ql::{
+            ast::Entity,
+            dml::{self, InsertStatement},
+        },
+    };
+
+    #[test]
+    fn insert_tuple_mini() {
+        let x = lex(br#"
+                insert into twitter.users ("sayan")
+            "#)
+        .unwrap();
+        let r = dml::parse_insert_full(&x[1..]).unwrap();
+        let e = InsertStatement {
+            entity: Entity::Full("twitter".into(), "users".into()),
+            data: into_array_nullable!["sayan"].to_vec().into(),
+        };
+        assert_eq!(e, r);
+    }
+    #[test]
+    fn insert_tuple() {
+        let x = lex(br#"
+                insert into twitter.users (
+                    "sayan",
+                    "Sayan",
+                    "sayan@example.com",
+                    true,
+                    12345,
+                    67890
+                )
+            "#)
+        .unwrap();
+        let r = dml::parse_insert_full(&x[1..]).unwrap();
+        let e = InsertStatement {
+            entity: Entity::Full("twitter".into(), "users".into()),
+            data: into_array_nullable![
+                "sayan",
+                "Sayan",
+                "sayan@example.com",
+                true,
+                12345,
+                67890
+            ]
+            .to_vec()
+            .into(),
+        };
+        assert_eq!(e, r);
+    }
+    #[test]
+    fn insert_tuple_pro() {
+        let x = lex(br#"
+                insert into twitter.users (
+                    "sayan",
+                    "Sayan",
+                    "sayan@example.com",
+                    true,
+                    12345,
+                    67890,
+                    null,
+                    12345,
+                    null
+                )
+            "#)
+        .unwrap();
+        let r = dml::parse_insert_full(&x[1..]).unwrap();
+        let e = InsertStatement {
+            entity: Entity::Full("twitter".into(), "users".into()),
+            data: into_array_nullable![
+                "sayan",
+                "Sayan",
+                "sayan@example.com",
+                true,
+                12345,
+                67890,
+                Null,
+                12345,
+                Null
+            ]
+            .to_vec()
+            .into(),
+        };
+        assert_eq!(e, r);
+    }
+    #[test]
+    fn insert_map_mini() {
+        let tok = lex(br#"
+                insert into jotsy.app { username: "sayan" }
+            "#)
+        .unwrap();
+        let r = dml::parse_insert_full(&tok[1..]).unwrap();
+        let e = InsertStatement {
+            entity: Entity::Full("jotsy".into(), "app".into()),
+            data: dict_nullable! {
+                "username".as_bytes() => "sayan"
+            }
+            .into(),
+        };
+        assert_eq!(e, r);
+    }
+    #[test]
+    fn insert_map() {
+        let tok = lex(br#"
+                insert into jotsy.app {
+                    username: "sayan",
+                    name: "Sayan",
+                    email: "sayan@example.com",
+                    verified: true,
+                    following: 12345,
+                    followers: 67890
+                }
+            "#)
+        .unwrap();
+        let r = dml::parse_insert_full(&tok[1..]).unwrap();
+        let e = InsertStatement {
+            entity: Entity::Full("jotsy".into(), "app".into()),
+            data: dict_nullable! {
+                "username".as_bytes() => "sayan",
+                "name".as_bytes() => "Sayan",
+                "email".as_bytes() => "sayan@example.com",
+                "verified".as_bytes() => true,
+                "following".as_bytes() => 12345,
+                "followers".as_bytes() => 67890
+            }
+            .into(),
+        };
+        assert_eq!(e, r);
+    }
+    #[test]
+    fn insert_map_pro() {
+        let tok = lex(br#"
+                insert into jotsy.app {
+                    username: "sayan",
+                    password: "pass123",
+                    email: "sayan@example.com",
+                    verified: true,
+                    following: 12345,
+                    followers: 67890,
+                    linked_smart_devices: null,
+                    bookmarks: 12345,
+                    other_linked_accounts: null
+                }
+            "#)
+        .unwrap();
+        let r = dml::parse_insert_full(&tok[1..]).unwrap();
+        let e = InsertStatement {
+            entity: Entity::Full("jotsy".into(), "app".into()),
+            data: dict_nullable! {
+                "username".as_bytes() => "sayan",
+                "password".as_bytes() => "pass123",
+                "email".as_bytes() => "sayan@example.com",
+                "verified".as_bytes() => true,
+                "following".as_bytes() => 12345,
+                "followers".as_bytes() => 67890,
+                "linked_smart_devices".as_bytes() => Null,
+                "bookmarks".as_bytes() => 12345,
+                "other_linked_accounts".as_bytes() => Null
+            }
+            .into(),
+        };
+        assert_eq!(r, e);
+    }
+}
+
+mod stmt_select {
+    use crate::engine::ql::dml::RelationalExpr;
+
+    use {
+        super::*,
+        crate::engine::ql::{
+            ast::Entity,
+            dml::{self, SelectStatement},
+        },
+    };
+    #[test]
+    fn select_mini() {
+        let tok = lex(br#"
+                select * from users where username = "sayan"
+            "#)
+        .unwrap();
+        let r = dml::parse_select_full(&tok[1..]).unwrap();
+        let username_where = "sayan".into();
+        let e = SelectStatement::new_test(
+            Entity::Single("users".into()),
+            [].to_vec(),
+            true,
+            dict! {
+                "username".as_bytes() => RelationalExpr::new(
+                    "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
+                ),
+            },
+        );
+        assert_eq!(r, e);
+    }
+    #[test]
+    fn select() {
+        let tok = lex(br#"
+                select field1 from users where username = "sayan"
+            "#)
+        .unwrap();
+        let r = dml::parse_select_full(&tok[1..]).unwrap();
+        let username_where = "sayan".into();
+        let e = SelectStatement::new_test(
+            Entity::Single("users".into()),
+            ["field1".into()].to_vec(),
+            false,
+            dict! {
+                "username".as_bytes() => RelationalExpr::new(
+                    "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
+                ),
+            },
+        );
+        assert_eq!(r, e);
+    }
+    #[test]
+    fn select_pro() {
+        let tok = lex(br#"
+                select field1 from twitter.users where username = "sayan"
+            "#)
+        .unwrap();
+        let r = dml::parse_select_full(&tok[1..]).unwrap();
+        let username_where = "sayan".into();
+        let e = SelectStatement::new_test(
+            Entity::Full("twitter".into(), "users".into()),
+            ["field1".into()].to_vec(),
+            false,
+            dict! {
+                "username".as_bytes() => RelationalExpr::new(
+                    "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
+                ),
+            },
+        );
+        assert_eq!(r, e);
+    }
+    #[test]
+    fn select_pro_max() {
+        let tok = lex(br#"
+                select field1, field2 from twitter.users where username = "sayan"
+            "#)
+        .unwrap();
+        let r = dml::parse_select_full(&tok[1..]).unwrap();
+        let username_where = "sayan".into();
+        let e = SelectStatement::new_test(
+            Entity::Full("twitter".into(), "users".into()),
+            ["field1".into(), "field2".into()].to_vec(),
+            false,
+            dict! {
+                "username".as_bytes() => RelationalExpr::new(
+                    "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
+                ),
+            },
+        );
+        assert_eq!(r, e);
+    }
+}
+mod expression_tests {
+    use {
+        super::*,
+        crate::engine::ql::{
+            dml::{self, AssignmentExpression, Operator},
+            lexer::Lit,
+        },
+    };
+    #[test]
+    fn expr_assign() {
+        let src = lex(b"username = 'sayan'").unwrap();
+        let r = dml::parse_expression_full(&src).unwrap();
+        assert_eq!(
+            r,
+            AssignmentExpression {
+                lhs: "username".into(),
+                rhs: &Lit::Str("sayan".into()),
+                operator_fn: Operator::Assign
+            }
+        );
+    }
+    #[test]
+    fn expr_add_assign() {
+        let src = lex(b"followers += 100").unwrap();
+        let r = dml::parse_expression_full(&src).unwrap();
+        assert_eq!(
+            r,
+            AssignmentExpression {
+                lhs: "followers".into(),
+                rhs: &(100.into()),
+                operator_fn: Operator::AddAssign
+            }
+        );
+    }
+    #[test]
+    fn expr_sub_assign() {
+        let src = lex(b"following -= 150").unwrap();
+        let r = dml::parse_expression_full(&src).unwrap();
+        assert_eq!(
+            r,
+            AssignmentExpression {
+                lhs: "following".into(),
+                rhs: &(150.into()),
+                operator_fn: Operator::SubAssign
+            }
+        );
+    }
+    #[test]
+    fn expr_mul_assign() {
+        let src = lex(b"product_qty *= 2").unwrap();
+        let r = dml::parse_expression_full(&src).unwrap();
+        assert_eq!(
+            r,
+            AssignmentExpression {
+                lhs: "product_qty".into(),
+                rhs: &(2.into()),
+                operator_fn: Operator::MulAssign
+            }
+        );
+    }
+    #[test]
+    fn expr_div_assign() {
+        let src = lex(b"image_crop_factor /= 2").unwrap();
+        let r = dml::parse_expression_full(&src).unwrap();
+        assert_eq!(
+            r,
+            AssignmentExpression {
+                lhs: "image_crop_factor".into(),
+                rhs: &(2.into()),
+                operator_fn: Operator::DivAssign
+            }
+        );
+    }
+}
+mod update_statement {
+    use {
+        super::*,
+        crate::engine::ql::{
+            ast::Entity,
+            dml::{
+                self, AssignmentExpression, Operator, RelationalExpr, UpdateStatement,
+                WhereClause,
+            },
+        },
+    };
+    #[test]
+    fn update_mini() {
+        let tok = lex(br#"
+                update app SET notes += "this is my new note" where username = "sayan"
+            "#)
+        .unwrap();
+        let where_username = "sayan".into();
+        let note = "this is my new note".to_string().into();
+        let r = dml::parse_update_full(&tok[1..]).unwrap();
+        let e = UpdateStatement {
+            entity: Entity::Single("app".into()),
+            expressions: vec![AssignmentExpression {
+                lhs: "notes".into(),
+                rhs: &note,
+                operator_fn: Operator::AddAssign,
+            }],
+            wc: WhereClause::new(dict! {
+                "username".as_bytes() => RelationalExpr::new(
+                    "username".as_bytes(),
+                    &where_username,
+                    RelationalExpr::OP_EQ
+                )
+            }),
+        };
+        assert_eq!(r, e);
+    }
+    #[test]
+    fn update() {
+        let tok = lex(br#"
+                update
+                    jotsy.app
+                SET
+                    notes += "this is my new note",
+                    email = "sayan@example.com"
+                WHERE
+                    username = "sayan"
+            "#)
+        .unwrap();
+        let r = dml::parse_update_full(&tok[1..]).unwrap();
+        let where_username = "sayan".into();
+        let field_note = "this is my new note".into();
+        let field_email = "sayan@example.com".into();
+        let e = UpdateStatement {
+            entity: ("jotsy", "app").into(),
+            expressions: vec![
+                AssignmentExpression::new("notes".into(), &field_note, Operator::AddAssign),
+                AssignmentExpression::new("email".into(), &field_email, Operator::Assign),
+            ],
+            wc: WhereClause::new(dict! {
+                "username".as_bytes() => RelationalExpr::new(
+                    "username".as_bytes(),
+                    &where_username,
+                    RelationalExpr::OP_EQ
+                )
+            }),
+        };
+
+        assert_eq!(r, e);
+    }
+}
+mod delete_stmt {
+    use {
+        super::*,
+        crate::engine::ql::{
+            ast::Entity,
+            dml::{self, DeleteStatement, RelationalExpr},
+        },
+    };
+
+    #[test]
+    fn delete_mini() {
+        let tok = lex(br#"
+                delete from users where username = "sayan"
+            "#)
+        .unwrap();
+        let primary_key = "sayan".into();
+        let e = DeleteStatement::new_test(
+            Entity::Single("users".into()),
+            dict! {
+                "username".as_bytes() => RelationalExpr::new(
+                    "username".as_bytes(),
+                    &primary_key,
+                    RelationalExpr::OP_EQ
+                )
+            },
+        );
+        let r = dml::parse_delete_full(&tok[1..]).unwrap();
+        assert_eq!(r, e);
+    }
+    #[test]
+    fn delete() {
+        let tok = lex(br#"
+                delete from twitter.users where username = "sayan"
+            "#)
+        .unwrap();
+        let primary_key = "sayan".into();
+        let e = DeleteStatement::new_test(
+            ("twitter", "users").into(),
+            dict! {
+                "username".as_bytes() => RelationalExpr::new(
+                    "username".as_bytes(),
+                    &primary_key,
+                    RelationalExpr::OP_EQ
+                )
+            },
+        );
+        let r = dml::parse_delete_full(&tok[1..]).unwrap();
+        assert_eq!(r, e);
+    }
+}
+mod relational_expr {
+    use {
+        super::*,
+        crate::engine::ql::dml::{self, RelationalExpr},
+    };
+
+    #[test]
+    fn expr_eq() {
+        let expr = lex(b"primary_key = 10").unwrap();
+        let r = dml::parse_relexpr_full(&expr).unwrap();
+        assert_eq!(
+            r,
+            RelationalExpr {
+                rhs: &(10.into()),
+                lhs: "primary_key".as_bytes(),
+                opc: RelationalExpr::OP_EQ
+            }
+        );
+    }
+    #[test]
+    fn expr_ne() {
+        let expr = lex(b"primary_key != 10").unwrap();
+        let r = dml::parse_relexpr_full(&expr).unwrap();
+        assert_eq!(
+            r,
+            RelationalExpr {
+                rhs: &(10.into()),
+                lhs: "primary_key".as_bytes(),
+                opc: RelationalExpr::OP_NE
+            }
+        );
+    }
+    #[test]
+    fn expr_gt() {
+        let expr = lex(b"primary_key > 10").unwrap();
+        let r = dml::parse_relexpr_full(&expr).unwrap();
+        assert_eq!(
+            r,
+            RelationalExpr {
+                rhs: &(10.into()),
+                lhs: "primary_key".as_bytes(),
+                opc: RelationalExpr::OP_GT
+            }
+        );
+    }
+    #[test]
+    fn expr_ge() {
+        let expr = lex(b"primary_key >= 10").unwrap();
+        let r = dml::parse_relexpr_full(&expr).unwrap();
+        assert_eq!(
+            r,
+            RelationalExpr {
+                rhs: &(10.into()),
+                lhs: "primary_key".as_bytes(),
+                opc: RelationalExpr::OP_GE
+            }
+        );
+    }
+    #[test]
+    fn expr_lt() {
+        let expr = lex(b"primary_key < 10").unwrap();
+        let r = dml::parse_relexpr_full(&expr).unwrap();
+        assert_eq!(
+            r,
+            RelationalExpr {
+                rhs: &(10.into()),
+                lhs: "primary_key".as_bytes(),
+                opc: RelationalExpr::OP_LT
+            }
+        );
+    }
+    #[test]
+    fn expr_le() {
+        let expr = lex(b"primary_key <= 10").unwrap();
+        let r = dml::parse_relexpr_full(&expr).unwrap();
+        assert_eq!(
+            r,
+            RelationalExpr {
+                rhs: &(10.into()),
+                lhs: "primary_key".as_bytes(),
+                opc: RelationalExpr::OP_LE
+            }
+        );
+    }
+}
+mod where_clause {
+    use {
+        super::*,
+        crate::engine::ql::dml::{self, RelationalExpr, WhereClause},
+    };
+    #[test]
+    fn where_single() {
+        let tok = lex(br#"
+                x = 100
+            "#)
+        .unwrap();
+        let rhs_hundred = 100.into();
+        let expected = WhereClause::new(dict! {
+            "x".as_bytes() => RelationalExpr {
+                rhs: &rhs_hundred,
+                lhs: "x".as_bytes(),
+                opc: RelationalExpr::OP_EQ
+            }
+        });
+        assert_eq!(expected, dml::parse_where_clause_full(&tok).unwrap());
+    }
+    #[test]
+    fn where_double() {
+        let tok = lex(br#"
+                userid = 100 and pass = "password"
+            "#)
+        .unwrap();
+        let rhs_hundred = 100.into();
+        let rhs_password = "password".into();
+        let expected = WhereClause::new(dict! {
+            "userid".as_bytes() => RelationalExpr {
+                rhs: &rhs_hundred,
+                lhs: "userid".as_bytes(),
+                opc: RelationalExpr::OP_EQ
+            },
+            "pass".as_bytes() => RelationalExpr {
+                rhs: &rhs_password,
+                lhs: "pass".as_bytes(),
+                opc: RelationalExpr::OP_EQ
+            }
+        });
+        assert_eq!(expected, dml::parse_where_clause_full(&tok).unwrap());
+    }
+    #[test]
+    fn where_duplicate_condition() {
+        let tok = lex(br#"
+                userid = 100 and userid > 200
+            "#)
+        .unwrap();
+        assert!(dml::parse_where_clause_full(&tok).is_none());
+    }
+}
diff --git a/server/src/engine/ql/tests/entity.rs b/server/src/engine/ql/tests/entity.rs
new file mode 100644
index 00000000..7bc3a117
--- /dev/null
+++ b/server/src/engine/ql/tests/entity.rs
@@ -0,0 +1,49 @@
+/*
+ * Created on Sun Dec 18 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::*;
+use crate::engine::ql::ast::{Compiler, Entity};
+#[test]
+fn entity_current() {
+    let t = lex(b"hello").unwrap();
+    let mut c = Compiler::new(&t);
+    let r = Entity::parse(&mut c).unwrap();
+    assert_eq!(r, Entity::Single("hello".into()))
+}
+#[test]
+fn entity_partial() {
+    let t = lex(b":hello").unwrap();
+    let mut c = Compiler::new(&t);
+    let r = Entity::parse(&mut c).unwrap();
+    assert_eq!(r, Entity::Partial("hello".into()))
+}
+#[test]
+fn entity_full() {
+    let t = lex(b"hello.world").unwrap();
+    let mut c = Compiler::new(&t);
+    let r = Entity::parse(&mut c).unwrap();
+    assert_eq!(r, Entity::Full("hello".into(), "world".into()))
+}
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
new file mode 100644
index 00000000..c4536566
--- /dev/null
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -0,0 +1,393 @@
+/*
+ * Created on Sun Dec 18 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{
+        super::lexer::{Lit, Token},
+        lex,
+    },
+    crate::engine::ql::LangError,
+};
+
+macro_rules! v(
+    ($e:literal) => {{
+        $e.as_bytes().to_vec()
+    }};
+    ($($e:literal),* $(,)?) => {{
+        ($(v!($e)),*)
+    }};
+);
+
+#[test]
+fn lex_ident() {
+    let src = v!("hello");
+    assert_eq!(lex(&src).unwrap(), vec![Token::Ident("hello".into())]);
+}
+
+// literals
+#[test]
+fn lex_unsigned_int() {
+    let number = v!("123456");
+    assert_eq!(
+        lex(&number).unwrap(),
+        vec![Token::Lit(Lit::UnsignedInt(123456))]
+    );
+}
+#[test]
+fn lex_signed_int() {
+    let number = v!("-123456");
+    assert_eq!(
+        lex(&number).unwrap(),
+        vec![Token::Lit(Lit::SignedInt(-123456))]
+    );
+}
+#[test]
+fn lex_bool() {
+    let (t, f) = v!("true", "false");
+    assert_eq!(lex(&t).unwrap(), vec![Token::Lit(Lit::Bool(true))]);
+    assert_eq!(lex(&f).unwrap(), vec![Token::Lit(Lit::Bool(false))]);
+}
+#[test]
+fn lex_string() {
+    let s = br#" "hello, world" "#;
+    assert_eq!(
+        lex(s).unwrap(),
+        vec![Token::Lit(Lit::Str("hello, world".into()))]
+    );
+    let s = br#" 'hello, world' "#;
+    assert_eq!(
+        lex(s).unwrap(),
+        vec![Token::Lit(Lit::Str("hello, world".into()))]
+    );
+}
+#[test]
+fn lex_string_test_escape_quote() {
+    let s = br#" "\"hello world\"" "#; // == "hello world"
+    assert_eq!(
+        lex(s).unwrap(),
+        vec![Token::Lit(Lit::Str("\"hello world\"".into()))]
+    );
+    let s = br#" '\'hello world\'' "#; // == 'hello world'
+    assert_eq!(
+        lex(s).unwrap(),
+        vec![Token::Lit(Lit::Str("'hello world'".into()))]
+    );
+}
+#[test]
+fn lex_string_use_different_quote_style() {
+    let s = br#" "he's on it" "#;
+    assert_eq!(
+        lex(s).unwrap(),
+        vec![Token::Lit(Lit::Str("he's on it".into()))]
+    );
+    let s = br#" 'he thinks that "that girl" fixed it' "#;
+    assert_eq!(
+        lex(s).unwrap(),
+        vec![Token::Lit(Lit::Str(
+            "he thinks that \"that girl\" fixed it".into()
+        ))]
+    )
+}
+#[test]
+fn lex_string_escape_bs() {
+    let s = v!(r#" "windows has c:\\" "#);
+    assert_eq!(
+        lex(&s).unwrap(),
+        vec![Token::Lit(Lit::Str("windows has c:\\".into()))]
+    );
+    let s = v!(r#" 'windows has c:\\' "#);
+    assert_eq!(
+        lex(&s).unwrap(),
+        vec![Token::Lit(Lit::Str("windows has c:\\".into()))]
+    );
+    let lol = v!(r#"'\\\\\\\\\\'"#);
+    assert_eq!(
+        lex(&lol).unwrap(),
+        vec![Token::Lit(Lit::Str("\\".repeat(5).into_boxed_str()))],
+        "lol"
+    )
+}
+#[test]
+fn lex_string_bad_escape() {
+    let wth = br#" '\a should be an alert on windows apparently' "#;
+    assert_eq!(lex(wth).unwrap_err(), LangError::InvalidStringLiteral);
+}
+#[test]
+fn lex_string_unclosed() {
+    let wth = br#" 'omg where did the end go "#;
+    assert_eq!(lex(wth).unwrap_err(), LangError::InvalidStringLiteral);
+    let wth = br#" 'see, we escaped the end\' "#;
+    assert_eq!(lex(wth).unwrap_err(), LangError::InvalidStringLiteral);
+}
+#[test]
+fn lex_unsafe_literal_mini() {
+    let usl = lex("\r0\n".as_bytes()).unwrap();
+    assert_eq!(usl.len(), 1);
+    assert_eq!(Token::Lit(Lit::Bin("".into())), usl[0]);
+}
+#[test]
+fn lex_unsafe_literal() {
+    let usl = lex("\r9\nabcdefghi".as_bytes()).unwrap();
+    assert_eq!(usl.len(), 1);
+    assert_eq!(Token::Lit(Lit::Bin("abcdefghi".into())), usl[0]);
+}
+#[test]
+fn lex_unsafe_literal_pro() {
+    let usl = lex("\r18\nabcdefghi123456789".as_bytes()).unwrap();
+    assert_eq!(usl.len(), 1);
+    assert_eq!(Token::Lit(Lit::Bin("abcdefghi123456789".into())), usl[0]);
+}
+
+mod safequery_params {
+    use rand::seq::SliceRandom;
+
+    use crate::engine::ql::lexer::{LitIR, SafeQueryData};
+    #[test]
+    fn param_uint() {
+        let src = b"12345\n";
+        let mut d = Vec::new();
+        let mut i = 0;
+        assert!(SafeQueryData::uint(src, &mut i, &mut d));
+        assert_eq!(i, src.len());
+        assert_eq!(d, vec![LitIR::UInt(12345)]);
+    }
+    #[test]
+    fn param_sint() {
+        let src = b"-12345\n";
+        let mut d = Vec::new();
+        let mut i = 0;
+        assert!(SafeQueryData::sint(src, &mut i, &mut d));
+        assert_eq!(i, src.len());
+        assert_eq!(d, vec![LitIR::SInt(-12345)]);
+    }
+    #[test]
+    fn param_bool_true() {
+        let src = b"true\n";
+        let mut d = Vec::new();
+        let mut i = 0;
+        assert!(SafeQueryData::bool(src, &mut i, &mut d));
+        assert_eq!(i, src.len());
+        assert_eq!(d, vec![LitIR::Bool(true)]);
+    }
+    #[test]
+    fn param_bool_false() {
+        let src = b"false\n";
+        let mut d = Vec::new();
+        let mut i = 0;
+        assert!(SafeQueryData::bool(src, &mut i, &mut d));
+        assert_eq!(i, src.len());
+        assert_eq!(d, vec![LitIR::Bool(false)]);
+    }
+    #[test]
+    fn param_float() {
+        let src = b"4\n3.14";
+        let mut d = Vec::new();
+        let mut i = 0;
+        assert!(SafeQueryData::float(src, &mut i, &mut d));
+        assert_eq!(i, src.len());
+        assert_eq!(d, vec![LitIR::Float(3.14)]);
+    }
+    #[test]
+    fn param_bin() {
+        let src = b"5\nsayan";
+        let mut d = Vec::new();
+        let mut i = 0;
+        assert!(SafeQueryData::bin(src, &mut i, &mut d));
+        assert_eq!(i, src.len());
+        assert_eq!(d, vec![LitIR::Bin(b"sayan")]);
+    }
+    #[test]
+    fn param_str() {
+        let src = b"5\nsayan";
+        let mut d = Vec::new();
+        let mut i = 0;
+        assert!(SafeQueryData::str(src, &mut i, &mut d));
+        assert_eq!(i, src.len());
+        assert_eq!(d, vec![LitIR::Str("sayan")]);
+    }
+    #[test]
+    fn param_full_uint() {
+        let src = b"\x0012345\n";
+        let r = SafeQueryData::p_revloop(src, 1).unwrap();
+        assert_eq!(r.as_ref(), [LitIR::UInt(12345)]);
+    }
+    #[test]
+    fn param_full_sint() {
+        let src = b"\x01-12345\n";
+        let r = SafeQueryData::p_revloop(src, 1).unwrap();
+        assert_eq!(r.as_ref(), [LitIR::SInt(-12345)]);
+    }
+    #[test]
+    fn param_full_bool() {
+        let src = b"\x02true\n";
+        let r = SafeQueryData::p_revloop(src, 1).unwrap();
+        assert_eq!(r.as_ref(), [LitIR::Bool(true)]);
+        let src = b"\x02false\n";
+        let r = SafeQueryData::p_revloop(src, 1).unwrap();
+        assert_eq!(r.as_ref(), [LitIR::Bool(false)]);
+    }
+    #[test]
+    fn param_full_float() {
+        let src = b"\x034\n3.14";
+        let r = SafeQueryData::p_revloop(src, 1).unwrap();
+        assert_eq!(r.as_ref(), [LitIR::Float(3.14)]);
+        let src = b"\x035\n-3.14";
+        let r = SafeQueryData::p_revloop(src, 1).unwrap();
+        assert_eq!(r.as_ref(), [LitIR::Float(-3.14)]);
+    }
+    #[test]
+    fn param_full_bin() {
+        let src = b"\x0412\nhello, world";
+        let r = SafeQueryData::p_revloop(src, 1).unwrap();
+        assert_eq!(r.as_ref(), [LitIR::Bin(b"hello, world")]);
+    }
+    #[test]
+    fn param_full_str() {
+        let src = b"\x0512\nhello, world";
+        let r = SafeQueryData::p_revloop(src, 1).unwrap();
+        assert_eq!(r.as_ref(), [LitIR::Str("hello, world")]);
+    }
+    #[test]
+    fn params_mix() {
+        let mut rng = rand::thread_rng();
+        const DATA: [&'static [u8]; 6] = [
+            b"\x0012345\n",
+            b"\x01-12345\n",
+            b"\x02true\n",
+            b"\x0311\n12345.67890",
+            b"\x0430\none two three four five binary",
+            b"\x0527\none two three four five str",
+        ];
+        const RETMAP: [LitIR; 6] = [
+            LitIR::UInt(12345),
+            LitIR::SInt(-12345),
+            LitIR::Bool(true),
+            LitIR::Float(12345.67890),
+            LitIR::Bin(b"one two three four five binary"),
+            LitIR::Str("one two three four five str"),
+        ];
+        for _ in 0..DATA.len().pow(2) {
+            let mut local_data = DATA;
+            local_data.shuffle(&mut rng);
+            let ret: Vec<LitIR> = local_data.iter().map(|v| RETMAP[v[0] as usize]).collect();
+            let src: Vec<u8> = local_data
+                .into_iter()
+                .map(|v| v.to_owned())
+                .flatten()
+                .collect();
+            let r = SafeQueryData::p_revloop(&src, 6).unwrap();
+            assert_eq!(r.as_ref(), ret);
+        }
+    }
+}
+
+mod safequery_full_param {
+    use crate::engine::ql::lexer::{LitIR, SafeQueryData, Token};
+    #[test]
+    fn p_mini() {
+        let query = b"select * from myapp where username = ?";
+        let params = b"\x055\nsayan";
+        let sq = SafeQueryData::parse(query, params, 1).unwrap();
+        assert_eq!(
+            sq,
+            SafeQueryData::new_test(
+                vec![LitIR::Str("sayan")].into_boxed_slice(),
+                vec![
+                    Token![select],
+                    Token![*],
+                    Token![from],
+                    Token::Ident("myapp".into()),
+                    Token![where],
+                    Token::Ident("username".into()),
+                    Token![=],
+                    Token![?]
+                ]
+            )
+        );
+    }
+    #[test]
+    fn p() {
+        let query = b"select * from myapp where username = ? and pass = ?";
+        let params = b"\x055\nsayan\x048\npass1234";
+        let sq = SafeQueryData::parse(query, params, 1).unwrap();
+        assert_eq!(
+            sq,
+            SafeQueryData::new_test(
+                vec![LitIR::Str("sayan"), LitIR::Str("pass1234")].into_boxed_slice(),
+                vec![
+                    Token![select],
+                    Token![*],
+                    Token![from],
+                    Token::Ident("myapp".into()),
+                    Token![where],
+                    Token::Ident("username".into()),
+                    Token![=],
+                    Token![?],
+                    Token![and],
+                    Token::Ident("pass".into()),
+                    Token![=],
+                    Token![?]
+                ]
+            )
+        );
+    }
+    #[test]
+    fn p_pro() {
+        let query = b"select $notes[~?] from myapp where username = ? and pass = ?";
+        let params = b"\x00100\n\x055\nsayan\x048\npass1234";
+        let sq = SafeQueryData::parse(query, params, 3).unwrap();
+        assert_eq!(
+            sq,
+            SafeQueryData::new_test(
+                vec![
+                    LitIR::UInt(100),
+                    LitIR::Str("sayan"),
+                    LitIR::Bin(b"pass1234")
+                ]
+                .into_boxed_slice(),
+                vec![
+                    Token![select],
+                    Token![$],
+                    Token::Ident("notes".into()),
+                    Token![open []],
+                    Token![~],
+                    Token![?],
+                    Token![close []],
+                    Token![from],
+                    Token::Ident("myapp".into()),
+                    Token![where],
+                    Token::Ident("username".into()),
+                    Token![=],
+                    Token![?],
+                    Token![and],
+                    Token::Ident("pass".into()),
+                    Token![=],
+                    Token![?]
+                ]
+            )
+        );
+    }
+}
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
new file mode 100644
index 00000000..116b65f9
--- /dev/null
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -0,0 +1,1219 @@
+/*
+ * Created on Sun Dec 18 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::{
+    super::{
+        lexer::{Lit, Token},
+        schema,
+    },
+    lex, *,
+};
+mod inspect {
+    use {
+        super::*,
+        crate::engine::ql::{
+            ast::{Entity, Statement},
+            ddl,
+        },
+    };
+    #[test]
+    fn inspect_space() {
+        let tok = lex(b"inspect space myspace").unwrap();
+        assert_eq!(
+            ddl::parse_inspect_full(&tok[1..]).unwrap(),
+            Statement::InspectSpace("myspace".into())
+        );
+    }
+    #[test]
+    fn inspect_model() {
+        let tok = lex(b"inspect model users").unwrap();
+        assert_eq!(
+            ddl::parse_inspect_full(&tok[1..]).unwrap(),
+            Statement::InspectModel(Entity::Single("users".into()))
+        );
+        let tok = lex(b"inspect model tweeter.users").unwrap();
+        assert_eq!(
+            ddl::parse_inspect_full(&tok[1..]).unwrap(),
+            Statement::InspectModel(("tweeter", "users").into())
+        );
+    }
+    #[test]
+    fn inspect_spaces() {
+        let tok = lex(b"inspect spaces").unwrap();
+        assert_eq!(
+            ddl::parse_inspect_full(&tok[1..]).unwrap(),
+            Statement::InspectSpaces
+        );
+    }
+}
+
+mod alter_space {
+    use {
+        super::*,
+        crate::engine::ql::{
+            lexer::Lit,
+            schema::{self, AlterSpace},
+        },
+    };
+    #[test]
+    fn alter_space_mini() {
+        let tok = lex(b"alter model mymodel with {}").unwrap();
+        let r = schema::alter_space_full(&tok[2..]).unwrap();
+        assert_eq!(
+            r,
+            AlterSpace {
+                space_name: "mymodel".into(),
+                updated_props: nullable_dict! {}
+            }
+        );
+    }
+    #[test]
+    fn alter_space() {
+        let tok = lex(br#"
+                alter model mymodel with {
+                    max_entry: 1000,
+                    driver: "ts-0.8"
+                }
+            "#)
+        .unwrap();
+        let r = schema::alter_space_full(&tok[2..]).unwrap();
+        assert_eq!(
+            r,
+            AlterSpace {
+                space_name: "mymodel".into(),
+                updated_props: nullable_dict! {
+                    "max_entry" => Lit::UnsignedInt(1000),
+                    "driver" => Lit::Str("ts-0.8".into())
+                }
+            }
+        );
+    }
+}
+mod tymeta {
+    use super::*;
+    #[test]
+    fn tymeta_mini() {
+        let tok = lex(b"}").unwrap();
+        let (res, ret) = schema::fold_tymeta(&tok);
+        assert!(res.is_okay());
+        assert!(!res.has_more());
+        assert_eq!(res.pos(), 1);
+        assert_eq!(ret, nullable_dict!());
+    }
+    #[test]
+    fn tymeta_mini_fail() {
+        let tok = lex(b",}").unwrap();
+        let (res, ret) = schema::fold_tymeta(&tok);
+        assert!(!res.is_okay());
+        assert!(!res.has_more());
+        assert_eq!(res.pos(), 0);
+        assert_eq!(ret, nullable_dict!());
+    }
+    #[test]
+    fn tymeta() {
+        let tok = lex(br#"hello: "world", loading: true, size: 100 }"#).unwrap();
+        let (res, ret) = schema::fold_tymeta(&tok);
+        assert!(res.is_okay());
+        assert!(!res.has_more());
+        assert_eq!(res.pos(), tok.len());
+        assert_eq!(
+            ret,
+            nullable_dict! {
+                "hello" => Lit::Str("world".into()),
+                "loading" => Lit::Bool(true),
+                "size" => Lit::UnsignedInt(100)
+            }
+        );
+    }
+    #[test]
+    fn tymeta_pro() {
+        // list { maxlen: 100, type string, unique: true }
+        //        ^^^^^^^^^^^^^^^^^^ cursor should be at string
+        let tok = lex(br#"maxlen: 100, type string, unique: true }"#).unwrap();
+        let (res1, ret1) = schema::fold_tymeta(&tok);
+        assert!(res1.is_okay());
+        assert!(res1.has_more());
+        assert_eq!(res1.pos(), 5);
+        let remslice = &tok[res1.pos() + 2..];
+        let (res2, ret2) = schema::fold_tymeta(remslice);
+        assert!(res2.is_okay());
+        assert!(!res2.has_more());
+        assert_eq!(res2.pos() + res1.pos() + 2, tok.len());
+        let mut final_ret = ret1;
+        final_ret.extend(ret2);
+        assert_eq!(
+            final_ret,
+            nullable_dict! {
+                "maxlen" => Lit::UnsignedInt(100),
+                "unique" => Lit::Bool(true)
+            }
+        )
+    }
+    #[test]
+    fn tymeta_pro_max() {
+        // list { maxlen: 100, this: { is: "cool" }, type string, unique: true }
+        //        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cursor should be at string
+        let tok =
+            lex(br#"maxlen: 100, this: { is: "cool" }, type string, unique: true }"#).unwrap();
+        let (res1, ret1) = schema::fold_tymeta(&tok);
+        assert!(res1.is_okay());
+        assert!(res1.has_more());
+        assert_eq!(res1.pos(), 13);
+        let remslice = &tok[res1.pos() + 2..];
+        let (res2, ret2) = schema::fold_tymeta(remslice);
+        assert!(res2.is_okay());
+        assert!(!res2.has_more());
+        assert_eq!(res2.pos() + res1.pos() + 2, tok.len());
+        let mut final_ret = ret1;
+        final_ret.extend(ret2);
+        assert_eq!(
+            final_ret,
+            nullable_dict! {
+                "maxlen" => Lit::UnsignedInt(100),
+                "unique" => Lit::Bool(true),
+                "this" => nullable_dict! {
+                    "is" => Lit::Str("cool".into())
+                }
+            }
+        )
+    }
+    #[test]
+    fn fuzz_tymeta_normal() {
+        // { maxlen: 10, unique: true, users: "sayan" }
+        //   ^start
+        let tok = lex(b"
+                    maxlen: 10,
+                    unique: true,
+                    auth: {
+                        maybe: true\x01
+                    },
+                    users: \"sayan\"\x01
+                }
+            ")
+        .unwrap();
+        let expected = nullable_dict! {
+            "maxlen" => Lit::UnsignedInt(10),
+            "unique" => Lit::Bool(true),
+            "auth" => nullable_dict! {
+                "maybe" => Lit::Bool(true),
+            },
+            "users" => Lit::Str("sayan".into())
+        };
+        fuzz_tokens(&tok, |should_pass, new_src| {
+            let (ret, dict) = schema::fold_tymeta(&new_src);
+            if should_pass {
+                assert!(ret.is_okay(), "{:?}", &new_src);
+                assert!(!ret.has_more());
+                assert_eq!(ret.pos(), new_src.len());
+                assert_eq!(dict, expected);
+            } else if ret.is_okay() {
+                panic!(
+                    "Expected failure but passed for token stream: `{:?}`",
+                    new_src
+                );
+            }
+        });
+    }
+    #[test]
+    fn fuzz_tymeta_with_ty() {
+        // list { maxlen: 10, unique: true, type string, users: "sayan" }
+        //   ^start
+        let tok = lex(b"
+                    maxlen: 10,
+                    unique: true,
+                    auth: {
+                        maybe: true\x01
+                    },
+                    type string,
+                    users: \"sayan\"\x01
+                }
+            ")
+        .unwrap();
+        let expected = nullable_dict! {
+            "maxlen" => Lit::UnsignedInt(10),
+            "unique" => Lit::Bool(true),
+            "auth" => nullable_dict! {
+                "maybe" => Lit::Bool(true),
+            },
+        };
+        fuzz_tokens(&tok, |should_pass, new_src| {
+            let (ret, dict) = schema::fold_tymeta(&new_src);
+            if should_pass {
+                assert!(ret.is_okay());
+                assert!(ret.has_more());
+                assert!(new_src[ret.pos()] == Token::Ident("string".into()));
+                assert_eq!(dict, expected);
+            } else if ret.is_okay() {
+                panic!("Expected failure but passed for token stream: `{:?}`", tok);
+            }
+        });
+    }
+}
+mod layer {
+    use super::*;
+    use crate::engine::ql::schema::Layer;
+    #[test]
+    fn layer_mini() {
+        let tok = lex(b"string)").unwrap();
+        let (layers, c, okay) = schema::fold_layers(&tok);
+        assert_eq!(c, tok.len() - 1);
+        assert!(okay);
+        assert_eq!(
+            layers,
+            vec![Layer::new_noreset("string".into(), nullable_dict! {})]
+        );
+    }
+    #[test]
+    fn layer() {
+        let tok = lex(b"string { maxlen: 100 }").unwrap();
+        let (layers, c, okay) = schema::fold_layers(&tok);
+        assert_eq!(c, tok.len());
+        assert!(okay);
+        assert_eq!(
+            layers,
+            vec![Layer::new_noreset(
+                "string".into(),
+                nullable_dict! {
+                    "maxlen" => Lit::UnsignedInt(100)
+                }
+            )]
+        );
+    }
+    #[test]
+    fn layer_plus() {
+        let tok = lex(b"list { type string }").unwrap();
+        let (layers, c, okay) = schema::fold_layers(&tok);
+        assert_eq!(c, tok.len());
+        assert!(okay);
+        assert_eq!(
+            layers,
+            vec![
+                Layer::new_noreset("string".into(), nullable_dict! {}),
+                Layer::new_noreset("list".into(), nullable_dict! {})
+            ]
+        );
+    }
+    #[test]
+    fn layer_pro() {
+        let tok = lex(b"list { unique: true, type string, maxlen: 10 }").unwrap();
+        let (layers, c, okay) = schema::fold_layers(&tok);
+        assert_eq!(c, tok.len());
+        assert!(okay);
+        assert_eq!(
+            layers,
+            vec![
+                Layer::new_noreset("string".into(), nullable_dict! {}),
+                Layer::new_noreset(
+                    "list".into(),
+                    nullable_dict! {
+                        "unique" => Lit::Bool(true),
+                        "maxlen" => Lit::UnsignedInt(10),
+                    }
+                )
+            ]
+        );
+    }
+    #[test]
+    fn layer_pro_max() {
+        let tok = lex(
+            b"list { unique: true, type string { ascii_only: true, maxlen: 255 }, maxlen: 10 }",
+        )
+        .unwrap();
+        let (layers, c, okay) = schema::fold_layers(&tok);
+        assert_eq!(c, tok.len());
+        assert!(okay);
+        assert_eq!(
+            layers,
+            vec![
+                Layer::new_noreset(
+                    "string".into(),
+                    nullable_dict! {
+                        "ascii_only" => Lit::Bool(true),
+                        "maxlen" => Lit::UnsignedInt(255)
+                    }
+                ),
+                Layer::new_noreset(
+                    "list".into(),
+                    nullable_dict! {
+                        "unique" => Lit::Bool(true),
+                        "maxlen" => Lit::UnsignedInt(10),
+                    }
+                )
+            ]
+        );
+    }
+
+    #[test]
+    fn fuzz_layer() {
+        let tok = lex(b"
+            list {
+                type list {
+                    maxlen: 100,
+                    type string\x01
+                },
+                unique: true\x01
+            }
+        ")
+        .unwrap();
+        let expected = vec![
+            Layer::new_noreset("string".into(), nullable_dict!()),
+            Layer::new_noreset(
+                "list".into(),
+                nullable_dict! {
+                    "maxlen" => Lit::UnsignedInt(100),
+                },
+            ),
+            Layer::new_noreset("list".into(), nullable_dict!("unique" => Lit::Bool(true))),
+        ];
+        fuzz_tokens(&tok, |should_pass, new_tok| {
+            let (layers, c, okay) = schema::fold_layers(&new_tok);
+            if should_pass {
+                assert!(okay);
+                assert_eq!(c, new_tok.len());
+                assert_eq!(layers, expected);
+            } else if okay {
+                panic!(
+                    "expected failure but passed for token stream: `{:?}`",
+                    new_tok
+                );
+            }
+        });
+    }
+}
+mod field_properties {
+    use {super::*, crate::engine::ql::schema::FieldProperties};
+
+    #[test]
+    fn field_properties_empty() {
+        let tok = lex(b"myfield:").unwrap();
+        let (props, c, okay) = schema::parse_field_properties(&tok);
+        assert!(okay);
+        assert_eq!(c, 0);
+        assert_eq!(props, FieldProperties::default());
+    }
+    #[test]
+    fn field_properties_full() {
+        let tok = lex(b"primary null myfield:").unwrap();
+        let (props, c, okay) = schema::parse_field_properties(&tok);
+        assert_eq!(c, 2);
+        assert_eq!(tok[c], Token::Ident("myfield".into()));
+        assert!(okay);
+        assert_eq!(
+            props,
+            FieldProperties {
+                properties: set!["primary", "null"],
+            }
+        )
+    }
+}
+mod fields {
+    use {
+        super::*,
+        crate::engine::ql::schema::{Field, Layer},
+    };
+    #[test]
+    fn field_mini() {
+        let tok = lex(b"
+                username: string,
+            ")
+        .unwrap();
+        let (c, f) = schema::parse_field(&tok).unwrap();
+        assert_eq!(c, tok.len() - 1);
+        assert_eq!(
+            f,
+            Field {
+                field_name: "username".into(),
+                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                props: set![],
+            }
+        )
+    }
+    #[test]
+    fn field() {
+        let tok = lex(b"
+                primary username: string,    
+            ")
+        .unwrap();
+        let (c, f) = schema::parse_field(&tok).unwrap();
+        assert_eq!(c, tok.len() - 1);
+        assert_eq!(
+            f,
+            Field {
+                field_name: "username".into(),
+                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                props: set!["primary"],
+            }
+        )
+    }
+    #[test]
+    fn field_pro() {
+        let tok = lex(b"
+                primary username: string {
+                    maxlen: 10,
+                    ascii_only: true,
+                }
+            ")
+        .unwrap();
+        let (c, f) = schema::parse_field(&tok).unwrap();
+        assert_eq!(c, tok.len());
+        assert_eq!(
+            f,
+            Field {
+                field_name: "username".into(),
+                layers: [Layer::new_noreset(
+                    "string".into(),
+                    nullable_dict! {
+                        "maxlen" => Lit::UnsignedInt(10),
+                        "ascii_only" => Lit::Bool(true),
+                    }
+                )]
+                .into(),
+                props: set!["primary"],
+            }
+        )
+    }
+    #[test]
+    fn field_pro_max() {
+        let tok = lex(b"
+                null notes: list {
+                    type string {
+                        maxlen: 255,
+                        ascii_only: true,
+                    },
+                    unique: true,
+                }
+            ")
+        .unwrap();
+        let (c, f) = schema::parse_field(&tok).unwrap();
+        assert_eq!(c, tok.len());
+        assert_eq!(
+            f,
+            Field {
+                field_name: "notes".into(),
+                layers: [
+                    Layer::new_noreset(
+                        "string".into(),
+                        nullable_dict! {
+                            "maxlen" => Lit::UnsignedInt(255),
+                            "ascii_only" => Lit::Bool(true),
+                        }
+                    ),
+                    Layer::new_noreset(
+                        "list".into(),
+                        nullable_dict! {
+                            "unique" => Lit::Bool(true)
+                        }
+                    ),
+                ]
+                .into(),
+                props: set!["null"],
+            }
+        )
+    }
+}
+mod schemas {
+    use crate::engine::ql::schema::{Field, Layer, Model};
+
+    use super::*;
+    #[test]
+    fn schema_mini() {
+        let tok = lex(b"
+                create model mymodel(
+                    primary username: string,
+                    password: binary,
+                )
+            ")
+        .unwrap();
+        let tok = &tok[2..];
+
+        // parse model
+        let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
+        assert_eq!(c, tok.len());
+        assert_eq!(
+            model,
+            Model {
+                model_name: "mymodel".into(),
+                fields: vec![
+                    Field {
+                        field_name: "username".into(),
+                        layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
+                        props: set!["primary"]
+                    },
+                    Field {
+                        field_name: "password".into(),
+                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        props: set![]
+                    }
+                ],
+                props: nullable_dict! {}
+            }
+        )
+    }
+    #[test]
+    fn schema() {
+        let tok = lex(b"
+                create model mymodel(
+                    primary username: string,
+                    password: binary,
+                    null profile_pic: binary,
+                )
+            ")
+        .unwrap();
+        let tok = &tok[2..];
+
+        // parse model
+        let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
+        assert_eq!(c, tok.len());
+        assert_eq!(
+            model,
+            Model {
+                model_name: "mymodel".into(),
+                fields: vec![
+                    Field {
+                        field_name: "username".into(),
+                        layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
+                        props: set!["primary"]
+                    },
+                    Field {
+                        field_name: "password".into(),
+                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        props: set![]
+                    },
+                    Field {
+                        field_name: "profile_pic".into(),
+                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        props: set!["null"]
+                    }
+                ],
+                props: nullable_dict! {}
+            }
+        )
+    }
+
+    #[test]
+    fn schema_pro() {
+        let tok = lex(b"
+                create model mymodel(
+                    primary username: string,
+                    password: binary,
+                    null profile_pic: binary,
+                    null notes: list {
+                        type string,
+                        unique: true,
+                    },
+                )
+            ")
+        .unwrap();
+        let tok = &tok[2..];
+
+        // parse model
+        let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
+        assert_eq!(c, tok.len());
+        assert_eq!(
+            model,
+            Model {
+                model_name: "mymodel".into(),
+                fields: vec![
+                    Field {
+                        field_name: "username".into(),
+                        layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
+                        props: set!["primary"]
+                    },
+                    Field {
+                        field_name: "password".into(),
+                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        props: set![]
+                    },
+                    Field {
+                        field_name: "profile_pic".into(),
+                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        props: set!["null"]
+                    },
+                    Field {
+                        field_name: "notes".into(),
+                        layers: vec![
+                            Layer::new_noreset("string".into(), nullable_dict! {}),
+                            Layer::new_noreset(
+                                "list".into(),
+                                nullable_dict! {
+                                    "unique" => Lit::Bool(true)
+                                }
+                            )
+                        ],
+                        props: set!["null"]
+                    }
+                ],
+                props: nullable_dict! {}
+            }
+        )
+    }
+
+    #[test]
+    fn schema_pro_max() {
+        let tok = lex(b"
+                create model mymodel(
+                    primary username: string,
+                    password: binary,
+                    null profile_pic: binary,
+                    null notes: list {
+                        type string,
+                        unique: true,
+                    },
+                ) with {
+                    env: {
+                        free_user_limit: 100,
+                    },
+                    storage_driver: \"skyheap\"
+                }
+            ")
+        .unwrap();
+        let tok = &tok[2..];
+
+        // parse model
+        let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
+        assert_eq!(c, tok.len());
+        assert_eq!(
+            model,
+            Model {
+                model_name: "mymodel".into(),
+                fields: vec![
+                    Field {
+                        field_name: "username".into(),
+                        layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
+                        props: set!["primary"]
+                    },
+                    Field {
+                        field_name: "password".into(),
+                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        props: set![]
+                    },
+                    Field {
+                        field_name: "profile_pic".into(),
+                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        props: set!["null"]
+                    },
+                    Field {
+                        field_name: "notes".into(),
+                        layers: vec![
+                            Layer::new_noreset("string".into(), nullable_dict! {}),
+                            Layer::new_noreset(
+                                "list".into(),
+                                nullable_dict! {
+                                    "unique" => Lit::Bool(true)
+                                }
+                            )
+                        ],
+                        props: set!["null"]
+                    }
+                ],
+                props: nullable_dict! {
+                    "env" => nullable_dict! {
+                        "free_user_limit" => Lit::UnsignedInt(100),
+                    },
+                    "storage_driver" => Lit::Str("skyheap".into()),
+                }
+            }
+        )
+    }
+}
+mod dict_field_syntax {
+    use super::*;
+    use crate::engine::ql::schema::{ExpandedField, Layer};
+    #[test]
+    fn field_syn_mini() {
+        let tok = lex(b"username { type string }").unwrap();
+        let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            ef,
+            ExpandedField {
+                field_name: "username".into(),
+                layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
+                props: nullable_dict! {},
+                reset: false
+            }
+        )
+    }
+    #[test]
+    fn field_syn() {
+        let tok = lex(b"
+                username {
+                    nullable: false,
+                    type string,
+                }
+            ")
+        .unwrap();
+        let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            ef,
+            ExpandedField {
+                field_name: "username".into(),
+                props: nullable_dict! {
+                    "nullable" => Lit::Bool(false),
+                },
+                layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
+                reset: false
+            }
+        );
+    }
+    #[test]
+    fn field_syn_pro() {
+        let tok = lex(b"
+                username {
+                    nullable: false,
+                    type string {
+                        minlen: 6,
+                        maxlen: 255,
+                    },
+                    jingle_bells: \"snow\"
+                }
+            ")
+        .unwrap();
+        let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            ef,
+            ExpandedField {
+                field_name: "username".into(),
+                props: nullable_dict! {
+                    "nullable" => Lit::Bool(false),
+                    "jingle_bells" => Lit::Str("snow".into()),
+                },
+                layers: vec![Layer::new_noreset(
+                    "string".into(),
+                    nullable_dict! {
+                        "minlen" => Lit::UnsignedInt(6),
+                        "maxlen" => Lit::UnsignedInt(255),
+                    }
+                )],
+                reset: false
+            }
+        );
+    }
+    #[test]
+    fn field_syn_pro_max() {
+        let tok = lex(b"
+                notes {
+                    nullable: true,
+                    type list {
+                        type string {
+                            ascii_only: true,
+                        },
+                        unique: true,
+                    },
+                    jingle_bells: \"snow\"
+                }
+            ")
+        .unwrap();
+        let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            ef,
+            ExpandedField {
+                field_name: "notes".into(),
+                props: nullable_dict! {
+                    "nullable" => Lit::Bool(true),
+                    "jingle_bells" => Lit::Str("snow".into()),
+                },
+                layers: vec![
+                    Layer::new_noreset(
+                        "string".into(),
+                        nullable_dict! {
+                            "ascii_only" => Lit::Bool(true),
+                        }
+                    ),
+                    Layer::new_noreset(
+                        "list".into(),
+                        nullable_dict! {
+                            "unique" => Lit::Bool(true),
+                        }
+                    )
+                ],
+                reset: false
+            }
+        );
+    }
+}
+mod alter_model_remove {
+    use super::*;
+    use crate::engine::ql::RawSlice;
+    #[test]
+    fn alter_mini() {
+        let tok = lex(b"alter model mymodel remove myfield").unwrap();
+        let mut i = 4;
+        let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(remove, [RawSlice::from("myfield")].into());
+    }
+    #[test]
+    fn alter_mini_2() {
+        let tok = lex(b"alter model mymodel remove (myfield)").unwrap();
+        let mut i = 4;
+        let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(remove, [RawSlice::from("myfield")].into());
+    }
+    #[test]
+    fn alter() {
+        let tok =
+            lex(b"alter model mymodel remove (myfield1, myfield2, myfield3, myfield4)").unwrap();
+        let mut i = 4;
+        let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            remove,
+            [
+                RawSlice::from("myfield1"),
+                RawSlice::from("myfield2"),
+                RawSlice::from("myfield3"),
+                RawSlice::from("myfield4")
+            ]
+            .into()
+        );
+    }
+}
+mod alter_model_add {
+    use super::*;
+    use crate::engine::ql::schema::{ExpandedField, Layer};
+    #[test]
+    fn add_mini() {
+        let tok = lex(b"
+                alter model mymodel add myfield { type string }
+            ")
+        .unwrap();
+        let mut i = 4;
+        let r = schema::alter_add(&tok[i..], &mut i).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            r.as_ref(),
+            [ExpandedField {
+                field_name: "myfield".into(),
+                props: nullable_dict! {},
+                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                reset: false
+            }]
+        );
+    }
+    #[test]
+    fn add() {
+        let tok = lex(b"
+                alter model mymodel add myfield { type string, nullable: true }
+            ")
+        .unwrap();
+        let mut i = 4;
+        let r = schema::alter_add(&tok[i..], &mut i).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            r.as_ref(),
+            [ExpandedField {
+                field_name: "myfield".into(),
+                props: nullable_dict! {
+                    "nullable" => Lit::Bool(true)
+                },
+                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                reset: false
+            }]
+        );
+    }
+    #[test]
+    fn add_pro() {
+        let tok = lex(b"
+                alter model mymodel add (myfield { type string, nullable: true })
+            ")
+        .unwrap();
+        let mut i = 4;
+        let r = schema::alter_add(&tok[i..], &mut i).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            r.as_ref(),
+            [ExpandedField {
+                field_name: "myfield".into(),
+                props: nullable_dict! {
+                    "nullable" => Lit::Bool(true)
+                },
+                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                reset: false
+            }]
+        );
+    }
+    #[test]
+    fn add_pro_max() {
+        let tok = lex(b"
+                alter model mymodel add (
+                    myfield {
+                        type string,
+                        nullable: true
+                    },
+                    another {
+                        type list {
+                            type string {
+                                maxlen: 255
+                            },
+                            unique: true
+                        },
+                        nullable: false,
+                    }
+                )
+            ")
+        .unwrap();
+        let mut i = 4;
+        let r = schema::alter_add(&tok[i..], &mut i).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            r.as_ref(),
+            [
+                ExpandedField {
+                    field_name: "myfield".into(),
+                    props: nullable_dict! {
+                        "nullable" => Lit::Bool(true)
+                    },
+                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                    reset: false
+                },
+                ExpandedField {
+                    field_name: "another".into(),
+                    props: nullable_dict! {
+                        "nullable" => Lit::Bool(false)
+                    },
+                    layers: [
+                        Layer::new_noreset(
+                            "string".into(),
+                            nullable_dict! {
+                                "maxlen" => Lit::UnsignedInt(255)
+                            }
+                        ),
+                        Layer::new_noreset(
+                            "list".into(),
+                            nullable_dict! {
+                               "unique" => Lit::Bool(true)
+                            },
+                        )
+                    ]
+                    .into(),
+                    reset: false
+                }
+            ]
+        );
+    }
+}
+mod alter_model_update {
+    use super::*;
+    use crate::engine::ql::schema::{ExpandedField, Layer};
+
+    #[test]
+    fn alter_mini() {
+        let tok = lex(b"
+                alter model mymodel update myfield { type string, .. }
+            ")
+        .unwrap();
+        let mut i = 4;
+        let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            r.as_ref(),
+            [ExpandedField {
+                field_name: "myfield".into(),
+                props: nullable_dict! {},
+                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                reset: true
+            }]
+        );
+    }
+    #[test]
+    fn alter_mini_2() {
+        let tok = lex(b"
+                alter model mymodel update (myfield { type string, .. })
+            ")
+        .unwrap();
+        let mut i = 4;
+        let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            r.as_ref(),
+            [ExpandedField {
+                field_name: "myfield".into(),
+                props: nullable_dict! {},
+                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                reset: true
+            }]
+        );
+    }
+    #[test]
+    fn alter() {
+        let tok = lex(b"
+                alter model mymodel update (
+                    myfield {
+                        type string,
+                        nullable: true,
+                        ..
+                    }
+                )
+            ")
+        .unwrap();
+        let mut i = 4;
+        let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            r.as_ref(),
+            [ExpandedField {
+                field_name: "myfield".into(),
+                props: nullable_dict! {
+                    "nullable" => Lit::Bool(true)
+                },
+                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                reset: true
+            }]
+        );
+    }
+    #[test]
+    fn alter_pro() {
+        let tok = lex(b"
+                alter model mymodel update (
+                    myfield {
+                        type string,
+                        nullable: true,
+                        ..
+                    },
+                    myfield2 {
+                        type string,
+                        ..
+                    }
+                )
+            ")
+        .unwrap();
+        let mut i = 4;
+        let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            r.as_ref(),
+            [
+                ExpandedField {
+                    field_name: "myfield".into(),
+                    props: nullable_dict! {
+                        "nullable" => Lit::Bool(true)
+                    },
+                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                    reset: true
+                },
+                ExpandedField {
+                    field_name: "myfield2".into(),
+                    props: nullable_dict! {},
+                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                    reset: true
+                }
+            ]
+        );
+    }
+    #[test]
+    fn alter_pro_max() {
+        let tok = lex(b"
+                alter model mymodel update (
+                    myfield {
+                        type string {..},
+                        nullable: true,
+                        ..
+                    },
+                    myfield2 {
+                        type string {
+                            maxlen: 255,
+                            ..
+                        },
+                        ..
+                    }
+                )
+            ")
+        .unwrap();
+        let mut i = 4;
+        let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+        assert_eq!(i, tok.len());
+        assert_eq!(
+            r.as_ref(),
+            [
+                ExpandedField {
+                    field_name: "myfield".into(),
+                    props: nullable_dict! {
+                        "nullable" => Lit::Bool(true)
+                    },
+                    layers: [Layer::new_reset("string".into(), nullable_dict! {})].into(),
+                    reset: true
+                },
+                ExpandedField {
+                    field_name: "myfield2".into(),
+                    props: nullable_dict! {},
+                    layers: [Layer::new_reset(
+                        "string".into(),
+                        nullable_dict! {"maxlen" => Lit::UnsignedInt(255)}
+                    )]
+                    .into(),
+                    reset: true
+                }
+            ]
+        );
+    }
+}
+
+mod ddl_other_query_tests {
+    use {
+        super::*,
+        crate::engine::ql::{
+            ast::{Entity, Statement},
+            ddl::{self, DropModel, DropSpace},
+        },
+    };
+    #[test]
+    fn drop_space() {
+        let src = lex(br"drop space myspace").unwrap();
+        assert_eq!(
+            ddl::parse_drop_full(&src[1..]).unwrap(),
+            Statement::DropSpace(DropSpace::new("myspace".into(), false))
+        );
+    }
+    #[test]
+    fn drop_space_force() {
+        let src = lex(br"drop space myspace force").unwrap();
+        assert_eq!(
+            ddl::parse_drop_full(&src[1..]).unwrap(),
+            Statement::DropSpace(DropSpace::new("myspace".into(), true))
+        );
+    }
+    #[test]
+    fn drop_model() {
+        let src = lex(br"drop model mymodel").unwrap();
+        assert_eq!(
+            ddl::parse_drop_full(&src[1..]).unwrap(),
+            Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), false))
+        );
+    }
+    #[test]
+    fn drop_model_force() {
+        let src = lex(br"drop model mymodel force").unwrap();
+        assert_eq!(
+            ddl::parse_drop_full(&src[1..]).unwrap(),
+            Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), true))
+        );
+    }
+}
diff --git a/server/src/engine/ql/tests/structure_syn.rs b/server/src/engine/ql/tests/structure_syn.rs
new file mode 100644
index 00000000..b2881ea7
--- /dev/null
+++ b/server/src/engine/ql/tests/structure_syn.rs
@@ -0,0 +1,302 @@
+/*
+ * Created on Sun Dec 18 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::*,
+    crate::engine::ql::{lexer::Lit, schema},
+};
+mod dict {
+    use super::*;
+
+    #[test]
+    fn dict_read_mini() {
+        let (d1, d2) = fold_dict! {
+            br#"{name: "sayan"}"#,
+            br#"{name: "sayan",}"#,
+        };
+        let r = nullable_dict!("name" => Lit::Str("sayan".into()));
+        multi_assert_eq!(d1, d2 => r);
+    }
+    #[test]
+    fn dict_read() {
+        let (d1, d2) = fold_dict! {
+            br#"
+                {
+                    name: "sayan",
+                    verified: true,
+                    burgers: 152
+                }
+            "#,
+            br#"
+                {
+                    name: "sayan",
+                    verified: true,
+                    burgers: 152,
+                }
+            "#,
+        };
+        let r = nullable_dict! (
+            "name" => Lit::Str("sayan".into()),
+            "verified" => Lit::Bool(true),
+            "burgers" => Lit::UnsignedInt(152),
+        );
+        multi_assert_eq!(d1, d2 => r);
+    }
+    #[test]
+    fn dict_read_pro() {
+        let (d1, d2, d3) = fold_dict! {
+            br#"
+                {
+                    name: "sayan",
+                    notes: {
+                        burgers: "all the time, extra mayo",
+                        taco: true,
+                        pretzels: 1
+                    }
+                }
+            "#,
+            br#"
+                {
+                    name: "sayan",
+                    notes: {
+                        burgers: "all the time, extra mayo",
+                        taco: true,
+                        pretzels: 1,
+                    }
+                }
+            "#,
+            br#"
+                {
+                    name: "sayan",
+                    notes: {
+                        burgers: "all the time, extra mayo",
+                        taco: true,
+                        pretzels: 1,
+                },
+            }"#
+        };
+        multi_assert_eq!(
+            d1, d2, d3 => nullable_dict! {
+                "name" => Lit::Str("sayan".into()),
+                "notes" => nullable_dict! {
+                    "burgers" => Lit::Str("all the time, extra mayo".into()),
+                    "taco" => Lit::Bool(true),
+                    "pretzels" => Lit::UnsignedInt(1),
+                }
+            }
+        );
+    }
+
+    #[test]
+    fn dict_read_pro_max() {
+        let (d1, d2, d3) = fold_dict! {
+            br#"
+                {
+                    well: {
+                        now: {
+                            this: {
+                                is: {
+                                    ridiculous: true
+                                }
+                            }
+                        }
+                    }
+                }
+            "#,
+            br#"
+                {
+                    well: {
+                        now: {
+                            this: {
+                                is: {
+                                    ridiculous: true,
+                                }
+                            }
+                        }
+                    }
+                }
+            "#,
+            br#"
+                {
+                    well: {
+                        now: {
+                            this: {
+                                is: {
+                                    ridiculous: true,
+                                },
+                            },
+                        },
+                    },
+                }
+            }"#
+        };
+        multi_assert_eq!(
+            d1, d2, d3 => nullable_dict! {
+                "well" => nullable_dict! {
+                    "now" => nullable_dict! {
+                        "this" => nullable_dict! {
+                            "is" => nullable_dict! {
+                                "ridiculous" => Lit::Bool(true),
+                            }
+                        }
+                    }
+                }
+            }
+        );
+    }
+
+    #[test]
+    fn fuzz_dict() {
+        let ret = lex(b"
+                {
+                    the_tradition_is: \"hello, world\",
+                    could_have_been: {
+                        this: true,
+                        or_maybe_this: 100,
+                        even_this: \"hello, universe!\"\x01
+                    },
+                    but_oh_well: \"it continues to be the 'annoying' phrase\",
+                    lorem: {
+                        ipsum: {
+                            dolor: \"sit amet\"\x01
+                        }\x01
+                    }\x01
+                }
+            ")
+        .unwrap();
+        let ret_dict = nullable_dict! {
+            "the_tradition_is" => Lit::Str("hello, world".into()),
+            "could_have_been" => nullable_dict! {
+                "this" => Lit::Bool(true),
+                "or_maybe_this" => Lit::UnsignedInt(100),
+                "even_this" => Lit::Str("hello, universe!".into()),
+            },
+            "but_oh_well" => Lit::Str("it continues to be the 'annoying' phrase".into()),
+            "lorem" => nullable_dict! {
+                "ipsum" => nullable_dict! {
+                    "dolor" => Lit::Str("sit amet".into())
+                }
+            }
+        };
+        fuzz_tokens(&ret, |should_pass, new_src| {
+            let r = schema::fold_dict(&new_src);
+            if should_pass {
+                assert_eq!(r.unwrap(), ret_dict)
+            } else if r.is_some() {
+                panic!(
+                    "expected failure, but passed for token stream: `{:?}`",
+                    new_src
+                );
+            }
+        });
+    }
+}
+mod nullable_dict_tests {
+    use super::*;
+    mod dict {
+        use {super::*, crate::engine::ql::lexer::Lit};
+
+        #[test]
+        fn null_mini() {
+            let d = fold_dict!(br"{ x: null }");
+            assert_eq!(
+                d,
+                nullable_dict! {
+                    "x" => Null,
+                }
+            );
+        }
+        #[test]
+        fn null() {
+            let d = fold_dict! {
+                br#"
+                        {
+                            this_is_non_null: "hello",
+                            but_this_is_null: null,
+                        }
+                    "#
+            };
+            assert_eq!(
+                d,
+                nullable_dict! {
+                    "this_is_non_null" => Lit::Str("hello".into()),
+                    "but_this_is_null" => Null,
+                }
+            )
+        }
+        #[test]
+        fn null_pro() {
+            let d = fold_dict! {
+                br#"
+                        {
+                            a_string: "this is a string",
+                            num: 1234,
+                            a_dict: {
+                                a_null: null,
+                            }
+                        }
+                    "#
+            };
+            assert_eq!(
+                d,
+                nullable_dict! {
+                    "a_string" => Lit::Str("this is a string".into()),
+                    "num" => Lit::UnsignedInt(1234),
+                    "a_dict" => nullable_dict! {
+                        "a_null" => Null,
+                    }
+                }
+            )
+        }
+        #[test]
+        fn null_pro_max() {
+            let d = fold_dict! {
+                br#"
+                        {
+                            a_string: "this is a string",
+                            num: 1234,
+                            a_dict: {
+                                a_null: null,
+                            },
+                            another_null: null,
+                        }
+                    "#
+            };
+            assert_eq!(
+                d,
+                nullable_dict! {
+                    "a_string" => Lit::Str("this is a string".into()),
+                    "num" => Lit::UnsignedInt(1234),
+                    "a_dict" => nullable_dict! {
+                        "a_null" => Null,
+                    },
+                    "another_null" => Null,
+                }
+            )
+        }
+    }
+    // TODO(@ohsayan): Add null tests
+}

From dae3faf87d42f8b40eb62ac07d25fb4e9bb2904c Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 18 Dec 2022 20:35:35 +0530
Subject: [PATCH 056/310] Implement insecure lexer

---
 server/src/engine/ql/benches.rs             |  18 +-
 server/src/engine/ql/lexer.rs               | 512 +++++++-------------
 server/src/engine/ql/tests.rs               |   9 +-
 server/src/engine/ql/tests/dml_tests.rs     |  80 +--
 server/src/engine/ql/tests/entity.rs        |   6 +-
 server/src/engine/ql/tests/lexer_tests.rs   |  46 +-
 server/src/engine/ql/tests/schema_tests.rs  | 100 ++--
 server/src/engine/ql/tests/structure_syn.rs |   2 +-
 8 files changed, 301 insertions(+), 472 deletions(-)

diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index 0b9d5578..226da785 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -37,7 +37,7 @@
 
 extern crate test;
 
-use {crate::engine::ql::tests::lex, test::Bencher};
+use {crate::engine::ql::tests::lex_insecure, test::Bencher};
 
 mod lexer {
     use {
@@ -51,25 +51,25 @@ mod lexer {
     fn lex_number(b: &mut Bencher) {
         let src = b"1234567890";
         let expected = vec![Token::Lit(1234567890.into())];
-        b.iter(|| assert_eq!(lex(src).unwrap(), expected));
+        b.iter(|| assert_eq!(lex_insecure(src).unwrap(), expected));
     }
     #[bench]
     fn lex_bool(b: &mut Bencher) {
         let s = b"true";
         let e = vec![Token::Lit(true.into())];
-        b.iter(|| assert_eq!(lex(s).unwrap(), e));
+        b.iter(|| assert_eq!(lex_insecure(s).unwrap(), e));
     }
     #[bench]
     fn lex_string_noescapes(b: &mut Bencher) {
         let s = br#"'hello, world!'"#;
         let e = vec![Token::Lit("hello, world!".into())];
-        b.iter(|| assert_eq!(lex(s).unwrap(), e));
+        b.iter(|| assert_eq!(lex_insecure(s).unwrap(), e));
     }
     #[bench]
     fn lex_string_with_escapes(b: &mut Bencher) {
         let s = br#"'hello, world! this is within a \'quote\''"#;
         let e = vec![Token::Lit("hello, world! this is within a 'quote'".into())];
-        b.iter(|| assert_eq!(lex(s).unwrap(), e));
+        b.iter(|| assert_eq!(lex_insecure(s).unwrap(), e));
     }
     #[bench]
     fn lex_raw_literal(b: &mut Bencher) {
@@ -77,7 +77,7 @@ mod lexer {
         let expected = vec![Token::Lit(Lit::Bin(RawSlice::from(
             "e69b10ffcc250ae5091dec6f299072e23b0b41d6a739",
         )))];
-        b.iter(|| assert_eq!(lex(src).unwrap(), expected));
+        b.iter(|| assert_eq!(lex_insecure(src).unwrap(), expected));
     }
 }
 
@@ -87,7 +87,7 @@ mod ast {
     fn parse_entity_single(b: &mut Bencher) {
         let e = Entity::Single("tweeter".into());
         b.iter(|| {
-            let src = lex(b"tweeter").unwrap();
+            let src = lex_insecure(b"tweeter").unwrap();
             let mut i = 0;
             assert_eq!(Entity::parse_from_tokens(&src, &mut i).unwrap(), e);
             assert_eq!(i, src.len());
@@ -97,7 +97,7 @@ mod ast {
     fn parse_entity_double(b: &mut Bencher) {
         let e = ("tweeter", "user").into();
         b.iter(|| {
-            let src = lex(b"tweeter.user").unwrap();
+            let src = lex_insecure(b"tweeter.user").unwrap();
             let mut i = 0;
             assert_eq!(Entity::parse_from_tokens(&src, &mut i).unwrap(), e);
             assert_eq!(i, src.len());
@@ -107,7 +107,7 @@ mod ast {
     fn parse_entity_partial(b: &mut Bencher) {
         let e = Entity::Partial("user".into());
         b.iter(|| {
-            let src = lex(b":user").unwrap();
+            let src = lex_insecure(b":user").unwrap();
             let mut i = 0;
             assert_eq!(Entity::parse_from_tokens(&src, &mut i).unwrap(), e);
             assert_eq!(i, src.len());
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 0d7549d6..03690e60 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -28,7 +28,7 @@ use std::ops::BitOr;
 
 use {
     super::{LangError, LangResult, RawSlice},
-    crate::util::{compiler, Life},
+    crate::util::compiler,
     core::{cmp, fmt, marker::PhantomData, mem::size_of, slice, str},
 };
 
@@ -375,26 +375,39 @@ fn kwof(key: &[u8]) -> Option<Keyword> {
     }
 }
 
+impl Token {
+    #[inline(always)]
+    pub(crate) const fn is_ident(&self) -> bool {
+        matches!(self, Token::Ident(_))
+    }
+    #[inline(always)]
+    pub(super) const fn is_lit(&self) -> bool {
+        matches!(self, Self::Lit(_))
+    }
+}
+
+impl AsRef<Token> for Token {
+    #[inline(always)]
+    fn as_ref(&self) -> &Token {
+        self
+    }
+}
+
 /*
     Lexer impl
 */
 
-pub(super) const LANG_MODE_INSECURE: u8 = 0;
-pub(super) const LANG_MODE_SECURE: u8 = 1;
-
-pub type OperatingMode = u8;
-pub type InsecureLexer<'a> = Lexer<'a, LANG_MODE_INSECURE>;
-pub type SecureLexer<'a> = Lexer<'a, LANG_MODE_SECURE>;
-
-pub struct Lexer<'a, const OPERATING_MODE: u8> {
+#[derive(Debug)]
+pub struct RawLexer<'a> {
     c: *const u8,
     e: *const u8,
-    last_error: Option<LangError>,
     tokens: Vec<Token>,
+    last_error: Option<LangError>,
     _lt: PhantomData<&'a [u8]>,
 }
 
-impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
+// ctor
+impl<'a> RawLexer<'a> {
     #[inline(always)]
     pub const fn new(src: &'a [u8]) -> Self {
         Self {
@@ -411,7 +424,7 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
 }
 
 // meta
-impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
+impl<'a> RawLexer<'a> {
     #[inline(always)]
     const fn cursor(&self) -> *const u8 {
         self.c
@@ -499,7 +512,8 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
     }
 }
 
-impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
+// high level methods
+impl<'a> RawLexer<'a> {
     #[inline(always)]
     fn scan_ident(&mut self) -> RawSlice {
         let s = self.cursor();
@@ -522,72 +536,6 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
         }
     }
     #[inline(always)]
-    fn scan_unsigned_integer(&mut self) {
-        let s = self.cursor();
-        unsafe {
-            while self.peek_is(|b| b.is_ascii_digit()) {
-                self.incr_cursor();
-            }
-            /*
-                1234; // valid
-                1234} // valid
-                1234{ // invalid
-                1234, // valid
-                1234a // invalid
-            */
-            let wseof = self.peek_is(|char| !char.is_ascii_alphabetic()) || self.exhausted();
-            match str::from_utf8_unchecked(slice::from_raw_parts(
-                s,
-                self.cursor().offset_from(s) as usize,
-            ))
-            .parse()
-            {
-                Ok(num) if compiler::likely(wseof) => {
-                    self.tokens.push(Token::Lit(Lit::UnsignedInt(num)))
-                }
-                _ => self.set_error(LangError::InvalidNumericLiteral),
-            }
-        }
-    }
-    #[inline(always)]
-    fn scan_quoted_string(&mut self, quote_style: u8) {
-        debug_assert!(
-            unsafe { self.deref_cursor() } == quote_style,
-            "illegal call to scan_quoted_string"
-        );
-        unsafe { self.incr_cursor() }
-        let mut buf = Vec::new();
-        unsafe {
-            while self.peek_neq(quote_style) {
-                match self.deref_cursor() {
-                    b if b != b'\\' => {
-                        buf.push(b);
-                    }
-                    _ => {
-                        self.incr_cursor();
-                        if self.exhausted() {
-                            break;
-                        }
-                        let b = self.deref_cursor();
-                        let quote = b == quote_style;
-                        let bs = b == b'\\';
-                        if quote | bs {
-                            buf.push(b);
-                        } else {
-                            break; // what on good earth is that escape?
-                        }
-                    }
-                }
-                self.incr_cursor();
-            }
-            let terminated = self.peek_eq_and_forward(quote_style);
-            match String::from_utf8(buf) {
-                Ok(st) if terminated => self.tokens.push(Token::Lit(st.into_boxed_str().into())),
-                _ => self.set_error(LangError::InvalidStringLiteral),
-            }
-        }
-    }
-    #[inline(always)]
     fn scan_byte(&mut self, byte: u8) {
         match symof(byte) {
             Some(tok) => self.push_token(tok),
@@ -597,278 +545,195 @@ impl<'a, const OPERATING_MODE: u8> Lexer<'a, OPERATING_MODE> {
             self.incr_cursor();
         }
     }
+}
+
+#[derive(Debug)]
+/// This implements the `opmode-dev` for BlueQL
+pub struct InsecureLexer<'a> {
+    base: RawLexer<'a>,
+}
+
+impl<'a> InsecureLexer<'a> {
     #[inline(always)]
-    fn scan_binary_literal(&mut self) {
-        unsafe {
-            self.incr_cursor();
+    pub const fn new(src: &'a [u8]) -> Self {
+        Self {
+            base: RawLexer::new(src),
         }
-        let mut size = 0usize;
-        let mut okay = true;
-        while self.not_exhausted() && unsafe { self.deref_cursor() != b'\n' } && okay {
-            /*
-                Don't ask me how stupid this is. Like, I was probably in some "mood" when I wrote this
-                and it works duh, but isn't the most elegant of things (could I have just used a parse?
-                nah, I'm just a hardcore numeric normie)
-                -- Sayan
-            */
-            let byte = unsafe { self.deref_cursor() };
-            okay &= byte.is_ascii_digit();
-            let (prod, of_flag) = size.overflowing_mul(10);
-            okay &= !of_flag;
-            let (sum, of_flag) = prod.overflowing_add((byte & 0x0F) as _);
-            size = sum;
-            okay &= !of_flag;
-            unsafe {
-                self.incr_cursor();
-            }
+    }
+    #[inline(always)]
+    pub fn lex(src: &'a [u8]) -> LangResult<Vec<Token>> {
+        let mut slf = Self::new(src);
+        slf._lex();
+        let RawLexer {
+            tokens, last_error, ..
+        } = slf.base;
+        match last_error {
+            None => Ok(tokens),
+            Some(e) => Err(e),
         }
-        okay &= self.peek_eq_and_forward(b'\n');
-        okay &= self.remaining() >= size;
-        if compiler::likely(okay) {
-            unsafe {
-                self.push_token(Lit::Bin(RawSlice::new(self.cursor(), size)));
-                self.incr_cursor_by(size);
+    }
+    #[inline(always)]
+    fn _lex(&mut self) {
+        let ref mut slf = self.base;
+        while slf.not_exhausted() && slf.no_error() {
+            match unsafe { slf.deref_cursor() } {
+                byte if byte.is_ascii_alphabetic() => slf.scan_ident_or_keyword(),
+                #[cfg(test)]
+                byte if byte == b'\x01' => {
+                    slf.push_token(Token::IgnorableComma);
+                    unsafe {
+                        // UNSAFE(@ohsayan): All good here. Already read the token
+                        slf.incr_cursor();
+                    }
+                }
+                byte if byte.is_ascii_digit() => Self::scan_unsigned_integer(slf),
+                b'\r' => Self::scan_binary_literal(slf),
+                b'-' => Self::scan_signed_integer(slf),
+                qs @ (b'\'' | b'"') => Self::scan_quoted_string(slf, qs),
+                // blank space or an arbitrary byte
+                b' ' | b'\n' | b'\t' => slf.trim_ahead(),
+                b => slf.scan_byte(b),
             }
-        } else {
-            self.set_error(LangError::InvalidSafeLiteral);
         }
     }
+}
+
+// high-level methods
+impl<'a> InsecureLexer<'a> {
     #[inline(always)]
-    fn scan_signed_integer(&mut self) {
+    fn scan_signed_integer(slf: &mut RawLexer<'a>) {
         unsafe {
-            self.incr_cursor();
+            slf.incr_cursor();
         }
-        if self.peek_is(|b| b.is_ascii_digit()) {
+        if slf.peek_is(|b| b.is_ascii_digit()) {
             // we have some digits
             let start = unsafe {
                 // UNSAFE(@ohsayan): Take the (-) into the parse
                 // TODO(@ohsayan): we can maybe look at a more efficient way later
-                self.cursor().sub(1)
+                slf.cursor().sub(1)
             };
-            while self.peek_is_and_forward(|b| b.is_ascii_digit()) {}
-            let wseof = self.peek_is(|char| !char.is_ascii_alphabetic()) || self.exhausted();
+            while slf.peek_is_and_forward(|b| b.is_ascii_digit()) {}
+            let wseof = slf.peek_is(|char| !char.is_ascii_alphabetic()) || slf.exhausted();
             match unsafe {
                 str::from_utf8_unchecked(slice::from_raw_parts(
                     start,
-                    self.cursor().offset_from(start) as usize,
+                    slf.cursor().offset_from(start) as usize,
                 ))
             }
             .parse::<i64>()
             {
                 Ok(num) if compiler::likely(wseof) => {
-                    self.push_token(Lit::SignedInt(num));
+                    slf.push_token(Lit::SignedInt(num));
                 }
                 _ => {
-                    compiler::cold_val(self.set_error(LangError::InvalidNumericLiteral));
+                    compiler::cold_val(slf.set_error(LangError::InvalidNumericLiteral));
                 }
             }
         } else {
-            self.push_token(Token![-]);
+            slf.push_token(Token![-]);
         }
     }
     #[inline(always)]
-    fn _lex(&mut self) {
-        while self.not_exhausted() && self.no_error() {
-            match unsafe { self.deref_cursor() } {
-                // secure features
-                byte if byte.is_ascii_alphabetic() => self.scan_ident_or_keyword(),
-                #[cfg(test)]
-                byte if byte == b'\x01' => {
-                    self.push_token(Token::IgnorableComma);
-                    unsafe {
-                        // UNSAFE(@ohsayan): All good here. Already read the token
-                        self.incr_cursor();
-                    }
-                }
-                // insecure features
-                byte if byte.is_ascii_digit() && OPERATING_MODE == LANG_MODE_INSECURE => {
-                    self.scan_unsigned_integer()
-                }
-                b'-' if OPERATING_MODE == LANG_MODE_INSECURE => self.scan_signed_integer(),
-                qs @ (b'\'' | b'"') if OPERATING_MODE == LANG_MODE_INSECURE => {
-                    self.scan_quoted_string(qs)
+    fn scan_unsigned_integer(slf: &mut RawLexer<'a>) {
+        let s = slf.cursor();
+        unsafe {
+            while slf.peek_is(|b| b.is_ascii_digit()) {
+                slf.incr_cursor();
+            }
+            /*
+                1234; // valid
+                1234} // valid
+                1234{ // invalid
+                1234, // valid
+                1234a // invalid
+            */
+            let wseof = slf.peek_is(|char| !char.is_ascii_alphabetic()) || slf.exhausted();
+            match str::from_utf8_unchecked(slice::from_raw_parts(
+                s,
+                slf.cursor().offset_from(s) as usize,
+            ))
+            .parse()
+            {
+                Ok(num) if compiler::likely(wseof) => {
+                    slf.tokens.push(Token::Lit(Lit::UnsignedInt(num)))
                 }
-                // blank space or an arbitrary byte
-                b' ' | b'\n' | b'\t' => self.trim_ahead(),
-                b => self.scan_byte(b),
+                _ => slf.set_error(LangError::InvalidNumericLiteral),
             }
         }
     }
-    #[inline(always)]
-    pub fn lex(src: &'a [u8]) -> LangResult<Life<'a, Vec<Token>>> {
-        let mut slf = Self::new(src);
-        slf._lex();
-        match slf.last_error {
-            None => Ok(Life::new(slf.tokens)),
-            Some(e) => Err(e),
-        }
-    }
-}
 
-impl Token {
-    #[inline(always)]
-    pub(crate) const fn is_ident(&self) -> bool {
-        matches!(self, Token::Ident(_))
-    }
     #[inline(always)]
-    pub(super) const fn is_lit(&self) -> bool {
-        matches!(self, Self::Lit(_))
-    }
-}
-
-impl AsRef<Token> for Token {
-    #[inline(always)]
-    fn as_ref(&self) -> &Token {
-        self
-    }
-}
-
-#[derive(Debug)]
-pub struct RawLexer<'a> {
-    c: *const u8,
-    e: *const u8,
-    tokens: Vec<Token>,
-    last_error: Option<LangError>,
-    _lt: PhantomData<&'a [u8]>,
-}
-
-// ctor
-impl<'a> RawLexer<'a> {
-    #[inline(always)]
-    pub const fn new(src: &'a [u8]) -> Self {
-        Self {
-            c: src.as_ptr(),
-            e: unsafe {
-                // UNSAFE(@ohsayan): Always safe (<= EOA)
-                src.as_ptr().add(src.len())
-            },
-            last_error: None,
-            tokens: Vec::new(),
-            _lt: PhantomData,
-        }
-    }
-}
-
-// meta
-impl<'a> RawLexer<'a> {
-    #[inline(always)]
-    const fn cursor(&self) -> *const u8 {
-        self.c
-    }
-    #[inline(always)]
-    const fn data_end_ptr(&self) -> *const u8 {
-        self.e
-    }
-    #[inline(always)]
-    fn not_exhausted(&self) -> bool {
-        self.data_end_ptr() > self.cursor()
-    }
-    #[inline(always)]
-    fn exhausted(&self) -> bool {
-        self.cursor() == self.data_end_ptr()
-    }
-    #[inline(always)]
-    fn remaining(&self) -> usize {
-        unsafe { self.e.offset_from(self.c) as usize }
-    }
-    #[inline(always)]
-    unsafe fn deref_cursor(&self) -> u8 {
-        *self.cursor()
-    }
-    #[inline(always)]
-    unsafe fn incr_cursor_by(&mut self, by: usize) {
-        debug_assert!(self.remaining() >= by);
-        self.c = self.cursor().add(by)
-    }
-    #[inline(always)]
-    unsafe fn incr_cursor(&mut self) {
-        self.incr_cursor_by(1)
-    }
-    #[inline(always)]
-    unsafe fn incr_cursor_if(&mut self, iff: bool) {
-        self.incr_cursor_by(iff as usize)
-    }
-    #[inline(always)]
-    fn push_token(&mut self, token: impl Into<Token>) {
-        self.tokens.push(token.into())
-    }
-    #[inline(always)]
-    fn peek_is(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
-        self.not_exhausted() && unsafe { f(self.deref_cursor()) }
-    }
-    #[inline(always)]
-    fn peek_is_and_forward(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
-        let did_fw = self.not_exhausted() && unsafe { f(self.deref_cursor()) };
+    fn scan_binary_literal(slf: &mut RawLexer<'a>) {
         unsafe {
-            self.incr_cursor_if(did_fw);
+            slf.incr_cursor();
         }
-        did_fw
-    }
-    #[inline(always)]
-    fn peek_eq_and_forward_or_eof(&mut self, eq: u8) -> bool {
-        unsafe {
-            let eq = self.not_exhausted() && self.deref_cursor() == eq;
-            self.incr_cursor_if(eq);
-            eq | self.exhausted()
-        }
-    }
-    #[inline(always)]
-    fn peek_neq(&self, b: u8) -> bool {
-        self.not_exhausted() && unsafe { self.deref_cursor() != b }
-    }
-    #[inline(always)]
-    fn peek_eq_and_forward(&mut self, b: u8) -> bool {
-        unsafe {
-            let r = self.not_exhausted() && self.deref_cursor() == b;
-            self.incr_cursor_if(r);
-            r
-        }
-    }
-    #[inline(always)]
-    fn trim_ahead(&mut self) {
-        while self.peek_is_and_forward(|b| b == b' ' || b == b'\t' || b == b'\n') {}
-    }
-    #[inline(always)]
-    fn set_error(&mut self, e: LangError) {
-        self.last_error = Some(e);
-    }
-    #[inline(always)]
-    fn no_error(&self) -> bool {
-        self.last_error.is_none()
-    }
-}
-
-// high level methods
-impl<'a> RawLexer<'a> {
-    #[inline(always)]
-    fn scan_ident(&mut self) -> RawSlice {
-        let s = self.cursor();
-        unsafe {
-            while self.peek_is(|b| b.is_ascii_alphanumeric() || b == b'_') {
-                self.incr_cursor();
+        let mut size = 0usize;
+        let mut okay = true;
+        while slf.not_exhausted() && unsafe { slf.deref_cursor() != b'\n' } && okay {
+            /*
+                Don't ask me how stupid this is. Like, I was probably in some "mood" when I wrote this
+                and it works duh, but isn't the most elegant of things (could I have just used a parse?
+                nah, I'm just a hardcore numeric normie)
+                -- Sayan
+            */
+            let byte = unsafe { slf.deref_cursor() };
+            okay &= byte.is_ascii_digit();
+            let (prod, of_flag) = size.overflowing_mul(10);
+            okay &= !of_flag;
+            let (sum, of_flag) = prod.overflowing_add((byte & 0x0F) as _);
+            size = sum;
+            okay &= !of_flag;
+            unsafe {
+                slf.incr_cursor();
             }
-            RawSlice::new(s, self.cursor().offset_from(s) as usize)
         }
-    }
-    #[inline(always)]
-    fn scan_ident_or_keyword(&mut self) {
-        let s = self.scan_ident();
-        let st = unsafe { s.as_slice() }.to_ascii_lowercase();
-        match kwof(&st) {
-            Some(kw) => self.tokens.push(kw.into()),
-            // FIXME(@ohsayan): Uh, mind fixing this? The only advantage is that I can keep the graph *memory* footprint small
-            None if st == b"true" || st == b"false" => self.push_token(Lit::Bool(st == b"true")),
-            None => self.tokens.push(Token::Ident(s)),
+        okay &= slf.peek_eq_and_forward(b'\n');
+        okay &= slf.remaining() >= size;
+        if compiler::likely(okay) {
+            unsafe {
+                slf.push_token(Lit::Bin(RawSlice::new(slf.cursor(), size)));
+                slf.incr_cursor_by(size);
+            }
+        } else {
+            slf.set_error(LangError::InvalidSafeLiteral);
         }
     }
     #[inline(always)]
-    fn scan_byte(&mut self, byte: u8) {
-        match symof(byte) {
-            Some(tok) => self.push_token(tok),
-            None => return self.set_error(LangError::UnexpectedChar),
-        }
+    fn scan_quoted_string(slf: &mut RawLexer<'a>, quote_style: u8) {
+        debug_assert!(
+            unsafe { slf.deref_cursor() } == quote_style,
+            "illegal call to scan_quoted_string"
+        );
+        unsafe { slf.incr_cursor() }
+        let mut buf = Vec::new();
         unsafe {
-            self.incr_cursor();
+            while slf.peek_neq(quote_style) {
+                match slf.deref_cursor() {
+                    b if b != b'\\' => {
+                        buf.push(b);
+                    }
+                    _ => {
+                        slf.incr_cursor();
+                        if slf.exhausted() {
+                            break;
+                        }
+                        let b = slf.deref_cursor();
+                        let quote = b == quote_style;
+                        let bs = b == b'\\';
+                        if quote | bs {
+                            buf.push(b);
+                        } else {
+                            break; // what on good earth is that escape?
+                        }
+                    }
+                }
+                slf.incr_cursor();
+            }
+            let terminated = slf.peek_eq_and_forward(quote_style);
+            match String::from_utf8(buf) {
+                Ok(st) if terminated => slf.tokens.push(Token::Lit(st.into_boxed_str().into())),
+                _ => slf.set_error(LangError::InvalidStringLiteral),
+            }
         }
     }
 }
@@ -1026,39 +891,6 @@ where
     number
 }
 
-#[inline(always)]
-fn decode_num_from_bounded_payload<N>(src: &[u8], flag: &mut bool) -> N
-where
-    N: NumberDefinition,
-{
-    let l = src.len();
-
-    let mut i = 0;
-    let mut number = N::zero();
-    let mut okay = l <= N::qualified_max_length();
-
-    if N::ALLOW_SIGNED {
-        let is_signed = i < l && src[i] == b'-';
-        if is_signed {
-            number.negate();
-        }
-        i += is_signed as usize;
-    }
-
-    while i < l && okay {
-        okay &= src[i].is_ascii_digit();
-        let (product, p_of) = number.overflowing_mul(10);
-        okay &= !p_of;
-        let (sum, s_of) = product.overflowing_add(src[i] & 0x0F);
-        okay &= !s_of;
-        number = sum;
-        i += 1;
-    }
-
-    *flag &= okay;
-    number
-}
-
 #[derive(PartialEq, Debug, Clone, Copy)]
 /// Intermediate literal repr
 pub enum LitIR<'a> {
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index f124bf6a..f3cac871 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -29,16 +29,13 @@ use {
         lexer::{InsecureLexer, Symbol, Token},
         LangResult,
     },
-    crate::{
-        engine::memory::DataType,
-        util::{test_utils, Life},
-    },
+    crate::{engine::memory::DataType, util::test_utils},
     rand::{self, Rng},
 };
 
 macro_rules! fold_dict {
     ($($input:expr),* $(,)?) => {
-        ($({$crate::engine::ql::schema::fold_dict(&super::lex($input).unwrap()).unwrap()}),*)
+        ($({$crate::engine::ql::schema::fold_dict(&super::lex_insecure($input).unwrap()).unwrap()}),*)
     }
 }
 
@@ -48,7 +45,7 @@ mod lexer_tests;
 mod schema_tests;
 mod structure_syn;
 
-pub(super) fn lex(src: &[u8]) -> LangResult<Life<Vec<Token>>> {
+pub(super) fn lex_insecure(src: &[u8]) -> LangResult<Vec<Token>> {
     InsecureLexer::lex(src)
 }
 
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 2f2b97be..44f3c457 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -31,7 +31,7 @@ mod list_parse {
 
     #[test]
     fn list_mini() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 []
             ")
         .unwrap();
@@ -41,7 +41,7 @@ mod list_parse {
 
     #[test]
     fn list() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 [1, 2, 3, 4]
             ")
         .unwrap();
@@ -51,7 +51,7 @@ mod list_parse {
 
     #[test]
     fn list_pro() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 [
                     [1, 2],
                     [3, 4],
@@ -74,7 +74,7 @@ mod list_parse {
 
     #[test]
     fn list_pro_max() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 [
                     [[1, 1], [2, 2]],
                     [[], [4, 4]],
@@ -101,14 +101,14 @@ mod tuple_syntax {
 
     #[test]
     fn tuple_mini() {
-        let tok = lex(b"()").unwrap();
+        let tok = lex_insecure(b"()").unwrap();
         let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
         assert_eq!(r, vec![]);
     }
 
     #[test]
     fn tuple() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 (1234, "email@example.com", true)
             "#)
         .unwrap();
@@ -121,7 +121,7 @@ mod tuple_syntax {
 
     #[test]
     fn tuple_pro() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 (
                     1234,
                     "email@example.com",
@@ -144,7 +144,7 @@ mod tuple_syntax {
 
     #[test]
     fn tuple_pro_max() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 (
                     1234,
                     "email@example.com",
@@ -185,14 +185,14 @@ mod map_syntax {
 
     #[test]
     fn map_mini() {
-        let tok = lex(b"{}").unwrap();
+        let tok = lex_insecure(b"{}").unwrap();
         let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
         assert_eq!(r, nullable_dict! {})
     }
 
     #[test]
     fn map() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 {
                     name: "John Appletree",
                     email: "john@example.com",
@@ -215,7 +215,7 @@ mod map_syntax {
 
     #[test]
     fn map_pro() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 {
                     name: "John Appletree",
                     email: "john@example.com",
@@ -240,7 +240,7 @@ mod map_syntax {
 
     #[test]
     fn map_pro_max() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 {
                     name: "John Appletree",
                     email: "john@example.com",
@@ -286,7 +286,7 @@ mod stmt_insert {
 
     #[test]
     fn insert_tuple_mini() {
-        let x = lex(br#"
+        let x = lex_insecure(br#"
                 insert into twitter.users ("sayan")
             "#)
         .unwrap();
@@ -299,7 +299,7 @@ mod stmt_insert {
     }
     #[test]
     fn insert_tuple() {
-        let x = lex(br#"
+        let x = lex_insecure(br#"
                 insert into twitter.users (
                     "sayan",
                     "Sayan",
@@ -328,7 +328,7 @@ mod stmt_insert {
     }
     #[test]
     fn insert_tuple_pro() {
-        let x = lex(br#"
+        let x = lex_insecure(br#"
                 insert into twitter.users (
                     "sayan",
                     "Sayan",
@@ -363,7 +363,7 @@ mod stmt_insert {
     }
     #[test]
     fn insert_map_mini() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 insert into jotsy.app { username: "sayan" }
             "#)
         .unwrap();
@@ -379,7 +379,7 @@ mod stmt_insert {
     }
     #[test]
     fn insert_map() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 insert into jotsy.app {
                     username: "sayan",
                     name: "Sayan",
@@ -407,7 +407,7 @@ mod stmt_insert {
     }
     #[test]
     fn insert_map_pro() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 insert into jotsy.app {
                     username: "sayan",
                     password: "pass123",
@@ -453,7 +453,7 @@ mod stmt_select {
     };
     #[test]
     fn select_mini() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 select * from users where username = "sayan"
             "#)
         .unwrap();
@@ -473,7 +473,7 @@ mod stmt_select {
     }
     #[test]
     fn select() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 select field1 from users where username = "sayan"
             "#)
         .unwrap();
@@ -493,7 +493,7 @@ mod stmt_select {
     }
     #[test]
     fn select_pro() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 select field1 from twitter.users where username = "sayan"
             "#)
         .unwrap();
@@ -513,7 +513,7 @@ mod stmt_select {
     }
     #[test]
     fn select_pro_max() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 select field1, field2 from twitter.users where username = "sayan"
             "#)
         .unwrap();
@@ -542,7 +542,7 @@ mod expression_tests {
     };
     #[test]
     fn expr_assign() {
-        let src = lex(b"username = 'sayan'").unwrap();
+        let src = lex_insecure(b"username = 'sayan'").unwrap();
         let r = dml::parse_expression_full(&src).unwrap();
         assert_eq!(
             r,
@@ -555,7 +555,7 @@ mod expression_tests {
     }
     #[test]
     fn expr_add_assign() {
-        let src = lex(b"followers += 100").unwrap();
+        let src = lex_insecure(b"followers += 100").unwrap();
         let r = dml::parse_expression_full(&src).unwrap();
         assert_eq!(
             r,
@@ -568,7 +568,7 @@ mod expression_tests {
     }
     #[test]
     fn expr_sub_assign() {
-        let src = lex(b"following -= 150").unwrap();
+        let src = lex_insecure(b"following -= 150").unwrap();
         let r = dml::parse_expression_full(&src).unwrap();
         assert_eq!(
             r,
@@ -581,7 +581,7 @@ mod expression_tests {
     }
     #[test]
     fn expr_mul_assign() {
-        let src = lex(b"product_qty *= 2").unwrap();
+        let src = lex_insecure(b"product_qty *= 2").unwrap();
         let r = dml::parse_expression_full(&src).unwrap();
         assert_eq!(
             r,
@@ -594,7 +594,7 @@ mod expression_tests {
     }
     #[test]
     fn expr_div_assign() {
-        let src = lex(b"image_crop_factor /= 2").unwrap();
+        let src = lex_insecure(b"image_crop_factor /= 2").unwrap();
         let r = dml::parse_expression_full(&src).unwrap();
         assert_eq!(
             r,
@@ -619,7 +619,7 @@ mod update_statement {
     };
     #[test]
     fn update_mini() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 update app SET notes += "this is my new note" where username = "sayan"
             "#)
         .unwrap();
@@ -645,7 +645,7 @@ mod update_statement {
     }
     #[test]
     fn update() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 update
                     jotsy.app
                 SET
@@ -688,7 +688,7 @@ mod delete_stmt {
 
     #[test]
     fn delete_mini() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 delete from users where username = "sayan"
             "#)
         .unwrap();
@@ -708,7 +708,7 @@ mod delete_stmt {
     }
     #[test]
     fn delete() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 delete from twitter.users where username = "sayan"
             "#)
         .unwrap();
@@ -735,7 +735,7 @@ mod relational_expr {
 
     #[test]
     fn expr_eq() {
-        let expr = lex(b"primary_key = 10").unwrap();
+        let expr = lex_insecure(b"primary_key = 10").unwrap();
         let r = dml::parse_relexpr_full(&expr).unwrap();
         assert_eq!(
             r,
@@ -748,7 +748,7 @@ mod relational_expr {
     }
     #[test]
     fn expr_ne() {
-        let expr = lex(b"primary_key != 10").unwrap();
+        let expr = lex_insecure(b"primary_key != 10").unwrap();
         let r = dml::parse_relexpr_full(&expr).unwrap();
         assert_eq!(
             r,
@@ -761,7 +761,7 @@ mod relational_expr {
     }
     #[test]
     fn expr_gt() {
-        let expr = lex(b"primary_key > 10").unwrap();
+        let expr = lex_insecure(b"primary_key > 10").unwrap();
         let r = dml::parse_relexpr_full(&expr).unwrap();
         assert_eq!(
             r,
@@ -774,7 +774,7 @@ mod relational_expr {
     }
     #[test]
     fn expr_ge() {
-        let expr = lex(b"primary_key >= 10").unwrap();
+        let expr = lex_insecure(b"primary_key >= 10").unwrap();
         let r = dml::parse_relexpr_full(&expr).unwrap();
         assert_eq!(
             r,
@@ -787,7 +787,7 @@ mod relational_expr {
     }
     #[test]
     fn expr_lt() {
-        let expr = lex(b"primary_key < 10").unwrap();
+        let expr = lex_insecure(b"primary_key < 10").unwrap();
         let r = dml::parse_relexpr_full(&expr).unwrap();
         assert_eq!(
             r,
@@ -800,7 +800,7 @@ mod relational_expr {
     }
     #[test]
     fn expr_le() {
-        let expr = lex(b"primary_key <= 10").unwrap();
+        let expr = lex_insecure(b"primary_key <= 10").unwrap();
         let r = dml::parse_relexpr_full(&expr).unwrap();
         assert_eq!(
             r,
@@ -819,7 +819,7 @@ mod where_clause {
     };
     #[test]
     fn where_single() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 x = 100
             "#)
         .unwrap();
@@ -835,7 +835,7 @@ mod where_clause {
     }
     #[test]
     fn where_double() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 userid = 100 and pass = "password"
             "#)
         .unwrap();
@@ -857,7 +857,7 @@ mod where_clause {
     }
     #[test]
     fn where_duplicate_condition() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 userid = 100 and userid > 200
             "#)
         .unwrap();
diff --git a/server/src/engine/ql/tests/entity.rs b/server/src/engine/ql/tests/entity.rs
index 7bc3a117..d1e63eec 100644
--- a/server/src/engine/ql/tests/entity.rs
+++ b/server/src/engine/ql/tests/entity.rs
@@ -28,21 +28,21 @@ use super::*;
 use crate::engine::ql::ast::{Compiler, Entity};
 #[test]
 fn entity_current() {
-    let t = lex(b"hello").unwrap();
+    let t = lex_insecure(b"hello").unwrap();
     let mut c = Compiler::new(&t);
     let r = Entity::parse(&mut c).unwrap();
     assert_eq!(r, Entity::Single("hello".into()))
 }
 #[test]
 fn entity_partial() {
-    let t = lex(b":hello").unwrap();
+    let t = lex_insecure(b":hello").unwrap();
     let mut c = Compiler::new(&t);
     let r = Entity::parse(&mut c).unwrap();
     assert_eq!(r, Entity::Partial("hello".into()))
 }
 #[test]
 fn entity_full() {
-    let t = lex(b"hello.world").unwrap();
+    let t = lex_insecure(b"hello.world").unwrap();
     let mut c = Compiler::new(&t);
     let r = Entity::parse(&mut c).unwrap();
     assert_eq!(r, Entity::Full("hello".into(), "world".into()))
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index c4536566..b31bc775 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -27,7 +27,7 @@
 use {
     super::{
         super::lexer::{Lit, Token},
-        lex,
+        lex_insecure,
     },
     crate::engine::ql::LangError,
 };
@@ -44,7 +44,7 @@ macro_rules! v(
 #[test]
 fn lex_ident() {
     let src = v!("hello");
-    assert_eq!(lex(&src).unwrap(), vec![Token::Ident("hello".into())]);
+    assert_eq!(lex_insecure(&src).unwrap(), vec![Token::Ident("hello".into())]);
 }
 
 // literals
@@ -52,7 +52,7 @@ fn lex_ident() {
 fn lex_unsigned_int() {
     let number = v!("123456");
     assert_eq!(
-        lex(&number).unwrap(),
+        lex_insecure(&number).unwrap(),
         vec![Token::Lit(Lit::UnsignedInt(123456))]
     );
 }
@@ -60,26 +60,26 @@ fn lex_unsigned_int() {
 fn lex_signed_int() {
     let number = v!("-123456");
     assert_eq!(
-        lex(&number).unwrap(),
+        lex_insecure(&number).unwrap(),
         vec![Token::Lit(Lit::SignedInt(-123456))]
     );
 }
 #[test]
 fn lex_bool() {
     let (t, f) = v!("true", "false");
-    assert_eq!(lex(&t).unwrap(), vec![Token::Lit(Lit::Bool(true))]);
-    assert_eq!(lex(&f).unwrap(), vec![Token::Lit(Lit::Bool(false))]);
+    assert_eq!(lex_insecure(&t).unwrap(), vec![Token::Lit(Lit::Bool(true))]);
+    assert_eq!(lex_insecure(&f).unwrap(), vec![Token::Lit(Lit::Bool(false))]);
 }
 #[test]
 fn lex_string() {
     let s = br#" "hello, world" "#;
     assert_eq!(
-        lex(s).unwrap(),
+        lex_insecure(s).unwrap(),
         vec![Token::Lit(Lit::Str("hello, world".into()))]
     );
     let s = br#" 'hello, world' "#;
     assert_eq!(
-        lex(s).unwrap(),
+        lex_insecure(s).unwrap(),
         vec![Token::Lit(Lit::Str("hello, world".into()))]
     );
 }
@@ -87,12 +87,12 @@ fn lex_string() {
 fn lex_string_test_escape_quote() {
     let s = br#" "\"hello world\"" "#; // == "hello world"
     assert_eq!(
-        lex(s).unwrap(),
+        lex_insecure(s).unwrap(),
         vec![Token::Lit(Lit::Str("\"hello world\"".into()))]
     );
     let s = br#" '\'hello world\'' "#; // == 'hello world'
     assert_eq!(
-        lex(s).unwrap(),
+        lex_insecure(s).unwrap(),
         vec![Token::Lit(Lit::Str("'hello world'".into()))]
     );
 }
@@ -100,12 +100,12 @@ fn lex_string_test_escape_quote() {
 fn lex_string_use_different_quote_style() {
     let s = br#" "he's on it" "#;
     assert_eq!(
-        lex(s).unwrap(),
+        lex_insecure(s).unwrap(),
         vec![Token::Lit(Lit::Str("he's on it".into()))]
     );
     let s = br#" 'he thinks that "that girl" fixed it' "#;
     assert_eq!(
-        lex(s).unwrap(),
+        lex_insecure(s).unwrap(),
         vec![Token::Lit(Lit::Str(
             "he thinks that \"that girl\" fixed it".into()
         ))]
@@ -115,17 +115,17 @@ fn lex_string_use_different_quote_style() {
 fn lex_string_escape_bs() {
     let s = v!(r#" "windows has c:\\" "#);
     assert_eq!(
-        lex(&s).unwrap(),
+        lex_insecure(&s).unwrap(),
         vec![Token::Lit(Lit::Str("windows has c:\\".into()))]
     );
     let s = v!(r#" 'windows has c:\\' "#);
     assert_eq!(
-        lex(&s).unwrap(),
+        lex_insecure(&s).unwrap(),
         vec![Token::Lit(Lit::Str("windows has c:\\".into()))]
     );
     let lol = v!(r#"'\\\\\\\\\\'"#);
     assert_eq!(
-        lex(&lol).unwrap(),
+        lex_insecure(&lol).unwrap(),
         vec![Token::Lit(Lit::Str("\\".repeat(5).into_boxed_str()))],
         "lol"
     )
@@ -133,30 +133,30 @@ fn lex_string_escape_bs() {
 #[test]
 fn lex_string_bad_escape() {
     let wth = br#" '\a should be an alert on windows apparently' "#;
-    assert_eq!(lex(wth).unwrap_err(), LangError::InvalidStringLiteral);
+    assert_eq!(lex_insecure(wth).unwrap_err(), LangError::InvalidStringLiteral);
 }
 #[test]
 fn lex_string_unclosed() {
     let wth = br#" 'omg where did the end go "#;
-    assert_eq!(lex(wth).unwrap_err(), LangError::InvalidStringLiteral);
+    assert_eq!(lex_insecure(wth).unwrap_err(), LangError::InvalidStringLiteral);
     let wth = br#" 'see, we escaped the end\' "#;
-    assert_eq!(lex(wth).unwrap_err(), LangError::InvalidStringLiteral);
+    assert_eq!(lex_insecure(wth).unwrap_err(), LangError::InvalidStringLiteral);
 }
 #[test]
 fn lex_unsafe_literal_mini() {
-    let usl = lex("\r0\n".as_bytes()).unwrap();
+    let usl = lex_insecure("\r0\n".as_bytes()).unwrap();
     assert_eq!(usl.len(), 1);
     assert_eq!(Token::Lit(Lit::Bin("".into())), usl[0]);
 }
 #[test]
 fn lex_unsafe_literal() {
-    let usl = lex("\r9\nabcdefghi".as_bytes()).unwrap();
+    let usl = lex_insecure("\r9\nabcdefghi".as_bytes()).unwrap();
     assert_eq!(usl.len(), 1);
     assert_eq!(Token::Lit(Lit::Bin("abcdefghi".into())), usl[0]);
 }
 #[test]
 fn lex_unsafe_literal_pro() {
-    let usl = lex("\r18\nabcdefghi123456789".as_bytes()).unwrap();
+    let usl = lex_insecure("\r18\nabcdefghi123456789".as_bytes()).unwrap();
     assert_eq!(usl.len(), 1);
     assert_eq!(Token::Lit(Lit::Bin("abcdefghi123456789".into())), usl[0]);
 }
@@ -332,11 +332,11 @@ mod safequery_full_param {
     fn p() {
         let query = b"select * from myapp where username = ? and pass = ?";
         let params = b"\x055\nsayan\x048\npass1234";
-        let sq = SafeQueryData::parse(query, params, 1).unwrap();
+        let sq = SafeQueryData::parse(query, params, 2).unwrap();
         assert_eq!(
             sq,
             SafeQueryData::new_test(
-                vec![LitIR::Str("sayan"), LitIR::Str("pass1234")].into_boxed_slice(),
+                vec![LitIR::Str("sayan"), LitIR::Bin(b"pass1234")].into_boxed_slice(),
                 vec![
                     Token![select],
                     Token![*],
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 116b65f9..a69c89c9 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -29,7 +29,7 @@ use super::{
         lexer::{Lit, Token},
         schema,
     },
-    lex, *,
+    lex_insecure, *,
 };
 mod inspect {
     use {
@@ -41,7 +41,7 @@ mod inspect {
     };
     #[test]
     fn inspect_space() {
-        let tok = lex(b"inspect space myspace").unwrap();
+        let tok = lex_insecure(b"inspect space myspace").unwrap();
         assert_eq!(
             ddl::parse_inspect_full(&tok[1..]).unwrap(),
             Statement::InspectSpace("myspace".into())
@@ -49,12 +49,12 @@ mod inspect {
     }
     #[test]
     fn inspect_model() {
-        let tok = lex(b"inspect model users").unwrap();
+        let tok = lex_insecure(b"inspect model users").unwrap();
         assert_eq!(
             ddl::parse_inspect_full(&tok[1..]).unwrap(),
             Statement::InspectModel(Entity::Single("users".into()))
         );
-        let tok = lex(b"inspect model tweeter.users").unwrap();
+        let tok = lex_insecure(b"inspect model tweeter.users").unwrap();
         assert_eq!(
             ddl::parse_inspect_full(&tok[1..]).unwrap(),
             Statement::InspectModel(("tweeter", "users").into())
@@ -62,7 +62,7 @@ mod inspect {
     }
     #[test]
     fn inspect_spaces() {
-        let tok = lex(b"inspect spaces").unwrap();
+        let tok = lex_insecure(b"inspect spaces").unwrap();
         assert_eq!(
             ddl::parse_inspect_full(&tok[1..]).unwrap(),
             Statement::InspectSpaces
@@ -80,7 +80,7 @@ mod alter_space {
     };
     #[test]
     fn alter_space_mini() {
-        let tok = lex(b"alter model mymodel with {}").unwrap();
+        let tok = lex_insecure(b"alter model mymodel with {}").unwrap();
         let r = schema::alter_space_full(&tok[2..]).unwrap();
         assert_eq!(
             r,
@@ -92,7 +92,7 @@ mod alter_space {
     }
     #[test]
     fn alter_space() {
-        let tok = lex(br#"
+        let tok = lex_insecure(br#"
                 alter model mymodel with {
                     max_entry: 1000,
                     driver: "ts-0.8"
@@ -116,7 +116,7 @@ mod tymeta {
     use super::*;
     #[test]
     fn tymeta_mini() {
-        let tok = lex(b"}").unwrap();
+        let tok = lex_insecure(b"}").unwrap();
         let (res, ret) = schema::fold_tymeta(&tok);
         assert!(res.is_okay());
         assert!(!res.has_more());
@@ -125,7 +125,7 @@ mod tymeta {
     }
     #[test]
     fn tymeta_mini_fail() {
-        let tok = lex(b",}").unwrap();
+        let tok = lex_insecure(b",}").unwrap();
         let (res, ret) = schema::fold_tymeta(&tok);
         assert!(!res.is_okay());
         assert!(!res.has_more());
@@ -134,7 +134,7 @@ mod tymeta {
     }
     #[test]
     fn tymeta() {
-        let tok = lex(br#"hello: "world", loading: true, size: 100 }"#).unwrap();
+        let tok = lex_insecure(br#"hello: "world", loading: true, size: 100 }"#).unwrap();
         let (res, ret) = schema::fold_tymeta(&tok);
         assert!(res.is_okay());
         assert!(!res.has_more());
@@ -152,7 +152,7 @@ mod tymeta {
     fn tymeta_pro() {
         // list { maxlen: 100, type string, unique: true }
         //        ^^^^^^^^^^^^^^^^^^ cursor should be at string
-        let tok = lex(br#"maxlen: 100, type string, unique: true }"#).unwrap();
+        let tok = lex_insecure(br#"maxlen: 100, type string, unique: true }"#).unwrap();
         let (res1, ret1) = schema::fold_tymeta(&tok);
         assert!(res1.is_okay());
         assert!(res1.has_more());
@@ -177,7 +177,7 @@ mod tymeta {
         // list { maxlen: 100, this: { is: "cool" }, type string, unique: true }
         //        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cursor should be at string
         let tok =
-            lex(br#"maxlen: 100, this: { is: "cool" }, type string, unique: true }"#).unwrap();
+            lex_insecure(br#"maxlen: 100, this: { is: "cool" }, type string, unique: true }"#).unwrap();
         let (res1, ret1) = schema::fold_tymeta(&tok);
         assert!(res1.is_okay());
         assert!(res1.has_more());
@@ -204,7 +204,7 @@ mod tymeta {
     fn fuzz_tymeta_normal() {
         // { maxlen: 10, unique: true, users: "sayan" }
         //   ^start
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                     maxlen: 10,
                     unique: true,
                     auth: {
@@ -241,7 +241,7 @@ mod tymeta {
     fn fuzz_tymeta_with_ty() {
         // list { maxlen: 10, unique: true, type string, users: "sayan" }
         //   ^start
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                     maxlen: 10,
                     unique: true,
                     auth: {
@@ -277,7 +277,7 @@ mod layer {
     use crate::engine::ql::schema::Layer;
     #[test]
     fn layer_mini() {
-        let tok = lex(b"string)").unwrap();
+        let tok = lex_insecure(b"string)").unwrap();
         let (layers, c, okay) = schema::fold_layers(&tok);
         assert_eq!(c, tok.len() - 1);
         assert!(okay);
@@ -288,7 +288,7 @@ mod layer {
     }
     #[test]
     fn layer() {
-        let tok = lex(b"string { maxlen: 100 }").unwrap();
+        let tok = lex_insecure(b"string { maxlen: 100 }").unwrap();
         let (layers, c, okay) = schema::fold_layers(&tok);
         assert_eq!(c, tok.len());
         assert!(okay);
@@ -304,7 +304,7 @@ mod layer {
     }
     #[test]
     fn layer_plus() {
-        let tok = lex(b"list { type string }").unwrap();
+        let tok = lex_insecure(b"list { type string }").unwrap();
         let (layers, c, okay) = schema::fold_layers(&tok);
         assert_eq!(c, tok.len());
         assert!(okay);
@@ -318,7 +318,7 @@ mod layer {
     }
     #[test]
     fn layer_pro() {
-        let tok = lex(b"list { unique: true, type string, maxlen: 10 }").unwrap();
+        let tok = lex_insecure(b"list { unique: true, type string, maxlen: 10 }").unwrap();
         let (layers, c, okay) = schema::fold_layers(&tok);
         assert_eq!(c, tok.len());
         assert!(okay);
@@ -338,7 +338,7 @@ mod layer {
     }
     #[test]
     fn layer_pro_max() {
-        let tok = lex(
+        let tok = lex_insecure(
             b"list { unique: true, type string { ascii_only: true, maxlen: 255 }, maxlen: 10 }",
         )
         .unwrap();
@@ -368,7 +368,7 @@ mod layer {
 
     #[test]
     fn fuzz_layer() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
             list {
                 type list {
                     maxlen: 100,
@@ -408,7 +408,7 @@ mod field_properties {
 
     #[test]
     fn field_properties_empty() {
-        let tok = lex(b"myfield:").unwrap();
+        let tok = lex_insecure(b"myfield:").unwrap();
         let (props, c, okay) = schema::parse_field_properties(&tok);
         assert!(okay);
         assert_eq!(c, 0);
@@ -416,7 +416,7 @@ mod field_properties {
     }
     #[test]
     fn field_properties_full() {
-        let tok = lex(b"primary null myfield:").unwrap();
+        let tok = lex_insecure(b"primary null myfield:").unwrap();
         let (props, c, okay) = schema::parse_field_properties(&tok);
         assert_eq!(c, 2);
         assert_eq!(tok[c], Token::Ident("myfield".into()));
@@ -436,7 +436,7 @@ mod fields {
     };
     #[test]
     fn field_mini() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 username: string,
             ")
         .unwrap();
@@ -453,7 +453,7 @@ mod fields {
     }
     #[test]
     fn field() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 primary username: string,    
             ")
         .unwrap();
@@ -470,7 +470,7 @@ mod fields {
     }
     #[test]
     fn field_pro() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 primary username: string {
                     maxlen: 10,
                     ascii_only: true,
@@ -497,7 +497,7 @@ mod fields {
     }
     #[test]
     fn field_pro_max() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 null notes: list {
                     type string {
                         maxlen: 255,
@@ -540,7 +540,7 @@ mod schemas {
     use super::*;
     #[test]
     fn schema_mini() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 create model mymodel(
                     primary username: string,
                     password: binary,
@@ -574,7 +574,7 @@ mod schemas {
     }
     #[test]
     fn schema() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 create model mymodel(
                     primary username: string,
                     password: binary,
@@ -615,7 +615,7 @@ mod schemas {
 
     #[test]
     fn schema_pro() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 create model mymodel(
                     primary username: string,
                     password: binary,
@@ -673,7 +673,7 @@ mod schemas {
 
     #[test]
     fn schema_pro_max() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 create model mymodel(
                     primary username: string,
                     password: binary,
@@ -744,7 +744,7 @@ mod dict_field_syntax {
     use crate::engine::ql::schema::{ExpandedField, Layer};
     #[test]
     fn field_syn_mini() {
-        let tok = lex(b"username { type string }").unwrap();
+        let tok = lex_insecure(b"username { type string }").unwrap();
         let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
@@ -759,7 +759,7 @@ mod dict_field_syntax {
     }
     #[test]
     fn field_syn() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 username {
                     nullable: false,
                     type string,
@@ -782,7 +782,7 @@ mod dict_field_syntax {
     }
     #[test]
     fn field_syn_pro() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 username {
                     nullable: false,
                     type string {
@@ -816,7 +816,7 @@ mod dict_field_syntax {
     }
     #[test]
     fn field_syn_pro_max() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 notes {
                     nullable: true,
                     type list {
@@ -863,7 +863,7 @@ mod alter_model_remove {
     use crate::engine::ql::RawSlice;
     #[test]
     fn alter_mini() {
-        let tok = lex(b"alter model mymodel remove myfield").unwrap();
+        let tok = lex_insecure(b"alter model mymodel remove myfield").unwrap();
         let mut i = 4;
         let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
@@ -871,7 +871,7 @@ mod alter_model_remove {
     }
     #[test]
     fn alter_mini_2() {
-        let tok = lex(b"alter model mymodel remove (myfield)").unwrap();
+        let tok = lex_insecure(b"alter model mymodel remove (myfield)").unwrap();
         let mut i = 4;
         let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
@@ -880,7 +880,7 @@ mod alter_model_remove {
     #[test]
     fn alter() {
         let tok =
-            lex(b"alter model mymodel remove (myfield1, myfield2, myfield3, myfield4)").unwrap();
+            lex_insecure(b"alter model mymodel remove (myfield1, myfield2, myfield3, myfield4)").unwrap();
         let mut i = 4;
         let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
@@ -901,7 +901,7 @@ mod alter_model_add {
     use crate::engine::ql::schema::{ExpandedField, Layer};
     #[test]
     fn add_mini() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 alter model mymodel add myfield { type string }
             ")
         .unwrap();
@@ -920,7 +920,7 @@ mod alter_model_add {
     }
     #[test]
     fn add() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 alter model mymodel add myfield { type string, nullable: true }
             ")
         .unwrap();
@@ -941,7 +941,7 @@ mod alter_model_add {
     }
     #[test]
     fn add_pro() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 alter model mymodel add (myfield { type string, nullable: true })
             ")
         .unwrap();
@@ -962,7 +962,7 @@ mod alter_model_add {
     }
     #[test]
     fn add_pro_max() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 alter model mymodel add (
                     myfield {
                         type string,
@@ -1026,7 +1026,7 @@ mod alter_model_update {
 
     #[test]
     fn alter_mini() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 alter model mymodel update myfield { type string, .. }
             ")
         .unwrap();
@@ -1045,7 +1045,7 @@ mod alter_model_update {
     }
     #[test]
     fn alter_mini_2() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 alter model mymodel update (myfield { type string, .. })
             ")
         .unwrap();
@@ -1064,7 +1064,7 @@ mod alter_model_update {
     }
     #[test]
     fn alter() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 alter model mymodel update (
                     myfield {
                         type string,
@@ -1091,7 +1091,7 @@ mod alter_model_update {
     }
     #[test]
     fn alter_pro() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 alter model mymodel update (
                     myfield {
                         type string,
@@ -1130,7 +1130,7 @@ mod alter_model_update {
     }
     #[test]
     fn alter_pro_max() {
-        let tok = lex(b"
+        let tok = lex_insecure(b"
                 alter model mymodel update (
                     myfield {
                         type string {..},
@@ -1186,7 +1186,7 @@ mod ddl_other_query_tests {
     };
     #[test]
     fn drop_space() {
-        let src = lex(br"drop space myspace").unwrap();
+        let src = lex_insecure(br"drop space myspace").unwrap();
         assert_eq!(
             ddl::parse_drop_full(&src[1..]).unwrap(),
             Statement::DropSpace(DropSpace::new("myspace".into(), false))
@@ -1194,7 +1194,7 @@ mod ddl_other_query_tests {
     }
     #[test]
     fn drop_space_force() {
-        let src = lex(br"drop space myspace force").unwrap();
+        let src = lex_insecure(br"drop space myspace force").unwrap();
         assert_eq!(
             ddl::parse_drop_full(&src[1..]).unwrap(),
             Statement::DropSpace(DropSpace::new("myspace".into(), true))
@@ -1202,7 +1202,7 @@ mod ddl_other_query_tests {
     }
     #[test]
     fn drop_model() {
-        let src = lex(br"drop model mymodel").unwrap();
+        let src = lex_insecure(br"drop model mymodel").unwrap();
         assert_eq!(
             ddl::parse_drop_full(&src[1..]).unwrap(),
             Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), false))
@@ -1210,7 +1210,7 @@ mod ddl_other_query_tests {
     }
     #[test]
     fn drop_model_force() {
-        let src = lex(br"drop model mymodel force").unwrap();
+        let src = lex_insecure(br"drop model mymodel force").unwrap();
         assert_eq!(
             ddl::parse_drop_full(&src[1..]).unwrap(),
             Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), true))
diff --git a/server/src/engine/ql/tests/structure_syn.rs b/server/src/engine/ql/tests/structure_syn.rs
index b2881ea7..9681d24b 100644
--- a/server/src/engine/ql/tests/structure_syn.rs
+++ b/server/src/engine/ql/tests/structure_syn.rs
@@ -170,7 +170,7 @@ mod dict {
 
     #[test]
     fn fuzz_dict() {
-        let ret = lex(b"
+        let ret = lex_insecure(b"
                 {
                     the_tradition_is: \"hello, world\",
                     could_have_been: {

From 5d527408cd970f9d60c970f31d4e0e6d013f5b9a Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 24 Dec 2022 06:03:27 -0800
Subject: [PATCH 057/310] Fix number decode algorithm for signed numbers

Also added preliminary tests for the same
---
 server/src/engine/ql/lexer.rs             |  62 ++++----
 server/src/engine/ql/tests.rs             |   1 +
 server/src/engine/ql/tests/dml_tests.rs   | 172 ++++++++++++++--------
 server/src/engine/ql/tests/lexer_tests.rs | 137 +++++++++++++++--
 server/src/main.rs                        |   4 +-
 5 files changed, 270 insertions(+), 106 deletions(-)

diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 03690e60..2a19efac 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -61,6 +61,7 @@ assertions! {
     size_of::<Symbol>() == 1,
     size_of::<Keyword>() == 1,
     size_of::<Lit>() == 24, // FIXME(@ohsayan): Ouch
+    size_of::<LitIR>() == 24,
 }
 
 enum_impls! {
@@ -784,9 +785,9 @@ const ALLOW_SIGNED: bool = true;
 
 pub trait NumberDefinition: Sized + fmt::Debug + Copy + Clone + BitOr<Self, Output = Self> {
     const ALLOW_SIGNED: bool;
-    fn overflowing_mul(&self, v: u8) -> (Self, bool);
-    fn overflowing_add(&self, v: u8) -> (Self, bool);
-    fn negate(&mut self);
+    fn mul_of(&self, v: u8) -> (Self, bool);
+    fn add_of(&self, v: u8) -> (Self, bool);
+    fn sub_of(&self, v: u8) -> (Self, bool);
     fn qualified_max_length() -> usize;
     fn zero() -> Self;
     fn b(self, b: bool) -> Self;
@@ -801,14 +802,11 @@ macro_rules! impl_number_def {
             #[inline(always)] fn zero() -> Self { 0 }
             #[inline(always)] fn b(self, b: bool) -> Self { b as Self * self }
 			#[inline(always)]
-			fn overflowing_mul(&self, v: u8) -> ($ty, bool) { <$ty>::overflowing_mul(*self, v as $ty) }
+			fn mul_of(&self, v: u8) -> ($ty, bool) { <$ty>::overflowing_mul(*self, v as $ty) }
 			#[inline(always)]
-			fn overflowing_add(&self, v: u8) -> ($ty, bool) { <$ty>::overflowing_add(*self, v as $ty) }
+			fn add_of(&self, v: u8) -> ($ty, bool) { <$ty>::overflowing_add(*self, v as $ty) }
 			#[inline(always)]
-			fn negate(&mut self) {
-				assert!(Self::ALLOW_SIGNED, "tried to negate an unsigned integer");
-                *self = !(*self - 1);
-			}
+			fn sub_of(&self, v: u8) -> ($ty, bool) { <$ty>::overflowing_sub(*self, v as $ty) }
             #[inline(always)] fn qualified_max_length() -> usize { $qualified_max_length }
 		})*
 	}
@@ -846,7 +844,7 @@ impl_number_def! {
 }
 
 #[inline(always)]
-fn decode_num_from_unbounded_payload<N>(src: &[u8], flag: &mut bool, cnt: &mut usize) -> N
+pub(super) fn decode_num_ub<N>(src: &[u8], flag: &mut bool, cnt: &mut usize) -> N
 where
     N: NumberDefinition,
 {
@@ -856,42 +854,50 @@ where
     let mut number = N::zero();
     let mut nx_stop = false;
 
-    let is_signed = if N::ALLOW_SIGNED {
-        let is_signed = i < l && src[i] == b'-';
-        i += is_signed as usize;
+    let is_signed;
+    if N::ALLOW_SIGNED {
+        let loc_s = i < l && src[i] == b'-';
+        i += loc_s as usize;
         okay &= (i + 2) <= l; // [-][digit][LF]
-        is_signed
+        is_signed = loc_s;
     } else {
-        false
-    };
+        is_signed = false;
+    }
 
     while i < l && okay && !nx_stop {
         // potential exit
         nx_stop = src[i] == b'\n';
         // potential entry
         let mut local_ok = src[i].is_ascii_digit();
-        let (p, p_of) = number.overflowing_mul(10);
+        let (p, p_of) = number.mul_of(10);
         local_ok &= !p_of;
-        let (s, s_of) = p.overflowing_add(src[i] & 0x0f);
-        local_ok &= !s_of;
+        let lfret;
+        if N::ALLOW_SIGNED && is_signed {
+            let (d, d_of) = p.sub_of(src[i] & 0x0f);
+            local_ok &= !d_of;
+            lfret = d;
+        } else {
+            let (s, s_of) = p.add_of(src[i] & 0x0f);
+            local_ok &= !s_of;
+            lfret = s;
+        }
         // reassign or assign
         let reassign = number.b(nx_stop);
-        let assign = s.b(!nx_stop);
+        let assign = lfret.b(!nx_stop);
         number = reassign | assign;
         okay &= local_ok | nx_stop;
         i += okay as usize;
     }
+    if N::ALLOW_SIGNED {
+        number = number.b(okay);
+    }
     okay &= nx_stop;
     *cnt += i;
     *flag &= okay;
-
-    if N::ALLOW_SIGNED && is_signed {
-        number.negate()
-    }
     number
 }
 
-#[derive(PartialEq, Debug, Clone, Copy)]
+#[derive(PartialEq, Debug, Clone)]
 /// Intermediate literal repr
 pub enum LitIR<'a> {
     Str(&'a str),
@@ -961,7 +967,7 @@ impl<'b> SafeQueryData<'b> {
     fn mxple<'a>(src: &'a [u8], cnt: &mut usize, flag: &mut bool) -> &'a [u8] {
         // find payload length
         let mut i = 0;
-        let payload_len = decode_num_from_unbounded_payload::<usize>(src, flag, &mut i);
+        let payload_len = decode_num_ub::<usize>(src, flag, &mut i);
         let src = &src[i..];
         // find payload
         *flag &= src.len() >= payload_len;
@@ -974,14 +980,14 @@ impl<'b> SafeQueryData<'b> {
     #[inline(always)]
     pub(super) fn uint<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
         let mut b = true;
-        let r = decode_num_from_unbounded_payload(src, &mut b, cnt);
+        let r = decode_num_ub(src, &mut b, cnt);
         data.push(LitIR::UInt(r));
         b
     }
     #[inline(always)]
     pub(super) fn sint<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
         let mut b = true;
-        let r = decode_num_from_unbounded_payload(src, &mut b, cnt);
+        let r = decode_num_ub(src, &mut b, cnt);
         data.push(LitIR::SInt(r));
         b
     }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index f3cac871..c9352c75 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -45,6 +45,7 @@ mod lexer_tests;
 mod schema_tests;
 mod structure_syn;
 
+/// Uses the [`InsecureLexer`] to lex the given input
 pub(super) fn lex_insecure(src: &[u8]) -> LangResult<Vec<Token>> {
     InsecureLexer::lex(src)
 }
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 44f3c457..8675e760 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -31,9 +31,11 @@ mod list_parse {
 
     #[test]
     fn list_mini() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 []
-            ")
+            ",
+        )
         .unwrap();
         let r = parse_list_full(&tok[1..]).unwrap();
         assert_eq!(r, vec![])
@@ -41,9 +43,11 @@ mod list_parse {
 
     #[test]
     fn list() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 [1, 2, 3, 4]
-            ")
+            ",
+        )
         .unwrap();
         let r = parse_list_full(&tok[1..]).unwrap();
         assert_eq!(r.as_slice(), into_array![1, 2, 3, 4])
@@ -51,14 +55,16 @@ mod list_parse {
 
     #[test]
     fn list_pro() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 [
                     [1, 2],
                     [3, 4],
                     [5, 6],
                     []
                 ]
-            ")
+            ",
+        )
         .unwrap();
         let r = parse_list_full(&tok[1..]).unwrap();
         assert_eq!(
@@ -74,14 +80,16 @@ mod list_parse {
 
     #[test]
     fn list_pro_max() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 [
                     [[1, 1], [2, 2]],
                     [[], [4, 4]],
                     [[5, 5], [6, 6]],
                     [[7, 7], []]
                 ]
-            ")
+            ",
+        )
         .unwrap();
         let r = parse_list_full(&tok[1..]).unwrap();
         assert_eq!(
@@ -108,9 +116,11 @@ mod tuple_syntax {
 
     #[test]
     fn tuple() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 (1234, "email@example.com", true)
-            "#)
+            "#,
+        )
         .unwrap();
         let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
         assert_eq!(
@@ -121,14 +131,16 @@ mod tuple_syntax {
 
     #[test]
     fn tuple_pro() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 (
                     1234,
                     "email@example.com",
                     true,
                     ["hello", "world", "and", "the", "universe"]
                 )
-            "#)
+            "#,
+        )
         .unwrap();
         let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
         assert_eq!(
@@ -144,7 +156,8 @@ mod tuple_syntax {
 
     #[test]
     fn tuple_pro_max() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 (
                     1234,
                     "email@example.com",
@@ -158,7 +171,8 @@ mod tuple_syntax {
                         []
                     ]
                 )
-            "#)
+            "#,
+        )
         .unwrap();
         let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
         assert_eq!(
@@ -192,14 +206,16 @@ mod map_syntax {
 
     #[test]
     fn map() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 {
                     name: "John Appletree",
                     email: "john@example.com",
                     verified: false,
                     followers: 12345
                 }
-            "#)
+            "#,
+        )
         .unwrap();
         let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
         assert_eq!(
@@ -215,7 +231,8 @@ mod map_syntax {
 
     #[test]
     fn map_pro() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 {
                     name: "John Appletree",
                     email: "john@example.com",
@@ -223,7 +240,8 @@ mod map_syntax {
                     followers: 12345,
                     tweets_by_day: []
                 }
-            "#)
+            "#,
+        )
         .unwrap();
         let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
         assert_eq!(
@@ -286,9 +304,11 @@ mod stmt_insert {
 
     #[test]
     fn insert_tuple_mini() {
-        let x = lex_insecure(br#"
+        let x = lex_insecure(
+            br#"
                 insert into twitter.users ("sayan")
-            "#)
+            "#,
+        )
         .unwrap();
         let r = dml::parse_insert_full(&x[1..]).unwrap();
         let e = InsertStatement {
@@ -299,7 +319,8 @@ mod stmt_insert {
     }
     #[test]
     fn insert_tuple() {
-        let x = lex_insecure(br#"
+        let x = lex_insecure(
+            br#"
                 insert into twitter.users (
                     "sayan",
                     "Sayan",
@@ -308,27 +329,22 @@ mod stmt_insert {
                     12345,
                     67890
                 )
-            "#)
+            "#,
+        )
         .unwrap();
         let r = dml::parse_insert_full(&x[1..]).unwrap();
         let e = InsertStatement {
             entity: Entity::Full("twitter".into(), "users".into()),
-            data: into_array_nullable![
-                "sayan",
-                "Sayan",
-                "sayan@example.com",
-                true,
-                12345,
-                67890
-            ]
-            .to_vec()
-            .into(),
+            data: into_array_nullable!["sayan", "Sayan", "sayan@example.com", true, 12345, 67890]
+                .to_vec()
+                .into(),
         };
         assert_eq!(e, r);
     }
     #[test]
     fn insert_tuple_pro() {
-        let x = lex_insecure(br#"
+        let x = lex_insecure(
+            br#"
                 insert into twitter.users (
                     "sayan",
                     "Sayan",
@@ -340,7 +356,8 @@ mod stmt_insert {
                     12345,
                     null
                 )
-            "#)
+            "#,
+        )
         .unwrap();
         let r = dml::parse_insert_full(&x[1..]).unwrap();
         let e = InsertStatement {
@@ -363,9 +380,11 @@ mod stmt_insert {
     }
     #[test]
     fn insert_map_mini() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 insert into jotsy.app { username: "sayan" }
-            "#)
+            "#,
+        )
         .unwrap();
         let r = dml::parse_insert_full(&tok[1..]).unwrap();
         let e = InsertStatement {
@@ -379,7 +398,8 @@ mod stmt_insert {
     }
     #[test]
     fn insert_map() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 insert into jotsy.app {
                     username: "sayan",
                     name: "Sayan",
@@ -388,7 +408,8 @@ mod stmt_insert {
                     following: 12345,
                     followers: 67890
                 }
-            "#)
+            "#,
+        )
         .unwrap();
         let r = dml::parse_insert_full(&tok[1..]).unwrap();
         let e = InsertStatement {
@@ -407,7 +428,8 @@ mod stmt_insert {
     }
     #[test]
     fn insert_map_pro() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 insert into jotsy.app {
                     username: "sayan",
                     password: "pass123",
@@ -419,7 +441,8 @@ mod stmt_insert {
                     bookmarks: 12345,
                     other_linked_accounts: null
                 }
-            "#)
+            "#,
+        )
         .unwrap();
         let r = dml::parse_insert_full(&tok[1..]).unwrap();
         let e = InsertStatement {
@@ -453,9 +476,11 @@ mod stmt_select {
     };
     #[test]
     fn select_mini() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 select * from users where username = "sayan"
-            "#)
+            "#,
+        )
         .unwrap();
         let r = dml::parse_select_full(&tok[1..]).unwrap();
         let username_where = "sayan".into();
@@ -473,9 +498,11 @@ mod stmt_select {
     }
     #[test]
     fn select() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 select field1 from users where username = "sayan"
-            "#)
+            "#,
+        )
         .unwrap();
         let r = dml::parse_select_full(&tok[1..]).unwrap();
         let username_where = "sayan".into();
@@ -493,9 +520,11 @@ mod stmt_select {
     }
     #[test]
     fn select_pro() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 select field1 from twitter.users where username = "sayan"
-            "#)
+            "#,
+        )
         .unwrap();
         let r = dml::parse_select_full(&tok[1..]).unwrap();
         let username_where = "sayan".into();
@@ -513,9 +542,11 @@ mod stmt_select {
     }
     #[test]
     fn select_pro_max() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 select field1, field2 from twitter.users where username = "sayan"
-            "#)
+            "#,
+        )
         .unwrap();
         let r = dml::parse_select_full(&tok[1..]).unwrap();
         let username_where = "sayan".into();
@@ -612,16 +643,17 @@ mod update_statement {
         crate::engine::ql::{
             ast::Entity,
             dml::{
-                self, AssignmentExpression, Operator, RelationalExpr, UpdateStatement,
-                WhereClause,
+                self, AssignmentExpression, Operator, RelationalExpr, UpdateStatement, WhereClause,
             },
         },
     };
     #[test]
     fn update_mini() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 update app SET notes += "this is my new note" where username = "sayan"
-            "#)
+            "#,
+        )
         .unwrap();
         let where_username = "sayan".into();
         let note = "this is my new note".to_string().into();
@@ -645,7 +677,8 @@ mod update_statement {
     }
     #[test]
     fn update() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 update
                     jotsy.app
                 SET
@@ -653,7 +686,8 @@ mod update_statement {
                     email = "sayan@example.com"
                 WHERE
                     username = "sayan"
-            "#)
+            "#,
+        )
         .unwrap();
         let r = dml::parse_update_full(&tok[1..]).unwrap();
         let where_username = "sayan".into();
@@ -688,9 +722,11 @@ mod delete_stmt {
 
     #[test]
     fn delete_mini() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 delete from users where username = "sayan"
-            "#)
+            "#,
+        )
         .unwrap();
         let primary_key = "sayan".into();
         let e = DeleteStatement::new_test(
@@ -708,9 +744,11 @@ mod delete_stmt {
     }
     #[test]
     fn delete() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 delete from twitter.users where username = "sayan"
-            "#)
+            "#,
+        )
         .unwrap();
         let primary_key = "sayan".into();
         let e = DeleteStatement::new_test(
@@ -819,9 +857,11 @@ mod where_clause {
     };
     #[test]
     fn where_single() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 x = 100
-            "#)
+            "#,
+        )
         .unwrap();
         let rhs_hundred = 100.into();
         let expected = WhereClause::new(dict! {
@@ -835,9 +875,11 @@ mod where_clause {
     }
     #[test]
     fn where_double() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 userid = 100 and pass = "password"
-            "#)
+            "#,
+        )
         .unwrap();
         let rhs_hundred = 100.into();
         let rhs_password = "password".into();
@@ -857,9 +899,11 @@ mod where_clause {
     }
     #[test]
     fn where_duplicate_condition() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 userid = 100 and userid > 200
-            "#)
+            "#,
+        )
         .unwrap();
         assert!(dml::parse_where_clause_full(&tok).is_none());
     }
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index b31bc775..d4a1a595 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -44,7 +44,10 @@ macro_rules! v(
 #[test]
 fn lex_ident() {
     let src = v!("hello");
-    assert_eq!(lex_insecure(&src).unwrap(), vec![Token::Ident("hello".into())]);
+    assert_eq!(
+        lex_insecure(&src).unwrap(),
+        vec![Token::Ident("hello".into())]
+    );
 }
 
 // literals
@@ -68,7 +71,10 @@ fn lex_signed_int() {
 fn lex_bool() {
     let (t, f) = v!("true", "false");
     assert_eq!(lex_insecure(&t).unwrap(), vec![Token::Lit(Lit::Bool(true))]);
-    assert_eq!(lex_insecure(&f).unwrap(), vec![Token::Lit(Lit::Bool(false))]);
+    assert_eq!(
+        lex_insecure(&f).unwrap(),
+        vec![Token::Lit(Lit::Bool(false))]
+    );
 }
 #[test]
 fn lex_string() {
@@ -133,14 +139,23 @@ fn lex_string_escape_bs() {
 #[test]
 fn lex_string_bad_escape() {
     let wth = br#" '\a should be an alert on windows apparently' "#;
-    assert_eq!(lex_insecure(wth).unwrap_err(), LangError::InvalidStringLiteral);
+    assert_eq!(
+        lex_insecure(wth).unwrap_err(),
+        LangError::InvalidStringLiteral
+    );
 }
 #[test]
 fn lex_string_unclosed() {
     let wth = br#" 'omg where did the end go "#;
-    assert_eq!(lex_insecure(wth).unwrap_err(), LangError::InvalidStringLiteral);
+    assert_eq!(
+        lex_insecure(wth).unwrap_err(),
+        LangError::InvalidStringLiteral
+    );
     let wth = br#" 'see, we escaped the end\' "#;
-    assert_eq!(lex_insecure(wth).unwrap_err(), LangError::InvalidStringLiteral);
+    assert_eq!(
+        lex_insecure(wth).unwrap_err(),
+        LangError::InvalidStringLiteral
+    );
 }
 #[test]
 fn lex_unsafe_literal_mini() {
@@ -161,6 +176,107 @@ fn lex_unsafe_literal_pro() {
     assert_eq!(Token::Lit(Lit::Bin("abcdefghi123456789".into())), usl[0]);
 }
 
+mod num_tests {
+    use crate::engine::ql::lexer::decode_num_ub as ubdc;
+    mod uint8 {
+        use super::*;
+        #[test]
+        fn ndecub_u8_ok() {
+            const SRC: &[u8] = b"123\n";
+            let mut i = 0;
+            let mut b = true;
+            let x = ubdc::<u8>(SRC, &mut b, &mut i);
+            assert!(b);
+            assert_eq!(i, SRC.len());
+            assert_eq!(x, 123);
+        }
+        #[test]
+        fn ndecub_u8_lb() {
+            const SRC: &[u8] = b"0\n";
+            let mut i = 0;
+            let mut b = true;
+            let x = ubdc::<u8>(SRC, &mut b, &mut i);
+            assert!(b);
+            assert_eq!(i, SRC.len());
+            assert_eq!(x, 0);
+        }
+        #[test]
+        fn ndecub_u8_ub() {
+            const SRC: &[u8] = b"255\n";
+            let mut i = 0;
+            let mut b = true;
+            let x = ubdc::<u8>(SRC, &mut b, &mut i);
+            assert!(b);
+            assert_eq!(i, SRC.len());
+            assert_eq!(x, 255);
+        }
+        #[test]
+        fn ndecub_u8_ub_of() {
+            const SRC: &[u8] = b"256\n";
+            let mut i = 0;
+            let mut b = true;
+            let x = ubdc::<u8>(SRC, &mut b, &mut i);
+            assert!(!b);
+            assert_eq!(i, 2);
+            assert_eq!(x, 0);
+        }
+    }
+    mod sint8 {
+        use super::*;
+        #[test]
+        pub(crate) fn ndecub_i8_ok() {
+            const SRC: &[u8] = b"-123\n";
+            let mut i = 0;
+            let mut b = true;
+            let x = ubdc::<i8>(SRC, &mut b, &mut i);
+            assert!(b);
+            assert_eq!(i, SRC.len());
+            assert_eq!(x, -123);
+        }
+        #[test]
+        pub(crate) fn ndecub_i8_lb() {
+            const SRC: &[u8] = b"-128\n";
+            let mut i = 0;
+            let mut b = true;
+            let x = ubdc::<i8>(SRC, &mut b, &mut i);
+            assert!(b);
+            assert_eq!(i, SRC.len());
+            assert_eq!(x, -128);
+        }
+
+        #[test]
+        pub(crate) fn ndecub_i8_lb_of() {
+            const SRC: &[u8] = b"-129\n";
+            let mut i = 0;
+            let mut b = true;
+            let x = ubdc::<i8>(SRC, &mut b, &mut i);
+            assert!(!b);
+            assert_eq!(i, 3);
+            assert_eq!(x, 0);
+        }
+        #[test]
+        pub(crate) fn ndecub_i8_ub() {
+            const SRC: &[u8] = b"127\n";
+            let mut i = 0;
+            let mut b = true;
+            let x = ubdc::<i8>(SRC, &mut b, &mut i);
+            assert!(b);
+            assert_eq!(i, SRC.len());
+            assert_eq!(x, 127);
+        }
+        #[test]
+        pub(crate) fn ndecub_i8_ub_of() {
+            const SRC: &[u8] = b"128\n";
+            let mut i = 0;
+            let mut b = true;
+            let x = ubdc::<i8>(SRC, &mut b, &mut i);
+            assert!(!b);
+            assert_eq!(i, 2);
+            assert_eq!(x, 0);
+        }
+    }
+}
+
 mod safequery_params {
     use rand::seq::SliceRandom;
 
@@ -273,7 +389,7 @@ mod safequery_params {
     #[test]
     fn params_mix() {
         let mut rng = rand::thread_rng();
-        const DATA: [&'static [u8]; 6] = [
+        const DATA: [&[u8]; 6] = [
             b"\x0012345\n",
             b"\x01-12345\n",
             b"\x02true\n",
@@ -292,12 +408,11 @@ mod safequery_params {
         for _ in 0..DATA.len().pow(2) {
             let mut local_data = DATA;
             local_data.shuffle(&mut rng);
-            let ret: Vec<LitIR> = local_data.iter().map(|v| RETMAP[v[0] as usize]).collect();
-            let src: Vec<u8> = local_data
-                .into_iter()
-                .map(|v| v.to_owned())
-                .flatten()
+            let ret: Vec<LitIR> = local_data
+                .iter()
+                .map(|v| RETMAP[v[0] as usize].clone())
                 .collect();
+            let src: Vec<u8> = local_data.into_iter().flat_map(|v| v.to_owned()).collect();
             let r = SafeQueryData::p_revloop(&src, 6).unwrap();
             assert_eq!(r.as_ref(), ret);
         }
diff --git a/server/src/main.rs b/server/src/main.rs
index 9dad3e91..53815237 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -24,9 +24,7 @@
  *
 */
 
-#![deny(unused_crate_dependencies)]
-#![deny(unused_imports)]
-#![deny(unused_must_use)]
+#![deny(unused_crate_dependencies, unused_imports, unused_must_use)]
 #![cfg_attr(feature = "nightly", feature(test))]
 
 //! # Skytable

From b918af6d96130eff7863e704de50e7d84021f74c Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 1 Jan 2023 05:02:38 -0800
Subject: [PATCH 058/310] Use flags for DT

---
 server/src/engine/macros.rs               | 38 ++++++++++++++-
 server/src/engine/memory/mod.rs           | 56 +++++++++++++----------
 server/src/engine/ql/dml.rs               | 15 ++++--
 server/src/engine/ql/mod.rs               |  2 +-
 server/src/engine/ql/tests/lexer_tests.rs |  1 -
 5 files changed, 82 insertions(+), 30 deletions(-)

diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index 3c68cab4..4301db40 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -41,11 +41,47 @@ macro_rules! multi_assert_eq {
 }
 
 macro_rules! enum_impls {
+    ($for:ident<$lt:lifetime> => {$($other:ty as $me:ident),*$(,)?}) => {
+        $(impl<$lt> ::core::convert::From<$other> for $for<$lt> {fn from(v: $other) -> Self {Self::$me(v.into())}})*
+    };
     ($for:ty => {$($other:ty as $me:ident),*$(,)?}) => {
         $(impl ::core::convert::From<$other> for $for {fn from(v: $other) -> Self {Self::$me(v.into())}})*
-    }
+    };
 }
 
 macro_rules! assertions {
     ($($assert:expr),*$(,)?) => {$(const _:()=::core::assert!($assert);)*}
 }
+
+macro_rules! constgrp {
+    ($(#[$attr:meta])* $vis:vis struct $group:ident: $ty:ty { $($const:ident = $expr:expr),* $(,)?}) => (
+        $(#[$attr])* $vis struct $group {r#const: $ty}
+        impl $group {
+            $(pub const $const: Self = Self { r#const: $expr };)*
+            #[inline(always)] pub const fn d(&self) -> $ty { self.r#const }
+            const _BASE_HB: $ty = 1 << (<$ty>::BITS - 1);
+            #[inline(always)] pub const fn name(&self) -> &'static str {
+                match self.r#const {$(capture if capture == $expr => ::core::stringify!($const),)* _ => ::core::unreachable!()}
+            }
+        }
+        impl ::core::fmt::Debug for $group {
+            fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
+                ::core::write!(f, "{}::{}", ::core::stringify!($group), Self::name(self))
+            }
+        }
+    );
+}
+
+macro_rules! union {
+    ($(#[$attr:meta])* $vis:vis union $name:ident $tail:tt) => (union!(@parse [$(#[$attr])* $vis union $name] [] $tail););
+    ($(#[$attr:meta])* $vis:vis union $name:ident<$($lt:lifetime),*> $tail:tt) => (union!(@parse [$(#[$attr])* $vis union $name<$($lt),*>] [] $tail););
+    (@parse $decl:tt [$($head:tt)*] {}) => (union!(@defeat0 $decl [$($head)*]););
+    (@parse $decl:tt [$($head:tt)*] {$(#[$attr:meta])* $vis:vis !$ident:ident:$ty:ty,$($tail:tt)*}) => (
+        union!(@parse $decl [$($head)* $(#[$attr])* $vis $ident: ::core::mem::ManuallyDrop::<$ty>,] {$($tail)*});
+    );
+    (@parse $decl:tt [$($head:tt)*] {$(#[$attr:meta])* $vis:vis $ident:ident:$ty:ty,$($tail:tt)*}) => (
+        union!(@parse $decl [$($head)* $(#[$attr])* $vis $ident: $ty, ] { $($tail)* });
+    );
+    (@defeat0 [$($decls:tt)*] [$($head:tt)*]) => (union!(@defeat1 $($decls)* { $($head)* }););
+    (@defeat1 $i:item) => ($i);
+}
diff --git a/server/src/engine/memory/mod.rs b/server/src/engine/memory/mod.rs
index 5449dff3..6416dfe5 100644
--- a/server/src/engine/memory/mod.rs
+++ b/server/src/engine/memory/mod.rs
@@ -26,7 +26,7 @@
 
 // TODO(@ohsayan): Change the underlying structures, there are just rudimentary ones used during integration with the QL
 
-use super::ql::{lexer::Lit, RawSlice};
+use super::ql::lexer::Lit;
 
 /// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
 /// be of one type.
@@ -54,8 +54,6 @@ pub enum DataType {
     /// elements to ensure correctness in this specific context
     /// FIXME(@ohsayan): Try enforcing this somehow
     List(Vec<Self>),
-    /// **☢ WARNING:** Not an actual data type but MUST be translated into an actual data type
-    AnonymousTypeNeedsEval(RawSlice),
 }
 
 enum_impls! {
@@ -71,13 +69,16 @@ enum_impls! {
 
 impl DataType {
     #[inline(always)]
-    pub(super) fn clone_from_lit(lit: &Lit) -> Self {
+    /// ## Safety
+    ///
+    /// Ensure validity of Lit::Bin
+    pub(super) unsafe fn clone_from_lit(lit: &Lit) -> Self {
         match lit {
             Lit::Str(s) => DataType::String(s.clone()),
             Lit::Bool(b) => DataType::Boolean(*b),
             Lit::UnsignedInt(u) => DataType::UnsignedInt(*u),
             Lit::SignedInt(i) => DataType::SignedInt(*i),
-            Lit::Bin(l) => DataType::AnonymousTypeNeedsEval(l.clone()),
+            Lit::Bin(l) => DataType::Binary(l.as_slice().to_owned()),
         }
     }
 }
@@ -88,23 +89,30 @@ impl<const N: usize> From<[DataType; N]> for DataType {
     }
 }
 
-#[repr(u8, align(1))]
-pub enum DataKind {
-    // primitive: integer unsigned
-    UInt8 = 0,
-    UInt16 = 1,
-    Uint32 = 2,
-    UInt64 = 3,
-    // primitive: integer unsigned
-    SInt8 = 4,
-    SInt16 = 5,
-    SInt32 = 6,
-    SInt64 = 7,
-    // primitive: misc
-    Bool = 8,
-    // compound: flat
-    String = 9,
-    Binary = 10,
-    // compound: recursive
-    List = 11,
+constgrp! {
+    #[derive(PartialEq, Eq, Clone, Copy)]
+    pub struct DataKind: u8 {
+        // primitive: integer unsigned
+        UINT8 = 0,
+        UINT16 = 1,
+        UINT32 = 2,
+        UINT64 = 3,
+        // primitive: integer unsigned
+        SINT8 = 4,
+        SINT16 = 5,
+        SINT32 = 6,
+        SINT64 = 7,
+        // primitive: misc
+        BOOL = 8,
+        // primitive: floating point
+        FLOAT32 = 9,
+        FLOAT64 = 10,
+        // compound: flat
+        STR = 11,
+        STR_BX = Self::_BASE_HB | Self::STR.d(),
+        BIN = 12,
+        BIN_BX = Self::_BASE_HB | Self::BIN.d(),
+        // compound: recursive
+        LIST = 13,
+    }
 }
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 9f053fe8..2c417c94 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -237,7 +237,10 @@ pub(super) fn parse_list(
     let mut prev_nlist_dscr = None;
     while i < l && okay && !stop {
         let d = match &tok[i] {
-            Token::Lit(l) => DataType::clone_from_lit(l),
+            Token::Lit(l) => unsafe {
+                // UNSAFE(@ohsayan): Token LT0 guarantees LT0 > LT1 for lit
+                DataType::clone_from_lit(l)
+            },
             Token::Symbol(Symbol::TtOpenSqBracket) => {
                 // a nested list
                 let mut nested_list = Vec::new();
@@ -294,7 +297,10 @@ pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<Option<DataType>>,
     let mut data = Vec::new();
     while i < l && okay && !stop {
         match &tok[i] {
-            Token::Lit(l) => data.push(Some(DataType::clone_from_lit(l))),
+            Token::Lit(l) => data.push(Some(unsafe {
+                // UNSAFE(@ohsayan): Token LT0 guarantees LT0 > LT1 for lit
+                DataType::clone_from_lit(l)
+            })),
             Token::Symbol(Symbol::TtOpenSqBracket) => {
                 // ah, a list
                 let mut l = Vec::new();
@@ -351,7 +357,10 @@ pub(super) fn parse_data_map_syntax<'a>(
                             // UNSAFE(@ohsayan): Token lifetime ensures slice validity
                             id.as_slice()
                         },
-                        Some(DataType::clone_from_lit(l)),
+                        Some(unsafe {
+                            // UNSAFE(@ohsayan): Token LT0 guarantees LT0 > LT1 for lit
+                            DataType::clone_from_lit(l)
+                        }),
                     )
                     .is_none();
             }
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 52d22f91..14b1a898 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -99,7 +99,7 @@ impl RawSlice {
     const unsafe fn new_from_str(s: &str) -> Self {
         Self::new(s.as_bytes().as_ptr(), s.as_bytes().len())
     }
-    unsafe fn as_slice(&self) -> &[u8] {
+    pub unsafe fn as_slice(&self) -> &[u8] {
         slice::from_raw_parts(self.ptr.as_ptr(), self.len)
     }
     unsafe fn as_str(&self) -> &str {
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index d4a1a595..77794c28 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -243,7 +243,6 @@ mod num_tests {
             assert_eq!(i, SRC.len());
             assert_eq!(x, -128);
         }
-
         #[test]
         pub(crate) fn ndecub_i8_lb_of() {
             const SRC: &[u8] = b"-129\n";

From 4c5e919954de440d16a87c2413f1c468307cae49 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 2 Jan 2023 21:26:25 -0800
Subject: [PATCH 059/310] Store rel expr operands as `LitIR` reprs

---
 server/src/engine/ql/dml.rs             | 16 ++---
 server/src/engine/ql/lexer.rs           | 12 ++++
 server/src/engine/ql/macros.rs          |  2 +-
 server/src/engine/ql/tests.rs           |  1 +
 server/src/engine/ql/tests/dml_tests.rs | 90 ++++++++++++-------------
 5 files changed, 66 insertions(+), 55 deletions(-)

diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 2c417c94..82a5d730 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -32,7 +32,7 @@
 use {
     super::{
         ast::Entity,
-        lexer::{Lit, Symbol, Token},
+        lexer::{Lit, LitIR, Symbol, Token},
         LangError, LangResult, RawSlice,
     },
     crate::{
@@ -77,13 +77,13 @@ fn process_entity(tok: &[Token], d: &mut MaybeInit<Entity>, i: &mut usize) -> bo
 #[derive(Debug, PartialEq)]
 pub struct RelationalExpr<'a> {
     pub(super) lhs: &'a [u8],
-    pub(super) rhs: &'a Lit,
+    pub(super) rhs: LitIR<'a>,
     pub(super) opc: u8,
 }
 
 impl<'a> RelationalExpr<'a> {
     #[inline(always)]
-    pub(super) fn new(lhs: &'a [u8], rhs: &'a Lit, opc: u8) -> Self {
+    pub(super) fn new(lhs: &'a [u8], rhs: LitIR<'a>, opc: u8) -> RelationalExpr<'a> {
         Self { lhs, rhs, opc }
     }
     pub(super) const OP_EQ: u8 = 1;
@@ -138,11 +138,11 @@ impl<'a> RelationalExpr<'a> {
         if compiler::likely(okay) {
             Some(unsafe {
                 // UNSAFE(@ohsayan): tok[0] is checked for being an ident, tok[lit_idx] also checked to be a lit
-                Self {
-                    lhs: extract!(tok[0], Token::Ident(ref id) => id.as_slice()),
-                    rhs: extract!(tok[lit_idx], Token::Lit(ref l) => l),
-                    opc: operator,
-                }
+                Self::new(
+                    extract!(tok[0], Token::Ident(ref id) => id.as_slice()),
+                    extract!(tok[lit_idx], Token::Lit(ref l) => l.as_ir()),
+                    operator,
+                )
             })
         } else {
             compiler::cold_val(None)
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 2a19efac..a10d6839 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -82,6 +82,18 @@ pub enum Lit {
     Bin(RawSlice),
 }
 
+impl Lit {
+    pub(super) unsafe fn as_ir<'a>(&'a self) -> LitIR<'a> {
+        match self {
+            Self::Str(s) => LitIR::Str(s.as_ref()),
+            Self::Bool(b) => LitIR::Bool(*b),
+            Self::UnsignedInt(u) => LitIR::UInt(*u),
+            Self::SignedInt(s) => LitIR::SInt(*s),
+            Self::Bin(b) => LitIR::Bin(b.as_slice()),
+        }
+    }
+}
+
 impl From<&'static str> for Lit {
     fn from(s: &'static str) -> Self {
         Self::Str(s.into())
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index f75ccbe6..f36431f5 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -321,6 +321,6 @@ macro_rules! statictbl {
     ($name:ident: $kind:ty => [$($expr:expr),*]) => {{
         const LEN: usize = {let mut i = 0;$(let _ = $expr; i += 1;)*i};
         static $name: [$kind; LEN] = [$($expr),*];
-        $name
+        &'static $name
     }};
 }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index c9352c75..93e54f06 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -46,6 +46,7 @@ mod schema_tests;
 mod structure_syn;
 
 /// Uses the [`InsecureLexer`] to lex the given input
+#[inline(always)]
 pub(super) fn lex_insecure(src: &[u8]) -> LangResult<Vec<Token>> {
     InsecureLexer::lex(src)
 }
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 8675e760..4e7e57d6 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -472,6 +472,7 @@ mod stmt_select {
         crate::engine::ql::{
             ast::Entity,
             dml::{self, SelectStatement},
+            lexer::LitIR,
         },
     };
     #[test]
@@ -483,14 +484,13 @@ mod stmt_select {
         )
         .unwrap();
         let r = dml::parse_select_full(&tok[1..]).unwrap();
-        let username_where = "sayan".into();
         let e = SelectStatement::new_test(
             Entity::Single("users".into()),
             [].to_vec(),
             true,
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
+                    "username".as_bytes(), LitIR::Str("username"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -505,14 +505,13 @@ mod stmt_select {
         )
         .unwrap();
         let r = dml::parse_select_full(&tok[1..]).unwrap();
-        let username_where = "sayan".into();
         let e = SelectStatement::new_test(
             Entity::Single("users".into()),
             ["field1".into()].to_vec(),
             false,
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
+                    "username".as_bytes(), LitIR::Str("username"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -527,14 +526,13 @@ mod stmt_select {
         )
         .unwrap();
         let r = dml::parse_select_full(&tok[1..]).unwrap();
-        let username_where = "sayan".into();
         let e = SelectStatement::new_test(
             Entity::Full("twitter".into(), "users".into()),
             ["field1".into()].to_vec(),
             false,
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
+                    "username".as_bytes(), LitIR::Str("username"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -549,14 +547,13 @@ mod stmt_select {
         )
         .unwrap();
         let r = dml::parse_select_full(&tok[1..]).unwrap();
-        let username_where = "sayan".into();
         let e = SelectStatement::new_test(
             Entity::Full("twitter".into(), "users".into()),
             ["field1".into(), "field2".into()].to_vec(),
             false,
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(), &username_where, RelationalExpr::OP_EQ
+                    "username".as_bytes(), LitIR::Str("sayan"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -645,6 +642,7 @@ mod update_statement {
             dml::{
                 self, AssignmentExpression, Operator, RelationalExpr, UpdateStatement, WhereClause,
             },
+            lexer::LitIR,
         },
     };
     #[test]
@@ -655,7 +653,6 @@ mod update_statement {
             "#,
         )
         .unwrap();
-        let where_username = "sayan".into();
         let note = "this is my new note".to_string().into();
         let r = dml::parse_update_full(&tok[1..]).unwrap();
         let e = UpdateStatement {
@@ -668,7 +665,7 @@ mod update_statement {
             wc: WhereClause::new(dict! {
                 "username".as_bytes() => RelationalExpr::new(
                     "username".as_bytes(),
-                    &where_username,
+                    LitIR::Str("sayan"),
                     RelationalExpr::OP_EQ
                 )
             }),
@@ -690,7 +687,6 @@ mod update_statement {
         )
         .unwrap();
         let r = dml::parse_update_full(&tok[1..]).unwrap();
-        let where_username = "sayan".into();
         let field_note = "this is my new note".into();
         let field_email = "sayan@example.com".into();
         let e = UpdateStatement {
@@ -702,7 +698,7 @@ mod update_statement {
             wc: WhereClause::new(dict! {
                 "username".as_bytes() => RelationalExpr::new(
                     "username".as_bytes(),
-                    &where_username,
+                    LitIR::Str("sayan"),
                     RelationalExpr::OP_EQ
                 )
             }),
@@ -717,6 +713,7 @@ mod delete_stmt {
         crate::engine::ql::{
             ast::Entity,
             dml::{self, DeleteStatement, RelationalExpr},
+            lexer::LitIR,
         },
     };
 
@@ -728,13 +725,12 @@ mod delete_stmt {
             "#,
         )
         .unwrap();
-        let primary_key = "sayan".into();
         let e = DeleteStatement::new_test(
             Entity::Single("users".into()),
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
                     "username".as_bytes(),
-                    &primary_key,
+                    LitIR::Str("sayan"),
                     RelationalExpr::OP_EQ
                 )
             },
@@ -750,13 +746,12 @@ mod delete_stmt {
             "#,
         )
         .unwrap();
-        let primary_key = "sayan".into();
         let e = DeleteStatement::new_test(
             ("twitter", "users").into(),
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
                     "username".as_bytes(),
-                    &primary_key,
+                    LitIR::Str("user"),
                     RelationalExpr::OP_EQ
                 )
             },
@@ -768,7 +763,10 @@ mod delete_stmt {
 mod relational_expr {
     use {
         super::*,
-        crate::engine::ql::dml::{self, RelationalExpr},
+        crate::engine::ql::{
+            dml::{self, RelationalExpr},
+            lexer::LitIR,
+        },
     };
 
     #[test]
@@ -778,7 +776,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: &(10.into()),
+                rhs: LitIR::UInt(10),
                 lhs: "primary_key".as_bytes(),
                 opc: RelationalExpr::OP_EQ
             }
@@ -791,7 +789,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: &(10.into()),
+                rhs: LitIR::UInt(10),
                 lhs: "primary_key".as_bytes(),
                 opc: RelationalExpr::OP_NE
             }
@@ -804,7 +802,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: &(10.into()),
+                rhs: LitIR::UInt(10),
                 lhs: "primary_key".as_bytes(),
                 opc: RelationalExpr::OP_GT
             }
@@ -817,7 +815,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: &(10.into()),
+                rhs: LitIR::UInt(10),
                 lhs: "primary_key".as_bytes(),
                 opc: RelationalExpr::OP_GE
             }
@@ -830,7 +828,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: &(10.into()),
+                rhs: LitIR::UInt(10),
                 lhs: "primary_key".as_bytes(),
                 opc: RelationalExpr::OP_LT
             }
@@ -842,18 +840,21 @@ mod relational_expr {
         let r = dml::parse_relexpr_full(&expr).unwrap();
         assert_eq!(
             r,
-            RelationalExpr {
-                rhs: &(10.into()),
-                lhs: "primary_key".as_bytes(),
-                opc: RelationalExpr::OP_LE
-            }
+            RelationalExpr::new(
+                "primary_key".as_bytes(),
+                LitIR::UInt(10),
+                RelationalExpr::OP_LE
+            )
         );
     }
 }
 mod where_clause {
     use {
         super::*,
-        crate::engine::ql::dml::{self, RelationalExpr, WhereClause},
+        crate::engine::ql::{
+            dml::{self, RelationalExpr, WhereClause},
+            lexer::LitIR,
+        },
     };
     #[test]
     fn where_single() {
@@ -863,13 +864,12 @@ mod where_clause {
             "#,
         )
         .unwrap();
-        let rhs_hundred = 100.into();
         let expected = WhereClause::new(dict! {
-            "x".as_bytes() => RelationalExpr {
-                rhs: &rhs_hundred,
-                lhs: "x".as_bytes(),
-                opc: RelationalExpr::OP_EQ
-            }
+            "x".as_bytes() => RelationalExpr::new(
+                "x".as_bytes(),
+                LitIR::UInt(100),
+                RelationalExpr::OP_EQ
+            )
         });
         assert_eq!(expected, dml::parse_where_clause_full(&tok).unwrap());
     }
@@ -881,19 +881,17 @@ mod where_clause {
             "#,
         )
         .unwrap();
-        let rhs_hundred = 100.into();
-        let rhs_password = "password".into();
         let expected = WhereClause::new(dict! {
-            "userid".as_bytes() => RelationalExpr {
-                rhs: &rhs_hundred,
-                lhs: "userid".as_bytes(),
-                opc: RelationalExpr::OP_EQ
-            },
-            "pass".as_bytes() => RelationalExpr {
-                rhs: &rhs_password,
-                lhs: "pass".as_bytes(),
-                opc: RelationalExpr::OP_EQ
-            }
+            "userid".as_bytes() => RelationalExpr::new(
+                "userid".as_bytes(),
+                LitIR::UInt(100),
+                RelationalExpr::OP_EQ
+            ),
+            "pass".as_bytes() => RelationalExpr::new(
+                "pass".as_bytes(),
+                LitIR::Str("password"),
+                RelationalExpr::OP_EQ
+            )
         });
         assert_eq!(expected, dml::parse_where_clause_full(&tok).unwrap());
     }

From 82f8cea5d1da8e40b9ecdeaf0b8500df576f3736 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 2 Jan 2023 23:13:32 -0800
Subject: [PATCH 060/310] Fix tests in `tests::dml_tests`

---
 server/src/engine/ql/lexer.rs           | 13 ++++++++++---
 server/src/engine/ql/tests/dml_tests.rs |  8 ++++----
 server/src/util/macros.rs               |  7 +++++++
 3 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index a10d6839..3fca09ab 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -56,12 +56,19 @@ impl PartialEq<Symbol> for Token {
     }
 }
 
+const SIZEOF_USIZE: usize = size_of::<usize>();
+const LT_SZ: usize = if is_64b!() {
+    size_of::<usize>() * 3
+} else {
+    size_of::<usize>() * 2
+};
+
 assertions! {
-    size_of::<Token>() == 24, // FIXME(@ohsayan): Damn, what?
+    size_of::<Token>() == LT_SZ,
     size_of::<Symbol>() == 1,
     size_of::<Keyword>() == 1,
-    size_of::<Lit>() == 24, // FIXME(@ohsayan): Ouch
-    size_of::<LitIR>() == 24,
+    size_of::<Lit>() == LT_SZ,
+    size_of::<LitIR>() == LT_SZ,
 }
 
 enum_impls! {
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 4e7e57d6..82b3b3f3 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -490,7 +490,7 @@ mod stmt_select {
             true,
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(), LitIR::Str("username"), RelationalExpr::OP_EQ
+                    "username".as_bytes(), LitIR::Str("sayan"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -511,7 +511,7 @@ mod stmt_select {
             false,
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(), LitIR::Str("username"), RelationalExpr::OP_EQ
+                    "username".as_bytes(), LitIR::Str("sayan"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -532,7 +532,7 @@ mod stmt_select {
             false,
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(), LitIR::Str("username"), RelationalExpr::OP_EQ
+                    "username".as_bytes(), LitIR::Str("sayan"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -751,7 +751,7 @@ mod delete_stmt {
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
                     "username".as_bytes(),
-                    LitIR::Str("user"),
+                    LitIR::Str("sayan"),
                     RelationalExpr::OP_EQ
                 )
             },
diff --git a/server/src/util/macros.rs b/server/src/util/macros.rs
index 3c37cc8c..0079e3da 100644
--- a/server/src/util/macros.rs
+++ b/server/src/util/macros.rs
@@ -269,3 +269,10 @@ macro_rules! bench {
         $vis mod $modname;
     };
 }
+
+#[macro_export]
+macro_rules! is_64b {
+    () => {
+        cfg!(target_pointer_width = "64")
+    };
+}

From 40b4c5042a86ab3a7272a6ad0146771a0e6393ca Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 3 Jan 2023 08:40:08 -0800
Subject: [PATCH 061/310] Define `QueryInterface`

---
 server/src/engine/ql/ast.rs   | 56 ++++++++++++++++++++++++++++++++++-
 server/src/engine/ql/lexer.rs |  2 +-
 2 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 410516b9..57c7afe9 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -27,13 +27,67 @@
 use {
     super::{
         ddl,
-        lexer::{InsecureLexer, Token},
+        lexer::{InsecureLexer, LitIR, Token},
         schema, LangError, LangResult, RawSlice,
     },
     crate::util::Life,
     core::{marker::PhantomData, slice},
 };
 
+/// A [`QueryInterface`] defines how a query is to be processed and how it is laid out
+pub trait QueryInterface {
+    /// Escaped data to be supplemented
+    type EscData<'a>;
+    /// Check if atleast one element of supplementary data is available
+    fn has_esc_data<'a>(d: Self::EscData<'a>) -> bool;
+    /// Attempt to read a lit instance from an appropriate source and return the available
+    /// lit using default substitution, i.e `0x3F`
+    fn read_append_lit_def_sub<'a>(
+        tok: &'a [Token],
+        esdt: Self::EscData<'a>,
+        v: &mut Vec<LitIR<'a>>,
+    ) -> bool;
+}
+
+pub struct InplaceQueryInterface;
+
+impl QueryInterface for InplaceQueryInterface {
+    type EscData<'a> = ();
+    #[inline(always)]
+    fn has_esc_data<'a>(_: ()) -> bool {
+        true
+    }
+    #[inline(always)]
+    fn read_append_lit_def_sub<'a>(tok: &'a [Token], _: (), v: &mut Vec<LitIR<'a>>) -> bool {
+        match tok[0] {
+            Token::Lit(ref l) => {
+                v.push(unsafe { l.as_ir() });
+                true
+            }
+            _ => false,
+        }
+    }
+}
+
+pub struct EscQueryInterface;
+
+impl QueryInterface for EscQueryInterface {
+    type EscData<'a> = &'a [LitIR<'a>];
+    #[inline(always)]
+    fn has_esc_data<'a>(d: Self::EscData<'a>) -> bool {
+        d.len() != 0
+    }
+    #[inline(always)]
+    fn read_append_lit_def_sub<'a>(
+        tok: &'a [Token],
+        d: Self::EscData<'a>,
+        v: &mut Vec<LitIR<'a>>,
+    ) -> bool {
+        v.push(d[0]);
+        tok[0] == Token![?]
+    }
+}
+
 /*
     AST
 */
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 3fca09ab..905edaab 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -916,7 +916,7 @@ where
     number
 }
 
-#[derive(PartialEq, Debug, Clone)]
+#[derive(PartialEq, Debug, Clone, Copy)]
 /// Intermediate literal repr
 pub enum LitIR<'a> {
     Str(&'a str),

From 3257798ee03bc75d87869def4b417f1be5ca8007 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 3 Jan 2023 19:09:36 -0800
Subject: [PATCH 062/310] Use `LitIR` throughout `dml`

---
 server/src/engine/ql/dml.rs             |  8 +++----
 server/src/engine/ql/tests/dml_tests.rs | 29 +++++++++++++++----------
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 82a5d730..87f4636f 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -32,7 +32,7 @@
 use {
     super::{
         ast::Entity,
-        lexer::{Lit, LitIR, Symbol, Token},
+        lexer::{LitIR, Symbol, Token},
         LangError, LangResult, RawSlice,
     },
     crate::{
@@ -676,13 +676,13 @@ pub struct AssignmentExpression<'a> {
     /// the LHS ident
     pub(super) lhs: RawSlice,
     /// the RHS lit
-    pub(super) rhs: &'a Lit,
+    pub(super) rhs: LitIR<'a>,
     /// operator
     pub(super) operator_fn: Operator,
 }
 
 impl<'a> AssignmentExpression<'a> {
-    pub(super) fn new(lhs: RawSlice, rhs: &'a Lit, operator_fn: Operator) -> Self {
+    pub(super) fn new(lhs: RawSlice, rhs: LitIR<'a>, operator_fn: Operator) -> Self {
         Self {
             lhs,
             rhs,
@@ -740,7 +740,7 @@ impl<'a> AssignmentExpression<'a> {
                 */
                 AssignmentExpression {
                     lhs: extract!(tok[0], Token::Ident(ref r) => r.clone()),
-                    rhs: extract!(tok[i], Token::Lit(ref l) => l),
+                    rhs: extract!(tok[i], Token::Lit(ref l) => l.as_ir()),
                     operator_fn: OPERATOR[operator_code as usize],
                 }
             };
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 82b3b3f3..7684d0e0 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -565,7 +565,7 @@ mod expression_tests {
         super::*,
         crate::engine::ql::{
             dml::{self, AssignmentExpression, Operator},
-            lexer::Lit,
+            lexer::LitIR,
         },
     };
     #[test]
@@ -576,7 +576,7 @@ mod expression_tests {
             r,
             AssignmentExpression {
                 lhs: "username".into(),
-                rhs: &Lit::Str("sayan".into()),
+                rhs: LitIR::Str("sayan"),
                 operator_fn: Operator::Assign
             }
         );
@@ -589,7 +589,7 @@ mod expression_tests {
             r,
             AssignmentExpression {
                 lhs: "followers".into(),
-                rhs: &(100.into()),
+                rhs: LitIR::UInt(100),
                 operator_fn: Operator::AddAssign
             }
         );
@@ -602,7 +602,7 @@ mod expression_tests {
             r,
             AssignmentExpression {
                 lhs: "following".into(),
-                rhs: &(150.into()),
+                rhs: LitIR::UInt(150),
                 operator_fn: Operator::SubAssign
             }
         );
@@ -615,7 +615,7 @@ mod expression_tests {
             r,
             AssignmentExpression {
                 lhs: "product_qty".into(),
-                rhs: &(2.into()),
+                rhs: LitIR::UInt(2),
                 operator_fn: Operator::MulAssign
             }
         );
@@ -628,7 +628,7 @@ mod expression_tests {
             r,
             AssignmentExpression {
                 lhs: "image_crop_factor".into(),
-                rhs: &(2.into()),
+                rhs: LitIR::UInt(2),
                 operator_fn: Operator::DivAssign
             }
         );
@@ -653,13 +653,12 @@ mod update_statement {
             "#,
         )
         .unwrap();
-        let note = "this is my new note".to_string().into();
         let r = dml::parse_update_full(&tok[1..]).unwrap();
         let e = UpdateStatement {
             entity: Entity::Single("app".into()),
             expressions: vec![AssignmentExpression {
                 lhs: "notes".into(),
-                rhs: &note,
+                rhs: LitIR::Str("this is my new note"),
                 operator_fn: Operator::AddAssign,
             }],
             wc: WhereClause::new(dict! {
@@ -687,13 +686,19 @@ mod update_statement {
         )
         .unwrap();
         let r = dml::parse_update_full(&tok[1..]).unwrap();
-        let field_note = "this is my new note".into();
-        let field_email = "sayan@example.com".into();
         let e = UpdateStatement {
             entity: ("jotsy", "app").into(),
             expressions: vec![
-                AssignmentExpression::new("notes".into(), &field_note, Operator::AddAssign),
-                AssignmentExpression::new("email".into(), &field_email, Operator::Assign),
+                AssignmentExpression::new(
+                    "notes".into(),
+                    LitIR::Str("this is my new note"),
+                    Operator::AddAssign,
+                ),
+                AssignmentExpression::new(
+                    "email".into(),
+                    LitIR::Str("sayan@example.com"),
+                    Operator::Assign,
+                ),
             ],
             wc: WhereClause::new(dict! {
                 "username".as_bytes() => RelationalExpr::new(

From ce8bba18cd3d6c0bdc26f43c5d05106743404528 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 5 Jan 2023 07:11:44 -0800
Subject: [PATCH 063/310] Add support for parameterization

---
 server/src/engine/memory/mod.rs |  14 ++-
 server/src/engine/ql/ast.rs     |  79 +++++++---------
 server/src/engine/ql/dml.rs     | 162 ++++++++++++++++++++------------
 3 files changed, 150 insertions(+), 105 deletions(-)

diff --git a/server/src/engine/memory/mod.rs b/server/src/engine/memory/mod.rs
index 6416dfe5..e6e768f1 100644
--- a/server/src/engine/memory/mod.rs
+++ b/server/src/engine/memory/mod.rs
@@ -26,7 +26,7 @@
 
 // TODO(@ohsayan): Change the underlying structures, there are just rudimentary ones used during integration with the QL
 
-use super::ql::lexer::Lit;
+use super::ql::lexer::{Lit, LitIR};
 
 /// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
 /// be of one type.
@@ -50,6 +50,8 @@ pub enum DataType {
     SignedInt(i64),
     /// A boolean
     Boolean(bool),
+    /// A float (64-bit)
+    Float(f64),
     /// A single-type list. Note, you **need** to keep up the invariant that the [`DataType`] disc. remains the same for all
     /// elements to ensure correctness in this specific context
     /// FIXME(@ohsayan): Try enforcing this somehow
@@ -81,6 +83,16 @@ impl DataType {
             Lit::Bin(l) => DataType::Binary(l.as_slice().to_owned()),
         }
     }
+    pub(super) fn clone_from_litir<'a>(lit: LitIR<'a>) -> Self {
+        match lit {
+            LitIR::Str(s) => Self::String(s.to_owned().into_boxed_str()),
+            LitIR::Bin(b) => Self::Binary(b.to_owned()),
+            LitIR::Float(f) => Self::Float(f),
+            LitIR::SInt(s) => Self::SignedInt(s),
+            LitIR::UInt(u) => Self::UnsignedInt(u),
+            LitIR::Bool(b) => Self::Boolean(b),
+        }
+    }
 }
 
 impl<const N: usize> From<[DataType; N]> for DataType {
diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 57c7afe9..46c3047d 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -34,57 +34,50 @@ use {
     core::{marker::PhantomData, slice},
 };
 
-/// A [`QueryInterface`] defines how a query is to be processed and how it is laid out
-pub trait QueryInterface {
-    /// Escaped data to be supplemented
-    type EscData<'a>;
-    /// Check if atleast one element of supplementary data is available
-    fn has_esc_data<'a>(d: Self::EscData<'a>) -> bool;
-    /// Attempt to read a lit instance from an appropriate source and return the available
-    /// lit using default substitution, i.e `0x3F`
-    fn read_append_lit_def_sub<'a>(
-        tok: &'a [Token],
-        esdt: Self::EscData<'a>,
-        v: &mut Vec<LitIR<'a>>,
-    ) -> bool;
+pub trait QueryData<'a> {
+    /// Check if the given token is a lit, while also checking `self`'s data if necessary
+    fn can_read_lit_from(&self, tok: &Token) -> bool;
+    /// Read a lit using the given token, using `self`'s data as necessary
+    ///
+    /// ## Safety
+    /// The current token **must match** the signature of a lit
+    unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a>;
 }
 
-pub struct InplaceQueryInterface;
+pub struct InplaceData;
+impl InplaceData {
+    pub const fn new() -> Self {
+        Self
+    }
+}
 
-impl QueryInterface for InplaceQueryInterface {
-    type EscData<'a> = ();
-    #[inline(always)]
-    fn has_esc_data<'a>(_: ()) -> bool {
-        true
+impl<'a> QueryData<'a> for InplaceData {
+    fn can_read_lit_from(&self, tok: &Token) -> bool {
+        tok.is_lit()
     }
-    #[inline(always)]
-    fn read_append_lit_def_sub<'a>(tok: &'a [Token], _: (), v: &mut Vec<LitIR<'a>>) -> bool {
-        match tok[0] {
-            Token::Lit(ref l) => {
-                v.push(unsafe { l.as_ir() });
-                true
-            }
-            _ => false,
-        }
+    unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a> {
+        extract!(tok, Token::Lit(l) => l.as_ir())
     }
 }
 
-pub struct EscQueryInterface;
-
-impl QueryInterface for EscQueryInterface {
-    type EscData<'a> = &'a [LitIR<'a>];
-    #[inline(always)]
-    fn has_esc_data<'a>(d: Self::EscData<'a>) -> bool {
-        d.len() != 0
+pub struct SubstitutedData<'a> {
+    data: &'a [LitIR<'a>],
+}
+impl<'a> SubstitutedData<'a> {
+    pub const fn new(src: &'a [LitIR<'a>]) -> Self {
+        Self { data: src }
     }
-    #[inline(always)]
-    fn read_append_lit_def_sub<'a>(
-        tok: &'a [Token],
-        d: Self::EscData<'a>,
-        v: &mut Vec<LitIR<'a>>,
-    ) -> bool {
-        v.push(d[0]);
-        tok[0] == Token![?]
+}
+
+impl<'a> QueryData<'a> for SubstitutedData<'a> {
+    fn can_read_lit_from(&self, tok: &Token) -> bool {
+        Token![?].eq(tok) && !self.data.is_empty()
+    }
+    unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a> {
+        debug_assert!(Token![?].eq(tok));
+        let ret = self.data[0];
+        self.data = &self.data[1..];
+        ret
     }
 }
 
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index 87f4636f..d12f961d 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -29,9 +29,11 @@
     should augment in future revisions of the QL engine
 */
 
+#[cfg(test)]
+use super::ast::InplaceData;
 use {
     super::{
-        ast::Entity,
+        ast::{Entity, QueryData},
         lexer::{LitIR, Symbol, Token},
         LangError, LangResult, RawSlice,
     },
@@ -46,6 +48,11 @@ use {
     },
 };
 
+#[inline(always)]
+fn minidx<T>(src: &[T], index: usize) -> usize {
+    cmp::min(src.len() - 1, index)
+}
+
 /*
     Misc
 */
@@ -117,7 +124,7 @@ impl<'a> RelationalExpr<'a> {
         opc
     }
     #[inline(always)]
-    fn try_parse(tok: &'a [Token], cnt: &mut usize) -> Option<Self> {
+    fn try_parse<Qd: QueryData<'a>>(tok: &'a [Token], d: &mut Qd, cnt: &mut usize) -> Option<Self> {
         /*
             Minimum length of an expression:
             [lhs] [operator] [rhs]
@@ -132,18 +139,19 @@ impl<'a> RelationalExpr<'a> {
         // let's get ourselves the operator
         let operator = Self::parse_operator(&tok[i..], &mut i, &mut okay);
         okay &= i < tok.len();
-        let lit_idx = cmp::min(i, tok.len() - 1);
-        okay &= tok[lit_idx].is_lit(); // LOL, I really like saving cycles
+        let lit_idx = minidx(tok, i);
+        okay &= Qd::can_read_lit_from(d, &tok[lit_idx]); // LOL, I really like saving cycles
         *cnt += i + okay as usize;
         if compiler::likely(okay) {
-            Some(unsafe {
+            unsafe {
                 // UNSAFE(@ohsayan): tok[0] is checked for being an ident, tok[lit_idx] also checked to be a lit
-                Self::new(
+                let lit = Qd::read_lit(d, &tok[lit_idx]);
+                Some(Self::new(
                     extract!(tok[0], Token::Ident(ref id) => id.as_slice()),
-                    extract!(tok[lit_idx], Token::Lit(ref l) => l.as_ir()),
+                    lit,
                     operator,
-                )
-            })
+                ))
+            }
         } else {
             compiler::cold_val(None)
         }
@@ -168,8 +176,9 @@ impl<'a> WhereClause<'a> {
     /// Notes:
     /// - Deny duplicate clauses
     /// - No enforcement on minimum number of clauses
-    fn parse_where_and_append_to(
+    fn parse_where_and_append_to<Qd: QueryData<'a>>(
         tok: &'a [Token],
+        d: &mut Qd,
         cnt: &mut usize,
         c: &mut WhereClauseCollection<'a>,
     ) -> bool {
@@ -178,10 +187,10 @@ impl<'a> WhereClause<'a> {
         let mut i = 0;
         let mut has_more = true;
         while okay && i < l && has_more {
-            okay &= RelationalExpr::try_parse(&tok[i..], &mut i)
+            okay &= RelationalExpr::try_parse(&tok[i..], d, &mut i)
                 .map(|clause| c.insert(clause.lhs, clause).is_none())
                 .unwrap_or(false);
-            has_more = tok[cmp::min(i, l - 1)] == Token![and] && i < l;
+            has_more = tok[minidx(tok, i)] == Token![and] && i < l;
             i += has_more as usize;
         }
         *cnt += i;
@@ -192,9 +201,14 @@ impl<'a> WhereClause<'a> {
     ///
     /// Notes:
     /// - Enforce a minimum of 1 clause
-    pub(super) fn parse_where(tok: &'a [Token], flag: &mut bool, cnt: &mut usize) -> Self {
+    pub(super) fn parse_where<Qd: QueryData<'a>>(
+        tok: &'a [Token],
+        d: &mut Qd,
+        flag: &mut bool,
+        cnt: &mut usize,
+    ) -> Self {
         let mut c = HashMap::with_capacity(2);
-        *flag &= Self::parse_where_and_append_to(tok, cnt, &mut c);
+        *flag &= Self::parse_where_and_append_to(tok, d, cnt, &mut c);
         *flag &= !c.is_empty();
         Self { c }
     }
@@ -204,7 +218,7 @@ impl<'a> WhereClause<'a> {
 pub(super) fn parse_where_clause_full<'a>(tok: &'a [Token]) -> Option<WhereClause<'a>> {
     let mut flag = true;
     let mut i = 0;
-    let ret = WhereClause::parse_where(tok, &mut flag, &mut i);
+    let ret = WhereClause::parse_where(tok, &mut InplaceData::new(), &mut flag, &mut i);
     assert_full_tt!(tok.len(), i);
     flag.then_some(ret)
 }
@@ -213,7 +227,7 @@ pub(super) fn parse_where_clause_full<'a>(tok: &'a [Token]) -> Option<WhereClaus
 #[inline(always)]
 pub(super) fn parse_relexpr_full<'a>(tok: &'a [Token]) -> Option<RelationalExpr<'a>> {
     let mut i = 0;
-    let okay = RelationalExpr::try_parse(tok, &mut i);
+    let okay = RelationalExpr::try_parse(tok, &mut InplaceData::new(), &mut i);
     assert_full_tt!(tok.len(), i);
     okay
 }
@@ -225,8 +239,9 @@ pub(super) fn parse_relexpr_full<'a>(tok: &'a [Token]) -> Option<RelationalExpr<
 /// Parse a list
 ///
 /// **NOTE:** This function will error if the `[` token is passed. Make sure this is forwarded by the caller
-pub(super) fn parse_list(
-    tok: &[Token],
+pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
+    tok: &'a [Token],
+    d: &mut Qd,
     list: &mut Vec<DataType>,
 ) -> (Option<Discriminant<DataType>>, usize, bool) {
     let l = tok.len();
@@ -237,22 +252,23 @@ pub(super) fn parse_list(
     let mut prev_nlist_dscr = None;
     while i < l && okay && !stop {
         let d = match &tok[i] {
-            Token::Lit(l) => unsafe {
-                // UNSAFE(@ohsayan): Token LT0 guarantees LT0 > LT1 for lit
-                DataType::clone_from_lit(l)
-            },
+            tok if Qd::can_read_lit_from(d, tok) => {
+                unsafe {
+                    // UNSAFE(@ohsayan): Token LT0 guarantees LT0 > LT1 for lit
+                    DataType::clone_from_litir(Qd::read_lit(d, tok))
+                }
+            }
             Token::Symbol(Symbol::TtOpenSqBracket) => {
                 // a nested list
                 let mut nested_list = Vec::new();
-                let (nlist_dscr, nlist_i, nlist_okay) = parse_list(&tok[i + 1..], &mut nested_list);
+                let (nlist_dscr, nlist_i, nlist_okay) =
+                    parse_list(&tok[i + 1..], d, &mut nested_list);
                 okay &= nlist_okay;
                 i += nlist_i;
                 // check type return
-                okay &= {
-                    prev_nlist_dscr.is_none()
-                        || nlist_dscr.is_none()
-                        || prev_nlist_dscr == nlist_dscr
-                };
+                okay &= prev_nlist_dscr.is_none()
+                    || nlist_dscr.is_none()
+                    || prev_nlist_dscr == nlist_dscr;
                 if prev_nlist_dscr.is_none() && nlist_dscr.is_some() {
                     prev_nlist_dscr = nlist_dscr;
                 }
@@ -279,7 +295,7 @@ pub(super) fn parse_list(
 #[cfg(test)]
 pub(super) fn parse_list_full(tok: &[Token]) -> Option<Vec<DataType>> {
     let mut l = Vec::new();
-    if matches!(parse_list(tok, &mut l), (_, i, true) if i == tok.len()) {
+    if matches!(parse_list(tok, &mut InplaceData::new(), &mut l), (_, i, true) if i == tok.len()) {
         Some(l)
     } else {
         None
@@ -289,7 +305,10 @@ pub(super) fn parse_list_full(tok: &[Token]) -> Option<Vec<DataType>> {
 /// Parse the tuple data passed in with an insert query.
 ///
 /// **Note:** Make sure you pass the `(` token
-pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<Option<DataType>>, usize, bool) {
+pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
+    tok: &'a [Token],
+    d: &mut Qd,
+) -> (Vec<Option<DataType>>, usize, bool) {
     let l = tok.len();
     let mut okay = l != 0;
     let mut stop = okay && tok[0] == Token::Symbol(Symbol::TtCloseParen);
@@ -297,14 +316,16 @@ pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<Option<DataType>>,
     let mut data = Vec::new();
     while i < l && okay && !stop {
         match &tok[i] {
-            Token::Lit(l) => data.push(Some(unsafe {
-                // UNSAFE(@ohsayan): Token LT0 guarantees LT0 > LT1 for lit
-                DataType::clone_from_lit(l)
-            })),
+            tok if Qd::can_read_lit_from(d, tok) => {
+                unsafe {
+                    // UNSAFE(@ohsayan): Token LT0 guarantees LT0 > LT1 for lit
+                    data.push(Some(DataType::clone_from_litir(Qd::read_lit(d, tok))));
+                }
+            }
             Token::Symbol(Symbol::TtOpenSqBracket) => {
                 // ah, a list
                 let mut l = Vec::new();
-                let (_, lst_i, lst_okay) = parse_list(&tok[i + 1..], &mut l);
+                let (_, lst_i, lst_okay) = parse_list(&tok[i + 1..], d, &mut l);
                 data.push(Some(l.into()));
                 i += lst_i;
                 okay &= lst_okay;
@@ -329,7 +350,7 @@ pub(super) fn parse_data_tuple_syntax(tok: &[Token]) -> (Vec<Option<DataType>>,
 
 #[cfg(test)]
 pub(super) fn parse_data_tuple_syntax_full(tok: &[Token]) -> Option<Vec<Option<DataType>>> {
-    let (ret, cnt, okay) = parse_data_tuple_syntax(tok);
+    let (ret, cnt, okay) = parse_data_tuple_syntax(tok, &mut InplaceData::new());
     assert!(cnt == tok.len(), "didn't use full length");
     if okay {
         Some(ret)
@@ -338,8 +359,9 @@ pub(super) fn parse_data_tuple_syntax_full(tok: &[Token]) -> Option<Vec<Option<D
     }
 }
 
-pub(super) fn parse_data_map_syntax<'a>(
+pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
     tok: &'a [Token],
+    d: &mut Qd,
 ) -> (HashMap<&'a [u8], Option<DataType>>, usize, bool) {
     let l = tok.len();
     let mut okay = l != 0;
@@ -350,7 +372,7 @@ pub(super) fn parse_data_map_syntax<'a>(
         let (field, colon, expression) = (&tok[i], &tok[i + 1], &tok[i + 2]);
         okay &= colon == &Symbol::SymColon;
         match (field, expression) {
-            (Token::Ident(id), Token::Lit(l)) => {
+            (Token::Ident(id), tok) if Qd::can_read_lit_from(d, tok) => {
                 okay &= data
                     .insert(
                         unsafe {
@@ -359,7 +381,7 @@ pub(super) fn parse_data_map_syntax<'a>(
                         },
                         Some(unsafe {
                             // UNSAFE(@ohsayan): Token LT0 guarantees LT0 > LT1 for lit
-                            DataType::clone_from_lit(l)
+                            DataType::clone_from_litir(Qd::read_lit(d, tok))
                         }),
                     )
                     .is_none();
@@ -367,7 +389,7 @@ pub(super) fn parse_data_map_syntax<'a>(
             (Token::Ident(id), Token::Symbol(Symbol::TtOpenSqBracket)) => {
                 // ooh a list
                 let mut l = Vec::new();
-                let (_, lst_i, lst_ok) = parse_list(&tok[i + 3..], &mut l);
+                let (_, lst_i, lst_ok) = parse_list(&tok[i + 3..], d, &mut l);
                 okay &= lst_ok;
                 i += lst_i;
                 okay &= data
@@ -410,7 +432,7 @@ pub(super) fn parse_data_map_syntax<'a>(
 pub(super) fn parse_data_map_syntax_full(
     tok: &[Token],
 ) -> Option<HashMap<Box<str>, Option<DataType>>> {
-    let (dat, i, ok) = parse_data_map_syntax(tok);
+    let (dat, i, ok) = parse_data_map_syntax(tok, &mut InplaceData::new());
     assert!(i == tok.len(), "didn't use full length");
     if ok {
         Some(
@@ -469,8 +491,9 @@ fn parse_entity(tok: &[Token], entity: &mut MaybeInit<Entity>, i: &mut usize) ->
     is_full | is_half
 }
 
-pub(super) fn parse_insert<'a>(
+pub(super) fn parse_insert<'a, Qd: QueryData<'a>>(
     tok: &'a [Token],
+    d: &mut Qd,
     counter: &mut usize,
 ) -> LangResult<InsertStatement<'a>> {
     /*
@@ -495,13 +518,13 @@ pub(super) fn parse_insert<'a>(
     }
     match tok[i] {
         Token![() open] => {
-            let (this_data, incr, ok) = parse_data_tuple_syntax(&tok[i + 1..]);
+            let (this_data, incr, ok) = parse_data_tuple_syntax(&tok[i + 1..], d);
             okay &= ok;
             i += incr + 1;
             data = Some(InsertData::Ordered(this_data));
         }
         Token![open {}] => {
-            let (this_data, incr, ok) = parse_data_map_syntax(&tok[i + 1..]);
+            let (this_data, incr, ok) = parse_data_map_syntax(&tok[i + 1..], d);
             okay &= ok;
             i += incr + 1;
             data = Some(InsertData::Map(this_data));
@@ -529,7 +552,7 @@ pub(super) fn parse_insert<'a>(
 #[cfg(test)]
 pub(super) fn parse_insert_full<'a>(tok: &'a [Token]) -> Option<InsertStatement<'a>> {
     let mut z = 0;
-    let s = self::parse_insert(tok, &mut z);
+    let s = self::parse_insert(tok, &mut InplaceData::new(), &mut z);
     assert!(z == tok.len(), "didn't use full length");
     s.ok()
 }
@@ -577,8 +600,9 @@ impl<'a> SelectStatement<'a> {
 
 /// Parse a `select` query. The cursor should have already passed the `select` token when this
 /// function is called.
-pub(super) fn parse_select<'a>(
+pub(super) fn parse_select<'a, Qd: QueryData<'a>>(
     tok: &'a [Token],
+    d: &mut Qd,
     counter: &mut usize,
 ) -> LangResult<SelectStatement<'a>> {
     /*
@@ -604,7 +628,7 @@ pub(super) fn parse_select<'a>(
             }
         }
         i += 1;
-        let nx_idx = cmp::min(i, l);
+        let nx_idx = minidx(tok, i);
         let nx_comma = tok[nx_idx] == Token![,] && i < l;
         let nx_from = tok[nx_idx] == Token![from];
         okay &= nx_comma | nx_from;
@@ -620,11 +644,11 @@ pub(super) fn parse_select<'a>(
     // now process entity
     let mut entity = MaybeInit::uninit();
     okay &= process_entity(&tok[i..], &mut entity, &mut i);
-    let has_where = tok[cmp::min(i, l)] == Token![where];
+    let has_where = tok[minidx(tok, i)] == Token![where];
     i += has_where as usize;
     let mut clauses = <_ as Default>::default();
     if has_where {
-        okay &= WhereClause::parse_where_and_append_to(&tok[i..], &mut i, &mut clauses);
+        okay &= WhereClause::parse_where_and_append_to(&tok[i..], d, &mut i, &mut clauses);
         okay &= !clauses.is_empty(); // append doesn't enforce clause arity
     }
     *counter += i;
@@ -647,7 +671,7 @@ pub(super) fn parse_select<'a>(
 /// **test-mode only** parse for a `select` where the full token stream is exhausted
 pub(super) fn parse_select_full<'a>(tok: &'a [Token]) -> Option<SelectStatement<'a>> {
     let mut i = 0;
-    let r = self::parse_select(tok, &mut i);
+    let r = self::parse_select(tok, &mut InplaceData::new(), &mut i);
     assert_full_tt!(i, tok.len());
     r.ok()
 }
@@ -692,8 +716,9 @@ impl<'a> AssignmentExpression<'a> {
     /// Attempt to parse an expression and then append it to the given vector of expressions. This will return `true`
     /// if the expression was parsed correctly, otherwise `false` is returned
     #[inline(always)]
-    fn parse_and_append_expression(
+    fn parse_and_append_expression<Qd: QueryData<'a>>(
         tok: &'a [Token],
+        d: &mut Qd,
         expressions: &mut Vec<Self>,
         counter: &mut usize,
     ) -> bool {
@@ -728,7 +753,7 @@ impl<'a> AssignmentExpression<'a> {
         okay &= single_assign_okay | double_assign_okay;
         i += double_assign_okay as usize; // skip on <op>assign
 
-        let has_rhs = i < l && tok[i].is_lit();
+        let has_rhs = Qd::can_read_lit_from(d, &tok[minidx(tok, i)]);
         okay &= has_rhs;
         *counter += i + has_rhs as usize;
 
@@ -738,9 +763,10 @@ impl<'a> AssignmentExpression<'a> {
                    UNSAFE(@ohsayan): tok[0] is checked for being an ident early on; second, tok[i]
                    is also checked for being a lit and then `okay` ensures correctness
                 */
+                let rhs = Qd::read_lit(d, &tok[i]);
                 AssignmentExpression {
                     lhs: extract!(tok[0], Token::Ident(ref r) => r.clone()),
-                    rhs: extract!(tok[i], Token::Lit(ref l) => l.as_ir()),
+                    rhs,
                     operator_fn: OPERATOR[operator_code as usize],
                 }
             };
@@ -755,7 +781,12 @@ impl<'a> AssignmentExpression<'a> {
 pub(super) fn parse_expression_full<'a>(tok: &'a [Token]) -> Option<AssignmentExpression<'a>> {
     let mut i = 0;
     let mut exprs = Vec::new();
-    if AssignmentExpression::parse_and_append_expression(tok, &mut exprs, &mut i) {
+    if AssignmentExpression::parse_and_append_expression(
+        tok,
+        &mut InplaceData::new(),
+        &mut exprs,
+        &mut i,
+    ) {
         assert_full_tt!(i, tok.len());
         Some(exprs.remove(0))
     } else {
@@ -797,7 +828,11 @@ impl<'a> UpdateStatement<'a> {
         }
     }
     #[inline(always)]
-    pub(super) fn parse_update(tok: &'a [Token], counter: &mut usize) -> LangResult<Self> {
+    pub(super) fn parse_update<Qd: QueryData<'a>>(
+        tok: &'a [Token],
+        d: &mut Qd,
+        counter: &mut usize,
+    ) -> LangResult<Self> {
         /*
             TODO(@ohsayan): Allow volcanoes
             smallest tt:
@@ -824,10 +859,11 @@ impl<'a> UpdateStatement<'a> {
         while i < l && okay && !nx_where {
             okay &= AssignmentExpression::parse_and_append_expression(
                 &tok[i..],
+                d,
                 &mut expressions,
                 &mut i,
             );
-            let nx_idx = cmp::min(i, l);
+            let nx_idx = minidx(tok, i);
             let nx_comma = tok[nx_idx] == Token![,] && i < l;
             // NOTE: volcano
             nx_where = tok[nx_idx] == Token![where] && i < l;
@@ -838,7 +874,7 @@ impl<'a> UpdateStatement<'a> {
         i += okay as usize;
         // now process expressions
         let mut clauses = <_ as Default>::default();
-        okay &= WhereClause::parse_where_and_append_to(&tok[i..], &mut i, &mut clauses);
+        okay &= WhereClause::parse_where_and_append_to(&tok[i..], d, &mut i, &mut clauses);
         okay &= !clauses.is_empty(); // NOTE: volcano
         *counter += i;
         if okay {
@@ -859,7 +895,7 @@ impl<'a> UpdateStatement<'a> {
 #[cfg(test)]
 pub(super) fn parse_update_full<'a>(tok: &'a [Token]) -> LangResult<UpdateStatement<'a>> {
     let mut i = 0;
-    let r = UpdateStatement::parse_update(tok, &mut i);
+    let r = UpdateStatement::parse_update(tok, &mut InplaceData::new(), &mut i);
     assert_full_tt!(i, tok.len());
     r
 }
@@ -887,7 +923,11 @@ impl<'a> DeleteStatement<'a> {
     pub(super) fn new_test(entity: Entity, wc: WhereClauseCollection<'a>) -> Self {
         Self::new(entity, WhereClause::new(wc))
     }
-    pub(super) fn parse_delete(tok: &'a [Token], counter: &mut usize) -> LangResult<Self> {
+    pub(super) fn parse_delete<Qd: QueryData<'a>>(
+        tok: &'a [Token],
+        d: &mut Qd,
+        counter: &mut usize,
+    ) -> LangResult<Self> {
         /*
             TODO(@ohsayan): Volcano
             smallest tt:
@@ -912,7 +952,7 @@ impl<'a> DeleteStatement<'a> {
         okay &= tok[i] == Token![where]; // NOTE: volcano
         i += 1; // skip even if incorrect
         let mut clauses = <_ as Default>::default();
-        okay &= WhereClause::parse_where_and_append_to(&tok[i..], &mut i, &mut clauses);
+        okay &= WhereClause::parse_where_and_append_to(&tok[i..], d, &mut i, &mut clauses);
         okay &= !clauses.is_empty();
         *counter += i;
         if okay {
@@ -932,7 +972,7 @@ impl<'a> DeleteStatement<'a> {
 #[cfg(test)]
 pub(super) fn parse_delete_full<'a>(tok: &'a [Token]) -> LangResult<DeleteStatement<'a>> {
     let mut i = 0_usize;
-    let r = DeleteStatement::parse_delete(tok, &mut i);
+    let r = DeleteStatement::parse_delete(tok, &mut InplaceData::new(), &mut i);
     assert_full_tt!(i, tok.len());
     r
 }

From 16fb7a7d1b4ad80da6a175495f34711612ed9d5d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 5 Jan 2023 09:01:56 -0800
Subject: [PATCH 064/310] Support parameters in DDL queries

---
 server/src/engine/ql/ast.rs                 |  11 +-
 server/src/engine/ql/lexer.rs               |  27 ++-
 server/src/engine/ql/schema.rs              | 231 ++++++++++++++------
 server/src/engine/ql/tests.rs               |   4 +-
 server/src/engine/ql/tests/schema_tests.rs  | 192 ++++++++++------
 server/src/engine/ql/tests/structure_syn.rs |   6 +-
 6 files changed, 322 insertions(+), 149 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 46c3047d..20580879 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -324,7 +324,8 @@ impl<'a> Compiler<'a> {
     /// Alter model
     fn alter_model(&mut self) -> Result<Statement, LangError> {
         let mut c = 0;
-        let r = schema::parse_alter_kind_from_tokens(self.remslice(), &mut c);
+        let r =
+            schema::parse_alter_kind_from_tokens(self.remslice(), &mut InplaceData::new(), &mut c);
         unsafe {
             self.incr_cursor_by(c);
         }
@@ -333,7 +334,8 @@ impl<'a> Compiler<'a> {
     #[inline(always)]
     /// Alter space
     fn alter_space(&mut self) -> Result<Statement, LangError> {
-        let (alter, i) = schema::parse_alter_space_from_tokens(self.remslice())?;
+        let (alter, i) =
+            schema::parse_alter_space_from_tokens(self.remslice(), &mut InplaceData::new())?;
         unsafe {
             self.incr_cursor_by(i);
         }
@@ -358,7 +360,8 @@ impl<'a> Compiler<'a> {
     #[inline(always)]
     /// Create model
     fn c_model0(&mut self) -> Result<Statement, LangError> {
-        let (model, i) = schema::parse_schema_from_tokens(self.remslice())?;
+        let (model, i) =
+            schema::parse_schema_from_tokens(self.remslice(), &mut InplaceData::new())?;
         unsafe {
             self.incr_cursor_by(i);
         }
@@ -367,7 +370,7 @@ impl<'a> Compiler<'a> {
     #[inline(always)]
     /// Create space
     fn c_space0(&mut self) -> Result<Statement, LangError> {
-        let (space, i) = schema::parse_space_from_tokens(self.remslice())?;
+        let (space, i) = schema::parse_space_from_tokens(self.remslice(), &mut InplaceData::new())?;
         unsafe {
             self.incr_cursor_by(i);
         }
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 905edaab..1e1b7b90 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -24,12 +24,10 @@
  *
 */
 
-use std::ops::BitOr;
-
 use {
     super::{LangError, LangResult, RawSlice},
     crate::util::compiler,
-    core::{cmp, fmt, marker::PhantomData, mem::size_of, slice, str},
+    core::{cmp, fmt, marker::PhantomData, mem::size_of, ops::BitOr, slice, str},
 };
 
 /*
@@ -927,6 +925,29 @@ pub enum LitIR<'a> {
     Float(f64),
 }
 
+impl<'a> LitIR<'a> {
+    pub fn to_litir_owned(&self) -> LitIROwned {
+        match self {
+            Self::Str(s) => LitIROwned::Str(s.to_string().into_boxed_str()),
+            Self::Bin(b) => LitIROwned::Bin(b.to_vec().into_boxed_slice()),
+            Self::UInt(u) => LitIROwned::UInt(*u),
+            Self::SInt(s) => LitIROwned::SInt(*s),
+            Self::Bool(b) => LitIROwned::Bool(*b),
+            Self::Float(f) => LitIROwned::Float(*f),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum LitIROwned {
+    Str(Box<str>),
+    Bin(Box<[u8]>),
+    UInt(u64),
+    SInt(i64),
+    Bool(bool),
+    Float(f64),
+}
+
 #[derive(Debug, PartialEq)]
 /// Data constructed from `opmode-safe`
 pub struct SafeQueryData<'a> {
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 4c62e4a0..a5e69700 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -46,13 +46,17 @@
 
 use {
     super::{
-        lexer::{Lit, Symbol, Token},
+        ast::QueryData,
+        lexer::{LitIR, LitIROwned, Symbol, Token},
         LangError, LangResult, RawSlice,
     },
     crate::util::MaybeInit,
     std::collections::{HashMap, HashSet},
 };
 
+#[cfg(test)]
+use crate::engine::ql::ast::InplaceData;
+
 /*
     Meta
 */
@@ -62,7 +66,7 @@ use {
 /// **DO NOT** construct states manually
 macro_rules! states {
     ($(#[$attr:meta])+$vis:vis struct $stateid:ident: $statebase:ty {$($(#[$tyattr:meta])*$v:vis$state:ident = $statexp:expr),+ $(,)?}) => {
-        #[::core::prelude::v1::derive(::core::cmp::PartialEq, ::core::cmp::Eq, ::core::clone::Clone, ::core::marker::Copy)]
+        #[::core::prelude::v1::derive(::core::cmp::PartialEq,::core::cmp::Eq,::core::clone::Clone,::core::marker::Copy)]
         $(#[$attr])+$vis struct $stateid {__base: $statebase}
         impl $stateid {$($(#[$tyattr])*$v const $state:Self=$stateid{__base: $statexp,};)*}
         impl ::core::fmt::Debug for $stateid {
@@ -86,13 +90,13 @@ const ALLOW_RESET_SYNTAX: bool = true;
 #[derive(Debug, PartialEq)]
 /// A dictionary entry type. Either a literal or another dictionary
 pub enum DictEntry {
-    Lit(Lit),
+    Lit(LitIROwned),
     Map(Dict),
 }
 
-impl From<Lit> for DictEntry {
-    fn from(l: Lit) -> Self {
-        Self::Lit(l)
+impl<'a> From<LitIR<'a>> for DictEntry {
+    fn from(l: LitIR<'a>) -> Self {
+        Self::Lit(l.to_litir_owned())
     }
 }
 
@@ -221,7 +225,12 @@ states! {
 }
 
 /// Fold a dictionary
-pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dict) -> u64 {
+pub(super) fn rfold_dict<'a, Qd: QueryData<'a>>(
+    mut state: DictFoldState,
+    tok: &'a [Token],
+    d: &mut Qd,
+    dict: &mut Dict,
+) -> u64 {
     /*
         NOTE: Assume rules wherever applicable
 
@@ -263,15 +272,17 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
                 i += 1;
                 state = DictFoldState::LIT_OR_OB;
             }
-            (Token::Lit(l), DictFoldState::LIT_OR_OB) => {
+            (tok, DictFoldState::LIT_OR_OB) if Qd::can_read_lit_from(d, tok) => {
                 i += 1;
                 // found literal; so push in k/v pair and then expect a comma or close brace
-                okay &= dict
-                    .insert(
-                        unsafe { tmp.assume_init_ref() }.to_string(),
-                        Some(l.clone().into()),
-                    )
-                    .is_none();
+                unsafe {
+                    okay &= dict
+                        .insert(
+                            tmp.assume_init_ref().to_string(),
+                            Some(Qd::read_lit(d, tok).into()),
+                        )
+                        .is_none();
+                }
                 state = DictFoldState::COMMA_OR_CB;
             }
             (Token![null], DictFoldState::LIT_OR_OB) => {
@@ -292,7 +303,7 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
                 i += 1;
                 // we found an open brace, so this is a dict
                 let mut new_dict = Dict::new();
-                let ret = rfold_dict(DictFoldState::CB_OR_IDENT, &tok[i..], &mut new_dict);
+                let ret = rfold_dict(DictFoldState::CB_OR_IDENT, &tok[i..], d, &mut new_dict);
                 okay &= ret & HIBIT == HIBIT;
                 i += (ret & !HIBIT) as usize;
                 okay &= dict
@@ -318,7 +329,7 @@ pub(super) fn rfold_dict(mut state: DictFoldState, tok: &[Token], dict: &mut Dic
 /// Fold a dictionary (**test-only**)
 pub fn fold_dict(tok: &[Token]) -> Option<Dict> {
     let mut d = Dict::new();
-    let r = rfold_dict(DictFoldState::OB, tok, &mut d);
+    let r = rfold_dict(DictFoldState::OB, tok, &mut InplaceData::new(), &mut d);
     if r & HIBIT == HIBIT {
         Some(d)
     } else {
@@ -418,9 +429,10 @@ impl TyMetaFoldResult {
 }
 
 /// Fold type metadata (flag setup dependent on caller)
-pub(super) fn rfold_tymeta<const ALLOW_RESET: bool>(
+pub(super) fn rfold_tymeta<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
     mut state: TyMetaFoldState,
-    tok: &[Token],
+    tok: &'a [Token],
+    d: &mut Qd,
     dict: &mut Dict,
 ) -> TyMetaFoldResult {
     let l = tok.len();
@@ -463,15 +475,17 @@ pub(super) fn rfold_tymeta<const ALLOW_RESET: bool>(
                 // we just saw a colon. now we want a literal or openbrace
                 state = TyMetaFoldState::LIT_OR_OB;
             }
-            (Token::Lit(lit), TyMetaFoldState::LIT_OR_OB) => {
+            (tok, TyMetaFoldState::LIT_OR_OB) if Qd::can_read_lit_from(d, tok) => {
                 r.incr();
-                r.record(
-                    dict.insert(
-                        unsafe { tmp.assume_init_ref() }.to_string(),
-                        Some(lit.clone().into()),
-                    )
-                    .is_none(),
-                );
+                unsafe {
+                    r.record(
+                        dict.insert(
+                            tmp.assume_init_ref().to_string(),
+                            Some(Qd::read_lit(d, tok).into()),
+                        )
+                        .is_none(),
+                    );
+                }
                 // saw a literal. next is either comma or close brace
                 state = TyMetaFoldState::COMMA_OR_CB;
             }
@@ -492,11 +506,12 @@ pub(super) fn rfold_tymeta<const ALLOW_RESET: bool>(
             (Token::Symbol(Symbol::TtOpenBrace), TyMetaFoldState::LIT_OR_OB) => {
                 r.incr();
                 // another dict in here
-                let mut d = Dict::new();
-                let ret = rfold_tymeta::<ALLOW_RESET>(
+                let mut nd = Dict::new();
+                let ret = rfold_tymeta::<Qd, ALLOW_RESET>(
                     TyMetaFoldState::IDENT_OR_CB,
                     &tok[r.pos()..],
-                    &mut d,
+                    d,
+                    &mut nd,
                 );
                 r.incr_by(ret.pos());
                 r.record(ret.is_okay());
@@ -504,8 +519,11 @@ pub(super) fn rfold_tymeta<const ALLOW_RESET: bool>(
                 r.record(!ret.has_more());
                 // end of definition or comma followed by something
                 r.record(
-                    dict.insert(unsafe { tmp.assume_init_ref() }.to_string(), Some(d.into()))
-                        .is_none(),
+                    dict.insert(
+                        unsafe { tmp.assume_init_ref() }.to_string(),
+                        Some(nd.into()),
+                    )
+                    .is_none(),
                 );
                 state = TyMetaFoldState::COMMA_OR_CB;
             }
@@ -523,7 +541,12 @@ pub(super) fn rfold_tymeta<const ALLOW_RESET: bool>(
 /// (**test-only**) fold type metadata
 pub(super) fn fold_tymeta(tok: &[Token]) -> (TyMetaFoldResult, Dict) {
     let mut d = Dict::new();
-    let r = rfold_tymeta::<DISALLOW_RESET_SYNTAX>(TyMetaFoldState::IDENT_OR_CB, tok, &mut d);
+    let r = rfold_tymeta::<InplaceData, DISALLOW_RESET_SYNTAX>(
+        TyMetaFoldState::IDENT_OR_CB,
+        tok,
+        &mut InplaceData::new(),
+        &mut d,
+    );
     (r, d)
 }
 
@@ -542,9 +565,10 @@ states! {
 }
 
 /// Fold layers
-pub(super) fn rfold_layers<const ALLOW_RESET: bool>(
+pub(super) fn rfold_layers<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
     start: LayerFoldState,
-    tok: &[Token],
+    tok: &'a [Token],
+    qd: &mut Qd,
     layers: &mut Vec<Layer>,
 ) -> u64 {
     /*
@@ -577,13 +601,18 @@ pub(super) fn rfold_layers<const ALLOW_RESET: bool>(
             (Token::Symbol(Symbol::TtOpenBrace), LayerFoldState::END_OR_OB) => {
                 i += 1;
                 // since we found an open brace, this type has some meta
-                let ret =
-                    rfold_tymeta::<ALLOW_RESET>(TyMetaFoldState::IDENT_OR_CB, &tok[i..], &mut dict);
+                let ret = rfold_tymeta::<Qd, ALLOW_RESET>(
+                    TyMetaFoldState::IDENT_OR_CB,
+                    &tok[i..],
+                    qd,
+                    &mut dict,
+                );
                 i += ret.pos();
                 okay &= ret.is_okay();
                 if ret.has_more() {
                     // more layers
-                    let ret = rfold_layers::<ALLOW_RESET>(LayerFoldState::TY, &tok[i..], layers);
+                    let ret =
+                        rfold_layers::<Qd, ALLOW_RESET>(LayerFoldState::TY, &tok[i..], qd, layers);
                     okay &= ret & HIBIT == HIBIT;
                     i += (ret & !HIBIT) as usize;
                     state = LayerFoldState::FOLD_DICT_INCOMPLETE;
@@ -601,8 +630,12 @@ pub(super) fn rfold_layers<const ALLOW_RESET: bool>(
             (Token::Symbol(Symbol::SymComma), LayerFoldState::FOLD_DICT_INCOMPLETE) => {
                 // there is a comma at the end of this
                 i += 1;
-                let ret =
-                    rfold_tymeta::<ALLOW_RESET>(TyMetaFoldState::IDENT_OR_CB, &tok[i..], &mut dict);
+                let ret = rfold_tymeta::<Qd, ALLOW_RESET>(
+                    TyMetaFoldState::IDENT_OR_CB,
+                    &tok[i..],
+                    qd,
+                    &mut dict,
+                );
                 i += ret.pos();
                 okay &= ret.is_okay();
                 okay &= !ret.has_more(); // not more than one type depth
@@ -653,7 +686,12 @@ pub(super) fn rfold_layers<const ALLOW_RESET: bool>(
 /// (**test-only**) fold layers
 pub(super) fn fold_layers(tok: &[Token]) -> (Vec<Layer>, usize, bool) {
     let mut l = Vec::new();
-    let r = rfold_layers::<DISALLOW_RESET_SYNTAX>(LayerFoldState::TY, tok, &mut l);
+    let r = rfold_layers::<InplaceData, DISALLOW_RESET_SYNTAX>(
+        LayerFoldState::TY,
+        tok,
+        &mut InplaceData::new(),
+        &mut l,
+    );
     (l, (r & !HIBIT) as _, r & HIBIT == HIBIT)
 }
 
@@ -688,9 +726,17 @@ pub(super) fn parse_field_properties(tok: &[Token]) -> (FieldProperties, usize,
     (p, (r & !HIBIT) as _, r & HIBIT == HIBIT)
 }
 
+#[cfg(test)]
+pub(super) fn parse_field_full(tok: &[Token]) -> LangResult<(usize, Field)> {
+    self::parse_field(tok, &mut InplaceData::new())
+}
+
 #[inline(always)]
 /// Parse a field using the declaration-syntax (not field syntax)
-pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
+pub(super) fn parse_field<'a, Qd: QueryData<'a>>(
+    tok: &'a [Token],
+    qd: &mut Qd,
+) -> LangResult<(usize, Field)> {
     let l = tok.len();
     let mut i = 0;
     let mut okay = true;
@@ -712,7 +758,8 @@ pub(super) fn parse_field(tok: &[Token]) -> LangResult<(usize, Field)> {
 
     // layers
     let mut layers = Vec::new();
-    let r = rfold_layers::<DISALLOW_RESET_SYNTAX>(LayerFoldState::TY, &tok[i..], &mut layers);
+    let r =
+        rfold_layers::<Qd, DISALLOW_RESET_SYNTAX>(LayerFoldState::TY, &tok[i..], qd, &mut layers);
     okay &= r & HIBIT == HIBIT;
     i += (r & !HIBIT) as usize;
 
@@ -740,9 +787,17 @@ states! {
     }
 }
 
+#[cfg(test)]
+pub(super) fn parse_schema_from_tokens_full(tok: &[Token]) -> LangResult<(Model, usize)> {
+    self::parse_schema_from_tokens::<InplaceData>(tok, &mut InplaceData::new())
+}
+
 #[inline(always)]
 /// Parse a fresh schema with declaration-syntax fields
-pub(super) fn parse_schema_from_tokens(tok: &[Token]) -> LangResult<(Model, usize)> {
+pub(super) fn parse_schema_from_tokens<'a, Qd: QueryData<'a>>(
+    tok: &'a [Token],
+    qd: &mut Qd,
+) -> LangResult<(Model, usize)> {
     // parse fields
     let l = tok.len();
     let mut i = 0;
@@ -763,7 +818,7 @@ pub(super) fn parse_schema_from_tokens(tok: &[Token]) -> LangResult<(Model, usiz
                 SchemaParseState::FIELD | SchemaParseState::END_OR_FIELD,
             ) => {
                 // fine, we found a field. let's see what we've got
-                let (c, f) = self::parse_field(&tok[i..])?;
+                let (c, f) = self::parse_field(&tok[i..], qd)?;
                 fields.push(f);
                 i += c;
                 state = SchemaParseState::COMMA_OR_END;
@@ -804,7 +859,7 @@ pub(super) fn parse_schema_from_tokens(tok: &[Token]) -> LangResult<(Model, usiz
 
         // great, parse the dict
         let mut dict = Dict::new();
-        let r = self::rfold_dict(DictFoldState::OB, &tok[i..], &mut dict);
+        let r = self::rfold_dict(DictFoldState::OB, &tok[i..], qd, &mut dict);
         i += (r & !HIBIT) as usize;
 
         if r & HIBIT == HIBIT {
@@ -835,7 +890,10 @@ pub(super) fn parse_schema_from_tokens(tok: &[Token]) -> LangResult<(Model, usiz
 
 #[inline(always)]
 /// Parse space data from the given tokens
-pub(super) fn parse_space_from_tokens(tok: &[Token]) -> LangResult<(Space, usize)> {
+pub(super) fn parse_space_from_tokens<'a, Qd: QueryData<'a>>(
+    tok: &'a [Token],
+    qd: &mut Qd,
+) -> LangResult<(Space, usize)> {
     let l = tok.len();
     let mut okay = !tok.is_empty() && tok[0].is_ident();
     let mut i = 0;
@@ -847,7 +905,7 @@ pub(super) fn parse_space_from_tokens(tok: &[Token]) -> LangResult<(Space, usize
     let mut d = Dict::new();
 
     if has_more_properties && okay {
-        let ret = self::rfold_dict(DictFoldState::OB, &tok[1..], &mut d);
+        let ret = self::rfold_dict(DictFoldState::OB, &tok[1..], qd, &mut d);
         i += (ret & !HIBIT) as usize;
         okay &= ret & HIBIT == HIBIT;
     }
@@ -867,7 +925,10 @@ pub(super) fn parse_space_from_tokens(tok: &[Token]) -> LangResult<(Space, usize
 
 #[inline(always)]
 /// Parse alter space from tokens
-pub(super) fn parse_alter_space_from_tokens(tok: &[Token]) -> LangResult<(AlterSpace, usize)> {
+pub(super) fn parse_alter_space_from_tokens<'a, Qd: QueryData<'a>>(
+    tok: &'a [Token],
+    qd: &mut Qd,
+) -> LangResult<(AlterSpace, usize)> {
     let mut i = 0;
     let l = tok.len();
 
@@ -882,7 +943,7 @@ pub(super) fn parse_alter_space_from_tokens(tok: &[Token]) -> LangResult<(AlterS
     i += 3;
 
     let mut d = Dict::new();
-    let ret = rfold_dict(DictFoldState::CB_OR_IDENT, &tok[i..], &mut d);
+    let ret = rfold_dict(DictFoldState::CB_OR_IDENT, &tok[i..], qd, &mut d);
     i += (ret & !HIBIT) as usize;
 
     if ret & HIBIT == HIBIT {
@@ -900,8 +961,8 @@ pub(super) fn parse_alter_space_from_tokens(tok: &[Token]) -> LangResult<(AlterS
 
 #[cfg(test)]
 pub(super) fn alter_space_full(tok: &[Token]) -> LangResult<AlterSpace> {
-    let (r, i) = self::parse_alter_space_from_tokens(tok)?;
-    assert_eq!(i, tok.len(), "full token stream not used");
+    let (r, i) = self::parse_alter_space_from_tokens(tok, &mut InplaceData::new())?;
+    assert_full_tt!(i, tok.len());
     Ok(r)
 }
 
@@ -924,10 +985,18 @@ pub struct ExpandedField {
     pub(super) reset: bool,
 }
 
+#[cfg(test)]
+pub fn parse_field_syntax_full<const ALLOW_RESET: bool>(
+    tok: &[Token],
+) -> LangResult<(ExpandedField, usize)> {
+    self::parse_field_syntax::<InplaceData, ALLOW_RESET>(tok, &mut InplaceData::new())
+}
+
 #[inline(always)]
 /// Parse a field declared using the field syntax
-pub(super) fn parse_field_syntax<const ALLOW_RESET: bool>(
-    tok: &[Token],
+pub(super) fn parse_field_syntax<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
+    tok: &'a [Token],
+    qd: &mut Qd,
 ) -> LangResult<(ExpandedField, usize)> {
     let l = tok.len();
     let mut i = 0_usize;
@@ -947,18 +1016,20 @@ pub(super) fn parse_field_syntax<const ALLOW_RESET: bool>(
             }
             (Token::Symbol(Symbol::TtOpenBrace), FieldSyntaxParseState::OB) => {
                 i += 1;
-                let r = self::rfold_tymeta::<ALLOW_RESET>(
+                let r = self::rfold_tymeta::<Qd, ALLOW_RESET>(
                     TyMetaFoldState::IDENT_OR_CB,
                     &tok[i..],
+                    qd,
                     &mut props,
                 );
                 okay &= r.is_okay();
                 i += r.pos();
                 if r.has_more() && i < l {
                     // now parse layers
-                    let r = self::rfold_layers::<ALLOW_RESET>(
+                    let r = self::rfold_layers::<Qd, ALLOW_RESET>(
                         LayerFoldState::TY,
                         &tok[i..],
+                        qd,
                         &mut layers,
                     );
                     okay &= r & HIBIT == HIBIT;
@@ -971,9 +1042,10 @@ pub(super) fn parse_field_syntax<const ALLOW_RESET: bool>(
             }
             (Token::Symbol(Symbol::SymComma), FieldSyntaxParseState::FOLD_DICT_INCOMPLETE) => {
                 i += 1;
-                let r = self::rfold_tymeta::<ALLOW_RESET>(
+                let r = self::rfold_tymeta::<Qd, ALLOW_RESET>(
                     TyMetaFoldState::IDENT_OR_CB,
                     &tok[i..],
+                    qd,
                     &mut props,
                 );
                 okay &= r.is_okay() && !r.has_more();
@@ -1040,8 +1112,9 @@ pub enum AlterKind {
 
 #[inline(always)]
 /// Parse an [`AlterKind`] from the given token stream
-pub(super) fn parse_alter_kind_from_tokens(
-    tok: &[Token],
+pub(super) fn parse_alter_kind_from_tokens<'a, Qd: QueryData<'a>>(
+    tok: &'a [Token],
+    qd: &mut Qd,
     current: &mut usize,
 ) -> LangResult<Alter> {
     let l = tok.len();
@@ -1052,13 +1125,13 @@ pub(super) fn parse_alter_kind_from_tokens(
     *current += 2;
     let model_name = unsafe { extract!(tok[0], Token::Ident(ref l) => l.clone()) };
     match tok[1] {
-        Token![add] => alter_add(&tok[1..], current)
+        Token![add] => alter_add(&tok[1..], qd, current)
             .map(AlterKind::Add)
             .map(|kind| Alter::new(model_name, kind)),
         Token![remove] => alter_remove(&tok[1..], current)
             .map(AlterKind::Remove)
             .map(|kind| Alter::new(model_name, kind)),
-        Token![update] => alter_update(&tok[1..], current)
+        Token![update] => alter_update(&tok[1..], qd, current)
             .map(AlterKind::Update)
             .map(|kind| Alter::new(model_name, kind)),
         _ => return Err(LangError::ExpectedStatement),
@@ -1067,8 +1140,9 @@ pub(super) fn parse_alter_kind_from_tokens(
 
 #[inline(always)]
 /// Parse multiple fields declared using the field syntax. Flag setting allows or disallows reset syntax
-pub(super) fn parse_multiple_field_syntax<const ALLOW_RESET: bool>(
-    tok: &[Token],
+pub(super) fn parse_multiple_field_syntax<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
+    tok: &'a [Token],
+    qd: &mut Qd,
     current: &mut usize,
 ) -> LangResult<Box<[ExpandedField]>> {
     const DEFAULT_ADD_COL_CNT: usize = 4;
@@ -1086,7 +1160,7 @@ pub(super) fn parse_multiple_field_syntax<const ALLOW_RESET: bool>(
     }
     match tok[0] {
         Token::Ident(_) => {
-            let (r, i) = parse_field_syntax::<ALLOW_RESET>(&tok)?;
+            let (r, i) = parse_field_syntax::<Qd, ALLOW_RESET>(&tok, qd)?;
             *current += i;
             Ok([r].into())
         }
@@ -1098,7 +1172,7 @@ pub(super) fn parse_multiple_field_syntax<const ALLOW_RESET: bool>(
             while i < l && okay && !stop {
                 match tok[i] {
                     Token::Ident(_) => {
-                        let (r, cnt) = parse_field_syntax::<ALLOW_RESET>(&tok[i..])?;
+                        let (r, cnt) = parse_field_syntax::<Qd, ALLOW_RESET>(&tok[i..], qd)?;
                         i += cnt;
                         cols.push(r);
                         let nx_comma = i < l && tok[i] == Token::Symbol(Symbol::SymComma);
@@ -1126,8 +1200,20 @@ pub(super) fn parse_multiple_field_syntax<const ALLOW_RESET: bool>(
 
 #[inline(always)]
 /// Parse the expression for `alter model <> add (..)`
-pub(super) fn alter_add(tok: &[Token], current: &mut usize) -> LangResult<Box<[ExpandedField]>> {
-    self::parse_multiple_field_syntax::<DISALLOW_RESET_SYNTAX>(tok, current)
+pub(super) fn alter_add<'a, Qd: QueryData<'a>>(
+    tok: &'a [Token],
+    qd: &mut Qd,
+    current: &mut usize,
+) -> LangResult<Box<[ExpandedField]>> {
+    self::parse_multiple_field_syntax::<Qd, DISALLOW_RESET_SYNTAX>(tok, qd, current)
+}
+
+#[cfg(test)]
+pub(super) fn alter_add_full(
+    tok: &[Token],
+    current: &mut usize,
+) -> LangResult<Box<[ExpandedField]>> {
+    self::alter_add(tok, &mut InplaceData::new(), current)
 }
 
 #[inline(always)]
@@ -1184,6 +1270,15 @@ pub(super) fn alter_remove(tok: &[Token], current: &mut usize) -> LangResult<Box
 
 #[inline(always)]
 /// Parse the expression for `alter model <> update (..)`
-pub(super) fn alter_update(tok: &[Token], current: &mut usize) -> LangResult<Box<[ExpandedField]>> {
-    self::parse_multiple_field_syntax::<ALLOW_RESET_SYNTAX>(tok, current)
+pub(super) fn alter_update<'a, Qd: QueryData<'a>>(
+    tok: &'a [Token],
+    qd: &mut Qd,
+    current: &mut usize,
+) -> LangResult<Box<[ExpandedField]>> {
+    self::parse_multiple_field_syntax::<Qd, ALLOW_RESET_SYNTAX>(tok, qd, current)
+}
+
+#[cfg(test)]
+pub(super) fn alter_update_full(tok: &[Token], i: &mut usize) -> LangResult<Box<[ExpandedField]>> {
+    self::alter_update(tok, &mut InplaceData::new(), i)
 }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 93e54f06..10e14350 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -88,7 +88,9 @@ impl NullableMapEntry for Null {
 
 impl NullableMapEntry for super::lexer::Lit {
     fn data(self) -> Option<super::schema::DictEntry> {
-        Some(super::schema::DictEntry::Lit(self))
+        Some(super::schema::DictEntry::Lit(
+            unsafe { self.as_ir() }.to_litir_owned(),
+        ))
     }
 }
 
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index a69c89c9..43d5b5b5 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -92,12 +92,14 @@ mod alter_space {
     }
     #[test]
     fn alter_space() {
-        let tok = lex_insecure(br#"
+        let tok = lex_insecure(
+            br#"
                 alter model mymodel with {
                     max_entry: 1000,
                     driver: "ts-0.8"
                 }
-            "#)
+            "#,
+        )
         .unwrap();
         let r = schema::alter_space_full(&tok[2..]).unwrap();
         assert_eq!(
@@ -177,7 +179,8 @@ mod tymeta {
         // list { maxlen: 100, this: { is: "cool" }, type string, unique: true }
         //        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cursor should be at string
         let tok =
-            lex_insecure(br#"maxlen: 100, this: { is: "cool" }, type string, unique: true }"#).unwrap();
+            lex_insecure(br#"maxlen: 100, this: { is: "cool" }, type string, unique: true }"#)
+                .unwrap();
         let (res1, ret1) = schema::fold_tymeta(&tok);
         assert!(res1.is_okay());
         assert!(res1.has_more());
@@ -204,7 +207,8 @@ mod tymeta {
     fn fuzz_tymeta_normal() {
         // { maxlen: 10, unique: true, users: "sayan" }
         //   ^start
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                     maxlen: 10,
                     unique: true,
                     auth: {
@@ -212,7 +216,8 @@ mod tymeta {
                     },
                     users: \"sayan\"\x01
                 }
-            ")
+            ",
+        )
         .unwrap();
         let expected = nullable_dict! {
             "maxlen" => Lit::UnsignedInt(10),
@@ -241,7 +246,8 @@ mod tymeta {
     fn fuzz_tymeta_with_ty() {
         // list { maxlen: 10, unique: true, type string, users: "sayan" }
         //   ^start
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                     maxlen: 10,
                     unique: true,
                     auth: {
@@ -250,7 +256,8 @@ mod tymeta {
                     type string,
                     users: \"sayan\"\x01
                 }
-            ")
+            ",
+        )
         .unwrap();
         let expected = nullable_dict! {
             "maxlen" => Lit::UnsignedInt(10),
@@ -368,7 +375,8 @@ mod layer {
 
     #[test]
     fn fuzz_layer() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
             list {
                 type list {
                     maxlen: 100,
@@ -376,7 +384,8 @@ mod layer {
                 },
                 unique: true\x01
             }
-        ")
+        ",
+        )
         .unwrap();
         let expected = vec![
             Layer::new_noreset("string".into(), nullable_dict!()),
@@ -436,11 +445,13 @@ mod fields {
     };
     #[test]
     fn field_mini() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 username: string,
-            ")
+            ",
+        )
         .unwrap();
-        let (c, f) = schema::parse_field(&tok).unwrap();
+        let (c, f) = schema::parse_field_full(&tok).unwrap();
         assert_eq!(c, tok.len() - 1);
         assert_eq!(
             f,
@@ -453,11 +464,13 @@ mod fields {
     }
     #[test]
     fn field() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 primary username: string,    
-            ")
+            ",
+        )
         .unwrap();
-        let (c, f) = schema::parse_field(&tok).unwrap();
+        let (c, f) = schema::parse_field_full(&tok).unwrap();
         assert_eq!(c, tok.len() - 1);
         assert_eq!(
             f,
@@ -470,14 +483,16 @@ mod fields {
     }
     #[test]
     fn field_pro() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 primary username: string {
                     maxlen: 10,
                     ascii_only: true,
                 }
-            ")
+            ",
+        )
         .unwrap();
-        let (c, f) = schema::parse_field(&tok).unwrap();
+        let (c, f) = schema::parse_field_full(&tok).unwrap();
         assert_eq!(c, tok.len());
         assert_eq!(
             f,
@@ -497,7 +512,8 @@ mod fields {
     }
     #[test]
     fn field_pro_max() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 null notes: list {
                     type string {
                         maxlen: 255,
@@ -505,9 +521,10 @@ mod fields {
                     },
                     unique: true,
                 }
-            ")
+            ",
+        )
         .unwrap();
-        let (c, f) = schema::parse_field(&tok).unwrap();
+        let (c, f) = schema::parse_field_full(&tok).unwrap();
         assert_eq!(c, tok.len());
         assert_eq!(
             f,
@@ -540,17 +557,19 @@ mod schemas {
     use super::*;
     #[test]
     fn schema_mini() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 create model mymodel(
                     primary username: string,
                     password: binary,
                 )
-            ")
+            ",
+        )
         .unwrap();
         let tok = &tok[2..];
 
         // parse model
-        let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
+        let (model, c) = schema::parse_schema_from_tokens_full(tok).unwrap();
         assert_eq!(c, tok.len());
         assert_eq!(
             model,
@@ -574,18 +593,20 @@ mod schemas {
     }
     #[test]
     fn schema() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 create model mymodel(
                     primary username: string,
                     password: binary,
                     null profile_pic: binary,
                 )
-            ")
+            ",
+        )
         .unwrap();
         let tok = &tok[2..];
 
         // parse model
-        let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
+        let (model, c) = schema::parse_schema_from_tokens_full(tok).unwrap();
         assert_eq!(c, tok.len());
         assert_eq!(
             model,
@@ -615,7 +636,8 @@ mod schemas {
 
     #[test]
     fn schema_pro() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 create model mymodel(
                     primary username: string,
                     password: binary,
@@ -625,12 +647,13 @@ mod schemas {
                         unique: true,
                     },
                 )
-            ")
+            ",
+        )
         .unwrap();
         let tok = &tok[2..];
 
         // parse model
-        let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
+        let (model, c) = schema::parse_schema_from_tokens_full(tok).unwrap();
         assert_eq!(c, tok.len());
         assert_eq!(
             model,
@@ -673,7 +696,8 @@ mod schemas {
 
     #[test]
     fn schema_pro_max() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 create model mymodel(
                     primary username: string,
                     password: binary,
@@ -688,12 +712,13 @@ mod schemas {
                     },
                     storage_driver: \"skyheap\"
                 }
-            ")
+            ",
+        )
         .unwrap();
         let tok = &tok[2..];
 
         // parse model
-        let (model, c) = schema::parse_schema_from_tokens(tok).unwrap();
+        let (model, c) = schema::parse_schema_from_tokens_full(tok).unwrap();
         assert_eq!(c, tok.len());
         assert_eq!(
             model,
@@ -745,7 +770,7 @@ mod dict_field_syntax {
     #[test]
     fn field_syn_mini() {
         let tok = lex_insecure(b"username { type string }").unwrap();
-        let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
+        let (ef, i) = schema::parse_field_syntax_full::<true>(&tok).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             ef,
@@ -759,14 +784,16 @@ mod dict_field_syntax {
     }
     #[test]
     fn field_syn() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 username {
                     nullable: false,
                     type string,
                 }
-            ")
+            ",
+        )
         .unwrap();
-        let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
+        let (ef, i) = schema::parse_field_syntax_full::<true>(&tok).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             ef,
@@ -782,7 +809,8 @@ mod dict_field_syntax {
     }
     #[test]
     fn field_syn_pro() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 username {
                     nullable: false,
                     type string {
@@ -791,9 +819,10 @@ mod dict_field_syntax {
                     },
                     jingle_bells: \"snow\"
                 }
-            ")
+            ",
+        )
         .unwrap();
-        let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
+        let (ef, i) = schema::parse_field_syntax_full::<true>(&tok).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             ef,
@@ -816,7 +845,8 @@ mod dict_field_syntax {
     }
     #[test]
     fn field_syn_pro_max() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 notes {
                     nullable: true,
                     type list {
@@ -827,9 +857,10 @@ mod dict_field_syntax {
                     },
                     jingle_bells: \"snow\"
                 }
-            ")
+            ",
+        )
         .unwrap();
-        let (ef, i) = schema::parse_field_syntax::<true>(&tok).unwrap();
+        let (ef, i) = schema::parse_field_syntax_full::<true>(&tok).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             ef,
@@ -880,7 +911,8 @@ mod alter_model_remove {
     #[test]
     fn alter() {
         let tok =
-            lex_insecure(b"alter model mymodel remove (myfield1, myfield2, myfield3, myfield4)").unwrap();
+            lex_insecure(b"alter model mymodel remove (myfield1, myfield2, myfield3, myfield4)")
+                .unwrap();
         let mut i = 4;
         let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
@@ -901,12 +933,14 @@ mod alter_model_add {
     use crate::engine::ql::schema::{ExpandedField, Layer};
     #[test]
     fn add_mini() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 alter model mymodel add myfield { type string }
-            ")
+            ",
+        )
         .unwrap();
         let mut i = 4;
-        let r = schema::alter_add(&tok[i..], &mut i).unwrap();
+        let r = schema::alter_add_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             r.as_ref(),
@@ -920,12 +954,14 @@ mod alter_model_add {
     }
     #[test]
     fn add() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 alter model mymodel add myfield { type string, nullable: true }
-            ")
+            ",
+        )
         .unwrap();
         let mut i = 4;
-        let r = schema::alter_add(&tok[i..], &mut i).unwrap();
+        let r = schema::alter_add_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             r.as_ref(),
@@ -941,12 +977,14 @@ mod alter_model_add {
     }
     #[test]
     fn add_pro() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 alter model mymodel add (myfield { type string, nullable: true })
-            ")
+            ",
+        )
         .unwrap();
         let mut i = 4;
-        let r = schema::alter_add(&tok[i..], &mut i).unwrap();
+        let r = schema::alter_add_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             r.as_ref(),
@@ -962,7 +1000,8 @@ mod alter_model_add {
     }
     #[test]
     fn add_pro_max() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 alter model mymodel add (
                     myfield {
                         type string,
@@ -978,10 +1017,11 @@ mod alter_model_add {
                         nullable: false,
                     }
                 )
-            ")
+            ",
+        )
         .unwrap();
         let mut i = 4;
-        let r = schema::alter_add(&tok[i..], &mut i).unwrap();
+        let r = schema::alter_add_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             r.as_ref(),
@@ -1026,12 +1066,14 @@ mod alter_model_update {
 
     #[test]
     fn alter_mini() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 alter model mymodel update myfield { type string, .. }
-            ")
+            ",
+        )
         .unwrap();
         let mut i = 4;
-        let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+        let r = schema::alter_update_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             r.as_ref(),
@@ -1045,12 +1087,14 @@ mod alter_model_update {
     }
     #[test]
     fn alter_mini_2() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 alter model mymodel update (myfield { type string, .. })
-            ")
+            ",
+        )
         .unwrap();
         let mut i = 4;
-        let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+        let r = schema::alter_update_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             r.as_ref(),
@@ -1064,7 +1108,8 @@ mod alter_model_update {
     }
     #[test]
     fn alter() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 alter model mymodel update (
                     myfield {
                         type string,
@@ -1072,10 +1117,11 @@ mod alter_model_update {
                         ..
                     }
                 )
-            ")
+            ",
+        )
         .unwrap();
         let mut i = 4;
-        let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+        let r = schema::alter_update_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             r.as_ref(),
@@ -1091,7 +1137,8 @@ mod alter_model_update {
     }
     #[test]
     fn alter_pro() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 alter model mymodel update (
                     myfield {
                         type string,
@@ -1103,10 +1150,11 @@ mod alter_model_update {
                         ..
                     }
                 )
-            ")
+            ",
+        )
         .unwrap();
         let mut i = 4;
-        let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+        let r = schema::alter_update_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             r.as_ref(),
@@ -1130,7 +1178,8 @@ mod alter_model_update {
     }
     #[test]
     fn alter_pro_max() {
-        let tok = lex_insecure(b"
+        let tok = lex_insecure(
+            b"
                 alter model mymodel update (
                     myfield {
                         type string {..},
@@ -1145,10 +1194,11 @@ mod alter_model_update {
                         ..
                     }
                 )
-            ")
+            ",
+        )
         .unwrap();
         let mut i = 4;
-        let r = schema::alter_update(&tok[i..], &mut i).unwrap();
+        let r = schema::alter_update_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             r.as_ref(),
diff --git a/server/src/engine/ql/tests/structure_syn.rs b/server/src/engine/ql/tests/structure_syn.rs
index 9681d24b..ff7e01fd 100644
--- a/server/src/engine/ql/tests/structure_syn.rs
+++ b/server/src/engine/ql/tests/structure_syn.rs
@@ -170,7 +170,8 @@ mod dict {
 
     #[test]
     fn fuzz_dict() {
-        let ret = lex_insecure(b"
+        let ret = lex_insecure(
+            b"
                 {
                     the_tradition_is: \"hello, world\",
                     could_have_been: {
@@ -185,7 +186,8 @@ mod dict {
                         }\x01
                     }\x01
                 }
-            ")
+            ",
+        )
         .unwrap();
         let ret_dict = nullable_dict! {
             "the_tradition_is" => Lit::Str("hello, world".into()),

From 255d058793fa33d667657e07136ca8704be60bed Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 5 Jan 2023 20:34:47 -0800
Subject: [PATCH 065/310] Add tests for secure list parsing

---
 server/src/engine/ql/dml.rs             |   7 +-
 server/src/engine/ql/tests.rs           |   9 +-
 server/src/engine/ql/tests/dml_tests.rs | 108 ++++++++++++++++++++++--
 3 files changed, 112 insertions(+), 12 deletions(-)

diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index d12f961d..ea7cdd88 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -293,9 +293,12 @@ pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
 }
 
 #[cfg(test)]
-pub(super) fn parse_list_full(tok: &[Token]) -> Option<Vec<DataType>> {
+pub(super) fn parse_list_full<'a>(
+    tok: &'a [Token],
+    qd: &mut impl QueryData<'a>,
+) -> Option<Vec<DataType>> {
     let mut l = Vec::new();
-    if matches!(parse_list(tok, &mut InplaceData::new(), &mut l), (_, i, true) if i == tok.len()) {
+    if matches!(parse_list(tok, qd, &mut l), (_, i, true) if i == tok.len()) {
         Some(l)
     } else {
         None
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 10e14350..398909ab 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -26,7 +26,7 @@
 
 use {
     super::{
-        lexer::{InsecureLexer, Symbol, Token},
+        lexer::{InsecureLexer, SafeLexer, Symbol, Token},
         LangResult,
     },
     crate::{engine::memory::DataType, util::test_utils},
@@ -45,11 +45,16 @@ mod lexer_tests;
 mod schema_tests;
 mod structure_syn;
 
-/// Uses the [`InsecureLexer`] to lex the given input
 #[inline(always)]
+/// Uses the [`InsecureLexer`] to lex the given input
 pub(super) fn lex_insecure(src: &[u8]) -> LangResult<Vec<Token>> {
     InsecureLexer::lex(src)
 }
+#[inline(always)]
+/// Uses the [`SafeLexer`] to lex the given input
+pub(super) fn lex_secure(src: &[u8]) -> LangResult<Vec<Token>> {
+    SafeLexer::lex(src)
+}
 
 pub trait NullableData<T> {
     fn data(self) -> Option<T>;
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 7684d0e0..7707fd4d 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -27,7 +27,11 @@
 use super::*;
 mod list_parse {
     use super::*;
-    use crate::engine::ql::dml::parse_list_full;
+    use crate::engine::ql::{
+        ast::{InplaceData, SubstitutedData},
+        dml::parse_list_full,
+        lexer::LitIR,
+    };
 
     #[test]
     fn list_mini() {
@@ -37,10 +41,9 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_list_full(&tok[1..]).unwrap();
+        let r = parse_list_full(&tok[1..], &mut InplaceData::new()).unwrap();
         assert_eq!(r, vec![])
     }
-
     #[test]
     fn list() {
         let tok = lex_insecure(
@@ -49,10 +52,27 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_list_full(&tok[1..]).unwrap();
+        let r = parse_list_full(&tok[1..], &mut InplaceData::new()).unwrap();
+        assert_eq!(r.as_slice(), into_array![1, 2, 3, 4])
+    }
+    #[test]
+    fn list_param() {
+        let tok = lex_secure(
+            b"
+                [?, ?, ?, ?]
+            ",
+        )
+        .unwrap();
+        let data = [
+            LitIR::UInt(1),
+            LitIR::UInt(2),
+            LitIR::UInt(3),
+            LitIR::UInt(4),
+        ];
+        let mut param = SubstitutedData::new(&data);
+        let r = parse_list_full(&tok[1..], &mut param).unwrap();
         assert_eq!(r.as_slice(), into_array![1, 2, 3, 4])
     }
-
     #[test]
     fn list_pro() {
         let tok = lex_insecure(
@@ -66,7 +86,40 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_list_full(&tok[1..]).unwrap();
+        let r = parse_list_full(&tok[1..], &mut InplaceData::new()).unwrap();
+        assert_eq!(
+            r.as_slice(),
+            into_array![
+                into_array![1, 2],
+                into_array![3, 4],
+                into_array![5, 6],
+                into_array![]
+            ]
+        )
+    }
+    #[test]
+    fn list_pro_param() {
+        let tok = lex_secure(
+            b"
+                [
+                    [?, ?],
+                    [?, ?],
+                    [?, ?],
+                    []
+                ]
+            ",
+        )
+        .unwrap();
+        let data = [
+            LitIR::UInt(1),
+            LitIR::UInt(2),
+            LitIR::UInt(3),
+            LitIR::UInt(4),
+            LitIR::UInt(5),
+            LitIR::UInt(6),
+        ];
+        let mut param = SubstitutedData::new(&data);
+        let r = parse_list_full(&tok[1..], &mut param).unwrap();
         assert_eq!(
             r.as_slice(),
             into_array![
@@ -77,7 +130,6 @@ mod list_parse {
             ]
         )
     }
-
     #[test]
     fn list_pro_max() {
         let tok = lex_insecure(
@@ -91,7 +143,46 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_list_full(&tok[1..]).unwrap();
+        let r = parse_list_full(&tok[1..], &mut InplaceData::new()).unwrap();
+        assert_eq!(
+            r.as_slice(),
+            into_array![
+                into_array![into_array![1, 1], into_array![2, 2]],
+                into_array![into_array![], into_array![4, 4]],
+                into_array![into_array![5, 5], into_array![6, 6]],
+                into_array![into_array![7, 7], into_array![]],
+            ]
+        )
+    }
+    #[test]
+    fn list_pro_max_param() {
+        let tok = lex_secure(
+            b"
+                [
+                    [[?, ?], [?, ?]],
+                    [[], [?, ?]],
+                    [[?, ?], [?, ?]],
+                    [[?, ?], []]
+                ]
+            ",
+        )
+        .unwrap();
+        let data = [
+            LitIR::UInt(1),
+            LitIR::UInt(1),
+            LitIR::UInt(2),
+            LitIR::UInt(2),
+            LitIR::UInt(4),
+            LitIR::UInt(4),
+            LitIR::UInt(5),
+            LitIR::UInt(5),
+            LitIR::UInt(6),
+            LitIR::UInt(6),
+            LitIR::UInt(7),
+            LitIR::UInt(7),
+        ];
+        let mut param = SubstitutedData::new(&data);
+        let r = parse_list_full(&tok[1..], &mut param).unwrap();
         assert_eq!(
             r.as_slice(),
             into_array![
@@ -103,6 +194,7 @@ mod list_parse {
         )
     }
 }
+
 mod tuple_syntax {
     use super::*;
     use crate::engine::ql::dml::parse_data_tuple_syntax_full;

From 1df6f330322c20ea30effe18be27a0db19890550 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 6 Jan 2023 06:53:15 -0800
Subject: [PATCH 066/310] Remove `RawSlice` usage

This hasn't been an easy task, but I'm finally very happy that we've
gotten rid of all usage instances of `RawSlice` within `engine::ql`.
This is a great feat in terms of codebase cleanup, removing unnecessary
usage of `unsafe` code.

However, this had made lifetimes slightly more complex; for example, we
can no longer directly return from methods that use `&[Token]` since all
returned data is now bound to the same lifetime.

Either way, with some care such issues can be avoided.
---
 server/src/engine/memory/mod.rs            |  12 +-
 server/src/engine/ql/ast.rs                | 336 +++++----------------
 server/src/engine/ql/benches.rs            | 106 ++++---
 server/src/engine/ql/ddl.rs                |  52 ++--
 server/src/engine/ql/dml.rs                |  69 ++---
 server/src/engine/ql/lexer.rs              | 114 ++++---
 server/src/engine/ql/mod.rs                | 102 +------
 server/src/engine/ql/schema.rs             | 130 ++++----
 server/src/engine/ql/tests.rs              |   6 +-
 server/src/engine/ql/tests/dml_tests.rs    |  50 +--
 server/src/engine/ql/tests/entity.rs       |  17 +-
 server/src/engine/ql/tests/lexer_tests.rs  |  29 +-
 server/src/engine/ql/tests/schema_tests.rs | 207 +++++++------
 13 files changed, 481 insertions(+), 749 deletions(-)

diff --git a/server/src/engine/memory/mod.rs b/server/src/engine/memory/mod.rs
index e6e768f1..1bb7be69 100644
--- a/server/src/engine/memory/mod.rs
+++ b/server/src/engine/memory/mod.rs
@@ -35,7 +35,7 @@ pub enum DataType {
     /// An UTF-8 string
     String(Box<str>),
     /// Bytes
-    Binary(Vec<u8>),
+    Binary(Box<[u8]>),
     /// An unsigned integer
     ///
     /// **NOTE:** This is the default evaluated type for unsigned integers by the query processor. It is the
@@ -71,22 +71,20 @@ enum_impls! {
 
 impl DataType {
     #[inline(always)]
-    /// ## Safety
-    ///
-    /// Ensure validity of Lit::Bin
-    pub(super) unsafe fn clone_from_lit(lit: &Lit) -> Self {
+    pub(super) fn clone_from_lit(lit: &Lit) -> Self {
         match lit {
             Lit::Str(s) => DataType::String(s.clone()),
             Lit::Bool(b) => DataType::Boolean(*b),
             Lit::UnsignedInt(u) => DataType::UnsignedInt(*u),
             Lit::SignedInt(i) => DataType::SignedInt(*i),
-            Lit::Bin(l) => DataType::Binary(l.as_slice().to_owned()),
+            Lit::Bin(l) => DataType::Binary(l.to_vec().into_boxed_slice()),
         }
     }
+    #[inline(always)]
     pub(super) fn clone_from_litir<'a>(lit: LitIR<'a>) -> Self {
         match lit {
             LitIR::Str(s) => Self::String(s.to_owned().into_boxed_str()),
-            LitIR::Bin(b) => Self::Binary(b.to_owned()),
+            LitIR::Bin(b) => Self::Binary(b.to_owned().into_boxed_slice()),
             LitIR::Float(f) => Self::Float(f),
             LitIR::SInt(s) => Self::SignedInt(s),
             LitIR::UInt(u) => Self::UnsignedInt(u),
diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 20580879..02814cc9 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -26,12 +26,11 @@
 
 use {
     super::{
-        ddl,
-        lexer::{InsecureLexer, LitIR, Token},
-        schema, LangError, LangResult, RawSlice,
+        ddl, dml,
+        lexer::{LitIR, Slice, Token},
+        schema, LangError, LangResult,
     },
-    crate::util::Life,
-    core::{marker::PhantomData, slice},
+    crate::util::compiler,
 };
 
 pub trait QueryData<'a> {
@@ -46,15 +45,18 @@ pub trait QueryData<'a> {
 
 pub struct InplaceData;
 impl InplaceData {
+    #[inline(always)]
     pub const fn new() -> Self {
         Self
     }
 }
 
 impl<'a> QueryData<'a> for InplaceData {
+    #[inline(always)]
     fn can_read_lit_from(&self, tok: &Token) -> bool {
         tok.is_lit()
     }
+    #[inline(always)]
     unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a> {
         extract!(tok, Token::Lit(l) => l.as_ir())
     }
@@ -64,15 +66,18 @@ pub struct SubstitutedData<'a> {
     data: &'a [LitIR<'a>],
 }
 impl<'a> SubstitutedData<'a> {
+    #[inline(always)]
     pub const fn new(src: &'a [LitIR<'a>]) -> Self {
         Self { data: src }
     }
 }
 
 impl<'a> QueryData<'a> for SubstitutedData<'a> {
+    #[inline(always)]
     fn can_read_lit_from(&self, tok: &Token) -> bool {
         Token![?].eq(tok) && !self.data.is_empty()
     }
+    #[inline(always)]
     unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a> {
         debug_assert!(Token![?].eq(tok));
         let ret = self.data[0];
@@ -87,7 +92,7 @@ impl<'a> QueryData<'a> for SubstitutedData<'a> {
 
 #[derive(Debug, PartialEq)]
 /// An [`Entity`] represents the location for a specific structure, such as a model
-pub enum Entity {
+pub enum Entity<'a> {
     /// A partial entity is used when switching to a model wrt the currently set space (commonly used
     /// when running `use` queries)
     ///
@@ -95,7 +100,7 @@ pub enum Entity {
     /// ```sql
     /// :model
     /// ```
-    Partial(RawSlice),
+    Partial(Slice<'a>),
     /// A single entity is used when switching to a model wrt the currently set space (commonly used
     /// when running DML queries)
     ///
@@ -103,7 +108,7 @@ pub enum Entity {
     /// ```sql
     /// model
     /// ```
-    Single(RawSlice),
+    Single(Slice<'a>),
     /// A full entity is a complete definition to a model wrt to the given space (commonly used with
     /// DML queries)
     ///
@@ -111,16 +116,17 @@ pub enum Entity {
     /// ```sql
     /// space.model
     /// ```
-    Full(RawSlice, RawSlice),
+    Full(Slice<'a>, Slice<'a>),
 }
 
-impl<T: Into<RawSlice>, U: Into<RawSlice>> From<(T, U)> for Entity {
-    fn from((space, model): (T, U)) -> Self {
-        Self::Full(space.into(), model.into())
+impl<'a> From<(Slice<'a>, Slice<'a>)> for Entity<'a> {
+    #[inline(always)]
+    fn from((space, model): (Slice<'a>, Slice<'a>)) -> Self {
+        Self::Full(space, model)
     }
 }
 
-impl Entity {
+impl<'a> Entity<'a> {
     #[inline(always)]
     /// Parse a full entity from the given slice
     ///
@@ -128,7 +134,7 @@ impl Entity {
     ///
     /// Caller guarantees that the token stream matches the exact stream of tokens
     /// expected for a full entity
-    pub(super) unsafe fn full_entity_from_slice(sl: &[Token]) -> Self {
+    pub(super) unsafe fn full_entity_from_slice(sl: &'a [Token]) -> Self {
         Entity::Full(
             extract!(&sl[0], Token::Ident(sl) => sl.clone()),
             extract!(&sl[2], Token::Ident(sl) => sl.clone()),
@@ -141,7 +147,7 @@ impl Entity {
     ///
     /// Caller guarantees that the token stream matches the exact stream of tokens
     /// expected for a single entity
-    pub(super) unsafe fn single_entity_from_slice(sl: &[Token]) -> Self {
+    pub(super) unsafe fn single_entity_from_slice(sl: &'a [Token]) -> Self {
         Entity::Single(extract!(&sl[0], Token::Ident(sl) => sl.clone()))
     }
     #[inline(always)]
@@ -151,7 +157,7 @@ impl Entity {
     ///
     /// Caller guarantees that the token stream matches the exact stream of tokens
     /// expected for a partial entity
-    pub(super) unsafe fn partial_entity_from_slice(sl: &[Token]) -> Self {
+    pub(super) unsafe fn partial_entity_from_slice(sl: &'a [Token]) -> Self {
         Entity::Partial(extract!(&sl[1], Token::Ident(sl) => sl.clone()))
     }
     #[inline(always)]
@@ -174,7 +180,7 @@ impl Entity {
     #[inline(always)]
     /// Attempt to parse an entity using the given token stream. It also accepts a counter
     /// argument to forward the cursor
-    pub(super) fn parse_from_tokens(tok: &[Token], c: &mut usize) -> LangResult<Self> {
+    pub fn parse_from_tokens(tok: &'a [Token], c: &mut usize) -> LangResult<Self> {
         let is_partial = Self::tokens_with_partial(tok);
         let is_current = Self::tokens_with_single(tok);
         let is_full = Self::tokens_with_full(tok);
@@ -195,272 +201,94 @@ impl Entity {
         };
         Ok(r)
     }
-    #[inline(always)]
-    /// Parse an entity using the given [`Compiler`] instance. Internally this just evalutes it
-    /// using a token stream, finally forwarding the [`Compiler`]'s internal cursor depending on the
-    /// number of bytes consumed
-    pub(super) fn parse(cm: &mut Compiler) -> LangResult<Self> {
-        let sl = cm.remslice();
-        let mut c = 0;
-        let r = Self::parse_from_tokens(sl, &mut c);
-        unsafe {
-            cm.incr_cursor_by(c);
-        }
-        r
-    }
 }
 
 #[cfg_attr(test, derive(Debug, PartialEq))]
 /// A [`Statement`] is a fully BlueQL statement that can be executed by the query engine
 // TODO(@ohsayan): Determine whether we actually need this
-pub enum Statement {
+pub enum Statement<'a> {
     /// DDL query to switch between spaces and models
-    Use(Entity),
+    Use(Entity<'a>),
     /// DDL query to create a model
-    CreateModel(schema::Model),
+    CreateModel(schema::Model<'a>),
     /// DDL query to create a space
-    CreateSpace(schema::Space),
+    CreateSpace(schema::Space<'a>),
     /// DDL query to alter a space (properties)
-    AlterSpace(schema::AlterSpace),
+    AlterSpace(schema::AlterSpace<'a>),
     /// DDL query to alter a model (properties, field types, etc)
-    AlterModel(schema::Alter),
+    AlterModel(schema::Alter<'a>),
     /// DDL query to drop a model
     ///
     /// Conditions:
     /// - Model view is empty
     /// - Model is not in active use
-    DropModel(ddl::DropModel),
+    DropModel(ddl::DropModel<'a>),
     /// DDL query to drop a space
     ///
     /// Conditions:
     /// - Space doesn't have any other structures
     /// - Space is not in active use
-    DropSpace(ddl::DropSpace),
+    DropSpace(ddl::DropSpace<'a>),
     /// DDL query to inspect a space (returns a list of models in the space)
-    InspectSpace(RawSlice),
+    InspectSpace(Slice<'a>),
     /// DDL query to inspect a model (returns the model definition)
-    InspectModel(Entity),
+    InspectModel(Entity<'a>),
     /// DDL query to inspect all spaces (returns a list of spaces in the database)
     InspectSpaces,
+    /// DML insert
+    Insert(dml::InsertStatement<'a>),
+    /// DML select
+    Select(dml::SelectStatement<'a>),
+    /// DML update
+    Update(dml::UpdateStatement<'a>),
+    /// DML delete
+    Delete(dml::DeleteStatement<'a>),
 }
 
-/// A [`Compiler`] for BlueQL queries
-// TODO(@ohsayan): Decide whether we need this
-pub struct Compiler<'a> {
-    c: *const Token,
-    e: *const Token,
-    _lt: PhantomData<&'a [u8]>,
-}
-
-impl<'a> Compiler<'a> {
-    /// Compile a BlueQL query
-    pub fn compile(src: &'a [u8]) -> LangResult<Life<'a, Statement>> {
-        let token_stream = InsecureLexer::lex(src)?;
-        Self::new(&token_stream).compile_link_lt()
-    }
-    #[inline(always)]
-    /// Create a new [`Compiler`] instance
-    pub(super) const fn new(token_stream: &[Token]) -> Self {
-        unsafe {
-            Self {
-                c: token_stream.as_ptr(),
-                e: token_stream.as_ptr().add(token_stream.len()),
-                _lt: PhantomData,
+pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], mut qd: Qd) -> LangResult<Statement<'a>> {
+    let mut i = 0;
+    let ref mut qd = qd;
+    if compiler::unlikely(tok.len() < 2) {
+        return Err(LangError::UnexpectedEndofStatement);
+    }
+    match tok[0] {
+        // DDL
+        Token![use] => Entity::parse_from_tokens(&tok[1..], &mut i).map(Statement::Use),
+        Token![create] => match tok[1] {
+            Token![model] => schema::parse_schema_from_tokens(&tok[2..], qd).map(|(q, c)| {
+                i += c;
+                Statement::CreateModel(q)
+            }),
+            Token![space] => schema::parse_space_from_tokens(&tok[2..], qd).map(|(q, c)| {
+                i += c;
+                Statement::CreateSpace(q)
+            }),
+            _ => compiler::cold_rerr(LangError::UnknownCreateStatement),
+        },
+        Token![drop] if tok.len() >= 3 => ddl::parse_drop(&tok[1..], &mut i),
+        Token![alter] => match tok[1] {
+            Token![model] => schema::parse_alter_kind_from_tokens(&tok[2..], qd, &mut i)
+                .map(Statement::AlterModel),
+            Token![space] => {
+                schema::parse_alter_space_from_tokens(&tok[2..], qd).map(|(q, incr)| {
+                    i += incr;
+                    Statement::AlterSpace(q)
+                })
             }
+            _ => compiler::cold_rerr(LangError::UnknownAlterStatement),
+        },
+        Token::Ident(id) if id.eq_ignore_ascii_case(b"inspect") => {
+            ddl::parse_inspect(&tok[1..], &mut i)
         }
-    }
-    #[inline(always)]
-    /// Utility method to link a lifetime to the statement since the statement makes use of some
-    /// unsafe lifetime-free code that would otherwise cause the program to crash and burn
-    fn compile_link_lt(mut self) -> LangResult<Life<'a, Statement>> {
-        match self.stage0() {
-            Ok(t) if self.exhausted() => Ok(Life::new(t)),
-            Err(e) => Err(e),
-            _ => Err(LangError::UnexpectedToken),
-        }
-    }
-    #[inline(always)]
-    /// Stage 0: what statement
-    fn stage0(&mut self) -> Result<Statement, LangError> {
-        match self.nxtok_opt_forward() {
-            Some(Token![create]) => self.create0(),
-            Some(Token![drop]) => self.drop0(),
-            Some(Token![alter]) => self.alter0(),
-            Some(Token![describe]) => self.inspect0(),
-            Some(Token![use]) => self.use0(),
-            _ => Err(LangError::ExpectedStatement),
-        }
-    }
-    #[inline(always)]
-    /// Create 0: Create what (model/space)
-    fn create0(&mut self) -> Result<Statement, LangError> {
-        match self.nxtok_opt_forward() {
-            Some(Token![model]) => self.c_model0(),
-            Some(Token![space]) => self.c_space0(),
-            _ => Err(LangError::UnexpectedEndofStatement),
-        }
-    }
-    #[inline(always)]
-    /// Drop 0: Drop what (model/space)
-    fn drop0(&mut self) -> Result<Statement, LangError> {
-        let mut i = 0;
-        let r = ddl::parse_drop(self.remslice(), &mut i);
-        unsafe {
-            self.incr_cursor_by(i);
-        }
-        r
-    }
-    #[inline(always)]
-    /// Alter 0: Alter what (model/space)
-    fn alter0(&mut self) -> Result<Statement, LangError> {
-        match self.nxtok_opt_forward() {
-            Some(Token![model]) => self.alter_model(),
-            Some(Token![space]) => self.alter_space(),
-            Some(_) => Err(LangError::ExpectedStatement),
-            None => Err(LangError::UnexpectedEndofStatement),
-        }
-    }
-    #[inline(always)]
-    /// Alter model
-    fn alter_model(&mut self) -> Result<Statement, LangError> {
-        let mut c = 0;
-        let r =
-            schema::parse_alter_kind_from_tokens(self.remslice(), &mut InplaceData::new(), &mut c);
-        unsafe {
-            self.incr_cursor_by(c);
-        }
-        r.map(Statement::AlterModel)
-    }
-    #[inline(always)]
-    /// Alter space
-    fn alter_space(&mut self) -> Result<Statement, LangError> {
-        let (alter, i) =
-            schema::parse_alter_space_from_tokens(self.remslice(), &mut InplaceData::new())?;
-        unsafe {
-            self.incr_cursor_by(i);
-        }
-        Ok(Statement::AlterSpace(alter))
-    }
-    #[inline(always)]
-    /// Inspect 0: Inpsect what (model/space/spaces)
-    fn inspect0(&mut self) -> Result<Statement, LangError> {
-        let mut i = 0;
-        let r = ddl::parse_inspect(self.remslice(), &mut i);
-        unsafe {
-            self.incr_cursor_by(i);
-        }
-        r
-    }
-    #[inline(always)]
-    /// Parse an `use` query
-    fn use0(&mut self) -> Result<Statement, LangError> {
-        let entity = Entity::parse(self)?;
-        Ok(Statement::Use(entity))
-    }
-    #[inline(always)]
-    /// Create model
-    fn c_model0(&mut self) -> Result<Statement, LangError> {
-        let (model, i) =
-            schema::parse_schema_from_tokens(self.remslice(), &mut InplaceData::new())?;
-        unsafe {
-            self.incr_cursor_by(i);
-        }
-        Ok(Statement::CreateModel(model))
-    }
-    #[inline(always)]
-    /// Create space
-    fn c_space0(&mut self) -> Result<Statement, LangError> {
-        let (space, i) = schema::parse_space_from_tokens(self.remslice(), &mut InplaceData::new())?;
-        unsafe {
-            self.incr_cursor_by(i);
-        }
-        Ok(Statement::CreateSpace(space))
-    }
-}
-
-impl<'a> Compiler<'a> {
-    #[inline(always)]
-    /// Attempt to read the next token and forward the interal cursor if there is a cursor ahead
-    pub(super) fn nxtok_opt_forward<'b>(&mut self) -> Option<&'b Token>
-    where
-        'a: 'b,
-    {
-        if self.not_exhausted() {
-            unsafe {
-                let r = Some(&*self.c);
-                self.incr_cursor();
-                r
-            }
-        } else {
-            None
+        // DML
+        Token![insert] => dml::parse_insert(&tok[1..], qd, &mut i).map(Statement::Insert),
+        Token![select] => dml::parse_select(&tok[1..], qd, &mut i).map(Statement::Select),
+        Token![update] => {
+            dml::UpdateStatement::parse_update(&tok[1..], qd, &mut i).map(Statement::Update)
         }
-    }
-    #[inline(always)]
-    /// Returns the cursor
-    pub(super) const fn cursor(&self) -> *const Token {
-        self.c
-    }
-    #[inline(always)]
-    /// Returns the remaining buffer as a slice
-    pub(super) fn remslice(&'a self) -> &'a [Token] {
-        unsafe { slice::from_raw_parts(self.c, self.remaining()) }
-    }
-    #[inline(always)]
-    /// Check if the buffer is not exhausted
-    pub(super) fn not_exhausted(&self) -> bool {
-        self.c != self.e
-    }
-    #[inline(always)]
-    /// Check if the buffer is exhausted
-    pub(super) fn exhausted(&self) -> bool {
-        self.c == self.e
-    }
-    #[inline(always)]
-    /// Check the remaining bytes in the buffer
-    pub(super) fn remaining(&self) -> usize {
-        unsafe { self.e.offset_from(self.c) as usize }
-    }
-    /// Deref the cursor
-    ///
-    /// ## Safety
-    ///
-    /// Have to ensure it isn't pointing to garbage i.e beyond EOA
-    pub(super) unsafe fn deref_cursor(&self) -> &Token {
-        &*self.c
-    }
-    /// Increment the cursor if the next token matches the given token
-    pub(super) fn peek_eq_and_forward(&mut self, t: Token) -> bool {
-        let did_fw = self.not_exhausted() && unsafe { self.deref_cursor() == &t };
-        unsafe {
-            self.incr_cursor_if(did_fw);
+        Token![delete] => {
+            dml::DeleteStatement::parse_delete(&tok[1..], qd, &mut i).map(Statement::Delete)
         }
-        did_fw
-    }
-    #[inline(always)]
-    /// Increment the cursor
-    ///
-    /// ## Safety
-    ///
-    /// Should be >= EOA
-    pub(super) unsafe fn incr_cursor(&mut self) {
-        self.incr_cursor_by(1)
-    }
-    /// Increment the cursor if the given boolean expr is satisified
-    ///
-    /// ## Safety
-    ///
-    /// Should be >= EOA (if true)
-    pub(super) unsafe fn incr_cursor_if(&mut self, did_fw: bool) {
-        self.incr_cursor_by(did_fw as _)
-    }
-    #[inline(always)]
-    /// Increment the cursor by the given count
-    ///
-    /// ## Safety
-    ///
-    /// >= EOA (if nonzero)
-    pub(super) unsafe fn incr_cursor_by(&mut self, by: usize) {
-        debug_assert!(self.remaining() >= by);
-        self.c = self.c.add(by);
+        _ => compiler::cold_rerr(LangError::ExpectedStatement),
     }
 }
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index 226da785..b24c1c3a 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -42,10 +42,7 @@ use {crate::engine::ql::tests::lex_insecure, test::Bencher};
 mod lexer {
     use {
         super::*,
-        crate::engine::ql::{
-            lexer::{Lit, Token},
-            RawSlice,
-        },
+        crate::engine::ql::lexer::{Lit, Token},
     };
     #[bench]
     fn lex_number(b: &mut Bencher) {
@@ -74,9 +71,9 @@ mod lexer {
     #[bench]
     fn lex_raw_literal(b: &mut Bencher) {
         let src = b"\r44\ne69b10ffcc250ae5091dec6f299072e23b0b41d6a739";
-        let expected = vec![Token::Lit(Lit::Bin(RawSlice::from(
-            "e69b10ffcc250ae5091dec6f299072e23b0b41d6a739",
-        )))];
+        let expected = vec![Token::Lit(Lit::Bin(
+            b"e69b10ffcc250ae5091dec6f299072e23b0b41d6a739",
+        ))];
         b.iter(|| assert_eq!(lex_insecure(src).unwrap(), expected));
     }
 }
@@ -85,7 +82,7 @@ mod ast {
     use {super::*, crate::engine::ql::ast::Entity};
     #[bench]
     fn parse_entity_single(b: &mut Bencher) {
-        let e = Entity::Single("tweeter".into());
+        let e = Entity::Single(b"tweeter");
         b.iter(|| {
             let src = lex_insecure(b"tweeter").unwrap();
             let mut i = 0;
@@ -95,7 +92,7 @@ mod ast {
     }
     #[bench]
     fn parse_entity_double(b: &mut Bencher) {
-        let e = ("tweeter", "user").into();
+        let e = Entity::Full(b"tweeter", b"user");
         b.iter(|| {
             let src = lex_insecure(b"tweeter.user").unwrap();
             let mut i = 0;
@@ -105,7 +102,7 @@ mod ast {
     }
     #[bench]
     fn parse_entity_partial(b: &mut Bencher) {
-        let e = Entity::Partial("user".into());
+        let e = Entity::Partial(b"user");
         b.iter(|| {
             let src = lex_insecure(b":user").unwrap();
             let mut i = 0;
@@ -118,21 +115,30 @@ mod ast {
 mod ddl_queries {
     use {
         super::*,
-        crate::engine::ql::ast::{Compiler, Entity, Statement},
+        crate::engine::ql::{
+            ast::{compile, Entity, InplaceData, Statement},
+            lexer::InsecureLexer,
+        },
     };
     mod use_stmt {
         use super::*;
         #[bench]
         fn use_space(b: &mut Bencher) {
             let src = b"use myspace";
-            let expected = Statement::Use(Entity::Single("myspace".into()));
-            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+            let expected = Statement::Use(Entity::Single(b"myspace"));
+            b.iter(|| {
+                let lexed = InsecureLexer::lex(src).unwrap();
+                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
+            });
         }
         #[bench]
         fn use_model(b: &mut Bencher) {
             let src = b"use myspace.mymodel";
-            let expected = Statement::Use(("myspace", "mymodel").into());
-            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+            let expected = Statement::Use(Entity::Full(b"myspace", b"mymodel"));
+            b.iter(|| {
+                let lexed = InsecureLexer::lex(src).unwrap();
+                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
+            });
         }
     }
     mod inspect_stmt {
@@ -140,26 +146,38 @@ mod ddl_queries {
         #[bench]
         fn inspect_space(b: &mut Bencher) {
             let src = b"inspect space myspace";
-            let expected = Statement::InspectSpace("myspace".into());
-            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+            let expected = Statement::InspectSpace(b"myspace");
+            b.iter(|| {
+                let lexed = InsecureLexer::lex(src).unwrap();
+                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
+            });
         }
         #[bench]
         fn inspect_model_single_entity(b: &mut Bencher) {
             let src = b"inspect model mymodel";
-            let expected = Statement::InspectModel(Entity::Single("mymodel".into()));
-            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+            let expected = Statement::InspectModel(Entity::Single(b"mymodel"));
+            b.iter(|| {
+                let lexed = InsecureLexer::lex(src).unwrap();
+                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
+            });
         }
         #[bench]
         fn inspect_model_full_entity(b: &mut Bencher) {
             let src = b"inspect model myspace.mymodel";
-            let expected = Statement::InspectModel(("myspace", "mymodel").into());
-            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+            let expected = Statement::InspectModel(Entity::Full(b"myspace", b"mymodel"));
+            b.iter(|| {
+                let lexed = InsecureLexer::lex(src).unwrap();
+                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
+            });
         }
         #[bench]
         fn inspect_spaces(b: &mut Bencher) {
             let src = b"inspect spaces";
             let expected = Statement::InspectSpaces;
-            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+            b.iter(|| {
+                let lexed = InsecureLexer::lex(src).unwrap();
+                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
+            });
         }
     }
     mod drop_stmt {
@@ -170,42 +188,58 @@ mod ddl_queries {
         #[bench]
         fn drop_space(b: &mut Bencher) {
             let src = b"drop space myspace";
-            let expected = Statement::DropSpace(DropSpace::new("myspace".into(), false));
-            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+            let expected = Statement::DropSpace(DropSpace::new(b"myspace", false));
+            b.iter(|| {
+                let lexed = InsecureLexer::lex(src).unwrap();
+                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
+            });
         }
         #[bench]
         fn drop_space_force(b: &mut Bencher) {
             let src = b"drop space myspace force";
-            let expected = Statement::DropSpace(DropSpace::new("myspace".into(), true));
-            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+            let expected = Statement::DropSpace(DropSpace::new(b"myspace", true));
+            b.iter(|| {
+                let lexed = InsecureLexer::lex(src).unwrap();
+                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
+            });
         }
         #[bench]
         fn drop_model_single(b: &mut Bencher) {
             let src = b"drop model mymodel";
-            let expected =
-                Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), false));
-            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+            let expected = Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), false));
+            b.iter(|| {
+                let lexed = InsecureLexer::lex(src).unwrap();
+                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
+            });
         }
         #[bench]
         fn drop_model_single_force(b: &mut Bencher) {
             let src = b"drop model mymodel force";
-            let expected =
-                Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), true));
-            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+            let expected = Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), true));
+            b.iter(|| {
+                let lexed = InsecureLexer::lex(src).unwrap();
+                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
+            });
         }
         #[bench]
         fn drop_model_full(b: &mut Bencher) {
             let src = b"drop model myspace.mymodel";
             let expected =
-                Statement::DropModel(DropModel::new(("myspace", "mymodel").into(), false));
-            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+                Statement::DropModel(DropModel::new(Entity::Full(b"myspace", b"mymodel"), false));
+            b.iter(|| {
+                let lexed = InsecureLexer::lex(src).unwrap();
+                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
+            });
         }
         #[bench]
         fn drop_model_full_force(b: &mut Bencher) {
             let src = b"drop model myspace.mymodel force";
             let expected =
-                Statement::DropModel(DropModel::new(("myspace", "mymodel").into(), true));
-            b.iter(|| assert_eq!(Compiler::compile(src).unwrap(), expected));
+                Statement::DropModel(DropModel::new(Entity::Full(b"myspace", b"mymodel"), true));
+            b.iter(|| {
+                let lexed = InsecureLexer::lex(src).unwrap();
+                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
+            });
         }
     }
 }
diff --git a/server/src/engine/ql/ddl.rs b/server/src/engine/ql/ddl.rs
index 396dbb7e..d984ecd0 100644
--- a/server/src/engine/ql/ddl.rs
+++ b/server/src/engine/ql/ddl.rs
@@ -26,46 +26,49 @@
 
 use super::{
     ast::{Entity, Statement},
-    lexer::Token,
-    LangError, LangResult, RawSlice,
+    lexer::{Slice, Token},
+    LangError, LangResult,
 };
 
 #[derive(Debug, PartialEq)]
 /// A generic representation of `drop` query
-pub struct DropSpace {
-    pub(super) space: RawSlice,
+pub struct DropSpace<'a> {
+    pub(super) space: Slice<'a>,
     pub(super) force: bool,
 }
 
-impl DropSpace {
+impl<'a> DropSpace<'a> {
     #[inline(always)]
     /// Instantiate
-    pub(super) const fn new(space: RawSlice, force: bool) -> Self {
+    pub(super) const fn new(space: Slice<'a>, force: bool) -> Self {
         Self { space, force }
     }
 }
 
 #[derive(Debug, PartialEq)]
-pub struct DropModel {
-    pub(super) entity: Entity,
+pub struct DropModel<'a> {
+    pub(super) entity: Entity<'a>,
     pub(super) force: bool,
 }
 
-impl DropModel {
+impl<'a> DropModel<'a> {
     #[inline(always)]
-    pub fn new(entity: Entity, force: bool) -> Self {
+    pub fn new(entity: Entity<'a>, force: bool) -> Self {
         Self { entity, force }
     }
 }
 
 // drop (<space> | <model>) <ident> [<force>]
-pub(super) fn parse_drop(tok: &[Token], counter: &mut usize) -> LangResult<Statement> {
-    match tok.get(0) {
-        Some(Token![model]) => {
+/// ## Panic
+///
+/// If token stream length is < 2
+pub(super) fn parse_drop<'a>(tok: &'a [Token], counter: &mut usize) -> LangResult<Statement<'a>> {
+    match tok[0] {
+        Token![model] => {
             // we have a model. now parse entity and see if we should force deletion
             let mut i = 1;
             let e = Entity::parse_from_tokens(&tok[1..], &mut i)?;
-            let force = i < tok.len() && tok[i] == Token::Ident("force".into());
+            let force = i < tok.len() && tok[i] == Token::Ident(b"force");
             i += force as usize;
             *counter += i;
             // if we've exhausted the stream, we're good to go (either `force`, or nothing)
@@ -73,10 +76,10 @@ pub(super) fn parse_drop(tok: &[Token], counter: &mut usize) -> LangResult<State
                 return Ok(Statement::DropModel(DropModel::new(e, force)));
             }
         }
-        Some(Token![space]) if tok.len() > 1 && tok[1].is_ident() => {
+        Token![space] if tok[1].is_ident() => {
             let mut i = 2; // (`space` and space name)
                            // should we force drop?
-            let force = i < tok.len() && tok[i] == Token::Ident("force".into());
+            let force = i < tok.len() && tok[i] == Token::Ident(b"force");
             i += force as usize;
             *counter += i;
             // either `force` or nothing
@@ -84,7 +87,7 @@ pub(super) fn parse_drop(tok: &[Token], counter: &mut usize) -> LangResult<State
                 return Ok(Statement::DropSpace(DropSpace::new(
                     unsafe {
                         // UNSAFE(@ohsayan): Safe because the match predicate ensures that tok[1] is indeed an ident
-                        extract!(tok[1], Token::Ident(ref space) => space.clone())
+                        extract!(tok[1], Token::Ident(ref space) => *space)
                     },
                     force,
                 )));
@@ -96,14 +99,14 @@ pub(super) fn parse_drop(tok: &[Token], counter: &mut usize) -> LangResult<State
 }
 
 #[cfg(test)]
-pub(super) fn parse_drop_full(tok: &[Token]) -> LangResult<Statement> {
+pub(super) fn parse_drop_full<'a>(tok: &'a [Token]) -> LangResult<Statement<'a>> {
     let mut i = 0;
     let r = self::parse_drop(tok, &mut i);
     assert_full_tt!(i, tok.len());
     r
 }
 
-pub(super) fn parse_inspect(tok: &[Token], c: &mut usize) -> LangResult<Statement> {
+pub(super) fn parse_inspect<'a>(tok: &'a [Token], c: &mut usize) -> LangResult<Statement<'a>> {
     /*
         inpsect model <entity>
         inspect space <entity>
@@ -118,15 +121,10 @@ pub(super) fn parse_inspect(tok: &[Token], c: &mut usize) -> LangResult<Statemen
             *c += 1;
             Ok(Statement::InspectSpace(unsafe {
                 // UNSAFE(@ohsayan): Safe because of the match predicate
-                extract!(tok[1], Token::Ident(ref space) => space.clone())
+                extract!(tok[1], Token::Ident(ref space) => space)
             }))
         }
-        Some(Token::Ident(id))
-            if unsafe {
-                // UNSAFE(@ohsayan): Token lifetime ensures validity of slice
-                id.as_slice().eq_ignore_ascii_case(b"spaces")
-            } && tok.len() == 1 =>
-        {
+        Some(Token::Ident(id)) if id.eq_ignore_ascii_case(b"spaces") && tok.len() == 1 => {
             Ok(Statement::InspectSpaces)
         }
         _ => Err(LangError::ExpectedStatement),
@@ -134,7 +132,7 @@ pub(super) fn parse_inspect(tok: &[Token], c: &mut usize) -> LangResult<Statemen
 }
 
 #[cfg(test)]
-pub(super) fn parse_inspect_full(tok: &[Token]) -> LangResult<Statement> {
+pub(super) fn parse_inspect_full<'a>(tok: &'a [Token]) -> LangResult<Statement<'a>> {
     let mut i = 0;
     let r = self::parse_inspect(tok, &mut i);
     assert_full_tt!(i, tok.len());
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
index ea7cdd88..7d28f386 100644
--- a/server/src/engine/ql/dml.rs
+++ b/server/src/engine/ql/dml.rs
@@ -35,7 +35,7 @@ use {
     super::{
         ast::{Entity, QueryData},
         lexer::{LitIR, Symbol, Token},
-        LangError, LangResult, RawSlice,
+        LangError, LangResult,
     },
     crate::{
         engine::memory::DataType,
@@ -58,7 +58,7 @@ fn minidx<T>(src: &[T], index: usize) -> usize {
 */
 
 #[inline(always)]
-fn process_entity(tok: &[Token], d: &mut MaybeInit<Entity>, i: &mut usize) -> bool {
+fn process_entity<'a>(tok: &'a [Token<'a>], d: &mut MaybeInit<Entity<'a>>, i: &mut usize) -> bool {
     let is_full = Entity::tokens_with_full(tok);
     let is_single = Entity::tokens_with_single(tok);
     if is_full {
@@ -147,7 +147,7 @@ impl<'a> RelationalExpr<'a> {
                 // UNSAFE(@ohsayan): tok[0] is checked for being an ident, tok[lit_idx] also checked to be a lit
                 let lit = Qd::read_lit(d, &tok[lit_idx]);
                 Some(Self::new(
-                    extract!(tok[0], Token::Ident(ref id) => id.as_slice()),
+                    extract!(tok[0], Token::Ident(ref id) => id),
                     lit,
                     operator,
                 ))
@@ -378,10 +378,7 @@ pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
             (Token::Ident(id), tok) if Qd::can_read_lit_from(d, tok) => {
                 okay &= data
                     .insert(
-                        unsafe {
-                            // UNSAFE(@ohsayan): Token lifetime ensures slice validity
-                            id.as_slice()
-                        },
+                        *id,
                         Some(unsafe {
                             // UNSAFE(@ohsayan): Token LT0 guarantees LT0 > LT1 for lit
                             DataType::clone_from_litir(Qd::read_lit(d, tok))
@@ -395,26 +392,10 @@ pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
                 let (_, lst_i, lst_ok) = parse_list(&tok[i + 3..], d, &mut l);
                 okay &= lst_ok;
                 i += lst_i;
-                okay &= data
-                    .insert(
-                        unsafe {
-                            // UNSAFE(@ohsayan): Token lifetime ensures validity
-                            id.as_slice()
-                        },
-                        Some(l.into()),
-                    )
-                    .is_none();
+                okay &= data.insert(id, Some(l.into())).is_none();
             }
             (Token::Ident(id), Token![null]) => {
-                okay &= data
-                    .insert(
-                        unsafe {
-                            // UNSAFE(@ohsayan): Token lifetime ensures validity
-                            id.as_slice()
-                        },
-                        None,
-                    )
-                    .is_none();
+                okay &= data.insert(id, None).is_none();
             }
             _ => {
                 okay = false;
@@ -473,12 +454,12 @@ impl<'a> From<HashMap<&'static [u8], Option<DataType>>> for InsertData<'a> {
 
 #[derive(Debug, PartialEq)]
 pub struct InsertStatement<'a> {
-    pub(super) entity: Entity,
+    pub(super) entity: Entity<'a>,
     pub(super) data: InsertData<'a>,
 }
 
 #[inline(always)]
-fn parse_entity(tok: &[Token], entity: &mut MaybeInit<Entity>, i: &mut usize) -> bool {
+fn parse_entity<'a>(tok: &'a [Token], entity: &mut MaybeInit<Entity<'a>>, i: &mut usize) -> bool {
     let is_full = tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident();
     let is_half = tok[0].is_ident();
     unsafe {
@@ -565,11 +546,11 @@ pub(super) fn parse_insert_full<'a>(tok: &'a [Token]) -> Option<InsertStatement<
 */
 
 #[derive(Debug, PartialEq)]
-pub(super) struct SelectStatement<'a> {
+pub struct SelectStatement<'a> {
     /// the entity
-    pub(super) entity: Entity,
+    pub(super) entity: Entity<'a>,
     /// fields in order of querying. will be zero when wildcard is set
-    pub(super) fields: Vec<RawSlice>,
+    pub(super) fields: Vec<&'a [u8]>,
     /// whether a wildcard was passed
     pub(super) wildcard: bool,
     /// where clause
@@ -578,8 +559,8 @@ pub(super) struct SelectStatement<'a> {
 impl<'a> SelectStatement<'a> {
     #[inline(always)]
     pub(crate) fn new_test(
-        entity: Entity,
-        fields: Vec<RawSlice>,
+        entity: Entity<'a>,
+        fields: Vec<&'a [u8]>,
         wildcard: bool,
         clauses: WhereClauseCollection<'a>,
     ) -> SelectStatement<'a> {
@@ -587,8 +568,8 @@ impl<'a> SelectStatement<'a> {
     }
     #[inline(always)]
     fn new(
-        entity: Entity,
-        fields: Vec<RawSlice>,
+        entity: Entity<'a>,
+        fields: Vec<&'a [u8]>,
         wildcard: bool,
         clauses: WhereClauseCollection<'a>,
     ) -> SelectStatement<'a> {
@@ -701,7 +682,7 @@ static OPERATOR: [Operator; 6] = [
 #[derive(Debug, PartialEq)]
 pub struct AssignmentExpression<'a> {
     /// the LHS ident
-    pub(super) lhs: RawSlice,
+    pub(super) lhs: &'a [u8],
     /// the RHS lit
     pub(super) rhs: LitIR<'a>,
     /// operator
@@ -709,7 +690,7 @@ pub struct AssignmentExpression<'a> {
 }
 
 impl<'a> AssignmentExpression<'a> {
-    pub(super) fn new(lhs: RawSlice, rhs: LitIR<'a>, operator_fn: Operator) -> Self {
+    pub(super) fn new(lhs: &'a [u8], rhs: LitIR<'a>, operator_fn: Operator) -> Self {
         Self {
             lhs,
             rhs,
@@ -768,7 +749,7 @@ impl<'a> AssignmentExpression<'a> {
                 */
                 let rhs = Qd::read_lit(d, &tok[i]);
                 AssignmentExpression {
-                    lhs: extract!(tok[0], Token::Ident(ref r) => r.clone()),
+                    lhs: extract!(tok[0], Token::Ident(ref r) => r),
                     rhs,
                     operator_fn: OPERATOR[operator_code as usize],
                 }
@@ -803,7 +784,7 @@ pub(super) fn parse_expression_full<'a>(tok: &'a [Token]) -> Option<AssignmentEx
 
 #[derive(Debug, PartialEq)]
 pub struct UpdateStatement<'a> {
-    pub(super) entity: Entity,
+    pub(super) entity: Entity<'a>,
     pub(super) expressions: Vec<AssignmentExpression<'a>>,
     pub(super) wc: WhereClause<'a>,
 }
@@ -812,7 +793,7 @@ impl<'a> UpdateStatement<'a> {
     #[inline(always)]
     #[cfg(test)]
     pub fn new_test(
-        entity: Entity,
+        entity: Entity<'a>,
         expressions: Vec<AssignmentExpression<'a>>,
         wc: WhereClauseCollection<'a>,
     ) -> Self {
@@ -820,7 +801,7 @@ impl<'a> UpdateStatement<'a> {
     }
     #[inline(always)]
     pub fn new(
-        entity: Entity,
+        entity: Entity<'a>,
         expressions: Vec<AssignmentExpression<'a>>,
         wc: WhereClause<'a>,
     ) -> Self {
@@ -911,19 +892,19 @@ pub(super) fn parse_update_full<'a>(tok: &'a [Token]) -> LangResult<UpdateStatem
 */
 
 #[derive(Debug, PartialEq)]
-pub(super) struct DeleteStatement<'a> {
-    pub(super) entity: Entity,
+pub struct DeleteStatement<'a> {
+    pub(super) entity: Entity<'a>,
     pub(super) wc: WhereClause<'a>,
 }
 
 impl<'a> DeleteStatement<'a> {
     #[inline(always)]
-    pub(super) fn new(entity: Entity, wc: WhereClause<'a>) -> Self {
+    pub(super) fn new(entity: Entity<'a>, wc: WhereClause<'a>) -> Self {
         Self { entity, wc }
     }
     #[inline(always)]
     #[cfg(test)]
-    pub(super) fn new_test(entity: Entity, wc: WhereClauseCollection<'a>) -> Self {
+    pub(super) fn new_test(entity: Entity<'a>, wc: WhereClauseCollection<'a>) -> Self {
         Self::new(entity, WhereClause::new(wc))
     }
     pub(super) fn parse_delete<Qd: QueryData<'a>>(
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 1e1b7b90..e64d530b 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -25,27 +25,29 @@
 */
 
 use {
-    super::{LangError, LangResult, RawSlice},
+    super::{LangError, LangResult},
     crate::util::compiler,
-    core::{cmp, fmt, marker::PhantomData, mem::size_of, ops::BitOr, slice, str},
+    core::{cmp, fmt, mem::size_of, ops::BitOr, slice, str},
 };
 
+pub type Slice<'a> = &'a [u8];
+
 /*
     Lex meta
 */
 
 #[derive(Debug, PartialEq, Clone)]
-pub enum Token {
+pub enum Token<'a> {
     Symbol(Symbol),
     Keyword(Keyword),
-    Ident(RawSlice),
+    Ident(Slice<'a>),
     #[cfg(test)]
     /// A comma that can be ignored (used for fuzzing)
     IgnorableComma,
-    Lit(Lit), // literal
+    Lit(Lit<'a>), // literal
 }
 
-impl PartialEq<Symbol> for Token {
+impl<'a> PartialEq<Symbol> for Token<'a> {
     fn eq(&self, other: &Symbol) -> bool {
         match self {
             Self::Symbol(s) => s == other,
@@ -70,43 +72,43 @@ assertions! {
 }
 
 enum_impls! {
-    Token => {
+    Token<'a> => {
         Keyword as Keyword,
         Symbol as Symbol,
-        Lit as Lit,
+        Lit<'a> as Lit,
     }
 }
 
 #[derive(Debug, PartialEq, Clone)]
 #[repr(u8)]
-pub enum Lit {
+pub enum Lit<'a> {
     Str(Box<str>),
     Bool(bool),
     UnsignedInt(u64),
     SignedInt(i64),
-    Bin(RawSlice),
+    Bin(Slice<'a>),
 }
 
-impl Lit {
-    pub(super) unsafe fn as_ir<'a>(&'a self) -> LitIR<'a> {
+impl<'a> Lit<'a> {
+    pub(super) fn as_ir(&'a self) -> LitIR<'a> {
         match self {
             Self::Str(s) => LitIR::Str(s.as_ref()),
             Self::Bool(b) => LitIR::Bool(*b),
             Self::UnsignedInt(u) => LitIR::UInt(*u),
             Self::SignedInt(s) => LitIR::SInt(*s),
-            Self::Bin(b) => LitIR::Bin(b.as_slice()),
+            Self::Bin(b) => LitIR::Bin(b),
         }
     }
 }
 
-impl From<&'static str> for Lit {
+impl<'a> From<&'static str> for Lit<'a> {
     fn from(s: &'static str) -> Self {
         Self::Str(s.into())
     }
 }
 
 enum_impls! {
-    Lit => {
+    Lit<'a> => {
         Box<str> as Str,
         String as Str,
         bool as Bool,
@@ -183,7 +185,7 @@ pub enum Keyword {
     Select,
     Exists,
     Update,
-    Delere,
+    Delete,
     Into,
     From,
     As,
@@ -210,10 +212,7 @@ pub enum Keyword {
     And,
     Or,
     Not,
-    User,
-    Revoke,
     Null,
-    Infinity,
 }
 
 /*
@@ -294,7 +293,7 @@ fn symof(sym: u8) -> Option<Symbol> {
     }
 }
 
-static KW_LUT: [(&[u8], Keyword); 60] = [
+static KW_LUT: [(&[u8], Keyword); 57] = [
     (b"table", Keyword::Table),
     (b"model", Keyword::Model),
     (b"space", Keyword::Space),
@@ -324,7 +323,7 @@ static KW_LUT: [(&[u8], Keyword); 60] = [
     (b"select", Keyword::Select),
     (b"exists", Keyword::Exists),
     (b"update", Keyword::Update),
-    (b"delere", Keyword::Delere),
+    (b"delete", Keyword::Delete),
     (b"into", Keyword::Into),
     (b"from", Keyword::From),
     (b"as", Keyword::As),
@@ -351,20 +350,17 @@ static KW_LUT: [(&[u8], Keyword); 60] = [
     (b"and", Keyword::And),
     (b"or", Keyword::Or),
     (b"not", Keyword::Not),
-    (b"user", Keyword::User),
-    (b"revoke", Keyword::Revoke),
     (b"null", Keyword::Null),
-    (b"infinity", Keyword::Infinity),
 ];
 
-static KWG: [u8; 64] = [
-    0, 55, 32, 25, 4, 21, 51, 43, 28, 59, 34, 1, 9, 39, 5, 49, 0, 16, 29, 0, 48, 0, 17, 60, 19, 21,
-    26, 18, 0, 41, 55, 10, 48, 62, 55, 35, 56, 18, 29, 41, 5, 46, 25, 52, 32, 26, 27, 17, 61, 60,
-    61, 59, 24, 12, 17, 30, 53, 4, 17, 0, 6, 2, 45, 56,
+static KWG: [u8; 63] = [
+    0, 24, 15, 29, 51, 53, 44, 38, 43, 4, 27, 1, 37, 57, 32, 0, 46, 24, 59, 45, 32, 52, 8, 0, 23,
+    19, 33, 48, 56, 60, 33, 53, 18, 47, 49, 53, 2, 19, 1, 34, 19, 58, 11, 5, 0, 41, 27, 24, 20, 2,
+    0, 0, 48, 2, 42, 46, 43, 0, 18, 33, 21, 12, 41,
 ];
 
-const KWMG_1: [u8; 11] = *b"nJEcjrLflKX";
-const KWMG_2: [u8; 11] = *b"KWHPUPK3Fh3";
+const KWMG_1: [u8; 11] = *b"MpVBwC1vsCy";
+const KWMG_2: [u8; 11] = *b"m7sNd9mtGzC";
 const KWMG_S: usize = KWMG_1.len();
 
 #[inline(always)]
@@ -393,7 +389,7 @@ fn kwof(key: &[u8]) -> Option<Keyword> {
     }
 }
 
-impl Token {
+impl<'a> Token<'a> {
     #[inline(always)]
     pub(crate) const fn is_ident(&self) -> bool {
         matches!(self, Token::Ident(_))
@@ -404,9 +400,9 @@ impl Token {
     }
 }
 
-impl AsRef<Token> for Token {
+impl<'a> AsRef<Token<'a>> for Token<'a> {
     #[inline(always)]
-    fn as_ref(&self) -> &Token {
+    fn as_ref(&self) -> &Token<'a> {
         self
     }
 }
@@ -419,15 +415,14 @@ impl AsRef<Token> for Token {
 pub struct RawLexer<'a> {
     c: *const u8,
     e: *const u8,
-    tokens: Vec<Token>,
+    tokens: Vec<Token<'a>>,
     last_error: Option<LangError>,
-    _lt: PhantomData<&'a [u8]>,
 }
 
 // ctor
 impl<'a> RawLexer<'a> {
     #[inline(always)]
-    pub const fn new(src: &'a [u8]) -> Self {
+    pub const fn new(src: Slice<'a>) -> Self {
         Self {
             c: src.as_ptr(),
             e: unsafe {
@@ -436,7 +431,6 @@ impl<'a> RawLexer<'a> {
             },
             last_error: None,
             tokens: Vec::new(),
-            _lt: PhantomData,
         }
     }
 }
@@ -481,7 +475,7 @@ impl<'a> RawLexer<'a> {
         self.incr_cursor_by(iff as usize)
     }
     #[inline(always)]
-    fn push_token(&mut self, token: impl Into<Token>) {
+    fn push_token(&mut self, token: impl Into<Token<'a>>) {
         self.tokens.push(token.into())
     }
     #[inline(always)]
@@ -533,19 +527,19 @@ impl<'a> RawLexer<'a> {
 // high level methods
 impl<'a> RawLexer<'a> {
     #[inline(always)]
-    fn scan_ident(&mut self) -> RawSlice {
+    fn scan_ident(&mut self) -> Slice<'a> {
         let s = self.cursor();
         unsafe {
             while self.peek_is(|b| b.is_ascii_alphanumeric() || b == b'_') {
                 self.incr_cursor();
             }
-            RawSlice::new(s, self.cursor().offset_from(s) as usize)
+            slice::from_raw_parts(s, self.cursor().offset_from(s) as usize)
         }
     }
     #[inline(always)]
     fn scan_ident_or_keyword(&mut self) {
         let s = self.scan_ident();
-        let st = unsafe { s.as_slice() }.to_ascii_lowercase();
+        let st = s.to_ascii_lowercase();
         match kwof(&st) {
             Some(kw) => self.tokens.push(kw.into()),
             // FIXME(@ohsayan): Uh, mind fixing this? The only advantage is that I can keep the graph *memory* footprint small
@@ -573,13 +567,13 @@ pub struct InsecureLexer<'a> {
 
 impl<'a> InsecureLexer<'a> {
     #[inline(always)]
-    pub const fn new(src: &'a [u8]) -> Self {
+    pub const fn new(src: Slice<'a>) -> Self {
         Self {
             base: RawLexer::new(src),
         }
     }
     #[inline(always)]
-    pub fn lex(src: &'a [u8]) -> LangResult<Vec<Token>> {
+    pub fn lex(src: Slice<'a>) -> LangResult<Vec<Token<'a>>> {
         let mut slf = Self::new(src);
         slf._lex();
         let RawLexer {
@@ -709,7 +703,7 @@ impl<'a> InsecureLexer<'a> {
         okay &= slf.remaining() >= size;
         if compiler::likely(okay) {
             unsafe {
-                slf.push_token(Lit::Bin(RawSlice::new(slf.cursor(), size)));
+                slf.push_token(Lit::Bin(slice::from_raw_parts(slf.cursor(), size)));
                 slf.incr_cursor_by(size);
             }
         } else {
@@ -764,17 +758,17 @@ pub struct SafeLexer<'a> {
 
 impl<'a> SafeLexer<'a> {
     #[inline(always)]
-    pub const fn new(src: &'a [u8]) -> Self {
+    pub const fn new(src: Slice<'a>) -> Self {
         Self {
             base: RawLexer::new(src),
         }
     }
     #[inline(always)]
-    pub fn lex(src: &'a [u8]) -> LangResult<Vec<Token>> {
+    pub fn lex(src: Slice<'a>) -> LangResult<Vec<Token>> {
         Self::new(src)._lex()
     }
     #[inline(always)]
-    fn _lex(self) -> LangResult<Vec<Token>> {
+    fn _lex(self) -> LangResult<Vec<Token<'a>>> {
         let Self { base: mut l } = self;
         while l.not_exhausted() && l.no_error() {
             let b = unsafe { l.deref_cursor() };
@@ -918,7 +912,7 @@ where
 /// Intermediate literal repr
 pub enum LitIR<'a> {
     Str(&'a str),
-    Bin(&'a [u8]),
+    Bin(Slice<'a>),
     UInt(u64),
     SInt(i64),
     Bool(bool),
@@ -952,16 +946,16 @@ pub enum LitIROwned {
 /// Data constructed from `opmode-safe`
 pub struct SafeQueryData<'a> {
     p: Box<[LitIR<'a>]>,
-    t: Vec<Token>,
+    t: Vec<Token<'a>>,
 }
 
 impl<'a> SafeQueryData<'a> {
     #[cfg(test)]
-    pub fn new_test(p: Box<[LitIR<'a>]>, t: Vec<Token>) -> Self {
+    pub fn new_test(p: Box<[LitIR<'a>]>, t: Vec<Token<'a>>) -> Self {
         Self { p, t }
     }
     #[inline(always)]
-    pub fn parse(qf: &'a [u8], pf: &'a [u8], pf_sz: usize) -> LangResult<Self> {
+    pub fn parse(qf: Slice<'a>, pf: Slice<'a>, pf_sz: usize) -> LangResult<Self> {
         let q = SafeLexer::lex(qf);
         let p = Self::p_revloop(pf, pf_sz);
         let (Ok(t), Ok(p)) = (q, p) else {
@@ -970,8 +964,8 @@ impl<'a> SafeQueryData<'a> {
         Ok(Self { p, t })
     }
     #[inline]
-    pub(super) fn p_revloop(mut src: &'a [u8], size: usize) -> LangResult<Box<[LitIR<'a>]>> {
-        static LITIR_TF: [for<'a> fn(&'a [u8], &mut usize, &mut Vec<LitIR<'a>>) -> bool; 7] = [
+    pub(super) fn p_revloop(mut src: Slice<'a>, size: usize) -> LangResult<Box<[LitIR<'a>]>> {
+        static LITIR_TF: [for<'a> fn(Slice<'a>, &mut usize, &mut Vec<LitIR<'a>>) -> bool; 7] = [
             SafeQueryData::uint,  // tc: 0
             SafeQueryData::sint,  // tc: 1
             SafeQueryData::bool,  // tc: 2
@@ -1004,7 +998,7 @@ impl<'a> SafeQueryData<'a> {
 // low level methods
 impl<'b> SafeQueryData<'b> {
     #[inline(always)]
-    fn mxple<'a>(src: &'a [u8], cnt: &mut usize, flag: &mut bool) -> &'a [u8] {
+    fn mxple<'a>(src: Slice<'a>, cnt: &mut usize, flag: &mut bool) -> Slice<'a> {
         // find payload length
         let mut i = 0;
         let payload_len = decode_num_ub::<usize>(src, flag, &mut i);
@@ -1018,21 +1012,21 @@ impl<'b> SafeQueryData<'b> {
         unsafe { slice::from_raw_parts(src.as_ptr(), mx_extract) }
     }
     #[inline(always)]
-    pub(super) fn uint<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
+    pub(super) fn uint<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
         let mut b = true;
         let r = decode_num_ub(src, &mut b, cnt);
         data.push(LitIR::UInt(r));
         b
     }
     #[inline(always)]
-    pub(super) fn sint<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
+    pub(super) fn sint<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
         let mut b = true;
         let r = decode_num_ub(src, &mut b, cnt);
         data.push(LitIR::SInt(r));
         b
     }
     #[inline(always)]
-    pub(super) fn bool<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
+    pub(super) fn bool<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
         // `true\n` or `false\n`
         let mx = cmp::min(6, src.len());
         let slice = &src[..mx];
@@ -1044,7 +1038,7 @@ impl<'b> SafeQueryData<'b> {
         v_true | v_false
     }
     #[inline(always)]
-    pub(super) fn float<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
+    pub(super) fn float<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
         let mut okay = true;
         let payload = Self::mxple(src, cnt, &mut okay);
         match String::from_utf8_lossy(payload).parse() {
@@ -1056,14 +1050,14 @@ impl<'b> SafeQueryData<'b> {
         okay
     }
     #[inline(always)]
-    pub(super) fn bin<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
+    pub(super) fn bin<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
         let mut okay = true;
         let payload = Self::mxple(src, cnt, &mut okay);
         data.push(LitIR::Bin(payload));
         okay
     }
     #[inline(always)]
-    pub(super) fn str<'a>(src: &'a [u8], cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
+    pub(super) fn str<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
         let mut okay = true;
         let payload = Self::mxple(src, cnt, &mut okay);
         match str::from_utf8(payload) {
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 14b1a898..4da225ea 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -37,10 +37,6 @@ pub(super) mod schema;
 #[cfg(test)]
 mod tests;
 
-#[cfg(test)]
-use core::{fmt, ops::Deref};
-use core::{mem, ptr::NonNull, slice, str};
-
 /*
     Lang errors
 */
@@ -61,100 +57,6 @@ pub enum LangError {
     InvalidTypeDefinition,
     InvalidSafeLiteral,
     BadPframe,
-}
-
-/*
-    Utils
-*/
-
-/// An unsafe, C-like slice that holds a ptr and length. Construction and usage is at the risk of the user
-///
-/// Notes:
-/// - [`Clone`] is implemented for [`RawSlice`] because it is a simple bitwise copy of the fat ptr
-/// - [`fmt::Debug`] is implemented in different ways
-///     - For test builds like test and bench, it will output a slice
-///     - In release mode, it will output the fat ptr meta
-/// - [`PartialEq`] is implemented in debug mode with slice comparison, but is **NOT implemented for release mode in the
-///   way you'd expect it to**. In release mode (non-test), a comparison will simply panic.
-#[cfg_attr(not(test), derive(Debug))]
-#[derive(Clone)]
-pub struct RawSlice {
-    ptr: NonNull<u8>,
-    len: usize,
-}
-
-// again, caller's responsibility
-unsafe impl Send for RawSlice {}
-unsafe impl Sync for RawSlice {}
-
-impl RawSlice {
-    const _EALIGN: () = assert!(mem::align_of::<Self>() == mem::align_of::<&[u8]>());
-    const FAKE_SLICE: Self = unsafe { Self::new_from_str("") };
-    const unsafe fn new(ptr: *const u8, len: usize) -> Self {
-        Self {
-            ptr: NonNull::new_unchecked(ptr.cast_mut()),
-            len,
-        }
-    }
-    const unsafe fn new_from_str(s: &str) -> Self {
-        Self::new(s.as_bytes().as_ptr(), s.as_bytes().len())
-    }
-    pub unsafe fn as_slice(&self) -> &[u8] {
-        slice::from_raw_parts(self.ptr.as_ptr(), self.len)
-    }
-    unsafe fn as_str(&self) -> &str {
-        str::from_utf8_unchecked(self.as_slice())
-    }
-    pub fn ptr(&self) -> NonNull<u8> {
-        self.ptr
-    }
-    pub fn len(&self) -> usize {
-        self.len
-    }
-}
-
-#[cfg(test)]
-impl fmt::Debug for RawSlice {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str(unsafe {
-            // UNSAFE(@ohsayan): Only implemented in debug
-            self.as_str()
-        })
-    }
-}
-
-#[cfg(test)]
-impl PartialEq for RawSlice {
-    fn eq(&self, other: &Self) -> bool {
-        unsafe {
-            // UNSAFE(@ohsayan): Callers must ensure validity during usage
-            self.as_slice() == other.as_slice()
-        }
-    }
-}
-
-#[cfg(not(test))]
-impl PartialEq for RawSlice {
-    fn eq(&self, _other: &Self) -> bool {
-        panic!("Called partialeq on rawslice in release mode");
-    }
-}
-
-#[cfg(test)]
-impl<U> PartialEq<U> for RawSlice
-where
-    U: Deref<Target = [u8]>,
-{
-    fn eq(&self, other: &U) -> bool {
-        unsafe {
-            // UNSAFE(@ohsayan): Callers must ensure validity during usage
-            self.as_slice() == other.deref()
-        }
-    }
-}
-
-impl From<&'static str> for RawSlice {
-    fn from(st: &'static str) -> Self {
-        unsafe { Self::new(st.as_bytes().as_ptr(), st.as_bytes().len()) }
-    }
+    UnknownCreateStatement,
+    UnknownAlterStatement,
 }
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index a5e69700..0d64f39d 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -47,10 +47,11 @@
 use {
     super::{
         ast::QueryData,
-        lexer::{LitIR, LitIROwned, Symbol, Token},
-        LangError, LangResult, RawSlice,
+        lexer::{LitIR, LitIROwned, Slice, Symbol, Token},
+        LangError, LangResult,
     },
     crate::util::MaybeInit,
+    core::str,
     std::collections::{HashMap, HashSet},
 };
 
@@ -111,23 +112,23 @@ pub type Dict = HashMap<String, Option<DictEntry>>;
 
 #[derive(Debug, PartialEq)]
 /// A layer contains a type and corresponding metadata
-pub struct Layer {
-    ty: RawSlice,
+pub struct Layer<'a> {
+    ty: Slice<'a>,
     props: Dict,
     reset: bool,
 }
 
-impl Layer {
+impl<'a> Layer<'a> {
     //// Create a new layer
-    pub(super) const fn new(ty: RawSlice, props: Dict, reset: bool) -> Self {
+    pub(super) const fn new(ty: Slice<'a>, props: Dict, reset: bool) -> Self {
         Self { ty, props, reset }
     }
     /// Create a new layer that doesn't have any reset
-    pub(super) const fn new_noreset(ty: RawSlice, props: Dict) -> Self {
+    pub(super) const fn new_noreset(ty: Slice<'a>, props: Dict) -> Self {
         Self::new(ty, props, false)
     }
     /// Create a new layer that adds a reset
-    pub(super) const fn new_reset(ty: RawSlice, props: Dict) -> Self {
+    pub(super) const fn new_reset(ty: Slice<'a>, props: Dict) -> Self {
         Self::new(ty, props, true)
     }
 }
@@ -150,18 +151,18 @@ impl FieldProperties {
 
 #[derive(Debug, PartialEq)]
 /// A field definition
-pub struct Field {
+pub struct Field<'a> {
     /// the field name
-    pub(super) field_name: RawSlice,
+    pub(super) field_name: Slice<'a>,
     /// layers
-    pub(super) layers: Vec<Layer>,
+    pub(super) layers: Vec<Layer<'a>>,
     /// properties
     pub(super) props: HashSet<StaticStr>,
 }
 
-impl Field {
+impl<'a> Field<'a> {
     #[inline(always)]
-    pub fn new(field_name: RawSlice, layers: Vec<Layer>, props: HashSet<StaticStr>) -> Self {
+    pub fn new(field_name: Slice<'a>, layers: Vec<Layer<'a>>, props: HashSet<StaticStr>) -> Self {
         Self {
             field_name,
             layers,
@@ -172,18 +173,18 @@ impl Field {
 
 #[derive(Debug, PartialEq)]
 /// A model definition
-pub struct Model {
+pub struct Model<'a> {
     /// the model name
-    pub(super) model_name: RawSlice,
+    pub(super) model_name: Slice<'a>,
     /// the fields
-    pub(super) fields: Vec<Field>,
+    pub(super) fields: Vec<Field<'a>>,
     /// properties
     pub(super) props: Dict,
 }
 
-impl Model {
+impl<'a> Model<'a> {
     #[inline(always)]
-    pub fn new(model_name: RawSlice, fields: Vec<Field>, props: Dict) -> Self {
+    pub fn new(model_name: Slice<'a>, fields: Vec<Field<'a>>, props: Dict) -> Self {
         Self {
             model_name,
             fields,
@@ -194,17 +195,17 @@ impl Model {
 
 #[derive(Debug, PartialEq)]
 /// A space
-pub struct Space {
+pub struct Space<'a> {
     /// the space name
-    pub(super) space_name: RawSlice,
+    pub(super) space_name: Slice<'a>,
     /// properties
     pub(super) props: Dict,
 }
 
 #[derive(Debug, PartialEq)]
 /// An alter space query with corresponding data
-pub struct AlterSpace {
-    pub(super) space_name: RawSlice,
+pub struct AlterSpace<'a> {
+    pub(super) space_name: Slice<'a>,
     pub(super) updated_props: Dict,
 }
 
@@ -264,7 +265,7 @@ pub(super) fn rfold_dict<'a, Qd: QueryData<'a>>(
             (Token::Ident(id), DictFoldState::CB_OR_IDENT) => {
                 // found ident, so expect colon
                 i += 1;
-                tmp = MaybeInit::new(unsafe { id.as_str() });
+                tmp = MaybeInit::new(unsafe { str::from_utf8_unchecked(id) });
                 state = DictFoldState::COLON;
             }
             (Token::Symbol(Symbol::SymColon), DictFoldState::COLON) => {
@@ -466,7 +467,7 @@ pub(super) fn rfold_tymeta<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
             }
             (Token::Ident(ident), TyMetaFoldState::IDENT_OR_CB) => {
                 r.incr();
-                tmp = MaybeInit::new(unsafe { ident.as_str() });
+                tmp = MaybeInit::new(unsafe { str::from_utf8_unchecked(ident) });
                 // we just saw an ident, so we expect to see a colon
                 state = TyMetaFoldState::COLON;
             }
@@ -569,7 +570,7 @@ pub(super) fn rfold_layers<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
     start: LayerFoldState,
     tok: &'a [Token],
     qd: &mut Qd,
-    layers: &mut Vec<Layer>,
+    layers: &mut Vec<Layer<'a>>,
 ) -> u64 {
     /*
         NOTE: Assume rules wherever applicable
@@ -684,7 +685,7 @@ pub(super) fn rfold_layers<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
 #[cfg(test)]
 #[inline(always)]
 /// (**test-only**) fold layers
-pub(super) fn fold_layers(tok: &[Token]) -> (Vec<Layer>, usize, bool) {
+pub(super) fn fold_layers<'a>(tok: &'a [Token]) -> (Vec<Layer<'a>>, usize, bool) {
     let mut l = Vec::new();
     let r = rfold_layers::<InplaceData, DISALLOW_RESET_SYNTAX>(
         LayerFoldState::TY,
@@ -727,7 +728,7 @@ pub(super) fn parse_field_properties(tok: &[Token]) -> (FieldProperties, usize,
 }
 
 #[cfg(test)]
-pub(super) fn parse_field_full(tok: &[Token]) -> LangResult<(usize, Field)> {
+pub(super) fn parse_field_full<'a>(tok: &'a [Token]) -> LangResult<(usize, Field<'a>)> {
     self::parse_field(tok, &mut InplaceData::new())
 }
 
@@ -736,7 +737,7 @@ pub(super) fn parse_field_full(tok: &[Token]) -> LangResult<(usize, Field)> {
 pub(super) fn parse_field<'a, Qd: QueryData<'a>>(
     tok: &'a [Token],
     qd: &mut Qd,
-) -> LangResult<(usize, Field)> {
+) -> LangResult<(usize, Field<'a>)> {
     let l = tok.len();
     let mut i = 0;
     let mut okay = true;
@@ -788,7 +789,9 @@ states! {
 }
 
 #[cfg(test)]
-pub(super) fn parse_schema_from_tokens_full(tok: &[Token]) -> LangResult<(Model, usize)> {
+pub(super) fn parse_schema_from_tokens_full<'a>(
+    tok: &'a [Token],
+) -> LangResult<(Model<'a>, usize)> {
     self::parse_schema_from_tokens::<InplaceData>(tok, &mut InplaceData::new())
 }
 
@@ -797,7 +800,7 @@ pub(super) fn parse_schema_from_tokens_full(tok: &[Token]) -> LangResult<(Model,
 pub(super) fn parse_schema_from_tokens<'a, Qd: QueryData<'a>>(
     tok: &'a [Token],
     qd: &mut Qd,
-) -> LangResult<(Model, usize)> {
+) -> LangResult<(Model<'a>, usize)> {
     // parse fields
     let l = tok.len();
     let mut i = 0;
@@ -893,7 +896,7 @@ pub(super) fn parse_schema_from_tokens<'a, Qd: QueryData<'a>>(
 pub(super) fn parse_space_from_tokens<'a, Qd: QueryData<'a>>(
     tok: &'a [Token],
     qd: &mut Qd,
-) -> LangResult<(Space, usize)> {
+) -> LangResult<(Space<'a>, usize)> {
     let l = tok.len();
     let mut okay = !tok.is_empty() && tok[0].is_ident();
     let mut i = 0;
@@ -928,7 +931,7 @@ pub(super) fn parse_space_from_tokens<'a, Qd: QueryData<'a>>(
 pub(super) fn parse_alter_space_from_tokens<'a, Qd: QueryData<'a>>(
     tok: &'a [Token],
     qd: &mut Qd,
-) -> LangResult<(AlterSpace, usize)> {
+) -> LangResult<(AlterSpace<'a>, usize)> {
     let mut i = 0;
     let l = tok.len();
 
@@ -960,7 +963,7 @@ pub(super) fn parse_alter_space_from_tokens<'a, Qd: QueryData<'a>>(
 }
 
 #[cfg(test)]
-pub(super) fn alter_space_full(tok: &[Token]) -> LangResult<AlterSpace> {
+pub(super) fn alter_space_full<'a>(tok: &'a [Token]) -> LangResult<AlterSpace<'a>> {
     let (r, i) = self::parse_alter_space_from_tokens(tok, &mut InplaceData::new())?;
     assert_full_tt!(i, tok.len());
     Ok(r)
@@ -978,17 +981,17 @@ states! {
 
 #[derive(Debug, PartialEq)]
 /// An [`ExpandedField`] is a full field definition with advanced metadata
-pub struct ExpandedField {
-    pub(super) field_name: RawSlice,
+pub struct ExpandedField<'a> {
+    pub(super) field_name: Slice<'a>,
     pub(super) props: Dict,
-    pub(super) layers: Vec<Layer>,
+    pub(super) layers: Vec<Layer<'a>>,
     pub(super) reset: bool,
 }
 
 #[cfg(test)]
-pub fn parse_field_syntax_full<const ALLOW_RESET: bool>(
-    tok: &[Token],
-) -> LangResult<(ExpandedField, usize)> {
+pub fn parse_field_syntax_full<'a, const ALLOW_RESET: bool>(
+    tok: &'a [Token],
+) -> LangResult<(ExpandedField<'a>, usize)> {
     self::parse_field_syntax::<InplaceData, ALLOW_RESET>(tok, &mut InplaceData::new())
 }
 
@@ -997,7 +1000,7 @@ pub fn parse_field_syntax_full<const ALLOW_RESET: bool>(
 pub(super) fn parse_field_syntax<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
     tok: &'a [Token],
     qd: &mut Qd,
-) -> LangResult<(ExpandedField, usize)> {
+) -> LangResult<(ExpandedField<'a>, usize)> {
     let l = tok.len();
     let mut i = 0_usize;
     let mut state = FieldSyntaxParseState::IDENT;
@@ -1086,28 +1089,25 @@ pub(super) fn parse_field_syntax<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>
 
 #[derive(Debug)]
 #[cfg_attr(test, derive(PartialEq))]
-pub struct Alter {
-    model: RawSlice,
-    kind: AlterKind,
+pub struct Alter<'a> {
+    model: Slice<'a>,
+    kind: AlterKind<'a>,
 }
 
-impl Alter {
+impl<'a> Alter<'a> {
     #[inline(always)]
-    pub(super) fn new(model: RawSlice, kind: AlterKind) -> Self {
-        Self {
-            model,
-            kind: kind.into(),
-        }
+    pub(super) fn new(model: Slice<'a>, kind: AlterKind<'a>) -> Self {
+        Self { model, kind }
     }
 }
 
 #[derive(Debug)]
 #[cfg_attr(test, derive(PartialEq))]
 /// The alter operation kind
-pub enum AlterKind {
-    Add(Box<[ExpandedField]>),
-    Remove(Box<[RawSlice]>),
-    Update(Box<[ExpandedField]>),
+pub enum AlterKind<'a> {
+    Add(Box<[ExpandedField<'a>]>),
+    Remove(Box<[Slice<'a>]>),
+    Update(Box<[ExpandedField<'a>]>),
 }
 
 #[inline(always)]
@@ -1116,7 +1116,7 @@ pub(super) fn parse_alter_kind_from_tokens<'a, Qd: QueryData<'a>>(
     tok: &'a [Token],
     qd: &mut Qd,
     current: &mut usize,
-) -> LangResult<Alter> {
+) -> LangResult<Alter<'a>> {
     let l = tok.len();
     let okay = l > 2 && tok[0].is_ident();
     if !okay {
@@ -1144,7 +1144,7 @@ pub(super) fn parse_multiple_field_syntax<'a, Qd: QueryData<'a>, const ALLOW_RES
     tok: &'a [Token],
     qd: &mut Qd,
     current: &mut usize,
-) -> LangResult<Box<[ExpandedField]>> {
+) -> LangResult<Box<[ExpandedField<'a>]>> {
     const DEFAULT_ADD_COL_CNT: usize = 4;
     /*
         WARNING: No trailing commas allowed
@@ -1204,21 +1204,24 @@ pub(super) fn alter_add<'a, Qd: QueryData<'a>>(
     tok: &'a [Token],
     qd: &mut Qd,
     current: &mut usize,
-) -> LangResult<Box<[ExpandedField]>> {
+) -> LangResult<Box<[ExpandedField<'a>]>> {
     self::parse_multiple_field_syntax::<Qd, DISALLOW_RESET_SYNTAX>(tok, qd, current)
 }
 
 #[cfg(test)]
-pub(super) fn alter_add_full(
-    tok: &[Token],
+pub(super) fn alter_add_full<'a>(
+    tok: &'a [Token],
     current: &mut usize,
-) -> LangResult<Box<[ExpandedField]>> {
+) -> LangResult<Box<[ExpandedField<'a>]>> {
     self::alter_add(tok, &mut InplaceData::new(), current)
 }
 
 #[inline(always)]
 /// Parse the expression for `alter model <> remove (..)`
-pub(super) fn alter_remove(tok: &[Token], current: &mut usize) -> LangResult<Box<[RawSlice]>> {
+pub(super) fn alter_remove<'a>(
+    tok: &'a [Token],
+    current: &mut usize,
+) -> LangResult<Box<[Slice<'a>]>> {
     const DEFAULT_REMOVE_COL_CNT: usize = 4;
     /*
         WARNING: No trailing commas allowed
@@ -1274,11 +1277,14 @@ pub(super) fn alter_update<'a, Qd: QueryData<'a>>(
     tok: &'a [Token],
     qd: &mut Qd,
     current: &mut usize,
-) -> LangResult<Box<[ExpandedField]>> {
+) -> LangResult<Box<[ExpandedField<'a>]>> {
     self::parse_multiple_field_syntax::<Qd, ALLOW_RESET_SYNTAX>(tok, qd, current)
 }
 
 #[cfg(test)]
-pub(super) fn alter_update_full(tok: &[Token], i: &mut usize) -> LangResult<Box<[ExpandedField]>> {
+pub(super) fn alter_update_full<'a>(
+    tok: &'a [Token],
+    i: &mut usize,
+) -> LangResult<Box<[ExpandedField<'a>]>> {
     self::alter_update(tok, &mut InplaceData::new(), i)
 }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 398909ab..a8bec0ac 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -91,11 +91,9 @@ impl NullableMapEntry for Null {
     }
 }
 
-impl NullableMapEntry for super::lexer::Lit {
+impl<'a> NullableMapEntry for super::lexer::Lit<'a> {
     fn data(self) -> Option<super::schema::DictEntry> {
-        Some(super::schema::DictEntry::Lit(
-            unsafe { self.as_ir() }.to_litir_owned(),
-        ))
+        Some(super::schema::DictEntry::Lit(self.as_ir().to_litir_owned()))
     }
 }
 
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 7707fd4d..d2909292 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -404,7 +404,7 @@ mod stmt_insert {
         .unwrap();
         let r = dml::parse_insert_full(&x[1..]).unwrap();
         let e = InsertStatement {
-            entity: Entity::Full("twitter".into(), "users".into()),
+            entity: Entity::Full(b"twitter", b"users"),
             data: into_array_nullable!["sayan"].to_vec().into(),
         };
         assert_eq!(e, r);
@@ -426,7 +426,7 @@ mod stmt_insert {
         .unwrap();
         let r = dml::parse_insert_full(&x[1..]).unwrap();
         let e = InsertStatement {
-            entity: Entity::Full("twitter".into(), "users".into()),
+            entity: Entity::Full(b"twitter", b"users"),
             data: into_array_nullable!["sayan", "Sayan", "sayan@example.com", true, 12345, 67890]
                 .to_vec()
                 .into(),
@@ -453,7 +453,7 @@ mod stmt_insert {
         .unwrap();
         let r = dml::parse_insert_full(&x[1..]).unwrap();
         let e = InsertStatement {
-            entity: Entity::Full("twitter".into(), "users".into()),
+            entity: Entity::Full(b"twitter", b"users"),
             data: into_array_nullable![
                 "sayan",
                 "Sayan",
@@ -480,7 +480,7 @@ mod stmt_insert {
         .unwrap();
         let r = dml::parse_insert_full(&tok[1..]).unwrap();
         let e = InsertStatement {
-            entity: Entity::Full("jotsy".into(), "app".into()),
+            entity: Entity::Full(b"jotsy", b"app"),
             data: dict_nullable! {
                 "username".as_bytes() => "sayan"
             }
@@ -505,7 +505,7 @@ mod stmt_insert {
         .unwrap();
         let r = dml::parse_insert_full(&tok[1..]).unwrap();
         let e = InsertStatement {
-            entity: Entity::Full("jotsy".into(), "app".into()),
+            entity: Entity::Full(b"jotsy", b"app"),
             data: dict_nullable! {
                 "username".as_bytes() => "sayan",
                 "name".as_bytes() => "Sayan",
@@ -538,7 +538,7 @@ mod stmt_insert {
         .unwrap();
         let r = dml::parse_insert_full(&tok[1..]).unwrap();
         let e = InsertStatement {
-            entity: Entity::Full("jotsy".into(), "app".into()),
+            entity: Entity::Full(b"jotsy", b"app"),
             data: dict_nullable! {
                 "username".as_bytes() => "sayan",
                 "password".as_bytes() => "pass123",
@@ -577,7 +577,7 @@ mod stmt_select {
         .unwrap();
         let r = dml::parse_select_full(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
-            Entity::Single("users".into()),
+            Entity::Single(b"users"),
             [].to_vec(),
             true,
             dict! {
@@ -598,8 +598,8 @@ mod stmt_select {
         .unwrap();
         let r = dml::parse_select_full(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
-            Entity::Single("users".into()),
-            ["field1".into()].to_vec(),
+            Entity::Single(b"users"),
+            [b"field1".as_slice()].to_vec(),
             false,
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
@@ -619,8 +619,8 @@ mod stmt_select {
         .unwrap();
         let r = dml::parse_select_full(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
-            Entity::Full("twitter".into(), "users".into()),
-            ["field1".into()].to_vec(),
+            Entity::Full(b"twitter", b"users"),
+            [b"field1".as_slice()].to_vec(),
             false,
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
@@ -640,8 +640,8 @@ mod stmt_select {
         .unwrap();
         let r = dml::parse_select_full(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
-            Entity::Full("twitter".into(), "users".into()),
-            ["field1".into(), "field2".into()].to_vec(),
+            Entity::Full(b"twitter", b"users"),
+            [b"field1".as_slice(), b"field2".as_slice()].to_vec(),
             false,
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
@@ -667,7 +667,7 @@ mod expression_tests {
         assert_eq!(
             r,
             AssignmentExpression {
-                lhs: "username".into(),
+                lhs: b"username",
                 rhs: LitIR::Str("sayan"),
                 operator_fn: Operator::Assign
             }
@@ -680,7 +680,7 @@ mod expression_tests {
         assert_eq!(
             r,
             AssignmentExpression {
-                lhs: "followers".into(),
+                lhs: b"followers",
                 rhs: LitIR::UInt(100),
                 operator_fn: Operator::AddAssign
             }
@@ -693,7 +693,7 @@ mod expression_tests {
         assert_eq!(
             r,
             AssignmentExpression {
-                lhs: "following".into(),
+                lhs: b"following",
                 rhs: LitIR::UInt(150),
                 operator_fn: Operator::SubAssign
             }
@@ -706,7 +706,7 @@ mod expression_tests {
         assert_eq!(
             r,
             AssignmentExpression {
-                lhs: "product_qty".into(),
+                lhs: b"product_qty",
                 rhs: LitIR::UInt(2),
                 operator_fn: Operator::MulAssign
             }
@@ -719,7 +719,7 @@ mod expression_tests {
         assert_eq!(
             r,
             AssignmentExpression {
-                lhs: "image_crop_factor".into(),
+                lhs: b"image_crop_factor",
                 rhs: LitIR::UInt(2),
                 operator_fn: Operator::DivAssign
             }
@@ -747,9 +747,9 @@ mod update_statement {
         .unwrap();
         let r = dml::parse_update_full(&tok[1..]).unwrap();
         let e = UpdateStatement {
-            entity: Entity::Single("app".into()),
+            entity: Entity::Single(b"app"),
             expressions: vec![AssignmentExpression {
-                lhs: "notes".into(),
+                lhs: b"notes",
                 rhs: LitIR::Str("this is my new note"),
                 operator_fn: Operator::AddAssign,
             }],
@@ -779,15 +779,15 @@ mod update_statement {
         .unwrap();
         let r = dml::parse_update_full(&tok[1..]).unwrap();
         let e = UpdateStatement {
-            entity: ("jotsy", "app").into(),
+            entity: Entity::Full(b"jotsy", b"app"),
             expressions: vec![
                 AssignmentExpression::new(
-                    "notes".into(),
+                    b"notes",
                     LitIR::Str("this is my new note"),
                     Operator::AddAssign,
                 ),
                 AssignmentExpression::new(
-                    "email".into(),
+                    b"email",
                     LitIR::Str("sayan@example.com"),
                     Operator::Assign,
                 ),
@@ -823,7 +823,7 @@ mod delete_stmt {
         )
         .unwrap();
         let e = DeleteStatement::new_test(
-            Entity::Single("users".into()),
+            Entity::Single(b"users"),
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
                     "username".as_bytes(),
@@ -844,7 +844,7 @@ mod delete_stmt {
         )
         .unwrap();
         let e = DeleteStatement::new_test(
-            ("twitter", "users").into(),
+            Entity::Full(b"twitter", b"users"),
             dict! {
                 "username".as_bytes() => RelationalExpr::new(
                     "username".as_bytes(),
diff --git a/server/src/engine/ql/tests/entity.rs b/server/src/engine/ql/tests/entity.rs
index d1e63eec..4b8699e6 100644
--- a/server/src/engine/ql/tests/entity.rs
+++ b/server/src/engine/ql/tests/entity.rs
@@ -25,25 +25,22 @@
 */
 
 use super::*;
-use crate::engine::ql::ast::{Compiler, Entity};
+use crate::engine::ql::ast::Entity;
 #[test]
 fn entity_current() {
     let t = lex_insecure(b"hello").unwrap();
-    let mut c = Compiler::new(&t);
-    let r = Entity::parse(&mut c).unwrap();
-    assert_eq!(r, Entity::Single("hello".into()))
+    let r = Entity::parse_from_tokens(&t, &mut 0).unwrap();
+    assert_eq!(r, Entity::Single(b"hello"))
 }
 #[test]
 fn entity_partial() {
     let t = lex_insecure(b":hello").unwrap();
-    let mut c = Compiler::new(&t);
-    let r = Entity::parse(&mut c).unwrap();
-    assert_eq!(r, Entity::Partial("hello".into()))
+    let r = Entity::parse_from_tokens(&t, &mut 0).unwrap();
+    assert_eq!(r, Entity::Partial(b"hello"))
 }
 #[test]
 fn entity_full() {
     let t = lex_insecure(b"hello.world").unwrap();
-    let mut c = Compiler::new(&t);
-    let r = Entity::parse(&mut c).unwrap();
-    assert_eq!(r, Entity::Full("hello".into(), "world".into()))
+    let r = Entity::parse_from_tokens(&t, &mut 0).unwrap();
+    assert_eq!(r, Entity::Full(b"hello", b"world"))
 }
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index 77794c28..b551b72d 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -44,10 +44,7 @@ macro_rules! v(
 #[test]
 fn lex_ident() {
     let src = v!("hello");
-    assert_eq!(
-        lex_insecure(&src).unwrap(),
-        vec![Token::Ident("hello".into())]
-    );
+    assert_eq!(lex_insecure(&src).unwrap(), vec![Token::Ident(b"hello")]);
 }
 
 // literals
@@ -161,19 +158,19 @@ fn lex_string_unclosed() {
 fn lex_unsafe_literal_mini() {
     let usl = lex_insecure("\r0\n".as_bytes()).unwrap();
     assert_eq!(usl.len(), 1);
-    assert_eq!(Token::Lit(Lit::Bin("".into())), usl[0]);
+    assert_eq!(Token::Lit(Lit::Bin(b"")), usl[0]);
 }
 #[test]
 fn lex_unsafe_literal() {
     let usl = lex_insecure("\r9\nabcdefghi".as_bytes()).unwrap();
     assert_eq!(usl.len(), 1);
-    assert_eq!(Token::Lit(Lit::Bin("abcdefghi".into())), usl[0]);
+    assert_eq!(Token::Lit(Lit::Bin(b"abcdefghi")), usl[0]);
 }
 #[test]
 fn lex_unsafe_literal_pro() {
     let usl = lex_insecure("\r18\nabcdefghi123456789".as_bytes()).unwrap();
     assert_eq!(usl.len(), 1);
-    assert_eq!(Token::Lit(Lit::Bin("abcdefghi123456789".into())), usl[0]);
+    assert_eq!(Token::Lit(Lit::Bin(b"abcdefghi123456789")), usl[0]);
 }
 
 mod num_tests {
@@ -433,9 +430,9 @@ mod safequery_full_param {
                     Token![select],
                     Token![*],
                     Token![from],
-                    Token::Ident("myapp".into()),
+                    Token::Ident(b"myapp"),
                     Token![where],
-                    Token::Ident("username".into()),
+                    Token::Ident(b"username"),
                     Token![=],
                     Token![?]
                 ]
@@ -455,13 +452,13 @@ mod safequery_full_param {
                     Token![select],
                     Token![*],
                     Token![from],
-                    Token::Ident("myapp".into()),
+                    Token::Ident(b"myapp"),
                     Token![where],
-                    Token::Ident("username".into()),
+                    Token::Ident(b"username"),
                     Token![=],
                     Token![?],
                     Token![and],
-                    Token::Ident("pass".into()),
+                    Token::Ident(b"pass"),
                     Token![=],
                     Token![?]
                 ]
@@ -485,19 +482,19 @@ mod safequery_full_param {
                 vec![
                     Token![select],
                     Token![$],
-                    Token::Ident("notes".into()),
+                    Token::Ident(b"notes"),
                     Token![open []],
                     Token![~],
                     Token![?],
                     Token![close []],
                     Token![from],
-                    Token::Ident("myapp".into()),
+                    Token::Ident(b"myapp"),
                     Token![where],
-                    Token::Ident("username".into()),
+                    Token::Ident(b"username"),
                     Token![=],
                     Token![?],
                     Token![and],
-                    Token::Ident("pass".into()),
+                    Token::Ident(b"pass"),
                     Token![=],
                     Token![?]
                 ]
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 43d5b5b5..7c39a416 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -44,7 +44,7 @@ mod inspect {
         let tok = lex_insecure(b"inspect space myspace").unwrap();
         assert_eq!(
             ddl::parse_inspect_full(&tok[1..]).unwrap(),
-            Statement::InspectSpace("myspace".into())
+            Statement::InspectSpace(b"myspace")
         );
     }
     #[test]
@@ -52,12 +52,12 @@ mod inspect {
         let tok = lex_insecure(b"inspect model users").unwrap();
         assert_eq!(
             ddl::parse_inspect_full(&tok[1..]).unwrap(),
-            Statement::InspectModel(Entity::Single("users".into()))
+            Statement::InspectModel(Entity::Single(b"users"))
         );
         let tok = lex_insecure(b"inspect model tweeter.users").unwrap();
         assert_eq!(
             ddl::parse_inspect_full(&tok[1..]).unwrap(),
-            Statement::InspectModel(("tweeter", "users").into())
+            Statement::InspectModel(Entity::Full(b"tweeter", b"users"))
         );
     }
     #[test]
@@ -85,7 +85,7 @@ mod alter_space {
         assert_eq!(
             r,
             AlterSpace {
-                space_name: "mymodel".into(),
+                space_name: b"mymodel",
                 updated_props: nullable_dict! {}
             }
         );
@@ -105,7 +105,7 @@ mod alter_space {
         assert_eq!(
             r,
             AlterSpace {
-                space_name: "mymodel".into(),
+                space_name: b"mymodel",
                 updated_props: nullable_dict! {
                     "max_entry" => Lit::UnsignedInt(1000),
                     "driver" => Lit::Str("ts-0.8".into())
@@ -271,7 +271,7 @@ mod tymeta {
             if should_pass {
                 assert!(ret.is_okay());
                 assert!(ret.has_more());
-                assert!(new_src[ret.pos()] == Token::Ident("string".into()));
+                assert!(new_src[ret.pos()] == Token::Ident(b"string"));
                 assert_eq!(dict, expected);
             } else if ret.is_okay() {
                 panic!("Expected failure but passed for token stream: `{:?}`", tok);
@@ -290,7 +290,7 @@ mod layer {
         assert!(okay);
         assert_eq!(
             layers,
-            vec![Layer::new_noreset("string".into(), nullable_dict! {})]
+            vec![Layer::new_noreset(b"string", nullable_dict! {})]
         );
     }
     #[test]
@@ -302,7 +302,7 @@ mod layer {
         assert_eq!(
             layers,
             vec![Layer::new_noreset(
-                "string".into(),
+                b"string",
                 nullable_dict! {
                     "maxlen" => Lit::UnsignedInt(100)
                 }
@@ -318,8 +318,8 @@ mod layer {
         assert_eq!(
             layers,
             vec![
-                Layer::new_noreset("string".into(), nullable_dict! {}),
-                Layer::new_noreset("list".into(), nullable_dict! {})
+                Layer::new_noreset(b"string", nullable_dict! {}),
+                Layer::new_noreset(b"list", nullable_dict! {})
             ]
         );
     }
@@ -332,9 +332,9 @@ mod layer {
         assert_eq!(
             layers,
             vec![
-                Layer::new_noreset("string".into(), nullable_dict! {}),
+                Layer::new_noreset(b"string", nullable_dict! {}),
                 Layer::new_noreset(
-                    "list".into(),
+                    b"list",
                     nullable_dict! {
                         "unique" => Lit::Bool(true),
                         "maxlen" => Lit::UnsignedInt(10),
@@ -356,14 +356,14 @@ mod layer {
             layers,
             vec![
                 Layer::new_noreset(
-                    "string".into(),
+                    b"string",
                     nullable_dict! {
                         "ascii_only" => Lit::Bool(true),
                         "maxlen" => Lit::UnsignedInt(255)
                     }
                 ),
                 Layer::new_noreset(
-                    "list".into(),
+                    b"list",
                     nullable_dict! {
                         "unique" => Lit::Bool(true),
                         "maxlen" => Lit::UnsignedInt(10),
@@ -388,14 +388,14 @@ mod layer {
         )
         .unwrap();
         let expected = vec![
-            Layer::new_noreset("string".into(), nullable_dict!()),
+            Layer::new_noreset(b"string", nullable_dict!()),
             Layer::new_noreset(
-                "list".into(),
+                b"list",
                 nullable_dict! {
                     "maxlen" => Lit::UnsignedInt(100),
                 },
             ),
-            Layer::new_noreset("list".into(), nullable_dict!("unique" => Lit::Bool(true))),
+            Layer::new_noreset(b"list", nullable_dict!("unique" => Lit::Bool(true))),
         ];
         fuzz_tokens(&tok, |should_pass, new_tok| {
             let (layers, c, okay) = schema::fold_layers(&new_tok);
@@ -428,7 +428,7 @@ mod field_properties {
         let tok = lex_insecure(b"primary null myfield:").unwrap();
         let (props, c, okay) = schema::parse_field_properties(&tok);
         assert_eq!(c, 2);
-        assert_eq!(tok[c], Token::Ident("myfield".into()));
+        assert_eq!(tok[c], Token::Ident(b"myfield"));
         assert!(okay);
         assert_eq!(
             props,
@@ -456,8 +456,8 @@ mod fields {
         assert_eq!(
             f,
             Field {
-                field_name: "username".into(),
-                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                field_name: b"username",
+                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
                 props: set![],
             }
         )
@@ -475,8 +475,8 @@ mod fields {
         assert_eq!(
             f,
             Field {
-                field_name: "username".into(),
-                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                field_name: b"username",
+                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
                 props: set!["primary"],
             }
         )
@@ -497,9 +497,9 @@ mod fields {
         assert_eq!(
             f,
             Field {
-                field_name: "username".into(),
+                field_name: b"username",
                 layers: [Layer::new_noreset(
-                    "string".into(),
+                    b"string",
                     nullable_dict! {
                         "maxlen" => Lit::UnsignedInt(10),
                         "ascii_only" => Lit::Bool(true),
@@ -529,17 +529,17 @@ mod fields {
         assert_eq!(
             f,
             Field {
-                field_name: "notes".into(),
+                field_name: b"notes",
                 layers: [
                     Layer::new_noreset(
-                        "string".into(),
+                        b"string",
                         nullable_dict! {
                             "maxlen" => Lit::UnsignedInt(255),
                             "ascii_only" => Lit::Bool(true),
                         }
                     ),
                     Layer::new_noreset(
-                        "list".into(),
+                        b"list",
                         nullable_dict! {
                             "unique" => Lit::Bool(true)
                         }
@@ -574,16 +574,16 @@ mod schemas {
         assert_eq!(
             model,
             Model {
-                model_name: "mymodel".into(),
+                model_name: b"mymodel",
                 fields: vec![
                     Field {
-                        field_name: "username".into(),
-                        layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
+                        field_name: b"username",
+                        layers: vec![Layer::new_noreset(b"string", nullable_dict! {})],
                         props: set!["primary"]
                     },
                     Field {
-                        field_name: "password".into(),
-                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        field_name: b"password",
+                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
                         props: set![]
                     }
                 ],
@@ -611,21 +611,21 @@ mod schemas {
         assert_eq!(
             model,
             Model {
-                model_name: "mymodel".into(),
+                model_name: b"mymodel",
                 fields: vec![
                     Field {
-                        field_name: "username".into(),
-                        layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
+                        field_name: b"username",
+                        layers: vec![Layer::new_noreset(b"string", nullable_dict! {})],
                         props: set!["primary"]
                     },
                     Field {
-                        field_name: "password".into(),
-                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        field_name: b"password",
+                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
                         props: set![]
                     },
                     Field {
-                        field_name: "profile_pic".into(),
-                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        field_name: b"profile_pic",
+                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
                         props: set!["null"]
                     }
                 ],
@@ -658,29 +658,29 @@ mod schemas {
         assert_eq!(
             model,
             Model {
-                model_name: "mymodel".into(),
+                model_name: b"mymodel",
                 fields: vec![
                     Field {
-                        field_name: "username".into(),
-                        layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
+                        field_name: b"username",
+                        layers: vec![Layer::new_noreset(b"string", nullable_dict! {})],
                         props: set!["primary"]
                     },
                     Field {
-                        field_name: "password".into(),
-                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        field_name: b"password",
+                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
                         props: set![]
                     },
                     Field {
-                        field_name: "profile_pic".into(),
-                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        field_name: b"profile_pic",
+                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
                         props: set!["null"]
                     },
                     Field {
-                        field_name: "notes".into(),
+                        field_name: b"notes",
                         layers: vec![
-                            Layer::new_noreset("string".into(), nullable_dict! {}),
+                            Layer::new_noreset(b"string", nullable_dict! {}),
                             Layer::new_noreset(
-                                "list".into(),
+                                b"list",
                                 nullable_dict! {
                                     "unique" => Lit::Bool(true)
                                 }
@@ -723,29 +723,29 @@ mod schemas {
         assert_eq!(
             model,
             Model {
-                model_name: "mymodel".into(),
+                model_name: b"mymodel",
                 fields: vec![
                     Field {
-                        field_name: "username".into(),
-                        layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
+                        field_name: b"username",
+                        layers: vec![Layer::new_noreset(b"string", nullable_dict! {})],
                         props: set!["primary"]
                     },
                     Field {
-                        field_name: "password".into(),
-                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        field_name: b"password",
+                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
                         props: set![]
                     },
                     Field {
-                        field_name: "profile_pic".into(),
-                        layers: vec![Layer::new_noreset("binary".into(), nullable_dict! {})],
+                        field_name: b"profile_pic",
+                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
                         props: set!["null"]
                     },
                     Field {
-                        field_name: "notes".into(),
+                        field_name: b"notes",
                         layers: vec![
-                            Layer::new_noreset("string".into(), nullable_dict! {}),
+                            Layer::new_noreset(b"string", nullable_dict! {}),
                             Layer::new_noreset(
-                                "list".into(),
+                                b"list",
                                 nullable_dict! {
                                     "unique" => Lit::Bool(true)
                                 }
@@ -775,8 +775,8 @@ mod dict_field_syntax {
         assert_eq!(
             ef,
             ExpandedField {
-                field_name: "username".into(),
-                layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
+                field_name: b"username",
+                layers: vec![Layer::new_noreset(b"string", nullable_dict! {})],
                 props: nullable_dict! {},
                 reset: false
             }
@@ -798,11 +798,11 @@ mod dict_field_syntax {
         assert_eq!(
             ef,
             ExpandedField {
-                field_name: "username".into(),
+                field_name: b"username",
                 props: nullable_dict! {
                     "nullable" => Lit::Bool(false),
                 },
-                layers: vec![Layer::new_noreset("string".into(), nullable_dict! {})],
+                layers: vec![Layer::new_noreset(b"string", nullable_dict! {})],
                 reset: false
             }
         );
@@ -827,13 +827,13 @@ mod dict_field_syntax {
         assert_eq!(
             ef,
             ExpandedField {
-                field_name: "username".into(),
+                field_name: b"username",
                 props: nullable_dict! {
                     "nullable" => Lit::Bool(false),
                     "jingle_bells" => Lit::Str("snow".into()),
                 },
                 layers: vec![Layer::new_noreset(
-                    "string".into(),
+                    b"string",
                     nullable_dict! {
                         "minlen" => Lit::UnsignedInt(6),
                         "maxlen" => Lit::UnsignedInt(255),
@@ -865,20 +865,20 @@ mod dict_field_syntax {
         assert_eq!(
             ef,
             ExpandedField {
-                field_name: "notes".into(),
+                field_name: b"notes",
                 props: nullable_dict! {
                     "nullable" => Lit::Bool(true),
                     "jingle_bells" => Lit::Str("snow".into()),
                 },
                 layers: vec![
                     Layer::new_noreset(
-                        "string".into(),
+                        b"string",
                         nullable_dict! {
                             "ascii_only" => Lit::Bool(true),
                         }
                     ),
                     Layer::new_noreset(
-                        "list".into(),
+                        b"list",
                         nullable_dict! {
                             "unique" => Lit::Bool(true),
                         }
@@ -891,14 +891,13 @@ mod dict_field_syntax {
 }
 mod alter_model_remove {
     use super::*;
-    use crate::engine::ql::RawSlice;
     #[test]
     fn alter_mini() {
         let tok = lex_insecure(b"alter model mymodel remove myfield").unwrap();
         let mut i = 4;
         let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
-        assert_eq!(remove, [RawSlice::from("myfield")].into());
+        assert_eq!(remove, [b"myfield".as_slice()].into());
     }
     #[test]
     fn alter_mini_2() {
@@ -906,7 +905,7 @@ mod alter_model_remove {
         let mut i = 4;
         let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
-        assert_eq!(remove, [RawSlice::from("myfield")].into());
+        assert_eq!(remove, [b"myfield".as_slice()].into());
     }
     #[test]
     fn alter() {
@@ -919,10 +918,10 @@ mod alter_model_remove {
         assert_eq!(
             remove,
             [
-                RawSlice::from("myfield1"),
-                RawSlice::from("myfield2"),
-                RawSlice::from("myfield3"),
-                RawSlice::from("myfield4")
+                b"myfield1".as_slice(),
+                b"myfield2".as_slice(),
+                b"myfield3".as_slice(),
+                b"myfield4".as_slice(),
             ]
             .into()
         );
@@ -945,9 +944,9 @@ mod alter_model_add {
         assert_eq!(
             r.as_ref(),
             [ExpandedField {
-                field_name: "myfield".into(),
+                field_name: b"myfield",
                 props: nullable_dict! {},
-                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
                 reset: false
             }]
         );
@@ -966,11 +965,11 @@ mod alter_model_add {
         assert_eq!(
             r.as_ref(),
             [ExpandedField {
-                field_name: "myfield".into(),
+                field_name: b"myfield",
                 props: nullable_dict! {
                     "nullable" => Lit::Bool(true)
                 },
-                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
                 reset: false
             }]
         );
@@ -989,11 +988,11 @@ mod alter_model_add {
         assert_eq!(
             r.as_ref(),
             [ExpandedField {
-                field_name: "myfield".into(),
+                field_name: b"myfield",
                 props: nullable_dict! {
                     "nullable" => Lit::Bool(true)
                 },
-                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
                 reset: false
             }]
         );
@@ -1027,27 +1026,27 @@ mod alter_model_add {
             r.as_ref(),
             [
                 ExpandedField {
-                    field_name: "myfield".into(),
+                    field_name: b"myfield",
                     props: nullable_dict! {
                         "nullable" => Lit::Bool(true)
                     },
-                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                    layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
                     reset: false
                 },
                 ExpandedField {
-                    field_name: "another".into(),
+                    field_name: b"another",
                     props: nullable_dict! {
                         "nullable" => Lit::Bool(false)
                     },
                     layers: [
                         Layer::new_noreset(
-                            "string".into(),
+                            b"string",
                             nullable_dict! {
                                 "maxlen" => Lit::UnsignedInt(255)
                             }
                         ),
                         Layer::new_noreset(
-                            "list".into(),
+                            b"list",
                             nullable_dict! {
                                "unique" => Lit::Bool(true)
                             },
@@ -1078,9 +1077,9 @@ mod alter_model_update {
         assert_eq!(
             r.as_ref(),
             [ExpandedField {
-                field_name: "myfield".into(),
+                field_name: b"myfield",
                 props: nullable_dict! {},
-                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
                 reset: true
             }]
         );
@@ -1099,9 +1098,9 @@ mod alter_model_update {
         assert_eq!(
             r.as_ref(),
             [ExpandedField {
-                field_name: "myfield".into(),
+                field_name: b"myfield",
                 props: nullable_dict! {},
-                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
                 reset: true
             }]
         );
@@ -1126,11 +1125,11 @@ mod alter_model_update {
         assert_eq!(
             r.as_ref(),
             [ExpandedField {
-                field_name: "myfield".into(),
+                field_name: b"myfield",
                 props: nullable_dict! {
                     "nullable" => Lit::Bool(true)
                 },
-                layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
                 reset: true
             }]
         );
@@ -1160,17 +1159,17 @@ mod alter_model_update {
             r.as_ref(),
             [
                 ExpandedField {
-                    field_name: "myfield".into(),
+                    field_name: b"myfield",
                     props: nullable_dict! {
                         "nullable" => Lit::Bool(true)
                     },
-                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                    layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
                     reset: true
                 },
                 ExpandedField {
-                    field_name: "myfield2".into(),
+                    field_name: b"myfield2",
                     props: nullable_dict! {},
-                    layers: [Layer::new_noreset("string".into(), nullable_dict! {})].into(),
+                    layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
                     reset: true
                 }
             ]
@@ -1204,18 +1203,18 @@ mod alter_model_update {
             r.as_ref(),
             [
                 ExpandedField {
-                    field_name: "myfield".into(),
+                    field_name: b"myfield",
                     props: nullable_dict! {
                         "nullable" => Lit::Bool(true)
                     },
-                    layers: [Layer::new_reset("string".into(), nullable_dict! {})].into(),
+                    layers: [Layer::new_reset(b"string", nullable_dict! {})].into(),
                     reset: true
                 },
                 ExpandedField {
-                    field_name: "myfield2".into(),
+                    field_name: b"myfield2",
                     props: nullable_dict! {},
                     layers: [Layer::new_reset(
-                        "string".into(),
+                        b"string",
                         nullable_dict! {"maxlen" => Lit::UnsignedInt(255)}
                     )]
                     .into(),
@@ -1239,7 +1238,7 @@ mod ddl_other_query_tests {
         let src = lex_insecure(br"drop space myspace").unwrap();
         assert_eq!(
             ddl::parse_drop_full(&src[1..]).unwrap(),
-            Statement::DropSpace(DropSpace::new("myspace".into(), false))
+            Statement::DropSpace(DropSpace::new(b"myspace", false))
         );
     }
     #[test]
@@ -1247,7 +1246,7 @@ mod ddl_other_query_tests {
         let src = lex_insecure(br"drop space myspace force").unwrap();
         assert_eq!(
             ddl::parse_drop_full(&src[1..]).unwrap(),
-            Statement::DropSpace(DropSpace::new("myspace".into(), true))
+            Statement::DropSpace(DropSpace::new(b"myspace", true))
         );
     }
     #[test]
@@ -1255,7 +1254,7 @@ mod ddl_other_query_tests {
         let src = lex_insecure(br"drop model mymodel").unwrap();
         assert_eq!(
             ddl::parse_drop_full(&src[1..]).unwrap(),
-            Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), false))
+            Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), false))
         );
     }
     #[test]
@@ -1263,7 +1262,7 @@ mod ddl_other_query_tests {
         let src = lex_insecure(br"drop model mymodel force").unwrap();
         assert_eq!(
             ddl::parse_drop_full(&src[1..]).unwrap(),
-            Statement::DropModel(DropModel::new(Entity::Single("mymodel".into()), true))
+            Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), true))
         );
     }
 }

From 0f064e10872e35951305e4278e812f6568b2f772 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 7 Jan 2023 10:00:18 -0800
Subject: [PATCH 067/310] Use `State` for more reliable QP

---
 server/src/engine/ql/ast.rs             | 174 ++++-
 server/src/engine/ql/dml.rs             | 962 ------------------------
 server/src/engine/ql/dml/delete.rs      | 106 +++
 server/src/engine/ql/dml/insert.rs      | 314 ++++++++
 server/src/engine/ql/dml/mod.rs         | 213 ++++++
 server/src/engine/ql/dml/select.rs      | 146 ++++
 server/src/engine/ql/dml/update.rs      | 236 ++++++
 server/src/engine/ql/macros.rs          |   8 +
 server/src/engine/ql/tests/dml_tests.rs | 186 +++--
 9 files changed, 1268 insertions(+), 1077 deletions(-)
 delete mode 100644 server/src/engine/ql/dml.rs
 create mode 100644 server/src/engine/ql/dml/delete.rs
 create mode 100644 server/src/engine/ql/dml/insert.rs
 create mode 100644 server/src/engine/ql/dml/mod.rs
 create mode 100644 server/src/engine/ql/dml/select.rs
 create mode 100644 server/src/engine/ql/dml/update.rs

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 02814cc9..4d501635 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -31,8 +31,143 @@ use {
         schema, LangError, LangResult,
     },
     crate::util::compiler,
+    core::cmp,
 };
 
+#[inline(always)]
+pub fn minidx<T>(src: &[T], index: usize) -> usize {
+    cmp::min(src.len() - 1, index)
+}
+
+#[derive(Debug, PartialEq)]
+/// Query parse state
+pub struct State<'a, Qd> {
+    t: &'a [Token<'a>],
+    d: Qd,
+    i: usize,
+    f: bool,
+}
+
+impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
+    #[inline(always)]
+    /// Create a new [`State`] instance using the given tokens and data
+    pub const fn new(t: &'a [Token<'a>], d: Qd) -> Self {
+        Self {
+            i: 0,
+            f: true,
+            t,
+            d,
+        }
+    }
+    #[inline(always)]
+    /// Returns `true` if the state is okay
+    pub const fn okay(&self) -> bool {
+        self.f
+    }
+    #[inline(always)]
+    /// Poison the state flag
+    pub fn poison(&mut self) {
+        self.f = false;
+    }
+    #[inline(always)]
+    /// Poison the state flag if the expression is satisfied
+    pub fn poison_if(&mut self, fuse: bool) {
+        self.f &= !fuse;
+    }
+    #[inline(always)]
+    /// Poison the state flag if the expression is not satisfied
+    pub fn poison_if_not(&mut self, fuse: bool) {
+        self.poison_if(!fuse);
+    }
+    #[inline(always)]
+    /// Move the cursor ahead by 1
+    pub fn cursor_ahead(&mut self) {
+        self.cursor_ahead_by(1)
+    }
+    #[inline(always)]
+    /// Move the cursor ahead by the given count
+    pub fn cursor_ahead_by(&mut self, by: usize) {
+        self.i += by;
+    }
+    #[inline(always)]
+    /// Move the cursor ahead by 1 if the expression is satisfied
+    pub fn cursor_ahead_if(&mut self, iff: bool) {
+        self.cursor_ahead_by(iff as _);
+    }
+    #[inline(always)]
+    /// Read the cursor
+    pub fn read(&self) -> &'a Token<'a> {
+        &self.t[self.i]
+    }
+    #[inline(always)]
+    /// Return a subslice of the tokens using the current state
+    pub fn current(&self) -> &'a [Token<'a>] {
+        &self.t[self.i..]
+    }
+    #[inline(always)]
+    /// Returns a count of the number of consumable tokens remaining
+    pub fn remaining(&self) -> usize {
+        self.t.len() - self.i
+    }
+    #[inline(always)]
+    /// Read and forward the cursor
+    pub fn fw_read(&mut self) -> &'a Token<'a> {
+        let r = self.read();
+        self.cursor_ahead();
+        r
+    }
+    #[inline(always)]
+    /// Check if the token stream has alteast `many` count of tokens
+    pub fn has_remaining(&mut self, many: usize) -> bool {
+        self.remaining() >= many
+    }
+    #[inline(always)]
+    /// Returns true if the token stream has been exhausted
+    pub fn exhausted(&self) -> bool {
+        self.remaining() == 0
+    }
+    #[inline(always)]
+    /// Returns true if the token stream has **not** been exhausted
+    pub fn not_exhausted(&self) -> bool {
+        self.remaining() != 0
+    }
+    #[inline(always)]
+    /// Check if the current cursor can read a lit (with context from the data source); rounded
+    pub fn can_read_lit_rounded(&self) -> bool {
+        let mx = minidx(self.t, self.i);
+        Qd::can_read_lit_from(&self.d, &self.t[mx]) && mx == self.i
+    }
+    /// Check if a lit can be read using the given token with context from the data source
+    pub fn can_read_lit_from(&self, tok: &'a Token<'a>) -> bool {
+        Qd::can_read_lit_from(&self.d, tok)
+    }
+    #[inline(always)]
+    /// Read a lit from the cursor and data source
+    ///
+    /// ## Safety
+    /// - Must ensure that `Self::can_read_lit_rounded` is true
+    pub unsafe fn read_cursor_lit_unchecked(&mut self) -> LitIR<'a> {
+        let tok = self.read();
+        Qd::read_lit(&mut self.d, tok)
+    }
+    #[inline(always)]
+    /// Check if the cursor equals the given token; rounded
+    pub fn cursor_rounded_eq(&self, tok: Token<'a>) -> bool {
+        let mx = minidx(self.t, self.i);
+        self.t[mx] == tok && mx == self.i
+    }
+    #[inline(always)]
+    /// Check if the cursor equals the given token
+    pub(crate) fn cursor_eq(&self, token: Token) -> bool {
+        self.t[self.i] == token
+    }
+    #[inline(always)]
+    /// Read ahead from the cursor by the given positions
+    pub(crate) fn read_ahead(&self, ahead: usize) -> &'a Token<'a> {
+        &self.t[self.i + ahead]
+    }
+}
+
 pub trait QueryData<'a> {
     /// Check if the given token is a lit, while also checking `self`'s data if necessary
     fn can_read_lit_from(&self, tok: &Token) -> bool;
@@ -43,6 +178,7 @@ pub trait QueryData<'a> {
     unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a>;
 }
 
+#[derive(Debug)]
 pub struct InplaceData;
 impl InplaceData {
     #[inline(always)]
@@ -62,6 +198,7 @@ impl<'a> QueryData<'a> for InplaceData {
     }
 }
 
+#[derive(Debug)]
 pub struct SubstitutedData<'a> {
     data: &'a [LitIR<'a>],
 }
@@ -236,18 +373,18 @@ pub enum Statement<'a> {
     /// DDL query to inspect all spaces (returns a list of spaces in the database)
     InspectSpaces,
     /// DML insert
-    Insert(dml::InsertStatement<'a>),
+    Insert(dml::insert::InsertStatement<'a>),
     /// DML select
-    Select(dml::SelectStatement<'a>),
+    Select(dml::select::SelectStatement<'a>),
     /// DML update
-    Update(dml::UpdateStatement<'a>),
+    Update(dml::update::UpdateStatement<'a>),
     /// DML delete
-    Delete(dml::DeleteStatement<'a>),
+    Delete(dml::delete::DeleteStatement<'a>),
 }
 
-pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], mut qd: Qd) -> LangResult<Statement<'a>> {
+pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], mut d: Qd) -> LangResult<Statement<'a>> {
     let mut i = 0;
-    let ref mut qd = qd;
+    let ref mut qd = d;
     if compiler::unlikely(tok.len() < 2) {
         return Err(LangError::UnexpectedEndofStatement);
     }
@@ -281,14 +418,23 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], mut qd: Qd) -> LangResul
             ddl::parse_inspect(&tok[1..], &mut i)
         }
         // DML
-        Token![insert] => dml::parse_insert(&tok[1..], qd, &mut i).map(Statement::Insert),
-        Token![select] => dml::parse_select(&tok[1..], qd, &mut i).map(Statement::Select),
-        Token![update] => {
-            dml::UpdateStatement::parse_update(&tok[1..], qd, &mut i).map(Statement::Update)
-        }
-        Token![delete] => {
-            dml::DeleteStatement::parse_delete(&tok[1..], qd, &mut i).map(Statement::Delete)
+        ref stmt => {
+            let mut state = State::new(&tok[1..], d);
+            match stmt {
+                Token![insert] => {
+                    dml::insert::InsertStatement::parse_insert(&mut state).map(Statement::Insert)
+                }
+                Token![select] => {
+                    dml::select::SelectStatement::parse_select(&mut state).map(Statement::Select)
+                }
+                Token![update] => {
+                    dml::update::UpdateStatement::parse_update(&mut state).map(Statement::Update)
+                }
+                Token![delete] => {
+                    dml::delete::DeleteStatement::parse_delete(&mut state).map(Statement::Delete)
+                }
+                _ => compiler::cold_rerr(LangError::ExpectedStatement),
+            }
         }
-        _ => compiler::cold_rerr(LangError::ExpectedStatement),
     }
 }
diff --git a/server/src/engine/ql/dml.rs b/server/src/engine/ql/dml.rs
deleted file mode 100644
index 7d28f386..00000000
--- a/server/src/engine/ql/dml.rs
+++ /dev/null
@@ -1,962 +0,0 @@
-/*
- * Created on Fri Oct 14 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
-    TODO(@ohsayan): For now we've settled for an imprecise error site reporting for simplicity, which we
-    should augment in future revisions of the QL engine
-*/
-
-#[cfg(test)]
-use super::ast::InplaceData;
-use {
-    super::{
-        ast::{Entity, QueryData},
-        lexer::{LitIR, Symbol, Token},
-        LangError, LangResult,
-    },
-    crate::{
-        engine::memory::DataType,
-        util::{compiler, MaybeInit},
-    },
-    std::{
-        cmp,
-        collections::HashMap,
-        mem::{discriminant, Discriminant},
-    },
-};
-
-#[inline(always)]
-fn minidx<T>(src: &[T], index: usize) -> usize {
-    cmp::min(src.len() - 1, index)
-}
-
-/*
-    Misc
-*/
-
-#[inline(always)]
-fn process_entity<'a>(tok: &'a [Token<'a>], d: &mut MaybeInit<Entity<'a>>, i: &mut usize) -> bool {
-    let is_full = Entity::tokens_with_full(tok);
-    let is_single = Entity::tokens_with_single(tok);
-    if is_full {
-        *i += 3;
-        *d = MaybeInit::new(unsafe {
-            // UNSAFE(@ohsayan): Predicate ensures validity
-            Entity::full_entity_from_slice(tok)
-        })
-    } else if is_single {
-        *i += 1;
-        *d = MaybeInit::new(unsafe {
-            // UNSAFE(@ohsayan): Predicate ensures validity
-            Entity::single_entity_from_slice(tok)
-        });
-    }
-    is_full | is_single
-}
-
-/*
-    Contexts
-*/
-
-#[derive(Debug, PartialEq)]
-pub struct RelationalExpr<'a> {
-    pub(super) lhs: &'a [u8],
-    pub(super) rhs: LitIR<'a>,
-    pub(super) opc: u8,
-}
-
-impl<'a> RelationalExpr<'a> {
-    #[inline(always)]
-    pub(super) fn new(lhs: &'a [u8], rhs: LitIR<'a>, opc: u8) -> RelationalExpr<'a> {
-        Self { lhs, rhs, opc }
-    }
-    pub(super) const OP_EQ: u8 = 1;
-    pub(super) const OP_NE: u8 = 2;
-    pub(super) const OP_GT: u8 = 3;
-    pub(super) const OP_GE: u8 = 4;
-    pub(super) const OP_LT: u8 = 5;
-    pub(super) const OP_LE: u8 = 6;
-    fn filter_hint_none(&self) -> bool {
-        self.opc == Self::OP_EQ
-    }
-    #[inline(always)]
-    fn parse_operator(tok: &[Token], i: &mut usize, okay: &mut bool) -> u8 {
-        /*
-            FIXME(@ohsayan): This is relatively messy right now, but does the job. Will
-            re-implement later.
-        */
-        #[inline(always)]
-        fn u(b: bool) -> u8 {
-            b as _
-        }
-        let op_eq = u(tok[0] == Token![=]) * Self::OP_EQ;
-        let op_ne = u(tok[0] == Token![!] && tok[1] == Token![=]) * Self::OP_NE;
-        let op_ge = u(tok[0] == Token![>] && tok[1] == Token![=]) * Self::OP_GE;
-        let op_gt = u(tok[0] == Token![>] && op_ge == 0) * Self::OP_GT;
-        let op_le = u(tok[0] == Token![<] && tok[1] == Token![=]) * Self::OP_LE;
-        let op_lt = u(tok[0] == Token![<] && op_le == 0) * Self::OP_LT;
-        let opc = op_eq + op_ne + op_ge + op_gt + op_le + op_lt;
-        *okay &= opc != 0;
-        *i += 1 + (opc & 1 == 0) as usize;
-        opc
-    }
-    #[inline(always)]
-    fn try_parse<Qd: QueryData<'a>>(tok: &'a [Token], d: &mut Qd, cnt: &mut usize) -> Option<Self> {
-        /*
-            Minimum length of an expression:
-            [lhs] [operator] [rhs]
-        */
-        let mut okay = tok.len() >= 3;
-        let mut i = 0_usize;
-        if compiler::unlikely(!okay) {
-            return None;
-        }
-        okay &= tok[0].is_ident();
-        i += 1;
-        // let's get ourselves the operator
-        let operator = Self::parse_operator(&tok[i..], &mut i, &mut okay);
-        okay &= i < tok.len();
-        let lit_idx = minidx(tok, i);
-        okay &= Qd::can_read_lit_from(d, &tok[lit_idx]); // LOL, I really like saving cycles
-        *cnt += i + okay as usize;
-        if compiler::likely(okay) {
-            unsafe {
-                // UNSAFE(@ohsayan): tok[0] is checked for being an ident, tok[lit_idx] also checked to be a lit
-                let lit = Qd::read_lit(d, &tok[lit_idx]);
-                Some(Self::new(
-                    extract!(tok[0], Token::Ident(ref id) => id),
-                    lit,
-                    operator,
-                ))
-            }
-        } else {
-            compiler::cold_val(None)
-        }
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub struct WhereClause<'a> {
-    c: WhereClauseCollection<'a>,
-}
-
-type WhereClauseCollection<'a> = HashMap<&'a [u8], RelationalExpr<'a>>;
-
-impl<'a> WhereClause<'a> {
-    #[inline(always)]
-    pub(super) fn new(c: WhereClauseCollection<'a>) -> Self {
-        Self { c }
-    }
-    #[inline(always)]
-    /// Parse the expressions in a `where` context, appending it to the given map
-    ///
-    /// Notes:
-    /// - Deny duplicate clauses
-    /// - No enforcement on minimum number of clauses
-    fn parse_where_and_append_to<Qd: QueryData<'a>>(
-        tok: &'a [Token],
-        d: &mut Qd,
-        cnt: &mut usize,
-        c: &mut WhereClauseCollection<'a>,
-    ) -> bool {
-        let l = tok.len();
-        let mut okay = true;
-        let mut i = 0;
-        let mut has_more = true;
-        while okay && i < l && has_more {
-            okay &= RelationalExpr::try_parse(&tok[i..], d, &mut i)
-                .map(|clause| c.insert(clause.lhs, clause).is_none())
-                .unwrap_or(false);
-            has_more = tok[minidx(tok, i)] == Token![and] && i < l;
-            i += has_more as usize;
-        }
-        *cnt += i;
-        okay
-    }
-    #[inline(always)]
-    /// Parse a where context
-    ///
-    /// Notes:
-    /// - Enforce a minimum of 1 clause
-    pub(super) fn parse_where<Qd: QueryData<'a>>(
-        tok: &'a [Token],
-        d: &mut Qd,
-        flag: &mut bool,
-        cnt: &mut usize,
-    ) -> Self {
-        let mut c = HashMap::with_capacity(2);
-        *flag &= Self::parse_where_and_append_to(tok, d, cnt, &mut c);
-        *flag &= !c.is_empty();
-        Self { c }
-    }
-}
-
-#[cfg(test)]
-pub(super) fn parse_where_clause_full<'a>(tok: &'a [Token]) -> Option<WhereClause<'a>> {
-    let mut flag = true;
-    let mut i = 0;
-    let ret = WhereClause::parse_where(tok, &mut InplaceData::new(), &mut flag, &mut i);
-    assert_full_tt!(tok.len(), i);
-    flag.then_some(ret)
-}
-
-#[cfg(test)]
-#[inline(always)]
-pub(super) fn parse_relexpr_full<'a>(tok: &'a [Token]) -> Option<RelationalExpr<'a>> {
-    let mut i = 0;
-    let okay = RelationalExpr::try_parse(tok, &mut InplaceData::new(), &mut i);
-    assert_full_tt!(tok.len(), i);
-    okay
-}
-
-/*
-    Impls for insert
-*/
-
-/// Parse a list
-///
-/// **NOTE:** This function will error if the `[` token is passed. Make sure this is forwarded by the caller
-pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
-    tok: &'a [Token],
-    d: &mut Qd,
-    list: &mut Vec<DataType>,
-) -> (Option<Discriminant<DataType>>, usize, bool) {
-    let l = tok.len();
-    let mut okay = l != 0;
-    let mut stop = okay && tok[0] == Symbol::TtCloseSqBracket;
-    let mut i = stop as usize;
-    let mut overall_dscr = None;
-    let mut prev_nlist_dscr = None;
-    while i < l && okay && !stop {
-        let d = match &tok[i] {
-            tok if Qd::can_read_lit_from(d, tok) => {
-                unsafe {
-                    // UNSAFE(@ohsayan): Token LT0 guarantees LT0 > LT1 for lit
-                    DataType::clone_from_litir(Qd::read_lit(d, tok))
-                }
-            }
-            Token::Symbol(Symbol::TtOpenSqBracket) => {
-                // a nested list
-                let mut nested_list = Vec::new();
-                let (nlist_dscr, nlist_i, nlist_okay) =
-                    parse_list(&tok[i + 1..], d, &mut nested_list);
-                okay &= nlist_okay;
-                i += nlist_i;
-                // check type return
-                okay &= prev_nlist_dscr.is_none()
-                    || nlist_dscr.is_none()
-                    || prev_nlist_dscr == nlist_dscr;
-                if prev_nlist_dscr.is_none() && nlist_dscr.is_some() {
-                    prev_nlist_dscr = nlist_dscr;
-                }
-                DataType::List(nested_list)
-            }
-            _ => {
-                okay = false;
-                break;
-            }
-        };
-        i += 1;
-        okay &= list.is_empty() || discriminant(&d) == discriminant(&list[0]);
-        overall_dscr = Some(discriminant(&d));
-        list.push(d);
-        let nx_comma = i < l && tok[i] == Symbol::SymComma;
-        let nx_csqrb = i < l && tok[i] == Symbol::TtCloseSqBracket;
-        okay &= nx_comma | nx_csqrb;
-        i += okay as usize;
-        stop = nx_csqrb;
-    }
-    (overall_dscr, i, okay && stop)
-}
-
-#[cfg(test)]
-pub(super) fn parse_list_full<'a>(
-    tok: &'a [Token],
-    qd: &mut impl QueryData<'a>,
-) -> Option<Vec<DataType>> {
-    let mut l = Vec::new();
-    if matches!(parse_list(tok, qd, &mut l), (_, i, true) if i == tok.len()) {
-        Some(l)
-    } else {
-        None
-    }
-}
-
-/// Parse the tuple data passed in with an insert query.
-///
-/// **Note:** Make sure you pass the `(` token
-pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
-    tok: &'a [Token],
-    d: &mut Qd,
-) -> (Vec<Option<DataType>>, usize, bool) {
-    let l = tok.len();
-    let mut okay = l != 0;
-    let mut stop = okay && tok[0] == Token::Symbol(Symbol::TtCloseParen);
-    let mut i = stop as usize;
-    let mut data = Vec::new();
-    while i < l && okay && !stop {
-        match &tok[i] {
-            tok if Qd::can_read_lit_from(d, tok) => {
-                unsafe {
-                    // UNSAFE(@ohsayan): Token LT0 guarantees LT0 > LT1 for lit
-                    data.push(Some(DataType::clone_from_litir(Qd::read_lit(d, tok))));
-                }
-            }
-            Token::Symbol(Symbol::TtOpenSqBracket) => {
-                // ah, a list
-                let mut l = Vec::new();
-                let (_, lst_i, lst_okay) = parse_list(&tok[i + 1..], d, &mut l);
-                data.push(Some(l.into()));
-                i += lst_i;
-                okay &= lst_okay;
-            }
-            Token![null] => {
-                data.push(None);
-            }
-            _ => {
-                okay = false;
-                break;
-            }
-        }
-        i += 1;
-        let nx_comma = i < l && tok[i] == Symbol::SymComma;
-        let nx_csprn = i < l && tok[i] == Symbol::TtCloseParen;
-        okay &= nx_comma | nx_csprn;
-        i += okay as usize;
-        stop = nx_csprn;
-    }
-    (data, i, okay && stop)
-}
-
-#[cfg(test)]
-pub(super) fn parse_data_tuple_syntax_full(tok: &[Token]) -> Option<Vec<Option<DataType>>> {
-    let (ret, cnt, okay) = parse_data_tuple_syntax(tok, &mut InplaceData::new());
-    assert!(cnt == tok.len(), "didn't use full length");
-    if okay {
-        Some(ret)
-    } else {
-        None
-    }
-}
-
-pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
-    tok: &'a [Token],
-    d: &mut Qd,
-) -> (HashMap<&'a [u8], Option<DataType>>, usize, bool) {
-    let l = tok.len();
-    let mut okay = l != 0;
-    let mut stop = okay && tok[0] == Token::Symbol(Symbol::TtCloseBrace);
-    let mut i = stop as usize;
-    let mut data = HashMap::new();
-    while i + 3 < l && okay && !stop {
-        let (field, colon, expression) = (&tok[i], &tok[i + 1], &tok[i + 2]);
-        okay &= colon == &Symbol::SymColon;
-        match (field, expression) {
-            (Token::Ident(id), tok) if Qd::can_read_lit_from(d, tok) => {
-                okay &= data
-                    .insert(
-                        *id,
-                        Some(unsafe {
-                            // UNSAFE(@ohsayan): Token LT0 guarantees LT0 > LT1 for lit
-                            DataType::clone_from_litir(Qd::read_lit(d, tok))
-                        }),
-                    )
-                    .is_none();
-            }
-            (Token::Ident(id), Token::Symbol(Symbol::TtOpenSqBracket)) => {
-                // ooh a list
-                let mut l = Vec::new();
-                let (_, lst_i, lst_ok) = parse_list(&tok[i + 3..], d, &mut l);
-                okay &= lst_ok;
-                i += lst_i;
-                okay &= data.insert(id, Some(l.into())).is_none();
-            }
-            (Token::Ident(id), Token![null]) => {
-                okay &= data.insert(id, None).is_none();
-            }
-            _ => {
-                okay = false;
-                break;
-            }
-        }
-        i += 3;
-        let nx_comma = i < l && tok[i] == Symbol::SymComma;
-        let nx_csbrc = i < l && tok[i] == Symbol::TtCloseBrace;
-        okay &= nx_comma | nx_csbrc;
-        i += okay as usize;
-        stop = nx_csbrc;
-    }
-    (data, i, okay && stop)
-}
-
-#[cfg(test)]
-pub(super) fn parse_data_map_syntax_full(
-    tok: &[Token],
-) -> Option<HashMap<Box<str>, Option<DataType>>> {
-    let (dat, i, ok) = parse_data_map_syntax(tok, &mut InplaceData::new());
-    assert!(i == tok.len(), "didn't use full length");
-    if ok {
-        Some(
-            dat.into_iter()
-                .map(|(ident, val)| {
-                    (
-                        String::from_utf8_lossy(ident).to_string().into_boxed_str(),
-                        val,
-                    )
-                })
-                .collect(),
-        )
-    } else {
-        None
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub enum InsertData<'a> {
-    Ordered(Vec<Option<DataType>>),
-    Map(HashMap<&'a [u8], Option<DataType>>),
-}
-
-impl<'a> From<Vec<Option<DataType>>> for InsertData<'a> {
-    fn from(v: Vec<Option<DataType>>) -> Self {
-        Self::Ordered(v)
-    }
-}
-
-impl<'a> From<HashMap<&'static [u8], Option<DataType>>> for InsertData<'a> {
-    fn from(m: HashMap<&'static [u8], Option<DataType>>) -> Self {
-        Self::Map(m)
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub struct InsertStatement<'a> {
-    pub(super) entity: Entity<'a>,
-    pub(super) data: InsertData<'a>,
-}
-
-#[inline(always)]
-fn parse_entity<'a>(tok: &'a [Token], entity: &mut MaybeInit<Entity<'a>>, i: &mut usize) -> bool {
-    let is_full = tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident();
-    let is_half = tok[0].is_ident();
-    unsafe {
-        // UNSAFE(@ohsayan): The branch predicates assert their correctness
-        if is_full {
-            *i += 3;
-            *entity = MaybeInit::new(Entity::full_entity_from_slice(&tok));
-        } else if is_half {
-            *i += 1;
-            *entity = MaybeInit::new(Entity::single_entity_from_slice(&tok));
-        }
-    }
-    is_full | is_half
-}
-
-pub(super) fn parse_insert<'a, Qd: QueryData<'a>>(
-    tok: &'a [Token],
-    d: &mut Qd,
-    counter: &mut usize,
-) -> LangResult<InsertStatement<'a>> {
-    /*
-        smallest:
-        insert into model (primarykey)
-               ^1    ^2   ^3      ^4 ^5
-    */
-    let l = tok.len();
-    if compiler::unlikely(l < 5) {
-        return compiler::cold_val(Err(LangError::UnexpectedEndofStatement));
-    }
-    let mut okay = tok[0] == Token![into];
-    let mut i = okay as usize;
-    let mut entity = MaybeInit::uninit();
-    okay &= parse_entity(&tok[i..], &mut entity, &mut i);
-    let mut data = None;
-    if !(i < l) {
-        unsafe {
-            // UNSAFE(@ohsayan): ALWAYS true because 1 + 3 for entity; early exit if smaller
-            impossible!();
-        }
-    }
-    match tok[i] {
-        Token![() open] => {
-            let (this_data, incr, ok) = parse_data_tuple_syntax(&tok[i + 1..], d);
-            okay &= ok;
-            i += incr + 1;
-            data = Some(InsertData::Ordered(this_data));
-        }
-        Token![open {}] => {
-            let (this_data, incr, ok) = parse_data_map_syntax(&tok[i + 1..], d);
-            okay &= ok;
-            i += incr + 1;
-            data = Some(InsertData::Map(this_data));
-        }
-        _ => okay = false,
-    }
-    *counter += i;
-    if okay {
-        let data = unsafe {
-            // UNSAFE(@ohsayan): Will be safe because of `okay` since it ensures that entity has been initialized
-            data.unwrap_unchecked()
-        };
-        Ok(InsertStatement {
-            entity: unsafe {
-                // UNSAFE(@ohsayan): Will be safe because of `okay` since it ensures that entity has been initialized
-                entity.assume_init()
-            },
-            data,
-        })
-    } else {
-        Err(LangError::UnexpectedToken)
-    }
-}
-
-#[cfg(test)]
-pub(super) fn parse_insert_full<'a>(tok: &'a [Token]) -> Option<InsertStatement<'a>> {
-    let mut z = 0;
-    let s = self::parse_insert(tok, &mut InplaceData::new(), &mut z);
-    assert!(z == tok.len(), "didn't use full length");
-    s.ok()
-}
-
-/*
-    Impls for select
-*/
-
-#[derive(Debug, PartialEq)]
-pub struct SelectStatement<'a> {
-    /// the entity
-    pub(super) entity: Entity<'a>,
-    /// fields in order of querying. will be zero when wildcard is set
-    pub(super) fields: Vec<&'a [u8]>,
-    /// whether a wildcard was passed
-    pub(super) wildcard: bool,
-    /// where clause
-    pub(super) clause: WhereClause<'a>,
-}
-impl<'a> SelectStatement<'a> {
-    #[inline(always)]
-    pub(crate) fn new_test(
-        entity: Entity<'a>,
-        fields: Vec<&'a [u8]>,
-        wildcard: bool,
-        clauses: WhereClauseCollection<'a>,
-    ) -> SelectStatement<'a> {
-        Self::new(entity, fields, wildcard, clauses)
-    }
-    #[inline(always)]
-    fn new(
-        entity: Entity<'a>,
-        fields: Vec<&'a [u8]>,
-        wildcard: bool,
-        clauses: WhereClauseCollection<'a>,
-    ) -> SelectStatement<'a> {
-        Self {
-            entity,
-            fields,
-            wildcard,
-            clause: WhereClause::new(clauses),
-        }
-    }
-}
-
-/// Parse a `select` query. The cursor should have already passed the `select` token when this
-/// function is called.
-pub(super) fn parse_select<'a, Qd: QueryData<'a>>(
-    tok: &'a [Token],
-    d: &mut Qd,
-    counter: &mut usize,
-) -> LangResult<SelectStatement<'a>> {
-    /*
-        Smallest query:
-        select * from model
-               ^ ^    ^
-               1 2    3
-    */
-    let l = tok.len();
-    if compiler::unlikely(l < 3) {
-        return compiler::cold_val(Err(LangError::UnexpectedEndofStatement));
-    }
-    let mut i = 0;
-    let mut okay = true;
-    let mut select_fields = Vec::new();
-    let is_wildcard = tok[0] == Token![*];
-    i += is_wildcard as usize;
-    while i < l && okay && !is_wildcard {
-        match tok[i] {
-            Token::Ident(ref id) => select_fields.push(id.clone()),
-            _ => {
-                break;
-            }
-        }
-        i += 1;
-        let nx_idx = minidx(tok, i);
-        let nx_comma = tok[nx_idx] == Token![,] && i < l;
-        let nx_from = tok[nx_idx] == Token![from];
-        okay &= nx_comma | nx_from;
-        i += nx_comma as usize;
-    }
-    okay &= is_wildcard | !select_fields.is_empty();
-    okay &= (i + 2) <= l;
-    if compiler::unlikely(!okay) {
-        return compiler::cold_val(Err(LangError::UnexpectedToken));
-    }
-    okay &= tok[i] == Token![from];
-    i += okay as usize;
-    // now process entity
-    let mut entity = MaybeInit::uninit();
-    okay &= process_entity(&tok[i..], &mut entity, &mut i);
-    let has_where = tok[minidx(tok, i)] == Token![where];
-    i += has_where as usize;
-    let mut clauses = <_ as Default>::default();
-    if has_where {
-        okay &= WhereClause::parse_where_and_append_to(&tok[i..], d, &mut i, &mut clauses);
-        okay &= !clauses.is_empty(); // append doesn't enforce clause arity
-    }
-    *counter += i;
-    if okay {
-        Ok(SelectStatement {
-            entity: unsafe {
-                // UNSAFE(@ohsayan): `process_entity` and `okay` assert correctness
-                entity.assume_init()
-            },
-            fields: select_fields,
-            wildcard: is_wildcard,
-            clause: WhereClause::new(clauses),
-        })
-    } else {
-        Err(LangError::UnexpectedToken)
-    }
-}
-
-#[cfg(test)]
-/// **test-mode only** parse for a `select` where the full token stream is exhausted
-pub(super) fn parse_select_full<'a>(tok: &'a [Token]) -> Option<SelectStatement<'a>> {
-    let mut i = 0;
-    let r = self::parse_select(tok, &mut InplaceData::new(), &mut i);
-    assert_full_tt!(i, tok.len());
-    r.ok()
-}
-
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
-/// TODO(@ohsayan): This only helps with the parser test for now. Replace this with actual operator expressions
-pub(super) enum Operator {
-    Assign,
-    AddAssign,
-    SubAssign,
-    MulAssign,
-    DivAssign,
-}
-
-static OPERATOR: [Operator; 6] = [
-    Operator::Assign,
-    Operator::Assign,
-    Operator::AddAssign,
-    Operator::SubAssign,
-    Operator::MulAssign,
-    Operator::DivAssign,
-];
-
-#[derive(Debug, PartialEq)]
-pub struct AssignmentExpression<'a> {
-    /// the LHS ident
-    pub(super) lhs: &'a [u8],
-    /// the RHS lit
-    pub(super) rhs: LitIR<'a>,
-    /// operator
-    pub(super) operator_fn: Operator,
-}
-
-impl<'a> AssignmentExpression<'a> {
-    pub(super) fn new(lhs: &'a [u8], rhs: LitIR<'a>, operator_fn: Operator) -> Self {
-        Self {
-            lhs,
-            rhs,
-            operator_fn,
-        }
-    }
-    /// Attempt to parse an expression and then append it to the given vector of expressions. This will return `true`
-    /// if the expression was parsed correctly, otherwise `false` is returned
-    #[inline(always)]
-    fn parse_and_append_expression<Qd: QueryData<'a>>(
-        tok: &'a [Token],
-        d: &mut Qd,
-        expressions: &mut Vec<Self>,
-        counter: &mut usize,
-    ) -> bool {
-        /*
-            smallest expression:
-            <ident> <operator> <lit>
-        */
-        let l = tok.len();
-        let mut i = 0;
-        let mut okay = tok.len() > 2 && tok[0].is_ident();
-        i += okay as usize;
-
-        let op_assign = (i < l && tok[i] == Token![=]) as usize * 1;
-        let op_add = (i < l && tok[i] == Token![+]) as usize * 2;
-        let op_sub = (i < l && tok[i] == Token![-]) as usize * 3;
-        let op_mul = (i < l && tok[i] == Token![*]) as usize * 4;
-        let op_div = (i < l && tok[i] == Token![/]) as usize * 5;
-
-        let operator_code = op_assign + op_add + op_sub + op_mul + op_div;
-        unsafe {
-            // UNSAFE(@ohsayan): Inherently obvious, just a hint
-            if operator_code > 5 {
-                impossible!()
-            }
-        }
-        okay &= operator_code != 0;
-        i += okay as usize;
-
-        let has_double_assign = i < l && tok[i] == Token![=];
-        let double_assign_okay = operator_code != 1 && has_double_assign;
-        let single_assign_okay = operator_code == 1 && !double_assign_okay;
-        okay &= single_assign_okay | double_assign_okay;
-        i += double_assign_okay as usize; // skip on <op>assign
-
-        let has_rhs = Qd::can_read_lit_from(d, &tok[minidx(tok, i)]);
-        okay &= has_rhs;
-        *counter += i + has_rhs as usize;
-
-        if okay {
-            let expression = unsafe {
-                /*
-                   UNSAFE(@ohsayan): tok[0] is checked for being an ident early on; second, tok[i]
-                   is also checked for being a lit and then `okay` ensures correctness
-                */
-                let rhs = Qd::read_lit(d, &tok[i]);
-                AssignmentExpression {
-                    lhs: extract!(tok[0], Token::Ident(ref r) => r),
-                    rhs,
-                    operator_fn: OPERATOR[operator_code as usize],
-                }
-            };
-            expressions.push(expression);
-        }
-
-        okay
-    }
-}
-
-#[cfg(test)]
-pub(super) fn parse_expression_full<'a>(tok: &'a [Token]) -> Option<AssignmentExpression<'a>> {
-    let mut i = 0;
-    let mut exprs = Vec::new();
-    if AssignmentExpression::parse_and_append_expression(
-        tok,
-        &mut InplaceData::new(),
-        &mut exprs,
-        &mut i,
-    ) {
-        assert_full_tt!(i, tok.len());
-        Some(exprs.remove(0))
-    } else {
-        None
-    }
-}
-
-/*
-    Impls for update
-*/
-
-#[derive(Debug, PartialEq)]
-pub struct UpdateStatement<'a> {
-    pub(super) entity: Entity<'a>,
-    pub(super) expressions: Vec<AssignmentExpression<'a>>,
-    pub(super) wc: WhereClause<'a>,
-}
-
-impl<'a> UpdateStatement<'a> {
-    #[inline(always)]
-    #[cfg(test)]
-    pub fn new_test(
-        entity: Entity<'a>,
-        expressions: Vec<AssignmentExpression<'a>>,
-        wc: WhereClauseCollection<'a>,
-    ) -> Self {
-        Self::new(entity, expressions, WhereClause::new(wc))
-    }
-    #[inline(always)]
-    pub fn new(
-        entity: Entity<'a>,
-        expressions: Vec<AssignmentExpression<'a>>,
-        wc: WhereClause<'a>,
-    ) -> Self {
-        Self {
-            entity,
-            expressions,
-            wc,
-        }
-    }
-    #[inline(always)]
-    pub(super) fn parse_update<Qd: QueryData<'a>>(
-        tok: &'a [Token],
-        d: &mut Qd,
-        counter: &mut usize,
-    ) -> LangResult<Self> {
-        /*
-            TODO(@ohsayan): Allow volcanoes
-            smallest tt:
-            update model SET x  =  1 where x = 1
-                   ^1    ^2  ^3 ^4 ^5^6    ^7^8^9
-        */
-        let l = tok.len();
-        if compiler::unlikely(l < 9) {
-            return compiler::cold_val(Err(LangError::UnexpectedEndofStatement));
-        }
-        let mut i = 0;
-        let mut entity = MaybeInit::uninit();
-        let mut okay = parse_entity(&tok[i..], &mut entity, &mut i);
-        if !((i + 6) <= l) {
-            unsafe {
-                // UNSAFE(@ohsayan): Obvious, just a hint; entity can fw by 3 max
-                impossible!();
-            }
-        }
-        okay &= tok[i] == Token![set];
-        i += 1; // ignore whatever we have here, even if it's broken
-        let mut nx_where = false;
-        let mut expressions = Vec::new();
-        while i < l && okay && !nx_where {
-            okay &= AssignmentExpression::parse_and_append_expression(
-                &tok[i..],
-                d,
-                &mut expressions,
-                &mut i,
-            );
-            let nx_idx = minidx(tok, i);
-            let nx_comma = tok[nx_idx] == Token![,] && i < l;
-            // NOTE: volcano
-            nx_where = tok[nx_idx] == Token![where] && i < l;
-            okay &= nx_comma | nx_where; // NOTE: volcano
-            i += nx_comma as usize;
-        }
-        okay &= nx_where;
-        i += okay as usize;
-        // now process expressions
-        let mut clauses = <_ as Default>::default();
-        okay &= WhereClause::parse_where_and_append_to(&tok[i..], d, &mut i, &mut clauses);
-        okay &= !clauses.is_empty(); // NOTE: volcano
-        *counter += i;
-        if okay {
-            Ok(Self {
-                entity: unsafe {
-                    // UNSAFE(@ohsayan): This is safe because of `parse_entity` and `okay`
-                    entity.assume_init()
-                },
-                expressions,
-                wc: WhereClause::new(clauses),
-            })
-        } else {
-            Err(LangError::UnexpectedToken)
-        }
-    }
-}
-
-#[cfg(test)]
-pub(super) fn parse_update_full<'a>(tok: &'a [Token]) -> LangResult<UpdateStatement<'a>> {
-    let mut i = 0;
-    let r = UpdateStatement::parse_update(tok, &mut InplaceData::new(), &mut i);
-    assert_full_tt!(i, tok.len());
-    r
-}
-
-/*
-    Impls for delete
-    ---
-    Smallest statement:
-    delete model:primary_key
-*/
-
-#[derive(Debug, PartialEq)]
-pub struct DeleteStatement<'a> {
-    pub(super) entity: Entity<'a>,
-    pub(super) wc: WhereClause<'a>,
-}
-
-impl<'a> DeleteStatement<'a> {
-    #[inline(always)]
-    pub(super) fn new(entity: Entity<'a>, wc: WhereClause<'a>) -> Self {
-        Self { entity, wc }
-    }
-    #[inline(always)]
-    #[cfg(test)]
-    pub(super) fn new_test(entity: Entity<'a>, wc: WhereClauseCollection<'a>) -> Self {
-        Self::new(entity, WhereClause::new(wc))
-    }
-    pub(super) fn parse_delete<Qd: QueryData<'a>>(
-        tok: &'a [Token],
-        d: &mut Qd,
-        counter: &mut usize,
-    ) -> LangResult<Self> {
-        /*
-            TODO(@ohsayan): Volcano
-            smallest tt:
-            delete from model where x = 1
-                   ^1   ^2    ^3    ^4  ^5
-        */
-        let l = tok.len();
-        if compiler::unlikely(l < 5) {
-            return compiler::cold_val(Err(LangError::UnexpectedEndofStatement));
-        }
-        let mut i = 0;
-        let mut okay = tok[i] == Token![from];
-        i += 1; // skip even if incorrect
-        let mut entity = MaybeInit::uninit();
-        okay &= parse_entity(&tok[i..], &mut entity, &mut i);
-        if !(i < l) {
-            unsafe {
-                // UNSAFE(@ohsayan): Obvious, we have atleast 5, used max 4
-                impossible!();
-            }
-        }
-        okay &= tok[i] == Token![where]; // NOTE: volcano
-        i += 1; // skip even if incorrect
-        let mut clauses = <_ as Default>::default();
-        okay &= WhereClause::parse_where_and_append_to(&tok[i..], d, &mut i, &mut clauses);
-        okay &= !clauses.is_empty();
-        *counter += i;
-        if okay {
-            Ok(Self {
-                entity: unsafe {
-                    // UNSAFE(@ohsayan): obvious due to `okay` and `parse_entity`
-                    entity.assume_init()
-                },
-                wc: WhereClause::new(clauses),
-            })
-        } else {
-            Err(LangError::UnexpectedToken)
-        }
-    }
-}
-
-#[cfg(test)]
-pub(super) fn parse_delete_full<'a>(tok: &'a [Token]) -> LangResult<DeleteStatement<'a>> {
-    let mut i = 0_usize;
-    let r = DeleteStatement::parse_delete(tok, &mut InplaceData::new(), &mut i);
-    assert_full_tt!(i, tok.len());
-    r
-}
diff --git a/server/src/engine/ql/dml/delete.rs b/server/src/engine/ql/dml/delete.rs
new file mode 100644
index 00000000..3990324a
--- /dev/null
+++ b/server/src/engine/ql/dml/delete.rs
@@ -0,0 +1,106 @@
+/*
+ * Created on Fri Jan 06 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[cfg(test)]
+use {
+    super::WhereClauseCollection,
+    crate::engine::ql::{ast::InplaceData, lexer::Token},
+};
+use {
+    super::{parse_entity, WhereClause},
+    crate::{
+        engine::ql::{
+            ast::{Entity, QueryData, State},
+            LangError, LangResult,
+        },
+        util::{compiler, MaybeInit},
+    },
+};
+
+/*
+    Impls for delete
+    ---
+    Smallest statement:
+    delete model:primary_key
+*/
+
+#[derive(Debug, PartialEq)]
+pub struct DeleteStatement<'a> {
+    pub(super) entity: Entity<'a>,
+    pub(super) wc: WhereClause<'a>,
+}
+
+impl<'a> DeleteStatement<'a> {
+    #[inline(always)]
+    pub(super) fn new(entity: Entity<'a>, wc: WhereClause<'a>) -> Self {
+        Self { entity, wc }
+    }
+    #[inline(always)]
+    #[cfg(test)]
+    pub fn new_test(entity: Entity<'a>, wc: WhereClauseCollection<'a>) -> Self {
+        Self::new(entity, WhereClause::new(wc))
+    }
+    #[inline(always)]
+    pub fn parse_delete<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        /*
+            TODO(@ohsayan): Volcano
+            smallest tt:
+            delete from model where x = 1
+                   ^1   ^2    ^3    ^4  ^5
+        */
+        if compiler::unlikely(state.remaining() < 5) {
+            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+        }
+        // from + entity
+        state.poison_if_not(state.cursor_eq(Token![from]));
+        state.cursor_ahead(); // ignore errors (if any)
+        let mut entity = MaybeInit::uninit();
+        parse_entity(state, &mut entity);
+        // where + clauses
+        state.poison_if_not(state.cursor_eq(Token![where]));
+        state.cursor_ahead(); // ignore errors
+        let wc = WhereClause::parse_where(state);
+        if compiler::likely(state.okay()) {
+            Ok(Self {
+                entity: unsafe {
+                    // UNSAFE(@ohsayan): Safety guaranteed by state
+                    entity.assume_init()
+                },
+                wc,
+            })
+        } else {
+            compiler::cold_rerr(LangError::UnexpectedToken)
+        }
+    }
+}
+
+#[cfg(test)]
+pub fn parse_delete_full<'a>(tok: &'a [Token]) -> LangResult<DeleteStatement<'a>> {
+    let mut state = State::new(tok, InplaceData::new());
+    let ret = DeleteStatement::parse_delete(&mut state);
+    assert_full_tt!(state);
+    ret
+}
diff --git a/server/src/engine/ql/dml/insert.rs b/server/src/engine/ql/dml/insert.rs
new file mode 100644
index 00000000..deeff326
--- /dev/null
+++ b/server/src/engine/ql/dml/insert.rs
@@ -0,0 +1,314 @@
+/*
+ * Created on Fri Jan 06 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[cfg(test)]
+use crate::engine::ql::ast::InplaceData;
+use {
+    super::parse_entity,
+    crate::{
+        engine::{
+            memory::DataType,
+            ql::{
+                ast::{Entity, QueryData, State},
+                lexer::Token,
+                LangError, LangResult,
+            },
+        },
+        util::{compiler, MaybeInit},
+    },
+    core::mem::{discriminant, Discriminant},
+    std::collections::HashMap,
+};
+
+/*
+    Impls for insert
+*/
+
+/// ## Panics
+/// - If tt length is less than 1
+pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
+    state: &mut State<'a, Qd>,
+    list: &mut Vec<DataType>,
+) -> Option<Discriminant<DataType>> {
+    let mut stop = state.cursor_eq(Token![close []]);
+    state.cursor_ahead_if(stop);
+    let mut overall_dscr = None;
+    let mut prev_nlist_dscr = None;
+    while state.not_exhausted() && state.okay() && !stop {
+        let d = match state.read() {
+            tok if state.can_read_lit_from(tok) => {
+                let r = unsafe {
+                    // UNSAFE(@ohsayan): the if guard guarantees correctness
+                    DataType::clone_from_litir(state.read_cursor_lit_unchecked())
+                };
+                state.cursor_ahead();
+                r
+            }
+            Token![open []] => {
+                state.cursor_ahead();
+                // a nested list
+                let mut nested_list = Vec::new();
+                let nlist_dscr = parse_list(state, &mut nested_list);
+                // check type return
+                state.poison_if_not(
+                    prev_nlist_dscr.is_none()
+                        || nlist_dscr.is_none()
+                        || prev_nlist_dscr == nlist_dscr,
+                );
+                if prev_nlist_dscr.is_none() && nlist_dscr.is_some() {
+                    prev_nlist_dscr = nlist_dscr;
+                }
+                DataType::List(nested_list)
+            }
+            _ => {
+                state.poison();
+                break;
+            }
+        };
+        state.poison_if_not(list.is_empty() || discriminant(&d) == discriminant(&list[0]));
+        overall_dscr = Some(discriminant(&d));
+        list.push(d);
+        let nx_comma = state.cursor_rounded_eq(Token![,]);
+        let nx_csqrb = state.cursor_rounded_eq(Token![close []]);
+        state.poison_if_not(nx_comma | nx_csqrb);
+        state.cursor_ahead_if(state.okay());
+        stop = nx_csqrb;
+    }
+    overall_dscr
+}
+
+#[cfg(test)]
+pub fn parse_list_full<'a>(tok: &'a [Token], qd: impl QueryData<'a>) -> Option<Vec<DataType>> {
+    let mut l = Vec::new();
+    let mut state = State::new(tok, qd);
+    parse_list(&mut state, &mut l);
+    assert_full_tt!(state);
+    state.okay().then_some(l)
+}
+
+/// ## Panics
+/// - If tt is empty
+pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
+    state: &mut State<'a, Qd>,
+) -> Vec<Option<DataType>> {
+    let mut stop = state.cursor_eq(Token![() close]);
+    state.cursor_ahead_if(stop);
+    let mut data = Vec::new();
+    while state.not_exhausted() && state.okay() && !stop {
+        match state.read() {
+            tok if state.can_read_lit_from(tok) => {
+                unsafe {
+                    // UNSAFE(@ohsayan): if guard guarantees correctness
+                    data.push(Some(DataType::clone_from_litir(
+                        state.read_cursor_lit_unchecked(),
+                    )))
+                }
+                state.cursor_ahead();
+            }
+            Token![open []] if state.remaining() >= 2 => {
+                state.cursor_ahead();
+                let mut l = Vec::new();
+                let _ = parse_list(state, &mut l);
+                data.push(Some(l.into()));
+            }
+            Token![null] => {
+                state.cursor_ahead();
+                data.push(None);
+            }
+            _ => {
+                state.poison();
+                break;
+            }
+        }
+        let nx_comma = state.cursor_rounded_eq(Token![,]);
+        let nx_csprn = state.cursor_rounded_eq(Token![() close]);
+        state.poison_if_not(nx_comma | nx_csprn);
+        state.cursor_ahead_if(state.okay());
+        stop = nx_csprn;
+    }
+    data
+}
+
+#[cfg(test)]
+pub fn parse_data_tuple_syntax_full(tok: &[Token]) -> Option<Vec<Option<DataType>>> {
+    let mut state = State::new(tok, InplaceData::new());
+    let ret = parse_data_tuple_syntax(&mut state);
+    assert_full_tt!(state);
+    state.okay().then_some(ret)
+}
+
+/// ## Panics
+/// Panics if tt is empty
+pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
+    state: &mut State<'a, Qd>,
+) -> HashMap<&'a [u8], Option<DataType>> {
+    let mut stop = state.cursor_eq(Token![close {}]);
+    state.cursor_ahead_if(stop);
+    let mut data = HashMap::with_capacity(2);
+    while state.has_remaining(3) && state.okay() && !stop {
+        let field = state.read();
+        let colon = state.read_ahead(1);
+        let expr = state.read_ahead(2);
+        state.poison_if_not(Token![:].eq(colon));
+        match (field, expr) {
+            (Token::Ident(id), tok) if state.can_read_lit_from(tok) => {
+                state.cursor_ahead_by(2); // ident + colon
+                let ldata = Some(DataType::clone_from_litir(unsafe {
+                    // UNSAFE(@ohsayan): The if guard guarantees correctness
+                    state.read_cursor_lit_unchecked()
+                }));
+                state.cursor_ahead();
+                state.poison_if_not(data.insert(*id, ldata).is_none());
+            }
+            (Token::Ident(id), Token![null]) => {
+                state.cursor_ahead_by(3);
+                state.poison_if_not(data.insert(*id, None).is_none());
+            }
+            (Token::Ident(id), Token![open []]) if state.remaining() >= 4 => {
+                state.cursor_ahead_by(3);
+                let mut l = Vec::new();
+                let _ = parse_list(state, &mut l);
+                state.poison_if_not(data.insert(*id, Some(l.into())).is_none());
+            }
+            _ => {
+                state.poison();
+                break;
+            }
+        }
+        let nx_comma = state.cursor_rounded_eq(Token![,]);
+        let nx_csbrc = state.cursor_rounded_eq(Token![close {}]);
+        state.poison_if_not(nx_comma | nx_csbrc);
+        state.cursor_ahead_if(state.okay());
+        stop = nx_csbrc;
+    }
+    data
+}
+
+#[cfg(test)]
+pub fn parse_data_map_syntax_full(tok: &[Token]) -> Option<HashMap<Box<str>, Option<DataType>>> {
+    let mut state = State::new(tok, InplaceData::new());
+    let r = parse_data_map_syntax(&mut state);
+    assert_full_tt!(state);
+    state.okay().then_some(
+        r.into_iter()
+            .map(|(ident, val)| {
+                (
+                    String::from_utf8_lossy(ident).to_string().into_boxed_str(),
+                    val,
+                )
+            })
+            .collect(),
+    )
+}
+
+#[derive(Debug, PartialEq)]
+pub enum InsertData<'a> {
+    Ordered(Vec<Option<DataType>>),
+    Map(HashMap<&'a [u8], Option<DataType>>),
+}
+
+impl<'a> From<Vec<Option<DataType>>> for InsertData<'a> {
+    fn from(v: Vec<Option<DataType>>) -> Self {
+        Self::Ordered(v)
+    }
+}
+
+impl<'a> From<HashMap<&'static [u8], Option<DataType>>> for InsertData<'a> {
+    fn from(m: HashMap<&'static [u8], Option<DataType>>) -> Self {
+        Self::Map(m)
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub struct InsertStatement<'a> {
+    pub(super) entity: Entity<'a>,
+    pub(super) data: InsertData<'a>,
+}
+
+impl<'a> InsertStatement<'a> {
+    #[inline(always)]
+    pub fn new(entity: Entity<'a>, data: InsertData<'a>) -> Self {
+        Self { entity, data }
+    }
+}
+
+impl<'a> InsertStatement<'a> {
+    pub fn parse_insert<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        /*
+            smallest:
+            insert into model (primarykey)
+                   ^1    ^2   ^3      ^4 ^5
+        */
+        if compiler::unlikely(state.remaining() < 5) {
+            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+        }
+        state.poison_if_not(state.cursor_eq(Token![into]));
+        state.cursor_ahead(); // ignore errors
+
+        // entity
+        let mut entity = MaybeInit::uninit();
+        parse_entity(state, &mut entity);
+        let what_data = state.read();
+        state.cursor_ahead(); // ignore errors for now
+        let mut data = None;
+        match what_data {
+            Token![() open] if state.not_exhausted() => {
+                let this_data = parse_data_tuple_syntax(state);
+                data = Some(InsertData::Ordered(this_data));
+            }
+            Token![open {}] if state.not_exhausted() => {
+                let this_data = parse_data_map_syntax(state);
+                data = Some(InsertData::Map(this_data));
+            }
+            _ => {
+                state.poison();
+            }
+        }
+        if state.okay() {
+            let data = unsafe {
+                // UNSAFE(@ohsayan): state's flag guarantees correctness
+                data.unwrap_unchecked()
+            };
+            Ok(InsertStatement {
+                entity: unsafe {
+                    // UNSAFE(@ohsayan): state's flag ensures correctness
+                    entity.assume_init()
+                },
+                data,
+            })
+        } else {
+            compiler::cold_rerr(LangError::UnexpectedToken)
+        }
+    }
+}
+
+#[cfg(test)]
+pub fn parse_insert_full<'a>(tok: &'a [Token]) -> Option<InsertStatement<'a>> {
+    let mut state = State::new(tok, InplaceData::new());
+    let ret = InsertStatement::parse_insert(&mut state);
+    assert_full_tt!(state);
+    ret.ok()
+}
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
new file mode 100644
index 00000000..89a5082d
--- /dev/null
+++ b/server/src/engine/ql/dml/mod.rs
@@ -0,0 +1,213 @@
+/*
+ * Created on Fri Oct 14 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/*
+    TODO(@ohsayan): For now we've settled for an imprecise error site reporting for simplicity, which we
+    should augment in future revisions of the QL engine
+*/
+
+pub mod delete;
+pub mod insert;
+pub mod select;
+pub mod update;
+
+#[cfg(test)]
+use super::ast::InplaceData;
+use {
+    super::{
+        ast::{Entity, QueryData, State},
+        lexer::{LitIR, Token},
+    },
+    crate::util::{compiler, MaybeInit},
+    std::collections::HashMap,
+};
+
+#[inline(always)]
+unsafe fn read_ident<'a>(tok: &'a Token<'a>) -> &'a [u8] {
+    extract!(tok, Token::Ident(ref tok) => tok)
+}
+
+#[inline(always)]
+fn u(b: bool) -> u8 {
+    b as _
+}
+
+/*
+    Misc
+*/
+
+#[inline(always)]
+fn attempt_process_entity<'a, Qd: QueryData<'a>>(
+    state: &mut State<'a, Qd>,
+    d: &mut MaybeInit<Entity<'a>>,
+) {
+    let tok = state.current();
+    let is_full = Entity::tokens_with_full(tok);
+    let is_single = Entity::tokens_with_single(tok);
+    unsafe {
+        if is_full {
+            state.cursor_ahead_by(3);
+            *d = MaybeInit::new(Entity::full_entity_from_slice(tok));
+        } else if is_single {
+            state.cursor_ahead();
+            *d = MaybeInit::new(Entity::single_entity_from_slice(tok));
+        }
+    }
+    state.poison_if_not(is_full | is_single);
+}
+
+fn parse_entity<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>, d: &mut MaybeInit<Entity<'a>>) {
+    let tok = state.current();
+    let is_full = tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident();
+    let is_single = tok[0].is_ident();
+    unsafe {
+        if is_full {
+            state.cursor_ahead_by(3);
+            *d = MaybeInit::new(Entity::full_entity_from_slice(tok));
+        } else if is_single {
+            state.cursor_ahead();
+            *d = MaybeInit::new(Entity::single_entity_from_slice(tok));
+        }
+    }
+    state.poison_if_not(is_full | is_single);
+}
+
+/*
+    Contexts
+*/
+
+#[derive(Debug, PartialEq)]
+pub struct RelationalExpr<'a> {
+    pub(super) lhs: &'a [u8],
+    pub(super) rhs: LitIR<'a>,
+    pub(super) opc: u8,
+}
+
+impl<'a> RelationalExpr<'a> {
+    #[inline(always)]
+    pub(super) fn new(lhs: &'a [u8], rhs: LitIR<'a>, opc: u8) -> RelationalExpr<'a> {
+        Self { lhs, rhs, opc }
+    }
+    pub(super) const OP_EQ: u8 = 1;
+    pub(super) const OP_NE: u8 = 2;
+    pub(super) const OP_GT: u8 = 3;
+    pub(super) const OP_GE: u8 = 4;
+    pub(super) const OP_LT: u8 = 5;
+    pub(super) const OP_LE: u8 = 6;
+    fn filter_hint_none(&self) -> bool {
+        self.opc == Self::OP_EQ
+    }
+    #[inline(always)]
+    fn parse_operator<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> u8 {
+        let tok = state.current();
+        let op_eq = u(tok[0] == Token![=]) * Self::OP_EQ;
+        let op_ne = u(tok[0] == Token![!] && tok[1] == Token![=]) * Self::OP_NE;
+        let op_ge = u(tok[0] == Token![>] && tok[1] == Token![=]) * Self::OP_GE;
+        let op_gt = u(tok[0] == Token![>] && op_ge == 0) * Self::OP_GT;
+        let op_le = u(tok[0] == Token![<] && tok[1] == Token![=]) * Self::OP_LE;
+        let op_lt = u(tok[0] == Token![<] && op_le == 0) * Self::OP_LT;
+        let opc = op_eq + op_ne + op_ge + op_gt + op_le + op_lt;
+        state.poison_if_not(opc != 0);
+        state.cursor_ahead_by(1 + (opc & 1 == 0) as usize);
+        opc
+    }
+    #[inline(always)]
+    fn try_parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> Option<Self> {
+        if compiler::likely(state.remaining() < 3) {
+            return compiler::cold_val(None);
+        }
+        let ident = state.read();
+        state.poison_if_not(ident.is_ident());
+        state.cursor_ahead(); // ignore any errors
+        let operator = Self::parse_operator(state);
+        state.poison_if_not(state.can_read_lit_rounded());
+        if compiler::likely(state.okay()) {
+            unsafe {
+                let lit = state.read_cursor_lit_unchecked();
+                state.cursor_ahead();
+                Some(Self::new(read_ident(ident), lit, operator))
+            }
+        } else {
+            None
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub struct WhereClause<'a> {
+    c: WhereClauseCollection<'a>,
+}
+
+type WhereClauseCollection<'a> = HashMap<&'a [u8], RelationalExpr<'a>>;
+
+impl<'a> WhereClause<'a> {
+    #[inline(always)]
+    pub(super) fn new(c: WhereClauseCollection<'a>) -> Self {
+        Self { c }
+    }
+    #[inline(always)]
+    fn parse_where_and_append_to<Qd: QueryData<'a>>(
+        state: &mut State<'a, Qd>,
+        c: &mut WhereClauseCollection<'a>,
+    ) {
+        let mut has_more = true;
+        while has_more && state.not_exhausted() && state.okay() {
+            if let Some(expr) = RelationalExpr::try_parse(state) {
+                state.poison_if_not(c.insert(expr.lhs, expr).is_none());
+            }
+            has_more = state.cursor_rounded_eq(Token![and]);
+            state.cursor_ahead_if(has_more);
+        }
+    }
+    #[inline(always)]
+    /// Parse a where context
+    ///
+    /// Notes:
+    /// - Enforce a minimum of 1 clause
+    pub(super) fn parse_where<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> Self {
+        let mut c = HashMap::with_capacity(2);
+        Self::parse_where_and_append_to(state, &mut c);
+        state.poison_if(c.is_empty());
+        Self { c }
+    }
+}
+
+#[cfg(test)]
+pub(super) fn parse_where_clause_full<'a>(tok: &'a [Token]) -> Option<WhereClause<'a>> {
+    let mut state = State::new(tok, InplaceData::new());
+    let ret = WhereClause::parse_where(&mut state);
+    assert_full_tt!(state);
+    state.okay().then_some(ret)
+}
+
+#[cfg(test)]
+#[inline(always)]
+pub(super) fn parse_relexpr_full<'a>(tok: &'a [Token]) -> Option<RelationalExpr<'a>> {
+    let mut state = State::new(tok, InplaceData::new());
+    let ret = RelationalExpr::try_parse(&mut state);
+    assert_full_tt!(state);
+    ret
+}
diff --git a/server/src/engine/ql/dml/select.rs b/server/src/engine/ql/dml/select.rs
new file mode 100644
index 00000000..9f9d2c81
--- /dev/null
+++ b/server/src/engine/ql/dml/select.rs
@@ -0,0 +1,146 @@
+/*
+ * Created on Fri Jan 06 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[cfg(test)]
+use crate::engine::ql::ast::InplaceData;
+use {
+    super::{attempt_process_entity, WhereClause, WhereClauseCollection},
+    crate::{
+        engine::ql::{
+            ast::{Entity, QueryData, State},
+            lexer::Token,
+            LangError, LangResult,
+        },
+        util::{compiler, MaybeInit},
+    },
+};
+
+/*
+    Impls for select
+*/
+
+#[derive(Debug, PartialEq)]
+pub struct SelectStatement<'a> {
+    /// the entity
+    pub(super) entity: Entity<'a>,
+    /// fields in order of querying. will be zero when wildcard is set
+    pub(super) fields: Vec<&'a [u8]>,
+    /// whether a wildcard was passed
+    pub(super) wildcard: bool,
+    /// where clause
+    pub(super) clause: WhereClause<'a>,
+}
+
+impl<'a> SelectStatement<'a> {
+    #[inline(always)]
+    pub(crate) fn new_test(
+        entity: Entity<'a>,
+        fields: Vec<&'a [u8]>,
+        wildcard: bool,
+        clauses: WhereClauseCollection<'a>,
+    ) -> SelectStatement<'a> {
+        Self::new(entity, fields, wildcard, clauses)
+    }
+    #[inline(always)]
+    fn new(
+        entity: Entity<'a>,
+        fields: Vec<&'a [u8]>,
+        wildcard: bool,
+        clauses: WhereClauseCollection<'a>,
+    ) -> SelectStatement<'a> {
+        Self {
+            entity,
+            fields,
+            wildcard,
+            clause: WhereClause::new(clauses),
+        }
+    }
+}
+
+#[cfg(test)]
+/// **test-mode only** parse for a `select` where the full token stream is exhausted
+pub fn parse_select_full<'a>(tok: &'a [Token]) -> Option<SelectStatement<'a>> {
+    let mut state = State::new(tok, InplaceData::new());
+    let r = SelectStatement::parse_select(&mut state);
+    assert_full_tt!(state);
+    r.ok()
+}
+
+impl<'a> SelectStatement<'a> {
+    pub fn parse_select<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        /*
+            Smallest query:
+            select * from model
+                   ^ ^    ^
+                   1 2    3
+        */
+        if compiler::unlikely(state.remaining() < 3) {
+            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+        }
+        let mut select_fields = Vec::new();
+        let is_wildcard = state.cursor_eq(Token![*]);
+        state.cursor_ahead_if(is_wildcard);
+        while state.not_exhausted() && state.okay() && !is_wildcard {
+            match state.read() {
+                Token::Ident(id) => select_fields.push(*id),
+                _ => break,
+            }
+            state.cursor_ahead();
+            let nx_comma = state.cursor_rounded_eq(Token![,]);
+            let nx_from = state.cursor_rounded_eq(Token![from]);
+            state.poison_if_not(nx_comma | nx_from);
+            state.cursor_ahead_if(nx_comma);
+        }
+        state.poison_if_not(is_wildcard | !select_fields.is_empty());
+        // we should have from + model
+        if compiler::unlikely(state.remaining() < 2 || !state.okay()) {
+            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+        }
+        state.poison_if_not(state.cursor_eq(Token![from]));
+        state.cursor_ahead(); // ignore errors
+        let mut entity = MaybeInit::uninit();
+        attempt_process_entity(state, &mut entity);
+        let mut clauses = <_ as Default>::default();
+        if state.cursor_rounded_eq(Token![where]) {
+            state.cursor_ahead();
+            WhereClause::parse_where_and_append_to(state, &mut clauses);
+            state.poison_if(clauses.is_empty());
+        }
+        if compiler::likely(state.okay()) {
+            Ok(SelectStatement {
+                entity: unsafe {
+                    // UNSAFE(@ohsayan): `process_entity` and `okay` assert correctness
+                    entity.assume_init()
+                },
+                fields: select_fields,
+                wildcard: is_wildcard,
+                clause: WhereClause::new(clauses),
+            })
+        } else {
+            compiler::cold_rerr(LangError::UnexpectedToken)
+        }
+    }
+}
diff --git a/server/src/engine/ql/dml/update.rs b/server/src/engine/ql/dml/update.rs
new file mode 100644
index 00000000..5cd0f032
--- /dev/null
+++ b/server/src/engine/ql/dml/update.rs
@@ -0,0 +1,236 @@
+/*
+ * Created on Fri Jan 06 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[cfg(test)]
+use {
+    super::WhereClauseCollection,
+    crate::engine::ql::{ast::InplaceData, lexer::Token},
+};
+use {
+    super::{parse_entity, read_ident, u, WhereClause},
+    crate::{
+        engine::ql::{
+            ast::{Entity, QueryData, State},
+            lexer::LitIR,
+            LangError, LangResult,
+        },
+        util::{compiler, MaybeInit},
+    },
+};
+
+/*
+    Impls for update
+*/
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+/// TODO(@ohsayan): This only helps with the parser test for now. Replace this with actual operator expressions
+pub enum Operator {
+    Assign,
+    AddAssign,
+    SubAssign,
+    MulAssign,
+    DivAssign,
+}
+
+static OPERATOR: [Operator; 6] = [
+    Operator::Assign,
+    Operator::Assign,
+    Operator::AddAssign,
+    Operator::SubAssign,
+    Operator::MulAssign,
+    Operator::DivAssign,
+];
+
+#[derive(Debug, PartialEq)]
+pub struct AssignmentExpression<'a> {
+    /// the LHS ident
+    pub(super) lhs: &'a [u8],
+    /// the RHS lit
+    pub(super) rhs: LitIR<'a>,
+    /// operator
+    pub(super) operator_fn: Operator,
+}
+
+impl<'a> AssignmentExpression<'a> {
+    pub fn new(lhs: &'a [u8], rhs: LitIR<'a>, operator_fn: Operator) -> Self {
+        Self {
+            lhs,
+            rhs,
+            operator_fn,
+        }
+    }
+    fn parse_and_append_expression<Qd: QueryData<'a>>(
+        state: &mut State<'a, Qd>,
+        expressions: &mut Vec<Self>,
+    ) {
+        /*
+            smallest expr:
+            x = y
+        */
+        if compiler::unlikely(state.remaining() < 3) {
+            state.poison();
+            return;
+        }
+        let lhs = state.fw_read();
+        state.poison_if_not(lhs.is_ident());
+        let op_ass = u(state.cursor_eq(Token![=])) * 1;
+        let op_add = u(state.cursor_eq(Token![+])) * 2;
+        let op_sub = u(state.cursor_eq(Token![-])) * 3;
+        let op_mul = u(state.cursor_eq(Token![*])) * 4;
+        let op_div = u(state.cursor_eq(Token![/])) * 5;
+        let operator_code = op_ass + op_add + op_sub + op_mul + op_div;
+        unsafe {
+            // UNSAFE(@ohsayan): A hint, obvious from above
+            if operator_code > 5 {
+                impossible!();
+            }
+        }
+        state.cursor_ahead();
+        state.poison_if(operator_code == 0);
+        let has_double_assign = state.cursor_rounded_eq(Token![=]);
+        let double_assign_okay = operator_code != 1 && has_double_assign;
+        let single_assign_okay = operator_code == 1 && !double_assign_okay;
+        state.poison_if_not(single_assign_okay | double_assign_okay);
+        state.cursor_ahead_if(double_assign_okay);
+        state.poison_if_not(state.can_read_lit_rounded());
+
+        if state.okay() {
+            unsafe {
+                // UNSAFE(@ohsayan): Checked lit, state flag ensures we have ident for lhs
+                let rhs = state.read_cursor_lit_unchecked();
+                state.cursor_ahead();
+                expressions.push(AssignmentExpression::new(
+                    read_ident(lhs),
+                    rhs,
+                    OPERATOR[operator_code as usize],
+                ))
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+pub fn parse_assn_expression_full<'a>(tok: &'a [Token]) -> Option<AssignmentExpression<'a>> {
+    let mut state = State::new(tok, InplaceData::new());
+    let mut exprs = Vec::new();
+    AssignmentExpression::parse_and_append_expression(&mut state, &mut exprs);
+    assert_full_tt!(state);
+    if state.okay() {
+        assert_eq!(exprs.len(), 1, "expected one expression, found multiple");
+        Some(exprs.remove(0))
+    } else {
+        None
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub struct UpdateStatement<'a> {
+    pub(super) entity: Entity<'a>,
+    pub(super) expressions: Vec<AssignmentExpression<'a>>,
+    pub(super) wc: WhereClause<'a>,
+}
+
+impl<'a> UpdateStatement<'a> {
+    #[inline(always)]
+    #[cfg(test)]
+    pub fn new_test(
+        entity: Entity<'a>,
+        expressions: Vec<AssignmentExpression<'a>>,
+        wc: WhereClauseCollection<'a>,
+    ) -> Self {
+        Self::new(entity, expressions, WhereClause::new(wc))
+    }
+    #[inline(always)]
+    pub fn new(
+        entity: Entity<'a>,
+        expressions: Vec<AssignmentExpression<'a>>,
+        wc: WhereClause<'a>,
+    ) -> Self {
+        Self {
+            entity,
+            expressions,
+            wc,
+        }
+    }
+    #[inline(always)]
+    pub fn parse_update<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        /*
+            TODO(@ohsayan): Allow volcanoes
+            smallest tt:
+            update model SET x  =  1 where x = 1
+                   ^1    ^2  ^3 ^4 ^5^6    ^7^8^9
+        */
+        if compiler::unlikely(state.remaining() < 9) {
+            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+        }
+        // parse entity
+        let mut entity = MaybeInit::uninit();
+        parse_entity(state, &mut entity);
+        if !(state.has_remaining(6)) {
+            unsafe {
+                // UNSAFE(@ohsayan): Obvious from above, max 3 fw
+                impossible!();
+            }
+        }
+        state.poison_if_not(state.cursor_eq(Token![set]));
+        state.cursor_ahead(); // ignore errors if any
+        let mut nx_where = false;
+        let mut expressions = Vec::new();
+        while state.not_exhausted() && state.okay() && !nx_where {
+            AssignmentExpression::parse_and_append_expression(state, &mut expressions);
+            let nx_comma = state.cursor_rounded_eq(Token![,]);
+            nx_where = state.cursor_rounded_eq(Token![where]); // NOTE: volcano
+            state.poison_if_not(nx_comma | nx_where);
+            state.cursor_ahead_if(nx_comma);
+        }
+        state.poison_if_not(nx_where);
+        state.cursor_ahead_if(state.okay());
+        // check where clauses
+        let mut clauses = <_ as Default>::default();
+        WhereClause::parse_where_and_append_to(state, &mut clauses);
+        state.poison_if(clauses.is_empty()); // NOTE: volcano
+        if compiler::likely(state.okay()) {
+            Ok(Self {
+                entity: unsafe {
+                    // UNSAFE(@ohsayan): This is safe because of `parse_entity` and `okay`
+                    entity.assume_init()
+                },
+                expressions,
+                wc: WhereClause::new(clauses),
+            })
+        } else {
+            compiler::cold_rerr(LangError::UnexpectedToken)
+        }
+    }
+}
+
+#[cfg(test)]
+pub fn parse_update_full<'a>(tok: &'a [Token]) -> LangResult<UpdateStatement<'a>> {
+    let mut state = State::new(tok, InplaceData::new());
+    let r = UpdateStatement::parse_update(&mut state);
+    assert_full_tt!(state);
+    r
+}
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index f36431f5..afdcff3b 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -29,6 +29,14 @@ macro_rules! assert_full_tt {
     ($a:expr, $b:expr) => {
         assert_eq!($a, $b, "full token stream not utilized")
     };
+    ($a:expr) => {
+        assert!(
+            crate::engine::ql::ast::State::exhausted(&$a),
+            "full tt not utilized at: {}:{}",
+            ::core::file!(),
+            ::core::line!()
+        )
+    };
 }
 
 macro_rules! __sym_token {
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index d2909292..08d27fa5 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -29,7 +29,7 @@ mod list_parse {
     use super::*;
     use crate::engine::ql::{
         ast::{InplaceData, SubstitutedData},
-        dml::parse_list_full,
+        dml::insert::parse_list_full,
         lexer::LitIR,
     };
 
@@ -41,7 +41,7 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_list_full(&tok[1..], &mut InplaceData::new()).unwrap();
+        let r = parse_list_full(&tok[1..], InplaceData::new()).unwrap();
         assert_eq!(r, vec![])
     }
     #[test]
@@ -52,7 +52,7 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_list_full(&tok[1..], &mut InplaceData::new()).unwrap();
+        let r = parse_list_full(&tok[1..], InplaceData::new()).unwrap();
         assert_eq!(r.as_slice(), into_array![1, 2, 3, 4])
     }
     #[test]
@@ -69,8 +69,7 @@ mod list_parse {
             LitIR::UInt(3),
             LitIR::UInt(4),
         ];
-        let mut param = SubstitutedData::new(&data);
-        let r = parse_list_full(&tok[1..], &mut param).unwrap();
+        let r = parse_list_full(&tok[1..], SubstitutedData::new(&data)).unwrap();
         assert_eq!(r.as_slice(), into_array![1, 2, 3, 4])
     }
     #[test]
@@ -86,7 +85,7 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_list_full(&tok[1..], &mut InplaceData::new()).unwrap();
+        let r = parse_list_full(&tok[1..], InplaceData::new()).unwrap();
         assert_eq!(
             r.as_slice(),
             into_array![
@@ -118,8 +117,7 @@ mod list_parse {
             LitIR::UInt(5),
             LitIR::UInt(6),
         ];
-        let mut param = SubstitutedData::new(&data);
-        let r = parse_list_full(&tok[1..], &mut param).unwrap();
+        let r = parse_list_full(&tok[1..], SubstitutedData::new(&data)).unwrap();
         assert_eq!(
             r.as_slice(),
             into_array![
@@ -143,7 +141,7 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_list_full(&tok[1..], &mut InplaceData::new()).unwrap();
+        let r = parse_list_full(&tok[1..], InplaceData::new()).unwrap();
         assert_eq!(
             r.as_slice(),
             into_array![
@@ -181,8 +179,7 @@ mod list_parse {
             LitIR::UInt(7),
             LitIR::UInt(7),
         ];
-        let mut param = SubstitutedData::new(&data);
-        let r = parse_list_full(&tok[1..], &mut param).unwrap();
+        let r = parse_list_full(&tok[1..], SubstitutedData::new(&data)).unwrap();
         assert_eq!(
             r.as_slice(),
             into_array![
@@ -197,7 +194,7 @@ mod list_parse {
 
 mod tuple_syntax {
     use super::*;
-    use crate::engine::ql::dml::parse_data_tuple_syntax_full;
+    use crate::engine::ql::dml::insert::parse_data_tuple_syntax_full;
 
     #[test]
     fn tuple_mini() {
@@ -287,7 +284,7 @@ mod tuple_syntax {
 }
 mod map_syntax {
     use super::*;
-    use crate::engine::ql::dml::parse_data_map_syntax_full;
+    use crate::engine::ql::dml::insert::parse_data_map_syntax_full;
 
     #[test]
     fn map_mini() {
@@ -390,7 +387,7 @@ mod stmt_insert {
         super::*,
         crate::engine::ql::{
             ast::Entity,
-            dml::{self, InsertStatement},
+            dml::{self, insert::InsertStatement},
         },
     };
 
@@ -402,11 +399,11 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::parse_insert_full(&x[1..]).unwrap();
-        let e = InsertStatement {
-            entity: Entity::Full(b"twitter", b"users"),
-            data: into_array_nullable!["sayan"].to_vec().into(),
-        };
+        let r = dml::insert::parse_insert_full(&x[1..]).unwrap();
+        let e = InsertStatement::new(
+            Entity::Full(b"twitter", b"users"),
+            into_array_nullable!["sayan"].to_vec().into(),
+        );
         assert_eq!(e, r);
     }
     #[test]
@@ -424,13 +421,13 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::parse_insert_full(&x[1..]).unwrap();
-        let e = InsertStatement {
-            entity: Entity::Full(b"twitter", b"users"),
-            data: into_array_nullable!["sayan", "Sayan", "sayan@example.com", true, 12345, 67890]
+        let r = dml::insert::parse_insert_full(&x[1..]).unwrap();
+        let e = InsertStatement::new(
+            Entity::Full(b"twitter", b"users"),
+            into_array_nullable!["sayan", "Sayan", "sayan@example.com", true, 12345, 67890]
                 .to_vec()
                 .into(),
-        };
+        );
         assert_eq!(e, r);
     }
     #[test]
@@ -451,10 +448,10 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::parse_insert_full(&x[1..]).unwrap();
-        let e = InsertStatement {
-            entity: Entity::Full(b"twitter", b"users"),
-            data: into_array_nullable![
+        let r = dml::insert::parse_insert_full(&x[1..]).unwrap();
+        let e = InsertStatement::new(
+            Entity::Full(b"twitter", b"users"),
+            into_array_nullable![
                 "sayan",
                 "Sayan",
                 "sayan@example.com",
@@ -467,7 +464,7 @@ mod stmt_insert {
             ]
             .to_vec()
             .into(),
-        };
+        );
         assert_eq!(e, r);
     }
     #[test]
@@ -478,14 +475,14 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::parse_insert_full(&tok[1..]).unwrap();
-        let e = InsertStatement {
-            entity: Entity::Full(b"jotsy", b"app"),
-            data: dict_nullable! {
+        let r = dml::insert::parse_insert_full(&tok[1..]).unwrap();
+        let e = InsertStatement::new(
+            Entity::Full(b"jotsy", b"app"),
+            dict_nullable! {
                 "username".as_bytes() => "sayan"
             }
             .into(),
-        };
+        );
         assert_eq!(e, r);
     }
     #[test]
@@ -503,10 +500,10 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::parse_insert_full(&tok[1..]).unwrap();
-        let e = InsertStatement {
-            entity: Entity::Full(b"jotsy", b"app"),
-            data: dict_nullable! {
+        let r = dml::insert::parse_insert_full(&tok[1..]).unwrap();
+        let e = InsertStatement::new(
+            Entity::Full(b"jotsy", b"app"),
+            dict_nullable! {
                 "username".as_bytes() => "sayan",
                 "name".as_bytes() => "Sayan",
                 "email".as_bytes() => "sayan@example.com",
@@ -515,7 +512,7 @@ mod stmt_insert {
                 "followers".as_bytes() => 67890
             }
             .into(),
-        };
+        );
         assert_eq!(e, r);
     }
     #[test]
@@ -536,10 +533,10 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::parse_insert_full(&tok[1..]).unwrap();
-        let e = InsertStatement {
-            entity: Entity::Full(b"jotsy", b"app"),
-            data: dict_nullable! {
+        let r = dml::insert::parse_insert_full(&tok[1..]).unwrap();
+        let e = InsertStatement::new(
+            Entity::Full(b"jotsy", b"app"),
+            dict_nullable! {
                 "username".as_bytes() => "sayan",
                 "password".as_bytes() => "pass123",
                 "email".as_bytes() => "sayan@example.com",
@@ -551,7 +548,7 @@ mod stmt_insert {
                 "other_linked_accounts".as_bytes() => Null
             }
             .into(),
-        };
+        );
         assert_eq!(r, e);
     }
 }
@@ -563,7 +560,7 @@ mod stmt_select {
         super::*,
         crate::engine::ql::{
             ast::Entity,
-            dml::{self, SelectStatement},
+            dml::{self, select::SelectStatement},
             lexer::LitIR,
         },
     };
@@ -575,7 +572,7 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = dml::parse_select_full(&tok[1..]).unwrap();
+        let r = dml::select::parse_select_full(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
             Entity::Single(b"users"),
             [].to_vec(),
@@ -596,7 +593,7 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = dml::parse_select_full(&tok[1..]).unwrap();
+        let r = dml::select::parse_select_full(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
             Entity::Single(b"users"),
             [b"field1".as_slice()].to_vec(),
@@ -617,7 +614,7 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = dml::parse_select_full(&tok[1..]).unwrap();
+        let r = dml::select::parse_select_full(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
             Entity::Full(b"twitter", b"users"),
             [b"field1".as_slice()].to_vec(),
@@ -638,7 +635,7 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = dml::parse_select_full(&tok[1..]).unwrap();
+        let r = dml::select::parse_select_full(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
             Entity::Full(b"twitter", b"users"),
             [b"field1".as_slice(), b"field2".as_slice()].to_vec(),
@@ -656,73 +653,56 @@ mod expression_tests {
     use {
         super::*,
         crate::engine::ql::{
-            dml::{self, AssignmentExpression, Operator},
+            dml::{
+                self,
+                update::{AssignmentExpression, Operator},
+            },
             lexer::LitIR,
         },
     };
     #[test]
     fn expr_assign() {
         let src = lex_insecure(b"username = 'sayan'").unwrap();
-        let r = dml::parse_expression_full(&src).unwrap();
+        let r = dml::update::parse_assn_expression_full(&src).unwrap();
         assert_eq!(
             r,
-            AssignmentExpression {
-                lhs: b"username",
-                rhs: LitIR::Str("sayan"),
-                operator_fn: Operator::Assign
-            }
+            AssignmentExpression::new(b"username", LitIR::Str("sayan"), Operator::Assign)
         );
     }
     #[test]
     fn expr_add_assign() {
         let src = lex_insecure(b"followers += 100").unwrap();
-        let r = dml::parse_expression_full(&src).unwrap();
+        let r = dml::update::parse_assn_expression_full(&src).unwrap();
         assert_eq!(
             r,
-            AssignmentExpression {
-                lhs: b"followers",
-                rhs: LitIR::UInt(100),
-                operator_fn: Operator::AddAssign
-            }
+            AssignmentExpression::new(b"followers", LitIR::UInt(100), Operator::AddAssign)
         );
     }
     #[test]
     fn expr_sub_assign() {
         let src = lex_insecure(b"following -= 150").unwrap();
-        let r = dml::parse_expression_full(&src).unwrap();
+        let r = dml::update::parse_assn_expression_full(&src).unwrap();
         assert_eq!(
             r,
-            AssignmentExpression {
-                lhs: b"following",
-                rhs: LitIR::UInt(150),
-                operator_fn: Operator::SubAssign
-            }
+            AssignmentExpression::new(b"following", LitIR::UInt(150), Operator::SubAssign)
         );
     }
     #[test]
     fn expr_mul_assign() {
         let src = lex_insecure(b"product_qty *= 2").unwrap();
-        let r = dml::parse_expression_full(&src).unwrap();
+        let r = dml::update::parse_assn_expression_full(&src).unwrap();
         assert_eq!(
             r,
-            AssignmentExpression {
-                lhs: b"product_qty",
-                rhs: LitIR::UInt(2),
-                operator_fn: Operator::MulAssign
-            }
+            AssignmentExpression::new(b"product_qty", LitIR::UInt(2), Operator::MulAssign)
         );
     }
     #[test]
     fn expr_div_assign() {
         let src = lex_insecure(b"image_crop_factor /= 2").unwrap();
-        let r = dml::parse_expression_full(&src).unwrap();
+        let r = dml::update::parse_assn_expression_full(&src).unwrap();
         assert_eq!(
             r,
-            AssignmentExpression {
-                lhs: b"image_crop_factor",
-                rhs: LitIR::UInt(2),
-                operator_fn: Operator::DivAssign
-            }
+            AssignmentExpression::new(b"image_crop_factor", LitIR::UInt(2), Operator::DivAssign)
         );
     }
 }
@@ -732,7 +712,9 @@ mod update_statement {
         crate::engine::ql::{
             ast::Entity,
             dml::{
-                self, AssignmentExpression, Operator, RelationalExpr, UpdateStatement, WhereClause,
+                self,
+                update::{AssignmentExpression, Operator, UpdateStatement},
+                RelationalExpr, WhereClause,
             },
             lexer::LitIR,
         },
@@ -745,22 +727,22 @@ mod update_statement {
             "#,
         )
         .unwrap();
-        let r = dml::parse_update_full(&tok[1..]).unwrap();
-        let e = UpdateStatement {
-            entity: Entity::Single(b"app"),
-            expressions: vec![AssignmentExpression {
-                lhs: b"notes",
-                rhs: LitIR::Str("this is my new note"),
-                operator_fn: Operator::AddAssign,
-            }],
-            wc: WhereClause::new(dict! {
+        let r = dml::update::parse_update_full(&tok[1..]).unwrap();
+        let e = UpdateStatement::new(
+            Entity::Single(b"app"),
+            vec![AssignmentExpression::new(
+                b"notes",
+                LitIR::Str("this is my new note"),
+                Operator::AddAssign,
+            )],
+            WhereClause::new(dict! {
                 "username".as_bytes() => RelationalExpr::new(
                     "username".as_bytes(),
                     LitIR::Str("sayan"),
                     RelationalExpr::OP_EQ
                 )
             }),
-        };
+        );
         assert_eq!(r, e);
     }
     #[test]
@@ -777,10 +759,10 @@ mod update_statement {
             "#,
         )
         .unwrap();
-        let r = dml::parse_update_full(&tok[1..]).unwrap();
-        let e = UpdateStatement {
-            entity: Entity::Full(b"jotsy", b"app"),
-            expressions: vec![
+        let r = dml::update::parse_update_full(&tok[1..]).unwrap();
+        let e = UpdateStatement::new(
+            Entity::Full(b"jotsy", b"app"),
+            vec![
                 AssignmentExpression::new(
                     b"notes",
                     LitIR::Str("this is my new note"),
@@ -792,15 +774,14 @@ mod update_statement {
                     Operator::Assign,
                 ),
             ],
-            wc: WhereClause::new(dict! {
+            WhereClause::new(dict! {
                 "username".as_bytes() => RelationalExpr::new(
                     "username".as_bytes(),
                     LitIR::Str("sayan"),
                     RelationalExpr::OP_EQ
                 )
             }),
-        };
-
+        );
         assert_eq!(r, e);
     }
 }
@@ -809,7 +790,10 @@ mod delete_stmt {
         super::*,
         crate::engine::ql::{
             ast::Entity,
-            dml::{self, DeleteStatement, RelationalExpr},
+            dml::{
+                delete::{self, DeleteStatement},
+                RelationalExpr,
+            },
             lexer::LitIR,
         },
     };
@@ -832,7 +816,7 @@ mod delete_stmt {
                 )
             },
         );
-        let r = dml::parse_delete_full(&tok[1..]).unwrap();
+        let r = delete::parse_delete_full(&tok[1..]).unwrap();
         assert_eq!(r, e);
     }
     #[test]
@@ -853,7 +837,7 @@ mod delete_stmt {
                 )
             },
         );
-        let r = dml::parse_delete_full(&tok[1..]).unwrap();
+        let r = delete::parse_delete_full(&tok[1..]).unwrap();
         assert_eq!(r, e);
     }
 }

From 4d140c74fd9f8fd626ecb019f434b09350db6966 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 9 Jan 2023 21:12:17 -0800
Subject: [PATCH 068/310] Support built-in fn substitution

---
 Cargo.lock                              |  23 +++
 server/Cargo.toml                       |   1 +
 server/src/engine/ql/ast.rs             |  38 ++++-
 server/src/engine/ql/dml/insert.rs      | 186 ++++++++++++++++++++----
 server/src/engine/ql/tests/dml_tests.rs |  30 ++++
 5 files changed, 245 insertions(+), 33 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ba939077..cb09d4bf 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1332,6 +1332,7 @@ dependencies = [
  "tokio",
  "tokio-openssl",
  "toml",
+ "uuid",
  "winapi",
 ]
 
@@ -1603,6 +1604,28 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
 
+[[package]]
+name = "uuid"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c"
+dependencies = [
+ "getrandom",
+ "rand",
+ "uuid-macro-internal",
+]
+
+[[package]]
+name = "uuid-macro-internal"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73bc89f2894593e665241e0052c3791999e6787b7c4831daa0a5c2e637e276d8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "vcpkg"
 version = "0.2.15"
diff --git a/server/Cargo.toml b/server/Cargo.toml
index b40f218b..cf930e69 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -26,6 +26,7 @@ tokio = { version = "1.21.0", features = ["full"] }
 tokio-openssl = "0.6.3"
 toml = "0.5.9"
 base64 = "0.13.0"
+uuid = { version = "1.2.2", features = ["v4", "fast-rng", "macro-diagnostics"] }
 
 [target.'cfg(all(not(target_env = "msvc"), not(miri)))'.dependencies]
 # external deps
diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 4d501635..4a2c3526 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -118,7 +118,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     }
     #[inline(always)]
     /// Check if the token stream has alteast `many` count of tokens
-    pub fn has_remaining(&mut self, many: usize) -> bool {
+    pub fn has_remaining(&self, many: usize) -> bool {
         self.remaining() >= many
     }
     #[inline(always)]
@@ -151,6 +151,14 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
         Qd::read_lit(&mut self.d, tok)
     }
     #[inline(always)]
+    /// Read a lit from the given token
+    ///
+    /// ## Safety
+    /// - Must ensure that `Self::can_read_lit_from` is true for the token
+    pub unsafe fn read_lit_unchecked_from(&mut self, tok: &'a Token<'a>) -> LitIR<'a> {
+        Qd::read_lit(&mut self.d, tok)
+    }
+    #[inline(always)]
     /// Check if the cursor equals the given token; rounded
     pub fn cursor_rounded_eq(&self, tok: Token<'a>) -> bool {
         let mx = minidx(self.t, self.i);
@@ -166,6 +174,34 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     pub(crate) fn read_ahead(&self, ahead: usize) -> &'a Token<'a> {
         &self.t[self.i + ahead]
     }
+    #[inline(always)]
+    /// Move the cursor back by 1
+    pub(crate) fn cursor_back(&mut self) {
+        self.cursor_back_by(1);
+    }
+    #[inline(always)]
+    /// Move the cursor back by the given count
+    pub(crate) fn cursor_back_by(&mut self, by: usize) {
+        self.i -= by;
+    }
+    #[inline(always)]
+    pub(crate) fn cursor_has_ident_rounded(&self) -> bool {
+        self.t[minidx(self.t, self.i)].is_ident() && self.not_exhausted()
+    }
+    #[inline(always)]
+    /// Check if the current token stream matches the signature of an arity(0) fn; rounded
+    ///
+    /// NOTE: Consider using a direct comparison without rounding
+    pub(crate) fn cursor_signature_match_fn_arity0_rounded(&self) -> bool {
+        let rem = self.has_remaining(3);
+        let idx_a = self.i * rem as usize;
+        let idx_b = (self.i + 1) * rem as usize;
+        let idx_c = (self.i + 2) * rem as usize;
+        (self.t[idx_a].is_ident())
+            & (self.t[idx_b] == Token![() open])
+            & (self.t[idx_c] == Token![() close])
+            & rem
+    }
 }
 
 pub trait QueryData<'a> {
diff --git a/server/src/engine/ql/dml/insert.rs b/server/src/engine/ql/dml/insert.rs
index deeff326..a93a52e4 100644
--- a/server/src/engine/ql/dml/insert.rs
+++ b/server/src/engine/ql/dml/insert.rs
@@ -27,7 +27,7 @@
 #[cfg(test)]
 use crate::engine::ql::ast::InplaceData;
 use {
-    super::parse_entity,
+    super::{parse_entity, read_ident},
     crate::{
         engine::{
             memory::DataType,
@@ -39,14 +39,105 @@ use {
         },
         util::{compiler, MaybeInit},
     },
-    core::mem::{discriminant, Discriminant},
-    std::collections::HashMap,
+    core::{
+        cmp,
+        mem::{discriminant, Discriminant},
+    },
+    std::{
+        collections::HashMap,
+        time::{Duration, SystemTime, UNIX_EPOCH},
+    },
+    uuid::Uuid,
 };
 
 /*
     Impls for insert
 */
 
+pub const T_UUIDSTR: &str = "4593264b-0231-43e9-b0aa-50784f14e204";
+pub const T_UUIDBIN: &[u8] = T_UUIDSTR.as_bytes();
+pub const T_TIMESEC: u64 = 1673187839_u64;
+
+type ProducerFn = fn() -> DataType;
+
+// base
+#[inline(always)]
+fn pfnbase_time() -> Duration {
+    if cfg!(debug_assertions) {
+        Duration::from_secs(T_TIMESEC)
+    } else {
+        SystemTime::now().duration_since(UNIX_EPOCH).unwrap()
+    }
+}
+#[inline(always)]
+fn pfnbase_uuid() -> Uuid {
+    if cfg!(debug_assertions) {
+        Uuid::parse_str(T_UUIDSTR).unwrap()
+    } else {
+        Uuid::new_v4()
+    }
+}
+// impl
+#[inline(always)]
+fn pfn_timesec() -> DataType {
+    DataType::UnsignedInt(pfnbase_time().as_secs())
+}
+#[inline(always)]
+fn pfn_uuidstr() -> DataType {
+    DataType::String(pfnbase_uuid().to_string().into_boxed_str())
+}
+#[inline(always)]
+fn pfn_uuidbin() -> DataType {
+    DataType::Binary(pfnbase_uuid().as_bytes().to_vec().into_boxed_slice())
+}
+
+static PRODUCER_G: [u8; 4] = [0, 2, 3, 0];
+static PRODUCER_F: [(&[u8], ProducerFn); 3] = [
+    (b"uuidstr", pfn_uuidstr),
+    (b"uuidbin", pfn_uuidbin),
+    (b"timesec", pfn_timesec),
+];
+const MAGIC_1: [u8; 7] = *b"cp21rLd";
+const MAGIC_2: [u8; 7] = *b"zS8zgaK";
+const MAGIC_L: usize = MAGIC_1.len();
+
+#[inline(always)]
+fn hashf(key: &[u8], m: &[u8]) -> u32 {
+    let mut i = 0;
+    let mut s = 0;
+    while i < key.len() {
+        s += m[(i % MAGIC_L) as usize] as u32 * key[i] as u32;
+        i += 1;
+    }
+    s % PRODUCER_G.len() as u32
+}
+#[inline(always)]
+fn hashp(key: &[u8]) -> u32 {
+    (PRODUCER_G[hashf(key, &MAGIC_1) as usize] + PRODUCER_G[hashf(key, &MAGIC_2) as usize]) as u32
+        % PRODUCER_G.len() as u32
+}
+#[inline(always)]
+fn ldfunc(func: &[u8]) -> Option<ProducerFn> {
+    let ph = hashp(func) as usize;
+    let min = cmp::min(ph, PRODUCER_F.len() - 1);
+    let data = PRODUCER_F[min as usize];
+    if data.0 == func {
+        Some(data.1)
+    } else {
+        None
+    }
+}
+#[inline(always)]
+fn ldfunc_exists(func: &[u8]) -> bool {
+    ldfunc(func).is_some()
+}
+#[inline(always)]
+unsafe fn ldfunc_unchecked(func: &[u8]) -> ProducerFn {
+    let ph = hashp(func) as usize;
+    debug_assert_eq!(PRODUCER_F[ph as usize].0, func);
+    PRODUCER_F[ph as usize].1
+}
+
 /// ## Panics
 /// - If tt length is less than 1
 pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
@@ -58,17 +149,15 @@ pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
     let mut overall_dscr = None;
     let mut prev_nlist_dscr = None;
     while state.not_exhausted() && state.okay() && !stop {
-        let d = match state.read() {
+        let d = match state.fw_read() {
             tok if state.can_read_lit_from(tok) => {
                 let r = unsafe {
                     // UNSAFE(@ohsayan): the if guard guarantees correctness
-                    DataType::clone_from_litir(state.read_cursor_lit_unchecked())
+                    DataType::clone_from_litir(state.read_lit_unchecked_from(tok))
                 };
-                state.cursor_ahead();
                 r
             }
             Token![open []] => {
-                state.cursor_ahead();
                 // a nested list
                 let mut nested_list = Vec::new();
                 let nlist_dscr = parse_list(state, &mut nested_list);
@@ -83,7 +172,18 @@ pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
                 }
                 DataType::List(nested_list)
             }
+            Token![@] if state.cursor_signature_match_fn_arity0_rounded() => match unsafe {
+                // UNSAFE(@ohsayan): Just verified at guard
+                handle_func_sub(state)
+            } {
+                Some(value) => value,
+                None => {
+                    state.poison();
+                    break;
+                }
+            },
             _ => {
+                state.cursor_back();
                 state.poison();
                 break;
             }
@@ -100,6 +200,15 @@ pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
     overall_dscr
 }
 
+#[inline(always)]
+/// ## Safety
+/// - Cursor must match arity(0) function signature
+unsafe fn handle_func_sub<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> Option<DataType> {
+    let func = read_ident(state.fw_read());
+    state.cursor_ahead_by(2); // skip tt:paren
+    ldfunc(func).map(move |f| f())
+}
+
 #[cfg(test)]
 pub fn parse_list_full<'a>(tok: &'a [Token], qd: impl QueryData<'a>) -> Option<Vec<DataType>> {
     let mut l = Vec::new();
@@ -118,27 +227,33 @@ pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
     state.cursor_ahead_if(stop);
     let mut data = Vec::new();
     while state.not_exhausted() && state.okay() && !stop {
-        match state.read() {
-            tok if state.can_read_lit_from(tok) => {
-                unsafe {
-                    // UNSAFE(@ohsayan): if guard guarantees correctness
-                    data.push(Some(DataType::clone_from_litir(
-                        state.read_cursor_lit_unchecked(),
-                    )))
-                }
-                state.cursor_ahead();
-            }
-            Token![open []] if state.remaining() >= 2 => {
-                state.cursor_ahead();
+        match state.fw_read() {
+            tok if state.can_read_lit_from(tok) => unsafe {
+                // UNSAFE(@ohsayan): if guard guarantees correctness
+                data.push(Some(DataType::clone_from_litir(
+                    state.read_lit_unchecked_from(tok),
+                )))
+            },
+            Token![open []] if state.not_exhausted() => {
                 let mut l = Vec::new();
                 let _ = parse_list(state, &mut l);
                 data.push(Some(l.into()));
             }
             Token![null] => {
-                state.cursor_ahead();
                 data.push(None);
             }
+            Token![@] if state.cursor_signature_match_fn_arity0_rounded() => match unsafe {
+                // UNSAFE(@ohsayan): Just verified at guard
+                handle_func_sub(state)
+            } {
+                Some(value) => data.push(Some(value)),
+                None => {
+                    state.poison();
+                    break;
+                }
+            },
             _ => {
+                state.cursor_back();
                 state.poison();
                 break;
             }
@@ -169,31 +284,40 @@ pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
     state.cursor_ahead_if(stop);
     let mut data = HashMap::with_capacity(2);
     while state.has_remaining(3) && state.okay() && !stop {
-        let field = state.read();
-        let colon = state.read_ahead(1);
-        let expr = state.read_ahead(2);
+        let field = state.fw_read();
+        let colon = state.fw_read();
+        let expr = state.fw_read();
         state.poison_if_not(Token![:].eq(colon));
         match (field, expr) {
             (Token::Ident(id), tok) if state.can_read_lit_from(tok) => {
-                state.cursor_ahead_by(2); // ident + colon
                 let ldata = Some(DataType::clone_from_litir(unsafe {
                     // UNSAFE(@ohsayan): The if guard guarantees correctness
-                    state.read_cursor_lit_unchecked()
+                    state.read_lit_unchecked_from(tok)
                 }));
-                state.cursor_ahead();
                 state.poison_if_not(data.insert(*id, ldata).is_none());
             }
             (Token::Ident(id), Token![null]) => {
-                state.cursor_ahead_by(3);
                 state.poison_if_not(data.insert(*id, None).is_none());
             }
-            (Token::Ident(id), Token![open []]) if state.remaining() >= 4 => {
-                state.cursor_ahead_by(3);
+            (Token::Ident(id), Token![open []]) if state.not_exhausted() => {
                 let mut l = Vec::new();
                 let _ = parse_list(state, &mut l);
                 state.poison_if_not(data.insert(*id, Some(l.into())).is_none());
             }
+            (Token::Ident(id), Token![@]) if state.cursor_signature_match_fn_arity0_rounded() => {
+                match unsafe {
+                    // UNSAFE(@ohsayan): Just verified at guard
+                    handle_func_sub(state)
+                } {
+                    Some(value) => state.poison_if_not(data.insert(*id, Some(value)).is_none()),
+                    None => {
+                        state.poison();
+                        break;
+                    }
+                }
+            }
             _ => {
+                state.cursor_back_by(3);
                 state.poison();
                 break;
             }
@@ -271,10 +395,8 @@ impl<'a> InsertStatement<'a> {
         // entity
         let mut entity = MaybeInit::uninit();
         parse_entity(state, &mut entity);
-        let what_data = state.read();
-        state.cursor_ahead(); // ignore errors for now
         let mut data = None;
-        match what_data {
+        match state.fw_read() {
             Token![() open] if state.not_exhausted() => {
                 let this_data = parse_data_tuple_syntax(state);
                 data = Some(InsertData::Ordered(this_data));
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 08d27fa5..9f12332a 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -551,6 +551,36 @@ mod stmt_insert {
         );
         assert_eq!(r, e);
     }
+    #[test]
+    fn insert_tuple_fnsub() {
+        let tok =
+            lex_insecure(br#"insert into jotsy.app(@uuidstr(), "sayan", @timesec())"#).unwrap();
+        let ret = dml::insert::parse_insert_full(&tok[1..]).unwrap();
+        let expected = InsertStatement::new(
+            Entity::Full(b"jotsy", b"app"),
+            into_array_nullable![dml::insert::T_UUIDSTR, "sayan", dml::insert::T_TIMESEC]
+                .to_vec()
+                .into(),
+        );
+        assert_eq!(ret, expected);
+    }
+    #[test]
+    fn insert_map_fnsub() {
+        let tok = lex_insecure(
+            br#"insert into jotsy.app { uuid: @uuidstr(), username: "sayan", signup_time: @timesec() }"#
+        ).unwrap();
+        let ret = dml::insert::parse_insert_full(&tok[1..]).unwrap();
+        let expected = InsertStatement::new(
+            Entity::Full(b"jotsy", b"app"),
+            dict_nullable! {
+                "uuid".as_bytes() => dml::insert::T_UUIDSTR,
+                "username".as_bytes() => "sayan",
+                "signup_time".as_bytes() => dml::insert::T_TIMESEC,
+            }
+            .into(),
+        );
+        assert_eq!(ret, expected);
+    }
 }
 
 mod stmt_select {

From 2756b1d070fa09119bfb31f4fce37c3f1f5183c1 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 9 Jan 2023 23:22:24 -0800
Subject: [PATCH 069/310] Simplify lit to owned data clone

---
 server/src/engine/ql/ast.rs        | 27 ++++++++++++++++++++++++++-
 server/src/engine/ql/dml/insert.rs | 12 +++++-------
 2 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 4a2c3526..d628d386 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -30,7 +30,7 @@ use {
         lexer::{LitIR, Slice, Token},
         schema, LangError, LangResult,
     },
-    crate::util::compiler,
+    crate::{engine::memory::DataType, util::compiler},
     core::cmp,
 };
 
@@ -202,6 +202,15 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
             & (self.t[idx_c] == Token![() close])
             & rem
     }
+    /// Reads a lit using the given token and the internal data source and return a data type
+    ///
+    /// ## Safety
+    ///
+    /// Caller should have checked that the token matches a lit signature and that enough data is available
+    /// in the data source. (ideally should run `can_read_lit_from` or `can_read_lit_rounded`)
+    pub unsafe fn read_lit_into_data_type_unchecked_from(&mut self, tok: &'a Token) -> DataType {
+        self.d.read_data_type(tok)
+    }
 }
 
 pub trait QueryData<'a> {
@@ -212,6 +221,11 @@ pub trait QueryData<'a> {
     /// ## Safety
     /// The current token **must match** the signature of a lit
     unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a>;
+    /// Read a lit using the given token and then copy it into a [`DataType`]
+    ///
+    /// ## Safety
+    /// The current token must match the signature of a lit
+    unsafe fn read_data_type(&mut self, tok: &'a Token) -> DataType;
 }
 
 #[derive(Debug)]
@@ -232,6 +246,10 @@ impl<'a> QueryData<'a> for InplaceData {
     unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a> {
         extract!(tok, Token::Lit(l) => l.as_ir())
     }
+    #[inline(always)]
+    unsafe fn read_data_type(&mut self, tok: &'a Token) -> DataType {
+        DataType::clone_from_lit(extract!(tok, Token::Lit(ref l) => l))
+    }
 }
 
 #[derive(Debug)]
@@ -257,6 +275,13 @@ impl<'a> QueryData<'a> for SubstitutedData<'a> {
         self.data = &self.data[1..];
         ret
     }
+    #[inline(always)]
+    unsafe fn read_data_type(&mut self, tok: &'a Token) -> DataType {
+        debug_assert!(Token![?].eq(tok));
+        let ret = self.data[0];
+        self.data = &self.data[1..];
+        DataType::clone_from_litir(ret)
+    }
 }
 
 /*
diff --git a/server/src/engine/ql/dml/insert.rs b/server/src/engine/ql/dml/insert.rs
index a93a52e4..54c54315 100644
--- a/server/src/engine/ql/dml/insert.rs
+++ b/server/src/engine/ql/dml/insert.rs
@@ -153,7 +153,7 @@ pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
             tok if state.can_read_lit_from(tok) => {
                 let r = unsafe {
                     // UNSAFE(@ohsayan): the if guard guarantees correctness
-                    DataType::clone_from_litir(state.read_lit_unchecked_from(tok))
+                    state.read_lit_into_data_type_unchecked_from(tok)
                 };
                 r
             }
@@ -230,9 +230,7 @@ pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
         match state.fw_read() {
             tok if state.can_read_lit_from(tok) => unsafe {
                 // UNSAFE(@ohsayan): if guard guarantees correctness
-                data.push(Some(DataType::clone_from_litir(
-                    state.read_lit_unchecked_from(tok),
-                )))
+                data.push(Some(state.read_lit_into_data_type_unchecked_from(tok)));
             },
             Token![open []] if state.not_exhausted() => {
                 let mut l = Vec::new();
@@ -290,10 +288,10 @@ pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
         state.poison_if_not(Token![:].eq(colon));
         match (field, expr) {
             (Token::Ident(id), tok) if state.can_read_lit_from(tok) => {
-                let ldata = Some(DataType::clone_from_litir(unsafe {
+                let ldata = Some(unsafe {
                     // UNSAFE(@ohsayan): The if guard guarantees correctness
-                    state.read_lit_unchecked_from(tok)
-                }));
+                    state.read_lit_into_data_type_unchecked_from(tok)
+                });
                 state.poison_if_not(data.insert(*id, ldata).is_none());
             }
             (Token::Ident(id), Token![null]) => {

From 763007a98c74c12327c87d3b2590c3c7f615f75e Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 10 Jan 2023 07:20:59 -0800
Subject: [PATCH 070/310] Use `State` for `CREATE` and `ALTER` statements

---
 server/src/engine/ql/ast.rs                |  77 +-
 server/src/engine/ql/lexer.rs              |   1 +
 server/src/engine/ql/schema.rs             | 803 +++++++++------------
 server/src/engine/ql/tests/schema_tests.rs | 104 +--
 4 files changed, 428 insertions(+), 557 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index d628d386..82eae2f2 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -202,6 +202,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
             & (self.t[idx_c] == Token![() close])
             & rem
     }
+    #[inline(always)]
     /// Reads a lit using the given token and the internal data source and return a data type
     ///
     /// ## Safety
@@ -211,6 +212,24 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     pub unsafe fn read_lit_into_data_type_unchecked_from(&mut self, tok: &'a Token) -> DataType {
         self.d.read_data_type(tok)
     }
+    #[inline(always)]
+    /// Loop condition for tt and non-poisoned state only
+    pub fn loop_tt(&self) -> bool {
+        self.not_exhausted() && self.okay()
+    }
+    #[inline(always)]
+    /// Loop condition for tt and non-poisoned state only
+    pub fn loop_data_tt(&self) -> bool {
+        self.not_exhausted() && self.okay() && self.d.nonzero()
+    }
+    #[inline(always)]
+    pub(crate) fn consumed(&self) -> usize {
+        self.t.len() - self.remaining()
+    }
+    #[inline(always)]
+    pub(crate) fn cursor(&self) -> usize {
+        self.i
+    }
 }
 
 pub trait QueryData<'a> {
@@ -226,6 +245,8 @@ pub trait QueryData<'a> {
     /// ## Safety
     /// The current token must match the signature of a lit
     unsafe fn read_data_type(&mut self, tok: &'a Token) -> DataType;
+    /// Returns true if the data source has enough data
+    fn nonzero(&self) -> bool;
 }
 
 #[derive(Debug)]
@@ -250,6 +271,10 @@ impl<'a> QueryData<'a> for InplaceData {
     unsafe fn read_data_type(&mut self, tok: &'a Token) -> DataType {
         DataType::clone_from_lit(extract!(tok, Token::Lit(ref l) => l))
     }
+    #[inline(always)]
+    fn nonzero(&self) -> bool {
+        true
+    }
 }
 
 #[derive(Debug)]
@@ -266,7 +291,7 @@ impl<'a> SubstitutedData<'a> {
 impl<'a> QueryData<'a> for SubstitutedData<'a> {
     #[inline(always)]
     fn can_read_lit_from(&self, tok: &Token) -> bool {
-        Token![?].eq(tok) && !self.data.is_empty()
+        Token![?].eq(tok) && self.nonzero()
     }
     #[inline(always)]
     unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a> {
@@ -282,6 +307,10 @@ impl<'a> QueryData<'a> for SubstitutedData<'a> {
         self.data = &self.data[1..];
         DataType::clone_from_litir(ret)
     }
+    #[inline(always)]
+    fn nonzero(&self) -> bool {
+        !self.data.is_empty()
+    }
 }
 
 /*
@@ -443,45 +472,41 @@ pub enum Statement<'a> {
     Delete(dml::delete::DeleteStatement<'a>),
 }
 
-pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], mut d: Qd) -> LangResult<Statement<'a>> {
+pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], d: Qd) -> LangResult<Statement<'a>> {
     let mut i = 0;
-    let ref mut qd = d;
     if compiler::unlikely(tok.len() < 2) {
         return Err(LangError::UnexpectedEndofStatement);
     }
     match tok[0] {
         // DDL
         Token![use] => Entity::parse_from_tokens(&tok[1..], &mut i).map(Statement::Use),
-        Token![create] => match tok[1] {
-            Token![model] => schema::parse_schema_from_tokens(&tok[2..], qd).map(|(q, c)| {
-                i += c;
-                Statement::CreateModel(q)
-            }),
-            Token![space] => schema::parse_space_from_tokens(&tok[2..], qd).map(|(q, c)| {
-                i += c;
-                Statement::CreateSpace(q)
-            }),
-            _ => compiler::cold_rerr(LangError::UnknownCreateStatement),
-        },
         Token![drop] if tok.len() >= 3 => ddl::parse_drop(&tok[1..], &mut i),
-        Token![alter] => match tok[1] {
-            Token![model] => schema::parse_alter_kind_from_tokens(&tok[2..], qd, &mut i)
-                .map(Statement::AlterModel),
-            Token![space] => {
-                schema::parse_alter_space_from_tokens(&tok[2..], qd).map(|(q, incr)| {
-                    i += incr;
-                    Statement::AlterSpace(q)
-                })
-            }
-            _ => compiler::cold_rerr(LangError::UnknownAlterStatement),
-        },
         Token::Ident(id) if id.eq_ignore_ascii_case(b"inspect") => {
             ddl::parse_inspect(&tok[1..], &mut i)
         }
-        // DML
         ref stmt => {
             let mut state = State::new(&tok[1..], d);
             match stmt {
+                // DDL
+                Token![create] => {
+                    match tok[1] {
+                        Token![model] => schema::parse_model_from_tokens(&mut state)
+                            .map(Statement::CreateModel),
+                        Token![space] => schema::parse_space_from_tokens(&mut state)
+                            .map(Statement::CreateSpace),
+                        _ => compiler::cold_rerr(LangError::UnknownCreateStatement),
+                    }
+                }
+                Token![alter] => match tok[1] {
+                    Token![model] => {
+                        schema::parse_alter_kind_from_tokens(&mut state).map(Statement::AlterModel)
+                    }
+                    Token![space] => {
+                        schema::parse_alter_space_from_tokens(&mut state).map(Statement::AlterSpace)
+                    }
+                    _ => compiler::cold_rerr(LangError::UnknownAlterStatement),
+                },
+                // DML
                 Token![insert] => {
                     dml::insert::InsertStatement::parse_insert(&mut state).map(Statement::Insert)
                 }
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index e64d530b..936b66a3 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -81,6 +81,7 @@ enum_impls! {
 
 #[derive(Debug, PartialEq, Clone)]
 #[repr(u8)]
+/// A [`Lit`] as represented by an insecure token stream
 pub enum Lit<'a> {
     Str(Box<str>),
     Bool(bool),
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 0d64f39d..399a73c8 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -44,20 +44,19 @@
     Sept. 15, 2022
 */
 
+#[cfg(test)]
+use crate::engine::ql::ast::InplaceData;
 use {
     super::{
-        ast::QueryData,
+        ast::{QueryData, State},
         lexer::{LitIR, LitIROwned, Slice, Symbol, Token},
         LangError, LangResult,
     },
-    crate::util::MaybeInit,
+    crate::util::{compiler, MaybeInit},
     core::str,
     std::collections::{HashMap, HashSet},
 };
 
-#[cfg(test)]
-use crate::engine::ql::ast::InplaceData;
-
 /*
     Meta
 */
@@ -227,11 +226,10 @@ states! {
 
 /// Fold a dictionary
 pub(super) fn rfold_dict<'a, Qd: QueryData<'a>>(
-    mut state: DictFoldState,
-    tok: &'a [Token],
-    d: &mut Qd,
+    mut mstate: DictFoldState,
+    state: &mut State<'a, Qd>,
     dict: &mut Dict,
-) -> u64 {
+) {
     /*
         NOTE: Assume rules wherever applicable
 
@@ -241,101 +239,84 @@ pub(super) fn rfold_dict<'a, Qd: QueryData<'a>>(
         <colon> ::= ":"
         <dict> ::= <openbrace> (<ident> <colon> (<lit> | <dict>) <comma>)* <comma>* <closebrace>
     */
-    let l = tok.len();
-    let mut i = 0;
-    let mut okay = true;
     let mut tmp = MaybeInit::uninit();
 
-    while i < l {
-        match (&tok[i], state) {
+    while state.loop_tt() {
+        match (state.fw_read(), mstate) {
             (Token::Symbol(Symbol::TtOpenBrace), DictFoldState::OB) => {
-                i += 1;
                 // we found a brace, expect a close brace or an ident
-                state = DictFoldState::CB_OR_IDENT;
+                mstate = DictFoldState::CB_OR_IDENT;
             }
             (
                 Token::Symbol(Symbol::TtCloseBrace),
                 DictFoldState::CB_OR_IDENT | DictFoldState::COMMA_OR_CB,
             ) => {
                 // end of stream
-                i += 1;
-                state = DictFoldState::FINAL;
+                mstate = DictFoldState::FINAL;
                 break;
             }
             (Token::Ident(id), DictFoldState::CB_OR_IDENT) => {
                 // found ident, so expect colon
-                i += 1;
                 tmp = MaybeInit::new(unsafe { str::from_utf8_unchecked(id) });
-                state = DictFoldState::COLON;
+                mstate = DictFoldState::COLON;
             }
             (Token::Symbol(Symbol::SymColon), DictFoldState::COLON) => {
                 // found colon, expect literal or openbrace
-                i += 1;
-                state = DictFoldState::LIT_OR_OB;
+                mstate = DictFoldState::LIT_OR_OB;
             }
-            (tok, DictFoldState::LIT_OR_OB) if Qd::can_read_lit_from(d, tok) => {
-                i += 1;
+            (tok, DictFoldState::LIT_OR_OB) if state.can_read_lit_from(tok) => {
                 // found literal; so push in k/v pair and then expect a comma or close brace
                 unsafe {
-                    okay &= dict
-                        .insert(
-                            tmp.assume_init_ref().to_string(),
-                            Some(Qd::read_lit(d, tok).into()),
-                        )
-                        .is_none();
+                    let v = Some(state.read_lit_unchecked_from(tok).into());
+                    state
+                        .poison_if_not(dict.insert(tmp.assume_init_ref().to_string(), v).is_none());
                 }
-                state = DictFoldState::COMMA_OR_CB;
+                mstate = DictFoldState::COMMA_OR_CB;
             }
             (Token![null], DictFoldState::LIT_OR_OB) => {
                 // null
-                i += 1;
-                okay &= dict
-                    .insert(unsafe { tmp.assume_init_ref() }.to_string(), None)
-                    .is_none();
-                state = DictFoldState::COMMA_OR_CB;
+                state.poison_if_not(
+                    dict.insert(unsafe { tmp.assume_init_ref() }.to_string(), None)
+                        .is_none(),
+                );
+                mstate = DictFoldState::COMMA_OR_CB;
             }
             // ONLY COMMA CAPTURE
             (Token::Symbol(Symbol::SymComma), DictFoldState::COMMA_OR_CB) => {
-                i += 1;
                 // we found a comma, expect a *strict* brace close or ident
-                state = DictFoldState::CB_OR_IDENT;
+                mstate = DictFoldState::CB_OR_IDENT;
             }
             (Token::Symbol(Symbol::TtOpenBrace), DictFoldState::LIT_OR_OB) => {
-                i += 1;
                 // we found an open brace, so this is a dict
                 let mut new_dict = Dict::new();
-                let ret = rfold_dict(DictFoldState::CB_OR_IDENT, &tok[i..], d, &mut new_dict);
-                okay &= ret & HIBIT == HIBIT;
-                i += (ret & !HIBIT) as usize;
-                okay &= dict
-                    .insert(
+                rfold_dict(DictFoldState::CB_OR_IDENT, state, &mut new_dict);
+                state.poison_if_not(
+                    dict.insert(
                         unsafe { tmp.assume_init_ref() }.to_string(),
                         Some(new_dict.into()),
                     )
-                    .is_none();
+                    .is_none(),
+                );
                 // at the end of a dict we either expect a comma or close brace
-                state = DictFoldState::COMMA_OR_CB;
+                mstate = DictFoldState::COMMA_OR_CB;
             }
             _ => {
-                okay = false;
+                state.poison();
+                state.cursor_back();
                 break;
             }
         }
     }
-    okay &= state == DictFoldState::FINAL;
-    i as u64 | ((okay as u64) << 63)
+    state.poison_if_not(mstate == DictFoldState::FINAL);
 }
 
 #[cfg(test)]
 /// Fold a dictionary (**test-only**)
 pub fn fold_dict(tok: &[Token]) -> Option<Dict> {
     let mut d = Dict::new();
-    let r = rfold_dict(DictFoldState::OB, tok, &mut InplaceData::new(), &mut d);
-    if r & HIBIT == HIBIT {
-        Some(d)
-    } else {
-        None
-    }
+    let mut state = State::new(tok, InplaceData::new());
+    rfold_dict(DictFoldState::OB, &mut state, &mut d);
+    state.okay().then_some(d)
 }
 
 /*
@@ -355,200 +336,137 @@ states! {
 }
 
 #[derive(Debug, PartialEq)]
-/// The result of a type metadata fold
-pub struct TyMetaFoldResult {
-    cnt: usize,
-    reset: bool,
+pub struct TyMetaReturn {
     more: bool,
-    okay: bool,
+    reset: bool,
 }
 
-impl TyMetaFoldResult {
+impl TyMetaReturn {
     #[inline(always)]
-    /// Create a new [`TyMetaFoldResult`] with the default settings:
-    /// - reset: false
-    /// - more: false
-    /// - okay: true
-    const fn new() -> Self {
+    pub const fn new() -> Self {
         Self {
-            cnt: 0,
-            reset: false,
             more: false,
-            okay: true,
+            reset: false,
         }
     }
     #[inline(always)]
-    /// Increment the position
-    fn incr(&mut self) {
-        self.incr_by(1)
-    }
-    #[inline(always)]
-    /// Increment the position by `by`
-    fn incr_by(&mut self, by: usize) {
-        self.cnt += by;
+    pub const fn has_more(&self) -> bool {
+        self.more
     }
     #[inline(always)]
-    /// Set fail
-    fn set_fail(&mut self) {
-        self.okay = false;
+    pub const fn has_reset(&self) -> bool {
+        self.reset
     }
     #[inline(always)]
-    /// Set has more
-    fn set_has_more(&mut self) {
+    pub fn set_has_more(&mut self) {
         self.more = true;
     }
     #[inline(always)]
-    /// Set reset
-    fn set_reset(&mut self) {
+    pub fn set_has_reset(&mut self) {
         self.reset = true;
     }
-    #[inline(always)]
-    /// Should the meta be reset?
-    pub fn should_reset(&self) -> bool {
-        self.reset
-    }
-    #[inline(always)]
-    /// Returns the cursor
-    pub fn pos(&self) -> usize {
-        self.cnt
-    }
-    #[inline(always)]
-    /// Returns if more layers are expected
-    pub fn has_more(&self) -> bool {
-        self.more
-    }
-    #[inline(always)]
-    /// Returns if the internal state is okay
-    pub fn is_okay(&self) -> bool {
-        self.okay
-    }
-    #[inline(always)]
-    /// Records an expression
-    fn record(&mut self, c: bool) {
-        self.okay &= c;
-    }
 }
 
 /// Fold type metadata (flag setup dependent on caller)
 pub(super) fn rfold_tymeta<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
-    mut state: TyMetaFoldState,
-    tok: &'a [Token],
-    d: &mut Qd,
+    mut mstate: TyMetaFoldState,
+    state: &mut State<'a, Qd>,
     dict: &mut Dict,
-) -> TyMetaFoldResult {
-    let l = tok.len();
-    let mut r = TyMetaFoldResult::new();
+) -> TyMetaReturn {
     let mut tmp = MaybeInit::uninit();
-    while r.pos() < l && r.is_okay() {
-        match (&tok[r.pos()], state) {
+    let mut tymr = TyMetaReturn::new();
+    while state.loop_tt() {
+        match (state.fw_read(), mstate) {
             (Token![type], TyMetaFoldState::IDENT_OR_CB) => {
                 // we were expecting an ident but found the type keyword! increase depth
-                r.incr();
-                r.set_has_more();
-                state = TyMetaFoldState::FINAL;
+                tymr.set_has_more();
+                mstate = TyMetaFoldState::FINAL;
                 break;
             }
-            (Token::Symbol(Symbol::SymPeriod), TyMetaFoldState::IDENT_OR_CB) if ALLOW_RESET => {
-                r.incr();
-                let reset = r.pos() < l && tok[r.pos()] == Token::Symbol(Symbol::SymPeriod);
-                r.incr_by(reset as _);
-                r.record(reset);
-                r.set_reset();
-                state = TyMetaFoldState::CB;
+            (Token![.], TyMetaFoldState::IDENT_OR_CB) if ALLOW_RESET => {
+                let reset = state.cursor_rounded_eq(Token![.]);
+                state.cursor_ahead_if(reset);
+                tymr.set_has_reset();
+                state.poison_if_not(reset);
+                mstate = TyMetaFoldState::CB;
             }
             (
                 Token::Symbol(Symbol::TtCloseBrace),
                 TyMetaFoldState::IDENT_OR_CB | TyMetaFoldState::COMMA_OR_CB | TyMetaFoldState::CB,
             ) => {
-                r.incr();
                 // found close brace. end of stream
-                state = TyMetaFoldState::FINAL;
+                mstate = TyMetaFoldState::FINAL;
                 break;
             }
             (Token::Ident(ident), TyMetaFoldState::IDENT_OR_CB) => {
-                r.incr();
                 tmp = MaybeInit::new(unsafe { str::from_utf8_unchecked(ident) });
                 // we just saw an ident, so we expect to see a colon
-                state = TyMetaFoldState::COLON;
+                mstate = TyMetaFoldState::COLON;
             }
             (Token::Symbol(Symbol::SymColon), TyMetaFoldState::COLON) => {
-                r.incr();
                 // we just saw a colon. now we want a literal or openbrace
-                state = TyMetaFoldState::LIT_OR_OB;
+                mstate = TyMetaFoldState::LIT_OR_OB;
             }
-            (tok, TyMetaFoldState::LIT_OR_OB) if Qd::can_read_lit_from(d, tok) => {
-                r.incr();
+            (tok, TyMetaFoldState::LIT_OR_OB) if state.can_read_lit_from(tok) => {
                 unsafe {
-                    r.record(
-                        dict.insert(
-                            tmp.assume_init_ref().to_string(),
-                            Some(Qd::read_lit(d, tok).into()),
-                        )
-                        .is_none(),
-                    );
+                    let v = Some(state.read_lit_unchecked_from(tok).into());
+                    state
+                        .poison_if_not(dict.insert(tmp.assume_init_ref().to_string(), v).is_none());
                 }
                 // saw a literal. next is either comma or close brace
-                state = TyMetaFoldState::COMMA_OR_CB;
+                mstate = TyMetaFoldState::COMMA_OR_CB;
             }
             (Token![null], TyMetaFoldState::LIT_OR_OB) => {
-                r.incr();
-                r.record(
+                state.poison_if_not(
                     dict.insert(unsafe { tmp.assume_init_ref() }.to_string(), None)
                         .is_none(),
                 );
                 // saw null, start parsing another entry
-                state = TyMetaFoldState::COMMA_OR_CB;
+                mstate = TyMetaFoldState::COMMA_OR_CB;
             }
             (Token::Symbol(Symbol::SymComma), TyMetaFoldState::COMMA_OR_CB) => {
-                r.incr();
                 // next is strictly a close brace or ident
-                state = TyMetaFoldState::IDENT_OR_CB;
+                mstate = TyMetaFoldState::IDENT_OR_CB;
             }
             (Token::Symbol(Symbol::TtOpenBrace), TyMetaFoldState::LIT_OR_OB) => {
-                r.incr();
                 // another dict in here
                 let mut nd = Dict::new();
-                let ret = rfold_tymeta::<Qd, ALLOW_RESET>(
-                    TyMetaFoldState::IDENT_OR_CB,
-                    &tok[r.pos()..],
-                    d,
-                    &mut nd,
-                );
-                r.incr_by(ret.pos());
-                r.record(ret.is_okay());
+                let ret =
+                    rfold_tymeta::<Qd, ALLOW_RESET>(TyMetaFoldState::IDENT_OR_CB, state, &mut nd);
                 // L2 cannot have type definitions
-                r.record(!ret.has_more());
+                state.poison_if(ret.has_more());
                 // end of definition or comma followed by something
-                r.record(
+                state.poison_if_not(
                     dict.insert(
                         unsafe { tmp.assume_init_ref() }.to_string(),
                         Some(nd.into()),
                     )
                     .is_none(),
                 );
-                state = TyMetaFoldState::COMMA_OR_CB;
+                mstate = TyMetaFoldState::COMMA_OR_CB;
             }
             _ => {
-                r.set_fail();
+                state.cursor_back();
+                state.poison();
                 break;
             }
         }
     }
-    r.record(state == TyMetaFoldState::FINAL);
-    r
+    state.poison_if_not(mstate == TyMetaFoldState::FINAL);
+    tymr
 }
 
 #[cfg(test)]
 /// (**test-only**) fold type metadata
-pub(super) fn fold_tymeta(tok: &[Token]) -> (TyMetaFoldResult, Dict) {
+pub(super) fn fold_tymeta(tok: &[Token]) -> (TyMetaReturn, bool, usize, Dict) {
+    let mut state = State::new(tok, InplaceData::new());
     let mut d = Dict::new();
-    let r = rfold_tymeta::<InplaceData, DISALLOW_RESET_SYNTAX>(
+    let ret = rfold_tymeta::<InplaceData, DISALLOW_RESET_SYNTAX>(
         TyMetaFoldState::IDENT_OR_CB,
-        tok,
-        &mut InplaceData::new(),
+        &mut state,
         &mut d,
     );
-    (r, d)
+    (ret, state.okay(), state.cursor(), d)
 }
 
 /*
@@ -568,10 +486,9 @@ states! {
 /// Fold layers
 pub(super) fn rfold_layers<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
     start: LayerFoldState,
-    tok: &'a [Token],
-    qd: &mut Qd,
+    state: &mut State<'a, Qd>,
     layers: &mut Vec<Layer<'a>>,
-) -> u64 {
+) {
     /*
         NOTE: Assume rules wherever applicable
 
@@ -585,76 +502,54 @@ pub(super) fn rfold_layers<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
             (<ident> <colon> (<lit> | <dict>) <comma>)*
             <comma>* <closebrace>
     */
-    let l = tok.len();
-    let mut i = 0;
-    let mut okay = true;
-    let mut state = start;
+    let mut mstate = start;
     let mut tmp = MaybeInit::uninit();
     let mut dict = Dict::new();
-    while i < l && okay {
-        match (&tok[i], state) {
+    while state.loop_tt() {
+        match (state.fw_read(), mstate) {
             (Token::Ident(ty), LayerFoldState::TY) => {
-                i += 1;
                 // expecting type, and found type. next is either end or an open brace or some arbitrary token
                 tmp = MaybeInit::new(ty.clone());
-                state = LayerFoldState::END_OR_OB;
+                mstate = LayerFoldState::END_OR_OB;
             }
             (Token::Symbol(Symbol::TtOpenBrace), LayerFoldState::END_OR_OB) => {
-                i += 1;
                 // since we found an open brace, this type has some meta
-                let ret = rfold_tymeta::<Qd, ALLOW_RESET>(
-                    TyMetaFoldState::IDENT_OR_CB,
-                    &tok[i..],
-                    qd,
-                    &mut dict,
-                );
-                i += ret.pos();
-                okay &= ret.is_okay();
+                let ret =
+                    rfold_tymeta::<Qd, ALLOW_RESET>(TyMetaFoldState::IDENT_OR_CB, state, &mut dict);
                 if ret.has_more() {
                     // more layers
-                    let ret =
-                        rfold_layers::<Qd, ALLOW_RESET>(LayerFoldState::TY, &tok[i..], qd, layers);
-                    okay &= ret & HIBIT == HIBIT;
-                    i += (ret & !HIBIT) as usize;
-                    state = LayerFoldState::FOLD_DICT_INCOMPLETE;
-                } else if okay {
+                    rfold_layers::<Qd, ALLOW_RESET>(LayerFoldState::TY, state, layers);
+                    mstate = LayerFoldState::FOLD_DICT_INCOMPLETE;
+                } else if state.okay() {
                     // done folding dictionary. nothing more expected. break
-                    state = LayerFoldState::FOLD_COMPLETED;
+                    mstate = LayerFoldState::FOLD_COMPLETED;
                     layers.push(Layer {
                         ty: unsafe { tmp.assume_init() }.clone(),
                         props: dict,
-                        reset: ret.should_reset(),
+                        reset: ret.has_reset(),
                     });
                     break;
                 }
             }
             (Token::Symbol(Symbol::SymComma), LayerFoldState::FOLD_DICT_INCOMPLETE) => {
                 // there is a comma at the end of this
-                i += 1;
-                let ret = rfold_tymeta::<Qd, ALLOW_RESET>(
-                    TyMetaFoldState::IDENT_OR_CB,
-                    &tok[i..],
-                    qd,
-                    &mut dict,
-                );
-                i += ret.pos();
-                okay &= ret.is_okay();
-                okay &= !ret.has_more(); // not more than one type depth
-                if okay {
+                let ret =
+                    rfold_tymeta::<Qd, ALLOW_RESET>(TyMetaFoldState::IDENT_OR_CB, state, &mut dict);
+                state.poison_if(ret.has_more()); // not more than one type depth
+                if state.okay() {
                     // done folding dict successfully. nothing more expected. break.
-                    state = LayerFoldState::FOLD_COMPLETED;
+                    mstate = LayerFoldState::FOLD_COMPLETED;
                     layers.push(Layer {
                         ty: unsafe { tmp.assume_init() }.clone(),
                         props: dict,
-                        reset: ret.should_reset(),
+                        reset: ret.has_reset(),
                     });
                     break;
                 }
             }
             (Token::Symbol(Symbol::TtCloseBrace), LayerFoldState::FOLD_DICT_INCOMPLETE) => {
                 // end of stream
-                i += 1;
-                state = LayerFoldState::FOLD_COMPLETED;
+                mstate = LayerFoldState::FOLD_COMPLETED;
                 layers.push(Layer {
                     ty: unsafe { tmp.assume_init() }.clone(),
                     props: dict,
@@ -663,8 +558,9 @@ pub(super) fn rfold_layers<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
                 break;
             }
             (_, LayerFoldState::END_OR_OB) => {
+                state.cursor_back();
                 // random arbitrary byte. finish append
-                state = LayerFoldState::FOLD_COMPLETED;
+                mstate = LayerFoldState::FOLD_COMPLETED;
                 layers.push(Layer {
                     ty: unsafe { tmp.assume_init() }.clone(),
                     props: dict,
@@ -673,106 +569,96 @@ pub(super) fn rfold_layers<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
                 break;
             }
             _ => {
-                okay = false;
+                state.cursor_back();
+                state.poison();
                 break;
             }
         }
     }
-    okay &= state == LayerFoldState::FOLD_COMPLETED;
-    i as u64 | ((okay as u64) << 63)
+    state.poison_if_not(mstate == LayerFoldState::FOLD_COMPLETED);
 }
 
 #[cfg(test)]
 #[inline(always)]
 /// (**test-only**) fold layers
 pub(super) fn fold_layers<'a>(tok: &'a [Token]) -> (Vec<Layer<'a>>, usize, bool) {
+    let mut state = State::new(tok, InplaceData::new());
     let mut l = Vec::new();
-    let r = rfold_layers::<InplaceData, DISALLOW_RESET_SYNTAX>(
-        LayerFoldState::TY,
-        tok,
-        &mut InplaceData::new(),
-        &mut l,
-    );
-    (l, (r & !HIBIT) as _, r & HIBIT == HIBIT)
+    rfold_layers::<InplaceData, DISALLOW_RESET_SYNTAX>(LayerFoldState::TY, &mut state, &mut l);
+    (l, state.consumed(), state.okay())
 }
 
 #[inline(always)]
 /// Collect field properties
-pub(super) fn collect_field_properties(tok: &[Token]) -> (FieldProperties, u64) {
+pub(super) fn collect_field_properties<'a, Qd: QueryData<'a>>(
+    state: &mut State<'a, Qd>,
+) -> FieldProperties {
     let mut props = FieldProperties::default();
-    let mut i = 0;
-    let mut okay = true;
-    while i < tok.len() {
-        match &tok[i] {
-            Token![primary] => okay &= props.properties.insert(FieldProperties::PRIMARY),
-            Token![null] => okay &= props.properties.insert(FieldProperties::NULL),
-            Token::Ident(_) => break,
+    while state.loop_tt() {
+        match state.fw_read() {
+            Token![primary] => {
+                state.poison_if_not(props.properties.insert(FieldProperties::PRIMARY))
+            }
+            Token![null] => state.poison_if_not(props.properties.insert(FieldProperties::NULL)),
+            Token::Ident(_) => {
+                state.cursor_back();
+                break;
+            }
             _ => {
                 // we could pass this over to the caller, but it's better if we do it since we're doing
                 // a linear scan anyways
-                okay = false;
+                state.cursor_back();
+                state.poison();
                 break;
             }
         }
-        i += 1;
     }
-    (props, i as u64 | ((okay as u64) << 63))
+    props
 }
 
 #[cfg(test)]
 #[inline(always)]
 /// (**test-only**) parse field properties
 pub(super) fn parse_field_properties(tok: &[Token]) -> (FieldProperties, usize, bool) {
-    let (p, r) = collect_field_properties(tok);
-    (p, (r & !HIBIT) as _, r & HIBIT == HIBIT)
+    let mut state = State::new(tok, InplaceData::new());
+    let p = collect_field_properties(&mut state);
+    (p, state.cursor(), state.okay())
 }
 
 #[cfg(test)]
 pub(super) fn parse_field_full<'a>(tok: &'a [Token]) -> LangResult<(usize, Field<'a>)> {
-    self::parse_field(tok, &mut InplaceData::new())
+    let mut state = State::new(tok, InplaceData::new());
+    self::parse_field(&mut state).map(|field| (state.cursor(), field))
 }
 
 #[inline(always)]
 /// Parse a field using the declaration-syntax (not field syntax)
+///
+/// Expected start token: field name (ident)
 pub(super) fn parse_field<'a, Qd: QueryData<'a>>(
-    tok: &'a [Token],
-    qd: &mut Qd,
-) -> LangResult<(usize, Field<'a>)> {
-    let l = tok.len();
-    let mut i = 0;
-    let mut okay = true;
+    state: &mut State<'a, Qd>,
+) -> LangResult<Field<'a>> {
     // parse field properties
-    let (props, r) = collect_field_properties(tok);
-    okay &= r & HIBIT == HIBIT;
-    i += (r & !HIBIT) as usize;
+    let props = collect_field_properties(state);
     // if exhauted or broken, simply return
-    if i == l || !okay || (l - i) == 1 {
+    if compiler::unlikely(state.exhausted() | !state.okay() || state.remaining() == 1) {
         return Err(LangError::UnexpectedEndofStatement);
     }
-
     // field name
-    let field_name = match (&tok[i], &tok[i + 1]) {
-        (Token::Ident(id), Token::Symbol(Symbol::SymColon)) => id,
+    let field_name = match (state.fw_read(), state.fw_read()) {
+        (Token::Ident(id), Token![:]) => id,
         _ => return Err(LangError::UnexpectedToken),
     };
-    i += 2;
 
     // layers
     let mut layers = Vec::new();
-    let r =
-        rfold_layers::<Qd, DISALLOW_RESET_SYNTAX>(LayerFoldState::TY, &tok[i..], qd, &mut layers);
-    okay &= r & HIBIT == HIBIT;
-    i += (r & !HIBIT) as usize;
-
-    if okay {
-        Ok((
-            i,
-            Field {
-                field_name: field_name.clone(),
-                layers,
-                props: props.properties,
-            },
-        ))
+    rfold_layers::<Qd, DISALLOW_RESET_SYNTAX>(LayerFoldState::TY, state, &mut layers);
+    if state.okay() {
+        Ok(Field {
+            field_name: field_name.clone(),
+            layers,
+            props: props.properties,
+        })
     } else {
         Err(LangError::UnexpectedToken)
     }
@@ -792,135 +678,123 @@ states! {
 pub(super) fn parse_schema_from_tokens_full<'a>(
     tok: &'a [Token],
 ) -> LangResult<(Model<'a>, usize)> {
-    self::parse_schema_from_tokens::<InplaceData>(tok, &mut InplaceData::new())
+    let mut state = State::new(tok, InplaceData::new());
+    self::parse_model_from_tokens::<InplaceData>(&mut state).map(|model| (model, state.cursor()))
 }
 
 #[inline(always)]
 /// Parse a fresh schema with declaration-syntax fields
-pub(super) fn parse_schema_from_tokens<'a, Qd: QueryData<'a>>(
-    tok: &'a [Token],
-    qd: &mut Qd,
-) -> LangResult<(Model<'a>, usize)> {
+pub(super) fn parse_model_from_tokens<'a, Qd: QueryData<'a>>(
+    state: &mut State<'a, Qd>,
+) -> LangResult<Model<'a>> {
     // parse fields
-    let l = tok.len();
-    let mut i = 0;
     // check if we have our model name
-    let mut okay = i < l && tok[i].is_ident();
-    i += okay as usize;
+    // smallest model declaration: create model mymodel(username: string, password: binary) -> 10 tokens
+    if compiler::unlikely(state.remaining() < 10) {
+        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+    }
+    let model_name = state.fw_read();
+    state.poison_if_not(model_name.is_ident());
     let mut fields = Vec::with_capacity(2);
-    let mut state = SchemaParseState::OPEN_PAREN;
+    let mut mstate = SchemaParseState::OPEN_PAREN;
 
-    while i < l && okay {
-        match (&tok[i], state) {
+    while state.loop_tt() {
+        match (state.fw_read(), mstate) {
             (Token::Symbol(Symbol::TtOpenParen), SchemaParseState::OPEN_PAREN) => {
-                i += 1;
-                state = SchemaParseState::FIELD;
+                mstate = SchemaParseState::FIELD;
             }
             (
                 Token![primary] | Token![null] | Token::Ident(_),
                 SchemaParseState::FIELD | SchemaParseState::END_OR_FIELD,
             ) => {
+                state.cursor_back();
                 // fine, we found a field. let's see what we've got
-                let (c, f) = self::parse_field(&tok[i..], qd)?;
+                let f = self::parse_field(state)?;
                 fields.push(f);
-                i += c;
-                state = SchemaParseState::COMMA_OR_END;
+                mstate = SchemaParseState::COMMA_OR_END;
             }
             (Token::Symbol(Symbol::SymComma), SchemaParseState::COMMA_OR_END) => {
-                i += 1;
                 // expect a field or close paren
-                state = SchemaParseState::END_OR_FIELD;
+                mstate = SchemaParseState::END_OR_FIELD;
             }
             (
                 Token::Symbol(Symbol::TtCloseParen),
                 SchemaParseState::COMMA_OR_END | SchemaParseState::END_OR_FIELD,
             ) => {
-                i += 1;
                 // end of stream
                 break;
             }
             _ => {
-                okay = false;
+                state.cursor_back();
+                state.poison();
                 break;
             }
         }
     }
 
     // model properties
-    if !okay {
+    if !state.okay() {
         return Err(LangError::UnexpectedToken);
     }
 
     let model_name = unsafe {
         // UNSAFE(@ohsayan): Now that we're sure that we have the model name ident, get it
-        extract!(tok[0], Token::Ident(ref model_name) => model_name.clone())
+        extract!(model_name, Token::Ident(ref model_name) => model_name.clone())
     };
 
-    if l > i && tok[i] == (Token![with]) {
+    if state.cursor_rounded_eq(Token![with]) {
         // we have some more input, and it should be a dict of properties
-        i += 1; // +WITH
+        state.cursor_ahead(); // +WITH
 
         // great, parse the dict
         let mut dict = Dict::new();
-        let r = self::rfold_dict(DictFoldState::OB, &tok[i..], qd, &mut dict);
-        i += (r & !HIBIT) as usize;
+        self::rfold_dict(DictFoldState::OB, state, &mut dict);
 
-        if r & HIBIT == HIBIT {
+        if state.okay() {
             // sweet, so we got our dict
-            Ok((
-                Model {
-                    model_name,
-                    props: dict,
-                    fields,
-                },
-                i,
-            ))
+            Ok(Model {
+                model_name,
+                props: dict,
+                fields,
+            })
         } else {
             Err(LangError::UnexpectedToken)
         }
     } else {
         // we've reached end of stream, so there's nothing more to parse
-        Ok((
-            Model {
-                model_name,
-                props: dict! {},
-                fields,
-            },
-            i,
-        ))
+        Ok(Model {
+            model_name,
+            props: dict! {},
+            fields,
+        })
     }
 }
 
 #[inline(always)]
 /// Parse space data from the given tokens
 pub(super) fn parse_space_from_tokens<'a, Qd: QueryData<'a>>(
-    tok: &'a [Token],
-    qd: &mut Qd,
-) -> LangResult<(Space<'a>, usize)> {
-    let l = tok.len();
-    let mut okay = !tok.is_empty() && tok[0].is_ident();
-    let mut i = 0;
-    i += okay as usize;
+    state: &mut State<'a, Qd>,
+) -> LangResult<Space<'a>> {
+    // smallest declaration: `create space myspace` -> >= 1 token
+    if compiler::unlikely(state.remaining() < 1) {
+        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+    }
+    let space_name = state.fw_read();
+    state.poison_if_not(space_name.is_ident());
     // either we have `with` or nothing. don't be stupid
-    let has_more_properties = i < l && tok[i] == Token![with];
-    okay &= has_more_properties | (i == l);
-    // properties
+    let has_more_properties = state.cursor_rounded_eq(Token![with]);
+    state.poison_if_not(has_more_properties | state.exhausted());
+    state.cursor_ahead_if(has_more_properties); // +WITH
     let mut d = Dict::new();
-
-    if has_more_properties && okay {
-        let ret = self::rfold_dict(DictFoldState::OB, &tok[1..], qd, &mut d);
-        i += (ret & !HIBIT) as usize;
-        okay &= ret & HIBIT == HIBIT;
+    // properties
+    if has_more_properties && state.okay() {
+        self::rfold_dict(DictFoldState::OB, state, &mut d);
     }
-
-    if okay {
-        Ok((
-            Space {
-                space_name: unsafe { extract!(tok[0], Token::Ident(ref id) => id.clone()) },
-                props: d,
-            },
-            i,
-        ))
+    if state.okay() {
+        Ok(Space {
+            space_name: unsafe { extract!(space_name, Token::Ident(ref id) => id.clone()) },
+            props: d,
+        })
     } else {
         Err(LangError::UnexpectedToken)
     }
@@ -929,34 +803,29 @@ pub(super) fn parse_space_from_tokens<'a, Qd: QueryData<'a>>(
 #[inline(always)]
 /// Parse alter space from tokens
 pub(super) fn parse_alter_space_from_tokens<'a, Qd: QueryData<'a>>(
-    tok: &'a [Token],
-    qd: &mut Qd,
-) -> LangResult<(AlterSpace<'a>, usize)> {
-    let mut i = 0;
-    let l = tok.len();
-
-    let okay = l > 3 && tok[0].is_ident() && tok[1] == Token![with] && tok[2] == Token![open {}];
+    state: &mut State<'a, Qd>,
+) -> LangResult<AlterSpace<'a>> {
+    if compiler::unlikely(state.remaining() <= 3) {
+        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+    }
+    let space_name = state.fw_read();
+    state.poison_if_not(state.cursor_eq(Token![with]));
+    state.cursor_ahead(); // ignore errors
+    state.poison_if_not(state.cursor_eq(Token![open {}]));
+    state.cursor_ahead(); // ignore errors
 
-    if !okay {
+    if compiler::unlikely(!state.okay()) {
         return Err(LangError::UnexpectedToken);
     }
 
-    let space_name = unsafe { extract!(tok[0], Token::Ident(ref space) => space.clone()) };
-
-    i += 3;
-
+    let space_name = unsafe { extract!(space_name, Token::Ident(ref space) => space.clone()) };
     let mut d = Dict::new();
-    let ret = rfold_dict(DictFoldState::CB_OR_IDENT, &tok[i..], qd, &mut d);
-    i += (ret & !HIBIT) as usize;
-
-    if ret & HIBIT == HIBIT {
-        Ok((
-            AlterSpace {
-                space_name,
-                updated_props: d,
-            },
-            i,
-        ))
+    rfold_dict(DictFoldState::CB_OR_IDENT, state, &mut d);
+    if state.okay() {
+        Ok(AlterSpace {
+            space_name,
+            updated_props: d,
+        })
     } else {
         Err(LangError::UnexpectedToken)
     }
@@ -964,9 +833,10 @@ pub(super) fn parse_alter_space_from_tokens<'a, Qd: QueryData<'a>>(
 
 #[cfg(test)]
 pub(super) fn alter_space_full<'a>(tok: &'a [Token]) -> LangResult<AlterSpace<'a>> {
-    let (r, i) = self::parse_alter_space_from_tokens(tok, &mut InplaceData::new())?;
-    assert_full_tt!(i, tok.len());
-    Ok(r)
+    let mut state = State::new(tok, InplaceData::new());
+    let a = self::parse_alter_space_from_tokens(&mut state)?;
+    assert_full_tt!(state);
+    Ok(a)
 }
 
 states! {
@@ -992,96 +862,75 @@ pub struct ExpandedField<'a> {
 pub fn parse_field_syntax_full<'a, const ALLOW_RESET: bool>(
     tok: &'a [Token],
 ) -> LangResult<(ExpandedField<'a>, usize)> {
-    self::parse_field_syntax::<InplaceData, ALLOW_RESET>(tok, &mut InplaceData::new())
+    let mut state = State::new(tok, InplaceData::new());
+    self::parse_field_syntax::<InplaceData, ALLOW_RESET>(&mut state)
+        .map(|efield| (efield, state.cursor()))
 }
 
 #[inline(always)]
 /// Parse a field declared using the field syntax
 pub(super) fn parse_field_syntax<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
-    tok: &'a [Token],
-    qd: &mut Qd,
-) -> LangResult<(ExpandedField<'a>, usize)> {
-    let l = tok.len();
-    let mut i = 0_usize;
-    let mut state = FieldSyntaxParseState::IDENT;
-    let mut okay = true;
+    state: &mut State<'a, Qd>,
+) -> LangResult<ExpandedField<'a>> {
+    let mut mstate = FieldSyntaxParseState::IDENT;
     let mut tmp = MaybeInit::uninit();
     let mut props = Dict::new();
     let mut layers = vec![];
     let mut reset = false;
-    while i < l && okay {
-        match (&tok[i], state) {
+    while state.loop_tt() {
+        match (state.fw_read(), mstate) {
             (Token::Ident(field), FieldSyntaxParseState::IDENT) => {
-                i += 1;
                 tmp = MaybeInit::new(field.clone());
                 // expect open brace
-                state = FieldSyntaxParseState::OB;
+                mstate = FieldSyntaxParseState::OB;
             }
             (Token::Symbol(Symbol::TtOpenBrace), FieldSyntaxParseState::OB) => {
-                i += 1;
                 let r = self::rfold_tymeta::<Qd, ALLOW_RESET>(
                     TyMetaFoldState::IDENT_OR_CB,
-                    &tok[i..],
-                    qd,
+                    state,
                     &mut props,
                 );
-                okay &= r.is_okay();
-                i += r.pos();
-                if r.has_more() && i < l {
+                if r.has_more() && state.not_exhausted() {
                     // now parse layers
-                    let r = self::rfold_layers::<Qd, ALLOW_RESET>(
-                        LayerFoldState::TY,
-                        &tok[i..],
-                        qd,
-                        &mut layers,
-                    );
-                    okay &= r & HIBIT == HIBIT;
-                    i += (r & !HIBIT) as usize;
-                    state = FieldSyntaxParseState::FOLD_DICT_INCOMPLETE;
+                    self::rfold_layers::<Qd, ALLOW_RESET>(LayerFoldState::TY, state, &mut layers);
+                    mstate = FieldSyntaxParseState::FOLD_DICT_INCOMPLETE;
                 } else {
-                    okay = false;
+                    state.poison();
                     break;
                 }
             }
             (Token::Symbol(Symbol::SymComma), FieldSyntaxParseState::FOLD_DICT_INCOMPLETE) => {
-                i += 1;
                 let r = self::rfold_tymeta::<Qd, ALLOW_RESET>(
                     TyMetaFoldState::IDENT_OR_CB,
-                    &tok[i..],
-                    qd,
+                    state,
                     &mut props,
                 );
-                okay &= r.is_okay() && !r.has_more();
-                i += r.pos();
-                reset = ALLOW_RESET && r.should_reset();
-                if okay {
-                    state = FieldSyntaxParseState::COMPLETED;
+                reset = ALLOW_RESET && r.has_reset();
+                if state.okay() {
+                    mstate = FieldSyntaxParseState::COMPLETED;
                     break;
                 }
             }
             (Token::Symbol(Symbol::TtCloseBrace), FieldSyntaxParseState::FOLD_DICT_INCOMPLETE) => {
-                i += 1;
                 // great, were done
-                state = FieldSyntaxParseState::COMPLETED;
+                mstate = FieldSyntaxParseState::COMPLETED;
                 break;
             }
             _ => {
-                okay = false;
+                state.cursor_back();
+                state.poison();
                 break;
             }
         }
     }
-    okay &= state == FieldSyntaxParseState::COMPLETED;
-    if okay {
-        Ok((
-            ExpandedField {
-                field_name: unsafe { tmp.assume_init() },
-                layers,
-                props,
-                reset,
-            },
-            i,
-        ))
+    state.poison_if_not(mstate == FieldSyntaxParseState::COMPLETED);
+    if state.okay() {
+        Ok(ExpandedField {
+            field_name: unsafe { tmp.assume_init() },
+            layers,
+            props,
+            reset,
+        })
     } else {
         Err(LangError::UnexpectedToken)
     }
@@ -1113,25 +962,21 @@ pub enum AlterKind<'a> {
 #[inline(always)]
 /// Parse an [`AlterKind`] from the given token stream
 pub(super) fn parse_alter_kind_from_tokens<'a, Qd: QueryData<'a>>(
-    tok: &'a [Token],
-    qd: &mut Qd,
-    current: &mut usize,
+    state: &mut State<'a, Qd>,
 ) -> LangResult<Alter<'a>> {
-    let l = tok.len();
-    let okay = l > 2 && tok[0].is_ident();
-    if !okay {
-        return Err(LangError::UnexpectedEndofStatement);
+    // alter model mymodel remove x
+    if state.remaining() <= 2 || !state.cursor_has_ident_rounded() {
+        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
     }
-    *current += 2;
-    let model_name = unsafe { extract!(tok[0], Token::Ident(ref l) => l.clone()) };
-    match tok[1] {
-        Token![add] => alter_add(&tok[1..], qd, current)
+    let model_name = unsafe { extract!(state.fw_read(), Token::Ident(ref l) => l.clone()) };
+    match state.fw_read() {
+        Token![add] => alter_add(state)
             .map(AlterKind::Add)
             .map(|kind| Alter::new(model_name, kind)),
-        Token![remove] => alter_remove(&tok[1..], current)
+        Token![remove] => alter_remove(state)
             .map(AlterKind::Remove)
             .map(|kind| Alter::new(model_name, kind)),
-        Token![update] => alter_update(&tok[1..], qd, current)
+        Token![update] => alter_update(state)
             .map(AlterKind::Update)
             .map(|kind| Alter::new(model_name, kind)),
         _ => return Err(LangError::ExpectedStatement),
@@ -1141,9 +986,7 @@ pub(super) fn parse_alter_kind_from_tokens<'a, Qd: QueryData<'a>>(
 #[inline(always)]
 /// Parse multiple fields declared using the field syntax. Flag setting allows or disallows reset syntax
 pub(super) fn parse_multiple_field_syntax<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
-    tok: &'a [Token],
-    qd: &mut Qd,
-    current: &mut usize,
+    state: &mut State<'a, Qd>,
 ) -> LangResult<Box<[ExpandedField<'a>]>> {
     const DEFAULT_ADD_COL_CNT: usize = 4;
     /*
@@ -1152,43 +995,39 @@ pub(super) fn parse_multiple_field_syntax<'a, Qd: QueryData<'a>, const ALLOW_RES
         <add> ::= (<field_syntax> <comma>)*
 
         Smallest length:
-        alter model add myfield { type string };
+        alter model add myfield { type string }
     */
-    let l = tok.len();
-    if l < 5 {
-        return Err(LangError::UnexpectedEndofStatement);
+    if compiler::unlikely(state.remaining() < 5) {
+        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
     }
-    match tok[0] {
+    match state.read() {
         Token::Ident(_) => {
-            let (r, i) = parse_field_syntax::<Qd, ALLOW_RESET>(&tok, qd)?;
-            *current += i;
-            Ok([r].into())
+            let ef = parse_field_syntax::<Qd, ALLOW_RESET>(state)?;
+            Ok([ef].into())
         }
         Token::Symbol(Symbol::TtOpenParen) => {
-            let mut i = 1;
-            let mut okay = true;
+            state.cursor_ahead();
             let mut stop = false;
             let mut cols = Vec::with_capacity(DEFAULT_ADD_COL_CNT);
-            while i < l && okay && !stop {
-                match tok[i] {
+            while state.loop_tt() && !stop {
+                match state.read() {
                     Token::Ident(_) => {
-                        let (r, cnt) = parse_field_syntax::<Qd, ALLOW_RESET>(&tok[i..], qd)?;
-                        i += cnt;
-                        cols.push(r);
-                        let nx_comma = i < l && tok[i] == Token::Symbol(Symbol::SymComma);
-                        let nx_close = i < l && tok[i] == Token::Symbol(Symbol::TtCloseParen);
+                        let ef = parse_field_syntax::<Qd, ALLOW_RESET>(state)?;
+                        cols.push(ef);
+                        let nx_comma = state.cursor_rounded_eq(Token![,]);
+                        let nx_close = state.cursor_rounded_eq(Token![() close]);
                         stop = nx_close;
-                        okay &= nx_comma | nx_close;
-                        i += (nx_comma | nx_close) as usize;
+                        state.poison_if_not(nx_comma | nx_close);
+                        state.cursor_ahead_if(state.okay());
                     }
                     _ => {
-                        okay = false;
+                        state.poison();
                         break;
                     }
                 }
             }
-            *current += i;
-            if okay && stop {
+            state.poison_if_not(stop);
+            if state.okay() {
                 Ok(cols.into_boxed_slice())
             } else {
                 Err(LangError::UnexpectedToken)
@@ -1201,11 +1040,9 @@ pub(super) fn parse_multiple_field_syntax<'a, Qd: QueryData<'a>, const ALLOW_RES
 #[inline(always)]
 /// Parse the expression for `alter model <> add (..)`
 pub(super) fn alter_add<'a, Qd: QueryData<'a>>(
-    tok: &'a [Token],
-    qd: &mut Qd,
-    current: &mut usize,
+    state: &mut State<'a, Qd>,
 ) -> LangResult<Box<[ExpandedField<'a>]>> {
-    self::parse_multiple_field_syntax::<Qd, DISALLOW_RESET_SYNTAX>(tok, qd, current)
+    self::parse_multiple_field_syntax::<Qd, DISALLOW_RESET_SYNTAX>(state)
 }
 
 #[cfg(test)]
@@ -1213,54 +1050,50 @@ pub(super) fn alter_add_full<'a>(
     tok: &'a [Token],
     current: &mut usize,
 ) -> LangResult<Box<[ExpandedField<'a>]>> {
-    self::alter_add(tok, &mut InplaceData::new(), current)
+    let mut state = State::new(tok, InplaceData::new());
+    let r = self::alter_add(&mut state);
+    *current += state.consumed();
+    r
 }
 
 #[inline(always)]
 /// Parse the expression for `alter model <> remove (..)`
-pub(super) fn alter_remove<'a>(
-    tok: &'a [Token],
-    current: &mut usize,
+pub(super) fn alter_remove<'a, Qd: QueryData<'a>>(
+    state: &mut State<'a, Qd>,
 ) -> LangResult<Box<[Slice<'a>]>> {
     const DEFAULT_REMOVE_COL_CNT: usize = 4;
     /*
         WARNING: No trailing commas allowed
         <remove> ::= <ident> | <openparen> (<ident> <comma>)*<closeparen>
     */
-    if tok.is_empty() {
-        return Err(LangError::UnexpectedEndofStatement);
+    if compiler::unlikely(state.exhausted()) {
+        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
     }
 
-    let r = match &tok[0] {
-        Token::Ident(id) => {
-            *current += 1;
-            Box::new([id.clone()])
-        }
+    let r = match state.fw_read() {
+        Token::Ident(id) => Box::new([id.clone()]),
         Token::Symbol(Symbol::TtOpenParen) => {
-            let l = tok.len();
-            let mut i = 1_usize;
-            let mut okay = true;
             let mut stop = false;
             let mut cols = Vec::with_capacity(DEFAULT_REMOVE_COL_CNT);
-            while i < tok.len() && okay && !stop {
-                match tok[i] {
+            while state.loop_tt() && !stop {
+                match state.fw_read() {
                     Token::Ident(ref ident) => {
                         cols.push(ident.clone());
-                        i += 1;
-                        let nx_comma = i < l && tok[i] == (Token::Symbol(Symbol::SymComma));
-                        let nx_close = i < l && tok[i] == (Token::Symbol(Symbol::TtCloseParen));
-                        okay &= nx_comma | nx_close;
+                        let nx_comma = state.cursor_rounded_eq(Token![,]);
+                        let nx_close = state.cursor_rounded_eq(Token![() close]);
+                        state.poison_if_not(nx_comma | nx_close);
                         stop = nx_close;
-                        i += (nx_comma | nx_close) as usize;
+                        state.cursor_ahead_if(state.okay());
                     }
                     _ => {
-                        okay = false;
+                        state.cursor_back();
+                        state.poison();
                         break;
                     }
                 }
             }
-            *current += i;
-            if okay && stop {
+            state.poison_if_not(stop);
+            if state.okay() {
                 cols.into_boxed_slice()
             } else {
                 return Err(LangError::UnexpectedToken);
@@ -1271,14 +1104,23 @@ pub(super) fn alter_remove<'a>(
     Ok(r)
 }
 
+#[cfg(test)]
+pub(super) fn alter_remove_full<'a>(
+    tok: &'a [Token<'a>],
+    i: &mut usize,
+) -> LangResult<Box<[Slice<'a>]>> {
+    let mut state = State::new(tok, InplaceData::new());
+    let r = self::alter_remove(&mut state);
+    *i += state.consumed();
+    r
+}
+
 #[inline(always)]
 /// Parse the expression for `alter model <> update (..)`
 pub(super) fn alter_update<'a, Qd: QueryData<'a>>(
-    tok: &'a [Token],
-    qd: &mut Qd,
-    current: &mut usize,
+    state: &mut State<'a, Qd>,
 ) -> LangResult<Box<[ExpandedField<'a>]>> {
-    self::parse_multiple_field_syntax::<Qd, ALLOW_RESET_SYNTAX>(tok, qd, current)
+    self::parse_multiple_field_syntax::<Qd, ALLOW_RESET_SYNTAX>(state)
 }
 
 #[cfg(test)]
@@ -1286,5 +1128,8 @@ pub(super) fn alter_update_full<'a>(
     tok: &'a [Token],
     i: &mut usize,
 ) -> LangResult<Box<[ExpandedField<'a>]>> {
-    self::alter_update(tok, &mut InplaceData::new(), i)
+    let mut state = State::new(tok, InplaceData::new());
+    let r = self::alter_update(&mut state);
+    *i += state.consumed();
+    r
 }
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 7c39a416..471f99ad 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -119,30 +119,30 @@ mod tymeta {
     #[test]
     fn tymeta_mini() {
         let tok = lex_insecure(b"}").unwrap();
-        let (res, ret) = schema::fold_tymeta(&tok);
-        assert!(res.is_okay());
-        assert!(!res.has_more());
-        assert_eq!(res.pos(), 1);
-        assert_eq!(ret, nullable_dict!());
+        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
+        assert!(okay);
+        assert!(!tymeta.has_more());
+        assert_eq!(cursor, 1);
+        assert_eq!(data, nullable_dict!());
     }
     #[test]
     fn tymeta_mini_fail() {
         let tok = lex_insecure(b",}").unwrap();
-        let (res, ret) = schema::fold_tymeta(&tok);
-        assert!(!res.is_okay());
-        assert!(!res.has_more());
-        assert_eq!(res.pos(), 0);
-        assert_eq!(ret, nullable_dict!());
+        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
+        assert!(!okay);
+        assert!(!tymeta.has_more());
+        assert_eq!(cursor, 0);
+        assert_eq!(data, nullable_dict!());
     }
     #[test]
     fn tymeta() {
         let tok = lex_insecure(br#"hello: "world", loading: true, size: 100 }"#).unwrap();
-        let (res, ret) = schema::fold_tymeta(&tok);
-        assert!(res.is_okay());
-        assert!(!res.has_more());
-        assert_eq!(res.pos(), tok.len());
+        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
+        assert!(okay);
+        assert!(!tymeta.has_more());
+        assert_eq!(cursor, tok.len());
         assert_eq!(
-            ret,
+            data,
             nullable_dict! {
                 "hello" => Lit::Str("world".into()),
                 "loading" => Lit::Bool(true),
@@ -155,17 +155,17 @@ mod tymeta {
         // list { maxlen: 100, type string, unique: true }
         //        ^^^^^^^^^^^^^^^^^^ cursor should be at string
         let tok = lex_insecure(br#"maxlen: 100, type string, unique: true }"#).unwrap();
-        let (res1, ret1) = schema::fold_tymeta(&tok);
-        assert!(res1.is_okay());
-        assert!(res1.has_more());
-        assert_eq!(res1.pos(), 5);
-        let remslice = &tok[res1.pos() + 2..];
-        let (res2, ret2) = schema::fold_tymeta(remslice);
-        assert!(res2.is_okay());
-        assert!(!res2.has_more());
-        assert_eq!(res2.pos() + res1.pos() + 2, tok.len());
-        let mut final_ret = ret1;
-        final_ret.extend(ret2);
+        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
+        assert!(okay);
+        assert!(tymeta.has_more());
+        assert_eq!(cursor, 5);
+        let remslice = &tok[cursor + 2..];
+        let (tymeta2, okay2, cursor2, data2) = schema::fold_tymeta(remslice);
+        assert!(okay2);
+        assert!(!tymeta2.has_more());
+        assert_eq!(cursor2 + cursor + 2, tok.len());
+        let mut final_ret = data;
+        final_ret.extend(data2);
         assert_eq!(
             final_ret,
             nullable_dict! {
@@ -181,17 +181,17 @@ mod tymeta {
         let tok =
             lex_insecure(br#"maxlen: 100, this: { is: "cool" }, type string, unique: true }"#)
                 .unwrap();
-        let (res1, ret1) = schema::fold_tymeta(&tok);
-        assert!(res1.is_okay());
-        assert!(res1.has_more());
-        assert_eq!(res1.pos(), 13);
-        let remslice = &tok[res1.pos() + 2..];
-        let (res2, ret2) = schema::fold_tymeta(remslice);
-        assert!(res2.is_okay());
-        assert!(!res2.has_more());
-        assert_eq!(res2.pos() + res1.pos() + 2, tok.len());
-        let mut final_ret = ret1;
-        final_ret.extend(ret2);
+        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
+        assert!(okay);
+        assert!(tymeta.has_more());
+        assert_eq!(cursor, 13);
+        let remslice = &tok[cursor + 2..];
+        let (tymeta2, okay2, cursor2, data2) = schema::fold_tymeta(remslice);
+        assert!(okay2);
+        assert!(!tymeta2.has_more());
+        assert_eq!(cursor2 + cursor + 2, tok.len());
+        let mut final_ret = data;
+        final_ret.extend(data2);
         assert_eq!(
             final_ret,
             nullable_dict! {
@@ -228,13 +228,13 @@ mod tymeta {
             "users" => Lit::Str("sayan".into())
         };
         fuzz_tokens(&tok, |should_pass, new_src| {
-            let (ret, dict) = schema::fold_tymeta(&new_src);
+            let (tymeta, okay, cursor, data) = schema::fold_tymeta(&new_src);
             if should_pass {
-                assert!(ret.is_okay(), "{:?}", &new_src);
-                assert!(!ret.has_more());
-                assert_eq!(ret.pos(), new_src.len());
-                assert_eq!(dict, expected);
-            } else if ret.is_okay() {
+                assert!(okay, "{:?}", &new_src);
+                assert!(!tymeta.has_more());
+                assert_eq!(cursor, new_src.len());
+                assert_eq!(data, expected);
+            } else if okay {
                 panic!(
                     "Expected failure but passed for token stream: `{:?}`",
                     new_src
@@ -267,13 +267,13 @@ mod tymeta {
             },
         };
         fuzz_tokens(&tok, |should_pass, new_src| {
-            let (ret, dict) = schema::fold_tymeta(&new_src);
+            let (tymeta, okay, cursor, data) = schema::fold_tymeta(&new_src);
             if should_pass {
-                assert!(ret.is_okay());
-                assert!(ret.has_more());
-                assert!(new_src[ret.pos()] == Token::Ident(b"string"));
-                assert_eq!(dict, expected);
-            } else if ret.is_okay() {
+                assert!(okay);
+                assert!(tymeta.has_more());
+                assert!(new_src[cursor] == Token::Ident(b"string"));
+                assert_eq!(data, expected);
+            } else if okay {
                 panic!("Expected failure but passed for token stream: `{:?}`", tok);
             }
         });
@@ -895,7 +895,7 @@ mod alter_model_remove {
     fn alter_mini() {
         let tok = lex_insecure(b"alter model mymodel remove myfield").unwrap();
         let mut i = 4;
-        let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+        let remove = schema::alter_remove_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(remove, [b"myfield".as_slice()].into());
     }
@@ -903,7 +903,7 @@ mod alter_model_remove {
     fn alter_mini_2() {
         let tok = lex_insecure(b"alter model mymodel remove (myfield)").unwrap();
         let mut i = 4;
-        let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+        let remove = schema::alter_remove_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(remove, [b"myfield".as_slice()].into());
     }
@@ -913,7 +913,7 @@ mod alter_model_remove {
             lex_insecure(b"alter model mymodel remove (myfield1, myfield2, myfield3, myfield4)")
                 .unwrap();
         let mut i = 4;
-        let remove = schema::alter_remove(&tok[i..], &mut i).unwrap();
+        let remove = schema::alter_remove_full(&tok[i..], &mut i).unwrap();
         assert_eq!(i, tok.len());
         assert_eq!(
             remove,

From fd22b511830d880c3a4b1173a30f8dcf21157418 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 10 Jan 2023 07:56:46 -0800
Subject: [PATCH 071/310] Use `State` for all other DDL statements

---
 server/src/engine/ql/ast.rs        | 139 ++++++++++++++++++++---------
 server/src/engine/ql/ddl.rs        |  84 +++++++++--------
 server/src/engine/ql/dml/delete.rs |  14 +--
 server/src/engine/ql/dml/insert.rs |   4 +-
 server/src/engine/ql/dml/mod.rs    |  40 +--------
 server/src/engine/ql/dml/select.rs |   4 +-
 server/src/engine/ql/dml/update.rs |   4 +-
 7 files changed, 158 insertions(+), 131 deletions(-)

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 82eae2f2..c1acec59 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -30,7 +30,10 @@ use {
         lexer::{LitIR, Slice, Token},
         schema, LangError, LangResult,
     },
-    crate::{engine::memory::DataType, util::compiler},
+    crate::{
+        engine::memory::DataType,
+        util::{compiler, MaybeInit},
+    },
     core::cmp,
 };
 
@@ -230,6 +233,10 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     pub(crate) fn cursor(&self) -> usize {
         self.i
     }
+    #[inline(always)]
+    pub(crate) fn cursor_is_ident(&self) -> bool {
+        self.read().is_ident()
+    }
 }
 
 pub trait QueryData<'a> {
@@ -428,6 +435,58 @@ impl<'a> Entity<'a> {
         };
         Ok(r)
     }
+    #[inline(always)]
+    pub fn attempt_process_entity_result<Qd: QueryData<'a>>(
+        state: &mut State<'a, Qd>,
+    ) -> LangResult<Self> {
+        let mut e = MaybeInit::uninit();
+        Self::attempt_process_entity(state, &mut e);
+        if state.okay() {
+            unsafe {
+                // UNSAFE(@ohsayan): just checked if okay
+                Ok(e.assume_init())
+            }
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    }
+    #[inline(always)]
+    pub fn attempt_process_entity<Qd: QueryData<'a>>(
+        state: &mut State<'a, Qd>,
+        d: &mut MaybeInit<Entity<'a>>,
+    ) {
+        let tok = state.current();
+        let is_full = Entity::tokens_with_full(tok);
+        let is_single = Entity::tokens_with_single(tok);
+        unsafe {
+            if is_full {
+                state.cursor_ahead_by(3);
+                *d = MaybeInit::new(Entity::full_entity_from_slice(tok));
+            } else if is_single {
+                state.cursor_ahead();
+                *d = MaybeInit::new(Entity::single_entity_from_slice(tok));
+            }
+        }
+        state.poison_if_not(is_full | is_single);
+    }
+    pub fn parse_entity<Qd: QueryData<'a>>(
+        state: &mut State<'a, Qd>,
+        d: &mut MaybeInit<Entity<'a>>,
+    ) {
+        let tok = state.current();
+        let is_full = tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident();
+        let is_single = tok[0].is_ident();
+        unsafe {
+            if is_full {
+                state.cursor_ahead_by(3);
+                *d = MaybeInit::new(Entity::full_entity_from_slice(tok));
+            } else if is_single {
+                state.cursor_ahead();
+                *d = MaybeInit::new(Entity::single_entity_from_slice(tok));
+            }
+        }
+        state.poison_if_not(is_full | is_single);
+    }
 }
 
 #[cfg_attr(test, derive(Debug, PartialEq))]
@@ -473,54 +532,46 @@ pub enum Statement<'a> {
 }
 
 pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], d: Qd) -> LangResult<Statement<'a>> {
-    let mut i = 0;
     if compiler::unlikely(tok.len() < 2) {
         return Err(LangError::UnexpectedEndofStatement);
     }
-    match tok[0] {
+    let mut state = State::new(tok, d);
+    match state.fw_read() {
         // DDL
-        Token![use] => Entity::parse_from_tokens(&tok[1..], &mut i).map(Statement::Use),
-        Token![drop] if tok.len() >= 3 => ddl::parse_drop(&tok[1..], &mut i),
-        Token::Ident(id) if id.eq_ignore_ascii_case(b"inspect") => {
-            ddl::parse_inspect(&tok[1..], &mut i)
-        }
-        ref stmt => {
-            let mut state = State::new(&tok[1..], d);
-            match stmt {
-                // DDL
-                Token![create] => {
-                    match tok[1] {
-                        Token![model] => schema::parse_model_from_tokens(&mut state)
-                            .map(Statement::CreateModel),
-                        Token![space] => schema::parse_space_from_tokens(&mut state)
-                            .map(Statement::CreateSpace),
-                        _ => compiler::cold_rerr(LangError::UnknownCreateStatement),
-                    }
-                }
-                Token![alter] => match tok[1] {
-                    Token![model] => {
-                        schema::parse_alter_kind_from_tokens(&mut state).map(Statement::AlterModel)
-                    }
-                    Token![space] => {
-                        schema::parse_alter_space_from_tokens(&mut state).map(Statement::AlterSpace)
-                    }
-                    _ => compiler::cold_rerr(LangError::UnknownAlterStatement),
-                },
-                // DML
-                Token![insert] => {
-                    dml::insert::InsertStatement::parse_insert(&mut state).map(Statement::Insert)
-                }
-                Token![select] => {
-                    dml::select::SelectStatement::parse_select(&mut state).map(Statement::Select)
-                }
-                Token![update] => {
-                    dml::update::UpdateStatement::parse_update(&mut state).map(Statement::Update)
-                }
-                Token![delete] => {
-                    dml::delete::DeleteStatement::parse_delete(&mut state).map(Statement::Delete)
-                }
-                _ => compiler::cold_rerr(LangError::ExpectedStatement),
+        Token![use] => Entity::attempt_process_entity_result(&mut state).map(Statement::Use),
+        Token![create] => match state.fw_read() {
+            Token![model] => {
+                schema::parse_model_from_tokens(&mut state).map(Statement::CreateModel)
+            }
+            Token![space] => {
+                schema::parse_space_from_tokens(&mut state).map(Statement::CreateSpace)
             }
+            _ => compiler::cold_rerr(LangError::UnknownCreateStatement),
+        },
+        Token![alter] => match state.fw_read() {
+            Token![model] => {
+                schema::parse_alter_kind_from_tokens(&mut state).map(Statement::AlterModel)
+            }
+            Token![space] => {
+                schema::parse_alter_space_from_tokens(&mut state).map(Statement::AlterSpace)
+            }
+            _ => compiler::cold_rerr(LangError::UnknownAlterStatement),
+        },
+        Token![drop] if state.remaining() >= 2 => ddl::parse_drop(&mut state),
+        Token::Ident(id) if id.eq_ignore_ascii_case(b"inspect") => ddl::parse_inspect(&mut state),
+        // DML
+        Token![insert] => {
+            dml::insert::InsertStatement::parse_insert(&mut state).map(Statement::Insert)
+        }
+        Token![select] => {
+            dml::select::SelectStatement::parse_select(&mut state).map(Statement::Select)
+        }
+        Token![update] => {
+            dml::update::UpdateStatement::parse_update(&mut state).map(Statement::Update)
+        }
+        Token![delete] => {
+            dml::delete::DeleteStatement::parse_delete(&mut state).map(Statement::Delete)
         }
+        _ => compiler::cold_rerr(LangError::ExpectedStatement),
     }
 }
diff --git a/server/src/engine/ql/ddl.rs b/server/src/engine/ql/ddl.rs
index d984ecd0..895fab67 100644
--- a/server/src/engine/ql/ddl.rs
+++ b/server/src/engine/ql/ddl.rs
@@ -24,10 +24,15 @@
  *
 */
 
-use super::{
-    ast::{Entity, Statement},
-    lexer::{Slice, Token},
-    LangError, LangResult,
+#[cfg(test)]
+use super::ast::InplaceData;
+use {
+    super::{
+        ast::{Entity, QueryData, State, Statement},
+        lexer::{Slice, Token},
+        LangError, LangResult,
+    },
+    crate::util::compiler,
 };
 
 #[derive(Debug, PartialEq)]
@@ -62,32 +67,31 @@ impl<'a> DropModel<'a> {
 /// ## Panic
 ///
 /// If token stream length is < 2
-pub(super) fn parse_drop<'a>(tok: &'a [Token], counter: &mut usize) -> LangResult<Statement<'a>> {
-    match tok[0] {
+pub(super) fn parse_drop<'a, Qd: QueryData<'a>>(
+    state: &mut State<'a, Qd>,
+) -> LangResult<Statement<'a>> {
+    match state.fw_read() {
         Token![model] => {
             // we have a model. now parse entity and see if we should force deletion
-            let mut i = 1;
-            let e = Entity::parse_from_tokens(&tok[1..], &mut i)?;
-            let force = i < tok.len() && tok[i] == Token::Ident(b"force");
-            i += force as usize;
-            *counter += i;
+            let e = Entity::attempt_process_entity_result(state)?;
+            let force = state.cursor_rounded_eq(Token::Ident(b"force"));
+            state.cursor_ahead_if(force);
             // if we've exhausted the stream, we're good to go (either `force`, or nothing)
-            if tok.len() == i {
+            if state.exhausted() {
                 return Ok(Statement::DropModel(DropModel::new(e, force)));
             }
         }
-        Token![space] if tok[1].is_ident() => {
-            let mut i = 2; // (`space` and space name)
-                           // should we force drop?
-            let force = i < tok.len() && tok[i] == Token::Ident(b"force");
-            i += force as usize;
-            *counter += i;
+        Token![space] if state.cursor_is_ident() => {
+            let ident = state.fw_read();
+            // should we force drop?
+            let force = state.cursor_rounded_eq(Token::Ident(b"force"));
+            state.cursor_ahead_if(force);
             // either `force` or nothing
-            if tok.len() == i {
+            if state.exhausted() {
                 return Ok(Statement::DropSpace(DropSpace::new(
                     unsafe {
                         // UNSAFE(@ohsayan): Safe because the match predicate ensures that tok[1] is indeed an ident
-                        extract!(tok[1], Token::Ident(ref space) => *space)
+                        extract!(ident, Token::Ident(ref space) => *space)
                     },
                     force,
                 )));
@@ -100,41 +104,49 @@ pub(super) fn parse_drop<'a>(tok: &'a [Token], counter: &mut usize) -> LangResul
 
 #[cfg(test)]
 pub(super) fn parse_drop_full<'a>(tok: &'a [Token]) -> LangResult<Statement<'a>> {
-    let mut i = 0;
-    let r = self::parse_drop(tok, &mut i);
-    assert_full_tt!(i, tok.len());
+    let mut state = State::new(tok, InplaceData::new());
+    let r = self::parse_drop(&mut state);
+    assert_full_tt!(state);
     r
 }
 
-pub(super) fn parse_inspect<'a>(tok: &'a [Token], c: &mut usize) -> LangResult<Statement<'a>> {
+pub(super) fn parse_inspect<'a, Qd: QueryData<'a>>(
+    state: &mut State<'a, Qd>,
+) -> LangResult<Statement<'a>> {
     /*
         inpsect model <entity>
         inspect space <entity>
         inspect spaces
+
+        min length -> (<model> | <space>) <model> = 2
     */
 
-    let nxt = tok.get(0);
-    *c += nxt.is_some() as usize;
-    match nxt {
-        Some(Token![model]) => Entity::parse_from_tokens(&tok[1..], c).map(Statement::InspectModel),
-        Some(Token![space]) if tok.len() == 2 && tok[1].is_ident() => {
-            *c += 1;
+    if compiler::unlikely(state.remaining() < 1) {
+        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+    }
+
+    match state.fw_read() {
+        Token![model] => Entity::attempt_process_entity_result(state).map(Statement::InspectModel),
+        Token![space] if state.cursor_has_ident_rounded() => {
             Ok(Statement::InspectSpace(unsafe {
                 // UNSAFE(@ohsayan): Safe because of the match predicate
-                extract!(tok[1], Token::Ident(ref space) => space)
+                extract!(state.fw_read(), Token::Ident(ref space) => space)
             }))
         }
-        Some(Token::Ident(id)) if id.eq_ignore_ascii_case(b"spaces") && tok.len() == 1 => {
+        Token::Ident(id) if id.eq_ignore_ascii_case(b"spaces") && state.exhausted() => {
             Ok(Statement::InspectSpaces)
         }
-        _ => Err(LangError::ExpectedStatement),
+        _ => {
+            state.cursor_back();
+            Err(LangError::ExpectedStatement)
+        }
     }
 }
 
 #[cfg(test)]
 pub(super) fn parse_inspect_full<'a>(tok: &'a [Token]) -> LangResult<Statement<'a>> {
-    let mut i = 0;
-    let r = self::parse_inspect(tok, &mut i);
-    assert_full_tt!(i, tok.len());
+    let mut state = State::new(tok, InplaceData::new());
+    let r = self::parse_inspect(&mut state);
+    assert_full_tt!(state);
     r
 }
diff --git a/server/src/engine/ql/dml/delete.rs b/server/src/engine/ql/dml/delete.rs
index 3990324a..92989c96 100644
--- a/server/src/engine/ql/dml/delete.rs
+++ b/server/src/engine/ql/dml/delete.rs
@@ -24,13 +24,8 @@
  *
 */
 
-#[cfg(test)]
-use {
-    super::WhereClauseCollection,
-    crate::engine::ql::{ast::InplaceData, lexer::Token},
-};
 use {
-    super::{parse_entity, WhereClause},
+    super::WhereClause,
     crate::{
         engine::ql::{
             ast::{Entity, QueryData, State},
@@ -39,6 +34,11 @@ use {
         util::{compiler, MaybeInit},
     },
 };
+#[cfg(test)]
+use {
+    super::WhereClauseCollection,
+    crate::engine::ql::{ast::InplaceData, lexer::Token},
+};
 
 /*
     Impls for delete
@@ -78,7 +78,7 @@ impl<'a> DeleteStatement<'a> {
         state.poison_if_not(state.cursor_eq(Token![from]));
         state.cursor_ahead(); // ignore errors (if any)
         let mut entity = MaybeInit::uninit();
-        parse_entity(state, &mut entity);
+        Entity::parse_entity(state, &mut entity);
         // where + clauses
         state.poison_if_not(state.cursor_eq(Token![where]));
         state.cursor_ahead(); // ignore errors
diff --git a/server/src/engine/ql/dml/insert.rs b/server/src/engine/ql/dml/insert.rs
index 54c54315..4852afc4 100644
--- a/server/src/engine/ql/dml/insert.rs
+++ b/server/src/engine/ql/dml/insert.rs
@@ -27,7 +27,7 @@
 #[cfg(test)]
 use crate::engine::ql::ast::InplaceData;
 use {
-    super::{parse_entity, read_ident},
+    super::read_ident,
     crate::{
         engine::{
             memory::DataType,
@@ -392,7 +392,7 @@ impl<'a> InsertStatement<'a> {
 
         // entity
         let mut entity = MaybeInit::uninit();
-        parse_entity(state, &mut entity);
+        Entity::parse_entity(state, &mut entity);
         let mut data = None;
         match state.fw_read() {
             Token![() open] if state.not_exhausted() => {
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index 89a5082d..65c9c918 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -38,10 +38,10 @@ pub mod update;
 use super::ast::InplaceData;
 use {
     super::{
-        ast::{Entity, QueryData, State},
+        ast::{QueryData, State},
         lexer::{LitIR, Token},
     },
-    crate::util::{compiler, MaybeInit},
+    crate::util::compiler,
     std::collections::HashMap,
 };
 
@@ -59,42 +59,6 @@ fn u(b: bool) -> u8 {
     Misc
 */
 
-#[inline(always)]
-fn attempt_process_entity<'a, Qd: QueryData<'a>>(
-    state: &mut State<'a, Qd>,
-    d: &mut MaybeInit<Entity<'a>>,
-) {
-    let tok = state.current();
-    let is_full = Entity::tokens_with_full(tok);
-    let is_single = Entity::tokens_with_single(tok);
-    unsafe {
-        if is_full {
-            state.cursor_ahead_by(3);
-            *d = MaybeInit::new(Entity::full_entity_from_slice(tok));
-        } else if is_single {
-            state.cursor_ahead();
-            *d = MaybeInit::new(Entity::single_entity_from_slice(tok));
-        }
-    }
-    state.poison_if_not(is_full | is_single);
-}
-
-fn parse_entity<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>, d: &mut MaybeInit<Entity<'a>>) {
-    let tok = state.current();
-    let is_full = tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident();
-    let is_single = tok[0].is_ident();
-    unsafe {
-        if is_full {
-            state.cursor_ahead_by(3);
-            *d = MaybeInit::new(Entity::full_entity_from_slice(tok));
-        } else if is_single {
-            state.cursor_ahead();
-            *d = MaybeInit::new(Entity::single_entity_from_slice(tok));
-        }
-    }
-    state.poison_if_not(is_full | is_single);
-}
-
 /*
     Contexts
 */
diff --git a/server/src/engine/ql/dml/select.rs b/server/src/engine/ql/dml/select.rs
index 9f9d2c81..a538b3c1 100644
--- a/server/src/engine/ql/dml/select.rs
+++ b/server/src/engine/ql/dml/select.rs
@@ -27,7 +27,7 @@
 #[cfg(test)]
 use crate::engine::ql::ast::InplaceData;
 use {
-    super::{attempt_process_entity, WhereClause, WhereClauseCollection},
+    super::{WhereClause, WhereClauseCollection},
     crate::{
         engine::ql::{
             ast::{Entity, QueryData, State},
@@ -122,7 +122,7 @@ impl<'a> SelectStatement<'a> {
         state.poison_if_not(state.cursor_eq(Token![from]));
         state.cursor_ahead(); // ignore errors
         let mut entity = MaybeInit::uninit();
-        attempt_process_entity(state, &mut entity);
+        Entity::attempt_process_entity(state, &mut entity);
         let mut clauses = <_ as Default>::default();
         if state.cursor_rounded_eq(Token![where]) {
             state.cursor_ahead();
diff --git a/server/src/engine/ql/dml/update.rs b/server/src/engine/ql/dml/update.rs
index 5cd0f032..6f0f7b05 100644
--- a/server/src/engine/ql/dml/update.rs
+++ b/server/src/engine/ql/dml/update.rs
@@ -30,7 +30,7 @@ use {
     crate::engine::ql::{ast::InplaceData, lexer::Token},
 };
 use {
-    super::{parse_entity, read_ident, u, WhereClause},
+    super::{read_ident, u, WhereClause},
     crate::{
         engine::ql::{
             ast::{Entity, QueryData, State},
@@ -188,7 +188,7 @@ impl<'a> UpdateStatement<'a> {
         }
         // parse entity
         let mut entity = MaybeInit::uninit();
-        parse_entity(state, &mut entity);
+        Entity::parse_entity(state, &mut entity);
         if !(state.has_remaining(6)) {
             unsafe {
                 // UNSAFE(@ohsayan): Obvious from above, max 3 fw

From a116e499534871bd9e470a188f7200b2c6567e45 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 12 Jan 2023 04:49:09 +0000
Subject: [PATCH 072/310] Add initial definition of `MemoryEngine`

---
 server/src/engine/memory/core.rs | 116 +++++++++++++++++++++++++++++++
 server/src/engine/memory/mod.rs  |   1 +
 server/src/engine/ql/ast.rs      |  23 +++++-
 server/src/engine/ql/benches.rs  |  27 ++++---
 server/src/engine/ql/lexer.rs    |   8 ++-
 5 files changed, 159 insertions(+), 16 deletions(-)
 create mode 100644 server/src/engine/memory/core.rs

diff --git a/server/src/engine/memory/core.rs b/server/src/engine/memory/core.rs
new file mode 100644
index 00000000..8e41e8fc
--- /dev/null
+++ b/server/src/engine/memory/core.rs
@@ -0,0 +1,116 @@
+/*
+ * Created on Wed Jan 11 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use core::{borrow::Borrow, hash::Hash};
+
+/// Any type implementing this trait can be used as a key inside memory engine structures
+pub trait AsKey: Hash + Eq + Clone {
+    /// Read the key
+    fn read_key(&self) -> &Self;
+    /// Read the key and return a clone
+    fn read_key_clone(&self) -> Self;
+}
+
+impl<T: Hash + Eq + Clone> AsKey for T {
+    #[inline(always)]
+    fn read_key(&self) -> &Self {
+        self
+    }
+    #[inline(always)]
+    fn read_key_clone(&self) -> Self {
+        Clone::clone(self)
+    }
+}
+
+/// Any type implementing this trait can be used as a value inside memory engine structures
+pub trait AsValue: Clone {
+    /// Read the value
+    fn read_value(&self) -> &Self;
+    /// Read the value and return a clone
+    fn read_value_clone(&self) -> Self;
+}
+
+impl<T: Clone> AsValue for T {
+    #[inline(always)]
+    fn read_value(&self) -> &Self {
+        self
+    }
+    #[inline(always)]
+    fn read_value_clone(&self) -> Self {
+        Clone::clone(self)
+    }
+}
+
+pub trait MemoryEngine<K, V>
+where
+    K: AsKey,
+    V: AsValue,
+{
+    const HAS_ORDER: bool;
+    type IterKV<'a>: Iterator<Item = (&'a K, &'a V)>
+    where
+        Self: 'a,
+        K: 'a,
+        V: 'a;
+    type IterKey<'a>: Iterator<Item = &'a K>
+    where
+        Self: 'a,
+        K: 'a;
+    // write
+    /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
+    /// violated
+    fn insert(&self, key: K, val: V) -> bool;
+    /// Returns a reference to the value corresponding to the key, if it exists
+
+    // read
+    fn get<Q>(&self, key: &Q) -> Option<&K>
+    where
+        K: Borrow<Q>;
+    /// Returns a clone of the value corresponding to the key, if it exists
+    fn get_cloned<Q>(&self, key: &Q) -> Option<K>
+    where
+        K: Borrow<Q>;
+
+    // update
+    /// Returns true if the entry is updated
+    fn update<Q>(&self, key: &Q, val: V) -> bool
+    where
+        K: Borrow<Q>;
+    /// Updates the entry and returns the old value, if it exists
+    fn update_return<Q>(&self, key: &Q, val: V) -> Option<K>
+    where
+        K: Borrow<Q>;
+
+    // delete
+    /// Returns true if the entry was deleted
+    fn delete<Q>(&self, key: &Q) -> bool
+    where
+        K: Borrow<Q>;
+    /// Removes the entry and returns it, if it exists
+    fn delete_return<Q>(&self, key: &Q) -> Option<K>
+    where
+        K: Borrow<Q>;
+}
diff --git a/server/src/engine/memory/mod.rs b/server/src/engine/memory/mod.rs
index 1bb7be69..3c126b4a 100644
--- a/server/src/engine/memory/mod.rs
+++ b/server/src/engine/memory/mod.rs
@@ -26,6 +26,7 @@
 
 // TODO(@ohsayan): Change the underlying structures, there are just rudimentary ones used during integration with the QL
 
+mod core;
 use super::ql::lexer::{Lit, LitIR};
 
 /// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index c1acec59..acbb0278 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -140,6 +140,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
         let mx = minidx(self.t, self.i);
         Qd::can_read_lit_from(&self.d, &self.t[mx]) && mx == self.i
     }
+    #[inline(always)]
     /// Check if a lit can be read using the given token with context from the data source
     pub fn can_read_lit_from(&self, tok: &'a Token<'a>) -> bool {
         Qd::can_read_lit_from(&self.d, tok)
@@ -206,6 +207,18 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
             & rem
     }
     #[inline(always)]
+    /// Check if the current token stream matches the signature of a full entity; rounded
+    ///
+    /// NOTE: Consider using a direct comparison without rounding; rounding is always slower
+    pub(crate) fn cursor_signature_match_entity_full_rounded(&self) -> bool {
+        let rem = self.has_remaining(3);
+        let rem_u = rem as usize;
+        let idx_a = self.i * rem_u;
+        let idx_b = (self.i + 1) * rem_u;
+        let idx_c = (self.i + 2) * rem_u;
+        (self.t[idx_a].is_ident()) & (self.t[idx_b] == Token![.]) & (self.t[idx_c].is_ident()) & rem
+    }
+    #[inline(always)]
     /// Reads a lit using the given token and the internal data source and return a data type
     ///
     /// ## Safety
@@ -226,14 +239,17 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
         self.not_exhausted() && self.okay() && self.d.nonzero()
     }
     #[inline(always)]
+    /// Returns the number of consumed tokens
     pub(crate) fn consumed(&self) -> usize {
         self.t.len() - self.remaining()
     }
     #[inline(always)]
+    /// Returns the position of the cursor
     pub(crate) fn cursor(&self) -> usize {
         self.i
     }
     #[inline(always)]
+    /// Returns true if the cursor is an ident
     pub(crate) fn cursor_is_ident(&self) -> bool {
         self.read().is_ident()
     }
@@ -441,7 +457,7 @@ impl<'a> Entity<'a> {
     ) -> LangResult<Self> {
         let mut e = MaybeInit::uninit();
         Self::attempt_process_entity(state, &mut e);
-        if state.okay() {
+        if compiler::likely(state.okay()) {
             unsafe {
                 // UNSAFE(@ohsayan): just checked if okay
                 Ok(e.assume_init())
@@ -456,8 +472,8 @@ impl<'a> Entity<'a> {
         d: &mut MaybeInit<Entity<'a>>,
     ) {
         let tok = state.current();
-        let is_full = Entity::tokens_with_full(tok);
-        let is_single = Entity::tokens_with_single(tok);
+        let is_full = state.cursor_signature_match_entity_full_rounded();
+        let is_single = state.cursor_has_ident_rounded();
         unsafe {
             if is_full {
                 state.cursor_ahead_by(3);
@@ -531,6 +547,7 @@ pub enum Statement<'a> {
     Delete(dml::delete::DeleteStatement<'a>),
 }
 
+#[inline(always)]
 pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], d: Qd) -> LangResult<Statement<'a>> {
     if compiler::unlikely(tok.len() < 2) {
         return Err(LangError::UnexpectedEndofStatement);
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index b24c1c3a..050f560e 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -79,26 +79,31 @@ mod lexer {
 }
 
 mod ast {
-    use {super::*, crate::engine::ql::ast::Entity};
+    use {
+        super::*,
+        crate::engine::ql::ast::{Entity, InplaceData, State},
+    };
     #[bench]
     fn parse_entity_single(b: &mut Bencher) {
-        let e = Entity::Single(b"tweeter");
+        let e = Entity::Single(b"user");
         b.iter(|| {
-            let src = lex_insecure(b"tweeter").unwrap();
-            let mut i = 0;
-            assert_eq!(Entity::parse_from_tokens(&src, &mut i).unwrap(), e);
-            assert_eq!(i, src.len());
-        });
+            let src = lex_insecure(b"user").unwrap();
+            let mut state = State::new(&src, InplaceData::new());
+            let re = Entity::attempt_process_entity_result(&mut state).unwrap();
+            assert_eq!(e, re);
+            assert!(state.exhausted());
+        })
     }
     #[bench]
     fn parse_entity_double(b: &mut Bencher) {
         let e = Entity::Full(b"tweeter", b"user");
         b.iter(|| {
             let src = lex_insecure(b"tweeter.user").unwrap();
-            let mut i = 0;
-            assert_eq!(Entity::parse_from_tokens(&src, &mut i).unwrap(), e);
-            assert_eq!(i, src.len());
-        });
+            let mut state = State::new(&src, InplaceData::new());
+            let re = Entity::attempt_process_entity_result(&mut state).unwrap();
+            assert_eq!(e, re);
+            assert!(state.exhausted());
+        })
     }
     #[bench]
     fn parse_entity_partial(b: &mut Bencher) {
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lexer.rs
index 936b66a3..7235cd59 100644
--- a/server/src/engine/ql/lexer.rs
+++ b/server/src/engine/ql/lexer.rs
@@ -956,6 +956,10 @@ impl<'a> SafeQueryData<'a> {
         Self { p, t }
     }
     #[inline(always)]
+    pub fn parse_data(pf: Slice<'a>, pf_sz: usize) -> LangResult<Box<[LitIR<'a>]>> {
+        Self::p_revloop(pf, pf_sz)
+    }
+    #[inline(always)]
     pub fn parse(qf: Slice<'a>, pf: Slice<'a>, pf_sz: usize) -> LangResult<Self> {
         let q = SafeLexer::lex(qf);
         let p = Self::p_revloop(pf, pf_sz);
@@ -1043,7 +1047,7 @@ impl<'b> SafeQueryData<'b> {
         let mut okay = true;
         let payload = Self::mxple(src, cnt, &mut okay);
         match String::from_utf8_lossy(payload).parse() {
-            Ok(p) if okay => {
+            Ok(p) if compiler::likely(okay) => {
                 data.push(LitIR::Float(p));
             }
             _ => {}
@@ -1062,7 +1066,7 @@ impl<'b> SafeQueryData<'b> {
         let mut okay = true;
         let payload = Self::mxple(src, cnt, &mut okay);
         match str::from_utf8(payload) {
-            Ok(s) if okay => {
+            Ok(s) if compiler::likely(okay) => {
                 data.push(LitIR::Str(s));
                 true
             }

From ee32df091651cefa932c87f35fe2c102c8450854 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 12 Jan 2023 14:22:35 +0000
Subject: [PATCH 073/310] Add iterator definitions to ME trait

---
 server/src/engine/memory/core.rs | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/server/src/engine/memory/core.rs b/server/src/engine/memory/core.rs
index 8e41e8fc..5e34bc46 100644
--- a/server/src/engine/memory/core.rs
+++ b/server/src/engine/memory/core.rs
@@ -69,16 +69,24 @@ where
     K: AsKey,
     V: AsValue,
 {
+    /// State whether the underlying structure provides any ordering on the iterators
     const HAS_ORDER: bool;
+    /// An iterator over the keys and values
     type IterKV<'a>: Iterator<Item = (&'a K, &'a V)>
     where
         Self: 'a,
         K: 'a,
         V: 'a;
+    /// An iterator over the keys
     type IterKey<'a>: Iterator<Item = &'a K>
     where
         Self: 'a,
         K: 'a;
+    /// An iterator over the values
+    type IterValue<'a>: Iterator<Item = &'a V>
+    where
+        Self: 'a,
+        V: 'a;
     // write
     /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
     /// violated
@@ -113,4 +121,12 @@ where
     fn delete_return<Q>(&self, key: &Q) -> Option<K>
     where
         K: Borrow<Q>;
+
+    // iter
+    /// Returns an iterator over a tuple of keys and values
+    fn iter_kv<'a>(&'a self) -> Self::IterKV<'a>;
+    /// Returns an iterator over the keys
+    fn iter_k<'a>(&'a self) -> Self::IterKey<'a>;
+    /// Returns an iterator over the values
+    fn iter_v<'a>(&'a self) -> Self::IterValue<'a>;
 }

From 603ea85e8c4a17cfa13ecb02d88d0ee672645823 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 16 Jan 2023 09:29:20 -0800
Subject: [PATCH 074/310] Add index definitions

---
 .../engine/{memory/core.rs => core/def.rs}    |  89 +++++++--
 server/src/engine/core/idx.rs                 | 184 ++++++++++++++++++
 server/src/engine/{memory => core}/mod.rs     |  18 +-
 server/src/engine/mod.rs                      |   2 +-
 server/src/engine/ql/ast.rs                   |   2 +-
 server/src/engine/ql/dml/insert.rs            |   2 +-
 server/src/engine/ql/tests.rs                 |   2 +-
 server/src/util/mod.rs                        |   7 +-
 8 files changed, 281 insertions(+), 25 deletions(-)
 rename server/src/engine/{memory/core.rs => core/def.rs} (56%)
 create mode 100644 server/src/engine/core/idx.rs
 rename server/src/engine/{memory => core}/mod.rs (92%)

diff --git a/server/src/engine/memory/core.rs b/server/src/engine/core/def.rs
similarity index 56%
rename from server/src/engine/memory/core.rs
rename to server/src/engine/core/def.rs
index 5e34bc46..f280f023 100644
--- a/server/src/engine/memory/core.rs
+++ b/server/src/engine/core/def.rs
@@ -64,7 +64,7 @@ impl<T: Clone> AsValue for T {
     }
 }
 
-pub trait MemoryEngine<K, V>
+pub trait MTIndex<K, V>
 where
     K: AsKey,
     V: AsValue,
@@ -90,43 +90,110 @@ where
     // write
     /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
     /// violated
-    fn insert(&self, key: K, val: V) -> bool;
+    fn me_insert(&self, key: K, val: V) -> bool;
+    /// Updates or inserts the given value
+    fn me_upsert(&self, key: K, val: V);
+
+    // read
     /// Returns a reference to the value corresponding to the key, if it exists
+    fn me_get<Q>(&self, key: &Q) -> Option<&K>
+    where
+        K: Borrow<Q>;
+    /// Returns a clone of the value corresponding to the key, if it exists
+    fn me_get_cloned<Q>(&self, key: &Q) -> Option<K>
+    where
+        K: Borrow<Q>;
+
+    // update
+    /// Returns true if the entry is updated
+    fn me_update<Q>(&self, key: &Q, val: V) -> bool
+    where
+        K: Borrow<Q>;
+    /// Updates the entry and returns the old value, if it exists
+    fn me_update_return<Q>(&self, key: &Q, val: V) -> Option<K>
+    where
+        K: Borrow<Q>;
+
+    // delete
+    /// Returns true if the entry was deleted
+    fn me_delete<Q>(&self, key: &Q) -> bool
+    where
+        K: Borrow<Q>;
+    /// Removes the entry and returns it, if it exists
+    fn me_delete_return<Q>(&self, key: &Q) -> Option<K>
+    where
+        K: Borrow<Q>;
+
+    // iter
+    /// Returns an iterator over a tuple of keys and values
+    fn me_iter_kv<'a>(&'a self) -> Self::IterKV<'a>;
+    /// Returns an iterator over the keys
+    fn me_iter_k<'a>(&'a self) -> Self::IterKey<'a>;
+    /// Returns an iterator over the values
+    fn me_iter_v<'a>(&'a self) -> Self::IterValue<'a>;
+}
+
+pub trait STIndex<K, V> {
+    /// State whether the underlying structure provides any ordering on the iterators
+    const HAS_ORDER: bool;
+    /// An iterator over the keys and values
+    type IterKV<'a>: Iterator<Item = (&'a K, &'a V)>
+    where
+        Self: 'a,
+        K: 'a,
+        V: 'a;
+    /// An iterator over the keys
+    type IterKey<'a>: Iterator<Item = &'a K>
+    where
+        Self: 'a,
+        K: 'a;
+    /// An iterator over the values
+    type IterValue<'a>: Iterator<Item = &'a V>
+    where
+        Self: 'a,
+        V: 'a;
+    // write
+    /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
+    /// violated
+    fn st_insert(&mut self, key: K, val: V) -> bool;
+    /// Updates or inserts the given value
+    fn st_upsert(&mut self, key: K, val: V);
 
     // read
-    fn get<Q>(&self, key: &Q) -> Option<&K>
+    /// Returns a reference to the value corresponding to the key, if it exists
+    fn st_get<Q>(&self, key: &Q) -> Option<&K>
     where
         K: Borrow<Q>;
     /// Returns a clone of the value corresponding to the key, if it exists
-    fn get_cloned<Q>(&self, key: &Q) -> Option<K>
+    fn st_get_cloned<Q>(&self, key: &Q) -> Option<K>
     where
         K: Borrow<Q>;
 
     // update
     /// Returns true if the entry is updated
-    fn update<Q>(&self, key: &Q, val: V) -> bool
+    fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
     where
         K: Borrow<Q>;
     /// Updates the entry and returns the old value, if it exists
-    fn update_return<Q>(&self, key: &Q, val: V) -> Option<K>
+    fn st_update_return<Q>(&mut self, key: &Q, val: V) -> Option<K>
     where
         K: Borrow<Q>;
 
     // delete
     /// Returns true if the entry was deleted
-    fn delete<Q>(&self, key: &Q) -> bool
+    fn st_delete<Q>(&mut self, key: &Q) -> bool
     where
         K: Borrow<Q>;
     /// Removes the entry and returns it, if it exists
-    fn delete_return<Q>(&self, key: &Q) -> Option<K>
+    fn st_delete_return<Q>(&mut self, key: &Q) -> Option<K>
     where
         K: Borrow<Q>;
 
     // iter
     /// Returns an iterator over a tuple of keys and values
-    fn iter_kv<'a>(&'a self) -> Self::IterKV<'a>;
+    fn st_iter_kv<'a>(&'a self) -> Self::IterKV<'a>;
     /// Returns an iterator over the keys
-    fn iter_k<'a>(&'a self) -> Self::IterKey<'a>;
+    fn st_iter_k<'a>(&'a self) -> Self::IterKey<'a>;
     /// Returns an iterator over the values
-    fn iter_v<'a>(&'a self) -> Self::IterValue<'a>;
+    fn st_iter_v<'a>(&'a self) -> Self::IterValue<'a>;
 }
diff --git a/server/src/engine/core/idx.rs b/server/src/engine/core/idx.rs
new file mode 100644
index 00000000..f0d0c57d
--- /dev/null
+++ b/server/src/engine/core/idx.rs
@@ -0,0 +1,184 @@
+/*
+ * Created on Mon Jan 16 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use std::{
+    alloc::{alloc as std_alloc, dealloc as std_dealloc, Layout},
+    borrow::Borrow,
+    collections::HashMap as StdMap,
+    hash::{Hash, Hasher},
+    mem,
+    ptr::{self, NonNull},
+};
+
+/*
+    For the ordered index impl, we resort to some crazy unsafe code, especially because there's no other way to
+    deal with non-primitive Ks. That's why we'll ENTIRELY AVOID exporting any structures; if we end up using a node
+    or a ptr struct anywhere inappropriate, it'll most likely SEGFAULT. So yeah, better be careful with this one.
+    Second note, I'm not a big fan of the DLL and will most likely try a different approach in the future; this one
+    is the most convenient option for now.
+
+    -- Sayan (@ohsayan) // Jan. 16 '23
+*/
+
+#[derive(Debug)]
+#[repr(transparent)]
+/// # WARNING: Segfault/UAF alert
+///
+/// Yeah, this type is going to segfault if you decide to use it in random places. Literally, don't use it if
+/// you're unsure about it's validity. For example, if you simply `==` this or attempt to use it an a hashmap,
+/// you can segfault. IFF, the ptr is valid will it not segfault
+struct Keyptr<K> {
+    p: *mut K,
+}
+
+impl<K: Hash> Hash for Keyptr<K> {
+    #[inline(always)]
+    fn hash<H>(&self, state: &mut H)
+    where
+        H: Hasher,
+    {
+        unsafe {
+            /*
+                UNSAFE(@ohsayan): BAD. THIS IS NOT SAFE, but dang it, it's the only way we can do this without
+                dynamic rule checking. I wish there was a `'self` lifetime
+            */
+            (*self.p).hash(state)
+        }
+    }
+}
+
+impl<K: PartialEq> PartialEq for Keyptr<K> {
+    #[inline(always)]
+    fn eq(&self, other: &Self) -> bool {
+        unsafe {
+            /*
+                UNSAFE(@ohsayan): BAD. THIS IS NOT SAFE, but dang it, it's the only way we can do this without
+                dynamic rule checking. I wish there was a `'self` lifetime
+            */
+            (*self.p).eq(&*other.p)
+        }
+    }
+}
+
+// stupid type for trait impl conflict riddance
+#[derive(Debug, Hash, PartialEq)]
+#[repr(transparent)]
+struct Qref<Q: ?Sized>(Q);
+
+impl<Q: ?Sized> Qref<Q> {
+    #[inline(always)]
+    unsafe fn from_ref(r: &Q) -> &Self {
+        mem::transmute(r)
+    }
+}
+
+impl<K, Q> Borrow<Qref<Q>> for Keyptr<K>
+where
+    K: Borrow<Q>,
+    Q: ?Sized,
+{
+    #[inline(always)]
+    fn borrow(&self) -> &Qref<Q> {
+        unsafe {
+            /*
+                UNSAFE(@ohsayan): BAD. This deref ain't safe either. ref is good though
+            */
+            Qref::from_ref((*self.p).borrow())
+        }
+    }
+}
+
+#[derive(Debug)]
+struct Node<K, V> {
+    k: K,
+    v: V,
+    n: *mut Self,
+    p: *mut Self,
+}
+
+impl<K, V> Node<K, V> {
+    const LAYOUT: Layout = Layout::new::<Self>();
+    #[inline(always)]
+    fn new(k: K, v: V, n: *mut Self, p: *mut Self) -> Self {
+        Self { k, v, n, p }
+    }
+    #[inline(always)]
+    fn new_null(k: K, v: V) -> Self {
+        Self::new(k, v, ptr::null_mut(), ptr::null_mut())
+    }
+    #[inline(always)]
+    fn _alloc<const WPTR_N: bool, const WPTR_P: bool>(Self { k, v, p, n }: Self) -> *mut Self {
+        unsafe {
+            // UNSAFE(@ohsayan): grow up, it's a malloc
+            let ptr = std_alloc(Self::LAYOUT) as *mut Self;
+            assert!(ptr.is_null(), "damn the allocator failed");
+            (*ptr).k = k;
+            (*ptr).v = v;
+            if WPTR_N {
+                (*ptr).n = n;
+            }
+            if WPTR_P {
+                (*ptr).p = p;
+            }
+            ptr
+        }
+    }
+    #[inline(always)]
+    fn alloc_null(k: K, v: V) -> *mut Self {
+        Self::_alloc::<false, false>(Self::new_null(k, v))
+    }
+    #[inline(always)]
+    fn alloc(k: K, v: V, p: *mut Self, n: *mut Self) -> *mut Self {
+        Self::_alloc::<true, true>(Self::new(k, v, p, n))
+    }
+    #[inline(always)]
+    unsafe fn dealloc(slf: *mut Self) {
+        let _ = Box::from_raw(slf);
+    }
+    #[inline(always)]
+    /// LEAK: K, V
+    unsafe fn dealloc_headless(slf: *mut Self) {
+        std_dealloc(slf as *mut u8, Self::LAYOUT)
+    }
+    #[inline(always)]
+    unsafe fn unlink(node: *mut Self) {
+        (*((*node).p)).n = (*node).n;
+        (*((*node).n)).p = (*node).p;
+    }
+    #[inline(always)]
+    unsafe fn link(from: *mut Self, to: *mut Self) {
+        (*to).n = (*from).n;
+        (*to).p = from;
+        (*from).n = to;
+        (*(*to).n).p = to;
+    }
+}
+
+pub struct OrdMap<K, V, S> {
+    m: StdMap<Keyptr<K>, NonNull<Node<K, V>>, S>,
+    h: *mut Node<K, V>,
+    f: *mut Node<K, V>,
+}
diff --git a/server/src/engine/memory/mod.rs b/server/src/engine/core/mod.rs
similarity index 92%
rename from server/src/engine/memory/mod.rs
rename to server/src/engine/core/mod.rs
index 3c126b4a..c7f973d7 100644
--- a/server/src/engine/memory/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -26,37 +26,39 @@
 
 // TODO(@ohsayan): Change the underlying structures, there are just rudimentary ones used during integration with the QL
 
-mod core;
+mod def;
+mod idx;
 use super::ql::lexer::{Lit, LitIR};
 
 /// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
 /// be of one type.
 #[derive(Debug, PartialEq, Clone)]
+#[repr(u8)]
 pub enum DataType {
     /// An UTF-8 string
-    String(Box<str>),
+    String(Box<str>) = DataKind::STR_BX.d(),
     /// Bytes
-    Binary(Box<[u8]>),
+    Binary(Box<[u8]>) = DataKind::BIN_BX.d(),
     /// An unsigned integer
     ///
     /// **NOTE:** This is the default evaluated type for unsigned integers by the query processor. It is the
     /// responsibility of the executor to ensure integrity checks depending on actual type width in the declared
     /// schema (if any)
-    UnsignedInt(u64),
+    UnsignedInt(u64) = DataKind::UINT64.d(),
     /// A signed integer
     ///
     /// **NOTE:** This is the default evaluated type for signed integers by the query processor. It is the
     /// responsibility of the executor to ensure integrity checks depending on actual type width in the declared
     /// schema (if any)
-    SignedInt(i64),
+    SignedInt(i64) = DataKind::SINT64.d(),
     /// A boolean
-    Boolean(bool),
+    Boolean(bool) = DataKind::BOOL.d(),
     /// A float (64-bit)
-    Float(f64),
+    Float(f64) = DataKind::FLOAT64.d(),
     /// A single-type list. Note, you **need** to keep up the invariant that the [`DataType`] disc. remains the same for all
     /// elements to ensure correctness in this specific context
     /// FIXME(@ohsayan): Try enforcing this somehow
-    List(Vec<Self>),
+    List(Vec<Self>) = DataKind::LIST.d(),
 }
 
 enum_impls! {
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index b1e0940f..e1227466 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -29,5 +29,5 @@
 
 #[macro_use]
 mod macros;
-mod memory;
+mod core;
 mod ql;
diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index acbb0278..3de85036 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -31,7 +31,7 @@ use {
         schema, LangError, LangResult,
     },
     crate::{
-        engine::memory::DataType,
+        engine::core::DataType,
         util::{compiler, MaybeInit},
     },
     core::cmp,
diff --git a/server/src/engine/ql/dml/insert.rs b/server/src/engine/ql/dml/insert.rs
index 4852afc4..2074d123 100644
--- a/server/src/engine/ql/dml/insert.rs
+++ b/server/src/engine/ql/dml/insert.rs
@@ -30,7 +30,7 @@ use {
     super::read_ident,
     crate::{
         engine::{
-            memory::DataType,
+            core::DataType,
             ql::{
                 ast::{Entity, QueryData, State},
                 lexer::Token,
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index a8bec0ac..e3c71fa3 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -29,7 +29,7 @@ use {
         lexer::{InsecureLexer, SafeLexer, Symbol, Token},
         LangResult,
     },
-    crate::{engine::memory::DataType, util::test_utils},
+    crate::{engine::core::DataType, util::test_utils},
     rand::{self, Rng},
 };
 
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index 252d4f53..c5f899f9 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -144,6 +144,7 @@ impl<T: Clone> Clone for Wrapper<T> {
 }
 
 #[derive(Debug, PartialEq)]
+#[repr(transparent)]
 /// This is yet another compiler hack and has no "actual impact" in terms of memory alignment.
 ///
 /// When it's hard to have a _split mutable borrow_, all across the source we use custom
@@ -163,18 +164,19 @@ impl<T: Clone> Clone for Wrapper<T> {
 /// to explicitly annotate bounds
 /// - this type derefs to the base type
 #[derive(Copy, Clone)]
-pub struct Life<'a, T> {
+pub struct Life<'a, T: 'a> {
     _lt: PhantomData<&'a T>,
     v: T,
 }
 
-impl<'a, T> Life<'a, T> {
+impl<'a, T: 'a> Life<'a, T> {
     /// Ensure compile-time alignment (this is just a sanity check)
     const _ENSURE_COMPILETIME_ALIGN: () =
         assert!(std::mem::align_of::<Life<T>>() == std::mem::align_of::<T>());
 
     #[inline(always)]
     pub const fn new(v: T) -> Self {
+        let _ = Self::_ENSURE_COMPILETIME_ALIGN;
         Life {
             v,
             _lt: PhantomData,
@@ -222,6 +224,7 @@ unsafe impl<'a, T: Sync> Sync for Life<'a, T> {}
 /// [`MaybeInit`] is a structure that is like an [`Option`] in debug mode and like
 /// [`MaybeUninit`] in release mode. This means that provided there are good enough test cases, most
 /// incorrect `assume_init` calls should be detected in the test phase.
+#[cfg_attr(not(test), repr(transparent))]
 pub struct MaybeInit<T> {
     #[cfg(test)]
     is_init: bool,

From 530e76eca4f751b06facfd0401555f64f0d8ed1f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 17 Jan 2023 01:19:44 -0800
Subject: [PATCH 075/310] Add primitive methods to ord idx impl

---
 server/src/engine/core/def.rs |  34 ++++++---
 server/src/engine/core/idx.rs | 133 +++++++++++++++++++++++++++++-----
 2 files changed, 138 insertions(+), 29 deletions(-)

diff --git a/server/src/engine/core/def.rs b/server/src/engine/core/def.rs
index f280f023..8b6ca346 100644
--- a/server/src/engine/core/def.rs
+++ b/server/src/engine/core/def.rs
@@ -71,6 +71,8 @@ where
 {
     /// State whether the underlying structure provides any ordering on the iterators
     const HAS_ORDER: bool;
+    /// Base support prealloc?
+    const PREALLOC: bool;
     /// An iterator over the keys and values
     type IterKV<'a>: Iterator<Item = (&'a K, &'a V)>
     where
@@ -87,55 +89,61 @@ where
     where
         Self: 'a,
         V: 'a;
+    /// Initialize an empty instance of the MTIndex
+    fn mt_init() -> Self;
+    /// Initialize a pre-loaded instance of the MTIndex
+    fn mt_init_with(s: Self) -> Self;
     // write
     /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
     /// violated
-    fn me_insert(&self, key: K, val: V) -> bool;
+    fn mt_insert(&self, key: K, val: V) -> bool;
     /// Updates or inserts the given value
-    fn me_upsert(&self, key: K, val: V);
+    fn mt_upsert(&self, key: K, val: V);
 
     // read
     /// Returns a reference to the value corresponding to the key, if it exists
-    fn me_get<Q>(&self, key: &Q) -> Option<&K>
+    fn mt_get<Q>(&self, key: &Q) -> Option<&K>
     where
         K: Borrow<Q>;
     /// Returns a clone of the value corresponding to the key, if it exists
-    fn me_get_cloned<Q>(&self, key: &Q) -> Option<K>
+    fn mt_get_cloned<Q>(&self, key: &Q) -> Option<K>
     where
         K: Borrow<Q>;
 
     // update
     /// Returns true if the entry is updated
-    fn me_update<Q>(&self, key: &Q, val: V) -> bool
+    fn mt_update<Q>(&self, key: &Q, val: V) -> bool
     where
         K: Borrow<Q>;
     /// Updates the entry and returns the old value, if it exists
-    fn me_update_return<Q>(&self, key: &Q, val: V) -> Option<K>
+    fn mt_update_return<Q>(&self, key: &Q, val: V) -> Option<K>
     where
         K: Borrow<Q>;
 
     // delete
     /// Returns true if the entry was deleted
-    fn me_delete<Q>(&self, key: &Q) -> bool
+    fn mt_delete<Q>(&self, key: &Q) -> bool
     where
         K: Borrow<Q>;
     /// Removes the entry and returns it, if it exists
-    fn me_delete_return<Q>(&self, key: &Q) -> Option<K>
+    fn mt_delete_return<Q>(&self, key: &Q) -> Option<K>
     where
         K: Borrow<Q>;
 
     // iter
     /// Returns an iterator over a tuple of keys and values
-    fn me_iter_kv<'a>(&'a self) -> Self::IterKV<'a>;
+    fn mt_iter_kv<'a>(&'a self) -> Self::IterKV<'a>;
     /// Returns an iterator over the keys
-    fn me_iter_k<'a>(&'a self) -> Self::IterKey<'a>;
+    fn mt_iter_k<'a>(&'a self) -> Self::IterKey<'a>;
     /// Returns an iterator over the values
-    fn me_iter_v<'a>(&'a self) -> Self::IterValue<'a>;
+    fn mt_iter_v<'a>(&'a self) -> Self::IterValue<'a>;
 }
 
 pub trait STIndex<K, V> {
     /// State whether the underlying structure provides any ordering on the iterators
     const HAS_ORDER: bool;
+    /// Base support prealloc?
+    const PREALLOC: bool;
     /// An iterator over the keys and values
     type IterKV<'a>: Iterator<Item = (&'a K, &'a V)>
     where
@@ -152,6 +160,10 @@ pub trait STIndex<K, V> {
     where
         Self: 'a,
         V: 'a;
+    /// Initialize an empty instance of the MTIndex
+    fn st_init() -> Self;
+    /// Initialize a pre-loaded instance of the MTIndex
+    fn st_init_with(s: Self) -> Self;
     // write
     /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
     /// violated
diff --git a/server/src/engine/core/idx.rs b/server/src/engine/core/idx.rs
index f0d0c57d..5ca05a27 100644
--- a/server/src/engine/core/idx.rs
+++ b/server/src/engine/core/idx.rs
@@ -24,15 +24,19 @@
  *
 */
 
+use super::def::{AsKey, AsValue};
 use std::{
     alloc::{alloc as std_alloc, dealloc as std_dealloc, Layout},
     borrow::Borrow,
-    collections::HashMap as StdMap,
-    hash::{Hash, Hasher},
+    collections::{hash_map::RandomState, HashMap as StdMap},
+    hash::{BuildHasher, Hash, Hasher},
     mem,
     ptr::{self, NonNull},
 };
 
+// re-exports for convenience
+pub type IndexSTSeq<K, V, S = RandomState> = IndexSTSeqDll<K, V, S>;
+
 /*
     For the ordered index impl, we resort to some crazy unsafe code, especially because there's no other way to
     deal with non-primitive Ks. That's why we'll ENTIRELY AVOID exporting any structures; if we end up using a node
@@ -50,11 +54,11 @@ use std::{
 /// Yeah, this type is going to segfault if you decide to use it in random places. Literally, don't use it if
 /// you're unsure about it's validity. For example, if you simply `==` this or attempt to use it an a hashmap,
 /// you can segfault. IFF, the ptr is valid will it not segfault
-struct Keyptr<K> {
+struct IndexSTSeqDllKeyptr<K> {
     p: *mut K,
 }
 
-impl<K: Hash> Hash for Keyptr<K> {
+impl<K: Hash> Hash for IndexSTSeqDllKeyptr<K> {
     #[inline(always)]
     fn hash<H>(&self, state: &mut H)
     where
@@ -70,7 +74,7 @@ impl<K: Hash> Hash for Keyptr<K> {
     }
 }
 
-impl<K: PartialEq> PartialEq for Keyptr<K> {
+impl<K: PartialEq> PartialEq for IndexSTSeqDllKeyptr<K> {
     #[inline(always)]
     fn eq(&self, other: &Self) -> bool {
         unsafe {
@@ -83,43 +87,45 @@ impl<K: PartialEq> PartialEq for Keyptr<K> {
     }
 }
 
+impl<K: Eq> Eq for IndexSTSeqDllKeyptr<K> {}
+
 // stupid type for trait impl conflict riddance
 #[derive(Debug, Hash, PartialEq)]
 #[repr(transparent)]
-struct Qref<Q: ?Sized>(Q);
+struct IndexSTSeqDllQref<Q: ?Sized>(Q);
 
-impl<Q: ?Sized> Qref<Q> {
+impl<Q: ?Sized> IndexSTSeqDllQref<Q> {
     #[inline(always)]
     unsafe fn from_ref(r: &Q) -> &Self {
         mem::transmute(r)
     }
 }
 
-impl<K, Q> Borrow<Qref<Q>> for Keyptr<K>
+impl<K, Q> Borrow<IndexSTSeqDllQref<Q>> for IndexSTSeqDllKeyptr<K>
 where
     K: Borrow<Q>,
     Q: ?Sized,
 {
     #[inline(always)]
-    fn borrow(&self) -> &Qref<Q> {
+    fn borrow(&self) -> &IndexSTSeqDllQref<Q> {
         unsafe {
             /*
                 UNSAFE(@ohsayan): BAD. This deref ain't safe either. ref is good though
             */
-            Qref::from_ref((*self.p).borrow())
+            IndexSTSeqDllQref::from_ref((*self.p).borrow())
         }
     }
 }
 
 #[derive(Debug)]
-struct Node<K, V> {
+struct IndexSTSeqDllNode<K, V> {
     k: K,
     v: V,
     n: *mut Self,
     p: *mut Self,
 }
 
-impl<K, V> Node<K, V> {
+impl<K, V> IndexSTSeqDllNode<K, V> {
     const LAYOUT: Layout = Layout::new::<Self>();
     #[inline(always)]
     fn new(k: K, v: V, n: *mut Self, p: *mut Self) -> Self {
@@ -130,11 +136,19 @@ impl<K, V> Node<K, V> {
         Self::new(k, v, ptr::null_mut(), ptr::null_mut())
     }
     #[inline(always)]
-    fn _alloc<const WPTR_N: bool, const WPTR_P: bool>(Self { k, v, p, n }: Self) -> *mut Self {
+    fn _alloc_with_garbage() -> *mut Self {
         unsafe {
-            // UNSAFE(@ohsayan): grow up, it's a malloc
+            // UNSAFE(@ohsayan): aight shut up, it's a malloc
             let ptr = std_alloc(Self::LAYOUT) as *mut Self;
             assert!(ptr.is_null(), "damn the allocator failed");
+            ptr
+        }
+    }
+    #[inline(always)]
+    fn _alloc<const WPTR_N: bool, const WPTR_P: bool>(Self { k, v, p, n }: Self) -> *mut Self {
+        unsafe {
+            // UNSAFE(@ohsayan): grow up, we're writing to a fresh block
+            let ptr = Self::_alloc_with_garbage();
             (*ptr).k = k;
             (*ptr).v = v;
             if WPTR_N {
@@ -177,8 +191,91 @@ impl<K, V> Node<K, V> {
     }
 }
 
-pub struct OrdMap<K, V, S> {
-    m: StdMap<Keyptr<K>, NonNull<Node<K, V>>, S>,
-    h: *mut Node<K, V>,
-    f: *mut Node<K, V>,
+type IndexSTSeqDllNodePtr<K, V> = NonNull<IndexSTSeqDllNode<K, V>>;
+
+/// An ST-index with ordering. Inefficient ordered scanning since not in block
+pub struct IndexSTSeqDll<K, V, S> {
+    m: StdMap<IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>, S>,
+    h: *mut IndexSTSeqDllNode<K, V>,
+    f: *mut IndexSTSeqDllNode<K, V>,
+}
+
+impl<K, V, S: BuildHasher> IndexSTSeqDll<K, V, S> {
+    const DEF_CAP: usize = 0;
+    #[inline(always)]
+    fn _new(
+        m: StdMap<IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>, S>,
+        h: *mut IndexSTSeqDllNode<K, V>,
+        f: *mut IndexSTSeqDllNode<K, V>,
+    ) -> IndexSTSeqDll<K, V, S> {
+        Self { m, h, f }
+    }
+    #[inline(always)]
+    fn _new_map(m: StdMap<IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>, S>) -> Self {
+        Self::_new(m, ptr::null_mut(), ptr::null_mut())
+    }
+    #[inline(always)]
+    pub fn with_hasher(hasher: S) -> Self {
+        Self::with_capacity_and_hasher(Self::DEF_CAP, hasher)
+    }
+    #[inline(always)]
+    pub fn with_capacity_and_hasher(cap: usize, hasher: S) -> Self {
+        Self::_new_map(StdMap::with_capacity_and_hasher(cap, hasher))
+    }
+}
+
+impl<K, V> IndexSTSeqDll<K, V, RandomState> {
+    #[inline(always)]
+    pub fn new() -> Self {
+        Self::with_capacity(Self::DEF_CAP)
+    }
+    #[inline(always)]
+    pub fn with_capacity(cap: usize) -> Self {
+        Self::with_capacity_and_hasher(cap, RandomState::default())
+    }
+}
+
+impl<K, V, S> IndexSTSeqDll<K, V, S> {
+    #[inline(always)]
+    fn ensure_sentinel(&mut self) {
+        let ptr = IndexSTSeqDllNode::_alloc_with_garbage();
+        unsafe {
+            self.h = ptr;
+            (*ptr).p = ptr;
+            (*ptr).n = ptr;
+        }
+    }
+    #[inline(always)]
+    /// ## Safety
+    ///
+    /// Head must not be null
+    unsafe fn drop_nodes_full(&mut self) {
+        let mut c = self.h;
+        while !c.is_null() {
+            let nx = (*c).n;
+            IndexSTSeqDllNode::dealloc(c);
+            c = nx;
+        }
+    }
+    #[inline(always)]
+    fn vacuum_free(&mut self) {
+        unsafe {
+            let mut c = self.f;
+            while !c.is_null() {
+                let nx = (*c).n;
+                IndexSTSeqDllNode::dealloc_headless(nx);
+                c = nx;
+            }
+        }
+        self.f = ptr::null_mut();
+    }
+}
+
+impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
+    #[inline(always)]
+    /// Clean up unused and cached memory
+    fn vacuum_full(&mut self) {
+        self.m.shrink_to_fit();
+        self.vacuum_free();
+    }
 }

From b5ec9da926f8f8ea1064922a98f85a8e3a90a2be Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 18 Jan 2023 04:37:58 -0800
Subject: [PATCH 076/310] Add ord index impls and fix UB

Added all basic impls for the ord index, also fixed dealloc on a nullptr
in drop impl
---
 server/src/engine/core/def.rs |   7 +
 server/src/engine/core/idx.rs | 383 ++++++++++++++++++++++++++++++++--
 2 files changed, 374 insertions(+), 16 deletions(-)

diff --git a/server/src/engine/core/def.rs b/server/src/engine/core/def.rs
index 8b6ca346..f57639f0 100644
--- a/server/src/engine/core/def.rs
+++ b/server/src/engine/core/def.rs
@@ -45,6 +45,9 @@ impl<T: Hash + Eq + Clone> AsKey for T {
     }
 }
 
+pub trait AsKeyRef: Hash + Eq {}
+impl<T: Hash + Eq + ?Sized> AsKeyRef for T {}
+
 /// Any type implementing this trait can be used as a value inside memory engine structures
 pub trait AsValue: Clone {
     /// Read the value
@@ -93,6 +96,8 @@ where
     fn mt_init() -> Self;
     /// Initialize a pre-loaded instance of the MTIndex
     fn mt_init_with(s: Self) -> Self;
+    /// Clears all the entries in the MTIndex
+    fn mt_clear(&self);
     // write
     /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
     /// violated
@@ -164,6 +169,8 @@ pub trait STIndex<K, V> {
     fn st_init() -> Self;
     /// Initialize a pre-loaded instance of the MTIndex
     fn st_init_with(s: Self) -> Self;
+    /// Clears all the entries in the STIndex
+    fn st_clear(&mut self);
     // write
     /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
     /// violated
diff --git a/server/src/engine/core/idx.rs b/server/src/engine/core/idx.rs
index 5ca05a27..7605060a 100644
--- a/server/src/engine/core/idx.rs
+++ b/server/src/engine/core/idx.rs
@@ -24,12 +24,17 @@
  *
 */
 
-use super::def::{AsKey, AsValue};
+use super::def::{AsKey, AsKeyRef, AsValue};
 use std::{
     alloc::{alloc as std_alloc, dealloc as std_dealloc, Layout},
     borrow::Borrow,
-    collections::{hash_map::RandomState, HashMap as StdMap},
+    collections::{
+        hash_map::{Iter, Keys as StdMapIterKey, RandomState, Values as StdMapIterVal},
+        HashMap as StdMap,
+    },
+    fmt::{self, Debug},
     hash::{BuildHasher, Hash, Hasher},
+    iter::FusedIterator,
     mem,
     ptr::{self, NonNull},
 };
@@ -47,7 +52,6 @@ pub type IndexSTSeq<K, V, S = RandomState> = IndexSTSeqDll<K, V, S>;
     -- Sayan (@ohsayan) // Jan. 16 '23
 */
 
-#[derive(Debug)]
 #[repr(transparent)]
 /// # WARNING: Segfault/UAF alert
 ///
@@ -55,7 +59,14 @@ pub type IndexSTSeq<K, V, S = RandomState> = IndexSTSeqDll<K, V, S>;
 /// you're unsure about it's validity. For example, if you simply `==` this or attempt to use it an a hashmap,
 /// you can segfault. IFF, the ptr is valid will it not segfault
 struct IndexSTSeqDllKeyptr<K> {
-    p: *mut K,
+    p: *const K,
+}
+
+impl<K> IndexSTSeqDllKeyptr<K> {
+    #[inline(always)]
+    fn new(r: &K) -> Self {
+        Self { p: r as *const _ }
+    }
 }
 
 impl<K: Hash> Hash for IndexSTSeqDllKeyptr<K> {
@@ -90,7 +101,7 @@ impl<K: PartialEq> PartialEq for IndexSTSeqDllKeyptr<K> {
 impl<K: Eq> Eq for IndexSTSeqDllKeyptr<K> {}
 
 // stupid type for trait impl conflict riddance
-#[derive(Debug, Hash, PartialEq)]
+#[derive(Debug, Hash, PartialEq, Eq)]
 #[repr(transparent)]
 struct IndexSTSeqDllQref<Q: ?Sized>(Q);
 
@@ -140,7 +151,7 @@ impl<K, V> IndexSTSeqDllNode<K, V> {
         unsafe {
             // UNSAFE(@ohsayan): aight shut up, it's a malloc
             let ptr = std_alloc(Self::LAYOUT) as *mut Self;
-            assert!(ptr.is_null(), "damn the allocator failed");
+            assert!(!ptr.is_null(), "damn the allocator failed");
             ptr
         }
     }
@@ -169,7 +180,7 @@ impl<K, V> IndexSTSeqDllNode<K, V> {
         Self::_alloc::<true, true>(Self::new(k, v, p, n))
     }
     #[inline(always)]
-    unsafe fn dealloc(slf: *mut Self) {
+    unsafe fn _drop(slf: *mut Self) {
         let _ = Box::from_raw(slf);
     }
     #[inline(always)]
@@ -189,6 +200,13 @@ impl<K, V> IndexSTSeqDllNode<K, V> {
         (*from).n = to;
         (*(*to).n).p = to;
     }
+    #[inline(always)]
+    fn alloc_box(node: IndexSTSeqDllNode<K, V>) -> NonNull<IndexSTSeqDllNode<K, V>> {
+        unsafe {
+            // UNSAFE(@ohsayan): Safe because of box alloc
+            NonNull::new_unchecked(Box::into_raw(Box::new(node)))
+        }
+    }
 }
 
 type IndexSTSeqDllNodePtr<K, V> = NonNull<IndexSTSeqDllNode<K, V>>;
@@ -238,11 +256,14 @@ impl<K, V> IndexSTSeqDll<K, V, RandomState> {
 impl<K, V, S> IndexSTSeqDll<K, V, S> {
     #[inline(always)]
     fn ensure_sentinel(&mut self) {
-        let ptr = IndexSTSeqDllNode::_alloc_with_garbage();
-        unsafe {
-            self.h = ptr;
-            (*ptr).p = ptr;
-            (*ptr).n = ptr;
+        if self.h.is_null() {
+            let ptr = IndexSTSeqDllNode::_alloc_with_garbage();
+            unsafe {
+                //  UNSAFE(@ohsayan): Fresh alloc
+                self.h = ptr;
+                (*ptr).p = ptr;
+                (*ptr).n = ptr;
+            }
         }
     }
     #[inline(always)]
@@ -250,25 +271,48 @@ impl<K, V, S> IndexSTSeqDll<K, V, S> {
     ///
     /// Head must not be null
     unsafe fn drop_nodes_full(&mut self) {
-        let mut c = self.h;
-        while !c.is_null() {
+        // don't drop sentinenl
+        let mut c = (*self.h).n;
+        while c != self.h {
             let nx = (*c).n;
-            IndexSTSeqDllNode::dealloc(c);
+            IndexSTSeqDllNode::_drop(c);
             c = nx;
         }
     }
     #[inline(always)]
     fn vacuum_free(&mut self) {
         unsafe {
+            // UNSAFE(@ohsayan): All nullck
             let mut c = self.f;
             while !c.is_null() {
                 let nx = (*c).n;
-                IndexSTSeqDllNode::dealloc_headless(nx);
+                IndexSTSeqDllNode::dealloc_headless(c);
                 c = nx;
             }
         }
         self.f = ptr::null_mut();
     }
+    #[inline(always)]
+    fn recycle_or_alloc(&mut self, node: IndexSTSeqDllNode<K, V>) -> IndexSTSeqDllNodePtr<K, V> {
+        if self.f.is_null() {
+            IndexSTSeqDllNode::alloc_box(node)
+        } else {
+            unsafe {
+                // UNSAFE(@ohsayan): Safe because we already did a nullptr check
+                let f = self.f;
+                self.f = (*self.f).n;
+                ptr::write(f, node);
+                IndexSTSeqDllNodePtr::new_unchecked(f)
+            }
+        }
+    }
+    #[inline(always)]
+    /// NOTE: `&mut Self` for aliasing
+    /// ## Safety
+    /// Ensure head is non null
+    unsafe fn link(&mut self, node: IndexSTSeqDllNodePtr<K, V>) {
+        IndexSTSeqDllNode::link(self.h, node.as_ptr())
+    }
 }
 
 impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
@@ -279,3 +323,310 @@ impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
         self.vacuum_free();
     }
 }
+
+impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
+    const GET_REFRESH: bool = true;
+    const GET_BYPASS: bool = false;
+    #[inline(always)]
+    fn _insert(&mut self, k: K, v: V) -> bool {
+        if self.m.contains_key(&IndexSTSeqDllKeyptr::new(&k)) {
+            return false;
+        }
+        self.__insert(k, v)
+    }
+    fn _get<Q: ?Sized>(&self, k: &Q) -> Option<&V>
+    where
+        K: Borrow<Q>,
+        Q: AsKeyRef,
+    {
+        self.m
+            .get(unsafe {
+                // UNSAFE(@ohsayan): Ref with correct bounds
+                IndexSTSeqDllQref::from_ref(k)
+            })
+            .map(|e| unsafe {
+                // UNSAFE(@ohsayan): ref is non-null and ensures aliasing reqs
+                &(e.as_ref()).read_value().v
+            })
+    }
+    #[inline(always)]
+    fn _update<Q: ?Sized>(&mut self, k: &Q, v: V) -> Option<V>
+    where
+        K: Borrow<Q>,
+        Q: AsKeyRef,
+    {
+        match self.m.get(unsafe {
+            // UNSAFE(@ohsayan): Just got a ref with the right bounds
+            IndexSTSeqDllQref::from_ref(k)
+        }) {
+            Some(e) => unsafe {
+                // UNSAFE(@ohsayan): Impl guarantees that entry presence == nullck head
+                self.__update(*e, v)
+            },
+            None => return None,
+        }
+    }
+    #[inline(always)]
+    fn _upsert(&mut self, k: K, v: V) -> Option<V> {
+        match self.m.get(&IndexSTSeqDllKeyptr::new(&k)) {
+            Some(e) => unsafe {
+                // UNSAFE(@ohsayan): Impl guarantees that entry presence == nullck head
+                self.__update(*e, v)
+            },
+            None => {
+                let _ = self.__insert(k, v);
+                None
+            }
+        }
+    }
+    #[inline(always)]
+    fn _remove<Q>(&mut self, k: &Q) -> Option<V>
+    where
+        K: Borrow<Q>,
+        Q: AsKeyRef + ?Sized,
+    {
+        self.m
+            .remove(unsafe {
+                // UNSAFE(@ohsayan): good trait bounds and type
+                IndexSTSeqDllQref::from_ref(k)
+            })
+            .map(|n| unsafe {
+                let n = n.as_ptr();
+                // UNSAFE(@ohsayan): Correct init and aligned to K
+                drop(ptr::read(&(*n).k));
+                // UNSAFE(@ohsayan): Correct init and aligned to V
+                let v = ptr::read(&(*n).v);
+                // UNSAFE(@ohsayan): non-null guaranteed by as_ptr
+                IndexSTSeqDllNode::unlink(n);
+                (*n).n = self.f;
+                self.f = n;
+                v
+            })
+    }
+    #[inline(always)]
+    fn __insert(&mut self, k: K, v: V) -> bool {
+        self.ensure_sentinel();
+        let node = self.recycle_or_alloc(IndexSTSeqDllNode::new_null(k, v));
+        let kptr = unsafe {
+            // UNSAFE(@ohsayan): All g, we allocated it rn
+            IndexSTSeqDllKeyptr::new(&node.as_ref().k)
+        };
+        let _ = self.m.insert(kptr, node);
+        unsafe {
+            // UNSAFE(@ohsayan): sentinel check done
+            self.link(node);
+        }
+        true
+    }
+    #[inline(always)]
+    /// ## Safety
+    ///
+    /// Has sentinel
+    unsafe fn __update(&mut self, e: NonNull<IndexSTSeqDllNode<K, V>>, v: V) -> Option<V> {
+        let old = unsafe {
+            // UNSAFE(@ohsayan): Same type layout, alignments and non-null
+            ptr::replace(&mut (*e.as_ptr()).v, v)
+        };
+        self._refresh(e);
+        Some(old)
+    }
+    #[inline(always)]
+    /// ## Safety
+    ///
+    /// Has sentinel
+    unsafe fn _refresh(&mut self, e: NonNull<IndexSTSeqDllNode<K, V>>) {
+        // UNSAFE(@ohsayan): Since it's in the collection, it is a valid ptr
+        IndexSTSeqDllNode::unlink(e.as_ptr());
+        // UNSAFE(@ohsayan): As we found a node, our impl guarantees that the head is not-null
+        self.link(e);
+    }
+    #[inline(always)]
+    fn _clear(&mut self) {
+        self.m.clear();
+        if !self.h.is_null() {
+            unsafe {
+                // UNSAFE(@ohsayan): nullck
+                self.drop_nodes_full();
+                // UNSAFE(@ohsayan): Drop won't kill sentinel; link back to self
+                (*self.h).p = self.h;
+                (*self.h).n = self.h;
+            }
+        }
+    }
+    #[inline(always)]
+    fn _iter_unord_kv<'a>(&'a self) -> IndexSTSeqDllIterUnordKV<'a, K, V> {
+        IndexSTSeqDllIterUnordKV::new(&self.m)
+    }
+    #[inline(always)]
+    fn _iter_unord_k<'a>(&'a self) -> IndexSTSeqDllIterUnordK<'a, K, V> {
+        IndexSTSeqDllIterUnordK::new(&self.m)
+    }
+    #[inline(always)]
+    fn _iter_unord_v<'a>(&'a self) -> IndexSTSeqDllIterUnordV<'a, K, V> {
+        IndexSTSeqDllIterUnordV::new(&self.m)
+    }
+}
+
+impl<K, V, S> Drop for IndexSTSeqDll<K, V, S> {
+    fn drop(&mut self) {
+        if !self.h.is_null() {
+            unsafe {
+                // UNSAFE(@ohsayan): nullck
+                self.drop_nodes_full();
+                // UNSAFE(@ohsayan): nullck: drop doesn't clear sentinel
+                IndexSTSeqDllNode::dealloc_headless(self.h);
+            }
+        }
+        self.vacuum_free();
+    }
+}
+
+unsafe impl<K: Send, V: Send, S: Send> Send for IndexSTSeqDll<K, V, S> {}
+unsafe impl<K: Sync, V: Sync, S: Sync> Sync for IndexSTSeqDll<K, V, S> {}
+
+macro_rules! unsafe_marker_impl {
+    ($ty:ty) => {
+        unsafe impl<'a, K: Send, V: Send> Send for $ty {}
+        unsafe impl<'a, K: Sync, V: Sync> Sync for $ty {}
+    };
+}
+
+pub struct IndexSTSeqDllIterUnordKV<'a, K: 'a, V: 'a> {
+    i: Iter<'a, IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>>,
+}
+
+// UNSAFE(@ohsayan): aliasing guarantees correctness
+unsafe_marker_impl!(IndexSTSeqDllIterUnordKV<'a, K, V>);
+
+impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordKV<'a, K, V> {
+    #[inline(always)]
+    fn new<S>(m: &'a StdMap<IndexSTSeqDllKeyptr<K>, NonNull<IndexSTSeqDllNode<K, V>>, S>) -> Self {
+        Self { i: m.iter() }
+    }
+}
+
+impl<K, V> Clone for IndexSTSeqDllIterUnordKV<'_, K, V> {
+    fn clone(&self) -> Self {
+        Self { i: self.i.clone() }
+    }
+}
+
+impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordKV<'a, K, V> {
+    type Item = (&'a K, &'a V);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.i.next().map(|(_, n)| {
+            let n = n.as_ptr();
+            unsafe {
+                // UNSAFE(@ohsayan): nullck
+                (&(*n).k, &(*n).v)
+            }
+        })
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordKV<'a, K, V> {
+    fn len(&self) -> usize {
+        self.i.len()
+    }
+}
+
+impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordKV<'a, K, V> {}
+
+impl<K: Debug, V: Debug> Debug for IndexSTSeqDllIterUnordKV<'_, K, V> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.clone()).finish()
+    }
+}
+
+pub struct IndexSTSeqDllIterUnordK<'a, K: 'a, V: 'a> {
+    k: StdMapIterKey<'a, IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>>,
+}
+
+// UNSAFE(@ohsayan): aliasing guarantees correctness
+unsafe_marker_impl!(IndexSTSeqDllIterUnordK<'a, K, V>);
+
+impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordK<'a, K, V> {
+    #[inline(always)]
+    fn new<S>(m: &'a StdMap<IndexSTSeqDllKeyptr<K>, NonNull<IndexSTSeqDllNode<K, V>>, S>) -> Self {
+        Self { k: m.keys() }
+    }
+}
+
+impl<K, V> Clone for IndexSTSeqDllIterUnordK<'_, K, V> {
+    fn clone(&self) -> Self {
+        Self { k: self.k.clone() }
+    }
+}
+
+impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordK<'a, K, V> {
+    type Item = &'a K;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.k.next().map(|k| {
+            unsafe {
+                // UNSAFE(@ohsayan): nullck
+                &*(*k).p
+            }
+        })
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordK<'a, K, V> {
+    fn len(&self) -> usize {
+        self.k.len()
+    }
+}
+
+impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordK<'a, K, V> {}
+
+impl<'a, K: Debug, V> Debug for IndexSTSeqDllIterUnordK<'_, K, V> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.clone()).finish()
+    }
+}
+
+pub struct IndexSTSeqDllIterUnordV<'a, K: 'a, V: 'a> {
+    v: StdMapIterVal<'a, IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>>,
+}
+
+// UNSAFE(@ohsayan): aliasing guarantees correctness
+unsafe_marker_impl!(IndexSTSeqDllIterUnordV<'a, K, V>);
+
+impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordV<'a, K, V> {
+    #[inline(always)]
+    fn new<S>(m: &'a StdMap<IndexSTSeqDllKeyptr<K>, NonNull<IndexSTSeqDllNode<K, V>>, S>) -> Self {
+        Self { v: m.values() }
+    }
+}
+
+impl<K, V> Clone for IndexSTSeqDllIterUnordV<'_, K, V> {
+    fn clone(&self) -> Self {
+        Self { v: self.v.clone() }
+    }
+}
+
+impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordV<'a, K, V> {
+    type Item = &'a V;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.v.next().map(|k| {
+            unsafe {
+                // UNSAFE(@ohsayan): nullck
+                &(*k.as_ptr()).v
+            }
+        })
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordV<'a, K, V> {
+    fn len(&self) -> usize {
+        self.v.len()
+    }
+}
+
+impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordV<'a, K, V> {}
+
+impl<'a, K, V: Debug> Debug for IndexSTSeqDllIterUnordV<'_, K, V> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.clone()).finish()
+    }
+}

From 29a4cceea1b2ee393c565dc53600af38cdb7291a Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 18 Jan 2023 09:01:26 -0800
Subject: [PATCH 077/310] Add ordered iter impls for ord idx

---
 server/src/engine/core/idx.rs | 303 ++++++++++++++++++++++++++++++++--
 1 file changed, 291 insertions(+), 12 deletions(-)

diff --git a/server/src/engine/core/idx.rs b/server/src/engine/core/idx.rs
index 7605060a..28cf055b 100644
--- a/server/src/engine/core/idx.rs
+++ b/server/src/engine/core/idx.rs
@@ -35,6 +35,7 @@ use std::{
     fmt::{self, Debug},
     hash::{BuildHasher, Hash, Hasher},
     iter::FusedIterator,
+    marker::PhantomData,
     mem,
     ptr::{self, NonNull},
 };
@@ -313,6 +314,9 @@ impl<K, V, S> IndexSTSeqDll<K, V, S> {
     unsafe fn link(&mut self, node: IndexSTSeqDllNodePtr<K, V>) {
         IndexSTSeqDllNode::link(self.h, node.as_ptr())
     }
+    pub fn len(&self) -> usize {
+        self.m.len()
+    }
 }
 
 impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
@@ -465,6 +469,24 @@ impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
     fn _iter_unord_v<'a>(&'a self) -> IndexSTSeqDllIterUnordV<'a, K, V> {
         IndexSTSeqDllIterUnordV::new(&self.m)
     }
+    #[inline(always)]
+    fn _iter_ord_kv<'a>(&'a self) -> IndexSTSeqDllIterOrdKV<'a, K, V> {
+        IndexSTSeqDllIterOrdKV {
+            i: IndexSTSeqDllIterOrdBase::new(self),
+        }
+    }
+    #[inline(always)]
+    fn _iter_ord_k<'a>(&'a self) -> IndexSTSeqDllIterOrdK<'a, K, V> {
+        IndexSTSeqDllIterOrdK {
+            i: IndexSTSeqDllIterOrdBase::new(self),
+        }
+    }
+    #[inline(always)]
+    fn _iter_ord_v<'a>(&'a self) -> IndexSTSeqDllIterOrdV<'a, K, V> {
+        IndexSTSeqDllIterOrdV {
+            i: IndexSTSeqDllIterOrdBase::new(self),
+        }
+    }
 }
 
 impl<K, V, S> Drop for IndexSTSeqDll<K, V, S> {
@@ -485,7 +507,7 @@ unsafe impl<K: Send, V: Send, S: Send> Send for IndexSTSeqDll<K, V, S> {}
 unsafe impl<K: Sync, V: Sync, S: Sync> Sync for IndexSTSeqDll<K, V, S> {}
 
 macro_rules! unsafe_marker_impl {
-    ($ty:ty) => {
+    (unsafe impl for $ty:ty) => {
         unsafe impl<'a, K: Send, V: Send> Send for $ty {}
         unsafe impl<'a, K: Sync, V: Sync> Sync for $ty {}
     };
@@ -496,7 +518,7 @@ pub struct IndexSTSeqDllIterUnordKV<'a, K: 'a, V: 'a> {
 }
 
 // UNSAFE(@ohsayan): aliasing guarantees correctness
-unsafe_marker_impl!(IndexSTSeqDllIterUnordKV<'a, K, V>);
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordKV<'a, K, V>);
 
 impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordKV<'a, K, V> {
     #[inline(always)]
@@ -505,7 +527,7 @@ impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordKV<'a, K, V> {
     }
 }
 
-impl<K, V> Clone for IndexSTSeqDllIterUnordKV<'_, K, V> {
+impl<'a, K, V> Clone for IndexSTSeqDllIterUnordKV<'a, K, V> {
     fn clone(&self) -> Self {
         Self { i: self.i.clone() }
     }
@@ -523,6 +545,9 @@ impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordKV<'a, K, V> {
             }
         })
     }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        <_ as Iterator>::size_hint(&self.i)
+    }
 }
 
 impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordKV<'a, K, V> {
@@ -533,7 +558,7 @@ impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordKV<'a, K, V> {
 
 impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordKV<'a, K, V> {}
 
-impl<K: Debug, V: Debug> Debug for IndexSTSeqDllIterUnordKV<'_, K, V> {
+impl<'a, K: 'a + Debug, V: 'a + Debug> Debug for IndexSTSeqDllIterUnordKV<'a, K, V> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_list().entries(self.clone()).finish()
     }
@@ -544,7 +569,7 @@ pub struct IndexSTSeqDllIterUnordK<'a, K: 'a, V: 'a> {
 }
 
 // UNSAFE(@ohsayan): aliasing guarantees correctness
-unsafe_marker_impl!(IndexSTSeqDllIterUnordK<'a, K, V>);
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordK<'a, K, V>);
 
 impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordK<'a, K, V> {
     #[inline(always)]
@@ -553,7 +578,7 @@ impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordK<'a, K, V> {
     }
 }
 
-impl<K, V> Clone for IndexSTSeqDllIterUnordK<'_, K, V> {
+impl<'a, K, V> Clone for IndexSTSeqDllIterUnordK<'a, K, V> {
     fn clone(&self) -> Self {
         Self { k: self.k.clone() }
     }
@@ -569,6 +594,9 @@ impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordK<'a, K, V> {
             }
         })
     }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        <_ as Iterator>::size_hint(&self.k)
+    }
 }
 
 impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordK<'a, K, V> {
@@ -579,7 +607,7 @@ impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordK<'a, K, V> {
 
 impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordK<'a, K, V> {}
 
-impl<'a, K: Debug, V> Debug for IndexSTSeqDllIterUnordK<'_, K, V> {
+impl<'a, K: Debug, V> Debug for IndexSTSeqDllIterUnordK<'a, K, V> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_list().entries(self.clone()).finish()
     }
@@ -590,7 +618,7 @@ pub struct IndexSTSeqDllIterUnordV<'a, K: 'a, V: 'a> {
 }
 
 // UNSAFE(@ohsayan): aliasing guarantees correctness
-unsafe_marker_impl!(IndexSTSeqDllIterUnordV<'a, K, V>);
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordV<'a, K, V>);
 
 impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordV<'a, K, V> {
     #[inline(always)]
@@ -599,7 +627,7 @@ impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordV<'a, K, V> {
     }
 }
 
-impl<K, V> Clone for IndexSTSeqDllIterUnordV<'_, K, V> {
+impl<'a, K, V> Clone for IndexSTSeqDllIterUnordV<'a, K, V> {
     fn clone(&self) -> Self {
         Self { v: self.v.clone() }
     }
@@ -608,13 +636,16 @@ impl<K, V> Clone for IndexSTSeqDllIterUnordV<'_, K, V> {
 impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordV<'a, K, V> {
     type Item = &'a V;
     fn next(&mut self) -> Option<Self::Item> {
-        self.v.next().map(|k| {
+        self.v.next().map(|n| {
             unsafe {
                 // UNSAFE(@ohsayan): nullck
-                &(*k.as_ptr()).v
+                &(*n.as_ptr()).v
             }
         })
     }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        <_ as Iterator>::size_hint(&self.v)
+    }
 }
 
 impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordV<'a, K, V> {
@@ -625,8 +656,256 @@ impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordV<'a, K, V> {
 
 impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordV<'a, K, V> {}
 
-impl<'a, K, V: Debug> Debug for IndexSTSeqDllIterUnordV<'_, K, V> {
+impl<'a, K, V: Debug> Debug for IndexSTSeqDllIterUnordV<'a, K, V> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_list().entries(self.clone()).finish()
     }
 }
+
+trait IndexSTSeqDllIterOrdConfig<K, V> {
+    type Ret<'a>
+    where
+        K: 'a,
+        V: 'a;
+    unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode<K, V>) -> Option<Self::Ret<'a>>
+    where
+        K: 'a,
+        V: 'a;
+    fn def_ret<'a>() -> Option<Self::Ret<'a>>;
+}
+
+struct IndexSTSeqDllIterOrdConfigFull;
+
+impl<K, V> IndexSTSeqDllIterOrdConfig<K, V> for IndexSTSeqDllIterOrdConfigFull {
+    type Ret<'a> = (&'a K, &'a V) where K: 'a, V: 'a;
+    #[inline(always)]
+    unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode<K, V>) -> Option<Self::Ret<'a>>
+    where
+        K: 'a,
+        V: 'a,
+    {
+        Some((&(*ptr).k, &(*ptr).v))
+    }
+    #[inline(always)]
+    fn def_ret<'a>() -> Option<Self::Ret<'a>> {
+        None
+    }
+}
+
+struct IndexSTSeqDllIterOrdConfigKey;
+
+impl<K, V> IndexSTSeqDllIterOrdConfig<K, V> for IndexSTSeqDllIterOrdConfigKey {
+    type Ret<'a> = &'a K
+    where
+        K: 'a,
+        V: 'a;
+    #[inline(always)]
+    unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode<K, V>) -> Option<&'a K>
+    where
+        K: 'a,
+        V: 'a,
+    {
+        Some(&(*ptr).k)
+    }
+    #[inline(always)]
+    fn def_ret<'a>() -> Option<&'a K>
+    where
+        K: 'a,
+        V: 'a,
+    {
+        None
+    }
+}
+
+struct IndexSTSeqDllIterOrdConfigValue;
+
+impl<K, V> IndexSTSeqDllIterOrdConfig<K, V> for IndexSTSeqDllIterOrdConfigValue {
+    type Ret<'a> = &'a V
+    where
+        K: 'a,
+        V: 'a;
+    #[inline(always)]
+    unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode<K, V>) -> Option<&'a V>
+    where
+        K: 'a,
+        V: 'a,
+    {
+        Some(&(*ptr).v)
+    }
+    #[inline(always)]
+    fn def_ret<'a>() -> Option<&'a V>
+    where
+        K: 'a,
+        V: 'a,
+    {
+        None
+    }
+}
+
+struct IndexSTSeqDllIterOrdBase<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> {
+    h: *const IndexSTSeqDllNode<K, V>,
+    t: *const IndexSTSeqDllNode<K, V>,
+    r: usize,
+    _l: PhantomData<(&'a K, &'a V, C)>,
+}
+
+impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> IndexSTSeqDllIterOrdBase<'a, K, V, C> {
+    #[inline(always)]
+    fn new<S>(idx: &'a IndexSTSeqDll<K, V, S>) -> Self {
+        Self {
+            h: if idx.h.is_null() {
+                ptr::null_mut()
+            } else {
+                unsafe {
+                    // UNSAFE(@ohsayan): nullck
+                    (*idx.h).p
+                }
+            },
+            t: idx.h,
+            r: idx.len(),
+            _l: PhantomData,
+        }
+    }
+    #[inline(always)]
+    fn _next(&mut self) -> Option<C::Ret<'a>> {
+        if self.h == self.t {
+            C::def_ret()
+        } else {
+            self.r -= 1;
+            unsafe {
+                // UNSAFE(@ohsayan): Assuming we had a legal init, this should be fine
+                let this = C::read_ret(self.h);
+                self.h = (*self.h).p;
+                this
+            }
+        }
+    }
+}
+
+impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> Debug
+    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
+where
+    C::Ret<'a>: Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.clone()).finish()
+    }
+}
+
+impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> Iterator
+    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
+{
+    type Item = C::Ret<'a>;
+    fn next(&mut self) -> Option<Self::Item> {
+        self._next()
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (self.r, Some(self.r))
+    }
+}
+
+impl<'a, K, V, C: IndexSTSeqDllIterOrdConfig<K, V>> ExactSizeIterator
+    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
+{
+    fn len(&self) -> usize {
+        self.r
+    }
+}
+
+impl<'a, K, V, C: IndexSTSeqDllIterOrdConfig<K, V>> Clone
+    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
+{
+    fn clone(&self) -> Self {
+        Self { ..*self }
+    }
+}
+
+#[derive(Debug)]
+pub struct IndexSTSeqDllIterOrdKV<'a, K: 'a, V: 'a> {
+    i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigFull>,
+}
+
+// UNSAFE(@ohsayan): aliasing guarantees correctness
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdKV<'a, K, V>);
+
+impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdKV<'a, K, V> {
+    type Item = (&'a K, &'a V);
+    fn next(&mut self) -> Option<Self::Item> {
+        self.i.next()
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.i.size_hint()
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdKV<'a, K, V> {
+    fn len(&self) -> usize {
+        self.i.len()
+    }
+}
+
+impl<'a, K, V> Clone for IndexSTSeqDllIterOrdKV<'a, K, V> {
+    fn clone(&self) -> Self {
+        Self { i: self.i.clone() }
+    }
+}
+
+#[derive(Debug)]
+pub struct IndexSTSeqDllIterOrdK<'a, K: 'a, V: 'a> {
+    i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigKey>,
+}
+
+// UNSAFE(@ohsayan): aliasing guarantees correctness
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdK<'a, K, V>);
+
+impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdK<'a, K, V> {
+    type Item = &'a K;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.i.next()
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.i.size_hint()
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdK<'a, K, V> {
+    fn len(&self) -> usize {
+        self.i.len()
+    }
+}
+
+impl<'a, K, V> Clone for IndexSTSeqDllIterOrdK<'a, K, V> {
+    fn clone(&self) -> Self {
+        Self { i: self.i.clone() }
+    }
+}
+
+#[derive(Debug)]
+pub struct IndexSTSeqDllIterOrdV<'a, K: 'a, V: 'a> {
+    i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigValue>,
+}
+
+// UNSAFE(@ohsayan): aliasing guarantees correctness
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdV<'a, K, V>);
+
+impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdV<'a, K, V> {
+    type Item = &'a V;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.i.next()
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.i.size_hint()
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdV<'a, K, V> {
+    fn len(&self) -> usize {
+        self.i.len()
+    }
+}
+
+impl<'a, K, V> Clone for IndexSTSeqDllIterOrdV<'a, K, V> {
+    fn clone(&self) -> Self {
+        Self { i: self.i.clone() }
+    }
+}

From ae516b7168443795944b06e81da797d727073f5c Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 18 Jan 2023 21:36:24 -0800
Subject: [PATCH 078/310] Add Index trait impls

---
 server/src/engine/core/def.rs | 164 ++++++++++---------
 server/src/engine/core/idx.rs | 291 +++++++++++++++++++++++++++-------
 2 files changed, 324 insertions(+), 131 deletions(-)

diff --git a/server/src/engine/core/def.rs b/server/src/engine/core/def.rs
index f57639f0..7fc7057f 100644
--- a/server/src/engine/core/def.rs
+++ b/server/src/engine/core/def.rs
@@ -67,14 +67,14 @@ impl<T: Clone> AsValue for T {
     }
 }
 
-pub trait MTIndex<K, V>
+/// The base spec for any index. Iterators have meaningless order, and that is intentional and oftentimes
+/// consequential. For more specialized impls, use the [`STIndex`], [`MTIndex`] or [`STIndexSeq`] traits
+pub trait IndexBaseSpec<K, V>
 where
     K: AsKey,
     V: AsValue,
 {
-    /// State whether the underlying structure provides any ordering on the iterators
-    const HAS_ORDER: bool;
-    /// Base support prealloc?
+    /// Index supports prealloc?
     const PREALLOC: bool;
     /// An iterator over the keys and values
     type IterKV<'a>: Iterator<Item = (&'a K, &'a V)>
@@ -92,10 +92,26 @@ where
     where
         Self: 'a,
         V: 'a;
-    /// Initialize an empty instance of the MTIndex
-    fn mt_init() -> Self;
-    /// Initialize a pre-loaded instance of the MTIndex
-    fn mt_init_with(s: Self) -> Self;
+    // init
+    /// Initialize an empty instance of the index
+    fn idx_init() -> Self;
+    /// Initialize a pre-loaded instance of the index
+    fn idx_init_with(s: Self) -> Self;
+    // iter
+    /// Returns an iterator over a tuple of keys and values
+    fn idx_iter_kv<'a>(&'a self) -> Self::IterKV<'a>;
+    /// Returns an iterator over the keys
+    fn idx_iter_key<'a>(&'a self) -> Self::IterKey<'a>;
+    /// Returns an iterator over the values
+    fn idx_iter_value<'a>(&'a self) -> Self::IterValue<'a>;
+}
+
+/// An unordered MTIndex
+pub trait MTIndex<K, V>: IndexBaseSpec<K, V>
+where
+    K: AsKey,
+    V: AsValue,
+{
     /// Clears all the entries in the MTIndex
     fn mt_clear(&self);
     // write
@@ -104,71 +120,47 @@ where
     fn mt_insert(&self, key: K, val: V) -> bool;
     /// Updates or inserts the given value
     fn mt_upsert(&self, key: K, val: V);
-
     // read
     /// Returns a reference to the value corresponding to the key, if it exists
-    fn mt_get<Q>(&self, key: &Q) -> Option<&K>
+    fn mt_get<Q>(&self, key: &Q) -> Option<&V>
     where
-        K: Borrow<Q>;
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef;
     /// Returns a clone of the value corresponding to the key, if it exists
-    fn mt_get_cloned<Q>(&self, key: &Q) -> Option<K>
+    fn mt_get_cloned<Q>(&self, key: &Q) -> Option<V>
     where
-        K: Borrow<Q>;
-
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef;
     // update
     /// Returns true if the entry is updated
     fn mt_update<Q>(&self, key: &Q, val: V) -> bool
     where
-        K: Borrow<Q>;
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef;
     /// Updates the entry and returns the old value, if it exists
-    fn mt_update_return<Q>(&self, key: &Q, val: V) -> Option<K>
+    fn mt_update_return<Q>(&self, key: &Q, val: V) -> Option<V>
     where
-        K: Borrow<Q>;
-
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef;
     // delete
     /// Returns true if the entry was deleted
     fn mt_delete<Q>(&self, key: &Q) -> bool
     where
-        K: Borrow<Q>;
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef;
     /// Removes the entry and returns it, if it exists
-    fn mt_delete_return<Q>(&self, key: &Q) -> Option<K>
+    fn mt_delete_return<Q>(&self, key: &Q) -> Option<V>
     where
-        K: Borrow<Q>;
-
-    // iter
-    /// Returns an iterator over a tuple of keys and values
-    fn mt_iter_kv<'a>(&'a self) -> Self::IterKV<'a>;
-    /// Returns an iterator over the keys
-    fn mt_iter_k<'a>(&'a self) -> Self::IterKey<'a>;
-    /// Returns an iterator over the values
-    fn mt_iter_v<'a>(&'a self) -> Self::IterValue<'a>;
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef;
 }
 
-pub trait STIndex<K, V> {
-    /// State whether the underlying structure provides any ordering on the iterators
-    const HAS_ORDER: bool;
-    /// Base support prealloc?
-    const PREALLOC: bool;
-    /// An iterator over the keys and values
-    type IterKV<'a>: Iterator<Item = (&'a K, &'a V)>
-    where
-        Self: 'a,
-        K: 'a,
-        V: 'a;
-    /// An iterator over the keys
-    type IterKey<'a>: Iterator<Item = &'a K>
-    where
-        Self: 'a,
-        K: 'a;
-    /// An iterator over the values
-    type IterValue<'a>: Iterator<Item = &'a V>
-    where
-        Self: 'a,
-        V: 'a;
-    /// Initialize an empty instance of the MTIndex
-    fn st_init() -> Self;
-    /// Initialize a pre-loaded instance of the MTIndex
-    fn st_init_with(s: Self) -> Self;
+/// An unordered STIndex
+pub trait STIndex<K, V>: IndexBaseSpec<K, V>
+where
+    K: AsKey,
+    V: AsValue,
+{
     /// Clears all the entries in the STIndex
     fn st_clear(&mut self);
     // write
@@ -177,42 +169,66 @@ pub trait STIndex<K, V> {
     fn st_insert(&mut self, key: K, val: V) -> bool;
     /// Updates or inserts the given value
     fn st_upsert(&mut self, key: K, val: V);
-
     // read
     /// Returns a reference to the value corresponding to the key, if it exists
-    fn st_get<Q>(&self, key: &Q) -> Option<&K>
+    fn st_get<Q>(&self, key: &Q) -> Option<&V>
     where
-        K: Borrow<Q>;
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef;
     /// Returns a clone of the value corresponding to the key, if it exists
-    fn st_get_cloned<Q>(&self, key: &Q) -> Option<K>
+    fn st_get_cloned<Q>(&self, key: &Q) -> Option<V>
     where
-        K: Borrow<Q>;
-
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef;
     // update
     /// Returns true if the entry is updated
     fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
     where
-        K: Borrow<Q>;
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef;
     /// Updates the entry and returns the old value, if it exists
-    fn st_update_return<Q>(&mut self, key: &Q, val: V) -> Option<K>
+    fn st_update_return<Q>(&mut self, key: &Q, val: V) -> Option<V>
     where
-        K: Borrow<Q>;
-
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef;
     // delete
     /// Returns true if the entry was deleted
     fn st_delete<Q>(&mut self, key: &Q) -> bool
     where
-        K: Borrow<Q>;
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef;
     /// Removes the entry and returns it, if it exists
-    fn st_delete_return<Q>(&mut self, key: &Q) -> Option<K>
+    fn st_delete_return<Q>(&mut self, key: &Q) -> Option<V>
     where
-        K: Borrow<Q>;
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef;
+}
 
-    // iter
-    /// Returns an iterator over a tuple of keys and values
-    fn st_iter_kv<'a>(&'a self) -> Self::IterKV<'a>;
-    /// Returns an iterator over the keys
-    fn st_iter_k<'a>(&'a self) -> Self::IterKey<'a>;
-    /// Returns an iterator over the values
-    fn st_iter_v<'a>(&'a self) -> Self::IterValue<'a>;
+pub trait STIndexSeq<K, V>: STIndex<K, V>
+where
+    K: AsKey,
+    V: AsValue,
+{
+    /// An ordered iterator over the keys and values
+    type IterOrdKV<'a>: Iterator<Item = (&'a K, &'a V)> + DoubleEndedIterator<Item = (&'a K, &'a V)>
+    where
+        Self: 'a,
+        K: 'a,
+        V: 'a;
+    /// An ordered iterator over the keys
+    type IterOrdKey<'a>: Iterator<Item = &'a K> + DoubleEndedIterator<Item = &'a K>
+    where
+        Self: 'a,
+        K: 'a;
+    /// An ordered iterator over the values
+    type IterOrdValue<'a>: Iterator<Item = &'a V> + DoubleEndedIterator<Item = &'a V>
+    where
+        Self: 'a,
+        V: 'a;
+    /// Returns an ordered iterator over the KV pairs
+    fn stseq_ord_kv<'a>(&'a self) -> Self::IterOrdKV<'a>;
+    /// Returns an ordered iterator over the keys
+    fn stseq_ord_key<'a>(&'a self) -> Self::IterOrdKey<'a>;
+    /// Returns an ordered iterator over the values
+    fn stseq_ord_value<'a>(&'a self) -> Self::IterOrdValue<'a>;
 }
diff --git a/server/src/engine/core/idx.rs b/server/src/engine/core/idx.rs
index 28cf055b..978f3ac8 100644
--- a/server/src/engine/core/idx.rs
+++ b/server/src/engine/core/idx.rs
@@ -24,7 +24,7 @@
  *
 */
 
-use super::def::{AsKey, AsKeyRef, AsValue};
+use super::def::{AsKey, AsKeyRef, AsValue, IndexBaseSpec, STIndex, STIndexSeq};
 use std::{
     alloc::{alloc as std_alloc, dealloc as std_dealloc, Layout},
     borrow::Borrow,
@@ -462,12 +462,12 @@ impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
         IndexSTSeqDllIterUnordKV::new(&self.m)
     }
     #[inline(always)]
-    fn _iter_unord_k<'a>(&'a self) -> IndexSTSeqDllIterUnordK<'a, K, V> {
-        IndexSTSeqDllIterUnordK::new(&self.m)
+    fn _iter_unord_k<'a>(&'a self) -> IndexSTSeqDllIterUnordKey<'a, K, V> {
+        IndexSTSeqDllIterUnordKey::new(&self.m)
     }
     #[inline(always)]
-    fn _iter_unord_v<'a>(&'a self) -> IndexSTSeqDllIterUnordV<'a, K, V> {
-        IndexSTSeqDllIterUnordV::new(&self.m)
+    fn _iter_unord_v<'a>(&'a self) -> IndexSTSeqDllIterUnordValue<'a, K, V> {
+        IndexSTSeqDllIterUnordValue::new(&self.m)
     }
     #[inline(always)]
     fn _iter_ord_kv<'a>(&'a self) -> IndexSTSeqDllIterOrdKV<'a, K, V> {
@@ -476,14 +476,14 @@ impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
         }
     }
     #[inline(always)]
-    fn _iter_ord_k<'a>(&'a self) -> IndexSTSeqDllIterOrdK<'a, K, V> {
-        IndexSTSeqDllIterOrdK {
+    fn _iter_ord_k<'a>(&'a self) -> IndexSTSeqDllIterOrdKey<'a, K, V> {
+        IndexSTSeqDllIterOrdKey {
             i: IndexSTSeqDllIterOrdBase::new(self),
         }
     }
     #[inline(always)]
-    fn _iter_ord_v<'a>(&'a self) -> IndexSTSeqDllIterOrdV<'a, K, V> {
-        IndexSTSeqDllIterOrdV {
+    fn _iter_ord_v<'a>(&'a self) -> IndexSTSeqDllIterOrdValue<'a, K, V> {
+        IndexSTSeqDllIterOrdValue {
             i: IndexSTSeqDllIterOrdBase::new(self),
         }
     }
@@ -503,6 +503,162 @@ impl<K, V, S> Drop for IndexSTSeqDll<K, V, S> {
     }
 }
 
+impl<K: AsKey, V: AsValue, S: BuildHasher + Default> FromIterator<(K, V)>
+    for IndexSTSeqDll<K, V, S>
+{
+    fn from_iter<T: IntoIterator<Item = (K, V)>>(iter: T) -> Self {
+        let mut slf = Self::with_hasher(S::default());
+        iter.into_iter()
+            .for_each(|(k, v)| assert!(slf._insert(k, v)));
+        slf
+    }
+}
+
+impl<K, V, S: BuildHasher + Default> IndexBaseSpec<K, V> for IndexSTSeqDll<K, V, S>
+where
+    K: AsKey,
+    V: AsValue,
+{
+    const PREALLOC: bool = true;
+
+    type IterKV<'a> = IndexSTSeqDllIterUnordKV<'a, K, V>
+    where
+        Self: 'a,
+        K: 'a,
+        V: 'a;
+
+    type IterKey<'a> = IndexSTSeqDllIterUnordKey<'a, K, V>
+    where
+        Self: 'a,
+        K: 'a;
+
+    type IterValue<'a> = IndexSTSeqDllIterUnordValue<'a, K, V>
+    where
+        Self: 'a,
+        V: 'a;
+
+    fn idx_init() -> Self {
+        Self::with_hasher(S::default())
+    }
+
+    fn idx_init_with(s: Self) -> Self {
+        Self::from(s)
+    }
+
+    fn idx_iter_kv<'a>(&'a self) -> Self::IterKV<'a> {
+        self._iter_unord_kv()
+    }
+
+    fn idx_iter_key<'a>(&'a self) -> Self::IterKey<'a> {
+        self._iter_unord_k()
+    }
+
+    fn idx_iter_value<'a>(&'a self) -> Self::IterValue<'a> {
+        self._iter_unord_v()
+    }
+}
+
+impl<K, V, S: BuildHasher + Default> STIndex<K, V> for IndexSTSeqDll<K, V, S>
+where
+    K: AsKey,
+    V: AsValue,
+{
+    fn st_clear(&mut self) {
+        self._clear()
+    }
+
+    fn st_insert(&mut self, key: K, val: V) -> bool {
+        self._insert(key, val)
+    }
+
+    fn st_upsert(&mut self, key: K, val: V) {
+        let _ = self._upsert(key, val);
+    }
+
+    fn st_get<Q>(&self, key: &Q) -> Option<&V>
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef,
+    {
+        self._get(key)
+    }
+
+    fn st_get_cloned<Q>(&self, key: &Q) -> Option<V>
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef,
+    {
+        self._get(key).cloned()
+    }
+
+    fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef,
+    {
+        self._update(key, val).is_none()
+    }
+
+    fn st_update_return<Q>(&mut self, key: &Q, val: V) -> Option<V>
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef,
+    {
+        self._update(key, val)
+    }
+
+    fn st_delete<Q>(&mut self, key: &Q) -> bool
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef,
+    {
+        self._remove(key).is_none()
+    }
+
+    fn st_delete_return<Q>(&mut self, key: &Q) -> Option<V>
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef,
+    {
+        self._remove(key)
+    }
+}
+
+impl<K, V, S> STIndexSeq<K, V> for IndexSTSeqDll<K, V, S>
+where
+    K: AsKey,
+    V: AsValue,
+    S: BuildHasher + Default,
+{
+    type IterOrdKV<'a> = IndexSTSeqDllIterOrdKV<'a, K, V>
+    where
+        Self: 'a,
+        K: 'a,
+        V: 'a;
+
+    type IterOrdKey<'a> = IndexSTSeqDllIterOrdKey<'a, K, V>
+    where
+        Self: 'a,
+        K: 'a;
+
+    type IterOrdValue<'a> = IndexSTSeqDllIterOrdValue<'a, K, V>
+    where
+        Self: 'a,
+        V: 'a;
+
+    fn stseq_ord_kv<'a>(&'a self) -> Self::IterOrdKV<'a> {
+        self._iter_ord_kv()
+    }
+
+    fn stseq_ord_key<'a>(&'a self) -> Self::IterOrdKey<'a> {
+        self._iter_ord_k()
+    }
+
+    fn stseq_ord_value<'a>(&'a self) -> Self::IterOrdValue<'a> {
+        self._iter_ord_v()
+    }
+}
+
 unsafe impl<K: Send, V: Send, S: Send> Send for IndexSTSeqDll<K, V, S> {}
 unsafe impl<K: Sync, V: Sync, S: Sync> Sync for IndexSTSeqDll<K, V, S> {}
 
@@ -564,27 +720,27 @@ impl<'a, K: 'a + Debug, V: 'a + Debug> Debug for IndexSTSeqDllIterUnordKV<'a, K,
     }
 }
 
-pub struct IndexSTSeqDllIterUnordK<'a, K: 'a, V: 'a> {
+pub struct IndexSTSeqDllIterUnordKey<'a, K: 'a, V: 'a> {
     k: StdMapIterKey<'a, IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>>,
 }
 
 // UNSAFE(@ohsayan): aliasing guarantees correctness
-unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordK<'a, K, V>);
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordKey<'a, K, V>);
 
-impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordK<'a, K, V> {
+impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordKey<'a, K, V> {
     #[inline(always)]
     fn new<S>(m: &'a StdMap<IndexSTSeqDllKeyptr<K>, NonNull<IndexSTSeqDllNode<K, V>>, S>) -> Self {
         Self { k: m.keys() }
     }
 }
 
-impl<'a, K, V> Clone for IndexSTSeqDllIterUnordK<'a, K, V> {
+impl<'a, K, V> Clone for IndexSTSeqDllIterUnordKey<'a, K, V> {
     fn clone(&self) -> Self {
         Self { k: self.k.clone() }
     }
 }
 
-impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordK<'a, K, V> {
+impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordKey<'a, K, V> {
     type Item = &'a K;
     fn next(&mut self) -> Option<Self::Item> {
         self.k.next().map(|k| {
@@ -599,41 +755,41 @@ impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordK<'a, K, V> {
     }
 }
 
-impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordK<'a, K, V> {
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordKey<'a, K, V> {
     fn len(&self) -> usize {
         self.k.len()
     }
 }
 
-impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordK<'a, K, V> {}
+impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordKey<'a, K, V> {}
 
-impl<'a, K: Debug, V> Debug for IndexSTSeqDllIterUnordK<'a, K, V> {
+impl<'a, K: Debug, V> Debug for IndexSTSeqDllIterUnordKey<'a, K, V> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_list().entries(self.clone()).finish()
     }
 }
 
-pub struct IndexSTSeqDllIterUnordV<'a, K: 'a, V: 'a> {
+pub struct IndexSTSeqDllIterUnordValue<'a, K: 'a, V: 'a> {
     v: StdMapIterVal<'a, IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>>,
 }
 
 // UNSAFE(@ohsayan): aliasing guarantees correctness
-unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordV<'a, K, V>);
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordValue<'a, K, V>);
 
-impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordV<'a, K, V> {
+impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordValue<'a, K, V> {
     #[inline(always)]
     fn new<S>(m: &'a StdMap<IndexSTSeqDllKeyptr<K>, NonNull<IndexSTSeqDllNode<K, V>>, S>) -> Self {
         Self { v: m.values() }
     }
 }
 
-impl<'a, K, V> Clone for IndexSTSeqDllIterUnordV<'a, K, V> {
+impl<'a, K, V> Clone for IndexSTSeqDllIterUnordValue<'a, K, V> {
     fn clone(&self) -> Self {
         Self { v: self.v.clone() }
     }
 }
 
-impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordV<'a, K, V> {
+impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordValue<'a, K, V> {
     type Item = &'a V;
     fn next(&mut self) -> Option<Self::Item> {
         self.v.next().map(|n| {
@@ -648,15 +804,15 @@ impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordV<'a, K, V> {
     }
 }
 
-impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordV<'a, K, V> {
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordValue<'a, K, V> {
     fn len(&self) -> usize {
         self.v.len()
     }
 }
 
-impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordV<'a, K, V> {}
+impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordValue<'a, K, V> {}
 
-impl<'a, K, V: Debug> Debug for IndexSTSeqDllIterUnordV<'a, K, V> {
+impl<'a, K, V: Debug> Debug for IndexSTSeqDllIterUnordValue<'a, K, V> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_list().entries(self.clone()).finish()
     }
@@ -667,11 +823,12 @@ trait IndexSTSeqDllIterOrdConfig<K, V> {
     where
         K: 'a,
         V: 'a;
+    /// ## Safety
+    /// Ptr must be non-null
     unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode<K, V>) -> Option<Self::Ret<'a>>
     where
         K: 'a,
         V: 'a;
-    fn def_ret<'a>() -> Option<Self::Ret<'a>>;
 }
 
 struct IndexSTSeqDllIterOrdConfigFull;
@@ -686,10 +843,6 @@ impl<K, V> IndexSTSeqDllIterOrdConfig<K, V> for IndexSTSeqDllIterOrdConfigFull {
     {
         Some((&(*ptr).k, &(*ptr).v))
     }
-    #[inline(always)]
-    fn def_ret<'a>() -> Option<Self::Ret<'a>> {
-        None
-    }
 }
 
 struct IndexSTSeqDllIterOrdConfigKey;
@@ -707,14 +860,6 @@ impl<K, V> IndexSTSeqDllIterOrdConfig<K, V> for IndexSTSeqDllIterOrdConfigKey {
     {
         Some(&(*ptr).k)
     }
-    #[inline(always)]
-    fn def_ret<'a>() -> Option<&'a K>
-    where
-        K: 'a,
-        V: 'a,
-    {
-        None
-    }
 }
 
 struct IndexSTSeqDllIterOrdConfigValue;
@@ -732,14 +877,6 @@ impl<K, V> IndexSTSeqDllIterOrdConfig<K, V> for IndexSTSeqDllIterOrdConfigValue
     {
         Some(&(*ptr).v)
     }
-    #[inline(always)]
-    fn def_ret<'a>() -> Option<&'a V>
-    where
-        K: 'a,
-        V: 'a,
-    {
-        None
-    }
 }
 
 struct IndexSTSeqDllIterOrdBase<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> {
@@ -769,7 +906,7 @@ impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> IndexSTSeqDllIterOrd
     #[inline(always)]
     fn _next(&mut self) -> Option<C::Ret<'a>> {
         if self.h == self.t {
-            C::def_ret()
+            None
         } else {
             self.r -= 1;
             unsafe {
@@ -780,6 +917,20 @@ impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> IndexSTSeqDllIterOrd
             }
         }
     }
+    #[inline(always)]
+    fn _next_back(&mut self) -> Option<C::Ret<'a>> {
+        if self.h == self.t {
+            None
+        } else {
+            self.r -= 1;
+            unsafe {
+                // UNSAFE(@ohsayan): legal init, then ok
+                self.t = (*self.t).n;
+                // UNSAFE(@ohsayan): non-null (sentinel)
+                C::read_ret(self.t)
+            }
+        }
+    }
 }
 
 impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> Debug
@@ -804,6 +955,14 @@ impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> Iterator
     }
 }
 
+impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> DoubleEndedIterator
+    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
+{
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self._next_back()
+    }
+}
+
 impl<'a, K, V, C: IndexSTSeqDllIterOrdConfig<K, V>> ExactSizeIterator
     for IndexSTSeqDllIterOrdBase<'a, K, V, C>
 {
@@ -838,6 +997,12 @@ impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdKV<'a, K, V> {
     }
 }
 
+impl<'a, K: 'a, V: 'a> DoubleEndedIterator for IndexSTSeqDllIterOrdKV<'a, K, V> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.i.next_back()
+    }
+}
+
 impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdKV<'a, K, V> {
     fn len(&self) -> usize {
         self.i.len()
@@ -851,14 +1016,14 @@ impl<'a, K, V> Clone for IndexSTSeqDllIterOrdKV<'a, K, V> {
 }
 
 #[derive(Debug)]
-pub struct IndexSTSeqDllIterOrdK<'a, K: 'a, V: 'a> {
+pub struct IndexSTSeqDllIterOrdKey<'a, K: 'a, V: 'a> {
     i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigKey>,
 }
 
 // UNSAFE(@ohsayan): aliasing guarantees correctness
-unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdK<'a, K, V>);
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdKey<'a, K, V>);
 
-impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdK<'a, K, V> {
+impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdKey<'a, K, V> {
     type Item = &'a K;
     fn next(&mut self) -> Option<Self::Item> {
         self.i.next()
@@ -868,27 +1033,33 @@ impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdK<'a, K, V> {
     }
 }
 
-impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdK<'a, K, V> {
+impl<'a, K: 'a, V: 'a> DoubleEndedIterator for IndexSTSeqDllIterOrdKey<'a, K, V> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.i.next_back()
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdKey<'a, K, V> {
     fn len(&self) -> usize {
         self.i.len()
     }
 }
 
-impl<'a, K, V> Clone for IndexSTSeqDllIterOrdK<'a, K, V> {
+impl<'a, K, V> Clone for IndexSTSeqDllIterOrdKey<'a, K, V> {
     fn clone(&self) -> Self {
         Self { i: self.i.clone() }
     }
 }
 
 #[derive(Debug)]
-pub struct IndexSTSeqDllIterOrdV<'a, K: 'a, V: 'a> {
+pub struct IndexSTSeqDllIterOrdValue<'a, K: 'a, V: 'a> {
     i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigValue>,
 }
 
 // UNSAFE(@ohsayan): aliasing guarantees correctness
-unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdV<'a, K, V>);
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdValue<'a, K, V>);
 
-impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdV<'a, K, V> {
+impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdValue<'a, K, V> {
     type Item = &'a V;
     fn next(&mut self) -> Option<Self::Item> {
         self.i.next()
@@ -898,13 +1069,19 @@ impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdV<'a, K, V> {
     }
 }
 
-impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdV<'a, K, V> {
+impl<'a, K: 'a, V: 'a> DoubleEndedIterator for IndexSTSeqDllIterOrdValue<'a, K, V> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.i.next_back()
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdValue<'a, K, V> {
     fn len(&self) -> usize {
         self.i.len()
     }
 }
 
-impl<'a, K, V> Clone for IndexSTSeqDllIterOrdV<'a, K, V> {
+impl<'a, K, V> Clone for IndexSTSeqDllIterOrdValue<'a, K, V> {
     fn clone(&self) -> Self {
         Self { i: self.i.clone() }
     }

From 01f2bca81f9b1243999dfa168a9347168be9c536 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 19 Jan 2023 01:22:46 -0800
Subject: [PATCH 079/310] Add basic tests for ord index

---
 server/src/engine/core/def.rs       | 10 ++++
 server/src/engine/core/idx.rs       | 65 ++++++++++++++++++++++--
 server/src/engine/core/mod.rs       |  2 +
 server/src/engine/core/tests/mod.rs | 77 +++++++++++++++++++++++++++++
 4 files changed, 151 insertions(+), 3 deletions(-)
 create mode 100644 server/src/engine/core/tests/mod.rs

diff --git a/server/src/engine/core/def.rs b/server/src/engine/core/def.rs
index 7fc7057f..08d4b969 100644
--- a/server/src/engine/core/def.rs
+++ b/server/src/engine/core/def.rs
@@ -67,6 +67,10 @@ impl<T: Clone> AsValue for T {
     }
 }
 
+#[cfg(debug_assertions)]
+/// A dummy metrics object
+pub struct DummyMetrics;
+
 /// The base spec for any index. Iterators have meaningless order, and that is intentional and oftentimes
 /// consequential. For more specialized impls, use the [`STIndex`], [`MTIndex`] or [`STIndexSeq`] traits
 pub trait IndexBaseSpec<K, V>
@@ -76,6 +80,9 @@ where
 {
     /// Index supports prealloc?
     const PREALLOC: bool;
+    #[cfg(debug_assertions)]
+    /// A type representing debug metrics
+    type Metrics;
     /// An iterator over the keys and values
     type IterKV<'a>: Iterator<Item = (&'a K, &'a V)>
     where
@@ -104,6 +111,9 @@ where
     fn idx_iter_key<'a>(&'a self) -> Self::IterKey<'a>;
     /// Returns an iterator over the values
     fn idx_iter_value<'a>(&'a self) -> Self::IterValue<'a>;
+    #[cfg(debug_assertions)]
+    /// Returns a reference to the index metrics
+    fn idx_metrics(&self) -> &Self::Metrics;
 }
 
 /// An unordered MTIndex
diff --git a/server/src/engine/core/idx.rs b/server/src/engine/core/idx.rs
index 978f3ac8..5bf72f31 100644
--- a/server/src/engine/core/idx.rs
+++ b/server/src/engine/core/idx.rs
@@ -41,7 +41,9 @@ use std::{
 };
 
 // re-exports for convenience
-pub type IndexSTSeq<K, V, S = RandomState> = IndexSTSeqDll<K, V, S>;
+pub type IndexSTSeq<K, V, S> = IndexSTSeqDll<K, V, S>;
+pub type IndexSTSeqDef<K, V> = IndexSTSeq<K, V, IndexSTSeqHasher>;
+pub type IndexSTSeqHasher = RandomState;
 
 /*
     For the ordered index impl, we resort to some crazy unsafe code, especially because there's no other way to
@@ -212,11 +214,27 @@ impl<K, V> IndexSTSeqDllNode<K, V> {
 
 type IndexSTSeqDllNodePtr<K, V> = NonNull<IndexSTSeqDllNode<K, V>>;
 
+#[cfg(debug_assertions)]
+pub struct IndexSTSeqDllMetrics {
+    stat_f: usize,
+}
+
+impl IndexSTSeqDllMetrics {
+    pub fn report_f(&self) -> usize {
+        self.stat_f
+    }
+    fn new() -> IndexSTSeqDllMetrics {
+        Self { stat_f: 0 }
+    }
+}
+
 /// An ST-index with ordering. Inefficient ordered scanning since not in block
 pub struct IndexSTSeqDll<K, V, S> {
     m: StdMap<IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>, S>,
     h: *mut IndexSTSeqDllNode<K, V>,
     f: *mut IndexSTSeqDllNode<K, V>,
+    #[cfg(debug_assertions)]
+    metrics: IndexSTSeqDllMetrics,
 }
 
 impl<K, V, S: BuildHasher> IndexSTSeqDll<K, V, S> {
@@ -227,7 +245,12 @@ impl<K, V, S: BuildHasher> IndexSTSeqDll<K, V, S> {
         h: *mut IndexSTSeqDllNode<K, V>,
         f: *mut IndexSTSeqDllNode<K, V>,
     ) -> IndexSTSeqDll<K, V, S> {
-        Self { m, h, f }
+        Self {
+            m,
+            h,
+            f,
+            metrics: IndexSTSeqDllMetrics::new(),
+        }
     }
     #[inline(always)]
     fn _new_map(m: StdMap<IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>, S>) -> Self {
@@ -255,6 +278,20 @@ impl<K, V> IndexSTSeqDll<K, V, RandomState> {
 }
 
 impl<K, V, S> IndexSTSeqDll<K, V, S> {
+    #[inline(always)]
+    fn metrics_update_f_decr(&mut self) {
+        #[cfg(debug_assertions)]
+        {
+            self.metrics.stat_f -= 1;
+        }
+    }
+    #[inline(always)]
+    fn metrics_update_f_incr(&mut self) {
+        #[cfg(debug_assertions)]
+        {
+            self.metrics.stat_f += 1;
+        }
+    }
     #[inline(always)]
     fn ensure_sentinel(&mut self) {
         if self.h.is_null() {
@@ -289,6 +326,7 @@ impl<K, V, S> IndexSTSeqDll<K, V, S> {
                 let nx = (*c).n;
                 IndexSTSeqDllNode::dealloc_headless(c);
                 c = nx;
+                self.metrics_update_f_decr();
             }
         }
         self.f = ptr::null_mut();
@@ -298,6 +336,7 @@ impl<K, V, S> IndexSTSeqDll<K, V, S> {
         if self.f.is_null() {
             IndexSTSeqDllNode::alloc_box(node)
         } else {
+            self.metrics_update_f_decr();
             unsafe {
                 // UNSAFE(@ohsayan): Safe because we already did a nullptr check
                 let f = self.f;
@@ -404,6 +443,7 @@ impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
                 IndexSTSeqDllNode::unlink(n);
                 (*n).n = self.f;
                 self.f = n;
+                self.metrics_update_f_incr();
                 v
             })
     }
@@ -521,6 +561,8 @@ where
 {
     const PREALLOC: bool = true;
 
+    type Metrics = IndexSTSeqDllMetrics;
+
     type IterKV<'a> = IndexSTSeqDllIterUnordKV<'a, K, V>
     where
         Self: 'a,
@@ -556,6 +598,11 @@ where
     fn idx_iter_value<'a>(&'a self) -> Self::IterValue<'a> {
         self._iter_unord_v()
     }
+
+    #[cfg(debug_assertions)]
+    fn idx_metrics(&self) -> &Self::Metrics {
+        &self.metrics
+    }
 }
 
 impl<K, V, S: BuildHasher + Default> STIndex<K, V> for IndexSTSeqDll<K, V, S>
@@ -596,7 +643,7 @@ where
         K: Borrow<Q>,
         Q: ?Sized + AsKeyRef,
     {
-        self._update(key, val).is_none()
+        self._update(key, val).is_some()
     }
 
     fn st_update_return<Q>(&mut self, key: &Q, val: V) -> Option<V>
@@ -659,6 +706,18 @@ where
     }
 }
 
+impl<K: AsKey, V: AsValue, S: BuildHasher + Default> Clone for IndexSTSeqDll<K, V, S> {
+    fn clone(&self) -> Self {
+        let mut slf = Self::with_capacity_and_hasher(self.len(), S::default());
+        self._iter_ord_kv()
+            .map(|(k, v)| (k.clone(), v.clone()))
+            .for_each(|(k, v)| {
+                slf._insert(k, v);
+            });
+        slf
+    }
+}
+
 unsafe impl<K: Send, V: Send, S: Send> Send for IndexSTSeqDll<K, V, S> {}
 unsafe impl<K: Sync, V: Sync, S: Sync> Sync for IndexSTSeqDll<K, V, S> {}
 
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index c7f973d7..ff1153b1 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -28,6 +28,8 @@
 
 mod def;
 mod idx;
+#[cfg(test)]
+mod tests;
 use super::ql::lexer::{Lit, LitIR};
 
 /// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
diff --git a/server/src/engine/core/tests/mod.rs b/server/src/engine/core/tests/mod.rs
new file mode 100644
index 00000000..66bdd100
--- /dev/null
+++ b/server/src/engine/core/tests/mod.rs
@@ -0,0 +1,77 @@
+/*
+ * Created on Wed Jan 18 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::*;
+
+mod idx_st_seq_dll {
+    use super::{
+        def::{IndexBaseSpec, STIndex},
+        idx::IndexSTSeqDef,
+    };
+    use rand::{distributions::Alphanumeric, Rng};
+
+    const SPAM_CNT: usize = 100_000;
+    const SPAM_SIZE: usize = 128;
+    type Index = IndexSTSeqDef<String, String>;
+
+    #[inline(always)]
+    fn s<'a>(s: &'a str) -> String {
+        s.to_owned()
+    }
+    fn ranstr(rand: &mut impl Rng) -> String {
+        rand.sample_iter(Alphanumeric)
+            .take(SPAM_SIZE)
+            .map(char::from)
+            .collect()
+    }
+    #[test]
+    fn empty_drop() {
+        let idx = Index::idx_init();
+        drop(idx);
+    }
+    #[test]
+    fn simple_crud() {
+        let mut idx = Index::idx_init();
+        assert!(idx.st_insert(s("hello"), s("world")));
+        assert_eq!(idx.st_get("hello").as_deref().unwrap(), "world");
+        assert!(idx.st_update("hello", s("world2")));
+        assert_eq!(idx.st_get("hello").as_deref().unwrap(), "world2");
+        assert_eq!(idx.st_delete_return("hello").unwrap(), "world2");
+        assert_eq!(idx.idx_metrics().report_f(), 1);
+    }
+    #[test]
+    fn spam_crud() {
+        let mut idx = IndexSTSeqDef::idx_init();
+        for int in 0..SPAM_CNT {
+            assert!(idx.st_insert(int, int + 1));
+            assert_eq!(*idx.st_get(&int).unwrap(), int + 1);
+            assert!(idx.st_update(&int, int + 2));
+            assert_eq!(*idx.st_get(&int).unwrap(), int + 2);
+            assert_eq!(idx.st_delete_return(&int).unwrap(), int + 2);
+        }
+        assert_eq!(idx.idx_metrics().report_f(), 1);
+    }
+}

From 9d3cef9685f402515a13386a6e92f75b00253c55 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 19 Jan 2023 05:56:03 -0800
Subject: [PATCH 080/310] Add tests for ord index

---
 server/src/engine/core/def.rs       |   4 +
 server/src/engine/core/idx.rs       |   8 +-
 server/src/engine/core/tests/mod.rs | 125 +++++++++++++++++++++++++++-
 3 files changed, 131 insertions(+), 6 deletions(-)

diff --git a/server/src/engine/core/def.rs b/server/src/engine/core/def.rs
index 08d4b969..3afb6243 100644
--- a/server/src/engine/core/def.rs
+++ b/server/src/engine/core/def.rs
@@ -122,6 +122,8 @@ where
     K: AsKey,
     V: AsValue,
 {
+    /// Attempts to compact the backing storage
+    fn mt_compact(&self) {}
     /// Clears all the entries in the MTIndex
     fn mt_clear(&self);
     // write
@@ -171,6 +173,8 @@ where
     K: AsKey,
     V: AsValue,
 {
+    /// Attempts to compact the backing storage
+    fn st_compact(&mut self) {}
     /// Clears all the entries in the STIndex
     fn st_clear(&mut self);
     // write
diff --git a/server/src/engine/core/idx.rs b/server/src/engine/core/idx.rs
index 5bf72f31..f2da91de 100644
--- a/server/src/engine/core/idx.rs
+++ b/server/src/engine/core/idx.rs
@@ -220,7 +220,7 @@ pub struct IndexSTSeqDllMetrics {
 }
 
 impl IndexSTSeqDllMetrics {
-    pub fn report_f(&self) -> usize {
+    pub fn raw_f(&self) -> usize {
         self.stat_f
     }
     fn new() -> IndexSTSeqDllMetrics {
@@ -610,6 +610,10 @@ where
     K: AsKey,
     V: AsValue,
 {
+    fn st_compact(&mut self) {
+        self.vacuum_full();
+    }
+
     fn st_clear(&mut self) {
         self._clear()
     }
@@ -659,7 +663,7 @@ where
         K: Borrow<Q>,
         Q: ?Sized + AsKeyRef,
     {
-        self._remove(key).is_none()
+        self._remove(key).is_some()
     }
 
     fn st_delete_return<Q>(&mut self, key: &Q) -> Option<V>
diff --git a/server/src/engine/core/tests/mod.rs b/server/src/engine/core/tests/mod.rs
index 66bdd100..decd07af 100644
--- a/server/src/engine/core/tests/mod.rs
+++ b/server/src/engine/core/tests/mod.rs
@@ -28,15 +28,29 @@ use super::*;
 
 mod idx_st_seq_dll {
     use super::{
-        def::{IndexBaseSpec, STIndex},
+        def::{IndexBaseSpec, STIndex, STIndexSeq},
         idx::IndexSTSeqDef,
     };
     use rand::{distributions::Alphanumeric, Rng};
 
-    const SPAM_CNT: usize = 100_000;
+    const SPAM_CNT: usize = 131_072;
     const SPAM_SIZE: usize = 128;
     type Index = IndexSTSeqDef<String, String>;
 
+    /// Returns an index with: `i -> "{i+1}"` starting from 0 upto the value of [`SPAM_CNT`]
+    fn mkidx() -> IndexSTSeqDef<usize, String> {
+        let mut idx = IndexSTSeqDef::idx_init();
+        for int in 0..SPAM_CNT {
+            assert!(idx.st_insert(int, (int + 1).to_string()));
+        }
+        // verify data
+        for int in 0..SPAM_CNT {
+            assert_eq!(idx.st_get(&int).unwrap().as_str(), (int + 1).to_string());
+        }
+        assert_eq!(idx.idx_metrics().raw_f(), 0);
+        idx
+    }
+
     #[inline(always)]
     fn s<'a>(s: &'a str) -> String {
         s.to_owned()
@@ -50,9 +64,38 @@ mod idx_st_seq_dll {
     #[test]
     fn empty_drop() {
         let idx = Index::idx_init();
+        assert_eq!(idx.idx_metrics().raw_f(), 0);
         drop(idx);
     }
     #[test]
+    fn spam_read_nx() {
+        let idx = IndexSTSeqDef::<usize, String>::new();
+        for int in SPAM_CNT..SPAM_CNT * 2 {
+            assert!(idx.st_get(&int).is_none());
+        }
+    }
+    #[test]
+    fn spam_insert_ex() {
+        let mut idx = mkidx();
+        for int in 0..SPAM_CNT {
+            assert!(!idx.st_insert(int, (int + 2).to_string()));
+        }
+    }
+    #[test]
+    fn spam_update_nx() {
+        let mut idx = IndexSTSeqDef::<usize, String>::new();
+        for int in 0..SPAM_CNT {
+            assert!(!idx.st_update(&int, (int + 2).to_string()));
+        }
+    }
+    #[test]
+    fn spam_delete_nx() {
+        let mut idx = IndexSTSeqDef::<usize, String>::new();
+        for int in 0..SPAM_CNT {
+            assert!(!idx.st_delete(&int));
+        }
+    }
+    #[test]
     fn simple_crud() {
         let mut idx = Index::idx_init();
         assert!(idx.st_insert(s("hello"), s("world")));
@@ -60,7 +103,7 @@ mod idx_st_seq_dll {
         assert!(idx.st_update("hello", s("world2")));
         assert_eq!(idx.st_get("hello").as_deref().unwrap(), "world2");
         assert_eq!(idx.st_delete_return("hello").unwrap(), "world2");
-        assert_eq!(idx.idx_metrics().report_f(), 1);
+        assert_eq!(idx.idx_metrics().raw_f(), 1);
     }
     #[test]
     fn spam_crud() {
@@ -72,6 +115,80 @@ mod idx_st_seq_dll {
             assert_eq!(*idx.st_get(&int).unwrap(), int + 2);
             assert_eq!(idx.st_delete_return(&int).unwrap(), int + 2);
         }
-        assert_eq!(idx.idx_metrics().report_f(), 1);
+        assert_eq!(idx.idx_metrics().raw_f(), 1);
+    }
+    #[test]
+    fn spam_read() {
+        let mut idx = IndexSTSeqDef::idx_init();
+        for int in 0..SPAM_CNT {
+            let v = (int + 1).to_string();
+            assert!(idx.st_insert(int, v.clone()));
+            assert_eq!(idx.st_get(&int).as_deref().unwrap(), &v);
+        }
+        assert_eq!(idx.idx_metrics().raw_f(), 0);
+    }
+    #[test]
+    fn spam_update() {
+        let mut idx = mkidx();
+        for int in 0..SPAM_CNT {
+            assert_eq!(
+                idx.st_update_return(&int, (int + 2).to_string()).unwrap(),
+                (int + 1).to_string()
+            );
+        }
+        assert_eq!(idx.idx_metrics().raw_f(), 0);
+    }
+    #[test]
+    fn spam_delete() {
+        let mut idx = mkidx();
+        for int in 0..SPAM_CNT {
+            assert_eq!(idx.st_delete_return(&int).unwrap(), (int + 1).to_string());
+            assert_eq!(idx.idx_metrics().raw_f(), int + 1);
+        }
+        assert_eq!(idx.idx_metrics().raw_f(), SPAM_CNT);
+    }
+    #[test]
+    fn compact() {
+        let mut idx = mkidx();
+        assert_eq!(idx.idx_metrics().raw_f(), 0);
+        for int in 0..SPAM_CNT {
+            let _ = idx.st_delete(&int);
+        }
+        assert_eq!(idx.idx_metrics().raw_f(), SPAM_CNT);
+        idx.st_clear();
+        assert_eq!(idx.idx_metrics().raw_f(), SPAM_CNT);
+        idx.st_compact();
+        assert_eq!(idx.idx_metrics().raw_f(), 0);
+    }
+    // pointless testing random iterators
+    #[test]
+    fn iter_ord() {
+        let idx1 = mkidx();
+        let idx2: Vec<(usize, String)> =
+            idx1.stseq_ord_kv().map(|(k, v)| (*k, v.clone())).collect();
+        (0..SPAM_CNT)
+            .into_iter()
+            .zip(idx2.into_iter())
+            .for_each(|(i, (k, v))| {
+                assert_eq!(i, k);
+                assert_eq!((i + 1).to_string(), v);
+            });
+    }
+    #[test]
+    fn iter_ord_rev() {
+        let idx1 = mkidx();
+        let idx2: Vec<(usize, String)> = idx1
+            .stseq_ord_kv()
+            .rev()
+            .map(|(k, v)| (*k, v.clone()))
+            .collect();
+        (0..SPAM_CNT)
+            .rev()
+            .into_iter()
+            .zip(idx2.into_iter())
+            .for_each(|(i, (k, v))| {
+                assert_eq!(i, k);
+                assert_eq!((i + 1).to_string(), v);
+            });
     }
 }

From 27e462fb4f5af1a336feaf280307a1051bcc7e9d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 19 Jan 2023 07:29:34 -0800
Subject: [PATCH 081/310] Refactor modules

---
 server/src/engine/core/mod.rs                 |  6 +-
 server/src/engine/{core/def.rs => idx/mod.rs} | 33 ++++++++
 .../src/engine/{core/idx.rs => idx/stseq.rs}  |  2 +-
 .../{core/tests/mod.rs => idx/tests.rs}       |  5 +-
 server/src/engine/mod.rs                      |  1 +
 server/src/engine/ql/ast.rs                   | 18 ++---
 server/src/engine/ql/benches.rs               |  4 +-
 server/src/engine/ql/ddl.rs                   |  2 +-
 .../src/engine/ql/dml/{delete.rs => del.rs}   |  2 +-
 .../src/engine/ql/dml/{insert.rs => ins.rs}   |  2 +-
 server/src/engine/ql/dml/mod.rs               | 10 +--
 .../src/engine/ql/dml/{select.rs => sel.rs}   |  2 +-
 .../src/engine/ql/dml/{update.rs => upd.rs}   |  4 +-
 server/src/engine/ql/{lexer.rs => lex.rs}     |  0
 server/src/engine/ql/macros.rs                |  4 +-
 server/src/engine/ql/mod.rs                   |  2 +-
 server/src/engine/ql/schema.rs                |  2 +-
 server/src/engine/ql/tests.rs                 |  4 +-
 server/src/engine/ql/tests/dml_tests.rs       | 78 +++++++++----------
 server/src/engine/ql/tests/lexer_tests.rs     |  8 +-
 server/src/engine/ql/tests/schema_tests.rs    |  4 +-
 server/src/engine/ql/tests/structure_syn.rs   |  4 +-
 22 files changed, 112 insertions(+), 85 deletions(-)
 rename server/src/engine/{core/def.rs => idx/mod.rs} (86%)
 rename server/src/engine/{core/idx.rs => idx/stseq.rs} (99%)
 rename server/src/engine/{core/tests/mod.rs => idx/tests.rs} (98%)
 rename server/src/engine/ql/dml/{delete.rs => del.rs} (98%)
 rename server/src/engine/ql/dml/{insert.rs => ins.rs} (99%)
 rename server/src/engine/ql/dml/{select.rs => sel.rs} (99%)
 rename server/src/engine/ql/dml/{update.rs => upd.rs} (98%)
 rename server/src/engine/ql/{lexer.rs => lex.rs} (100%)

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index ff1153b1..bef51661 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -26,11 +26,7 @@
 
 // TODO(@ohsayan): Change the underlying structures, there are just rudimentary ones used during integration with the QL
 
-mod def;
-mod idx;
-#[cfg(test)]
-mod tests;
-use super::ql::lexer::{Lit, LitIR};
+use super::ql::lex::{Lit, LitIR};
 
 /// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
 /// be of one type.
diff --git a/server/src/engine/core/def.rs b/server/src/engine/idx/mod.rs
similarity index 86%
rename from server/src/engine/core/def.rs
rename to server/src/engine/idx/mod.rs
index 3afb6243..cb6defc9 100644
--- a/server/src/engine/core/def.rs
+++ b/server/src/engine/idx/mod.rs
@@ -1,3 +1,29 @@
+/*
+ * Created on Thu Jan 19 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
 /*
  * Created on Wed Jan 11 2023
  *
@@ -26,6 +52,13 @@
 
 use core::{borrow::Borrow, hash::Hash};
 
+mod stseq;
+#[cfg(test)]
+mod tests;
+
+// re-exports
+pub use stseq::{IndexSTSeq, IndexSTSeqDef, IndexSTSeqHasher};
+
 /// Any type implementing this trait can be used as a key inside memory engine structures
 pub trait AsKey: Hash + Eq + Clone {
     /// Read the key
diff --git a/server/src/engine/core/idx.rs b/server/src/engine/idx/stseq.rs
similarity index 99%
rename from server/src/engine/core/idx.rs
rename to server/src/engine/idx/stseq.rs
index f2da91de..11e6683d 100644
--- a/server/src/engine/core/idx.rs
+++ b/server/src/engine/idx/stseq.rs
@@ -24,7 +24,7 @@
  *
 */
 
-use super::def::{AsKey, AsKeyRef, AsValue, IndexBaseSpec, STIndex, STIndexSeq};
+use super::{AsKey, AsKeyRef, AsValue, IndexBaseSpec, STIndex, STIndexSeq};
 use std::{
     alloc::{alloc as std_alloc, dealloc as std_dealloc, Layout},
     borrow::Borrow,
diff --git a/server/src/engine/core/tests/mod.rs b/server/src/engine/idx/tests.rs
similarity index 98%
rename from server/src/engine/core/tests/mod.rs
rename to server/src/engine/idx/tests.rs
index decd07af..07ce1de2 100644
--- a/server/src/engine/core/tests/mod.rs
+++ b/server/src/engine/idx/tests.rs
@@ -27,10 +27,7 @@
 use super::*;
 
 mod idx_st_seq_dll {
-    use super::{
-        def::{IndexBaseSpec, STIndex, STIndexSeq},
-        idx::IndexSTSeqDef,
-    };
+    use super::{IndexBaseSpec, IndexSTSeqDef, STIndex, STIndexSeq};
     use rand::{distributions::Alphanumeric, Rng};
 
     const SPAM_CNT: usize = 131_072;
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index e1227466..228ef3c6 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -30,4 +30,5 @@
 #[macro_use]
 mod macros;
 mod core;
+mod idx;
 mod ql;
diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index 3de85036..c1a00b7a 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -27,7 +27,7 @@
 use {
     super::{
         ddl, dml,
-        lexer::{LitIR, Slice, Token},
+        lex::{LitIR, Slice, Token},
         schema, LangError, LangResult,
     },
     crate::{
@@ -538,13 +538,13 @@ pub enum Statement<'a> {
     /// DDL query to inspect all spaces (returns a list of spaces in the database)
     InspectSpaces,
     /// DML insert
-    Insert(dml::insert::InsertStatement<'a>),
+    Insert(dml::ins::InsertStatement<'a>),
     /// DML select
-    Select(dml::select::SelectStatement<'a>),
+    Select(dml::sel::SelectStatement<'a>),
     /// DML update
-    Update(dml::update::UpdateStatement<'a>),
+    Update(dml::upd::UpdateStatement<'a>),
     /// DML delete
-    Delete(dml::delete::DeleteStatement<'a>),
+    Delete(dml::del::DeleteStatement<'a>),
 }
 
 #[inline(always)]
@@ -578,16 +578,16 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], d: Qd) -> LangResult<Sta
         Token::Ident(id) if id.eq_ignore_ascii_case(b"inspect") => ddl::parse_inspect(&mut state),
         // DML
         Token![insert] => {
-            dml::insert::InsertStatement::parse_insert(&mut state).map(Statement::Insert)
+            dml::ins::InsertStatement::parse_insert(&mut state).map(Statement::Insert)
         }
         Token![select] => {
-            dml::select::SelectStatement::parse_select(&mut state).map(Statement::Select)
+            dml::sel::SelectStatement::parse_select(&mut state).map(Statement::Select)
         }
         Token![update] => {
-            dml::update::UpdateStatement::parse_update(&mut state).map(Statement::Update)
+            dml::upd::UpdateStatement::parse_update(&mut state).map(Statement::Update)
         }
         Token![delete] => {
-            dml::delete::DeleteStatement::parse_delete(&mut state).map(Statement::Delete)
+            dml::del::DeleteStatement::parse_delete(&mut state).map(Statement::Delete)
         }
         _ => compiler::cold_rerr(LangError::ExpectedStatement),
     }
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index 050f560e..62f560ab 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -42,7 +42,7 @@ use {crate::engine::ql::tests::lex_insecure, test::Bencher};
 mod lexer {
     use {
         super::*,
-        crate::engine::ql::lexer::{Lit, Token},
+        crate::engine::ql::lex::{Lit, Token},
     };
     #[bench]
     fn lex_number(b: &mut Bencher) {
@@ -122,7 +122,7 @@ mod ddl_queries {
         super::*,
         crate::engine::ql::{
             ast::{compile, Entity, InplaceData, Statement},
-            lexer::InsecureLexer,
+            lex::InsecureLexer,
         },
     };
     mod use_stmt {
diff --git a/server/src/engine/ql/ddl.rs b/server/src/engine/ql/ddl.rs
index 895fab67..bb49e8b7 100644
--- a/server/src/engine/ql/ddl.rs
+++ b/server/src/engine/ql/ddl.rs
@@ -29,7 +29,7 @@ use super::ast::InplaceData;
 use {
     super::{
         ast::{Entity, QueryData, State, Statement},
-        lexer::{Slice, Token},
+        lex::{Slice, Token},
         LangError, LangResult,
     },
     crate::util::compiler,
diff --git a/server/src/engine/ql/dml/delete.rs b/server/src/engine/ql/dml/del.rs
similarity index 98%
rename from server/src/engine/ql/dml/delete.rs
rename to server/src/engine/ql/dml/del.rs
index 92989c96..54eea275 100644
--- a/server/src/engine/ql/dml/delete.rs
+++ b/server/src/engine/ql/dml/del.rs
@@ -37,7 +37,7 @@ use {
 #[cfg(test)]
 use {
     super::WhereClauseCollection,
-    crate::engine::ql::{ast::InplaceData, lexer::Token},
+    crate::engine::ql::{ast::InplaceData, lex::Token},
 };
 
 /*
diff --git a/server/src/engine/ql/dml/insert.rs b/server/src/engine/ql/dml/ins.rs
similarity index 99%
rename from server/src/engine/ql/dml/insert.rs
rename to server/src/engine/ql/dml/ins.rs
index 2074d123..8de9360d 100644
--- a/server/src/engine/ql/dml/insert.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -33,7 +33,7 @@ use {
             core::DataType,
             ql::{
                 ast::{Entity, QueryData, State},
-                lexer::Token,
+                lex::Token,
                 LangError, LangResult,
             },
         },
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index 65c9c918..90ec4969 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -29,17 +29,17 @@
     should augment in future revisions of the QL engine
 */
 
-pub mod delete;
-pub mod insert;
-pub mod select;
-pub mod update;
+pub mod del;
+pub mod ins;
+pub mod sel;
+pub mod upd;
 
 #[cfg(test)]
 use super::ast::InplaceData;
 use {
     super::{
         ast::{QueryData, State},
-        lexer::{LitIR, Token},
+        lex::{LitIR, Token},
     },
     crate::util::compiler,
     std::collections::HashMap,
diff --git a/server/src/engine/ql/dml/select.rs b/server/src/engine/ql/dml/sel.rs
similarity index 99%
rename from server/src/engine/ql/dml/select.rs
rename to server/src/engine/ql/dml/sel.rs
index a538b3c1..0242f910 100644
--- a/server/src/engine/ql/dml/select.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -31,7 +31,7 @@ use {
     crate::{
         engine::ql::{
             ast::{Entity, QueryData, State},
-            lexer::Token,
+            lex::Token,
             LangError, LangResult,
         },
         util::{compiler, MaybeInit},
diff --git a/server/src/engine/ql/dml/update.rs b/server/src/engine/ql/dml/upd.rs
similarity index 98%
rename from server/src/engine/ql/dml/update.rs
rename to server/src/engine/ql/dml/upd.rs
index 6f0f7b05..9245eed8 100644
--- a/server/src/engine/ql/dml/update.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -27,14 +27,14 @@
 #[cfg(test)]
 use {
     super::WhereClauseCollection,
-    crate::engine::ql::{ast::InplaceData, lexer::Token},
+    crate::engine::ql::{ast::InplaceData, lex::Token},
 };
 use {
     super::{read_ident, u, WhereClause},
     crate::{
         engine::ql::{
             ast::{Entity, QueryData, State},
-            lexer::LitIR,
+            lex::LitIR,
             LangError, LangResult,
         },
         util::{compiler, MaybeInit},
diff --git a/server/src/engine/ql/lexer.rs b/server/src/engine/ql/lex.rs
similarity index 100%
rename from server/src/engine/ql/lexer.rs
rename to server/src/engine/ql/lex.rs
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index afdcff3b..10b9ff99 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -41,13 +41,13 @@ macro_rules! assert_full_tt {
 
 macro_rules! __sym_token {
     ($ident:ident) => {
-        $crate::engine::ql::lexer::Token::Symbol($crate::engine::ql::lexer::Symbol::$ident)
+        $crate::engine::ql::lex::Token::Symbol($crate::engine::ql::lex::Symbol::$ident)
     };
 }
 
 macro_rules! __kw {
     ($ident:ident) => {
-        $crate::engine::ql::lexer::Token::Keyword($crate::engine::ql::lexer::Keyword::$ident)
+        $crate::engine::ql::lex::Token::Keyword($crate::engine::ql::lex::Keyword::$ident)
     };
 }
 
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 4da225ea..6d34fce3 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -32,7 +32,7 @@ pub(super) mod ast;
 mod benches;
 pub(super) mod ddl;
 pub(super) mod dml;
-pub(super) mod lexer;
+pub(super) mod lex;
 pub(super) mod schema;
 #[cfg(test)]
 mod tests;
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
index 399a73c8..844c669b 100644
--- a/server/src/engine/ql/schema.rs
+++ b/server/src/engine/ql/schema.rs
@@ -49,7 +49,7 @@ use crate::engine::ql::ast::InplaceData;
 use {
     super::{
         ast::{QueryData, State},
-        lexer::{LitIR, LitIROwned, Slice, Symbol, Token},
+        lex::{LitIR, LitIROwned, Slice, Symbol, Token},
         LangError, LangResult,
     },
     crate::util::{compiler, MaybeInit},
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index e3c71fa3..a9ed24e3 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -26,7 +26,7 @@
 
 use {
     super::{
-        lexer::{InsecureLexer, SafeLexer, Symbol, Token},
+        lex::{InsecureLexer, SafeLexer, Symbol, Token},
         LangResult,
     },
     crate::{engine::core::DataType, util::test_utils},
@@ -91,7 +91,7 @@ impl NullableMapEntry for Null {
     }
 }
 
-impl<'a> NullableMapEntry for super::lexer::Lit<'a> {
+impl<'a> NullableMapEntry for super::lex::Lit<'a> {
     fn data(self) -> Option<super::schema::DictEntry> {
         Some(super::schema::DictEntry::Lit(self.as_ir().to_litir_owned()))
     }
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 9f12332a..f6a8d90c 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -29,8 +29,8 @@ mod list_parse {
     use super::*;
     use crate::engine::ql::{
         ast::{InplaceData, SubstitutedData},
-        dml::insert::parse_list_full,
-        lexer::LitIR,
+        dml::ins::parse_list_full,
+        lex::LitIR,
     };
 
     #[test]
@@ -194,7 +194,7 @@ mod list_parse {
 
 mod tuple_syntax {
     use super::*;
-    use crate::engine::ql::dml::insert::parse_data_tuple_syntax_full;
+    use crate::engine::ql::dml::ins::parse_data_tuple_syntax_full;
 
     #[test]
     fn tuple_mini() {
@@ -284,7 +284,7 @@ mod tuple_syntax {
 }
 mod map_syntax {
     use super::*;
-    use crate::engine::ql::dml::insert::parse_data_map_syntax_full;
+    use crate::engine::ql::dml::ins::parse_data_map_syntax_full;
 
     #[test]
     fn map_mini() {
@@ -387,7 +387,7 @@ mod stmt_insert {
         super::*,
         crate::engine::ql::{
             ast::Entity,
-            dml::{self, insert::InsertStatement},
+            dml::{self, ins::InsertStatement},
         },
     };
 
@@ -399,7 +399,7 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::insert::parse_insert_full(&x[1..]).unwrap();
+        let r = dml::ins::parse_insert_full(&x[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(b"twitter", b"users"),
             into_array_nullable!["sayan"].to_vec().into(),
@@ -421,7 +421,7 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::insert::parse_insert_full(&x[1..]).unwrap();
+        let r = dml::ins::parse_insert_full(&x[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(b"twitter", b"users"),
             into_array_nullable!["sayan", "Sayan", "sayan@example.com", true, 12345, 67890]
@@ -448,7 +448,7 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::insert::parse_insert_full(&x[1..]).unwrap();
+        let r = dml::ins::parse_insert_full(&x[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(b"twitter", b"users"),
             into_array_nullable![
@@ -475,7 +475,7 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::insert::parse_insert_full(&tok[1..]).unwrap();
+        let r = dml::ins::parse_insert_full(&tok[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(b"jotsy", b"app"),
             dict_nullable! {
@@ -500,7 +500,7 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::insert::parse_insert_full(&tok[1..]).unwrap();
+        let r = dml::ins::parse_insert_full(&tok[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(b"jotsy", b"app"),
             dict_nullable! {
@@ -533,7 +533,7 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::insert::parse_insert_full(&tok[1..]).unwrap();
+        let r = dml::ins::parse_insert_full(&tok[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(b"jotsy", b"app"),
             dict_nullable! {
@@ -555,10 +555,10 @@ mod stmt_insert {
     fn insert_tuple_fnsub() {
         let tok =
             lex_insecure(br#"insert into jotsy.app(@uuidstr(), "sayan", @timesec())"#).unwrap();
-        let ret = dml::insert::parse_insert_full(&tok[1..]).unwrap();
+        let ret = dml::ins::parse_insert_full(&tok[1..]).unwrap();
         let expected = InsertStatement::new(
             Entity::Full(b"jotsy", b"app"),
-            into_array_nullable![dml::insert::T_UUIDSTR, "sayan", dml::insert::T_TIMESEC]
+            into_array_nullable![dml::ins::T_UUIDSTR, "sayan", dml::ins::T_TIMESEC]
                 .to_vec()
                 .into(),
         );
@@ -569,13 +569,13 @@ mod stmt_insert {
         let tok = lex_insecure(
             br#"insert into jotsy.app { uuid: @uuidstr(), username: "sayan", signup_time: @timesec() }"#
         ).unwrap();
-        let ret = dml::insert::parse_insert_full(&tok[1..]).unwrap();
+        let ret = dml::ins::parse_insert_full(&tok[1..]).unwrap();
         let expected = InsertStatement::new(
             Entity::Full(b"jotsy", b"app"),
             dict_nullable! {
-                "uuid".as_bytes() => dml::insert::T_UUIDSTR,
+                "uuid".as_bytes() => dml::ins::T_UUIDSTR,
                 "username".as_bytes() => "sayan",
-                "signup_time".as_bytes() => dml::insert::T_TIMESEC,
+                "signup_time".as_bytes() => dml::ins::T_TIMESEC,
             }
             .into(),
         );
@@ -590,8 +590,8 @@ mod stmt_select {
         super::*,
         crate::engine::ql::{
             ast::Entity,
-            dml::{self, select::SelectStatement},
-            lexer::LitIR,
+            dml::{self, sel::SelectStatement},
+            lex::LitIR,
         },
     };
     #[test]
@@ -602,7 +602,7 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = dml::select::parse_select_full(&tok[1..]).unwrap();
+        let r = dml::sel::parse_select_full(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
             Entity::Single(b"users"),
             [].to_vec(),
@@ -623,7 +623,7 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = dml::select::parse_select_full(&tok[1..]).unwrap();
+        let r = dml::sel::parse_select_full(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
             Entity::Single(b"users"),
             [b"field1".as_slice()].to_vec(),
@@ -644,7 +644,7 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = dml::select::parse_select_full(&tok[1..]).unwrap();
+        let r = dml::sel::parse_select_full(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
             Entity::Full(b"twitter", b"users"),
             [b"field1".as_slice()].to_vec(),
@@ -665,7 +665,7 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = dml::select::parse_select_full(&tok[1..]).unwrap();
+        let r = dml::sel::parse_select_full(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
             Entity::Full(b"twitter", b"users"),
             [b"field1".as_slice(), b"field2".as_slice()].to_vec(),
@@ -685,15 +685,15 @@ mod expression_tests {
         crate::engine::ql::{
             dml::{
                 self,
-                update::{AssignmentExpression, Operator},
+                upd::{AssignmentExpression, Operator},
             },
-            lexer::LitIR,
+            lex::LitIR,
         },
     };
     #[test]
     fn expr_assign() {
         let src = lex_insecure(b"username = 'sayan'").unwrap();
-        let r = dml::update::parse_assn_expression_full(&src).unwrap();
+        let r = dml::upd::parse_assn_expression_full(&src).unwrap();
         assert_eq!(
             r,
             AssignmentExpression::new(b"username", LitIR::Str("sayan"), Operator::Assign)
@@ -702,7 +702,7 @@ mod expression_tests {
     #[test]
     fn expr_add_assign() {
         let src = lex_insecure(b"followers += 100").unwrap();
-        let r = dml::update::parse_assn_expression_full(&src).unwrap();
+        let r = dml::upd::parse_assn_expression_full(&src).unwrap();
         assert_eq!(
             r,
             AssignmentExpression::new(b"followers", LitIR::UInt(100), Operator::AddAssign)
@@ -711,7 +711,7 @@ mod expression_tests {
     #[test]
     fn expr_sub_assign() {
         let src = lex_insecure(b"following -= 150").unwrap();
-        let r = dml::update::parse_assn_expression_full(&src).unwrap();
+        let r = dml::upd::parse_assn_expression_full(&src).unwrap();
         assert_eq!(
             r,
             AssignmentExpression::new(b"following", LitIR::UInt(150), Operator::SubAssign)
@@ -720,7 +720,7 @@ mod expression_tests {
     #[test]
     fn expr_mul_assign() {
         let src = lex_insecure(b"product_qty *= 2").unwrap();
-        let r = dml::update::parse_assn_expression_full(&src).unwrap();
+        let r = dml::upd::parse_assn_expression_full(&src).unwrap();
         assert_eq!(
             r,
             AssignmentExpression::new(b"product_qty", LitIR::UInt(2), Operator::MulAssign)
@@ -729,7 +729,7 @@ mod expression_tests {
     #[test]
     fn expr_div_assign() {
         let src = lex_insecure(b"image_crop_factor /= 2").unwrap();
-        let r = dml::update::parse_assn_expression_full(&src).unwrap();
+        let r = dml::upd::parse_assn_expression_full(&src).unwrap();
         assert_eq!(
             r,
             AssignmentExpression::new(b"image_crop_factor", LitIR::UInt(2), Operator::DivAssign)
@@ -743,10 +743,10 @@ mod update_statement {
             ast::Entity,
             dml::{
                 self,
-                update::{AssignmentExpression, Operator, UpdateStatement},
+                upd::{AssignmentExpression, Operator, UpdateStatement},
                 RelationalExpr, WhereClause,
             },
-            lexer::LitIR,
+            lex::LitIR,
         },
     };
     #[test]
@@ -757,7 +757,7 @@ mod update_statement {
             "#,
         )
         .unwrap();
-        let r = dml::update::parse_update_full(&tok[1..]).unwrap();
+        let r = dml::upd::parse_update_full(&tok[1..]).unwrap();
         let e = UpdateStatement::new(
             Entity::Single(b"app"),
             vec![AssignmentExpression::new(
@@ -789,7 +789,7 @@ mod update_statement {
             "#,
         )
         .unwrap();
-        let r = dml::update::parse_update_full(&tok[1..]).unwrap();
+        let r = dml::upd::parse_update_full(&tok[1..]).unwrap();
         let e = UpdateStatement::new(
             Entity::Full(b"jotsy", b"app"),
             vec![
@@ -821,10 +821,10 @@ mod delete_stmt {
         crate::engine::ql::{
             ast::Entity,
             dml::{
-                delete::{self, DeleteStatement},
+                del::{self, DeleteStatement},
                 RelationalExpr,
             },
-            lexer::LitIR,
+            lex::LitIR,
         },
     };
 
@@ -846,7 +846,7 @@ mod delete_stmt {
                 )
             },
         );
-        let r = delete::parse_delete_full(&tok[1..]).unwrap();
+        let r = del::parse_delete_full(&tok[1..]).unwrap();
         assert_eq!(r, e);
     }
     #[test]
@@ -867,7 +867,7 @@ mod delete_stmt {
                 )
             },
         );
-        let r = delete::parse_delete_full(&tok[1..]).unwrap();
+        let r = del::parse_delete_full(&tok[1..]).unwrap();
         assert_eq!(r, e);
     }
 }
@@ -876,7 +876,7 @@ mod relational_expr {
         super::*,
         crate::engine::ql::{
             dml::{self, RelationalExpr},
-            lexer::LitIR,
+            lex::LitIR,
         },
     };
 
@@ -964,7 +964,7 @@ mod where_clause {
         super::*,
         crate::engine::ql::{
             dml::{self, RelationalExpr, WhereClause},
-            lexer::LitIR,
+            lex::LitIR,
         },
     };
     #[test]
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index b551b72d..247cf5ad 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -26,7 +26,7 @@
 
 use {
     super::{
-        super::lexer::{Lit, Token},
+        super::lex::{Lit, Token},
         lex_insecure,
     },
     crate::engine::ql::LangError,
@@ -174,7 +174,7 @@ fn lex_unsafe_literal_pro() {
 }
 
 mod num_tests {
-    use crate::engine::ql::lexer::decode_num_ub as ubdc;
+    use crate::engine::ql::lex::decode_num_ub as ubdc;
     mod uint8 {
         use super::*;
         #[test]
@@ -276,7 +276,7 @@ mod num_tests {
 mod safequery_params {
     use rand::seq::SliceRandom;
 
-    use crate::engine::ql::lexer::{LitIR, SafeQueryData};
+    use crate::engine::ql::lex::{LitIR, SafeQueryData};
     #[test]
     fn param_uint() {
         let src = b"12345\n";
@@ -416,7 +416,7 @@ mod safequery_params {
 }
 
 mod safequery_full_param {
-    use crate::engine::ql::lexer::{LitIR, SafeQueryData, Token};
+    use crate::engine::ql::lex::{LitIR, SafeQueryData, Token};
     #[test]
     fn p_mini() {
         let query = b"select * from myapp where username = ?";
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 471f99ad..2a27f5c4 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -26,7 +26,7 @@
 
 use super::{
     super::{
-        lexer::{Lit, Token},
+        lex::{Lit, Token},
         schema,
     },
     lex_insecure, *,
@@ -74,7 +74,7 @@ mod alter_space {
     use {
         super::*,
         crate::engine::ql::{
-            lexer::Lit,
+            lex::Lit,
             schema::{self, AlterSpace},
         },
     };
diff --git a/server/src/engine/ql/tests/structure_syn.rs b/server/src/engine/ql/tests/structure_syn.rs
index ff7e01fd..fd78499b 100644
--- a/server/src/engine/ql/tests/structure_syn.rs
+++ b/server/src/engine/ql/tests/structure_syn.rs
@@ -26,7 +26,7 @@
 
 use {
     super::*,
-    crate::engine::ql::{lexer::Lit, schema},
+    crate::engine::ql::{lex::Lit, schema},
 };
 mod dict {
     use super::*;
@@ -219,7 +219,7 @@ mod dict {
 mod nullable_dict_tests {
     use super::*;
     mod dict {
-        use {super::*, crate::engine::ql::lexer::Lit};
+        use {super::*, crate::engine::ql::lex::Lit};
 
         #[test]
         fn null_mini() {

From 44aa57a25ab4f924bf369ac63bdf5ae47fbccc49 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 20 Jan 2023 14:35:56 +0000
Subject: [PATCH 082/310] Add sync module

---
 Cargo.lock                    | 522 ++++++++++++++++++++--------------
 server/Cargo.toml             |   1 +
 server/src/engine/idx/mod.rs  |  26 --
 server/src/engine/mod.rs      |   1 +
 server/src/engine/sync/atm.rs | 134 +++++++++
 server/src/engine/sync/mod.rs |  27 ++
 6 files changed, 476 insertions(+), 235 deletions(-)
 create mode 100644 server/src/engine/sync/atm.rs
 create mode 100644 server/src/engine/sync/mod.rs

diff --git a/Cargo.lock b/Cargo.lock
index cb09d4bf..2b76bbcf 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -33,9 +33,9 @@ dependencies = [
 
 [[package]]
 name = "ahash"
-version = "0.8.0"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57e6e951cfbb2db8de1828d49073a113a29fd7117b1596caa781a258c7e38d72"
+checksum = "bf6ccdb167abbf410dcb915cabd428929d7f6a04980b54a11f26a39f1c7f7107"
 dependencies = [
  "cfg-if",
  "getrandom",
@@ -45,9 +45,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "0.7.19"
+version = "0.7.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
+checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
 dependencies = [
  "memchr",
 ]
@@ -72,9 +72,9 @@ dependencies = [
 
 [[package]]
 name = "async-trait"
-version = "0.1.57"
+version = "0.1.62"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f"
+checksum = "689894c2db1ea643a50834b999abf1c110887402542955ff5451dab8f861f9ed"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -87,7 +87,7 @@ version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.1.19",
  "libc",
  "winapi",
 ]
@@ -100,15 +100,15 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "base64"
-version = "0.13.0"
+version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
 [[package]]
 name = "base64ct"
-version = "1.0.1"
+version = "1.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a32fd6af2b5827bce66c29053ba0e7c42b9dcab01835835058558c10851a46b"
+checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf"
 
 [[package]]
 name = "bb8"
@@ -159,9 +159,9 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.11.0"
+version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d"
+checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
 
 [[package]]
 name = "byteorder"
@@ -171,15 +171,15 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
 name = "bytes"
-version = "1.2.1"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
+checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c"
 
 [[package]]
 name = "bzip2"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6afcd980b5f3a45017c57e57a2fcccbb351cc43a356ce117ef760ef8052b89b0"
+checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8"
 dependencies = [
  "bzip2-sys",
  "libc",
@@ -198,9 +198,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.73"
+version = "1.0.78"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d"
 dependencies = [
  "jobserver",
 ]
@@ -213,15 +213,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.22"
+version = "0.4.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1"
+checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
 dependencies = [
  "iana-time-zone",
  "js-sys",
  "num-integer",
  "num-traits",
- "time 0.1.44",
+ "time 0.1.45",
  "wasm-bindgen",
  "winapi",
 ]
@@ -263,15 +263,25 @@ dependencies = [
 
 [[package]]
 name = "clipboard-win"
-version = "4.4.2"
+version = "4.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4ab1b92798304eedc095b53942963240037c0516452cb11aeba709d420b2219"
+checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362"
 dependencies = [
  "error-code",
  "str-buf",
  "winapi",
 ]
 
+[[package]]
+name = "codespan-reporting"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
+dependencies = [
+ "termcolor",
+ "unicode-width",
+]
+
 [[package]]
 name = "constant_time_eq"
 version = "0.1.5"
@@ -325,26 +335,24 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.10"
+version = "0.9.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1"
+checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
 dependencies = [
  "autocfg",
  "cfg-if",
  "crossbeam-utils",
  "memoffset",
- "once_cell",
  "scopeguard",
 ]
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.11"
+version = "0.8.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc"
+checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
 dependencies = [
  "cfg-if",
- "once_cell",
 ]
 
 [[package]]
@@ -382,6 +390,50 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "cxx"
+version = "1.0.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b61a7545f753a88bcbe0a70de1fcc0221e10bfc752f576754fa91e663db1622e"
+dependencies = [
+ "cc",
+ "cxxbridge-flags",
+ "cxxbridge-macro",
+ "link-cplusplus",
+]
+
+[[package]]
+name = "cxx-build"
+version = "1.0.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f464457d494b5ed6905c63b0c4704842aba319084a0a3561cdc1359536b53200"
+dependencies = [
+ "cc",
+ "codespan-reporting",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "scratch",
+ "syn",
+]
+
+[[package]]
+name = "cxxbridge-flags"
+version = "1.0.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43c7119ce3a3701ed81aca8410b9acf6fc399d2629d057b87e2efa4e63a3aaea"
+
+[[package]]
+name = "cxxbridge-macro"
+version = "1.0.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65e07508b90551e610910fa648a1878991d367064997a596135b86df30daf07e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "devtimer"
 version = "4.0.1"
@@ -390,9 +442,9 @@ checksum = "907339959a92f6b98846570500c0a567c9aecbb3871cef00561eb5d20d47b7c1"
 
 [[package]]
 name = "digest"
-version = "0.10.3"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
+checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"
 dependencies = [
  "block-buffer",
  "crypto-common",
@@ -434,9 +486,9 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
 
 [[package]]
 name = "env_logger"
-version = "0.9.0"
+version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3"
+checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7"
 dependencies = [
  "atty",
  "humantime",
@@ -478,9 +530,9 @@ dependencies = [
 
 [[package]]
 name = "fd-lock"
-version = "3.0.6"
+version = "3.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e11dcc7e4d79a8c89b9ab4c6f5c30b1fc4a83c420792da3542fd31179ed5f517"
+checksum = "bb21c69b9fea5e15dbc1049e4b77145dd0ba1c84019c488102de0dc4ea4b0a27"
 dependencies = [
  "cfg-if",
  "rustix",
@@ -489,9 +541,9 @@ dependencies = [
 
 [[package]]
 name = "flate2"
-version = "1.0.24"
+version = "1.0.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
+checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841"
 dependencies = [
  "crc32fast",
  "miniz_oxide",
@@ -520,30 +572,30 @@ checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
 
 [[package]]
 name = "futures-channel"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050"
+checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
 dependencies = [
  "futures-core",
 ]
 
 [[package]]
 name = "futures-core"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf"
+checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
 
 [[package]]
 name = "futures-task"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1"
+checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"
 
 [[package]]
 name = "futures-util"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90"
+checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -565,9 +617,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.7"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6"
+checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
 dependencies = [
  "cfg-if",
  "libc",
@@ -605,6 +657,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "hermit-abi"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "hmac"
 version = "0.12.1"
@@ -622,18 +683,28 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.47"
+version = "0.1.53"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c495f162af0bf17656d0014a0eded5f3cd2f365fdd204548c2869db89359dc7"
+checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
+ "iana-time-zone-haiku",
  "js-sys",
- "once_cell",
  "wasm-bindgen",
  "winapi",
 ]
 
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
+dependencies = [
+ "cxx",
+ "cxx-build",
+]
+
 [[package]]
 name = "inout"
 version = "0.1.3"
@@ -645,21 +716,25 @@ dependencies = [
 
 [[package]]
 name = "io-lifetimes"
-version = "0.7.3"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ea37f355c05dde75b84bba2d767906ad522e97cd9e2eef2be7a4ab7fb442c06"
+checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
 
 [[package]]
 name = "itoa"
-version = "1.0.3"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"
+checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
 
 [[package]]
 name = "jemalloc-sys"
-version = "0.5.1+5.3.0-patched"
+version = "0.5.2+5.3.0-patched"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7c2b313609b95939cb0c5a5c6917fb9b7c9394562aa3ef44eb66ffa51736432"
+checksum = "134163979b6eed9564c98637b710b40979939ba351f59952708234ea11b5f3f8"
 dependencies = [
  "cc",
  "fs_extra",
@@ -678,18 +753,18 @@ dependencies = [
 
 [[package]]
 name = "jobserver"
-version = "0.1.24"
+version = "0.1.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
+checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b"
 dependencies = [
  "libc",
 ]
 
 [[package]]
 name = "js-sys"
-version = "0.3.59"
+version = "0.3.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2"
+checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -702,9 +777,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.132"
+version = "0.2.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5"
+checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
 
 [[package]]
 name = "libsky"
@@ -720,17 +795,26 @@ dependencies = [
  "rayon",
 ]
 
+[[package]]
+name = "link-cplusplus"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "linux-raw-sys"
-version = "0.0.46"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d"
+checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
 
 [[package]]
 name = "lock_api"
-version = "0.4.8"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f80bf5aacaf25cbfc8210d1cfb718f2bf3b11c4c54e5afe36c236853a8ec390"
+checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"
 dependencies = [
  "autocfg",
  "scopeguard",
@@ -753,27 +837,27 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
 [[package]]
 name = "memoffset"
-version = "0.6.5"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
+checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
 dependencies = [
  "autocfg",
 ]
 
 [[package]]
 name = "miniz_oxide"
-version = "0.5.4"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
+checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa"
 dependencies = [
  "adler",
 ]
 
 [[package]]
 name = "mio"
-version = "0.8.4"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf"
+checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
 dependencies = [
  "libc",
  "log",
@@ -792,10 +876,11 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.24.2"
+version = "0.25.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc"
+checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4"
 dependencies = [
+ "autocfg",
  "bitflags",
  "cfg-if",
  "libc",
@@ -803,9 +888,9 @@ dependencies = [
 
 [[package]]
 name = "ntapi"
-version = "0.3.7"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f"
+checksum = "bc51db7b362b205941f71232e56c625156eb9a929f8cf74a428fd5bc094a4afc"
 dependencies = [
  "winapi",
 ]
@@ -831,28 +916,19 @@ dependencies = [
 
 [[package]]
 name = "num_cpus"
-version = "1.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
-dependencies = [
- "hermit-abi",
- "libc",
-]
-
-[[package]]
-name = "num_threads"
-version = "0.1.6"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44"
+checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
 dependencies = [
+ "hermit-abi 0.2.6",
  "libc",
 ]
 
 [[package]]
 name = "once_cell"
-version = "1.14.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0"
+checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
 
 [[package]]
 name = "opaque-debug"
@@ -862,9 +938,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
 [[package]]
 name = "openssl"
-version = "0.10.41"
+version = "0.10.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0"
+checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1"
 dependencies = [
  "bitflags",
  "cfg-if",
@@ -888,18 +964,18 @@ dependencies = [
 
 [[package]]
 name = "openssl-src"
-version = "111.22.0+1.1.1q"
+version = "111.24.0+1.1.1s"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f31f0d509d1c1ae9cada2f9539ff8f37933831fd5098879e482aa687d659853"
+checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd"
 dependencies = [
  "cc",
 ]
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.75"
+version = "0.9.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f"
+checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7"
 dependencies = [
  "autocfg",
  "cc",
@@ -921,9 +997,9 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.3"
+version = "0.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
+checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf"
 dependencies = [
  "cfg-if",
  "libc",
@@ -934,9 +1010,9 @@ dependencies = [
 
 [[package]]
 name = "password-hash"
-version = "0.3.2"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d791538a6dcc1e7cb7fe6f6b58aca40e7f79403c45b2bc274008b5e647af1d8"
+checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700"
 dependencies = [
  "base64ct",
  "rand_core",
@@ -945,9 +1021,9 @@ dependencies = [
 
 [[package]]
 name = "pbkdf2"
-version = "0.10.1"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "271779f35b581956db91a3e55737327a03aa051e90b1c47aeb189508533adfd7"
+checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917"
 dependencies = [
  "digest",
  "hmac",
@@ -969,9 +1045,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pkg-config"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
+checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
 
 [[package]]
 name = "powershell_script"
@@ -981,24 +1057,24 @@ checksum = "54bde2e1a395c0aee9423072d781610da37b7b120edf17d4da99f83d04f2cd54"
 
 [[package]]
 name = "ppv-lite86"
-version = "0.2.16"
+version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
+checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.43"
+version = "1.0.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab"
+checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.21"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
 dependencies = [
  "proc-macro2",
 ]
@@ -1047,30 +1123,28 @@ dependencies = [
 
 [[package]]
 name = "rand_core"
-version = "0.6.3"
+version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
  "getrandom",
 ]
 
 [[package]]
 name = "rayon"
-version = "1.5.3"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d"
+checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7"
 dependencies = [
- "autocfg",
- "crossbeam-deque",
  "either",
  "rayon-core",
 ]
 
 [[package]]
 name = "rayon-core"
-version = "1.9.3"
+version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f"
+checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"
 dependencies = [
  "crossbeam-channel",
  "crossbeam-deque",
@@ -1111,9 +1185,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.6.0"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
+checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1122,15 +1196,15 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.27"
+version = "0.6.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
+checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
 
 [[package]]
 name = "rustix"
-version = "0.35.9"
+version = "0.36.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72c825b8aa8010eb9ee99b75f05e10180b9278d161583034d7574c9d617aeada"
+checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03"
 dependencies = [
  "bitflags",
  "errno",
@@ -1142,9 +1216,9 @@ dependencies = [
 
 [[package]]
 name = "rustyline"
-version = "10.0.0"
+version = "10.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d1cd5ae51d3f7bf65d7969d579d502168ef578f289452bd8ccc91de28fda20e"
+checksum = "c1e83c32c3f3c33b08496e0d1df9ea8c64d39adb8eb36a1ebb1440c690697aef"
 dependencies = [
  "bitflags",
  "cfg-if",
@@ -1165,9 +1239,9 @@ dependencies = [
 
 [[package]]
 name = "ryu"
-version = "1.0.11"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
+checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
 
 [[package]]
 name = "scheduled-thread-pool"
@@ -1184,20 +1258,26 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "scratch"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2"
+
 [[package]]
 name = "serde"
-version = "1.0.144"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860"
+checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.144"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94ed3a816fb1d101812f83e789f888322c34e291f894f19590dc310963e87a00"
+checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1206,9 +1286,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.85"
+version = "1.0.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44"
+checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"
 dependencies = [
  "itoa",
  "ryu",
@@ -1217,9 +1297,9 @@ dependencies = [
 
 [[package]]
 name = "sha1"
-version = "0.10.4"
+version = "0.10.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "006769ba83e921b3085caa8334186b00cf92b4cb1a6cf4632fbccc8eff5c7549"
+checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -1228,9 +1308,9 @@ dependencies = [
 
 [[package]]
 name = "sha2"
-version = "0.10.5"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf9db03534dff993187064c4e0c05a5708d2a9728ace9a8959b77bedf415dac5"
+checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -1307,13 +1387,14 @@ dependencies = [
 name = "skyd"
 version = "0.8.0"
 dependencies = [
- "ahash 0.8.0",
+ "ahash 0.8.2",
  "base64",
  "bincode",
  "bytes",
  "cc",
  "chrono",
  "clap",
+ "crossbeam-epoch",
  "env_logger",
  "hashbrown",
  "jemallocator",
@@ -1352,7 +1433,7 @@ dependencies = [
 [[package]]
 name = "skytable"
 version = "0.8.0"
-source = "git+https://github.com/skytable/client-rust?branch=next#d6c43dbe226949b126008ca8fbf14e9b9c60e3fb"
+source = "git+https://github.com/skytable/client-rust?branch=next#c57e46e6ddce57c221e41cd038e7eb8296d2d732"
 dependencies = [
  "async-trait",
  "bb8",
@@ -1366,7 +1447,7 @@ dependencies = [
 [[package]]
 name = "skytable"
 version = "0.8.0"
-source = "git+https://github.com/skytable/client-rust.git#d6c43dbe226949b126008ca8fbf14e9b9c60e3fb"
+source = "git+https://github.com/skytable/client-rust.git#c57e46e6ddce57c221e41cd038e7eb8296d2d732"
 dependencies = [
  "r2d2",
 ]
@@ -1382,9 +1463,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.9.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"
+checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
 [[package]]
 name = "socket2"
@@ -1430,9 +1511,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
 [[package]]
 name = "syn"
-version = "1.0.99"
+version = "1.0.107"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13"
+checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1441,9 +1522,9 @@ dependencies = [
 
 [[package]]
 name = "sysinfo"
-version = "0.26.2"
+version = "0.26.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ae2421f3e16b3afd4aa692d23b83d0ba42ee9b0081d5deeb7d21428d7195fb1"
+checksum = "5c18a6156d1f27a9592ee18c1a846ca8dd5c258b7179fc193ae87c74ebb666f5"
 dependencies = [
  "cfg-if",
  "core-foundation-sys",
@@ -1456,9 +1537,9 @@ dependencies = [
 
 [[package]]
 name = "termcolor"
-version = "1.1.3"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
+checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
 dependencies = [
  "winapi-util",
 ]
@@ -1474,18 +1555,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.34"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c1b05ca9d106ba7d2e31a9dab4a64e7be2cce415321966ea3132c49a656e252"
+checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.34"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8f2591983642de85c921015f3f070c665a197ed69e417af436115e3a1407487"
+checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1494,9 +1575,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.1.44"
+version = "0.1.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
+checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
 dependencies = [
  "libc",
  "wasi 0.10.0+wasi-snapshot-preview1",
@@ -1505,27 +1586,36 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.14"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b"
+checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"
 dependencies = [
  "itoa",
- "libc",
- "num_threads",
+ "serde",
+ "time-core",
  "time-macros",
 ]
 
+[[package]]
+name = "time-core"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
+
 [[package]]
 name = "time-macros"
-version = "0.2.4"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792"
+checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2"
+dependencies = [
+ "time-core",
+]
 
 [[package]]
 name = "tokio"
-version = "1.21.0"
+version = "1.24.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89797afd69d206ccd11fb0ea560a44bbb87731d020670e79416d442919257d42"
+checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb"
 dependencies = [
  "autocfg",
  "bytes",
@@ -1533,20 +1623,19 @@ dependencies = [
  "memchr",
  "mio",
  "num_cpus",
- "once_cell",
  "parking_lot",
  "pin-project-lite",
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "winapi",
+ "windows-sys",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "1.8.0"
+version = "1.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484"
+checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1567,36 +1656,36 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.5.9"
+version = "0.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
+checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "typenum"
-version = "1.15.0"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.3"
+version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf"
+checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.9.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
+checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a"
 
 [[package]]
 name = "unicode-width"
-version = "0.1.9"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
+checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
 
 [[package]]
 name = "utf8parse"
@@ -1658,9 +1747,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.82"
+version = "0.2.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d"
+checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
 dependencies = [
  "cfg-if",
  "wasm-bindgen-macro",
@@ -1668,9 +1757,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.82"
+version = "0.2.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f"
+checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
 dependencies = [
  "bumpalo",
  "log",
@@ -1683,9 +1772,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.82"
+version = "0.2.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602"
+checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -1693,9 +1782,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.82"
+version = "0.2.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da"
+checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1706,9 +1795,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.82"
+version = "0.2.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a"
+checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
 
 [[package]]
 name = "winapi"
@@ -1743,46 +1832,60 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windows-sys"
-version = "0.36.1"
+version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
+checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
 dependencies = [
+ "windows_aarch64_gnullvm",
  "windows_aarch64_msvc",
  "windows_i686_gnu",
  "windows_i686_msvc",
  "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
  "windows_x86_64_msvc",
 ]
 
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
+
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.36.1"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
+checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.36.1"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
+checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.36.1"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
+checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.36.1"
+version = "0.42.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
+checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.36.1"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
+checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
 
 [[package]]
 name = "yaml-rust"
@@ -1792,9 +1895,9 @@ checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992"
 
 [[package]]
 name = "zip"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf225bcf73bb52cbb496e70475c7bd7a3f769df699c0020f6c7bd9a96dcf0b8d"
+checksum = "537ce7411d25e54e8ae21a7ce0b15840e7bfcff15b51d697ec3266cc76bdf080"
 dependencies = [
  "aes",
  "byteorder",
@@ -1806,24 +1909,24 @@ dependencies = [
  "hmac",
  "pbkdf2",
  "sha1",
- "time 0.3.14",
+ "time 0.3.17",
  "zstd",
 ]
 
 [[package]]
 name = "zstd"
-version = "0.10.2+zstd.1.5.2"
+version = "0.11.2+zstd.1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f4a6bd64f22b5e3e94b4e238669ff9f10815c27a5180108b849d24174a83847"
+checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4"
 dependencies = [
  "zstd-safe",
 ]
 
 [[package]]
 name = "zstd-safe"
-version = "4.1.6+zstd.1.5.2"
+version = "5.0.2+zstd.1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94b61c51bb270702d6167b8ce67340d2754b088d0c091b06e593aa772c3ee9bb"
+checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db"
 dependencies = [
  "libc",
  "zstd-sys",
@@ -1831,10 +1934,11 @@ dependencies = [
 
 [[package]]
 name = "zstd-sys"
-version = "1.6.3+zstd.1.5.2"
+version = "2.0.5+zstd.1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc49afa5c8d634e75761feda8c592051e7eeb4683ba827211eb0d731d3402ea8"
+checksum = "edc50ffce891ad571e9f9afe5039c4837bede781ac4bb13052ed7ae695518596"
 dependencies = [
  "cc",
  "libc",
+ "pkg-config",
 ]
diff --git a/server/Cargo.toml b/server/Cargo.toml
index cf930e69..d2c4862b 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -19,6 +19,7 @@ env_logger = "0.9.0"
 hashbrown = { version = "0.12.3", features = ["raw"] }
 log = "0.4.17"
 openssl = { version = "0.10.41", features = ["vendored"] }
+crossbeam-epoch = "0.9.13"
 parking_lot = "0.12.1"
 regex = "1.6.0"
 serde = { version = "1.0.144", features = ["derive"] }
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index cb6defc9..52d08cc2 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -24,32 +24,6 @@
  *
 */
 
-/*
- * Created on Wed Jan 11 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
 use core::{borrow::Borrow, hash::Hash};
 
 mod stseq;
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 228ef3c6..7d8a676a 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -32,3 +32,4 @@ mod macros;
 mod core;
 mod idx;
 mod ql;
+mod sync;
diff --git a/server/src/engine/sync/atm.rs b/server/src/engine/sync/atm.rs
new file mode 100644
index 00000000..dd4e2488
--- /dev/null
+++ b/server/src/engine/sync/atm.rs
@@ -0,0 +1,134 @@
+/*
+ * Created on Fri Jan 20 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use core::{
+    fmt,
+    mem::{self, size_of},
+    ops::Deref,
+};
+use crossbeam_epoch::{
+    Atomic as CBAtomic, CompareExchangeError, Guard, Pointable, Pointer, Shared,
+};
+use std::sync::atomic::Ordering;
+
+pub(super) const ORD_RLX: Ordering = Ordering::Relaxed;
+pub(super) const ORD_ACQ: Ordering = Ordering::Acquire;
+pub(super) const ORD_REL: Ordering = Ordering::Release;
+pub(super) const ORD_ACR: Ordering = Ordering::AcqRel;
+
+type CxResult<'g, T, P> = Result<Shared<'g, T>, CompareExchangeError<'g, T, P>>;
+
+pub(super) const fn ensure_flag_align<T>(fsize: usize) {
+    debug_assert!(mem::align_of::<T>().trailing_zeros() as usize >= fsize);
+}
+
+pub struct Atomic<T> {
+    a: CBAtomic<T>,
+}
+
+// the derive is stupid, it will enforce a debug constraint on T
+impl<T> fmt::Debug for Atomic<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self.a)
+    }
+}
+
+impl<T: Pointable> Atomic<T> {
+    // the compile time address size check ensures "first class" sanity
+    const _ENSURE_FLAG_STATIC_CHECK: () = ensure_flag_align::<T>(size_of::<Self>());
+    pub fn new_alloc(t: T) -> Self {
+        let _ = Self::_ENSURE_FLAG_STATIC_CHECK;
+        Self {
+            a: CBAtomic::new(t),
+        }
+    }
+    #[inline(always)]
+    pub(super) const fn null() -> Self {
+        Self {
+            a: CBAtomic::null(),
+        }
+    }
+    #[inline(always)]
+    pub(super) fn cx<'g, P>(
+        &self,
+        o: Shared<'g, T>,
+        n: P,
+        s: Ordering,
+        f: Ordering,
+        g: &'g Guard,
+    ) -> CxResult<'g, T, P>
+    where
+        P: Pointer<T>,
+    {
+        self.a.compare_exchange(o, n, s, f, g)
+    }
+    #[inline(always)]
+    pub(super) fn cx_weak<'g, P>(
+        &self,
+        o: Shared<'g, T>,
+        n: P,
+        s: Ordering,
+        f: Ordering,
+        g: &'g Guard,
+    ) -> CxResult<'g, T, P>
+    where
+        P: Pointer<T>,
+    {
+        self.a.compare_exchange_weak(o, n, s, f, g)
+    }
+    #[inline(always)]
+    pub(super) fn cx_rel<'g, P>(&self, o: Shared<'g, T>, n: P, g: &'g Guard) -> CxResult<'g, T, P>
+    where
+        P: Pointer<T>,
+    {
+        self.cx(o, n, ORD_REL, ORD_RLX, g)
+    }
+    #[inline(always)]
+    pub(super) fn ld<'g>(&self, o: Ordering, g: &'g Guard) -> Shared<'g, T> {
+        self.a.load(o, g)
+    }
+    #[inline(always)]
+    pub(super) fn ld_acq<'g>(&self, g: &'g Guard) -> Shared<'g, T> {
+        self.ld(ORD_ACQ, g)
+    }
+}
+
+impl<T, A> From<A> for Atomic<T>
+where
+    A: Into<CBAtomic<T>>,
+{
+    fn from(t: A) -> Self {
+        let _ = Self::_ENSURE_FLAG_STATIC_CHECK;
+        Self { a: Into::into(t) }
+    }
+}
+
+impl<T> Deref for Atomic<T> {
+    type Target = CBAtomic<T>;
+    fn deref(&self) -> &Self::Target {
+        &self.a
+    }
+}
diff --git a/server/src/engine/sync/mod.rs b/server/src/engine/sync/mod.rs
new file mode 100644
index 00000000..16a24114
--- /dev/null
+++ b/server/src/engine/sync/mod.rs
@@ -0,0 +1,27 @@
+/*
+ * Created on Thu Jan 19 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+pub(super) mod atm;

From 2dfe7227aa25100dc9e707635b149db76a05049a Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 22 Jan 2023 09:27:05 +0000
Subject: [PATCH 083/310] Add `TMCell`

---
 server/Cargo.toml              |   2 +-
 server/src/engine/sync/atm.rs  |  22 +++--
 server/src/engine/sync/cell.rs | 148 +++++++++++++++++++++++++++++++++
 server/src/engine/sync/mod.rs  |   1 +
 4 files changed, 163 insertions(+), 10 deletions(-)
 create mode 100644 server/src/engine/sync/cell.rs

diff --git a/server/Cargo.toml b/server/Cargo.toml
index d2c4862b..45b26e9a 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -19,7 +19,7 @@ env_logger = "0.9.0"
 hashbrown = { version = "0.12.3", features = ["raw"] }
 log = "0.4.17"
 openssl = { version = "0.10.41", features = ["vendored"] }
-crossbeam-epoch = "0.9.13"
+crossbeam-epoch = { version = "0.9.13" }
 parking_lot = "0.12.1"
 regex = "1.6.0"
 serde = { version = "1.0.144", features = ["derive"] }
diff --git a/server/src/engine/sync/atm.rs b/server/src/engine/sync/atm.rs
index dd4e2488..fd729148 100644
--- a/server/src/engine/sync/atm.rs
+++ b/server/src/engine/sync/atm.rs
@@ -24,15 +24,12 @@
  *
 */
 
-use core::{
-    fmt,
-    mem::{self, size_of},
-    ops::Deref,
+use core::{fmt, mem, ops::Deref, sync::atomic::Ordering};
+use crossbeam_epoch::{Atomic as CBAtomic, CompareExchangeError, Pointable, Pointer};
+// re-export here because we have some future plans ;) (@ohsayan)
+pub use crossbeam_epoch::{
+    pin as pin_current, unprotected as pin_unprotected, Guard, Owned, Shared,
 };
-use crossbeam_epoch::{
-    Atomic as CBAtomic, CompareExchangeError, Guard, Pointable, Pointer, Shared,
-};
-use std::sync::atomic::Ordering;
 
 pub(super) const ORD_RLX: Ordering = Ordering::Relaxed;
 pub(super) const ORD_ACQ: Ordering = Ordering::Acquire;
@@ -58,7 +55,10 @@ impl<T> fmt::Debug for Atomic<T> {
 
 impl<T: Pointable> Atomic<T> {
     // the compile time address size check ensures "first class" sanity
-    const _ENSURE_FLAG_STATIC_CHECK: () = ensure_flag_align::<T>(size_of::<Self>());
+    const _ENSURE_FLAG_STATIC_CHECK: () = ensure_flag_align::<T>(0);
+    /// Instantiates a new atomic
+    ///
+    /// **This will allocate**
     pub fn new_alloc(t: T) -> Self {
         let _ = Self::_ENSURE_FLAG_STATIC_CHECK;
         Self {
@@ -114,6 +114,10 @@ impl<T: Pointable> Atomic<T> {
     pub(super) fn ld_acq<'g>(&self, g: &'g Guard) -> Shared<'g, T> {
         self.ld(ORD_ACQ, g)
     }
+    #[inline(always)]
+    pub(crate) fn ld_rlx<'g>(&self, g: &'g Guard) -> Shared<'g, T> {
+        self.ld(ORD_RLX, g)
+    }
 }
 
 impl<T, A> From<A> for Atomic<T>
diff --git a/server/src/engine/sync/cell.rs b/server/src/engine/sync/cell.rs
new file mode 100644
index 00000000..132a31c9
--- /dev/null
+++ b/server/src/engine/sync/cell.rs
@@ -0,0 +1,148 @@
+/*
+ * Created on Sat Jan 21 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::atm::{pin_unprotected, Atomic, Guard, Owned, Shared, ORD_REL};
+use core::ops::Deref;
+use parking_lot::{Mutex, MutexGuard};
+use std::marker::PhantomData;
+
+/// A [`TMCell`] provides atomic reads and serialized writes; the `static` is a CB hack
+pub struct TMCell<T: 'static> {
+    a: Atomic<T>,
+    g: Mutex<()>,
+}
+
+impl<T: 'static> TMCell<T> {
+    pub fn new(v: T) -> Self {
+        Self {
+            a: Atomic::new_alloc(v),
+            g: Mutex::new(()),
+        }
+    }
+    pub fn begin_write_txn<'a, 'g>(&'a self, g: &'g Guard) -> TMCellWriteTxn<'a, 'g, T> {
+        let wg = self.g.lock();
+        let snapshot = self.a.ld_acq(g);
+        let data: &'g T = unsafe {
+            // UNSAFE(@ohsayan): first, non-null (TMCell is never null). second, the guard
+            snapshot.deref()
+        };
+        TMCellWriteTxn::new(data, &self.a, wg)
+    }
+    pub fn begin_read_txn<'a, 'g>(&'a self, g: &'g Guard) -> TMCellReadTxn<'a, 'g, T> {
+        let snapshot = self.a.ld_acq(g);
+        let data: &'g T = unsafe {
+            // UNSAFE(@ohsayan): non-null and the guard
+            snapshot.deref()
+        };
+        TMCellReadTxn::new(data)
+    }
+}
+
+impl<T> Drop for TMCell<T> {
+    fn drop(&mut self) {
+        unsafe {
+            // UNSAFE(@ohsayan): Sole owner with mutable access
+            let g = pin_unprotected();
+            let shptr = self.a.ld_rlx(&g);
+            g.defer_destroy(shptr);
+        }
+    }
+}
+
+pub struct TMCellReadTxn<'a, 'g, T: 'static> {
+    d: &'g T,
+    _m: PhantomData<&'a TMCell<T>>,
+}
+
+impl<'a, 'g, T> TMCellReadTxn<'a, 'g, T> {
+    #[inline(always)]
+    pub fn new(d: &'g T) -> Self {
+        Self { d, _m: PhantomData }
+    }
+    #[inline(always)]
+    pub fn read(&self) -> &'g T {
+        self.d
+    }
+}
+
+impl<'a, 'g, T: Copy> TMCellReadTxn<'a, 'g, T> {
+    fn read_copy(&self) -> T {
+        *self.d
+    }
+}
+
+impl<'a, 'g, T> Deref for TMCellReadTxn<'a, 'g, T> {
+    type Target = T;
+    fn deref(&self) -> &'g Self::Target {
+        self.d
+    }
+}
+
+pub struct TMCellWriteTxn<'a, 'g, T: 'static> {
+    d: &'g T,
+    a: &'a Atomic<T>,
+    g: MutexGuard<'a, ()>,
+}
+
+impl<'a, 'g, T> TMCellWriteTxn<'a, 'g, T> {
+    #[inline(always)]
+    pub fn new(d: &'g T, a: &'a Atomic<T>, g: MutexGuard<'a, ()>) -> Self {
+        Self { d, a, g }
+    }
+    pub fn publish_commit(self, new: T, g: &'g Guard) {
+        self._commit(new, g, |p| {
+            unsafe {
+                // UNSAFE(@ohsayan): Unlinked
+                g.defer_destroy(p);
+            }
+        })
+    }
+    fn _commit<F, R>(self, new: T, g: &'g Guard, f: F) -> R
+    where
+        F: FnOnce(Shared<T>) -> R,
+    {
+        let new = Owned::new(new);
+        let r = self.a.swap(new, ORD_REL, g);
+        f(r)
+    }
+    #[inline(always)]
+    pub fn read(&self) -> &'g T {
+        self.d
+    }
+}
+
+impl<'a, 'g, T: Copy> TMCellWriteTxn<'a, 'g, T> {
+    fn read_copy(&self) -> T {
+        *self.d
+    }
+}
+
+impl<'a, 'g, T> Deref for TMCellWriteTxn<'a, 'g, T> {
+    type Target = T;
+    fn deref(&self) -> &'g Self::Target {
+        self.d
+    }
+}
diff --git a/server/src/engine/sync/mod.rs b/server/src/engine/sync/mod.rs
index 16a24114..a5f67e40 100644
--- a/server/src/engine/sync/mod.rs
+++ b/server/src/engine/sync/mod.rs
@@ -25,3 +25,4 @@
 */
 
 pub(super) mod atm;
+pub(super) mod cell;

From fa17f7251934917bad236947a622a3ed8d0d0a70 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 22 Jan 2023 18:07:29 +0000
Subject: [PATCH 084/310] Add `VInline` impl

---
 server/src/engine/mem/mod.rs   | 254 +++++++++++++++++++++++++++++++++
 server/src/engine/mem/tests.rs |  57 ++++++++
 server/src/engine/mod.rs       |   1 +
 3 files changed, 312 insertions(+)
 create mode 100644 server/src/engine/mem/mod.rs
 create mode 100644 server/src/engine/mem/tests.rs

diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
new file mode 100644
index 00000000..45ee8fe9
--- /dev/null
+++ b/server/src/engine/mem/mod.rs
@@ -0,0 +1,254 @@
+/*
+ * Created on Sun Jan 22 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[cfg(test)]
+mod tests;
+
+use {
+    core::{
+        alloc::Layout,
+        fmt,
+        mem::{self, ManuallyDrop, MaybeUninit},
+        ops::{Deref, DerefMut},
+        ptr, slice,
+    },
+    std::alloc::{alloc, dealloc},
+};
+
+union VData<const N: usize, T> {
+    s: ManuallyDrop<[MaybeUninit<T>; N]>,
+    h: *mut T,
+}
+
+struct VInline<const N: usize, T> {
+    d: VData<N, T>,
+    l: usize,
+    c: usize,
+}
+
+impl<const N: usize, T> VInline<N, T> {
+    #[inline(always)]
+    pub const fn new() -> Self {
+        let _ = Self::_ENSURE_ALIGN;
+        Self {
+            d: VData {
+                s: ManuallyDrop::new(Self::INLINE_NULL_STACK),
+            },
+            l: 0,
+            c: N,
+        }
+    }
+    #[inline(always)]
+    pub const fn capacity(&self) -> usize {
+        self.c
+    }
+    #[inline(always)]
+    pub const fn len(&self) -> usize {
+        self.l
+    }
+    #[inline(always)]
+    pub fn push(&mut self, v: T) {
+        self.grow();
+        unsafe {
+            // UNSAFE(@ohsayan): grow allocated the cap we needed
+            self._as_mut_ptr().add(self.l).write(v);
+        }
+        self.l += 1;
+    }
+}
+
+impl<const N: usize, T> VInline<N, T> {
+    const INLINE_NULL: MaybeUninit<T> = MaybeUninit::uninit();
+    const INLINE_NULL_STACK: [MaybeUninit<T>; N] = [Self::INLINE_NULL; N];
+    const ALLOC_MULTIPLIER: usize = 2;
+    const _ENSURE_ALIGN: () =
+        debug_assert!(mem::align_of::<Vec<String>>() == mem::align_of::<VInline<N, String>>());
+    #[cfg(test)]
+    fn will_be_on_stack(&self) -> bool {
+        N >= self.l + 1
+    }
+    #[inline(always)]
+    fn on_stack(&self) -> bool {
+        self.c == N
+    }
+    #[inline(always)]
+    fn _as_ptr(&self) -> *const T {
+        unsafe {
+            // UNSAFE(@ohsayan): We make legal accesses by checking state
+            if self.on_stack() {
+                self.d.s.as_ptr() as *const T
+            } else {
+                self.d.h as *const T
+            }
+        }
+    }
+    #[inline(always)]
+    fn _as_mut_ptr(&mut self) -> *mut T {
+        unsafe {
+            // UNSAFE(@ohsayan): We make legal accesses by checking state
+            if self.on_stack() {
+                (&mut self.d).s.as_mut_ptr() as *mut T
+            } else {
+                (&mut self.d).h as *mut T
+            }
+        }
+    }
+    #[inline(always)]
+    fn _as_slice(&self) -> &[T] {
+        unsafe {
+            // UNSAFE(@ohsayan): _as_ptr() will ensure correct addresses
+            slice::from_raw_parts(self._as_ptr(), self.l)
+        }
+    }
+    #[inline(always)]
+    fn _as_slice_mut(&mut self) -> &mut [T] {
+        unsafe {
+            // UNSAFE(@ohsayan): _as_mut_ptr() will ensure correct addresses
+            slice::from_raw_parts_mut(self._as_mut_ptr(), self.l)
+        }
+    }
+    #[inline(always)]
+    fn layout(cap: usize) -> Layout {
+        Layout::array::<T>(cap).unwrap()
+    }
+    #[inline(always)]
+    fn ncap(&self) -> usize {
+        self.c * Self::ALLOC_MULTIPLIER
+    }
+    fn alloc_block(cap: usize) -> *mut T {
+        unsafe {
+            // UNSAFE(@ohsayan): malloc bro
+            let p = alloc(Self::layout(cap));
+            assert!(!p.is_null(), "alloc,0");
+            p as *mut T
+        }
+    }
+    #[inline]
+    fn grow(&mut self) {
+        if !(self.l == self.capacity()) {
+            return;
+        }
+        // allocate new block
+        let nc = self.ncap();
+        let nb = Self::alloc_block(nc);
+        if self.on_stack() {
+            // stack -> heap
+            unsafe {
+                // UNSAFE(@ohsayan): non-null; valid len
+                ptr::copy_nonoverlapping(self.d.s.as_ptr() as *const T, nb, self.l);
+            }
+        } else {
+            unsafe {
+                // UNSAFE(@ohsayan): non-null; valid len
+                ptr::copy_nonoverlapping(self.d.h.cast_const(), nb, self.l);
+                // UNSAFE(@ohsayan): non-null heap
+                self.dealloc_heap();
+            }
+        }
+        self.d.h = nb;
+        self.c = nc;
+    }
+    #[inline(always)]
+    unsafe fn dealloc_heap(&mut self) {
+        dealloc(self.d.h as *mut u8, Self::layout(self.capacity()))
+    }
+}
+
+impl<const N: usize, T> Deref for VInline<N, T> {
+    type Target = [T];
+    fn deref(&self) -> &Self::Target {
+        self._as_slice()
+    }
+}
+
+impl<const N: usize, T> DerefMut for VInline<N, T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self._as_slice_mut()
+    }
+}
+
+impl<const N: usize, T> Drop for VInline<N, T> {
+    fn drop(&mut self) {
+        unsafe {
+            // UNSAFE(@ohsayan): correct ptr guaranteed by safe impl of _as_slice_mut()
+            ptr::drop_in_place(self._as_slice_mut());
+            if !self.on_stack() {
+                // UNSAFE(@ohsayan): non-null heap
+                self.dealloc_heap();
+            }
+        }
+    }
+}
+
+impl<const N: usize, T: fmt::Debug> fmt::Debug for VInline<N, T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.iter()).finish()
+    }
+}
+
+impl<T, const N: usize> Extend<T> for VInline<N, T> {
+    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
+        // FIXME(@ohsayan): Optimize capacity match upfront
+        iter.into_iter().for_each(|item| self.push(item))
+    }
+}
+
+impl<T: Clone, const N: usize> Clone for VInline<N, T> {
+    fn clone(&self) -> Self {
+        unsafe {
+            if self.on_stack() {
+                // simple stack copy
+                let mut new_stack = Self::INLINE_NULL_STACK;
+                ptr::copy_nonoverlapping(self.d.s.as_ptr(), new_stack.as_mut_ptr(), self.l);
+                Self {
+                    d: VData {
+                        s: ManuallyDrop::new(new_stack),
+                    },
+                    l: self.l,
+                    c: N,
+                }
+            } else {
+                // new allocation
+                let nb = Self::alloc_block(self.len());
+                ptr::copy_nonoverlapping(self._as_ptr(), nb, self.l);
+                Self {
+                    d: VData { h: nb },
+                    l: self.l,
+                    c: self.l,
+                }
+            }
+        }
+    }
+}
+
+impl<T, const N: usize> FromIterator<T> for VInline<N, T> {
+    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
+        let it = iter.into_iter();
+        let mut slf = Self::new();
+        slf.extend(it);
+        slf
+    }
+}
diff --git a/server/src/engine/mem/tests.rs b/server/src/engine/mem/tests.rs
new file mode 100644
index 00000000..5e604a14
--- /dev/null
+++ b/server/src/engine/mem/tests.rs
@@ -0,0 +1,57 @@
+/*
+ * Created on Sun Jan 22 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::VInline;
+
+#[test]
+fn vinline_test() {
+    const CAP: usize = 512;
+    // first alloc on stack
+    let mut array = VInline::<CAP, String>::new();
+    (0..CAP).for_each(|i| array.push(format!("elem-{i}")));
+    // check meta methods
+    debug_assert!(array.on_stack());
+    debug_assert!(!array.will_be_on_stack());
+    // now iterate
+    array
+        .iter()
+        .enumerate()
+        .for_each(|(i, elem)| assert_eq!(elem, format!("elem-{i}").as_str()));
+    // now iter_mut
+    array
+        .iter_mut()
+        .enumerate()
+        .for_each(|(i, st)| *st = format!("elem-{}", i + 1));
+    // now let's get off the stack
+    (0..10).for_each(|i| array.push(format!("elem-{}", CAP + i + 1)));
+    // verify all elements
+    array
+        .iter()
+        .enumerate()
+        .for_each(|(i, st)| assert_eq!(st, format!("elem-{}", i + 1).as_str()));
+    debug_assert!(!array.on_stack());
+    debug_assert!(!array.will_be_on_stack());
+}
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 7d8a676a..9d5855f5 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -33,3 +33,4 @@ mod core;
 mod idx;
 mod ql;
 mod sync;
+mod mem;

From 7a2e8e4b58118572aebb8f6ce5c10b6df88a4279 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 23 Jan 2023 04:22:04 +0000
Subject: [PATCH 085/310] Impl idx traits for `StdMap`

---
 server/src/engine/idx/mod.rs                 |   5 +-
 server/src/engine/idx/stdhm.rs               | 167 +++++++++++++++++++
 server/src/engine/idx/{stseq.rs => stord.rs} |   0
 3 files changed, 170 insertions(+), 2 deletions(-)
 create mode 100644 server/src/engine/idx/stdhm.rs
 rename server/src/engine/idx/{stseq.rs => stord.rs} (100%)

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 52d08cc2..4d8175b7 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -26,12 +26,13 @@
 
 use core::{borrow::Borrow, hash::Hash};
 
-mod stseq;
+mod stord;
+mod stdhm;
 #[cfg(test)]
 mod tests;
 
 // re-exports
-pub use stseq::{IndexSTSeq, IndexSTSeqDef, IndexSTSeqHasher};
+pub use stord::{IndexSTSeq, IndexSTSeqDef, IndexSTSeqHasher};
 
 /// Any type implementing this trait can be used as a key inside memory engine structures
 pub trait AsKey: Hash + Eq + Clone {
diff --git a/server/src/engine/idx/stdhm.rs b/server/src/engine/idx/stdhm.rs
new file mode 100644
index 00000000..c7b9c07a
--- /dev/null
+++ b/server/src/engine/idx/stdhm.rs
@@ -0,0 +1,167 @@
+/*
+ * Created on Mon Jan 23 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::{AsKey, AsValue, DummyMetrics, IndexBaseSpec, STIndex};
+use std::{
+    collections::{
+        hash_map::{Entry, Iter as StdMapIterKV, Keys as StdMapIterKey, Values as StdMapIterVal},
+        HashMap as StdMap,
+    },
+    hash::BuildHasher,
+    mem,
+};
+
+impl<K, V, S> IndexBaseSpec<K, V> for StdMap<K, V, S>
+where
+    K: AsKey,
+    V: AsValue,
+    S: BuildHasher + Default,
+{
+    const PREALLOC: bool = true;
+
+    type Metrics = DummyMetrics;
+
+    type IterKV<'a> = StdMapIterKV<'a, K, V>
+    where
+        Self: 'a,
+        K: 'a,
+        V: 'a;
+
+    type IterKey<'a> = StdMapIterKey<'a, K, V>
+    where
+        Self: 'a,
+        K: 'a;
+
+    type IterValue<'a> = StdMapIterVal<'a, K, V>
+    where
+        Self: 'a,
+        V: 'a;
+
+    fn idx_init() -> Self {
+        StdMap::with_hasher(S::default())
+    }
+
+    fn idx_init_with(s: Self) -> Self {
+        s
+    }
+
+    fn idx_iter_kv<'a>(&'a self) -> Self::IterKV<'a> {
+        self.iter()
+    }
+
+    fn idx_iter_key<'a>(&'a self) -> Self::IterKey<'a> {
+        self.keys()
+    }
+
+    fn idx_iter_value<'a>(&'a self) -> Self::IterValue<'a> {
+        self.values()
+    }
+
+    fn idx_metrics(&self) -> &Self::Metrics {
+        &DummyMetrics
+    }
+}
+
+impl<K, V, S> STIndex<K, V> for StdMap<K, V, S>
+where
+    K: AsKey,
+    V: AsValue,
+    S: BuildHasher + Default,
+{
+    fn st_compact(&mut self) {
+        self.shrink_to_fit()
+    }
+
+    fn st_clear(&mut self) {
+        self.clear()
+    }
+
+    fn st_insert(&mut self, key: K, val: V) -> bool {
+        match self.entry(key) {
+            Entry::Vacant(ve) => {
+                ve.insert(val);
+                true
+            }
+            _ => false,
+        }
+    }
+
+    fn st_upsert(&mut self, key: K, val: V) {
+        let _ = self.insert(key, val);
+    }
+
+    fn st_get<Q>(&self, key: &Q) -> Option<&V>
+    where
+        K: std::borrow::Borrow<Q>,
+        Q: ?Sized + super::AsKeyRef,
+    {
+        self.get(key)
+    }
+
+    fn st_get_cloned<Q>(&self, key: &Q) -> Option<V>
+    where
+        K: std::borrow::Borrow<Q>,
+        Q: ?Sized + super::AsKeyRef,
+    {
+        self.get(key).cloned()
+    }
+
+    fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
+    where
+        K: std::borrow::Borrow<Q>,
+        Q: ?Sized + super::AsKeyRef,
+    {
+        self.get_mut(key).map(move |e| *e = val).is_some()
+    }
+
+    fn st_update_return<Q>(&mut self, key: &Q, val: V) -> Option<V>
+    where
+        K: std::borrow::Borrow<Q>,
+        Q: ?Sized + super::AsKeyRef,
+    {
+        self.get_mut(key).map(move |e| {
+            let mut new = val;
+            mem::swap(&mut new, e);
+            new
+        })
+    }
+
+    fn st_delete<Q>(&mut self, key: &Q) -> bool
+    where
+        K: std::borrow::Borrow<Q>,
+        Q: ?Sized + super::AsKeyRef,
+    {
+        self.remove(key).is_some()
+    }
+
+    fn st_delete_return<Q>(&mut self, key: &Q) -> Option<V>
+    where
+        K: std::borrow::Borrow<Q>,
+        Q: ?Sized + super::AsKeyRef,
+    {
+        self.remove(key)
+    }
+}
diff --git a/server/src/engine/idx/stseq.rs b/server/src/engine/idx/stord.rs
similarity index 100%
rename from server/src/engine/idx/stseq.rs
rename to server/src/engine/idx/stord.rs

From a9900e2bd6b7009ccc194713c8f55bfbbe2406b1 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 23 Jan 2023 04:38:13 +0000
Subject: [PATCH 086/310] Revise definitions for ST idx

---
 server/src/engine/idx/mod.rs   | 82 +++++++++++++++++++---------------
 server/src/engine/idx/stdhm.rs | 37 ++++++++-------
 server/src/engine/idx/stord.rs | 80 ++++++++++++++++++---------------
 3 files changed, 112 insertions(+), 87 deletions(-)

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 4d8175b7..d35bc158 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -26,8 +26,8 @@
 
 use core::{borrow::Borrow, hash::Hash};
 
-mod stord;
 mod stdhm;
+mod stord;
 #[cfg(test)]
 mod tests;
 
@@ -81,11 +81,7 @@ pub struct DummyMetrics;
 
 /// The base spec for any index. Iterators have meaningless order, and that is intentional and oftentimes
 /// consequential. For more specialized impls, use the [`STIndex`], [`MTIndex`] or [`STIndexSeq`] traits
-pub trait IndexBaseSpec<K, V>
-where
-    K: AsKey,
-    V: AsValue,
-{
+pub trait IndexBaseSpec<K, V> {
     /// Index supports prealloc?
     const PREALLOC: bool;
     #[cfg(debug_assertions)]
@@ -125,11 +121,7 @@ where
 }
 
 /// An unordered MTIndex
-pub trait MTIndex<K, V>: IndexBaseSpec<K, V>
-where
-    K: AsKey,
-    V: AsValue,
-{
+pub trait MTIndex<K, V>: IndexBaseSpec<K, V> {
     /// Attempts to compact the backing storage
     fn mt_compact(&self) {}
     /// Clears all the entries in the MTIndex
@@ -137,50 +129,58 @@ where
     // write
     /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
     /// violated
-    fn mt_insert(&self, key: K, val: V) -> bool;
+    fn mt_insert(&self, key: K, val: V) -> bool
+    where
+        K: AsKey,
+        V: AsValue;
     /// Updates or inserts the given value
-    fn mt_upsert(&self, key: K, val: V);
+    fn mt_upsert(&self, key: K, val: V)
+    where
+        K: AsKey,
+        V: AsValue;
     // read
+    fn mt_contains<Q>(&self, key: &Q) -> bool
+    where
+        K: Borrow<Q> + AsKey,
+        Q: ?Sized + AsKeyRef;
     /// Returns a reference to the value corresponding to the key, if it exists
     fn mt_get<Q>(&self, key: &Q) -> Option<&V>
     where
-        K: Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKeyRef;
     /// Returns a clone of the value corresponding to the key, if it exists
     fn mt_get_cloned<Q>(&self, key: &Q) -> Option<V>
     where
-        K: Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKeyRef;
     // update
     /// Returns true if the entry is updated
     fn mt_update<Q>(&self, key: &Q, val: V) -> bool
     where
-        K: Borrow<Q>,
+        K: AsKey + Borrow<Q>,
+        V: AsValue,
         Q: ?Sized + AsKeyRef;
     /// Updates the entry and returns the old value, if it exists
     fn mt_update_return<Q>(&self, key: &Q, val: V) -> Option<V>
     where
-        K: Borrow<Q>,
+        K: AsKey + Borrow<Q>,
+        V: AsValue,
         Q: ?Sized + AsKeyRef;
     // delete
     /// Returns true if the entry was deleted
     fn mt_delete<Q>(&self, key: &Q) -> bool
     where
-        K: Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKeyRef;
     /// Removes the entry and returns it, if it exists
     fn mt_delete_return<Q>(&self, key: &Q) -> Option<V>
     where
-        K: Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKeyRef;
 }
 
 /// An unordered STIndex
-pub trait STIndex<K, V>: IndexBaseSpec<K, V>
-where
-    K: AsKey,
-    V: AsValue,
-{
+pub trait STIndex<K, V>: IndexBaseSpec<K, V> {
     /// Attempts to compact the backing storage
     fn st_compact(&mut self) {}
     /// Clears all the entries in the STIndex
@@ -188,49 +188,57 @@ where
     // write
     /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
     /// violated
-    fn st_insert(&mut self, key: K, val: V) -> bool;
+    fn st_insert(&mut self, key: K, val: V) -> bool
+    where
+        K: AsKey,
+        V: AsValue;
     /// Updates or inserts the given value
-    fn st_upsert(&mut self, key: K, val: V);
+    fn st_upsert(&mut self, key: K, val: V)
+    where
+        K: AsKey,
+        V: AsValue;
     // read
+    fn st_contains<Q>(&self, key: &Q) -> bool
+    where
+        K: Borrow<Q> + AsKey,
+        Q: ?Sized + AsKeyRef;
     /// Returns a reference to the value corresponding to the key, if it exists
     fn st_get<Q>(&self, key: &Q) -> Option<&V>
     where
-        K: Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKeyRef;
     /// Returns a clone of the value corresponding to the key, if it exists
     fn st_get_cloned<Q>(&self, key: &Q) -> Option<V>
     where
-        K: Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKeyRef;
     // update
     /// Returns true if the entry is updated
     fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
     where
-        K: Borrow<Q>,
+        K: AsKey + Borrow<Q>,
+        V: AsValue,
         Q: ?Sized + AsKeyRef;
     /// Updates the entry and returns the old value, if it exists
     fn st_update_return<Q>(&mut self, key: &Q, val: V) -> Option<V>
     where
-        K: Borrow<Q>,
+        K: AsKey + Borrow<Q>,
+        V: AsValue,
         Q: ?Sized + AsKeyRef;
     // delete
     /// Returns true if the entry was deleted
     fn st_delete<Q>(&mut self, key: &Q) -> bool
     where
-        K: Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKeyRef;
     /// Removes the entry and returns it, if it exists
     fn st_delete_return<Q>(&mut self, key: &Q) -> Option<V>
     where
-        K: Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKeyRef;
 }
 
-pub trait STIndexSeq<K, V>: STIndex<K, V>
-where
-    K: AsKey,
-    V: AsValue,
-{
+pub trait STIndexSeq<K, V>: STIndex<K, V> {
     /// An ordered iterator over the keys and values
     type IterOrdKV<'a>: Iterator<Item = (&'a K, &'a V)> + DoubleEndedIterator<Item = (&'a K, &'a V)>
     where
diff --git a/server/src/engine/idx/stdhm.rs b/server/src/engine/idx/stdhm.rs
index c7b9c07a..d108ce09 100644
--- a/server/src/engine/idx/stdhm.rs
+++ b/server/src/engine/idx/stdhm.rs
@@ -24,8 +24,9 @@
  *
 */
 
-use super::{AsKey, AsValue, DummyMetrics, IndexBaseSpec, STIndex};
+use super::{AsKey, AsKeyRef, AsValue, DummyMetrics, IndexBaseSpec, STIndex};
 use std::{
+    borrow::Borrow,
     collections::{
         hash_map::{Entry, Iter as StdMapIterKV, Keys as StdMapIterKey, Values as StdMapIterVal},
         HashMap as StdMap,
@@ -36,8 +37,6 @@ use std::{
 
 impl<K, V, S> IndexBaseSpec<K, V> for StdMap<K, V, S>
 where
-    K: AsKey,
-    V: AsValue,
     S: BuildHasher + Default,
 {
     const PREALLOC: bool = true;
@@ -113,34 +112,42 @@ where
         let _ = self.insert(key, val);
     }
 
+    fn st_contains<Q>(&self, k: &Q) -> bool
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef,
+    {
+        self.contains_key(k)
+    }
+
     fn st_get<Q>(&self, key: &Q) -> Option<&V>
     where
-        K: std::borrow::Borrow<Q>,
-        Q: ?Sized + super::AsKeyRef,
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef,
     {
         self.get(key)
     }
 
     fn st_get_cloned<Q>(&self, key: &Q) -> Option<V>
     where
-        K: std::borrow::Borrow<Q>,
-        Q: ?Sized + super::AsKeyRef,
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef,
     {
         self.get(key).cloned()
     }
 
     fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
     where
-        K: std::borrow::Borrow<Q>,
-        Q: ?Sized + super::AsKeyRef,
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef,
     {
         self.get_mut(key).map(move |e| *e = val).is_some()
     }
 
     fn st_update_return<Q>(&mut self, key: &Q, val: V) -> Option<V>
     where
-        K: std::borrow::Borrow<Q>,
-        Q: ?Sized + super::AsKeyRef,
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef,
     {
         self.get_mut(key).map(move |e| {
             let mut new = val;
@@ -151,16 +158,16 @@ where
 
     fn st_delete<Q>(&mut self, key: &Q) -> bool
     where
-        K: std::borrow::Borrow<Q>,
-        Q: ?Sized + super::AsKeyRef,
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef,
     {
         self.remove(key).is_some()
     }
 
     fn st_delete_return<Q>(&mut self, key: &Q) -> Option<V>
     where
-        K: std::borrow::Borrow<Q>,
-        Q: ?Sized + super::AsKeyRef,
+        K: Borrow<Q>,
+        Q: ?Sized + AsKeyRef,
     {
         self.remove(key)
     }
diff --git a/server/src/engine/idx/stord.rs b/server/src/engine/idx/stord.rs
index 11e6683d..9a22eb35 100644
--- a/server/src/engine/idx/stord.rs
+++ b/server/src/engine/idx/stord.rs
@@ -358,6 +358,39 @@ impl<K, V, S> IndexSTSeqDll<K, V, S> {
     }
 }
 
+impl<K, V, S> IndexSTSeqDll<K, V, S> {
+    #[inline(always)]
+    fn _iter_unord_kv<'a>(&'a self) -> IndexSTSeqDllIterUnordKV<'a, K, V> {
+        IndexSTSeqDllIterUnordKV::new(&self.m)
+    }
+    #[inline(always)]
+    fn _iter_unord_k<'a>(&'a self) -> IndexSTSeqDllIterUnordKey<'a, K, V> {
+        IndexSTSeqDllIterUnordKey::new(&self.m)
+    }
+    #[inline(always)]
+    fn _iter_unord_v<'a>(&'a self) -> IndexSTSeqDllIterUnordValue<'a, K, V> {
+        IndexSTSeqDllIterUnordValue::new(&self.m)
+    }
+    #[inline(always)]
+    fn _iter_ord_kv<'a>(&'a self) -> IndexSTSeqDllIterOrdKV<'a, K, V> {
+        IndexSTSeqDllIterOrdKV {
+            i: IndexSTSeqDllIterOrdBase::new(self),
+        }
+    }
+    #[inline(always)]
+    fn _iter_ord_k<'a>(&'a self) -> IndexSTSeqDllIterOrdKey<'a, K, V> {
+        IndexSTSeqDllIterOrdKey {
+            i: IndexSTSeqDllIterOrdBase::new(self),
+        }
+    }
+    #[inline(always)]
+    fn _iter_ord_v<'a>(&'a self) -> IndexSTSeqDllIterOrdValue<'a, K, V> {
+        IndexSTSeqDllIterOrdValue {
+            i: IndexSTSeqDllIterOrdBase::new(self),
+        }
+    }
+}
+
 impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
     #[inline(always)]
     /// Clean up unused and cached memory
@@ -497,36 +530,6 @@ impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
             }
         }
     }
-    #[inline(always)]
-    fn _iter_unord_kv<'a>(&'a self) -> IndexSTSeqDllIterUnordKV<'a, K, V> {
-        IndexSTSeqDllIterUnordKV::new(&self.m)
-    }
-    #[inline(always)]
-    fn _iter_unord_k<'a>(&'a self) -> IndexSTSeqDllIterUnordKey<'a, K, V> {
-        IndexSTSeqDllIterUnordKey::new(&self.m)
-    }
-    #[inline(always)]
-    fn _iter_unord_v<'a>(&'a self) -> IndexSTSeqDllIterUnordValue<'a, K, V> {
-        IndexSTSeqDllIterUnordValue::new(&self.m)
-    }
-    #[inline(always)]
-    fn _iter_ord_kv<'a>(&'a self) -> IndexSTSeqDllIterOrdKV<'a, K, V> {
-        IndexSTSeqDllIterOrdKV {
-            i: IndexSTSeqDllIterOrdBase::new(self),
-        }
-    }
-    #[inline(always)]
-    fn _iter_ord_k<'a>(&'a self) -> IndexSTSeqDllIterOrdKey<'a, K, V> {
-        IndexSTSeqDllIterOrdKey {
-            i: IndexSTSeqDllIterOrdBase::new(self),
-        }
-    }
-    #[inline(always)]
-    fn _iter_ord_v<'a>(&'a self) -> IndexSTSeqDllIterOrdValue<'a, K, V> {
-        IndexSTSeqDllIterOrdValue {
-            i: IndexSTSeqDllIterOrdBase::new(self),
-        }
-    }
 }
 
 impl<K, V, S> Drop for IndexSTSeqDll<K, V, S> {
@@ -554,11 +557,7 @@ impl<K: AsKey, V: AsValue, S: BuildHasher + Default> FromIterator<(K, V)>
     }
 }
 
-impl<K, V, S: BuildHasher + Default> IndexBaseSpec<K, V> for IndexSTSeqDll<K, V, S>
-where
-    K: AsKey,
-    V: AsValue,
-{
+impl<K, V, S: BuildHasher + Default> IndexBaseSpec<K, V> for IndexSTSeqDll<K, V, S> {
     const PREALLOC: bool = true;
 
     type Metrics = IndexSTSeqDllMetrics;
@@ -626,6 +625,17 @@ where
         let _ = self._upsert(key, val);
     }
 
+    fn st_contains<Q>(&self, key: &Q) -> bool
+    where
+        K: Borrow<Q> + AsKey,
+        Q: ?Sized + AsKeyRef,
+    {
+        self.m.contains_key(unsafe {
+            // UNSAFE(@ohsayan): Valid ref with correct bounds
+            IndexSTSeqDllQref::from_ref(key)
+        })
+    }
+
     fn st_get<Q>(&self, key: &Q) -> Option<&V>
     where
         K: Borrow<Q>,

From 9d3115a7cf06229cca7a9927b82d2f755ff21372 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 23 Jan 2023 04:49:15 +0000
Subject: [PATCH 087/310] Enable idx init with cap

---
 server/src/engine/idx/mod.rs   | 11 ++++++++++-
 server/src/engine/idx/stdhm.rs |  4 ++++
 server/src/engine/idx/stord.rs |  4 ++++
 server/src/engine/ql/lex.rs    | 17 +----------------
 4 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index d35bc158..b7598ac5 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -81,7 +81,7 @@ pub struct DummyMetrics;
 
 /// The base spec for any index. Iterators have meaningless order, and that is intentional and oftentimes
 /// consequential. For more specialized impls, use the [`STIndex`], [`MTIndex`] or [`STIndexSeq`] traits
-pub trait IndexBaseSpec<K, V> {
+pub trait IndexBaseSpec<K, V>: Sized {
     /// Index supports prealloc?
     const PREALLOC: bool;
     #[cfg(debug_assertions)]
@@ -108,6 +108,15 @@ pub trait IndexBaseSpec<K, V> {
     fn idx_init() -> Self;
     /// Initialize a pre-loaded instance of the index
     fn idx_init_with(s: Self) -> Self;
+    /// Init the idx with the given cap
+    ///
+    /// By default doesn't attempt to allocate
+    fn idx_init_cap(_: usize) -> Self {
+        if Self::PREALLOC {
+            panic!("expected prealloc");
+        }
+        Self::idx_init()
+    }
     // iter
     /// Returns an iterator over a tuple of keys and values
     fn idx_iter_kv<'a>(&'a self) -> Self::IterKV<'a>;
diff --git a/server/src/engine/idx/stdhm.rs b/server/src/engine/idx/stdhm.rs
index d108ce09..a50f6827 100644
--- a/server/src/engine/idx/stdhm.rs
+++ b/server/src/engine/idx/stdhm.rs
@@ -67,6 +67,10 @@ where
         s
     }
 
+    fn idx_init_cap(cap: usize) -> Self {
+        Self::with_capacity_and_hasher(cap, S::default())
+    }
+
     fn idx_iter_kv<'a>(&'a self) -> Self::IterKV<'a> {
         self.iter()
     }
diff --git a/server/src/engine/idx/stord.rs b/server/src/engine/idx/stord.rs
index 9a22eb35..cbd4ff7b 100644
--- a/server/src/engine/idx/stord.rs
+++ b/server/src/engine/idx/stord.rs
@@ -586,6 +586,10 @@ impl<K, V, S: BuildHasher + Default> IndexBaseSpec<K, V> for IndexSTSeqDll<K, V,
         Self::from(s)
     }
 
+    fn idx_init_cap(cap: usize) -> Self {
+        Self::with_capacity_and_hasher(cap, S::default())
+    }
+
     fn idx_iter_kv<'a>(&'a self) -> Self::IterKV<'a> {
         self._iter_unord_kv()
     }
diff --git a/server/src/engine/ql/lex.rs b/server/src/engine/ql/lex.rs
index 7235cd59..f949e6bb 100644
--- a/server/src/engine/ql/lex.rs
+++ b/server/src/engine/ql/lex.rs
@@ -27,7 +27,7 @@
 use {
     super::{LangError, LangResult},
     crate::util::compiler,
-    core::{cmp, fmt, mem::size_of, ops::BitOr, slice, str},
+    core::{cmp, fmt, ops::BitOr, slice, str},
 };
 
 pub type Slice<'a> = &'a [u8];
@@ -56,21 +56,6 @@ impl<'a> PartialEq<Symbol> for Token<'a> {
     }
 }
 
-const SIZEOF_USIZE: usize = size_of::<usize>();
-const LT_SZ: usize = if is_64b!() {
-    size_of::<usize>() * 3
-} else {
-    size_of::<usize>() * 2
-};
-
-assertions! {
-    size_of::<Token>() == LT_SZ,
-    size_of::<Symbol>() == 1,
-    size_of::<Keyword>() == 1,
-    size_of::<Lit>() == LT_SZ,
-    size_of::<LitIR>() == LT_SZ,
-}
-
 enum_impls! {
     Token<'a> => {
         Keyword as Keyword,

From ab30a741732df125ffd415ffa7f308fff0134741 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 23 Jan 2023 06:12:12 +0000
Subject: [PATCH 088/310] Re-export at mod level

---
 server/src/engine/idx/mod.rs   |  4 +++-
 server/src/engine/idx/stord.rs |  6 ++----
 server/src/engine/idx/tests.rs | 18 +++++++++---------
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index b7598ac5..bf8e4f56 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -32,7 +32,9 @@ mod stord;
 mod tests;
 
 // re-exports
-pub use stord::{IndexSTSeq, IndexSTSeqDef, IndexSTSeqHasher};
+pub type IndexSTSeq<K, V, S> = stord::IndexSTSeqDll<K, V, S>;
+pub type IndexSTSeqDef<K, V> = IndexSTSeq<K, V, IndexSTSeqHasher>;
+pub type IndexSTSeqHasher = stord::IndexSTSeqDllHasher;
 
 /// Any type implementing this trait can be used as a key inside memory engine structures
 pub trait AsKey: Hash + Eq + Clone {
diff --git a/server/src/engine/idx/stord.rs b/server/src/engine/idx/stord.rs
index cbd4ff7b..5f4b2183 100644
--- a/server/src/engine/idx/stord.rs
+++ b/server/src/engine/idx/stord.rs
@@ -40,10 +40,8 @@ use std::{
     ptr::{self, NonNull},
 };
 
-// re-exports for convenience
-pub type IndexSTSeq<K, V, S> = IndexSTSeqDll<K, V, S>;
-pub type IndexSTSeqDef<K, V> = IndexSTSeq<K, V, IndexSTSeqHasher>;
-pub type IndexSTSeqHasher = RandomState;
+pub type IndexSTSeqDllDef<K, V> = IndexSTSeqDll<K, V, IndexSTSeqDllHasher>;
+pub type IndexSTSeqDllHasher = RandomState;
 
 /*
     For the ordered index impl, we resort to some crazy unsafe code, especially because there's no other way to
diff --git a/server/src/engine/idx/tests.rs b/server/src/engine/idx/tests.rs
index 07ce1de2..3cf57c03 100644
--- a/server/src/engine/idx/tests.rs
+++ b/server/src/engine/idx/tests.rs
@@ -27,16 +27,16 @@
 use super::*;
 
 mod idx_st_seq_dll {
-    use super::{IndexBaseSpec, IndexSTSeqDef, STIndex, STIndexSeq};
+    use super::{stord::IndexSTSeqDllDef, IndexBaseSpec, STIndex, STIndexSeq};
     use rand::{distributions::Alphanumeric, Rng};
 
     const SPAM_CNT: usize = 131_072;
     const SPAM_SIZE: usize = 128;
-    type Index = IndexSTSeqDef<String, String>;
+    type Index = IndexSTSeqDllDef<String, String>;
 
     /// Returns an index with: `i -> "{i+1}"` starting from 0 upto the value of [`SPAM_CNT`]
-    fn mkidx() -> IndexSTSeqDef<usize, String> {
-        let mut idx = IndexSTSeqDef::idx_init();
+    fn mkidx() -> IndexSTSeqDllDef<usize, String> {
+        let mut idx = IndexSTSeqDllDef::idx_init();
         for int in 0..SPAM_CNT {
             assert!(idx.st_insert(int, (int + 1).to_string()));
         }
@@ -66,7 +66,7 @@ mod idx_st_seq_dll {
     }
     #[test]
     fn spam_read_nx() {
-        let idx = IndexSTSeqDef::<usize, String>::new();
+        let idx = IndexSTSeqDllDef::<usize, String>::new();
         for int in SPAM_CNT..SPAM_CNT * 2 {
             assert!(idx.st_get(&int).is_none());
         }
@@ -80,14 +80,14 @@ mod idx_st_seq_dll {
     }
     #[test]
     fn spam_update_nx() {
-        let mut idx = IndexSTSeqDef::<usize, String>::new();
+        let mut idx = IndexSTSeqDllDef::<usize, String>::new();
         for int in 0..SPAM_CNT {
             assert!(!idx.st_update(&int, (int + 2).to_string()));
         }
     }
     #[test]
     fn spam_delete_nx() {
-        let mut idx = IndexSTSeqDef::<usize, String>::new();
+        let mut idx = IndexSTSeqDllDef::<usize, String>::new();
         for int in 0..SPAM_CNT {
             assert!(!idx.st_delete(&int));
         }
@@ -104,7 +104,7 @@ mod idx_st_seq_dll {
     }
     #[test]
     fn spam_crud() {
-        let mut idx = IndexSTSeqDef::idx_init();
+        let mut idx = IndexSTSeqDllDef::idx_init();
         for int in 0..SPAM_CNT {
             assert!(idx.st_insert(int, int + 1));
             assert_eq!(*idx.st_get(&int).unwrap(), int + 1);
@@ -116,7 +116,7 @@ mod idx_st_seq_dll {
     }
     #[test]
     fn spam_read() {
-        let mut idx = IndexSTSeqDef::idx_init();
+        let mut idx = IndexSTSeqDllDef::idx_init();
         for int in 0..SPAM_CNT {
             let v = (int + 1).to_string();
             assert!(idx.st_insert(int, v.clone()));

From 10a36fa77dd9d21dd522c2e8e1a0ce1ece8615b5 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 23 Jan 2023 09:31:40 +0000
Subject: [PATCH 089/310] Add `UArray` definition

---
 server/src/engine/mem/mod.rs   |  95 +++++++++++++++++++++++++++++
 server/src/engine/mem/tests.rs | 108 ++++++++++++++++++++++++---------
 2 files changed, 175 insertions(+), 28 deletions(-)

diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index 45ee8fe9..ee16b15f 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -252,3 +252,98 @@ impl<T, const N: usize> FromIterator<T> for VInline<N, T> {
         slf
     }
 }
+
+pub struct UArray<const N: usize, T> {
+    a: [MaybeUninit<T>; N],
+    l: usize,
+}
+
+impl<const N: usize, T> UArray<N, T> {
+    const NULL: MaybeUninit<T> = MaybeUninit::uninit();
+    const NULLED_ARRAY: [MaybeUninit<T>; N] = [Self::NULL; N];
+    #[inline(always)]
+    pub const fn new() -> Self {
+        Self {
+            a: Self::NULLED_ARRAY,
+            l: 0,
+        }
+    }
+    #[inline(always)]
+    pub const fn len(&self) -> usize {
+        self.l
+    }
+    #[inline(always)]
+    pub const fn capacity(&self) -> usize {
+        N
+    }
+    #[inline(always)]
+    pub const fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+    #[inline(always)]
+    unsafe fn incr_len(&mut self) {
+        self.l += 1;
+    }
+    #[inline(always)]
+    pub fn push(&mut self, v: T) {
+        if self.l == N {
+            panic!("stack,capof");
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): verified correct offsets (N)
+            self.a.as_mut_ptr().add(self.l).write(MaybeUninit::new(v));
+            // UNSAFE(@ohsayan): all G since l =< N
+            self.incr_len();
+        }
+    }
+    pub fn as_slice(&self) -> &[T] {
+        unsafe {
+            // UNSAFE(@ohsayan): ptr is always valid and len is correct, due to push impl
+            slice::from_raw_parts(self.a.as_ptr() as *const T, self.l)
+        }
+    }
+    pub fn as_slice_mut(&mut self) -> &mut [T] {
+        unsafe {
+            // UNSAFE(@ohsayan): ptr is always valid and len is correct, due to push impl
+            slice::from_raw_parts_mut(self.a.as_mut_ptr() as *mut T, self.l)
+        }
+    }
+}
+
+impl<const N: usize, T> Drop for UArray<N, T> {
+    fn drop(&mut self) {
+        if !self.is_empty() {
+            unsafe {
+                // UNSAFE(@ohsayan): as_slice_mut returns a correct offset
+                ptr::drop_in_place(self.as_slice_mut())
+            }
+        }
+    }
+}
+
+impl<const N: usize, T> Deref for UArray<N, T> {
+    type Target = [T];
+    fn deref(&self) -> &Self::Target {
+        self.as_slice()
+    }
+}
+
+impl<const N: usize, T> DerefMut for UArray<N, T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.as_slice_mut()
+    }
+}
+
+impl<const N: usize, T> FromIterator<T> for UArray<N, T> {
+    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
+        let mut slf = Self::new();
+        iter.into_iter().for_each(|v| slf.push(v));
+        slf
+    }
+}
+
+impl<const N: usize, T> Extend<T> for UArray<N, T> {
+    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
+        iter.into_iter().for_each(|v| self.push(v))
+    }
+}
diff --git a/server/src/engine/mem/tests.rs b/server/src/engine/mem/tests.rs
index 5e604a14..b4fd90b7 100644
--- a/server/src/engine/mem/tests.rs
+++ b/server/src/engine/mem/tests.rs
@@ -24,34 +24,86 @@
  *
 */
 
-use super::VInline;
+use super::*;
 
-#[test]
-fn vinline_test() {
+mod vinline {
+    use super::*;
     const CAP: usize = 512;
-    // first alloc on stack
-    let mut array = VInline::<CAP, String>::new();
-    (0..CAP).for_each(|i| array.push(format!("elem-{i}")));
-    // check meta methods
-    debug_assert!(array.on_stack());
-    debug_assert!(!array.will_be_on_stack());
-    // now iterate
-    array
-        .iter()
-        .enumerate()
-        .for_each(|(i, elem)| assert_eq!(elem, format!("elem-{i}").as_str()));
-    // now iter_mut
-    array
-        .iter_mut()
-        .enumerate()
-        .for_each(|(i, st)| *st = format!("elem-{}", i + 1));
-    // now let's get off the stack
-    (0..10).for_each(|i| array.push(format!("elem-{}", CAP + i + 1)));
-    // verify all elements
-    array
-        .iter()
-        .enumerate()
-        .for_each(|(i, st)| assert_eq!(st, format!("elem-{}", i + 1).as_str()));
-    debug_assert!(!array.on_stack());
-    debug_assert!(!array.will_be_on_stack());
+    #[test]
+    fn vinline_drop_empty() {
+        let array = VInline::<CAP, String>::new();
+        drop(array);
+    }
+
+    #[test]
+    fn vinline_test() {
+        // first alloc on stack
+        let mut array = VInline::<CAP, String>::new();
+        (0..CAP).for_each(|i| array.push(format!("elem-{i}")));
+        // check meta methods
+        debug_assert!(array.on_stack());
+        debug_assert!(!array.will_be_on_stack());
+        // now iterate
+        array
+            .iter()
+            .enumerate()
+            .for_each(|(i, elem)| assert_eq!(elem, format!("elem-{i}").as_str()));
+        // now iter_mut
+        array
+            .iter_mut()
+            .enumerate()
+            .for_each(|(i, st)| *st = format!("elem-{}", i + 1));
+        // now let's get off the stack
+        (0..10).for_each(|i| array.push(format!("elem-{}", CAP + i + 1)));
+        // verify all elements
+        array
+            .iter()
+            .enumerate()
+            .for_each(|(i, st)| assert_eq!(st, format!("elem-{}", i + 1).as_str()));
+        debug_assert!(!array.on_stack());
+        debug_assert!(!array.will_be_on_stack());
+    }
+}
+
+mod uarray {
+    use super::*;
+    const CAP: usize = 8;
+    #[test]
+    fn empty() {
+        let a = UArray::<CAP, u8>::new();
+        drop(a);
+    }
+    #[test]
+    fn push_okay() {
+        let mut a = UArray::<CAP, u8>::new();
+        a.push(1);
+        a.push(2);
+        a.push(3);
+        a.push(4);
+    }
+    #[test]
+    #[should_panic(expected = "stack,capof")]
+    fn push_panic() {
+        let mut a = UArray::<CAP, u8>::new();
+        a.push(1);
+        a.push(2);
+        a.push(3);
+        a.push(4);
+        a.push(5);
+        a.push(6);
+        a.push(7);
+        a.push(8);
+        a.push(9);
+    }
+    #[test]
+    fn slice() {
+        let a: UArray<CAP, _> = (1u8..=8).into_iter().collect();
+        assert_eq!(a.as_slice(), [1, 2, 3, 4, 5, 6, 7, 8]);
+    }
+    #[test]
+    fn slice_mut() {
+        let mut a: UArray<CAP, _> = (0u8..8).into_iter().collect();
+        a.iter_mut().for_each(|v| *v += 1);
+        assert_eq!(a.as_slice(), [1, 2, 3, 4, 5, 6, 7, 8])
+    }
 }

From 49dfac11b5a9e4f829e009b74ccf2c4da0054d4c Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 23 Jan 2023 13:07:48 +0000
Subject: [PATCH 090/310] Add remove and capacity optimization ops to VInline

Also updated tests to make debugging easier (the previous test simply
placed all operations in one test making it harder to debug any bugs)
---
 server/src/engine/mem/mod.rs   | 123 ++++++++++++++++++++++++++++----
 server/src/engine/mem/tests.rs | 126 +++++++++++++++++++++++++--------
 2 files changed, 206 insertions(+), 43 deletions(-)

diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index ee16b15f..fa0eb6e0 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -74,10 +74,51 @@ impl<const N: usize, T> VInline<N, T> {
         self.grow();
         unsafe {
             // UNSAFE(@ohsayan): grow allocated the cap we needed
-            self._as_mut_ptr().add(self.l).write(v);
+            self.push_unchecked(v);
         }
         self.l += 1;
     }
+    #[inline(always)]
+    pub fn clear(&mut self) {
+        unsafe {
+            // UNSAFE(@ohsayan): as_slice_mut will always give a valid ptr
+            ptr::drop_in_place(self._as_slice_mut());
+        }
+        self.l = 0;
+    }
+    #[inline(always)]
+    pub fn remove(&mut self, idx: usize) -> T {
+        if idx >= self.len() {
+            panic!("index out of range");
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): Verified index is within range
+            self.remove_unchecked(idx)
+        }
+    }
+    #[inline(always)]
+    pub fn remove_compact(&mut self, idx: usize) -> T {
+        let r = self.remove(idx);
+        self.optimize_capacity();
+        r
+    }
+    #[inline(always)]
+    /// SAFETY: `idx` must be < l
+    unsafe fn remove_unchecked(&mut self, idx: usize) -> T {
+        // UNSAFE(@ohsayan): idx is in range
+        let ptr = self.as_mut_ptr().add(idx);
+        // UNSAFE(@ohsayan): idx is in range and is valid
+        let ret = ptr::read(ptr);
+        // UNSAFE(@ohsayan): move all elements to the left
+        ptr::copy(ptr.add(1), ptr, self.len() - idx - 1);
+        // UNSAFE(@ohsayan): this is our new length
+        self.set_len(self.len() - 1);
+        ret
+    }
+    #[inline(always)]
+    unsafe fn set_len(&mut self, len: usize) {
+        self.l = len;
+    }
 }
 
 impl<const N: usize, T> VInline<N, T> {
@@ -86,9 +127,9 @@ impl<const N: usize, T> VInline<N, T> {
     const ALLOC_MULTIPLIER: usize = 2;
     const _ENSURE_ALIGN: () =
         debug_assert!(mem::align_of::<Vec<String>>() == mem::align_of::<VInline<N, String>>());
-    #[cfg(test)]
-    fn will_be_on_stack(&self) -> bool {
-        N >= self.l + 1
+    #[inline(always)]
+    fn on_heap(&self) -> bool {
+        self.c > N
     }
     #[inline(always)]
     fn on_stack(&self) -> bool {
@@ -146,6 +187,39 @@ impl<const N: usize, T> VInline<N, T> {
             p as *mut T
         }
     }
+    unsafe fn push_unchecked(&mut self, v: T) {
+        self._as_mut_ptr().add(self.l).write(v);
+    }
+    pub fn optimize_capacity(&mut self) {
+        if self.on_stack() || self.len() == self.capacity() {
+            return;
+        }
+        if self.l <= N {
+            unsafe {
+                // UNSAFE(@ohsayan): non-null heap
+                self.mv_to_stack();
+            }
+        } else {
+            let nb = Self::alloc_block(self.len());
+            unsafe {
+                // UNSAFE(@ohsayan): nonov; non-null
+                ptr::copy_nonoverlapping(self.d.h, nb, self.len());
+                // UNSAFE(@ohsayan): non-null heap
+                self.dealloc_heap(self.d.h);
+            }
+            self.d.h = nb;
+            self.c = self.len();
+        }
+    }
+    /// SAFETY: (1) non-null heap
+    unsafe fn mv_to_stack(&mut self) {
+        let heap = self.d.h;
+        // UNSAFE(@ohsayan): nonov; non-null (stack lol)
+        ptr::copy_nonoverlapping(self.d.h, (&mut self.d).s.as_mut_ptr() as *mut T, self.len());
+        // UNSAFE(@ohsayan): non-null heap
+        self.dealloc_heap(heap);
+        self.c = N;
+    }
     #[inline]
     fn grow(&mut self) {
         if !(self.l == self.capacity()) {
@@ -165,15 +239,15 @@ impl<const N: usize, T> VInline<N, T> {
                 // UNSAFE(@ohsayan): non-null; valid len
                 ptr::copy_nonoverlapping(self.d.h.cast_const(), nb, self.l);
                 // UNSAFE(@ohsayan): non-null heap
-                self.dealloc_heap();
+                self.dealloc_heap(self.d.h);
             }
         }
         self.d.h = nb;
         self.c = nc;
     }
     #[inline(always)]
-    unsafe fn dealloc_heap(&mut self) {
-        dealloc(self.d.h as *mut u8, Self::layout(self.capacity()))
+    unsafe fn dealloc_heap(&mut self, heap: *mut T) {
+        dealloc(heap as *mut u8, Self::layout(self.capacity()))
     }
 }
 
@@ -197,7 +271,7 @@ impl<const N: usize, T> Drop for VInline<N, T> {
             ptr::drop_in_place(self._as_slice_mut());
             if !self.on_stack() {
                 // UNSAFE(@ohsayan): non-null heap
-                self.dealloc_heap();
+                self.dealloc_heap(self.d.h);
             }
         }
     }
@@ -290,12 +364,37 @@ impl<const N: usize, T> UArray<N, T> {
             panic!("stack,capof");
         }
         unsafe {
-            // UNSAFE(@ohsayan): verified correct offsets (N)
-            self.a.as_mut_ptr().add(self.l).write(MaybeUninit::new(v));
-            // UNSAFE(@ohsayan): all G since l =< N
-            self.incr_len();
+            // UNSAFE(@ohsayan): verified length is smaller
+            self.push_unchecked(v);
         }
     }
+    pub fn remove(&mut self, idx: usize) -> T {
+        if idx >= self.len() {
+            panic!("out of range. idx is `{idx}` but len is `{}`", self.len());
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): verified idx < l
+            self.remove_unchecked(idx)
+        }
+    }
+    /// SAFETY: idx < self.l
+    unsafe fn remove_unchecked(&mut self, idx: usize) -> T {
+        // UNSAFE(@ohsayan): Verified idx
+        let target = self.a.as_mut_ptr().add(idx).cast::<T>();
+        // UNSAFE(@ohsayan): Verified idx
+        let ret = ptr::read(target);
+        // UNSAFE(@ohsayan): ov; not-null; correct len
+        ptr::copy(target.add(1), target, self.len() - idx - 1);
+        ret
+    }
+    #[inline(always)]
+    /// SAFETY: self.l < N
+    unsafe fn push_unchecked(&mut self, v: T) {
+        // UNSAFE(@ohsayan): verified correct offsets (N)
+        self.a.as_mut_ptr().add(self.l).write(MaybeUninit::new(v));
+        // UNSAFE(@ohsayan): all G since l =< N
+        self.incr_len();
+    }
     pub fn as_slice(&self) -> &[T] {
         unsafe {
             // UNSAFE(@ohsayan): ptr is always valid and len is correct, due to push impl
diff --git a/server/src/engine/mem/tests.rs b/server/src/engine/mem/tests.rs
index b4fd90b7..1043b9fa 100644
--- a/server/src/engine/mem/tests.rs
+++ b/server/src/engine/mem/tests.rs
@@ -28,40 +28,104 @@ use super::*;
 
 mod vinline {
     use super::*;
-    const CAP: usize = 512;
+    const CAP: usize = 8;
     #[test]
-    fn vinline_drop_empty() {
-        let array = VInline::<CAP, String>::new();
-        drop(array);
+    fn drop_empty() {
+        let vi = VInline::<CAP, String>::new();
+        drop(vi);
+    }
+    /// This will:
+    /// - returns an array [0..upto]
+    /// - verify length
+    /// - verify payload
+    /// - verify capacity (if upto <= CAP)
+    /// - verify stack/heap logic
+    fn mkvi(upto: usize) -> VInline<CAP, usize> {
+        let r: VInline<CAP, _> = (0..upto).into_iter().collect();
+        assert_eq!(r.len(), upto);
+        if upto <= CAP {
+            assert_eq!(r.capacity(), CAP);
+            assert!(r.on_stack());
+        } else {
+            assert!(r.on_heap());
+        }
+        assert!((0..upto)
+            .into_iter()
+            .zip(r.iter())
+            .all(|(x, y)| { x == *y }));
+        r
+    }
+    #[test]
+    fn push_on_stack() {
+        let vi = mkvi(CAP);
+        assert!(vi.on_stack());
+    }
+    #[test]
+    fn push_on_heap() {
+        let vi = mkvi(CAP + 1);
+        assert_eq!(vi.capacity(), CAP * 2);
+    }
+    #[test]
+    fn remove_on_stack() {
+        let mut vi = mkvi(CAP);
+        assert_eq!(vi.remove(6), 6);
+        assert_eq!(vi.len(), CAP - 1);
+        assert_eq!(vi.capacity(), CAP);
+        assert_eq!(vi.as_ref(), [0, 1, 2, 3, 4, 5, 7]);
+    }
+    #[test]
+    fn remove_on_heap() {
+        let mut vi = mkvi(CAP + 1);
+        assert_eq!(vi.remove(6), 6);
+        assert_eq!(vi.len(), CAP);
+        assert_eq!(vi.capacity(), CAP * 2);
+        assert_eq!(vi.as_ref(), [0, 1, 2, 3, 4, 5, 7, 8]);
+    }
+    #[test]
+    fn optimize_capacity_none_on_stack() {
+        let mut vi = mkvi(CAP);
+        vi.optimize_capacity();
+        assert_eq!(vi.capacity(), CAP);
+        assert!(vi.on_stack());
+    }
+    #[test]
+    fn optimize_capacity_none_on_heap() {
+        let mut vi = mkvi(CAP + 1);
+        assert_eq!(vi.capacity(), CAP * 2);
+        vi.extend((CAP + 1..CAP * 2).into_iter());
+        assert_eq!(vi.capacity(), CAP * 2);
+        vi.optimize_capacity();
+        assert_eq!(vi.capacity(), CAP * 2);
+    }
+    #[test]
+    fn optimize_capacity_on_heap() {
+        let mut vi = mkvi(CAP + 1);
+        assert_eq!(vi.capacity(), CAP * 2);
+        vi.optimize_capacity();
+        assert_eq!(vi.capacity(), CAP + 1);
+    }
+    #[test]
+    fn optimize_capacity_mv_stack() {
+        let mut vi = mkvi(CAP + 1);
+        assert_eq!(vi.capacity(), CAP * 2);
+        let _ = vi.remove_compact(0);
+        assert_eq!(vi.len(), CAP);
+        assert_eq!(vi.capacity(), CAP);
+        assert!(vi.on_stack());
+    }
+    #[test]
+    fn clear_stack() {
+        let mut vi = mkvi(CAP);
+        vi.clear();
+        assert_eq!(vi.capacity(), CAP);
+        assert_eq!(vi.len(), 0);
     }
-
     #[test]
-    fn vinline_test() {
-        // first alloc on stack
-        let mut array = VInline::<CAP, String>::new();
-        (0..CAP).for_each(|i| array.push(format!("elem-{i}")));
-        // check meta methods
-        debug_assert!(array.on_stack());
-        debug_assert!(!array.will_be_on_stack());
-        // now iterate
-        array
-            .iter()
-            .enumerate()
-            .for_each(|(i, elem)| assert_eq!(elem, format!("elem-{i}").as_str()));
-        // now iter_mut
-        array
-            .iter_mut()
-            .enumerate()
-            .for_each(|(i, st)| *st = format!("elem-{}", i + 1));
-        // now let's get off the stack
-        (0..10).for_each(|i| array.push(format!("elem-{}", CAP + i + 1)));
-        // verify all elements
-        array
-            .iter()
-            .enumerate()
-            .for_each(|(i, st)| assert_eq!(st, format!("elem-{}", i + 1).as_str()));
-        debug_assert!(!array.on_stack());
-        debug_assert!(!array.will_be_on_stack());
+    fn clear_heap() {
+        let mut vi = mkvi(CAP + 1);
+        vi.clear();
+        assert_eq!(vi.capacity(), CAP * 2);
+        assert_eq!(vi.len(), 0);
     }
 }
 

From 14c1d9402751dda4d70050484e2af08232b2af70 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 23 Jan 2023 13:16:49 +0000
Subject: [PATCH 091/310] Refactor `mem` modules

---
 server/src/engine/mem/mod.rs     | 423 +------------------------------
 server/src/engine/mem/tests.rs   |   4 +-
 server/src/engine/mem/uarray.rs  | 151 +++++++++++
 server/src/engine/mem/vinline.rs | 322 +++++++++++++++++++++++
 4 files changed, 479 insertions(+), 421 deletions(-)
 create mode 100644 server/src/engine/mem/uarray.rs
 create mode 100644 server/src/engine/mem/vinline.rs

diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index fa0eb6e0..d4c545dd 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -26,423 +26,8 @@
 
 #[cfg(test)]
 mod tests;
+mod uarray;
+mod vinline;
 
-use {
-    core::{
-        alloc::Layout,
-        fmt,
-        mem::{self, ManuallyDrop, MaybeUninit},
-        ops::{Deref, DerefMut},
-        ptr, slice,
-    },
-    std::alloc::{alloc, dealloc},
-};
-
-union VData<const N: usize, T> {
-    s: ManuallyDrop<[MaybeUninit<T>; N]>,
-    h: *mut T,
-}
-
-struct VInline<const N: usize, T> {
-    d: VData<N, T>,
-    l: usize,
-    c: usize,
-}
-
-impl<const N: usize, T> VInline<N, T> {
-    #[inline(always)]
-    pub const fn new() -> Self {
-        let _ = Self::_ENSURE_ALIGN;
-        Self {
-            d: VData {
-                s: ManuallyDrop::new(Self::INLINE_NULL_STACK),
-            },
-            l: 0,
-            c: N,
-        }
-    }
-    #[inline(always)]
-    pub const fn capacity(&self) -> usize {
-        self.c
-    }
-    #[inline(always)]
-    pub const fn len(&self) -> usize {
-        self.l
-    }
-    #[inline(always)]
-    pub fn push(&mut self, v: T) {
-        self.grow();
-        unsafe {
-            // UNSAFE(@ohsayan): grow allocated the cap we needed
-            self.push_unchecked(v);
-        }
-        self.l += 1;
-    }
-    #[inline(always)]
-    pub fn clear(&mut self) {
-        unsafe {
-            // UNSAFE(@ohsayan): as_slice_mut will always give a valid ptr
-            ptr::drop_in_place(self._as_slice_mut());
-        }
-        self.l = 0;
-    }
-    #[inline(always)]
-    pub fn remove(&mut self, idx: usize) -> T {
-        if idx >= self.len() {
-            panic!("index out of range");
-        }
-        unsafe {
-            // UNSAFE(@ohsayan): Verified index is within range
-            self.remove_unchecked(idx)
-        }
-    }
-    #[inline(always)]
-    pub fn remove_compact(&mut self, idx: usize) -> T {
-        let r = self.remove(idx);
-        self.optimize_capacity();
-        r
-    }
-    #[inline(always)]
-    /// SAFETY: `idx` must be < l
-    unsafe fn remove_unchecked(&mut self, idx: usize) -> T {
-        // UNSAFE(@ohsayan): idx is in range
-        let ptr = self.as_mut_ptr().add(idx);
-        // UNSAFE(@ohsayan): idx is in range and is valid
-        let ret = ptr::read(ptr);
-        // UNSAFE(@ohsayan): move all elements to the left
-        ptr::copy(ptr.add(1), ptr, self.len() - idx - 1);
-        // UNSAFE(@ohsayan): this is our new length
-        self.set_len(self.len() - 1);
-        ret
-    }
-    #[inline(always)]
-    unsafe fn set_len(&mut self, len: usize) {
-        self.l = len;
-    }
-}
-
-impl<const N: usize, T> VInline<N, T> {
-    const INLINE_NULL: MaybeUninit<T> = MaybeUninit::uninit();
-    const INLINE_NULL_STACK: [MaybeUninit<T>; N] = [Self::INLINE_NULL; N];
-    const ALLOC_MULTIPLIER: usize = 2;
-    const _ENSURE_ALIGN: () =
-        debug_assert!(mem::align_of::<Vec<String>>() == mem::align_of::<VInline<N, String>>());
-    #[inline(always)]
-    fn on_heap(&self) -> bool {
-        self.c > N
-    }
-    #[inline(always)]
-    fn on_stack(&self) -> bool {
-        self.c == N
-    }
-    #[inline(always)]
-    fn _as_ptr(&self) -> *const T {
-        unsafe {
-            // UNSAFE(@ohsayan): We make legal accesses by checking state
-            if self.on_stack() {
-                self.d.s.as_ptr() as *const T
-            } else {
-                self.d.h as *const T
-            }
-        }
-    }
-    #[inline(always)]
-    fn _as_mut_ptr(&mut self) -> *mut T {
-        unsafe {
-            // UNSAFE(@ohsayan): We make legal accesses by checking state
-            if self.on_stack() {
-                (&mut self.d).s.as_mut_ptr() as *mut T
-            } else {
-                (&mut self.d).h as *mut T
-            }
-        }
-    }
-    #[inline(always)]
-    fn _as_slice(&self) -> &[T] {
-        unsafe {
-            // UNSAFE(@ohsayan): _as_ptr() will ensure correct addresses
-            slice::from_raw_parts(self._as_ptr(), self.l)
-        }
-    }
-    #[inline(always)]
-    fn _as_slice_mut(&mut self) -> &mut [T] {
-        unsafe {
-            // UNSAFE(@ohsayan): _as_mut_ptr() will ensure correct addresses
-            slice::from_raw_parts_mut(self._as_mut_ptr(), self.l)
-        }
-    }
-    #[inline(always)]
-    fn layout(cap: usize) -> Layout {
-        Layout::array::<T>(cap).unwrap()
-    }
-    #[inline(always)]
-    fn ncap(&self) -> usize {
-        self.c * Self::ALLOC_MULTIPLIER
-    }
-    fn alloc_block(cap: usize) -> *mut T {
-        unsafe {
-            // UNSAFE(@ohsayan): malloc bro
-            let p = alloc(Self::layout(cap));
-            assert!(!p.is_null(), "alloc,0");
-            p as *mut T
-        }
-    }
-    unsafe fn push_unchecked(&mut self, v: T) {
-        self._as_mut_ptr().add(self.l).write(v);
-    }
-    pub fn optimize_capacity(&mut self) {
-        if self.on_stack() || self.len() == self.capacity() {
-            return;
-        }
-        if self.l <= N {
-            unsafe {
-                // UNSAFE(@ohsayan): non-null heap
-                self.mv_to_stack();
-            }
-        } else {
-            let nb = Self::alloc_block(self.len());
-            unsafe {
-                // UNSAFE(@ohsayan): nonov; non-null
-                ptr::copy_nonoverlapping(self.d.h, nb, self.len());
-                // UNSAFE(@ohsayan): non-null heap
-                self.dealloc_heap(self.d.h);
-            }
-            self.d.h = nb;
-            self.c = self.len();
-        }
-    }
-    /// SAFETY: (1) non-null heap
-    unsafe fn mv_to_stack(&mut self) {
-        let heap = self.d.h;
-        // UNSAFE(@ohsayan): nonov; non-null (stack lol)
-        ptr::copy_nonoverlapping(self.d.h, (&mut self.d).s.as_mut_ptr() as *mut T, self.len());
-        // UNSAFE(@ohsayan): non-null heap
-        self.dealloc_heap(heap);
-        self.c = N;
-    }
-    #[inline]
-    fn grow(&mut self) {
-        if !(self.l == self.capacity()) {
-            return;
-        }
-        // allocate new block
-        let nc = self.ncap();
-        let nb = Self::alloc_block(nc);
-        if self.on_stack() {
-            // stack -> heap
-            unsafe {
-                // UNSAFE(@ohsayan): non-null; valid len
-                ptr::copy_nonoverlapping(self.d.s.as_ptr() as *const T, nb, self.l);
-            }
-        } else {
-            unsafe {
-                // UNSAFE(@ohsayan): non-null; valid len
-                ptr::copy_nonoverlapping(self.d.h.cast_const(), nb, self.l);
-                // UNSAFE(@ohsayan): non-null heap
-                self.dealloc_heap(self.d.h);
-            }
-        }
-        self.d.h = nb;
-        self.c = nc;
-    }
-    #[inline(always)]
-    unsafe fn dealloc_heap(&mut self, heap: *mut T) {
-        dealloc(heap as *mut u8, Self::layout(self.capacity()))
-    }
-}
-
-impl<const N: usize, T> Deref for VInline<N, T> {
-    type Target = [T];
-    fn deref(&self) -> &Self::Target {
-        self._as_slice()
-    }
-}
-
-impl<const N: usize, T> DerefMut for VInline<N, T> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self._as_slice_mut()
-    }
-}
-
-impl<const N: usize, T> Drop for VInline<N, T> {
-    fn drop(&mut self) {
-        unsafe {
-            // UNSAFE(@ohsayan): correct ptr guaranteed by safe impl of _as_slice_mut()
-            ptr::drop_in_place(self._as_slice_mut());
-            if !self.on_stack() {
-                // UNSAFE(@ohsayan): non-null heap
-                self.dealloc_heap(self.d.h);
-            }
-        }
-    }
-}
-
-impl<const N: usize, T: fmt::Debug> fmt::Debug for VInline<N, T> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_list().entries(self.iter()).finish()
-    }
-}
-
-impl<T, const N: usize> Extend<T> for VInline<N, T> {
-    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
-        // FIXME(@ohsayan): Optimize capacity match upfront
-        iter.into_iter().for_each(|item| self.push(item))
-    }
-}
-
-impl<T: Clone, const N: usize> Clone for VInline<N, T> {
-    fn clone(&self) -> Self {
-        unsafe {
-            if self.on_stack() {
-                // simple stack copy
-                let mut new_stack = Self::INLINE_NULL_STACK;
-                ptr::copy_nonoverlapping(self.d.s.as_ptr(), new_stack.as_mut_ptr(), self.l);
-                Self {
-                    d: VData {
-                        s: ManuallyDrop::new(new_stack),
-                    },
-                    l: self.l,
-                    c: N,
-                }
-            } else {
-                // new allocation
-                let nb = Self::alloc_block(self.len());
-                ptr::copy_nonoverlapping(self._as_ptr(), nb, self.l);
-                Self {
-                    d: VData { h: nb },
-                    l: self.l,
-                    c: self.l,
-                }
-            }
-        }
-    }
-}
-
-impl<T, const N: usize> FromIterator<T> for VInline<N, T> {
-    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
-        let it = iter.into_iter();
-        let mut slf = Self::new();
-        slf.extend(it);
-        slf
-    }
-}
-
-pub struct UArray<const N: usize, T> {
-    a: [MaybeUninit<T>; N],
-    l: usize,
-}
-
-impl<const N: usize, T> UArray<N, T> {
-    const NULL: MaybeUninit<T> = MaybeUninit::uninit();
-    const NULLED_ARRAY: [MaybeUninit<T>; N] = [Self::NULL; N];
-    #[inline(always)]
-    pub const fn new() -> Self {
-        Self {
-            a: Self::NULLED_ARRAY,
-            l: 0,
-        }
-    }
-    #[inline(always)]
-    pub const fn len(&self) -> usize {
-        self.l
-    }
-    #[inline(always)]
-    pub const fn capacity(&self) -> usize {
-        N
-    }
-    #[inline(always)]
-    pub const fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-    #[inline(always)]
-    unsafe fn incr_len(&mut self) {
-        self.l += 1;
-    }
-    #[inline(always)]
-    pub fn push(&mut self, v: T) {
-        if self.l == N {
-            panic!("stack,capof");
-        }
-        unsafe {
-            // UNSAFE(@ohsayan): verified length is smaller
-            self.push_unchecked(v);
-        }
-    }
-    pub fn remove(&mut self, idx: usize) -> T {
-        if idx >= self.len() {
-            panic!("out of range. idx is `{idx}` but len is `{}`", self.len());
-        }
-        unsafe {
-            // UNSAFE(@ohsayan): verified idx < l
-            self.remove_unchecked(idx)
-        }
-    }
-    /// SAFETY: idx < self.l
-    unsafe fn remove_unchecked(&mut self, idx: usize) -> T {
-        // UNSAFE(@ohsayan): Verified idx
-        let target = self.a.as_mut_ptr().add(idx).cast::<T>();
-        // UNSAFE(@ohsayan): Verified idx
-        let ret = ptr::read(target);
-        // UNSAFE(@ohsayan): ov; not-null; correct len
-        ptr::copy(target.add(1), target, self.len() - idx - 1);
-        ret
-    }
-    #[inline(always)]
-    /// SAFETY: self.l < N
-    unsafe fn push_unchecked(&mut self, v: T) {
-        // UNSAFE(@ohsayan): verified correct offsets (N)
-        self.a.as_mut_ptr().add(self.l).write(MaybeUninit::new(v));
-        // UNSAFE(@ohsayan): all G since l =< N
-        self.incr_len();
-    }
-    pub fn as_slice(&self) -> &[T] {
-        unsafe {
-            // UNSAFE(@ohsayan): ptr is always valid and len is correct, due to push impl
-            slice::from_raw_parts(self.a.as_ptr() as *const T, self.l)
-        }
-    }
-    pub fn as_slice_mut(&mut self) -> &mut [T] {
-        unsafe {
-            // UNSAFE(@ohsayan): ptr is always valid and len is correct, due to push impl
-            slice::from_raw_parts_mut(self.a.as_mut_ptr() as *mut T, self.l)
-        }
-    }
-}
-
-impl<const N: usize, T> Drop for UArray<N, T> {
-    fn drop(&mut self) {
-        if !self.is_empty() {
-            unsafe {
-                // UNSAFE(@ohsayan): as_slice_mut returns a correct offset
-                ptr::drop_in_place(self.as_slice_mut())
-            }
-        }
-    }
-}
-
-impl<const N: usize, T> Deref for UArray<N, T> {
-    type Target = [T];
-    fn deref(&self) -> &Self::Target {
-        self.as_slice()
-    }
-}
-
-impl<const N: usize, T> DerefMut for UArray<N, T> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self.as_slice_mut()
-    }
-}
-
-impl<const N: usize, T> FromIterator<T> for UArray<N, T> {
-    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
-        let mut slf = Self::new();
-        iter.into_iter().for_each(|v| slf.push(v));
-        slf
-    }
-}
-
-impl<const N: usize, T> Extend<T> for UArray<N, T> {
-    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
-        iter.into_iter().for_each(|v| self.push(v))
-    }
-}
+pub use uarray::UArray;
+pub use vinline::VInline;
diff --git a/server/src/engine/mem/tests.rs b/server/src/engine/mem/tests.rs
index 1043b9fa..ba661a92 100644
--- a/server/src/engine/mem/tests.rs
+++ b/server/src/engine/mem/tests.rs
@@ -27,7 +27,7 @@
 use super::*;
 
 mod vinline {
-    use super::*;
+    use super::VInline;
     const CAP: usize = 8;
     #[test]
     fn drop_empty() {
@@ -130,7 +130,7 @@ mod vinline {
 }
 
 mod uarray {
-    use super::*;
+    use super::UArray;
     const CAP: usize = 8;
     #[test]
     fn empty() {
diff --git a/server/src/engine/mem/uarray.rs b/server/src/engine/mem/uarray.rs
new file mode 100644
index 00000000..154695f2
--- /dev/null
+++ b/server/src/engine/mem/uarray.rs
@@ -0,0 +1,151 @@
+/*
+ * Created on Mon Jan 23 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use core::{
+    mem::MaybeUninit,
+    ops::{Deref, DerefMut},
+    ptr, slice,
+};
+
+pub struct UArray<const N: usize, T> {
+    a: [MaybeUninit<T>; N],
+    l: usize,
+}
+
+impl<const N: usize, T> UArray<N, T> {
+    const NULL: MaybeUninit<T> = MaybeUninit::uninit();
+    const NULLED_ARRAY: [MaybeUninit<T>; N] = [Self::NULL; N];
+    #[inline(always)]
+    pub const fn new() -> Self {
+        Self {
+            a: Self::NULLED_ARRAY,
+            l: 0,
+        }
+    }
+    #[inline(always)]
+    pub const fn len(&self) -> usize {
+        self.l
+    }
+    #[inline(always)]
+    pub const fn capacity(&self) -> usize {
+        N
+    }
+    #[inline(always)]
+    pub const fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+    #[inline(always)]
+    unsafe fn incr_len(&mut self) {
+        self.l += 1;
+    }
+    #[inline(always)]
+    pub fn push(&mut self, v: T) {
+        if self.l == N {
+            panic!("stack,capof");
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): verified length is smaller
+            self.push_unchecked(v);
+        }
+    }
+    pub fn remove(&mut self, idx: usize) -> T {
+        if idx >= self.len() {
+            panic!("out of range. idx is `{idx}` but len is `{}`", self.len());
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): verified idx < l
+            self.remove_unchecked(idx)
+        }
+    }
+    /// SAFETY: idx < self.l
+    unsafe fn remove_unchecked(&mut self, idx: usize) -> T {
+        // UNSAFE(@ohsayan): Verified idx
+        let target = self.a.as_mut_ptr().add(idx).cast::<T>();
+        // UNSAFE(@ohsayan): Verified idx
+        let ret = ptr::read(target);
+        // UNSAFE(@ohsayan): ov; not-null; correct len
+        ptr::copy(target.add(1), target, self.len() - idx - 1);
+        ret
+    }
+    #[inline(always)]
+    /// SAFETY: self.l < N
+    unsafe fn push_unchecked(&mut self, v: T) {
+        // UNSAFE(@ohsayan): verified correct offsets (N)
+        self.a.as_mut_ptr().add(self.l).write(MaybeUninit::new(v));
+        // UNSAFE(@ohsayan): all G since l =< N
+        self.incr_len();
+    }
+    pub fn as_slice(&self) -> &[T] {
+        unsafe {
+            // UNSAFE(@ohsayan): ptr is always valid and len is correct, due to push impl
+            slice::from_raw_parts(self.a.as_ptr() as *const T, self.l)
+        }
+    }
+    pub fn as_slice_mut(&mut self) -> &mut [T] {
+        unsafe {
+            // UNSAFE(@ohsayan): ptr is always valid and len is correct, due to push impl
+            slice::from_raw_parts_mut(self.a.as_mut_ptr() as *mut T, self.l)
+        }
+    }
+}
+
+impl<const N: usize, T> Drop for UArray<N, T> {
+    fn drop(&mut self) {
+        if !self.is_empty() {
+            unsafe {
+                // UNSAFE(@ohsayan): as_slice_mut returns a correct offset
+                ptr::drop_in_place(self.as_slice_mut())
+            }
+        }
+    }
+}
+
+impl<const N: usize, T> Deref for UArray<N, T> {
+    type Target = [T];
+    fn deref(&self) -> &Self::Target {
+        self.as_slice()
+    }
+}
+
+impl<const N: usize, T> DerefMut for UArray<N, T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.as_slice_mut()
+    }
+}
+
+impl<const N: usize, T> FromIterator<T> for UArray<N, T> {
+    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
+        let mut slf = Self::new();
+        iter.into_iter().for_each(|v| slf.push(v));
+        slf
+    }
+}
+
+impl<const N: usize, T> Extend<T> for UArray<N, T> {
+    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
+        iter.into_iter().for_each(|v| self.push(v))
+    }
+}
diff --git a/server/src/engine/mem/vinline.rs b/server/src/engine/mem/vinline.rs
new file mode 100644
index 00000000..6b1bf19e
--- /dev/null
+++ b/server/src/engine/mem/vinline.rs
@@ -0,0 +1,322 @@
+/*
+ * Created on Mon Jan 23 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use std::{
+    alloc::{alloc, dealloc, Layout},
+    fmt,
+    mem::{self, ManuallyDrop, MaybeUninit},
+    ops::{Deref, DerefMut},
+    ptr, slice,
+};
+
+union VData<const N: usize, T> {
+    s: ManuallyDrop<[MaybeUninit<T>; N]>,
+    h: *mut T,
+}
+
+pub struct VInline<const N: usize, T> {
+    d: VData<N, T>,
+    l: usize,
+    c: usize,
+}
+
+impl<const N: usize, T> VInline<N, T> {
+    #[inline(always)]
+    pub const fn new() -> Self {
+        let _ = Self::_ENSURE_ALIGN;
+        Self {
+            d: VData {
+                s: ManuallyDrop::new(Self::INLINE_NULL_STACK),
+            },
+            l: 0,
+            c: N,
+        }
+    }
+    #[inline(always)]
+    pub const fn capacity(&self) -> usize {
+        self.c
+    }
+    #[inline(always)]
+    pub const fn len(&self) -> usize {
+        self.l
+    }
+    #[inline(always)]
+    pub fn push(&mut self, v: T) {
+        self.grow();
+        unsafe {
+            // UNSAFE(@ohsayan): grow allocated the cap we needed
+            self.push_unchecked(v);
+        }
+        self.l += 1;
+    }
+    #[inline(always)]
+    pub fn clear(&mut self) {
+        unsafe {
+            // UNSAFE(@ohsayan): as_slice_mut will always give a valid ptr
+            ptr::drop_in_place(self._as_slice_mut());
+        }
+        self.l = 0;
+    }
+    #[inline(always)]
+    pub fn remove(&mut self, idx: usize) -> T {
+        if idx >= self.len() {
+            panic!("index out of range");
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): Verified index is within range
+            self.remove_unchecked(idx)
+        }
+    }
+    #[inline(always)]
+    pub fn remove_compact(&mut self, idx: usize) -> T {
+        let r = self.remove(idx);
+        self.optimize_capacity();
+        r
+    }
+    #[inline(always)]
+    /// SAFETY: `idx` must be < l
+    unsafe fn remove_unchecked(&mut self, idx: usize) -> T {
+        // UNSAFE(@ohsayan): idx is in range
+        let ptr = self.as_mut_ptr().add(idx);
+        // UNSAFE(@ohsayan): idx is in range and is valid
+        let ret = ptr::read(ptr);
+        // UNSAFE(@ohsayan): move all elements to the left
+        ptr::copy(ptr.add(1), ptr, self.len() - idx - 1);
+        // UNSAFE(@ohsayan): this is our new length
+        self.set_len(self.len() - 1);
+        ret
+    }
+    #[inline(always)]
+    unsafe fn set_len(&mut self, len: usize) {
+        self.l = len;
+    }
+}
+
+impl<const N: usize, T> VInline<N, T> {
+    const INLINE_NULL: MaybeUninit<T> = MaybeUninit::uninit();
+    const INLINE_NULL_STACK: [MaybeUninit<T>; N] = [Self::INLINE_NULL; N];
+    const ALLOC_MULTIPLIER: usize = 2;
+    const _ENSURE_ALIGN: () =
+        debug_assert!(mem::align_of::<Vec<String>>() == mem::align_of::<VInline<N, String>>());
+    #[inline(always)]
+    pub fn on_heap(&self) -> bool {
+        self.c > N
+    }
+    #[inline(always)]
+    pub fn on_stack(&self) -> bool {
+        self.c == N
+    }
+    #[inline(always)]
+    fn _as_ptr(&self) -> *const T {
+        unsafe {
+            // UNSAFE(@ohsayan): We make legal accesses by checking state
+            if self.on_stack() {
+                self.d.s.as_ptr() as *const T
+            } else {
+                self.d.h as *const T
+            }
+        }
+    }
+    #[inline(always)]
+    fn _as_mut_ptr(&mut self) -> *mut T {
+        unsafe {
+            // UNSAFE(@ohsayan): We make legal accesses by checking state
+            if self.on_stack() {
+                (&mut self.d).s.as_mut_ptr() as *mut T
+            } else {
+                (&mut self.d).h as *mut T
+            }
+        }
+    }
+    #[inline(always)]
+    fn _as_slice(&self) -> &[T] {
+        unsafe {
+            // UNSAFE(@ohsayan): _as_ptr() will ensure correct addresses
+            slice::from_raw_parts(self._as_ptr(), self.l)
+        }
+    }
+    #[inline(always)]
+    fn _as_slice_mut(&mut self) -> &mut [T] {
+        unsafe {
+            // UNSAFE(@ohsayan): _as_mut_ptr() will ensure correct addresses
+            slice::from_raw_parts_mut(self._as_mut_ptr(), self.l)
+        }
+    }
+    #[inline(always)]
+    fn layout(cap: usize) -> Layout {
+        Layout::array::<T>(cap).unwrap()
+    }
+    #[inline(always)]
+    fn ncap(&self) -> usize {
+        self.c * Self::ALLOC_MULTIPLIER
+    }
+    fn alloc_block(cap: usize) -> *mut T {
+        unsafe {
+            // UNSAFE(@ohsayan): malloc bro
+            let p = alloc(Self::layout(cap));
+            assert!(!p.is_null(), "alloc,0");
+            p as *mut T
+        }
+    }
+    unsafe fn push_unchecked(&mut self, v: T) {
+        self._as_mut_ptr().add(self.l).write(v);
+    }
+    pub fn optimize_capacity(&mut self) {
+        if self.on_stack() || self.len() == self.capacity() {
+            return;
+        }
+        if self.l <= N {
+            unsafe {
+                // UNSAFE(@ohsayan): non-null heap
+                self.mv_to_stack();
+            }
+        } else {
+            let nb = Self::alloc_block(self.len());
+            unsafe {
+                // UNSAFE(@ohsayan): nonov; non-null
+                ptr::copy_nonoverlapping(self.d.h, nb, self.len());
+                // UNSAFE(@ohsayan): non-null heap
+                self.dealloc_heap(self.d.h);
+            }
+            self.d.h = nb;
+            self.c = self.len();
+        }
+    }
+    /// SAFETY: (1) non-null heap
+    unsafe fn mv_to_stack(&mut self) {
+        let heap = self.d.h;
+        // UNSAFE(@ohsayan): nonov; non-null (stack lol)
+        ptr::copy_nonoverlapping(self.d.h, (&mut self.d).s.as_mut_ptr() as *mut T, self.len());
+        // UNSAFE(@ohsayan): non-null heap
+        self.dealloc_heap(heap);
+        self.c = N;
+    }
+    #[inline]
+    fn grow(&mut self) {
+        if !(self.l == self.capacity()) {
+            return;
+        }
+        // allocate new block
+        let nc = self.ncap();
+        let nb = Self::alloc_block(nc);
+        if self.on_stack() {
+            // stack -> heap
+            unsafe {
+                // UNSAFE(@ohsayan): non-null; valid len
+                ptr::copy_nonoverlapping(self.d.s.as_ptr() as *const T, nb, self.l);
+            }
+        } else {
+            unsafe {
+                // UNSAFE(@ohsayan): non-null; valid len
+                ptr::copy_nonoverlapping(self.d.h.cast_const(), nb, self.l);
+                // UNSAFE(@ohsayan): non-null heap
+                self.dealloc_heap(self.d.h);
+            }
+        }
+        self.d.h = nb;
+        self.c = nc;
+    }
+    #[inline(always)]
+    unsafe fn dealloc_heap(&mut self, heap: *mut T) {
+        dealloc(heap as *mut u8, Self::layout(self.capacity()))
+    }
+}
+
+impl<const N: usize, T> Deref for VInline<N, T> {
+    type Target = [T];
+    fn deref(&self) -> &Self::Target {
+        self._as_slice()
+    }
+}
+
+impl<const N: usize, T> DerefMut for VInline<N, T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self._as_slice_mut()
+    }
+}
+
+impl<const N: usize, T> Drop for VInline<N, T> {
+    fn drop(&mut self) {
+        unsafe {
+            // UNSAFE(@ohsayan): correct ptr guaranteed by safe impl of _as_slice_mut()
+            ptr::drop_in_place(self._as_slice_mut());
+            if !self.on_stack() {
+                // UNSAFE(@ohsayan): non-null heap
+                self.dealloc_heap(self.d.h);
+            }
+        }
+    }
+}
+
+impl<const N: usize, T: fmt::Debug> fmt::Debug for VInline<N, T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.iter()).finish()
+    }
+}
+
+impl<T, const N: usize> Extend<T> for VInline<N, T> {
+    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
+        // FIXME(@ohsayan): Optimize capacity match upfront
+        iter.into_iter().for_each(|item| self.push(item))
+    }
+}
+
+impl<T: Clone, const N: usize> Clone for VInline<N, T> {
+    fn clone(&self) -> Self {
+        unsafe {
+            if self.on_stack() {
+                // simple stack copy
+                let mut new_stack = Self::INLINE_NULL_STACK;
+                ptr::copy_nonoverlapping(self.d.s.as_ptr(), new_stack.as_mut_ptr(), self.l);
+                Self {
+                    d: VData {
+                        s: ManuallyDrop::new(new_stack),
+                    },
+                    l: self.l,
+                    c: N,
+                }
+            } else {
+                // new allocation
+                let nb = Self::alloc_block(self.len());
+                ptr::copy_nonoverlapping(self._as_ptr(), nb, self.l);
+                Self {
+                    d: VData { h: nb },
+                    l: self.l,
+                    c: self.l,
+                }
+            }
+        }
+    }
+}
+
+impl<T, const N: usize> FromIterator<T> for VInline<N, T> {
+    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
+        let it = iter.into_iter();
+        let mut slf = Self::new();
+        slf.extend(it);
+        slf
+    }
+}

From f63ec2d37d9a947490bd8ee3de504d6298335a7d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 23 Jan 2023 14:01:46 +0000
Subject: [PATCH 092/310] Add `IntoIter` for `UArray`

---
 server/src/engine/mem/tests.rs  | 27 ++++++++++++
 server/src/engine/mem/uarray.rs | 74 +++++++++++++++++++++++++++++++--
 2 files changed, 97 insertions(+), 4 deletions(-)

diff --git a/server/src/engine/mem/tests.rs b/server/src/engine/mem/tests.rs
index ba661a92..f2d978c9 100644
--- a/server/src/engine/mem/tests.rs
+++ b/server/src/engine/mem/tests.rs
@@ -170,4 +170,31 @@ mod uarray {
         a.iter_mut().for_each(|v| *v += 1);
         assert_eq!(a.as_slice(), [1, 2, 3, 4, 5, 6, 7, 8])
     }
+    #[test]
+    fn into_iter_empty() {
+        let a: UArray<CAP, u8> = UArray::new();
+        let r: Vec<u8> = a.into_iter().collect();
+        assert!(r.is_empty());
+    }
+    #[test]
+    fn into_iter() {
+        let a: UArray<CAP, _> = (0u8..8).into_iter().collect();
+        let r: Vec<u8> = a.into_iter().collect();
+        (0..8)
+            .into_iter()
+            .zip(r.into_iter())
+            .for_each(|(x, y)| assert_eq!(x, y));
+    }
+    #[test]
+    fn into_iter_partial() {
+        let a: UArray<CAP, String> = (0u8..8)
+            .into_iter()
+            .map(|v| ToString::to_string(&v))
+            .collect();
+        let r: Vec<String> = a.into_iter().take(4).collect();
+        (0..4)
+            .into_iter()
+            .zip(r.into_iter())
+            .for_each(|(x, y)| assert_eq!(x.to_string(), y));
+    }
 }
diff --git a/server/src/engine/mem/uarray.rs b/server/src/engine/mem/uarray.rs
index 154695f2..90a9b63f 100644
--- a/server/src/engine/mem/uarray.rs
+++ b/server/src/engine/mem/uarray.rs
@@ -25,6 +25,7 @@
 */
 
 use core::{
+    iter::FusedIterator,
     mem::MaybeUninit,
     ops::{Deref, DerefMut},
     ptr, slice,
@@ -58,10 +59,6 @@ impl<const N: usize, T> UArray<N, T> {
         self.len() == 0
     }
     #[inline(always)]
-    unsafe fn incr_len(&mut self) {
-        self.l += 1;
-    }
-    #[inline(always)]
     pub fn push(&mut self, v: T) {
         if self.l == N {
             panic!("stack,capof");
@@ -110,6 +107,14 @@ impl<const N: usize, T> UArray<N, T> {
             slice::from_raw_parts_mut(self.a.as_mut_ptr() as *mut T, self.l)
         }
     }
+    #[inline(always)]
+    unsafe fn set_len(&mut self, l: usize) {
+        self.l = l;
+    }
+    #[inline(always)]
+    unsafe fn incr_len(&mut self) {
+        self.set_len(self.len() + 1)
+    }
 }
 
 impl<const N: usize, T> Drop for UArray<N, T> {
@@ -149,3 +154,64 @@ impl<const N: usize, T> Extend<T> for UArray<N, T> {
         iter.into_iter().for_each(|v| self.push(v))
     }
 }
+
+pub struct IntoIter<const N: usize, T> {
+    i: usize,
+    l: usize,
+    d: UArray<N, T>,
+}
+
+impl<const N: usize, T> IntoIter<N, T> {
+    fn _next(&mut self) -> Option<T> {
+        if self.i == self.l {
+            return None;
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): Below length, so this is legal
+            let target = self.d.a.as_ptr().add(self.i) as *mut T;
+            // UNSAFE(@ohsayan): Again, non-null and part of our stack
+            let ret = ptr::read(target);
+            self.i += 1;
+            Some(ret)
+        }
+    }
+}
+
+impl<const N: usize, T> Drop for IntoIter<N, T> {
+    fn drop(&mut self) {
+        if self.i < self.l {
+            unsafe {
+                // UNSAFE(@ohsayan): Len is verified, due to intoiter init
+                let ptr = self.d.a.as_mut_ptr().add(self.i) as *mut T;
+                let len = self.l - self.i;
+                // UNSAFE(@ohsayan): we know the segment to drop
+                ptr::drop_in_place(ptr::slice_from_raw_parts_mut(ptr, len))
+            }
+        }
+    }
+}
+
+impl<const N: usize, T> Iterator for IntoIter<N, T> {
+    type Item = T;
+    fn next(&mut self) -> Option<Self::Item> {
+        self._next()
+    }
+}
+impl<const N: usize, T> ExactSizeIterator for IntoIter<N, T> {}
+impl<const N: usize, T> FusedIterator for IntoIter<N, T> {}
+
+impl<const N: usize, T> IntoIterator for UArray<N, T> {
+    type Item = T;
+
+    type IntoIter = IntoIter<N, T>;
+
+    fn into_iter(mut self) -> Self::IntoIter {
+        let l = self.len();
+        unsafe {
+            // UNSAFE(@ohsayan): Leave drop to intoiter
+            // HACK(@ohsayan): sneaky trick to let drop be handled by intoiter
+            self.set_len(0);
+        }
+        Self::IntoIter { d: self, i: 0, l }
+    }
+}

From 28b6ce0d96704b77e8585119d8b7cf2995d5eee2 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 23 Jan 2023 14:17:55 +0000
Subject: [PATCH 093/310] Add misc trait impls for seq mem structures

---
 server/src/engine/mem/tests.rs   | 18 ++++++++++++++++++
 server/src/engine/mem/uarray.rs  | 19 +++++++++++++++++++
 server/src/engine/mem/vinline.rs |  6 ++++++
 3 files changed, 43 insertions(+)

diff --git a/server/src/engine/mem/tests.rs b/server/src/engine/mem/tests.rs
index f2d978c9..f3aae28f 100644
--- a/server/src/engine/mem/tests.rs
+++ b/server/src/engine/mem/tests.rs
@@ -127,6 +127,18 @@ mod vinline {
         assert_eq!(vi.capacity(), CAP * 2);
         assert_eq!(vi.len(), 0);
     }
+    #[test]
+    fn clone_stack() {
+        let v1 = mkvi(CAP);
+        let v2 = v1.clone();
+        assert_eq!(v1, v2);
+    }
+    #[test]
+    fn clone_heap() {
+        let v1 = mkvi(CAP + 1);
+        let v2 = v1.clone();
+        assert_eq!(v1, v2);
+    }
 }
 
 mod uarray {
@@ -197,4 +209,10 @@ mod uarray {
             .zip(r.into_iter())
             .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
+    #[test]
+    fn clone() {
+        let a: UArray<CAP, u8> = (0u8..CAP as _).into_iter().collect();
+        let b = a.clone();
+        assert_eq!(a, b);
+    }
 }
diff --git a/server/src/engine/mem/uarray.rs b/server/src/engine/mem/uarray.rs
index 90a9b63f..263f454d 100644
--- a/server/src/engine/mem/uarray.rs
+++ b/server/src/engine/mem/uarray.rs
@@ -25,6 +25,7 @@
 */
 
 use core::{
+    fmt,
     iter::FusedIterator,
     mem::MaybeUninit,
     ops::{Deref, DerefMut},
@@ -117,6 +118,18 @@ impl<const N: usize, T> UArray<N, T> {
     }
 }
 
+impl<const N: usize, T: Clone> Clone for UArray<N, T> {
+    fn clone(&self) -> Self {
+        self.iter().cloned().collect()
+    }
+}
+
+impl<const M: usize, const N: usize, T: PartialEq> PartialEq<UArray<M, T>> for UArray<N, T> {
+    fn eq(&self, other: &UArray<M, T>) -> bool {
+        self.as_slice() == other.as_slice()
+    }
+}
+
 impl<const N: usize, T> Drop for UArray<N, T> {
     fn drop(&mut self) {
         if !self.is_empty() {
@@ -155,6 +168,12 @@ impl<const N: usize, T> Extend<T> for UArray<N, T> {
     }
 }
 
+impl<const N: usize, T: fmt::Debug> fmt::Debug for UArray<N, T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.iter()).finish()
+    }
+}
+
 pub struct IntoIter<const N: usize, T> {
     i: usize,
     l: usize,
diff --git a/server/src/engine/mem/vinline.rs b/server/src/engine/mem/vinline.rs
index 6b1bf19e..ee126884 100644
--- a/server/src/engine/mem/vinline.rs
+++ b/server/src/engine/mem/vinline.rs
@@ -258,6 +258,12 @@ impl<const N: usize, T> DerefMut for VInline<N, T> {
     }
 }
 
+impl<const M: usize, const N: usize, T: PartialEq> PartialEq<VInline<M, T>> for VInline<N, T> {
+    fn eq(&self, other: &VInline<M, T>) -> bool {
+        self._as_slice() == other._as_slice()
+    }
+}
+
 impl<const N: usize, T> Drop for VInline<N, T> {
     fn drop(&mut self) {
         unsafe {

From 86585ff864d3e20ae9c64d35b54ee5b46a70135e Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 23 Jan 2023 14:50:30 +0000
Subject: [PATCH 094/310] Impl into_iter for `VInline`

---
 server/src/engine/mem/tests.rs   | 36 ++++++++++++++++++
 server/src/engine/mem/vinline.rs | 65 ++++++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)

diff --git a/server/src/engine/mem/tests.rs b/server/src/engine/mem/tests.rs
index f3aae28f..da30fb68 100644
--- a/server/src/engine/mem/tests.rs
+++ b/server/src/engine/mem/tests.rs
@@ -139,6 +139,42 @@ mod vinline {
         let v2 = v1.clone();
         assert_eq!(v1, v2);
     }
+    #[test]
+    fn into_iter_stack() {
+        let v1 = mkvi(CAP);
+        let v: Vec<usize> = v1.into_iter().collect();
+        (0..CAP)
+            .into_iter()
+            .zip(v.into_iter())
+            .for_each(|(x, y)| assert_eq!(x, y));
+    }
+    #[test]
+    fn into_iter_stack_partial() {
+        let v1 = mkvi(CAP);
+        let v: Vec<usize> = v1.into_iter().take(CAP / 2).collect();
+        (0..CAP / 2)
+            .into_iter()
+            .zip(v.into_iter())
+            .for_each(|(x, y)| assert_eq!(x, y));
+    }
+    #[test]
+    fn into_iter_heap() {
+        let v1 = mkvi(CAP + 2);
+        let v: Vec<usize> = v1.into_iter().collect();
+        (0..CAP)
+            .into_iter()
+            .zip(v.into_iter())
+            .for_each(|(x, y)| assert_eq!(x, y));
+    }
+    #[test]
+    fn into_iter_heap_partial() {
+        let v1 = mkvi(CAP + 2);
+        let v: Vec<usize> = v1.into_iter().take(CAP / 2).collect();
+        (0..CAP / 2)
+            .into_iter()
+            .zip(v.into_iter())
+            .for_each(|(x, y)| assert_eq!(x, y));
+    }
 }
 
 mod uarray {
diff --git a/server/src/engine/mem/vinline.rs b/server/src/engine/mem/vinline.rs
index ee126884..d74beb5b 100644
--- a/server/src/engine/mem/vinline.rs
+++ b/server/src/engine/mem/vinline.rs
@@ -27,6 +27,7 @@
 use std::{
     alloc::{alloc, dealloc, Layout},
     fmt,
+    iter::FusedIterator,
     mem::{self, ManuallyDrop, MaybeUninit},
     ops::{Deref, DerefMut},
     ptr, slice,
@@ -326,3 +327,67 @@ impl<T, const N: usize> FromIterator<T> for VInline<N, T> {
         slf
     }
 }
+
+pub struct IntoIter<const N: usize, T> {
+    v: VInline<N, T>,
+    l: usize,
+    i: usize,
+}
+
+impl<const N: usize, T> IntoIter<N, T> {
+    fn _next(&mut self) -> Option<T> {
+        if self.i == self.l {
+            return None;
+        }
+        unsafe {
+            let current = self.i;
+            self.i += 1;
+            // UNSAFE(@ohsayan): i < l; so in all cases we are behind EOA
+            ptr::read(self.v._as_ptr().add(current).cast())
+        }
+    }
+}
+
+impl<const N: usize, T> Drop for IntoIter<N, T> {
+    fn drop(&mut self) {
+        if self.i < self.l {
+            // sweet
+            unsafe {
+                // UNSAFE(@ohsayan): Safe because we maintain the EOA cond; second, the l is the remaining part
+                ptr::drop_in_place(ptr::slice_from_raw_parts_mut(
+                    self.v._as_mut_ptr().add(self.i),
+                    self.l - self.i,
+                ))
+            }
+        }
+    }
+}
+
+impl<const N: usize, T> Iterator for IntoIter<N, T> {
+    type Item = T;
+    fn next(&mut self) -> Option<Self::Item> {
+        self._next()
+    }
+}
+impl<const N: usize, T> ExactSizeIterator for IntoIter<N, T> {}
+impl<const N: usize, T> FusedIterator for IntoIter<N, T> {}
+
+impl<const N: usize, T> IntoIterator for VInline<N, T> {
+    type Item = T;
+
+    type IntoIter = IntoIter<N, T>;
+
+    fn into_iter(mut self) -> Self::IntoIter {
+        let real = self.len();
+        unsafe {
+            // UNSAFE(@ohsayan): drop work for intoiter
+            // HACK(@ohsayan): same juicy drop hack
+            self.set_len(0);
+        }
+        Self::IntoIter {
+            v: self,
+            l: real,
+            i: 0,
+        }
+    }
+}

From 89888483e209fc26a6ebd58a6adde1c89356219d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 23 Jan 2023 15:02:22 +0000
Subject: [PATCH 095/310] Enable bi-directional iter on `UArray`

---
 server/src/engine/mem/tests.rs  | 20 ++++++++++++++++++++
 server/src/engine/mem/uarray.rs | 17 +++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/server/src/engine/mem/tests.rs b/server/src/engine/mem/tests.rs
index da30fb68..c627ef72 100644
--- a/server/src/engine/mem/tests.rs
+++ b/server/src/engine/mem/tests.rs
@@ -251,4 +251,24 @@ mod uarray {
         let b = a.clone();
         assert_eq!(a, b);
     }
+    #[test]
+    fn into_iter_rev() {
+        let a: UArray<CAP, String> = (0u8..8).into_iter().map(|v| v.to_string()).collect();
+        let r: Vec<String> = a.into_iter().rev().collect();
+        (0..8)
+            .rev()
+            .into_iter()
+            .zip(r.into_iter())
+            .for_each(|(x, y)| assert_eq!(x.to_string(), y));
+    }
+    #[test]
+    fn into_iter_rev_partial() {
+        let a: UArray<CAP, String> = (0u8..8).into_iter().map(|v| v.to_string()).collect();
+        let r: Vec<String> = a.into_iter().rev().take(4).collect();
+        (4..8)
+            .rev()
+            .into_iter()
+            .zip(r.into_iter())
+            .for_each(|(x, y)| assert_eq!(x.to_string(), y));
+    }
 }
diff --git a/server/src/engine/mem/uarray.rs b/server/src/engine/mem/uarray.rs
index 263f454d..85ced38b 100644
--- a/server/src/engine/mem/uarray.rs
+++ b/server/src/engine/mem/uarray.rs
@@ -181,6 +181,7 @@ pub struct IntoIter<const N: usize, T> {
 }
 
 impl<const N: usize, T> IntoIter<N, T> {
+    #[inline(always)]
     fn _next(&mut self) -> Option<T> {
         if self.i == self.l {
             return None;
@@ -194,6 +195,17 @@ impl<const N: usize, T> IntoIter<N, T> {
             Some(ret)
         }
     }
+    #[inline(always)]
+    fn _next_back(&mut self) -> Option<T> {
+        if self.i == self.l {
+            return None;
+        }
+        unsafe {
+            self.l -= 1;
+            // UNSAFE(@ohsayan): we always ensure EOA condition
+            Some(ptr::read(self.d.a.as_ptr().add(self.l).cast()))
+        }
+    }
 }
 
 impl<const N: usize, T> Drop for IntoIter<N, T> {
@@ -218,6 +230,11 @@ impl<const N: usize, T> Iterator for IntoIter<N, T> {
 }
 impl<const N: usize, T> ExactSizeIterator for IntoIter<N, T> {}
 impl<const N: usize, T> FusedIterator for IntoIter<N, T> {}
+impl<const N: usize, T> DoubleEndedIterator for IntoIter<N, T> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self._next_back()
+    }
+}
 
 impl<const N: usize, T> IntoIterator for UArray<N, T> {
     type Item = T;

From d831eb3a7c8b642adff263295f868aa459e48a20 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 23 Jan 2023 15:17:34 +0000
Subject: [PATCH 096/310] Enable bi-directional iter on `VInline`

---
 server/src/engine/mem/tests.rs   | 79 ++++++++++++++++++++++++++------
 server/src/engine/mem/vinline.rs | 16 +++++++
 2 files changed, 81 insertions(+), 14 deletions(-)

diff --git a/server/src/engine/mem/tests.rs b/server/src/engine/mem/tests.rs
index c627ef72..2245b5c8 100644
--- a/server/src/engine/mem/tests.rs
+++ b/server/src/engine/mem/tests.rs
@@ -40,8 +40,12 @@ mod vinline {
     /// - verify payload
     /// - verify capacity (if upto <= CAP)
     /// - verify stack/heap logic
-    fn mkvi(upto: usize) -> VInline<CAP, usize> {
-        let r: VInline<CAP, _> = (0..upto).into_iter().collect();
+    fn cmkvi<F, T: PartialEq>(upto: usize, map: F) -> VInline<CAP, T>
+    where
+        F: Clone + FnMut(usize) -> T,
+    {
+        let map2 = map.clone();
+        let r: VInline<CAP, _> = (0..upto).into_iter().map(map).collect();
         assert_eq!(r.len(), upto);
         if upto <= CAP {
             assert_eq!(r.capacity(), CAP);
@@ -51,10 +55,17 @@ mod vinline {
         }
         assert!((0..upto)
             .into_iter()
+            .map(map2)
             .zip(r.iter())
             .all(|(x, y)| { x == *y }));
         r
     }
+    fn mkvi(upto: usize) -> VInline<CAP, usize> {
+        cmkvi(upto, |v| v)
+    }
+    fn mkvi_str(upto: usize) -> VInline<CAP, String> {
+        cmkvi(upto, |v| v.to_string())
+    }
     #[test]
     fn push_on_stack() {
         let vi = mkvi(CAP);
@@ -141,39 +152,79 @@ mod vinline {
     }
     #[test]
     fn into_iter_stack() {
-        let v1 = mkvi(CAP);
-        let v: Vec<usize> = v1.into_iter().collect();
+        let v1 = mkvi_str(CAP);
+        let v: Vec<String> = v1.into_iter().collect();
         (0..CAP)
             .into_iter()
             .zip(v.into_iter())
-            .for_each(|(x, y)| assert_eq!(x, y));
+            .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
     #[test]
     fn into_iter_stack_partial() {
-        let v1 = mkvi(CAP);
-        let v: Vec<usize> = v1.into_iter().take(CAP / 2).collect();
+        let v1 = mkvi_str(CAP);
+        let v: Vec<String> = v1.into_iter().take(CAP / 2).collect();
         (0..CAP / 2)
             .into_iter()
             .zip(v.into_iter())
-            .for_each(|(x, y)| assert_eq!(x, y));
+            .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
     #[test]
     fn into_iter_heap() {
-        let v1 = mkvi(CAP + 2);
-        let v: Vec<usize> = v1.into_iter().collect();
+        let v1 = mkvi_str(CAP + 2);
+        let v: Vec<String> = v1.into_iter().collect();
         (0..CAP)
             .into_iter()
             .zip(v.into_iter())
-            .for_each(|(x, y)| assert_eq!(x, y));
+            .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
     #[test]
     fn into_iter_heap_partial() {
-        let v1 = mkvi(CAP + 2);
-        let v: Vec<usize> = v1.into_iter().take(CAP / 2).collect();
+        let v1 = mkvi_str(CAP + 2);
+        let v: Vec<String> = v1.into_iter().take(CAP / 2).collect();
         (0..CAP / 2)
             .into_iter()
             .zip(v.into_iter())
-            .for_each(|(x, y)| assert_eq!(x, y));
+            .for_each(|(x, y)| assert_eq!(x.to_string(), y));
+    }
+    #[test]
+    fn into_iter_rev_stack() {
+        let v1 = mkvi_str(CAP);
+        let v: Vec<String> = v1.into_iter().rev().collect();
+        (0..CAP)
+            .rev()
+            .into_iter()
+            .zip(v.into_iter())
+            .for_each(|(x, y)| assert_eq!(x.to_string(), y));
+    }
+    #[test]
+    fn into_iter_rev_stack_partial() {
+        let v1 = mkvi_str(CAP);
+        let v: Vec<String> = v1.into_iter().rev().take(CAP / 2).collect();
+        (CAP..CAP / 2)
+            .rev()
+            .into_iter()
+            .zip(v.into_iter())
+            .for_each(|(x, y)| assert_eq!(x.to_string(), y));
+    }
+    #[test]
+    fn into_iter_rev_heap() {
+        let v1 = mkvi_str(CAP + 2);
+        let v: Vec<String> = v1.into_iter().rev().collect();
+        (0..CAP + 2)
+            .rev()
+            .into_iter()
+            .zip(v.into_iter())
+            .for_each(|(x, y)| assert_eq!(x.to_string(), y));
+    }
+    #[test]
+    fn into_iter_rev_heap_partial() {
+        let v1 = mkvi_str(CAP + 2);
+        let v: Vec<String> = v1.into_iter().rev().take(CAP / 2).collect();
+        (CAP..CAP / 2)
+            .rev()
+            .into_iter()
+            .zip(v.into_iter())
+            .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
 }
 
diff --git a/server/src/engine/mem/vinline.rs b/server/src/engine/mem/vinline.rs
index d74beb5b..e3f84c54 100644
--- a/server/src/engine/mem/vinline.rs
+++ b/server/src/engine/mem/vinline.rs
@@ -335,6 +335,7 @@ pub struct IntoIter<const N: usize, T> {
 }
 
 impl<const N: usize, T> IntoIter<N, T> {
+    #[inline(always)]
     fn _next(&mut self) -> Option<T> {
         if self.i == self.l {
             return None;
@@ -346,6 +347,16 @@ impl<const N: usize, T> IntoIter<N, T> {
             ptr::read(self.v._as_ptr().add(current).cast())
         }
     }
+    #[inline(always)]
+    fn _next_back(&mut self) -> Option<T> {
+        if self.i == self.l {
+            return None;
+        }
+        unsafe {
+            self.l -= 1;
+            ptr::read(self.v._as_ptr().add(self.l).cast())
+        }
+    }
 }
 
 impl<const N: usize, T> Drop for IntoIter<N, T> {
@@ -371,6 +382,11 @@ impl<const N: usize, T> Iterator for IntoIter<N, T> {
 }
 impl<const N: usize, T> ExactSizeIterator for IntoIter<N, T> {}
 impl<const N: usize, T> FusedIterator for IntoIter<N, T> {}
+impl<const N: usize, T> DoubleEndedIterator for IntoIter<N, T> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self._next_back()
+    }
+}
 
 impl<const N: usize, T> IntoIterator for VInline<N, T> {
     type Item = T;

From 548ed53e16a47f2089986be441f51ee4345e101f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 24 Jan 2023 01:20:17 -0800
Subject: [PATCH 097/310] Do not enforce a `Clone` bound in idx impls

---
 server/src/engine/idx/mod.rs   | 106 ++++++++++++++++++---------------
 server/src/engine/idx/stdhm.rs |  21 ++++---
 server/src/engine/idx/stord.rs |  28 +++++----
 3 files changed, 86 insertions(+), 69 deletions(-)

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index bf8e4f56..aa95e65a 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -24,53 +24,59 @@
  *
 */
 
-use core::{borrow::Borrow, hash::Hash};
-
 mod stdhm;
 mod stord;
 #[cfg(test)]
 mod tests;
 
+use core::{borrow::Borrow, hash::Hash};
+
 // re-exports
 pub type IndexSTSeq<K, V, S> = stord::IndexSTSeqDll<K, V, S>;
 pub type IndexSTSeqDef<K, V> = IndexSTSeq<K, V, IndexSTSeqHasher>;
 pub type IndexSTSeqHasher = stord::IndexSTSeqDllHasher;
 
 /// Any type implementing this trait can be used as a key inside memory engine structures
-pub trait AsKey: Hash + Eq + Clone {
+pub trait AsKey: Hash + Eq {
     /// Read the key
     fn read_key(&self) -> &Self;
-    /// Read the key and return a clone
-    fn read_key_clone(&self) -> Self;
 }
 
-impl<T: Hash + Eq + Clone> AsKey for T {
-    #[inline(always)]
+impl<T: Hash + Eq + ?Sized> AsKey for T {
     fn read_key(&self) -> &Self {
         self
     }
+}
+
+/// If your T can be cloned/copied and implements [`AsKey`], then this trait will automatically be implemented
+pub trait AsKeyClone: AsKey + Clone {
+    /// Read the key and return a clone
+    fn read_key_clone(&self) -> Self;
+}
+
+impl<T: AsKey + Clone + ?Sized> AsKeyClone for T {
     #[inline(always)]
     fn read_key_clone(&self) -> Self {
         Clone::clone(self)
     }
 }
 
-pub trait AsKeyRef: Hash + Eq {}
-impl<T: Hash + Eq + ?Sized> AsKeyRef for T {}
+pub trait AsValue {
+    fn read_value(&self) -> &Self;
+}
+impl<T: ?Sized> AsValue for T {
+    fn read_value(&self) -> &Self {
+        self
+    }
+}
 
 /// Any type implementing this trait can be used as a value inside memory engine structures
-pub trait AsValue: Clone {
-    /// Read the value
-    fn read_value(&self) -> &Self;
+pub trait AsValueClone: AsValue + Clone {
     /// Read the value and return a clone
     fn read_value_clone(&self) -> Self;
 }
 
-impl<T: Clone> AsValue for T {
-    #[inline(always)]
-    fn read_value(&self) -> &Self {
-        self
-    }
+impl<T: AsValue + Clone + ?Sized> AsValueClone for T {
     #[inline(always)]
     fn read_value_clone(&self) -> Self {
         Clone::clone(self)
@@ -142,52 +148,53 @@ pub trait MTIndex<K, V>: IndexBaseSpec<K, V> {
     /// violated
     fn mt_insert(&self, key: K, val: V) -> bool
     where
-        K: AsKey,
+        K: AsKeyClone,
         V: AsValue;
     /// Updates or inserts the given value
     fn mt_upsert(&self, key: K, val: V)
     where
-        K: AsKey,
+        K: AsKeyClone,
         V: AsValue;
     // read
     fn mt_contains<Q>(&self, key: &Q) -> bool
     where
-        K: Borrow<Q> + AsKey,
-        Q: ?Sized + AsKeyRef;
+        K: Borrow<Q> + AsKeyClone,
+        Q: ?Sized + AsKey;
     /// Returns a reference to the value corresponding to the key, if it exists
     fn mt_get<Q>(&self, key: &Q) -> Option<&V>
     where
-        K: AsKey + Borrow<Q>,
-        Q: ?Sized + AsKeyRef;
+        K: AsKeyClone + Borrow<Q>,
+        Q: ?Sized + AsKey;
     /// Returns a clone of the value corresponding to the key, if it exists
     fn mt_get_cloned<Q>(&self, key: &Q) -> Option<V>
     where
-        K: AsKey + Borrow<Q>,
-        Q: ?Sized + AsKeyRef;
+        K: AsKeyClone + Borrow<Q>,
+        Q: ?Sized + AsKey,
+        V: AsValueClone;
     // update
     /// Returns true if the entry is updated
     fn mt_update<Q>(&self, key: &Q, val: V) -> bool
     where
-        K: AsKey + Borrow<Q>,
+        K: AsKeyClone + Borrow<Q>,
         V: AsValue,
-        Q: ?Sized + AsKeyRef;
+        Q: ?Sized + AsKey;
     /// Updates the entry and returns the old value, if it exists
     fn mt_update_return<Q>(&self, key: &Q, val: V) -> Option<V>
     where
-        K: AsKey + Borrow<Q>,
+        K: AsKeyClone + Borrow<Q>,
         V: AsValue,
-        Q: ?Sized + AsKeyRef;
+        Q: ?Sized + AsKey;
     // delete
     /// Returns true if the entry was deleted
     fn mt_delete<Q>(&self, key: &Q) -> bool
     where
-        K: AsKey + Borrow<Q>,
-        Q: ?Sized + AsKeyRef;
+        K: AsKeyClone + Borrow<Q>,
+        Q: ?Sized + AsKey;
     /// Removes the entry and returns it, if it exists
     fn mt_delete_return<Q>(&self, key: &Q) -> Option<V>
     where
-        K: AsKey + Borrow<Q>,
-        Q: ?Sized + AsKeyRef;
+        K: AsKeyClone + Borrow<Q>,
+        Q: ?Sized + AsKey;
 }
 
 /// An unordered STIndex
@@ -201,52 +208,53 @@ pub trait STIndex<K, V>: IndexBaseSpec<K, V> {
     /// violated
     fn st_insert(&mut self, key: K, val: V) -> bool
     where
-        K: AsKey,
+        K: AsKeyClone,
         V: AsValue;
     /// Updates or inserts the given value
     fn st_upsert(&mut self, key: K, val: V)
     where
-        K: AsKey,
+        K: AsKeyClone,
         V: AsValue;
     // read
     fn st_contains<Q>(&self, key: &Q) -> bool
     where
-        K: Borrow<Q> + AsKey,
-        Q: ?Sized + AsKeyRef;
+        K: Borrow<Q> + AsKeyClone,
+        Q: ?Sized + AsKey;
     /// Returns a reference to the value corresponding to the key, if it exists
     fn st_get<Q>(&self, key: &Q) -> Option<&V>
     where
-        K: AsKey + Borrow<Q>,
-        Q: ?Sized + AsKeyRef;
+        K: AsKeyClone + Borrow<Q>,
+        Q: ?Sized + AsKey;
     /// Returns a clone of the value corresponding to the key, if it exists
     fn st_get_cloned<Q>(&self, key: &Q) -> Option<V>
     where
-        K: AsKey + Borrow<Q>,
-        Q: ?Sized + AsKeyRef;
+        K: AsKeyClone + Borrow<Q>,
+        Q: ?Sized + AsKey,
+        V: AsValueClone;
     // update
     /// Returns true if the entry is updated
     fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
     where
-        K: AsKey + Borrow<Q>,
+        K: AsKeyClone + Borrow<Q>,
         V: AsValue,
-        Q: ?Sized + AsKeyRef;
+        Q: ?Sized + AsKey;
     /// Updates the entry and returns the old value, if it exists
     fn st_update_return<Q>(&mut self, key: &Q, val: V) -> Option<V>
     where
-        K: AsKey + Borrow<Q>,
+        K: AsKeyClone + Borrow<Q>,
         V: AsValue,
-        Q: ?Sized + AsKeyRef;
+        Q: ?Sized + AsKey;
     // delete
     /// Returns true if the entry was deleted
     fn st_delete<Q>(&mut self, key: &Q) -> bool
     where
-        K: AsKey + Borrow<Q>,
-        Q: ?Sized + AsKeyRef;
+        K: AsKeyClone + Borrow<Q>,
+        Q: ?Sized + AsKey;
     /// Removes the entry and returns it, if it exists
     fn st_delete_return<Q>(&mut self, key: &Q) -> Option<V>
     where
-        K: AsKey + Borrow<Q>,
-        Q: ?Sized + AsKeyRef;
+        K: AsKeyClone + Borrow<Q>,
+        Q: ?Sized + AsKey;
 }
 
 pub trait STIndexSeq<K, V>: STIndex<K, V> {
diff --git a/server/src/engine/idx/stdhm.rs b/server/src/engine/idx/stdhm.rs
index a50f6827..436bdf7c 100644
--- a/server/src/engine/idx/stdhm.rs
+++ b/server/src/engine/idx/stdhm.rs
@@ -24,7 +24,9 @@
  *
 */
 
-use super::{AsKey, AsKeyRef, AsValue, DummyMetrics, IndexBaseSpec, STIndex};
+#[cfg(debug_assertions)]
+use super::DummyMetrics;
+use super::{AsKey, AsValue, AsValueClone, IndexBaseSpec, STIndex};
 use std::{
     borrow::Borrow,
     collections::{
@@ -41,6 +43,7 @@ where
 {
     const PREALLOC: bool = true;
 
+    #[cfg(debug_assertions)]
     type Metrics = DummyMetrics;
 
     type IterKV<'a> = StdMapIterKV<'a, K, V>
@@ -83,6 +86,7 @@ where
         self.values()
     }
 
+    #[cfg(debug_assertions)]
     fn idx_metrics(&self) -> &Self::Metrics {
         &DummyMetrics
     }
@@ -119,7 +123,7 @@ where
     fn st_contains<Q>(&self, k: &Q) -> bool
     where
         K: Borrow<Q>,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
     {
         self.contains_key(k)
     }
@@ -127,7 +131,7 @@ where
     fn st_get<Q>(&self, key: &Q) -> Option<&V>
     where
         K: Borrow<Q>,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
     {
         self.get(key)
     }
@@ -135,7 +139,8 @@ where
     fn st_get_cloned<Q>(&self, key: &Q) -> Option<V>
     where
         K: Borrow<Q>,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
+        V: AsValueClone,
     {
         self.get(key).cloned()
     }
@@ -143,7 +148,7 @@ where
     fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
     where
         K: Borrow<Q>,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
     {
         self.get_mut(key).map(move |e| *e = val).is_some()
     }
@@ -151,7 +156,7 @@ where
     fn st_update_return<Q>(&mut self, key: &Q, val: V) -> Option<V>
     where
         K: Borrow<Q>,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
     {
         self.get_mut(key).map(move |e| {
             let mut new = val;
@@ -163,7 +168,7 @@ where
     fn st_delete<Q>(&mut self, key: &Q) -> bool
     where
         K: Borrow<Q>,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
     {
         self.remove(key).is_some()
     }
@@ -171,7 +176,7 @@ where
     fn st_delete_return<Q>(&mut self, key: &Q) -> Option<V>
     where
         K: Borrow<Q>,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
     {
         self.remove(key)
     }
diff --git a/server/src/engine/idx/stord.rs b/server/src/engine/idx/stord.rs
index 5f4b2183..9c4becb1 100644
--- a/server/src/engine/idx/stord.rs
+++ b/server/src/engine/idx/stord.rs
@@ -24,7 +24,7 @@
  *
 */
 
-use super::{AsKey, AsKeyRef, AsValue, IndexBaseSpec, STIndex, STIndexSeq};
+use super::{AsKey, AsKeyClone, AsValue, AsValueClone, IndexBaseSpec, STIndex, STIndexSeq};
 use std::{
     alloc::{alloc as std_alloc, dealloc as std_dealloc, Layout},
     borrow::Borrow,
@@ -217,6 +217,7 @@ pub struct IndexSTSeqDllMetrics {
     stat_f: usize,
 }
 
+#[cfg(debug_assertions)]
 impl IndexSTSeqDllMetrics {
     pub fn raw_f(&self) -> usize {
         self.stat_f
@@ -247,6 +248,7 @@ impl<K, V, S: BuildHasher> IndexSTSeqDll<K, V, S> {
             m,
             h,
             f,
+            #[cfg(debug_assertions)]
             metrics: IndexSTSeqDllMetrics::new(),
         }
     }
@@ -411,7 +413,7 @@ impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
     fn _get<Q: ?Sized>(&self, k: &Q) -> Option<&V>
     where
         K: Borrow<Q>,
-        Q: AsKeyRef,
+        Q: AsKey,
     {
         self.m
             .get(unsafe {
@@ -427,7 +429,7 @@ impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
     fn _update<Q: ?Sized>(&mut self, k: &Q, v: V) -> Option<V>
     where
         K: Borrow<Q>,
-        Q: AsKeyRef,
+        Q: AsKey,
     {
         match self.m.get(unsafe {
             // UNSAFE(@ohsayan): Just got a ref with the right bounds
@@ -457,7 +459,7 @@ impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
     fn _remove<Q>(&mut self, k: &Q) -> Option<V>
     where
         K: Borrow<Q>,
-        Q: AsKeyRef + ?Sized,
+        Q: AsKey + ?Sized,
     {
         self.m
             .remove(unsafe {
@@ -558,6 +560,7 @@ impl<K: AsKey, V: AsValue, S: BuildHasher + Default> FromIterator<(K, V)>
 impl<K, V, S: BuildHasher + Default> IndexBaseSpec<K, V> for IndexSTSeqDll<K, V, S> {
     const PREALLOC: bool = true;
 
+    #[cfg(debug_assertions)]
     type Metrics = IndexSTSeqDllMetrics;
 
     type IterKV<'a> = IndexSTSeqDllIterUnordKV<'a, K, V>
@@ -630,7 +633,7 @@ where
     fn st_contains<Q>(&self, key: &Q) -> bool
     where
         K: Borrow<Q> + AsKey,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
     {
         self.m.contains_key(unsafe {
             // UNSAFE(@ohsayan): Valid ref with correct bounds
@@ -641,7 +644,7 @@ where
     fn st_get<Q>(&self, key: &Q) -> Option<&V>
     where
         K: Borrow<Q>,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
     {
         self._get(key)
     }
@@ -649,7 +652,8 @@ where
     fn st_get_cloned<Q>(&self, key: &Q) -> Option<V>
     where
         K: Borrow<Q>,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
+        V: AsValueClone,
     {
         self._get(key).cloned()
     }
@@ -657,7 +661,7 @@ where
     fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
     where
         K: Borrow<Q>,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
     {
         self._update(key, val).is_some()
     }
@@ -665,7 +669,7 @@ where
     fn st_update_return<Q>(&mut self, key: &Q, val: V) -> Option<V>
     where
         K: Borrow<Q>,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
     {
         self._update(key, val)
     }
@@ -673,7 +677,7 @@ where
     fn st_delete<Q>(&mut self, key: &Q) -> bool
     where
         K: Borrow<Q>,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
     {
         self._remove(key).is_some()
     }
@@ -681,7 +685,7 @@ where
     fn st_delete_return<Q>(&mut self, key: &Q) -> Option<V>
     where
         K: Borrow<Q>,
-        Q: ?Sized + AsKeyRef,
+        Q: ?Sized + AsKey,
     {
         self._remove(key)
     }
@@ -722,7 +726,7 @@ where
     }
 }
 
-impl<K: AsKey, V: AsValue, S: BuildHasher + Default> Clone for IndexSTSeqDll<K, V, S> {
+impl<K: AsKeyClone, V: AsValueClone, S: BuildHasher + Default> Clone for IndexSTSeqDll<K, V, S> {
     fn clone(&self) -> Self {
         let mut slf = Self::with_capacity_and_hasher(self.len(), S::default());
         self._iter_ord_kv()

From 6fcf87c0407e53d78686f2d2e87e26d2fe32590d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 24 Jan 2023 01:37:12 -0800
Subject: [PATCH 098/310] Add misc bounds and impls for `TMCell`

---
 server/src/engine/sync/atm.rs  |  4 +---
 server/src/engine/sync/cell.rs | 29 +++++++++++++++++++++++++++--
 2 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/server/src/engine/sync/atm.rs b/server/src/engine/sync/atm.rs
index fd729148..00eb094d 100644
--- a/server/src/engine/sync/atm.rs
+++ b/server/src/engine/sync/atm.rs
@@ -27,9 +27,7 @@
 use core::{fmt, mem, ops::Deref, sync::atomic::Ordering};
 use crossbeam_epoch::{Atomic as CBAtomic, CompareExchangeError, Pointable, Pointer};
 // re-export here because we have some future plans ;) (@ohsayan)
-pub use crossbeam_epoch::{
-    pin as pin_current, unprotected as pin_unprotected, Guard, Owned, Shared,
-};
+pub use crossbeam_epoch::{pin as cpin, unprotected as upin, Guard, Owned, Shared};
 
 pub(super) const ORD_RLX: Ordering = Ordering::Relaxed;
 pub(super) const ORD_ACQ: Ordering = Ordering::Acquire;
diff --git a/server/src/engine/sync/cell.rs b/server/src/engine/sync/cell.rs
index 132a31c9..ac11a629 100644
--- a/server/src/engine/sync/cell.rs
+++ b/server/src/engine/sync/cell.rs
@@ -24,12 +24,13 @@
  *
 */
 
-use super::atm::{pin_unprotected, Atomic, Guard, Owned, Shared, ORD_REL};
+use super::atm::{upin, Atomic, Guard, Owned, Shared, ORD_REL};
 use core::ops::Deref;
 use parking_lot::{Mutex, MutexGuard};
 use std::marker::PhantomData;
 
 /// A [`TMCell`] provides atomic reads and serialized writes; the `static` is a CB hack
+#[derive(Debug)]
 pub struct TMCell<T: 'static> {
     a: Atomic<T>,
     g: Mutex<()>,
@@ -65,13 +66,17 @@ impl<T> Drop for TMCell<T> {
     fn drop(&mut self) {
         unsafe {
             // UNSAFE(@ohsayan): Sole owner with mutable access
-            let g = pin_unprotected();
+            let g = upin();
             let shptr = self.a.ld_rlx(&g);
             g.defer_destroy(shptr);
         }
     }
 }
 
+unsafe impl<T: Send> Send for TMCell<T> {}
+unsafe impl<T: Sync> Sync for TMCell<T> {}
+
+#[derive(Debug)]
 pub struct TMCellReadTxn<'a, 'g, T: 'static> {
     d: &'g T,
     _m: PhantomData<&'a TMCell<T>>,
@@ -88,6 +93,13 @@ impl<'a, 'g, T> TMCellReadTxn<'a, 'g, T> {
     }
 }
 
+impl<'a, 'g, T: Clone> TMCellReadTxn<'a, 'g, T> {
+    #[inline(always)]
+    pub fn read_copied(&self) -> T {
+        self.read().clone()
+    }
+}
+
 impl<'a, 'g, T: Copy> TMCellReadTxn<'a, 'g, T> {
     fn read_copy(&self) -> T {
         *self.d
@@ -101,6 +113,10 @@ impl<'a, 'g, T> Deref for TMCellReadTxn<'a, 'g, T> {
     }
 }
 
+unsafe impl<'a, 'g, T: Send> Send for TMCellReadTxn<'a, 'g, T> {}
+unsafe impl<'a, 'g, T: Sync> Sync for TMCellReadTxn<'a, 'g, T> {}
+
+#[derive(Debug)]
 pub struct TMCellWriteTxn<'a, 'g, T: 'static> {
     d: &'g T,
     a: &'a Atomic<T>,
@@ -134,6 +150,13 @@ impl<'a, 'g, T> TMCellWriteTxn<'a, 'g, T> {
     }
 }
 
+impl<'a, 'g, T: Clone> TMCellWriteTxn<'a, 'g, T> {
+    #[inline(always)]
+    pub fn read_copied(&self) -> T {
+        self.read().clone()
+    }
+}
+
 impl<'a, 'g, T: Copy> TMCellWriteTxn<'a, 'g, T> {
     fn read_copy(&self) -> T {
         *self.d
@@ -146,3 +169,5 @@ impl<'a, 'g, T> Deref for TMCellWriteTxn<'a, 'g, T> {
         self.d
     }
 }
+
+unsafe impl<'a, 'g, T: Sync> Sync for TMCellWriteTxn<'a, 'g, T> {}

From f26e71512ee08c534dcfcdb729b537009a8c26d6 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 26 Jan 2023 04:43:19 -0800
Subject: [PATCH 099/310] Fix segfaults in UArray impl

---
 server/src/engine/mem/tests.rs  | 12 ++++++++++++
 server/src/engine/mem/uarray.rs | 27 +++++++++++++++++++++++++++
 2 files changed, 39 insertions(+)

diff --git a/server/src/engine/mem/tests.rs b/server/src/engine/mem/tests.rs
index 2245b5c8..ddfef1ec 100644
--- a/server/src/engine/mem/tests.rs
+++ b/server/src/engine/mem/tests.rs
@@ -322,4 +322,16 @@ mod uarray {
             .zip(r.into_iter())
             .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
+    #[test]
+    fn pop_array() {
+        let mut a: UArray<CAP, String> = (0u8..8).into_iter().map(|v| v.to_string()).collect();
+        assert_eq!(a.pop().unwrap(), "7");
+        assert_eq!(a.len(), CAP - 1);
+    }
+    #[test]
+    fn clear_array() {
+        let mut a: UArray<CAP, String> = (0u8..8).into_iter().map(|v| v.to_string()).collect();
+        a.clear();
+        assert!(a.is_empty());
+    }
 }
diff --git a/server/src/engine/mem/uarray.rs b/server/src/engine/mem/uarray.rs
index 85ced38b..c4e6933e 100644
--- a/server/src/engine/mem/uarray.rs
+++ b/server/src/engine/mem/uarray.rs
@@ -78,19 +78,42 @@ impl<const N: usize, T> UArray<N, T> {
             self.remove_unchecked(idx)
         }
     }
+    pub fn pop(&mut self) -> Option<T> {
+        if self.is_empty() {
+            None
+        } else {
+            unsafe {
+                // UNSAFE(@ohsayan): Non-empty checked
+                Some(self.remove_unchecked(self.len() - 1))
+            }
+        }
+    }
+    pub fn clear(&mut self) {
+        unsafe {
+            let ptr = self.as_slice_mut();
+            // UNSAFE(@ohsayan): We know this is the initialized length
+            ptr::drop_in_place(ptr);
+            // UNSAFE(@ohsayan): we've destroyed everything, so yeah, all g
+            self.set_len(0);
+        }
+    }
     /// SAFETY: idx < self.l
     unsafe fn remove_unchecked(&mut self, idx: usize) -> T {
+        debug_assert!(idx < self.len());
         // UNSAFE(@ohsayan): Verified idx
         let target = self.a.as_mut_ptr().add(idx).cast::<T>();
         // UNSAFE(@ohsayan): Verified idx
         let ret = ptr::read(target);
         // UNSAFE(@ohsayan): ov; not-null; correct len
         ptr::copy(target.add(1), target, self.len() - idx - 1);
+        // UNSAFE(@ohsayan): we just removed something, account for it
+        self.decr_len();
         ret
     }
     #[inline(always)]
     /// SAFETY: self.l < N
     unsafe fn push_unchecked(&mut self, v: T) {
+        debug_assert!(self.len() < N);
         // UNSAFE(@ohsayan): verified correct offsets (N)
         self.a.as_mut_ptr().add(self.l).write(MaybeUninit::new(v));
         // UNSAFE(@ohsayan): all G since l =< N
@@ -116,6 +139,10 @@ impl<const N: usize, T> UArray<N, T> {
     unsafe fn incr_len(&mut self) {
         self.set_len(self.len() + 1)
     }
+    #[inline(always)]
+    unsafe fn decr_len(&mut self) {
+        self.set_len(self.len() - 1)
+    }
 }
 
 impl<const N: usize, T: Clone> Clone for UArray<N, T> {

From a59823fc488f63a1c3a8eb54ac7b3aa9640d5a37 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 26 Jan 2023 04:53:32 -0800
Subject: [PATCH 100/310] Add misc lib changes

---
 server/src/engine/core/mod.rs |  6 +++---
 server/src/engine/macros.rs   | 18 ++++++++++++++----
 server/src/engine/sync/atm.rs | 30 +++++++++++++-----------------
 3 files changed, 30 insertions(+), 24 deletions(-)

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index bef51661..fae09acf 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -100,7 +100,7 @@ impl<const N: usize> From<[DataType; N]> for DataType {
     }
 }
 
-constgrp! {
+flags! {
     #[derive(PartialEq, Eq, Clone, Copy)]
     pub struct DataKind: u8 {
         // primitive: integer unsigned
@@ -120,9 +120,9 @@ constgrp! {
         FLOAT64 = 10,
         // compound: flat
         STR = 11,
-        STR_BX = Self::_BASE_HB | Self::STR.d(),
+        STR_BX = DataKind::_BASE_HB | DataKind::STR.d(),
         BIN = 12,
-        BIN_BX = Self::_BASE_HB | Self::BIN.d(),
+        BIN_BX = DataKind::_BASE_HB | DataKind::BIN.d(),
         // compound: recursive
         LIST = 13,
     }
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index 4301db40..c710e8af 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -11,7 +11,7 @@
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
+ * the Free Software fation, either version 3 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
@@ -53,9 +53,9 @@ macro_rules! assertions {
     ($($assert:expr),*$(,)?) => {$(const _:()=::core::assert!($assert);)*}
 }
 
-macro_rules! constgrp {
-    ($(#[$attr:meta])* $vis:vis struct $group:ident: $ty:ty { $($const:ident = $expr:expr),* $(,)?}) => (
-        $(#[$attr])* $vis struct $group {r#const: $ty}
+macro_rules! flags {
+    ($(#[$attr:meta])* $vis:vis struct $group:ident: $ty:ty { $($const:ident = $expr:expr),+ $(,)?}) => (
+        $(#[$attr])* #[repr(transparent)] $vis struct $group {r#const: $ty}
         impl $group {
             $(pub const $const: Self = Self { r#const: $expr };)*
             #[inline(always)] pub const fn d(&self) -> $ty { self.r#const }
@@ -63,9 +63,19 @@ macro_rules! constgrp {
             #[inline(always)] pub const fn name(&self) -> &'static str {
                 match self.r#const {$(capture if capture == $expr => ::core::stringify!($const),)* _ => ::core::unreachable!()}
             }
+            const LEN: usize = { let mut i = 0; $(let _ = $expr; i += 1;)+{i} };
+            const A: [$ty; $group::LEN] = [$($expr,)+];
+            const SANITY: () = {
+                let ref a = Self::A; let l = a.len(); let mut i = 0;
+                while i < l { let mut j = i + 1; while j < l { if a[i] == a[j] { panic!("conflict"); } j += 1; } i += 1; }
+            };
+            const ALL: $ty = { let mut r: $ty = 0; $( r |= $expr;)+ r };
+            pub const fn has_flags_in(v: $ty) -> bool { Self::ALL & v != 0 }
+            pub const fn bits() -> usize { let r: $ty = ($($expr+)+0); r.count_ones() as _ }
         }
         impl ::core::fmt::Debug for $group {
             fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
+                const _V : () = $group::SANITY;
                 ::core::write!(f, "{}::{}", ::core::stringify!($group), Self::name(self))
             }
         }
diff --git a/server/src/engine/sync/atm.rs b/server/src/engine/sync/atm.rs
index 00eb094d..57fa3142 100644
--- a/server/src/engine/sync/atm.rs
+++ b/server/src/engine/sync/atm.rs
@@ -29,15 +29,15 @@ use crossbeam_epoch::{Atomic as CBAtomic, CompareExchangeError, Pointable, Point
 // re-export here because we have some future plans ;) (@ohsayan)
 pub use crossbeam_epoch::{pin as cpin, unprotected as upin, Guard, Owned, Shared};
 
-pub(super) const ORD_RLX: Ordering = Ordering::Relaxed;
-pub(super) const ORD_ACQ: Ordering = Ordering::Acquire;
-pub(super) const ORD_REL: Ordering = Ordering::Release;
-pub(super) const ORD_ACR: Ordering = Ordering::AcqRel;
+pub const ORD_RLX: Ordering = Ordering::Relaxed;
+pub const ORD_ACQ: Ordering = Ordering::Acquire;
+pub const ORD_REL: Ordering = Ordering::Release;
+pub const ORD_ACR: Ordering = Ordering::AcqRel;
 
 type CxResult<'g, T, P> = Result<Shared<'g, T>, CompareExchangeError<'g, T, P>>;
 
-pub(super) const fn ensure_flag_align<T>(fsize: usize) {
-    debug_assert!(mem::align_of::<T>().trailing_zeros() as usize >= fsize);
+pub const fn ensure_flag_align<T, const FSIZE: usize>() -> bool {
+    mem::align_of::<T>().trailing_zeros() as usize >= FSIZE
 }
 
 pub struct Atomic<T> {
@@ -52,25 +52,22 @@ impl<T> fmt::Debug for Atomic<T> {
 }
 
 impl<T: Pointable> Atomic<T> {
-    // the compile time address size check ensures "first class" sanity
-    const _ENSURE_FLAG_STATIC_CHECK: () = ensure_flag_align::<T>(0);
     /// Instantiates a new atomic
     ///
     /// **This will allocate**
     pub fn new_alloc(t: T) -> Self {
-        let _ = Self::_ENSURE_FLAG_STATIC_CHECK;
         Self {
             a: CBAtomic::new(t),
         }
     }
     #[inline(always)]
-    pub(super) const fn null() -> Self {
+    pub const fn null() -> Self {
         Self {
             a: CBAtomic::null(),
         }
     }
     #[inline(always)]
-    pub(super) fn cx<'g, P>(
+    pub fn cx<'g, P>(
         &self,
         o: Shared<'g, T>,
         n: P,
@@ -84,7 +81,7 @@ impl<T: Pointable> Atomic<T> {
         self.a.compare_exchange(o, n, s, f, g)
     }
     #[inline(always)]
-    pub(super) fn cx_weak<'g, P>(
+    pub fn cx_weak<'g, P>(
         &self,
         o: Shared<'g, T>,
         n: P,
@@ -98,22 +95,22 @@ impl<T: Pointable> Atomic<T> {
         self.a.compare_exchange_weak(o, n, s, f, g)
     }
     #[inline(always)]
-    pub(super) fn cx_rel<'g, P>(&self, o: Shared<'g, T>, n: P, g: &'g Guard) -> CxResult<'g, T, P>
+    pub fn cx_rel<'g, P>(&self, o: Shared<'g, T>, n: P, g: &'g Guard) -> CxResult<'g, T, P>
     where
         P: Pointer<T>,
     {
         self.cx(o, n, ORD_REL, ORD_RLX, g)
     }
     #[inline(always)]
-    pub(super) fn ld<'g>(&self, o: Ordering, g: &'g Guard) -> Shared<'g, T> {
+    pub fn ld<'g>(&self, o: Ordering, g: &'g Guard) -> Shared<'g, T> {
         self.a.load(o, g)
     }
     #[inline(always)]
-    pub(super) fn ld_acq<'g>(&self, g: &'g Guard) -> Shared<'g, T> {
+    pub fn ld_acq<'g>(&self, g: &'g Guard) -> Shared<'g, T> {
         self.ld(ORD_ACQ, g)
     }
     #[inline(always)]
-    pub(crate) fn ld_rlx<'g>(&self, g: &'g Guard) -> Shared<'g, T> {
+    pub fn ld_rlx<'g>(&self, g: &'g Guard) -> Shared<'g, T> {
         self.ld(ORD_RLX, g)
     }
 }
@@ -123,7 +120,6 @@ where
     A: Into<CBAtomic<T>>,
 {
     fn from(t: A) -> Self {
-        let _ = Self::_ENSURE_FLAG_STATIC_CHECK;
         Self { a: Into::into(t) }
     }
 }

From 29baeb49968b7c0155785f10b2fb3b6eb6144b2d Mon Sep 17 00:00:00 2001
From: Sayan <nandansayan@outlook.com>
Date: Fri, 27 Jan 2023 22:53:01 +0530
Subject: [PATCH 101/310] Implement mtchm

---
 server/src/engine/idx/mod.rs          |   1 +
 server/src/engine/idx/mtchm/access.rs | 218 +++++++++
 server/src/engine/idx/mtchm/iter.rs   | 268 +++++++++++
 server/src/engine/idx/mtchm/meta.rs   | 123 +++++
 server/src/engine/idx/mtchm/mod.rs    | 658 ++++++++++++++++++++++++++
 server/src/engine/macros.rs           |   3 +
 server/src/engine/mem/vinline.rs      |   4 +-
 server/src/engine/mod.rs              |   3 +-
 server/src/engine/ql/macros.rs        |   6 +
 9 files changed, 1280 insertions(+), 4 deletions(-)
 create mode 100644 server/src/engine/idx/mtchm/access.rs
 create mode 100644 server/src/engine/idx/mtchm/iter.rs
 create mode 100644 server/src/engine/idx/mtchm/meta.rs
 create mode 100644 server/src/engine/idx/mtchm/mod.rs

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index aa95e65a..84d482fe 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -26,6 +26,7 @@
 
 mod stdhm;
 mod stord;
+mod mtchm;
 #[cfg(test)]
 mod tests;
 
diff --git a/server/src/engine/idx/mtchm/access.rs b/server/src/engine/idx/mtchm/access.rs
new file mode 100644
index 00000000..4b5177dc
--- /dev/null
+++ b/server/src/engine/idx/mtchm/access.rs
@@ -0,0 +1,218 @@
+/*
+ * Created on Fri Jan 27 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::meta::TreeElement;
+
+/// write mode flag
+type WriteFlag = u8;
+pub const WRITEMODE_DELETE: WriteFlag = 0xFF;
+/// fresh
+pub const WRITEMODE_FRESH: WriteFlag = 0b01;
+/// refresh
+pub const WRITEMODE_REFRESH: WriteFlag = 0b10;
+/// any
+pub const WRITEMODE_ANY: WriteFlag = 0b11;
+
+pub trait ReadMode<T>: 'static {
+    type Ret<'a>;
+    fn ex<'a>(v: &'a T) -> Self::Ret<'a>;
+    fn nx<'a>() -> Self::Ret<'a>;
+}
+
+pub struct RModeExists;
+impl<T> ReadMode<T> for RModeExists {
+    type Ret<'a> = bool;
+    fn ex<'a>(_: &'a T) -> Self::Ret<'a> {
+        true
+    }
+    fn nx<'a>() -> Self::Ret<'a> {
+        false
+    }
+}
+
+pub struct RModeRef;
+impl<T: TreeElement> ReadMode<T> for RModeRef {
+    type Ret<'a> = Option<&'a T::Value>;
+    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
+        Some(v.val())
+    }
+    fn nx<'a>() -> Self::Ret<'a> {
+        None
+    }
+}
+
+pub struct RModeClone;
+impl<T: TreeElement> ReadMode<T> for RModeClone {
+    type Ret<'a> = Option<T::Value>;
+    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
+        Some(v.val().clone())
+    }
+    fn nx<'a>() -> Self::Ret<'a> {
+        None
+    }
+}
+
+pub trait WriteMode<T>: 'static {
+    const WMODE: WriteFlag;
+    type Ret<'a>;
+    fn ex<'a>(v: &'a T) -> Self::Ret<'a>;
+    fn nx<'a>() -> Self::Ret<'a>;
+}
+
+pub struct WModeFresh;
+impl<T> WriteMode<T> for WModeFresh {
+    const WMODE: WriteFlag = WRITEMODE_FRESH;
+    type Ret<'a> = bool;
+    #[inline(always)]
+    fn ex(_: &T) -> Self::Ret<'static> {
+        false
+    }
+    #[inline(always)]
+    fn nx<'a>() -> Self::Ret<'a> {
+        true
+    }
+}
+
+pub struct WModeUpdate;
+impl<T> WriteMode<T> for WModeUpdate {
+    const WMODE: WriteFlag = WRITEMODE_REFRESH;
+    type Ret<'a> = bool;
+    #[inline(always)]
+    fn ex(_: &T) -> Self::Ret<'static> {
+        true
+    }
+    #[inline(always)]
+    fn nx<'a>() -> Self::Ret<'a> {
+        false
+    }
+}
+
+pub struct WModeUpdateRetClone;
+impl<T: TreeElement> WriteMode<T> for WModeUpdateRetClone {
+    const WMODE: WriteFlag = WRITEMODE_REFRESH;
+    type Ret<'a> = Option<T::Value>;
+    #[inline(always)]
+    fn ex(v: &T) -> Self::Ret<'static> {
+        Some(v.val().clone())
+    }
+    #[inline(always)]
+    fn nx<'a>() -> Self::Ret<'a> {
+        None
+    }
+}
+
+pub struct WModeUpdateRetRef;
+impl<T: TreeElement> WriteMode<T> for WModeUpdateRetRef {
+    const WMODE: WriteFlag = WRITEMODE_REFRESH;
+    type Ret<'a> = Option<&'a T::Value>;
+    #[inline(always)]
+    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
+        Some(v.val())
+    }
+    #[inline(always)]
+    fn nx<'a>() -> Self::Ret<'a> {
+        None
+    }
+}
+
+pub struct WModeUpsert;
+impl<T> WriteMode<T> for WModeUpsert {
+    const WMODE: WriteFlag = WRITEMODE_ANY;
+    type Ret<'a> = ();
+    #[inline(always)]
+    fn ex(_: &T) -> Self::Ret<'static> {
+        ()
+    }
+    #[inline(always)]
+    fn nx<'a>() -> Self::Ret<'a> {
+        ()
+    }
+}
+
+pub struct WModeUpsertRef;
+impl<T: TreeElement> WriteMode<T> for WModeUpsertRef {
+    const WMODE: WriteFlag = WRITEMODE_ANY;
+    type Ret<'a> = Option<&'a T::Value>;
+    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
+        Some(v.val())
+    }
+    fn nx<'a>() -> Self::Ret<'a> {
+        None
+    }
+}
+
+pub struct WModeUpsertClone;
+impl<T: TreeElement> WriteMode<T> for WModeUpsertClone {
+    const WMODE: WriteFlag = WRITEMODE_ANY;
+    type Ret<'a> = Option<T::Value>;
+    fn ex<'a>(v: &'a T) -> Self::Ret<'static> {
+        Some(v.val().clone())
+    }
+    fn nx<'a>() -> Self::Ret<'a> {
+        None
+    }
+}
+pub struct WModeDelete;
+impl<T: TreeElement> WriteMode<T> for WModeDelete {
+    const WMODE: WriteFlag = WRITEMODE_DELETE;
+    type Ret<'a> = bool;
+    #[inline(always)]
+    fn ex<'a>(_: &'a T) -> Self::Ret<'a> {
+        true
+    }
+    #[inline(always)]
+    fn nx<'a>() -> Self::Ret<'a> {
+        false
+    }
+}
+
+pub struct WModeDeleteRef;
+impl<T: TreeElement> WriteMode<T> for WModeDeleteRef {
+    const WMODE: WriteFlag = WRITEMODE_DELETE;
+    type Ret<'a> = Option<&'a T::Value>;
+    #[inline(always)]
+    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
+        Some(v.val())
+    }
+    #[inline(always)]
+    fn nx<'a>() -> Self::Ret<'a> {
+        None
+    }
+}
+
+pub struct WModeDeleteClone;
+impl<T: TreeElement> WriteMode<T> for WModeDeleteClone {
+    const WMODE: WriteFlag = WRITEMODE_DELETE;
+    type Ret<'a> = Option<T::Value>;
+    #[inline(always)]
+    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
+        Some(v.val().clone())
+    }
+    #[inline(always)]
+    fn nx<'a>() -> Self::Ret<'a> {
+        None
+    }
+}
diff --git a/server/src/engine/idx/mtchm/iter.rs b/server/src/engine/idx/mtchm/iter.rs
new file mode 100644
index 00000000..d856011a
--- /dev/null
+++ b/server/src/engine/idx/mtchm/iter.rs
@@ -0,0 +1,268 @@
+/*
+ * Created on Fri Jan 27 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::{
+    super::super::{
+        mem::UArray,
+        sync::atm::{Guard, Shared},
+    },
+    meta::{Config, DefConfig, NodeFlag, TreeElement},
+    Node, Tree,
+};
+use std::marker::PhantomData;
+
+pub struct IterKV<'t, 'g, 'v, T, S, C>
+where
+    't: 'v,
+    'g: 'v + 't,
+    C: Config,
+{
+    i: RawIter<'t, 'g, 'v, T, S, C, CfgIterKV>,
+}
+
+impl<'t, 'g, 'v, T, S, C> IterKV<'t, 'g, 'v, T, S, C>
+where
+    't: 'v,
+    'g: 'v + 't,
+    C: Config,
+{
+    pub fn new(t: &'t Tree<T, S, C>, g: &'g Guard) -> Self {
+        Self {
+            i: RawIter::new(t, g),
+        }
+    }
+}
+
+impl<'t, 'g, 'v, T, S, C> Iterator for IterKV<'t, 'g, 'v, T, S, C>
+where
+    't: 'v,
+    'g: 'v + 't,
+    C: Config,
+    T: TreeElement,
+{
+    type Item = &'v T;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.i.next()
+    }
+}
+
+pub struct IterKey<'t, 'g, 'v, T, S, C>
+where
+    't: 'v,
+    'g: 'v + 't,
+    C: Config,
+    T: TreeElement,
+{
+    i: RawIter<'t, 'g, 'v, T, S, C, CfgIterKey>,
+}
+
+impl<'t, 'g, 'v, T, S, C> IterKey<'t, 'g, 'v, T, S, C>
+where
+    't: 'v,
+    'g: 'v + 't,
+    C: Config,
+    T: TreeElement,
+{
+    pub fn new(t: &'t Tree<T, S, C>, g: &'g Guard) -> Self {
+        Self {
+            i: RawIter::new(t, g),
+        }
+    }
+}
+
+impl<'t, 'g, 'v, T, S, C> Iterator for IterKey<'t, 'g, 'v, T, S, C>
+where
+    't: 'v,
+    'g: 'v + 't,
+    C: Config,
+    T: TreeElement,
+{
+    type Item = &'v T::Key;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.i.next()
+    }
+}
+
+pub struct IterVal<'t, 'g, 'v, T, S, C>
+where
+    't: 'v,
+    'g: 'v + 't,
+    C: Config,
+    T: TreeElement,
+{
+    i: RawIter<'t, 'g, 'v, T, S, C, CfgIterVal>,
+}
+
+impl<'t, 'g, 'v, T, S, C> IterVal<'t, 'g, 'v, T, S, C>
+where
+    't: 'v,
+    'g: 'v + 't,
+    C: Config,
+    T: TreeElement,
+{
+    pub fn new(t: &'t Tree<T, S, C>, g: &'g Guard) -> Self {
+        Self {
+            i: RawIter::new(t, g),
+        }
+    }
+}
+
+impl<'t, 'g, 'v, T, S, C> Iterator for IterVal<'t, 'g, 'v, T, S, C>
+where
+    't: 'v,
+    'g: 'v + 't,
+    C: Config,
+    T: TreeElement,
+{
+    type Item = &'v T::Value;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.i.next()
+    }
+}
+
+trait IterConfig<T> {
+    type Ret<'a>
+    where
+        T: 'a;
+    fn some<'a>(v: &'a T) -> Option<Self::Ret<'a>>;
+}
+
+struct CfgIterKV;
+impl<T> IterConfig<T> for CfgIterKV {
+    type Ret<'a> = &'a T where T: 'a;
+    fn some<'a>(v: &'a T) -> Option<Self::Ret<'a>> {
+        Some(v)
+    }
+}
+
+struct CfgIterKey;
+impl<T: TreeElement> IterConfig<T> for CfgIterKey {
+    type Ret<'a> = &'a T::Key where T::Key: 'a;
+    fn some<'a>(v: &'a T) -> Option<Self::Ret<'a>> {
+        Some(v.key())
+    }
+}
+
+struct CfgIterVal;
+impl<T: TreeElement> IterConfig<T> for CfgIterVal {
+    type Ret<'a> = &'a T::Value where T::Value: 'a;
+    fn some<'a>(v: &'a T) -> Option<Self::Ret<'a>> {
+        Some(v.val())
+    }
+}
+
+struct DFSCNodeCtx<'g, C: Config> {
+    sptr: Shared<'g, Node<C>>,
+    idx: usize,
+}
+
+struct RawIter<'t, 'g, 'v, T, S, C, I>
+where
+    't: 'v,
+    'g: 'v + 't,
+    I: IterConfig<T>,
+    C: Config,
+{
+    g: &'g Guard,
+    stack: UArray<{ <DefConfig as Config>::BRANCH_MX + 1 }, DFSCNodeCtx<'g, C>>,
+    _m: PhantomData<(&'v T, C, &'t Tree<T, S, C>, I)>,
+}
+
+impl<'t, 'g, 'v, T, S, C, I> RawIter<'t, 'g, 'v, T, S, C, I>
+where
+    't: 'v,
+    'g: 'v + 't,
+    I: IterConfig<T>,
+    C: Config,
+{
+    pub(super) fn new(tree: &'t Tree<T, S, C>, g: &'g Guard) -> Self {
+        let mut stack = UArray::new();
+        let sptr = tree.root.ld_acq(g);
+        stack.push(DFSCNodeCtx { sptr, idx: 0 });
+        Self {
+            g,
+            stack,
+            _m: PhantomData,
+        }
+    }
+    /// depth-first search the tree
+    fn _next(&mut self) -> Option<I::Ret<'v>> {
+        while !self.stack.is_empty() {
+            let l = self.stack.len() - 1;
+            let ref mut current = self.stack[l];
+            let ref node = current.sptr;
+            let flag = super::ldfl(&current.sptr);
+            match flag {
+                _ if node.is_null() => {
+                    self.stack.pop();
+                }
+                flag if super::hf(flag, NodeFlag::DATA) => {
+                    let data = unsafe {
+                        // UNSAFE(@ohsayan): flagck
+                        Tree::<T, S, C>::read_data(current.sptr)
+                    };
+                    if current.idx < data.len() {
+                        let ref ret = data[current.idx];
+                        current.idx += 1;
+                        return I::some(ret);
+                    } else {
+                        self.stack.pop();
+                    }
+                }
+                _ if current.idx < C::MAX_TREE_HEIGHT => {
+                    let this_node = unsafe {
+                        // UNSAFE(@ohsayan): guard
+                        node.deref()
+                    };
+                    let sptr = this_node.branch[current.idx].ld_acq(&self.g);
+                    current.idx += 1;
+                    self.stack.push(DFSCNodeCtx { sptr, idx: 0 });
+                }
+                _ => {
+                    self.stack.pop();
+                }
+            }
+        }
+        None
+    }
+}
+
+impl<'t, 'g, 'v, T, S, C, I> Iterator for RawIter<'t, 'g, 'v, T, S, C, I>
+where
+    't: 'v,
+    'g: 'v + 't,
+    I: IterConfig<T>,
+    C: Config,
+{
+    type Item = I::Ret<'v>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self._next()
+    }
+}
diff --git a/server/src/engine/idx/mtchm/meta.rs b/server/src/engine/idx/mtchm/meta.rs
new file mode 100644
index 00000000..913b0db5
--- /dev/null
+++ b/server/src/engine/idx/mtchm/meta.rs
@@ -0,0 +1,123 @@
+/*
+ * Created on Thu Jan 26 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::super::{super::mem::VInline, AsKeyClone};
+use std::{hash::BuildHasher, sync::Arc};
+
+const LNODE_STACK: usize = 2;
+pub type DefConfig = Config2B;
+pub type LNode<T> = VInline<LNODE_STACK, T>;
+
+pub trait PreConfig: Sized + 'static {
+    const BITS: u32;
+}
+
+pub trait Config: PreConfig {
+    const BRANCH_MX: usize = <Self as PreConfig>::BITS as _;
+    const BRANCH_LG: usize = {
+        let mut index = <Self as Config>::BRANCH_MX;
+        let mut log = 0usize;
+        while {
+            index >>= 1;
+            index != 0
+        } {
+            log += 1;
+        }
+        log
+    };
+    const HASH_MASK: u64 = (<Self as PreConfig>::BITS - 1) as _;
+    const MAX_TREE_HEIGHT_UB: usize = 0x40;
+    const MAX_TREE_HEIGHT: usize =
+        <Self as Config>::MAX_TREE_HEIGHT_UB / <Self as Config>::BRANCH_LG;
+    const LEVEL_ZERO: usize = 0;
+}
+
+impl<T: PreConfig> Config for T {}
+
+macro_rules! impl_config {
+    ($($vis:vis $name:ident = $ty:ty),*) => {
+        $($vis struct $name; impl $crate::engine::idx::mtchm::meta::PreConfig for $name { const BITS: u32 = <$ty>::BITS; })*
+    }
+}
+
+impl_config!(pub Config2B = u16);
+
+pub trait Key: AsKeyClone + 'static {}
+impl<T> Key for T where T: AsKeyClone + 'static {}
+pub trait Value: Clone + 'static {}
+impl<T> Value for T where T: Clone + 'static {}
+pub trait AsHasher: BuildHasher + Default {}
+impl<T> AsHasher for T where T: BuildHasher + Default {}
+
+pub trait TreeElement: Clone + 'static {
+    type Key: Key;
+    type Value: Value;
+    fn key(&self) -> &Self::Key;
+    fn val(&self) -> &Self::Value;
+}
+
+impl<K: Key, V: Value> TreeElement for (K, V) {
+    type Key = K;
+    type Value = V;
+    #[inline(always)]
+    fn key(&self) -> &K {
+        &self.0
+    }
+    #[inline(always)]
+    fn val(&self) -> &V {
+        &self.1
+    }
+}
+
+impl<K: Key, V: Value> TreeElement for Arc<(K, V)> {
+    type Key = K;
+    type Value = V;
+    #[inline(always)]
+    fn key(&self) -> &K {
+        &self.0
+    }
+    #[inline(always)]
+    fn val(&self) -> &V {
+        &self.1
+    }
+}
+
+flags! {
+    pub struct NodeFlag: usize {
+        PENDING_DELETE = 0b01,
+        DATA = 0b10,
+    }
+}
+
+flags! {
+    #[derive(PartialEq, Eq)]
+    pub struct CompressState: u8 {
+        NULL = 0b00,
+        SNODE = 0b01,
+        CASFAIL = 0b10,
+        RESTORED = 0b11,
+    }
+}
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
new file mode 100644
index 00000000..b4dc70ec
--- /dev/null
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -0,0 +1,658 @@
+/*
+ * Created on Thu Jan 26 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod access;
+mod iter;
+mod meta;
+
+use self::{
+    access::{ReadMode, WriteMode},
+    iter::{IterKV, IterKey, IterVal},
+    meta::{AsHasher, CompressState, Config, DefConfig, LNode, NodeFlag, TreeElement},
+};
+use super::{
+    super::{
+        mem::UArray,
+        sync::atm::{self, cpin, upin, Atomic, Guard, Owned, Shared, ORD_ACQ, ORD_ACR, ORD_RLX},
+    },
+    AsKey,
+};
+use crossbeam_epoch::CompareExchangeError;
+use std::{borrow::Borrow, hash::Hasher, marker::PhantomData, mem, sync::atomic::AtomicUsize};
+
+/*
+    HACK(@ohsayan): Until https://github.com/rust-lang/rust/issues/76560 is stabilized which is likely to take a while,
+    we need to settle for trait objects
+*/
+
+pub struct Node<C: Config> {
+    branch: [Atomic<Self>; <DefConfig as Config>::BRANCH_MX],
+}
+
+impl<C: Config> Node<C> {
+    const NULL: Atomic<Self> = Atomic::null();
+    const NULL_BRANCH: [Atomic<Self>; <DefConfig as Config>::BRANCH_MX] =
+        [Self::NULL; <DefConfig as Config>::BRANCH_MX];
+    #[inline(always)]
+    const fn null() -> Self {
+        Self {
+            branch: Self::NULL_BRANCH,
+        }
+    }
+}
+
+#[inline(always)]
+fn gc(g: &Guard) {
+    g.flush();
+}
+
+#[inline(always)]
+fn ldfl<C: Config>(c: &Shared<Node<C>>) -> usize {
+    c.tag()
+}
+
+#[inline(always)]
+const fn hf(c: usize, f: NodeFlag) -> bool {
+    (c & f.d()) == f.d()
+}
+
+#[inline(always)]
+const fn cf(c: usize, r: NodeFlag) -> usize {
+    c & !r.d()
+}
+
+trait CTFlagAlign {
+    const FL_A: bool;
+    const FL_B: bool;
+    const FLCK_A: () = assert!(Self::FL_A & Self::FL_B);
+    const FLCK: () = Self::FLCK_A;
+}
+
+impl<T, S, C: Config> CTFlagAlign for Tree<T, S, C> {
+    const FL_A: bool = atm::ensure_flag_align::<LNode<T>, { NodeFlag::bits() }>();
+    const FL_B: bool = atm::ensure_flag_align::<Node<C>, { NodeFlag::bits() }>();
+}
+
+pub struct Tree<T, S, C: Config> {
+    root: Atomic<Node<C>>,
+    h: S,
+    l: AtomicUsize,
+    _m: PhantomData<T>,
+}
+
+impl<T, S, C: Config> Tree<T, S, C> {
+    #[inline(always)]
+    const fn _new(h: S) -> Self {
+        let _ = Self::FLCK;
+        Self {
+            root: Atomic::null(),
+            h,
+            l: AtomicUsize::new(0),
+            _m: PhantomData,
+        }
+    }
+    #[inline(always)]
+    fn len(&self) -> usize {
+        self.l.load(ORD_RLX)
+    }
+    #[inline(always)]
+    fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+    #[inline(always)]
+    pub const fn with_hasher(h: S) -> Self {
+        Self::_new(h)
+    }
+}
+
+impl<T, S: AsHasher, C: Config> Tree<T, S, C> {
+    #[inline(always)]
+    fn new() -> Self {
+        Self::_new(S::default())
+    }
+}
+
+impl<T, S: AsHasher, C: Config> Tree<T, S, C> {
+    fn hash<Q>(&self, k: &Q) -> u64
+    where
+        Q: ?Sized + AsKey,
+    {
+        let mut state = self.h.build_hasher();
+        k.hash(&mut state);
+        state.finish()
+    }
+}
+
+// iter
+impl<T: TreeElement, S, C: Config> Tree<T, S, C> {
+    fn iter_kv<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterKV<'t, 'g, 'v, T, S, C> {
+        IterKV::new(self, g)
+    }
+    fn iter_key<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterKey<'t, 'g, 'v, T, S, C> {
+        IterKey::new(self, g)
+    }
+    fn iter_val<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterVal<'t, 'g, 'v, T, S, C> {
+        IterVal::new(self, g)
+    }
+}
+
+impl<T: TreeElement, S: AsHasher, C: Config> Tree<T, S, C> {
+    fn insert(&self, elem: T, g: &Guard) -> bool {
+        self._insert::<access::WModeFresh>(elem, g)
+    }
+    fn update(&self, elem: T, g: &Guard) -> bool {
+        self._insert::<access::WModeUpdate>(elem, g)
+    }
+    fn update_return<'g>(&'g self, elem: T, g: &'g Guard) -> Option<&'g T::Value> {
+        self._insert::<access::WModeUpdateRetRef>(elem, g)
+    }
+    fn upsert(&self, elem: T, g: &Guard) {
+        self._insert::<access::WModeUpsert>(elem, g)
+    }
+    fn upsert_return<'g>(&'g self, elem: T, g: &'g Guard) -> Option<&'g T::Value> {
+        self._insert::<access::WModeUpsertRef>(elem, g)
+    }
+    fn _insert<'g, W: WriteMode<T>>(&'g self, elem: T, g: &'g Guard) -> W::Ret<'g> {
+        let hash = self.hash(elem.key());
+        let mut level = C::LEVEL_ZERO;
+        let mut current = &self.root;
+        let mut parent = None;
+        let mut child = None;
+        loop {
+            let node = current.ld_acq(g);
+            match ldfl(&node) {
+                flag if hf(flag, NodeFlag::PENDING_DELETE) => {
+                    /*
+                        FIXME(@ohsayan):
+                        this node is about to be deleted (well, maybe) so we'll attempt a cleanup as well. we might not exactly
+                        need to do this. also this is a potentially expensive thing since we're going all the way back to the root,
+                        we might be able to optimize this with a fixed-size queue.
+                    */
+                    unsafe {
+                        // UNSAFE(@ohsayan): we know that isn't the root and def doesn't have data (that's how the algorithm works)
+                        Self::compress(parent.unwrap(), child.unwrap(), g);
+                    }
+                    level = C::LEVEL_ZERO;
+                    current = &self.root;
+                    parent = None;
+                    child = None;
+                }
+                _ if node.is_null() => {
+                    // this is an empty slot
+                    if W::WMODE == access::WRITEMODE_REFRESH {
+                        // I call that a job well done
+                        return W::nx();
+                    }
+                    if (W::WMODE == access::WRITEMODE_ANY) | (W::WMODE == access::WRITEMODE_FRESH) {
+                        let new = Self::new_data(elem.clone());
+                        match current.cx_rel(node, new, g) {
+                            Ok(_) => {
+                                // we're done here
+                                self.incr_len();
+                                return W::nx();
+                            }
+                            Err(CompareExchangeError { new, .. }) => unsafe {
+                                /*
+                                    UNSAFE(@ohsayan): so we attempted to CAS it but the CAS failed. in that case, destroy the
+                                    lnode we created. We never published the value so no other thread has watched, making this
+                                    safe
+                                */
+                                Self::ldrop(new.into_shared(g));
+                            },
+                        }
+                    }
+                }
+                flag if hf(flag, NodeFlag::DATA) => {
+                    // so we have an lnode. well maybe an snode
+                    let data = unsafe {
+                        // UNSAFE(@ohsayan): flagck
+                        Self::read_data(node)
+                    };
+                    debug_assert!(!data.is_empty(), "logic,empty node not compressed");
+                    if data[0].key() != elem.key() && level < C::MAX_TREE_HEIGHT_UB {
+                        /*
+                            so this is a collision and since we haven't reached the max height, we should always
+                            create a new branch so let's do that
+                        */
+                        debug_assert_eq!(data.len(), 1, "logic,lnode before height ub");
+                        if W::WMODE == access::WRITEMODE_REFRESH {
+                            // another job well done; an snode with the wrong key; so basically it's missing
+                            return W::nx();
+                        }
+                        let next_chunk = (self.hash(data[0].key()) >> level) & C::HASH_MASK;
+                        let mut new_branch = Node::null();
+                        // stick this one in
+                        new_branch.branch[next_chunk as usize] = Atomic::from(node);
+                        // we don't care about what happens
+                        let _ = current.cx_rel(node, Owned::new(new_branch), g);
+                    } else {
+                        /*
+                            in this case we either have the same key or we found an lnode. resolve any conflicts and attempt
+                            to update
+                        */
+                        let p = data.iter().position(|e| e.key() == elem.key());
+                        match p {
+                            Some(v) if W::WMODE == access::WRITEMODE_FRESH => {
+                                return W::ex(&data[v])
+                            }
+                            Some(i)
+                                if W::WMODE == access::WRITEMODE_REFRESH
+                                    || W::WMODE == access::WRITEMODE_ANY =>
+                            {
+                                // update the entry and create a new node
+                                let mut new_ln = data.clone();
+                                new_ln[i] = elem.clone();
+                                match current.cx_rel(
+                                    node,
+                                    Self::new_lnode(new_ln).into_shared(g),
+                                    g,
+                                ) {
+                                    Ok(_) => {
+                                        unsafe {
+                                            /*
+                                                UNSAFE(@ohsayan): swapped out, and we'll be the last thread to see this once the epoch proceeds
+                                                sufficiently
+                                            */
+                                            g.defer_destroy(Shared::<LNode<T>>::from(
+                                                node.as_raw() as *const LNode<_>
+                                            ))
+                                        }
+                                        return W::ex(&data[i]);
+                                    }
+                                    Err(CompareExchangeError { new, .. }) => {
+                                        // failed to swap it in
+                                        unsafe {
+                                            // UNSAFE(@ohsayan): Failed to swap this in, and no one else saw it (well)
+                                            Self::ldrop(new)
+                                        }
+                                    }
+                                }
+                            }
+                            None if W::WMODE == access::WRITEMODE_ANY => {
+                                // no funk here
+                                let new_node = Self::new_data(elem.clone());
+                                match current.cx_rel(node, new_node.into_shared(g), g) {
+                                    Ok(_) => {
+                                        // swapped out
+                                        unsafe {
+                                            // UNSAFE(@ohsayan): last thread to see this (well, sorta)
+                                            g.defer_destroy(Shared::<LNode<T>>::from(
+                                                node.as_raw() as *const LNode<_>,
+                                            ));
+                                        }
+                                        self.incr_len();
+                                        return W::nx();
+                                    }
+                                    Err(CompareExchangeError { new, .. }) => {
+                                        // failed to swap it
+                                        unsafe {
+                                            // UNSAFE(@ohsayan): never published this, so we're the last one
+                                            Self::ldrop(new)
+                                        }
+                                    }
+                                }
+                            }
+                            None if W::WMODE == access::WRITEMODE_REFRESH => return W::nx(),
+                            _ => {
+                                unreachable!("logic,W::WMODE mismatch: `{}`", W::WMODE);
+                            }
+                        }
+                    }
+                }
+                _ => {
+                    // branch
+                    let nxidx = (hash >> level) & C::HASH_MASK;
+                    level += C::BRANCH_LG;
+                    parent = Some(current);
+                    child = Some(node);
+                    current = &unsafe { node.deref() }.branch[nxidx as usize];
+                }
+            }
+        }
+    }
+    fn contains_key<'g, Q, R: ReadMode<T>>(&'g self, k: &Q, g: &'g Guard) -> bool
+    where
+        T::Key: Borrow<Q>,
+        Q: AsKey + ?Sized,
+    {
+        self._lookup::<Q, access::RModeExists>(k, g)
+    }
+    fn get<'g, Q, R: ReadMode<T>>(&'g self, k: &Q, g: &'g Guard) -> Option<&'g T::Value>
+    where
+        T::Key: Borrow<Q>,
+        Q: AsKey + ?Sized,
+    {
+        self._lookup::<Q, access::RModeRef>(k, g)
+    }
+    fn _lookup<'g, Q, R: ReadMode<T>>(&'g self, k: &Q, g: &'g Guard) -> R::Ret<'g>
+    where
+        T::Key: Borrow<Q>,
+        Q: AsKey + ?Sized,
+    {
+        let mut hash = self.hash(k);
+        let mut current = &self.root;
+        loop {
+            let node = current.ld_acq(g);
+            match ldfl(&node) {
+                _ if node.is_null() => {
+                    // honestly, if this ran on the root I'm going to die laughing (@ohsayan)
+                    return R::nx();
+                }
+                flag if hf(flag, NodeFlag::DATA) => {
+                    let mut ret = R::nx();
+                    return unsafe {
+                        // UNSAFE(@ohsayan): checked flag + nullck
+                        Self::read_data(node).iter().find_map(|e| {
+                            e.key().borrow().eq(k).then_some({
+                                ret = R::ex(e);
+                                Some(())
+                            })
+                        });
+                        ret
+                    };
+                }
+                _ => {
+                    // branch
+                    current = &unsafe { node.deref() }.branch[(hash & C::HASH_MASK) as usize];
+                    hash >>= C::BRANCH_LG;
+                }
+            }
+        }
+    }
+    fn remove<'g, Q>(&'g self, k: &Q, g: &'g Guard) -> bool
+    where
+        T::Key: Borrow<Q>,
+        Q: AsKey + ?Sized,
+    {
+        self._remove::<Q, access::WModeDelete>(k, g)
+    }
+    fn remove_return<'g, Q>(&'g self, k: &Q, g: &'g Guard) -> Option<&'g T::Value>
+    where
+        T::Key: Borrow<Q>,
+        Q: AsKey + ?Sized,
+    {
+        self._remove::<Q, access::WModeDeleteRef>(k, g)
+    }
+    fn _remove<'g, Q, W: WriteMode<T>>(&'g self, k: &Q, g: &'g Guard) -> W::Ret<'g>
+    where
+        T::Key: Borrow<Q>,
+        Q: AsKey + ?Sized,
+    {
+        let hash = self.hash(k);
+        let mut current = &self.root;
+        let mut level = C::LEVEL_ZERO;
+        let mut levels = UArray::<{ <DefConfig as Config>::BRANCH_MX }, _>::new();
+        'retry: loop {
+            let node = current.ld_acq(g);
+            match ldfl(&node) {
+                _ if node.is_null() => {
+                    // lol
+                    return W::nx();
+                }
+                flag if hf(flag, NodeFlag::PENDING_DELETE) => {
+                    let (p, c) = levels.pop().unwrap();
+                    unsafe {
+                        /*
+                            we hit a node that might be deleted, we aren't allowed to change it, so we'll attempt a
+                            compression as well. same thing here as the other routines....can we do anything to avoid
+                            the expensive root traversal?
+                        */
+                        Self::compress(p, c, g);
+                    }
+                    levels.clear();
+                    level = C::LEVEL_ZERO;
+                    current = &self.root;
+                }
+                flag if hf(flag, NodeFlag::DATA) => {
+                    let data = unsafe {
+                        // UNSAFE(@ohsayan): flagck
+                        Self::read_data(node)
+                    };
+                    let mut ret = W::nx();
+                    // this node shouldn't be empty
+                    debug_assert!(!data.is_empty(), "logic,empty node not collected");
+                    // build new lnode
+                    let r: LNode<T> = data
+                        .iter()
+                        .filter_map(|this_elem| {
+                            if this_elem.key().borrow() == k {
+                                ret = W::ex(this_elem);
+                                None
+                            } else {
+                                Some(this_elem.clone())
+                            }
+                        })
+                        .collect();
+                    let replace = if r.is_empty() {
+                        // don't create dead nodes
+                        Shared::null()
+                    } else {
+                        Self::new_lnode(r).into_shared(g)
+                    };
+                    match current.cx_rel(node, replace, g) {
+                        Ok(_) => {
+                            // swapped it out
+                            unsafe {
+                                // UNSAFE(@ohsayan): flagck
+                                g.defer_destroy(Shared::<LNode<T>>::from(
+                                    node.as_raw() as *const LNode<_>
+                                ));
+                            }
+                        }
+                        Err(CompareExchangeError { new, .. }) if !new.is_null() => {
+                            // failed to swap it in, and it had some data
+                            unsafe {
+                                // UNSAFE(@ohsayan): Never published it, all ours
+                                g.defer_destroy(Shared::<LNode<T>>::from(
+                                    new.as_raw() as *const LNode<_>
+                                ));
+                            }
+                            continue 'retry;
+                        }
+                        Err(_) => continue 'retry,
+                    }
+                    // attempt compressions
+                    for (p, c) in levels.into_iter().rev() {
+                        let live_nodes = unsafe {
+                            // UNSAFE(@ohsayan): guard
+                            c.deref()
+                        }
+                        .branch
+                        .iter()
+                        .filter(|n| !n.ld_rlx(g).is_null())
+                        .count();
+                        if live_nodes > 1 {
+                            break;
+                        }
+                        if unsafe {
+                            // UNSAFE(@ohsayan): we know for a fact that we only have sensible levels
+                            Self::compress(p, c, g)
+                        } == CompressState::RESTORED
+                        {
+                            // simply restored the earlier state, so let's stop
+                            break;
+                        }
+                    }
+                    gc(g);
+                    return ret;
+                }
+                _ => {
+                    // branch
+                    levels.push((current, node));
+                    let nxidx = (hash >> level) & C::HASH_MASK;
+                    level += C::BRANCH_LG;
+                    current = &unsafe { node.deref() }.branch[nxidx as usize];
+                }
+            }
+        }
+    }
+}
+
+// low-level methods
+impl<T, S, C: Config> Tree<T, S, C> {
+    // hilarious enough but true, l doesn't affect safety but only creates an incorrect state
+    fn decr_len(&self) {
+        self.l.fetch_sub(1, ORD_ACQ);
+    }
+    fn incr_len(&self) {
+        self.l.fetch_add(1, ORD_ACQ);
+    }
+    #[inline(always)]
+    fn new_lnode(node: LNode<T>) -> Owned<Node<C>> {
+        unsafe {
+            Owned::<Node<_>>::from_raw(Box::into_raw(Box::new(node)) as *mut Node<_>)
+                .with_tag(NodeFlag::DATA.d())
+        }
+    }
+    /// Returns a new inner node, in the form of a data probe leaf
+    /// ☢ WARNING ☢: Do not drop this naively for god's sake
+    #[inline(always)]
+    fn new_data(data: T) -> Owned<Node<C>> {
+        let mut d = LNode::new();
+        unsafe {
+            // UNSAFE(@ohsayan): empty arr
+            d.push_unchecked(data)
+        };
+        Self::new_lnode(d)
+    }
+    unsafe fn read_data<'g>(d: Shared<'g, Node<C>>) -> &'g LNode<T> {
+        debug_assert!(hf(ldfl(&d), NodeFlag::DATA));
+        (d.as_raw() as *const LNode<_>)
+            .as_ref()
+            .expect("logic,nullptr in lnode")
+    }
+    /// SAFETY: Ensure you have some actual data and not random garbage
+    #[inline(always)]
+    unsafe fn ldrop(leaf: Shared<Node<C>>) {
+        debug_assert!(hf(ldfl(&leaf), NodeFlag::DATA));
+        drop(Owned::<LNode<T>>::from_raw(leaf.as_raw() as *mut _))
+    }
+    unsafe fn rdrop(n: &Atomic<Node<C>>) {
+        let g = upin();
+        let node = n.ld_acq(g);
+        match ldfl(&node) {
+            _ if node.is_null() => {}
+            flag if hf(flag, NodeFlag::DATA) => Self::ldrop(node),
+            _ => {
+                // a branch
+                let this_branch = node.into_owned();
+                for child in &this_branch.branch {
+                    Self::rdrop(child)
+                }
+                drop(this_branch);
+            }
+        }
+    }
+    unsafe fn compress<'g>(
+        parent: &Atomic<Node<C>>,
+        child: Shared<'g, Node<C>>,
+        g: &'g Guard,
+    ) -> CompressState {
+        /*
+            We look at the child's children and determine whether we can clean the child up. Although the amount of
+            memory we can save is not something very signficant but it becomes important with larger cardinalities
+        */
+        debug_assert!(!hf(ldfl(&child), NodeFlag::DATA), "logic,compress lnode");
+        debug_assert_eq!(ldfl(&child), 0, "logic,compress pending delete node");
+        let branch = child.deref();
+        let mut continue_compress = true;
+        let mut last_leaf = None;
+        let mut new_child = Node::null();
+        let mut cnt = 0_usize;
+
+        let mut i = 0;
+        while i < C::BRANCH_MX {
+            let ref child_ref = branch.branch[i];
+            let this_child = child_ref.fetch_or(NodeFlag::PENDING_DELETE.d(), ORD_ACR, g);
+            let this_child = this_child.with_tag(cf(ldfl(&this_child), NodeFlag::PENDING_DELETE));
+            match ldfl(&this_child) {
+                // lol, dangling child
+                _ if this_child.is_null() => {}
+                // some data in here
+                flag if hf(flag, NodeFlag::DATA) => {
+                    last_leaf = Some(this_child);
+                    cnt += Self::read_data(this_child).len();
+                }
+                // branch
+                _ => {
+                    continue_compress = false;
+                    cnt += 1;
+                }
+            }
+            new_child.branch[i] = Atomic::from(this_child);
+            i += 1;
+        }
+
+        let insert;
+        let ret;
+        let mut drop = None;
+
+        match last_leaf {
+            Some(node) if continue_compress && cnt == 1 => {
+                // snode
+                insert = node;
+                ret = CompressState::SNODE;
+            }
+            None if cnt == 0 => {
+                // a dangling branch
+                insert = Shared::null();
+                ret = CompressState::NULL;
+            }
+            _ => {
+                // we can't compress this since we have a lot of children
+                let new = Owned::new(new_child).into_shared(g);
+                insert = new;
+                drop = Some(new);
+                ret = CompressState::RESTORED;
+            }
+        }
+
+        // all logic done; let's see what fate the CAS brings us
+        match parent.cx_rel(child, insert, g) {
+            Ok(_) => {
+                unsafe {
+                    // UNSAFE(@ohsayan): We're the thread in the last epoch who's seeing this; so, we're good
+                    g.defer_destroy(child);
+                }
+                ret
+            }
+            Err(_) => {
+                mem::drop(drop.map(|n| Shared::into_owned(n)));
+                CompressState::CASFAIL
+            }
+        }
+    }
+}
+
+impl<T, S, C: Config> Drop for Tree<T, S, C> {
+    fn drop(&mut self) {
+        unsafe {
+            // UNSAFE(@ohsayan): sole live owner
+            Self::rdrop(&self.root);
+        }
+        gc(&cpin())
+    }
+}
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index c710e8af..a65c7f84 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -34,6 +34,7 @@ macro_rules! extract {
     };
 }
 
+#[cfg(test)]
 macro_rules! multi_assert_eq {
     ($($lhs:expr),* => $rhs:expr) => {
         $(assert_eq!($lhs, $rhs);)*
@@ -49,6 +50,7 @@ macro_rules! enum_impls {
     };
 }
 
+#[allow(unused_macros)]
 macro_rules! assertions {
     ($($assert:expr),*$(,)?) => {$(const _:()=::core::assert!($assert);)*}
 }
@@ -82,6 +84,7 @@ macro_rules! flags {
     );
 }
 
+#[allow(unused_macros)]
 macro_rules! union {
     ($(#[$attr:meta])* $vis:vis union $name:ident $tail:tt) => (union!(@parse [$(#[$attr])* $vis union $name] [] $tail););
     ($(#[$attr:meta])* $vis:vis union $name:ident<$($lt:lifetime),*> $tail:tt) => (union!(@parse [$(#[$attr])* $vis union $name<$($lt),*>] [] $tail););
diff --git a/server/src/engine/mem/vinline.rs b/server/src/engine/mem/vinline.rs
index e3f84c54..663ddbef 100644
--- a/server/src/engine/mem/vinline.rs
+++ b/server/src/engine/mem/vinline.rs
@@ -71,7 +71,6 @@ impl<const N: usize, T> VInline<N, T> {
             // UNSAFE(@ohsayan): grow allocated the cap we needed
             self.push_unchecked(v);
         }
-        self.l += 1;
     }
     #[inline(always)]
     pub fn clear(&mut self) {
@@ -182,8 +181,9 @@ impl<const N: usize, T> VInline<N, T> {
             p as *mut T
         }
     }
-    unsafe fn push_unchecked(&mut self, v: T) {
+    pub unsafe fn push_unchecked(&mut self, v: T) {
         self._as_mut_ptr().add(self.l).write(v);
+        self.l += 1;
     }
     pub fn optimize_capacity(&mut self) {
         if self.on_stack() || self.len() == self.capacity() {
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 9d5855f5..b367bf89 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -25,12 +25,11 @@
 */
 
 #![allow(dead_code)]
-#![allow(unused_macros)]
 
 #[macro_use]
 mod macros;
 mod core;
 mod idx;
+mod mem;
 mod ql;
 mod sync;
-mod mem;
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 10b9ff99..4b60718a 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -282,6 +282,7 @@ macro_rules! dict {
     }};
 }
 
+#[cfg(test)]
 macro_rules! nullable_dict {
     () => {
         dict! {}
@@ -295,6 +296,7 @@ macro_rules! nullable_dict {
     };
 }
 
+#[cfg(test)]
 macro_rules! dict_nullable {
     () => {
         <::std::collections::HashMap<_, _> as ::core::default::Default>::default()
@@ -306,6 +308,7 @@ macro_rules! dict_nullable {
     }};
 }
 
+#[cfg(test)]
 macro_rules! set {
     () => {
         <::std::collections::HashSet<_> as ::core::default::Default>::default()
@@ -317,14 +320,17 @@ macro_rules! set {
     }};
 }
 
+#[cfg(test)]
 macro_rules! into_array {
     ($($e:expr),* $(,)?) => { [$($e.into()),*] };
 }
 
+#[cfg(test)]
 macro_rules! into_array_nullable {
     ($($e:expr),* $(,)?) => { [$($crate::engine::ql::tests::nullable_datatype($e)),*] };
 }
 
+#[allow(unused_macros)]
 macro_rules! statictbl {
     ($name:ident: $kind:ty => [$($expr:expr),*]) => {{
         const LEN: usize = {let mut i = 0;$(let _ = $expr; i += 1;)*i};

From 48ada558ad1ae4a2970f9b226437ea5ab018295e Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 28 Jan 2023 02:18:36 -0800
Subject: [PATCH 102/310] Clarify idx trait definitions

MT idx impls would have a fairly different interface when compared to ST
equivalents.
---
 server/src/engine/idx/mod.rs       | 67 +++++++++++++++++++-----------
 server/src/engine/idx/mtchm/imp.rs | 56 +++++++++++++++++++++++++
 server/src/engine/idx/mtchm/mod.rs |  1 +
 server/src/engine/idx/stdhm.rs     | 56 ++++++++++++-------------
 server/src/engine/idx/stord.rs     | 56 ++++++++++++-------------
 5 files changed, 156 insertions(+), 80 deletions(-)
 create mode 100644 server/src/engine/idx/mtchm/imp.rs

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 84d482fe..27bab196 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -24,9 +24,9 @@
  *
 */
 
+mod mtchm;
 mod stdhm;
 mod stord;
-mod mtchm;
 #[cfg(test)]
 mod tests;
 
@@ -96,22 +96,6 @@ pub trait IndexBaseSpec<K, V>: Sized {
     #[cfg(debug_assertions)]
     /// A type representing debug metrics
     type Metrics;
-    /// An iterator over the keys and values
-    type IterKV<'a>: Iterator<Item = (&'a K, &'a V)>
-    where
-        Self: 'a,
-        K: 'a,
-        V: 'a;
-    /// An iterator over the keys
-    type IterKey<'a>: Iterator<Item = &'a K>
-    where
-        Self: 'a,
-        K: 'a;
-    /// An iterator over the values
-    type IterValue<'a>: Iterator<Item = &'a V>
-    where
-        Self: 'a,
-        V: 'a;
     // init
     /// Initialize an empty instance of the index
     fn idx_init() -> Self;
@@ -126,13 +110,6 @@ pub trait IndexBaseSpec<K, V>: Sized {
         }
         Self::idx_init()
     }
-    // iter
-    /// Returns an iterator over a tuple of keys and values
-    fn idx_iter_kv<'a>(&'a self) -> Self::IterKV<'a>;
-    /// Returns an iterator over the keys
-    fn idx_iter_key<'a>(&'a self) -> Self::IterKey<'a>;
-    /// Returns an iterator over the values
-    fn idx_iter_value<'a>(&'a self) -> Self::IterValue<'a>;
     #[cfg(debug_assertions)]
     /// Returns a reference to the index metrics
     fn idx_metrics(&self) -> &Self::Metrics;
@@ -140,6 +117,25 @@ pub trait IndexBaseSpec<K, V>: Sized {
 
 /// An unordered MTIndex
 pub trait MTIndex<K, V>: IndexBaseSpec<K, V> {
+    type IterKV<'t, 'g, 'v>: Iterator<Item = (&'v K, &'v V)>
+    where
+        'g: 't + 'v,
+        't: 'v,
+        K: 'v,
+        V: 'v,
+        Self: 't;
+    type IterKey<'t, 'g, 'v>: Iterator<Item = &'v K>
+    where
+        'g: 't + 'v,
+        't: 'v,
+        K: 'v,
+        Self: 't;
+    type IterVal<'t, 'g, 'v>: Iterator<Item = &'v V>
+    where
+        'g: 't + 'v,
+        't: 'v,
+        V: 'v,
+        Self: 't;
     /// Attempts to compact the backing storage
     fn mt_compact(&self) {}
     /// Clears all the entries in the MTIndex
@@ -200,6 +196,22 @@ pub trait MTIndex<K, V>: IndexBaseSpec<K, V> {
 
 /// An unordered STIndex
 pub trait STIndex<K, V>: IndexBaseSpec<K, V> {
+    /// An iterator over the keys and values
+    type IterKV<'a>: Iterator<Item = (&'a K, &'a V)>
+    where
+        Self: 'a,
+        K: 'a,
+        V: 'a;
+    /// An iterator over the keys
+    type IterKey<'a>: Iterator<Item = &'a K>
+    where
+        Self: 'a,
+        K: 'a;
+    /// An iterator over the values
+    type IterValue<'a>: Iterator<Item = &'a V>
+    where
+        Self: 'a,
+        V: 'a;
     /// Attempts to compact the backing storage
     fn st_compact(&mut self) {}
     /// Clears all the entries in the STIndex
@@ -256,6 +268,13 @@ pub trait STIndex<K, V>: IndexBaseSpec<K, V> {
     where
         K: AsKeyClone + Borrow<Q>,
         Q: ?Sized + AsKey;
+    // iter
+    /// Returns an iterator over a tuple of keys and values
+    fn st_iter_kv<'a>(&'a self) -> Self::IterKV<'a>;
+    /// Returns an iterator over the keys
+    fn st_iter_key<'a>(&'a self) -> Self::IterKey<'a>;
+    /// Returns an iterator over the values
+    fn st_iter_value<'a>(&'a self) -> Self::IterValue<'a>;
 }
 
 pub trait STIndexSeq<K, V>: STIndex<K, V> {
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
new file mode 100644
index 00000000..e290eb98
--- /dev/null
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -0,0 +1,56 @@
+/*
+ * Created on Sat Jan 28 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::{
+    super::{DummyMetrics, IndexBaseSpec},
+    meta::{AsHasher, Config},
+    Tree,
+};
+use std::sync::Arc;
+
+pub type MTArc<K, V, S, C> = Tree<Arc<(K, V)>, S, C>;
+
+impl<K, V, S, C> IndexBaseSpec<K, V> for MTArc<K, V, S, C>
+where
+    C: Config,
+    S: AsHasher,
+{
+    const PREALLOC: bool = false;
+
+    type Metrics = DummyMetrics;
+
+    fn idx_init() -> Self {
+        MTArc::new()
+    }
+
+    fn idx_init_with(s: Self) -> Self {
+        s
+    }
+
+    fn idx_metrics(&self) -> &Self::Metrics {
+        &DummyMetrics
+    }
+}
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index b4dc70ec..c0a00dd4 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -27,6 +27,7 @@
 mod access;
 mod iter;
 mod meta;
+mod imp;
 
 use self::{
     access::{ReadMode, WriteMode},
diff --git a/server/src/engine/idx/stdhm.rs b/server/src/engine/idx/stdhm.rs
index 436bdf7c..72cee783 100644
--- a/server/src/engine/idx/stdhm.rs
+++ b/server/src/engine/idx/stdhm.rs
@@ -46,22 +46,6 @@ where
     #[cfg(debug_assertions)]
     type Metrics = DummyMetrics;
 
-    type IterKV<'a> = StdMapIterKV<'a, K, V>
-    where
-        Self: 'a,
-        K: 'a,
-        V: 'a;
-
-    type IterKey<'a> = StdMapIterKey<'a, K, V>
-    where
-        Self: 'a,
-        K: 'a;
-
-    type IterValue<'a> = StdMapIterVal<'a, K, V>
-    where
-        Self: 'a,
-        V: 'a;
-
     fn idx_init() -> Self {
         StdMap::with_hasher(S::default())
     }
@@ -74,18 +58,6 @@ where
         Self::with_capacity_and_hasher(cap, S::default())
     }
 
-    fn idx_iter_kv<'a>(&'a self) -> Self::IterKV<'a> {
-        self.iter()
-    }
-
-    fn idx_iter_key<'a>(&'a self) -> Self::IterKey<'a> {
-        self.keys()
-    }
-
-    fn idx_iter_value<'a>(&'a self) -> Self::IterValue<'a> {
-        self.values()
-    }
-
     #[cfg(debug_assertions)]
     fn idx_metrics(&self) -> &Self::Metrics {
         &DummyMetrics
@@ -98,6 +70,22 @@ where
     V: AsValue,
     S: BuildHasher + Default,
 {
+    type IterKV<'a> = StdMapIterKV<'a, K, V>
+    where
+        Self: 'a,
+        K: 'a,
+        V: 'a;
+
+    type IterKey<'a> = StdMapIterKey<'a, K, V>
+    where
+        Self: 'a,
+        K: 'a;
+
+    type IterValue<'a> = StdMapIterVal<'a, K, V>
+    where
+        Self: 'a,
+        V: 'a;
+
     fn st_compact(&mut self) {
         self.shrink_to_fit()
     }
@@ -180,4 +168,16 @@ where
     {
         self.remove(key)
     }
+
+    fn st_iter_kv<'a>(&'a self) -> Self::IterKV<'a> {
+        self.iter()
+    }
+
+    fn st_iter_key<'a>(&'a self) -> Self::IterKey<'a> {
+        self.keys()
+    }
+
+    fn st_iter_value<'a>(&'a self) -> Self::IterValue<'a> {
+        self.values()
+    }
 }
diff --git a/server/src/engine/idx/stord.rs b/server/src/engine/idx/stord.rs
index 9c4becb1..ffd2e50e 100644
--- a/server/src/engine/idx/stord.rs
+++ b/server/src/engine/idx/stord.rs
@@ -563,22 +563,6 @@ impl<K, V, S: BuildHasher + Default> IndexBaseSpec<K, V> for IndexSTSeqDll<K, V,
     #[cfg(debug_assertions)]
     type Metrics = IndexSTSeqDllMetrics;
 
-    type IterKV<'a> = IndexSTSeqDllIterUnordKV<'a, K, V>
-    where
-        Self: 'a,
-        K: 'a,
-        V: 'a;
-
-    type IterKey<'a> = IndexSTSeqDllIterUnordKey<'a, K, V>
-    where
-        Self: 'a,
-        K: 'a;
-
-    type IterValue<'a> = IndexSTSeqDllIterUnordValue<'a, K, V>
-    where
-        Self: 'a,
-        V: 'a;
-
     fn idx_init() -> Self {
         Self::with_hasher(S::default())
     }
@@ -591,18 +575,6 @@ impl<K, V, S: BuildHasher + Default> IndexBaseSpec<K, V> for IndexSTSeqDll<K, V,
         Self::with_capacity_and_hasher(cap, S::default())
     }
 
-    fn idx_iter_kv<'a>(&'a self) -> Self::IterKV<'a> {
-        self._iter_unord_kv()
-    }
-
-    fn idx_iter_key<'a>(&'a self) -> Self::IterKey<'a> {
-        self._iter_unord_k()
-    }
-
-    fn idx_iter_value<'a>(&'a self) -> Self::IterValue<'a> {
-        self._iter_unord_v()
-    }
-
     #[cfg(debug_assertions)]
     fn idx_metrics(&self) -> &Self::Metrics {
         &self.metrics
@@ -614,6 +586,22 @@ where
     K: AsKey,
     V: AsValue,
 {
+    type IterKV<'a> = IndexSTSeqDllIterUnordKV<'a, K, V>
+    where
+        Self: 'a,
+        K: 'a,
+        V: 'a;
+
+    type IterKey<'a> = IndexSTSeqDllIterUnordKey<'a, K, V>
+    where
+        Self: 'a,
+        K: 'a;
+
+    type IterValue<'a> = IndexSTSeqDllIterUnordValue<'a, K, V>
+    where
+        Self: 'a,
+        V: 'a;
+
     fn st_compact(&mut self) {
         self.vacuum_full();
     }
@@ -689,6 +677,18 @@ where
     {
         self._remove(key)
     }
+
+    fn st_iter_kv<'a>(&'a self) -> Self::IterKV<'a> {
+        self._iter_unord_kv()
+    }
+
+    fn st_iter_key<'a>(&'a self) -> Self::IterKey<'a> {
+        self._iter_unord_k()
+    }
+
+    fn st_iter_value<'a>(&'a self) -> Self::IterValue<'a> {
+        self._iter_unord_v()
+    }
 }
 
 impl<K, V, S> STIndexSeq<K, V> for IndexSTSeqDll<K, V, S>

From 57c7a6447b9099ef9a454defe8809449bdefa1a9 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 28 Jan 2023 04:18:19 -0800
Subject: [PATCH 103/310] Add mt idx impls

---
 server/src/engine/idx/mod.rs        |  37 +++++----
 server/src/engine/idx/mtchm/imp.rs  | 114 +++++++++++++++++++++++++++-
 server/src/engine/idx/mtchm/iter.rs |  10 ++-
 server/src/engine/idx/mtchm/mod.rs  |  24 ++++--
 4 files changed, 157 insertions(+), 28 deletions(-)

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 27bab196..8bbd00a3 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -30,6 +30,7 @@ mod stord;
 #[cfg(test)]
 mod tests;
 
+use super::sync::atm::Guard;
 use core::{borrow::Borrow, hash::Hash};
 
 // re-exports
@@ -139,59 +140,65 @@ pub trait MTIndex<K, V>: IndexBaseSpec<K, V> {
     /// Attempts to compact the backing storage
     fn mt_compact(&self) {}
     /// Clears all the entries in the MTIndex
-    fn mt_clear(&self);
+    fn mt_clear(&self, g: &Guard);
     // write
     /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
     /// violated
-    fn mt_insert(&self, key: K, val: V) -> bool
+    fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool
     where
         K: AsKeyClone,
         V: AsValue;
     /// Updates or inserts the given value
-    fn mt_upsert(&self, key: K, val: V)
+    fn mt_upsert(&self, key: K, val: V, g: &Guard)
     where
         K: AsKeyClone,
         V: AsValue;
     // read
-    fn mt_contains<Q>(&self, key: &Q) -> bool
+    fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
     where
         K: Borrow<Q> + AsKeyClone,
         Q: ?Sized + AsKey;
     /// Returns a reference to the value corresponding to the key, if it exists
-    fn mt_get<Q>(&self, key: &Q) -> Option<&V>
+    fn mt_get<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
     where
         K: AsKeyClone + Borrow<Q>,
-        Q: ?Sized + AsKey;
+        Q: ?Sized + AsKey,
+        't: 'v,
+        'g: 't + 'v;
     /// Returns a clone of the value corresponding to the key, if it exists
-    fn mt_get_cloned<Q>(&self, key: &Q) -> Option<V>
+    fn mt_get_cloned<Q>(&self, key: &Q, g: &Guard) -> Option<V>
     where
         K: AsKeyClone + Borrow<Q>,
         Q: ?Sized + AsKey,
         V: AsValueClone;
     // update
     /// Returns true if the entry is updated
-    fn mt_update<Q>(&self, key: &Q, val: V) -> bool
+    fn mt_update<Q>(&self, key: K, val: V, g: &Guard) -> bool
     where
-        K: AsKeyClone + Borrow<Q>,
+        K: AsKeyClone,
         V: AsValue,
         Q: ?Sized + AsKey;
     /// Updates the entry and returns the old value, if it exists
-    fn mt_update_return<Q>(&self, key: &Q, val: V) -> Option<V>
+    fn mt_update_return<'t, 'g, 'v, Q>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
     where
-        K: AsKeyClone + Borrow<Q>,
+        K: AsKeyClone,
         V: AsValue,
-        Q: ?Sized + AsKey;
+        Q: ?Sized + AsKey,
+        't: 'v,
+        'g: 't + 'v;
     // delete
     /// Returns true if the entry was deleted
-    fn mt_delete<Q>(&self, key: &Q) -> bool
+    fn mt_delete<Q>(&self, key: &Q, g: &Guard) -> bool
     where
         K: AsKeyClone + Borrow<Q>,
         Q: ?Sized + AsKey;
     /// Removes the entry and returns it, if it exists
-    fn mt_delete_return<Q>(&self, key: &Q) -> Option<V>
+    fn mt_delete_return<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
     where
         K: AsKeyClone + Borrow<Q>,
-        Q: ?Sized + AsKey;
+        Q: ?Sized + AsKey,
+        't: 'v,
+        'g: 't + 'v;
 }
 
 /// An unordered STIndex
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index e290eb98..7fe86d9a 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -25,11 +25,17 @@
 */
 
 use super::{
-    super::{DummyMetrics, IndexBaseSpec},
-    meta::{AsHasher, Config},
+    super::{super::sync::atm::Guard, AsKey, DummyMetrics, IndexBaseSpec, MTIndex},
+    iter::{IterKV, IterKey, IterVal},
+    meta::{AsHasher, Config, Key, Value},
     Tree,
 };
-use std::sync::Arc;
+use std::{borrow::Borrow, sync::Arc};
+
+#[inline(always)]
+fn arc<K, V>(k: K, v: V) -> Arc<(K, V)> {
+    Arc::new((k, v))
+}
 
 pub type MTArc<K, V, S, C> = Tree<Arc<(K, V)>, S, C>;
 
@@ -54,3 +60,105 @@ where
         &DummyMetrics
     }
 }
+
+impl<K, V, S, C> MTIndex<K, V> for MTArc<K, V, S, C>
+where
+    C: Config,
+    S: AsHasher,
+    K: Key,
+    V: Value,
+{
+    type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, (K, V), S, C>
+    where
+        'g: 't + 'v,
+        't: 'v,
+        K: 'v,
+        V: 'v,
+        Self: 't;
+
+    type IterKey<'t, 'g, 'v> = IterKey<'t, 'g, 'v, (K, V), S, C>
+    where
+        'g: 't + 'v,
+        't: 'v,
+        K: 'v,
+        Self: 't;
+
+    type IterVal<'t, 'g, 'v> = IterVal<'t, 'g, 'v, (K, V), S, C>
+    where
+        'g: 't + 'v,
+        't: 'v,
+        V: 'v,
+        Self: 't;
+
+    fn mt_clear(&self, g: &Guard) {
+        self.nontransactional_clear(g)
+    }
+
+    fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool {
+        self.insert(arc(key, val), g)
+    }
+
+    fn mt_upsert(&self, key: K, val: V, g: &Guard) {
+        self.upsert(arc(key, val), g)
+    }
+
+    fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKey,
+    {
+        self.contains_key(key, g)
+    }
+
+    fn mt_get<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKey,
+        't: 'v,
+        'g: 't + 'v,
+    {
+        self.get(key, g)
+    }
+
+    fn mt_get_cloned<Q>(&self, key: &Q, g: &Guard) -> Option<V>
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKey,
+    {
+        self.get(key, g).cloned()
+    }
+
+    fn mt_update<Q>(&self, key: K, val: V, g: &Guard) -> bool
+    where
+        Q: ?Sized + AsKey,
+    {
+        self.update(arc(key, val), g)
+    }
+
+    fn mt_update_return<'t, 'g, 'v, Q>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
+    where
+        Q: ?Sized + AsKey,
+        't: 'v,
+        'g: 't + 'v,
+    {
+        self.update_return(arc(key, val), g)
+    }
+
+    fn mt_delete<Q>(&self, key: &Q, g: &Guard) -> bool
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKey,
+    {
+        self.remove(key, g)
+    }
+
+    fn mt_delete_return<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKey,
+        't: 'v,
+        'g: 't + 'v,
+    {
+        self.remove_return(key, g)
+    }
+}
diff --git a/server/src/engine/idx/mtchm/iter.rs b/server/src/engine/idx/mtchm/iter.rs
index d856011a..a650155f 100644
--- a/server/src/engine/idx/mtchm/iter.rs
+++ b/server/src/engine/idx/mtchm/iter.rs
@@ -39,6 +39,7 @@ where
     't: 'v,
     'g: 'v + 't,
     C: Config,
+    T: TreeElement,
 {
     i: RawIter<'t, 'g, 'v, T, S, C, CfgIterKV>,
 }
@@ -48,6 +49,7 @@ where
     't: 'v,
     'g: 'v + 't,
     C: Config,
+    T: TreeElement,
 {
     pub fn new(t: &'t Tree<T, S, C>, g: &'g Guard) -> Self {
         Self {
@@ -63,7 +65,7 @@ where
     C: Config,
     T: TreeElement,
 {
-    type Item = &'v T;
+    type Item = (&'v T::Key, &'v T::Value);
 
     fn next(&mut self) -> Option<Self::Item> {
         self.i.next()
@@ -154,10 +156,10 @@ trait IterConfig<T> {
 }
 
 struct CfgIterKV;
-impl<T> IterConfig<T> for CfgIterKV {
-    type Ret<'a> = &'a T where T: 'a;
+impl<T: TreeElement> IterConfig<T> for CfgIterKV {
+    type Ret<'a> = (&'a T::Key, &'a T::Value) where T: 'a;
     fn some<'a>(v: &'a T) -> Option<Self::Ret<'a>> {
-        Some(v)
+        Some((v.key(), v.val()))
     }
 }
 
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index c0a00dd4..c9837709 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -25,9 +25,9 @@
 */
 
 mod access;
+mod imp;
 mod iter;
 mod meta;
-mod imp;
 
 use self::{
     access::{ReadMode, WriteMode},
@@ -57,8 +57,12 @@ impl<C: Config> Node<C> {
     const NULL: Atomic<Self> = Atomic::null();
     const NULL_BRANCH: [Atomic<Self>; <DefConfig as Config>::BRANCH_MX] =
         [Self::NULL; <DefConfig as Config>::BRANCH_MX];
+    const _SZ: usize = mem::size_of::<Self>() / mem::size_of::<Atomic<Self>>();
+    const _ALIGN: usize = C::BRANCH_MX / Self::_SZ;
+    const _EQ: () = assert!(Self::_ALIGN == 1);
     #[inline(always)]
     const fn null() -> Self {
+        let _ = Self::_EQ;
         Self {
             branch: Self::NULL_BRANCH,
         }
@@ -161,6 +165,11 @@ impl<T: TreeElement, S, C: Config> Tree<T, S, C> {
 }
 
 impl<T: TreeElement, S: AsHasher, C: Config> Tree<T, S, C> {
+    fn nontransactional_clear(&self, g: &Guard) {
+        self.iter_key(g).for_each(|k| {
+            let _ = self.remove(k, g);
+        });
+    }
     fn insert(&self, elem: T, g: &Guard) -> bool {
         self._insert::<access::WModeFresh>(elem, g)
     }
@@ -334,14 +343,14 @@ impl<T: TreeElement, S: AsHasher, C: Config> Tree<T, S, C> {
             }
         }
     }
-    fn contains_key<'g, Q, R: ReadMode<T>>(&'g self, k: &Q, g: &'g Guard) -> bool
+    fn contains_key<'g, Q>(&'g self, k: &Q, g: &'g Guard) -> bool
     where
         T::Key: Borrow<Q>,
         Q: AsKey + ?Sized,
     {
         self._lookup::<Q, access::RModeExists>(k, g)
     }
-    fn get<'g, Q, R: ReadMode<T>>(&'g self, k: &Q, g: &'g Guard) -> Option<&'g T::Value>
+    fn get<'g, Q>(&'g self, k: &Q, g: &'g Guard) -> Option<&'g T::Value>
     where
         T::Key: Borrow<Q>,
         Q: AsKey + ?Sized,
@@ -551,9 +560,7 @@ impl<T, S, C: Config> Tree<T, S, C> {
         debug_assert!(hf(ldfl(&leaf), NodeFlag::DATA));
         drop(Owned::<LNode<T>>::from_raw(leaf.as_raw() as *mut _))
     }
-    unsafe fn rdrop(n: &Atomic<Node<C>>) {
-        let g = upin();
-        let node = n.ld_acq(g);
+    unsafe fn _rdrop(node: Shared<Node<C>>) {
         match ldfl(&node) {
             _ if node.is_null() => {}
             flag if hf(flag, NodeFlag::DATA) => Self::ldrop(node),
@@ -567,6 +574,11 @@ impl<T, S, C: Config> Tree<T, S, C> {
             }
         }
     }
+    unsafe fn rdrop(n: &Atomic<Node<C>>) {
+        let g = upin();
+        let node = n.ld_acq(g);
+        Self::_rdrop(node);
+    }
     unsafe fn compress<'g>(
         parent: &Atomic<Node<C>>,
         child: Shared<'g, Node<C>>,

From 900e10efd78eb22817f84f411374b7a32408ff3f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 28 Jan 2023 05:43:24 -0800
Subject: [PATCH 104/310] Add mt idx impl (copy)

---
 server/src/engine/idx/mod.rs       |   2 +
 server/src/engine/idx/mtchm/imp.rs | 134 ++++++++++++++++++++++++++++-
 server/src/engine/idx/mtchm/mod.rs |   4 +-
 3 files changed, 134 insertions(+), 6 deletions(-)

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 8bbd00a3..622f3cf7 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -37,6 +37,8 @@ use core::{borrow::Borrow, hash::Hash};
 pub type IndexSTSeq<K, V, S> = stord::IndexSTSeqDll<K, V, S>;
 pub type IndexSTSeqDef<K, V> = IndexSTSeq<K, V, IndexSTSeqHasher>;
 pub type IndexSTSeqHasher = stord::IndexSTSeqDllHasher;
+pub type IndexMTRC<K, V, S> = mtchm::imp::ChmArc<K, V, S, mtchm::meta::DefConfig>;
+pub type IndexMTCp<K, V, S> = mtchm::imp::ChmCopy<K, V, S, mtchm::meta::DefConfig>;
 
 /// Any type implementing this trait can be used as a key inside memory engine structures
 pub trait AsKey: Hash + Eq {
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index 7fe86d9a..3ff07294 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -37,9 +37,9 @@ fn arc<K, V>(k: K, v: V) -> Arc<(K, V)> {
     Arc::new((k, v))
 }
 
-pub type MTArc<K, V, S, C> = Tree<Arc<(K, V)>, S, C>;
+pub type ChmArc<K, V, S, C> = Tree<Arc<(K, V)>, S, C>;
 
-impl<K, V, S, C> IndexBaseSpec<K, V> for MTArc<K, V, S, C>
+impl<K, V, S, C> IndexBaseSpec<K, V> for ChmArc<K, V, S, C>
 where
     C: Config,
     S: AsHasher,
@@ -49,7 +49,7 @@ where
     type Metrics = DummyMetrics;
 
     fn idx_init() -> Self {
-        MTArc::new()
+        ChmArc::new()
     }
 
     fn idx_init_with(s: Self) -> Self {
@@ -61,7 +61,7 @@ where
     }
 }
 
-impl<K, V, S, C> MTIndex<K, V> for MTArc<K, V, S, C>
+impl<K, V, S, C> MTIndex<K, V> for ChmArc<K, V, S, C>
 where
     C: Config,
     S: AsHasher,
@@ -162,3 +162,129 @@ where
         self.remove_return(key, g)
     }
 }
+
+pub type ChmCopy<K, V, S, C> = Tree<(K, V), S, C>;
+
+impl<K, V, S, C> IndexBaseSpec<K, V> for ChmCopy<K, V, S, C>
+where
+    C: Config,
+    S: AsHasher,
+{
+    const PREALLOC: bool = false;
+
+    type Metrics = DummyMetrics;
+
+    fn idx_init() -> Self {
+        ChmCopy::new()
+    }
+
+    fn idx_init_with(s: Self) -> Self {
+        s
+    }
+
+    fn idx_metrics(&self) -> &Self::Metrics {
+        &DummyMetrics
+    }
+}
+
+impl<K, V, S, C> MTIndex<K, V> for ChmCopy<K, V, S, C>
+where
+    C: Config,
+    S: AsHasher,
+    K: Key,
+    V: Value,
+{
+    type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, (K, V), S, C>
+    where
+        'g: 't + 'v,
+        't: 'v,
+        K: 'v,
+        V: 'v,
+        Self: 't;
+
+    type IterKey<'t, 'g, 'v> = IterKey<'t, 'g, 'v, (K, V), S, C>
+    where
+        'g: 't + 'v,
+        't: 'v,
+        K: 'v,
+        Self: 't;
+
+    type IterVal<'t, 'g, 'v> = IterVal<'t, 'g, 'v, (K, V), S, C>
+    where
+        'g: 't + 'v,
+        't: 'v,
+        V: 'v,
+        Self: 't;
+
+    fn mt_clear(&self, g: &Guard) {
+        self.nontransactional_clear(g)
+    }
+
+    fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool {
+        self.insert((key, val), g)
+    }
+
+    fn mt_upsert(&self, key: K, val: V, g: &Guard) {
+        self.upsert((key, val), g)
+    }
+
+    fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKey,
+    {
+        self.contains_key(key, g)
+    }
+
+    fn mt_get<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKey,
+        't: 'v,
+        'g: 't + 'v,
+    {
+        self.get(key, g)
+    }
+
+    fn mt_get_cloned<Q>(&self, key: &Q, g: &Guard) -> Option<V>
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKey,
+    {
+        self.get(key, g).cloned()
+    }
+
+    fn mt_update<Q>(&self, key: K, val: V, g: &Guard) -> bool
+    where
+        Q: ?Sized + AsKey,
+    {
+        self.update((key, val), g)
+    }
+
+    fn mt_update_return<'t, 'g, 'v, Q>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
+    where
+        Q: ?Sized + AsKey,
+        't: 'v,
+        'g: 't + 'v,
+    {
+        self.update_return((key, val), g)
+    }
+
+    fn mt_delete<Q>(&self, key: &Q, g: &Guard) -> bool
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKey,
+    {
+        self.remove(key, g)
+    }
+
+    fn mt_delete_return<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + AsKey,
+        't: 'v,
+        'g: 't + 'v,
+    {
+        self.remove_return(key, g)
+    }
+}
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index c9837709..806fb47e 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -25,9 +25,9 @@
 */
 
 mod access;
-mod imp;
+pub(super) mod imp;
 mod iter;
-mod meta;
+pub(super) mod meta;
 
 use self::{
     access::{ReadMode, WriteMode},

From c9ea051bd3fd711d92a5c760104cae9e5d9f4991 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 29 Jan 2023 06:15:09 -0800
Subject: [PATCH 105/310] Put state into config

---
 server/src/engine/idx/mod.rs        |  4 +-
 server/src/engine/idx/mtchm/imp.rs  | 32 +++++-----
 server/src/engine/idx/mtchm/iter.rs | 42 ++++++-------
 server/src/engine/idx/mtchm/meta.rs | 15 +++--
 server/src/engine/idx/mtchm/mod.rs  | 96 ++++++++++++++++++++++-------
 server/src/engine/macros.rs         | 20 ++++++
 6 files changed, 141 insertions(+), 68 deletions(-)

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 622f3cf7..70b33e9c 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -37,8 +37,8 @@ use core::{borrow::Borrow, hash::Hash};
 pub type IndexSTSeq<K, V, S> = stord::IndexSTSeqDll<K, V, S>;
 pub type IndexSTSeqDef<K, V> = IndexSTSeq<K, V, IndexSTSeqHasher>;
 pub type IndexSTSeqHasher = stord::IndexSTSeqDllHasher;
-pub type IndexMTRC<K, V, S> = mtchm::imp::ChmArc<K, V, S, mtchm::meta::DefConfig>;
-pub type IndexMTCp<K, V, S> = mtchm::imp::ChmCopy<K, V, S, mtchm::meta::DefConfig>;
+pub type IndexMTRC<K, V> = mtchm::imp::ChmArc<K, V, mtchm::meta::DefConfig>;
+pub type IndexMTCp<K, V> = mtchm::imp::ChmCopy<K, V, mtchm::meta::DefConfig>;
 
 /// Any type implementing this trait can be used as a key inside memory engine structures
 pub trait AsKey: Hash + Eq {
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index 3ff07294..6d10e8f6 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -27,7 +27,7 @@
 use super::{
     super::{super::sync::atm::Guard, AsKey, DummyMetrics, IndexBaseSpec, MTIndex},
     iter::{IterKV, IterKey, IterVal},
-    meta::{AsHasher, Config, Key, Value},
+    meta::{Config, Key, Value},
     Tree,
 };
 use std::{borrow::Borrow, sync::Arc};
@@ -37,12 +37,11 @@ fn arc<K, V>(k: K, v: V) -> Arc<(K, V)> {
     Arc::new((k, v))
 }
 
-pub type ChmArc<K, V, S, C> = Tree<Arc<(K, V)>, S, C>;
+pub type ChmArc<K, V, C> = Tree<Arc<(K, V)>, C>;
 
-impl<K, V, S, C> IndexBaseSpec<K, V> for ChmArc<K, V, S, C>
+impl<K, V, C> IndexBaseSpec<K, V> for ChmArc<K, V, C>
 where
     C: Config,
-    S: AsHasher,
 {
     const PREALLOC: bool = false;
 
@@ -61,14 +60,14 @@ where
     }
 }
 
-impl<K, V, S, C> MTIndex<K, V> for ChmArc<K, V, S, C>
+impl<K, V, C> MTIndex<K, V> for ChmArc<K, V, C>
 where
     C: Config,
-    S: AsHasher,
+
     K: Key,
     V: Value,
 {
-    type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, (K, V), S, C>
+    type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, (K, V), C>
     where
         'g: 't + 'v,
         't: 'v,
@@ -76,14 +75,14 @@ where
         V: 'v,
         Self: 't;
 
-    type IterKey<'t, 'g, 'v> = IterKey<'t, 'g, 'v, (K, V), S, C>
+    type IterKey<'t, 'g, 'v> = IterKey<'t, 'g, 'v, (K, V), C>
     where
         'g: 't + 'v,
         't: 'v,
         K: 'v,
         Self: 't;
 
-    type IterVal<'t, 'g, 'v> = IterVal<'t, 'g, 'v, (K, V), S, C>
+    type IterVal<'t, 'g, 'v> = IterVal<'t, 'g, 'v, (K, V), C>
     where
         'g: 't + 'v,
         't: 'v,
@@ -163,12 +162,11 @@ where
     }
 }
 
-pub type ChmCopy<K, V, S, C> = Tree<(K, V), S, C>;
+pub type ChmCopy<K, V, C> = Tree<(K, V), C>;
 
-impl<K, V, S, C> IndexBaseSpec<K, V> for ChmCopy<K, V, S, C>
+impl<K, V, C> IndexBaseSpec<K, V> for ChmCopy<K, V, C>
 where
     C: Config,
-    S: AsHasher,
 {
     const PREALLOC: bool = false;
 
@@ -187,14 +185,14 @@ where
     }
 }
 
-impl<K, V, S, C> MTIndex<K, V> for ChmCopy<K, V, S, C>
+impl<K, V, C> MTIndex<K, V> for ChmCopy<K, V, C>
 where
     C: Config,
-    S: AsHasher,
+
     K: Key,
     V: Value,
 {
-    type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, (K, V), S, C>
+    type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, (K, V), C>
     where
         'g: 't + 'v,
         't: 'v,
@@ -202,14 +200,14 @@ where
         V: 'v,
         Self: 't;
 
-    type IterKey<'t, 'g, 'v> = IterKey<'t, 'g, 'v, (K, V), S, C>
+    type IterKey<'t, 'g, 'v> = IterKey<'t, 'g, 'v, (K, V), C>
     where
         'g: 't + 'v,
         't: 'v,
         K: 'v,
         Self: 't;
 
-    type IterVal<'t, 'g, 'v> = IterVal<'t, 'g, 'v, (K, V), S, C>
+    type IterVal<'t, 'g, 'v> = IterVal<'t, 'g, 'v, (K, V), C>
     where
         'g: 't + 'v,
         't: 'v,
diff --git a/server/src/engine/idx/mtchm/iter.rs b/server/src/engine/idx/mtchm/iter.rs
index a650155f..203d4637 100644
--- a/server/src/engine/idx/mtchm/iter.rs
+++ b/server/src/engine/idx/mtchm/iter.rs
@@ -34,31 +34,31 @@ use super::{
 };
 use std::marker::PhantomData;
 
-pub struct IterKV<'t, 'g, 'v, T, S, C>
+pub struct IterKV<'t, 'g, 'v, T, C>
 where
     't: 'v,
     'g: 'v + 't,
     C: Config,
     T: TreeElement,
 {
-    i: RawIter<'t, 'g, 'v, T, S, C, CfgIterKV>,
+    i: RawIter<'t, 'g, 'v, T, C, CfgIterKV>,
 }
 
-impl<'t, 'g, 'v, T, S, C> IterKV<'t, 'g, 'v, T, S, C>
+impl<'t, 'g, 'v, T, C> IterKV<'t, 'g, 'v, T, C>
 where
     't: 'v,
     'g: 'v + 't,
     C: Config,
     T: TreeElement,
 {
-    pub fn new(t: &'t Tree<T, S, C>, g: &'g Guard) -> Self {
+    pub fn new(t: &'t Tree<T, C>, g: &'g Guard) -> Self {
         Self {
             i: RawIter::new(t, g),
         }
     }
 }
 
-impl<'t, 'g, 'v, T, S, C> Iterator for IterKV<'t, 'g, 'v, T, S, C>
+impl<'t, 'g, 'v, T, C> Iterator for IterKV<'t, 'g, 'v, T, C>
 where
     't: 'v,
     'g: 'v + 't,
@@ -72,31 +72,31 @@ where
     }
 }
 
-pub struct IterKey<'t, 'g, 'v, T, S, C>
+pub struct IterKey<'t, 'g, 'v, T, C>
 where
     't: 'v,
     'g: 'v + 't,
     C: Config,
     T: TreeElement,
 {
-    i: RawIter<'t, 'g, 'v, T, S, C, CfgIterKey>,
+    i: RawIter<'t, 'g, 'v, T, C, CfgIterKey>,
 }
 
-impl<'t, 'g, 'v, T, S, C> IterKey<'t, 'g, 'v, T, S, C>
+impl<'t, 'g, 'v, T, C> IterKey<'t, 'g, 'v, T, C>
 where
     't: 'v,
     'g: 'v + 't,
     C: Config,
     T: TreeElement,
 {
-    pub fn new(t: &'t Tree<T, S, C>, g: &'g Guard) -> Self {
+    pub fn new(t: &'t Tree<T, C>, g: &'g Guard) -> Self {
         Self {
             i: RawIter::new(t, g),
         }
     }
 }
 
-impl<'t, 'g, 'v, T, S, C> Iterator for IterKey<'t, 'g, 'v, T, S, C>
+impl<'t, 'g, 'v, T, C> Iterator for IterKey<'t, 'g, 'v, T, C>
 where
     't: 'v,
     'g: 'v + 't,
@@ -110,31 +110,31 @@ where
     }
 }
 
-pub struct IterVal<'t, 'g, 'v, T, S, C>
+pub struct IterVal<'t, 'g, 'v, T, C>
 where
     't: 'v,
     'g: 'v + 't,
     C: Config,
     T: TreeElement,
 {
-    i: RawIter<'t, 'g, 'v, T, S, C, CfgIterVal>,
+    i: RawIter<'t, 'g, 'v, T, C, CfgIterVal>,
 }
 
-impl<'t, 'g, 'v, T, S, C> IterVal<'t, 'g, 'v, T, S, C>
+impl<'t, 'g, 'v, T, C> IterVal<'t, 'g, 'v, T, C>
 where
     't: 'v,
     'g: 'v + 't,
     C: Config,
     T: TreeElement,
 {
-    pub fn new(t: &'t Tree<T, S, C>, g: &'g Guard) -> Self {
+    pub fn new(t: &'t Tree<T, C>, g: &'g Guard) -> Self {
         Self {
             i: RawIter::new(t, g),
         }
     }
 }
 
-impl<'t, 'g, 'v, T, S, C> Iterator for IterVal<'t, 'g, 'v, T, S, C>
+impl<'t, 'g, 'v, T, C> Iterator for IterVal<'t, 'g, 'v, T, C>
 where
     't: 'v,
     'g: 'v + 't,
@@ -184,7 +184,7 @@ struct DFSCNodeCtx<'g, C: Config> {
     idx: usize,
 }
 
-struct RawIter<'t, 'g, 'v, T, S, C, I>
+struct RawIter<'t, 'g, 'v, T, C, I>
 where
     't: 'v,
     'g: 'v + 't,
@@ -193,17 +193,17 @@ where
 {
     g: &'g Guard,
     stack: UArray<{ <DefConfig as Config>::BRANCH_MX + 1 }, DFSCNodeCtx<'g, C>>,
-    _m: PhantomData<(&'v T, C, &'t Tree<T, S, C>, I)>,
+    _m: PhantomData<(&'v T, C, &'t Tree<T, C>, I)>,
 }
 
-impl<'t, 'g, 'v, T, S, C, I> RawIter<'t, 'g, 'v, T, S, C, I>
+impl<'t, 'g, 'v, T, C, I> RawIter<'t, 'g, 'v, T, C, I>
 where
     't: 'v,
     'g: 'v + 't,
     I: IterConfig<T>,
     C: Config,
 {
-    pub(super) fn new(tree: &'t Tree<T, S, C>, g: &'g Guard) -> Self {
+    pub(super) fn new(tree: &'t Tree<T, C>, g: &'g Guard) -> Self {
         let mut stack = UArray::new();
         let sptr = tree.root.ld_acq(g);
         stack.push(DFSCNodeCtx { sptr, idx: 0 });
@@ -227,7 +227,7 @@ where
                 flag if super::hf(flag, NodeFlag::DATA) => {
                     let data = unsafe {
                         // UNSAFE(@ohsayan): flagck
-                        Tree::<T, S, C>::read_data(current.sptr)
+                        Tree::<T, C>::read_data(current.sptr)
                     };
                     if current.idx < data.len() {
                         let ref ret = data[current.idx];
@@ -255,7 +255,7 @@ where
     }
 }
 
-impl<'t, 'g, 'v, T, S, C, I> Iterator for RawIter<'t, 'g, 'v, T, S, C, I>
+impl<'t, 'g, 'v, T, C, I> Iterator for RawIter<'t, 'g, 'v, T, C, I>
 where
     't: 'v,
     'g: 'v + 't,
diff --git a/server/src/engine/idx/mtchm/meta.rs b/server/src/engine/idx/mtchm/meta.rs
index 913b0db5..e0d7d699 100644
--- a/server/src/engine/idx/mtchm/meta.rs
+++ b/server/src/engine/idx/mtchm/meta.rs
@@ -25,13 +25,14 @@
 */
 
 use super::super::{super::mem::VInline, AsKeyClone};
-use std::{hash::BuildHasher, sync::Arc};
+use std::{collections::hash_map::RandomState, hash::BuildHasher, sync::Arc};
 
 const LNODE_STACK: usize = 2;
-pub type DefConfig = Config2B;
+pub type DefConfig = Config2BRandomState;
 pub type LNode<T> = VInline<LNODE_STACK, T>;
 
 pub trait PreConfig: Sized + 'static {
+    type HState: AsHasher;
     const BITS: u32;
 }
 
@@ -48,7 +49,7 @@ pub trait Config: PreConfig {
         }
         log
     };
-    const HASH_MASK: u64 = (<Self as PreConfig>::BITS - 1) as _;
+    const MASK: u64 = (<Self as PreConfig>::BITS - 1) as _;
     const MAX_TREE_HEIGHT_UB: usize = 0x40;
     const MAX_TREE_HEIGHT: usize =
         <Self as Config>::MAX_TREE_HEIGHT_UB / <Self as Config>::BRANCH_LG;
@@ -58,12 +59,14 @@ pub trait Config: PreConfig {
 impl<T: PreConfig> Config for T {}
 
 macro_rules! impl_config {
-    ($($vis:vis $name:ident = $ty:ty),*) => {
-        $($vis struct $name; impl $crate::engine::idx::mtchm::meta::PreConfig for $name { const BITS: u32 = <$ty>::BITS; })*
+    ($($vis:vis $name:ident: $state:ty = $ty:ty),*) => {
+        $($vis struct $name; impl $crate::engine::idx::mtchm::meta::PreConfig for $name {
+            type HState = $state; const BITS: u32 = <$ty>::BITS;
+        })*
     }
 }
 
-impl_config!(pub Config2B = u16);
+impl_config!(pub Config2BRandomState: RandomState = u16);
 
 pub trait Key: AsKeyClone + 'static {}
 impl<T> Key for T where T: AsKeyClone + 'static {}
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index 806fb47e..3c7a08e5 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -32,7 +32,7 @@ pub(super) mod meta;
 use self::{
     access::{ReadMode, WriteMode},
     iter::{IterKV, IterKey, IterVal},
-    meta::{AsHasher, CompressState, Config, DefConfig, LNode, NodeFlag, TreeElement},
+    meta::{CompressState, Config, DefConfig, LNode, NodeFlag, TreeElement},
 };
 use super::{
     super::{
@@ -42,13 +42,61 @@ use super::{
     AsKey,
 };
 use crossbeam_epoch::CompareExchangeError;
-use std::{borrow::Borrow, hash::Hasher, marker::PhantomData, mem, sync::atomic::AtomicUsize};
+use std::{
+    borrow::Borrow,
+    hash::{BuildHasher, Hasher},
+    marker::PhantomData,
+    mem,
+    sync::atomic::AtomicUsize,
+};
 
 /*
     HACK(@ohsayan): Until https://github.com/rust-lang/rust/issues/76560 is stabilized which is likely to take a while,
     we need to settle for trait objects
 */
 
+#[cfg(debug_assertions)]
+struct CHTMetricsData {
+    split: AtomicUsize,
+    hln: AtomicUsize,
+}
+
+pub struct CHTRuntimeLog {
+    #[cfg(debug_assertions)]
+    data: CHTMetricsData,
+    #[cfg(not(debug_assertions))]
+    data: (),
+}
+
+impl CHTRuntimeLog {
+    #[cfg(debug_assertions)]
+    const ZERO: AtomicUsize = AtomicUsize::new(0);
+    #[cfg(not(debug_assertions))]
+    const NEW: Self = Self { data: () };
+    #[cfg(debug_assertions)]
+    const NEW: Self = Self {
+        data: CHTMetricsData {
+            split: Self::ZERO,
+            hln: Self::ZERO,
+        },
+    };
+    const fn new() -> Self {
+        Self::NEW
+    }
+    dbgfn! {
+        fn hsplit(self: &Self) {
+            self.data.split.fetch_add(1, ORD_ACQ);
+        } else {
+            void!()
+        }
+        fn hlnode(self: &Self) {
+            self.data.hln.fetch_add(1, ORD_ACQ);
+        } else {
+            void!()
+        }
+    }
+}
+
 pub struct Node<C: Config> {
     branch: [Atomic<Self>; <DefConfig as Config>::BRANCH_MX],
 }
@@ -96,27 +144,29 @@ trait CTFlagAlign {
     const FLCK: () = Self::FLCK_A;
 }
 
-impl<T, S, C: Config> CTFlagAlign for Tree<T, S, C> {
+impl<T, C: Config> CTFlagAlign for Tree<T, C> {
     const FL_A: bool = atm::ensure_flag_align::<LNode<T>, { NodeFlag::bits() }>();
     const FL_B: bool = atm::ensure_flag_align::<Node<C>, { NodeFlag::bits() }>();
 }
 
-pub struct Tree<T, S, C: Config> {
+pub struct Tree<T, C: Config> {
     root: Atomic<Node<C>>,
-    h: S,
+    h: C::HState,
     l: AtomicUsize,
     _m: PhantomData<T>,
+    m: CHTRuntimeLog,
 }
 
-impl<T, S, C: Config> Tree<T, S, C> {
+impl<T, C: Config> Tree<T, C> {
     #[inline(always)]
-    const fn _new(h: S) -> Self {
+    const fn _new(h: C::HState) -> Self {
         let _ = Self::FLCK;
         Self {
             root: Atomic::null(),
             h,
             l: AtomicUsize::new(0),
             _m: PhantomData,
+            m: CHTRuntimeLog::new(),
         }
     }
     #[inline(always)]
@@ -128,19 +178,19 @@ impl<T, S, C: Config> Tree<T, S, C> {
         self.len() == 0
     }
     #[inline(always)]
-    pub const fn with_hasher(h: S) -> Self {
+    pub const fn with_state(h: C::HState) -> Self {
         Self::_new(h)
     }
 }
 
-impl<T, S: AsHasher, C: Config> Tree<T, S, C> {
+impl<T, C: Config> Tree<T, C> {
     #[inline(always)]
     fn new() -> Self {
-        Self::_new(S::default())
+        Self::_new(C::HState::default())
     }
 }
 
-impl<T, S: AsHasher, C: Config> Tree<T, S, C> {
+impl<T, C: Config> Tree<T, C> {
     fn hash<Q>(&self, k: &Q) -> u64
     where
         Q: ?Sized + AsKey,
@@ -152,19 +202,19 @@ impl<T, S: AsHasher, C: Config> Tree<T, S, C> {
 }
 
 // iter
-impl<T: TreeElement, S, C: Config> Tree<T, S, C> {
-    fn iter_kv<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterKV<'t, 'g, 'v, T, S, C> {
+impl<T: TreeElement, C: Config> Tree<T, C> {
+    fn iter_kv<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterKV<'t, 'g, 'v, T, C> {
         IterKV::new(self, g)
     }
-    fn iter_key<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterKey<'t, 'g, 'v, T, S, C> {
+    fn iter_key<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterKey<'t, 'g, 'v, T, C> {
         IterKey::new(self, g)
     }
-    fn iter_val<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterVal<'t, 'g, 'v, T, S, C> {
+    fn iter_val<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterVal<'t, 'g, 'v, T, C> {
         IterVal::new(self, g)
     }
 }
 
-impl<T: TreeElement, S: AsHasher, C: Config> Tree<T, S, C> {
+impl<T: TreeElement, C: Config> Tree<T, C> {
     fn nontransactional_clear(&self, g: &Guard) {
         self.iter_key(g).for_each(|k| {
             let _ = self.remove(k, g);
@@ -247,12 +297,13 @@ impl<T: TreeElement, S: AsHasher, C: Config> Tree<T, S, C> {
                             so this is a collision and since we haven't reached the max height, we should always
                             create a new branch so let's do that
                         */
+                        self.m.hsplit();
                         debug_assert_eq!(data.len(), 1, "logic,lnode before height ub");
                         if W::WMODE == access::WRITEMODE_REFRESH {
                             // another job well done; an snode with the wrong key; so basically it's missing
                             return W::nx();
                         }
-                        let next_chunk = (self.hash(data[0].key()) >> level) & C::HASH_MASK;
+                        let next_chunk = (self.hash(data[0].key()) >> level) & C::MASK;
                         let mut new_branch = Node::null();
                         // stick this one in
                         new_branch.branch[next_chunk as usize] = Atomic::from(node);
@@ -263,6 +314,7 @@ impl<T: TreeElement, S: AsHasher, C: Config> Tree<T, S, C> {
                             in this case we either have the same key or we found an lnode. resolve any conflicts and attempt
                             to update
                         */
+                        self.m.hlnode();
                         let p = data.iter().position(|e| e.key() == elem.key());
                         match p {
                             Some(v) if W::WMODE == access::WRITEMODE_FRESH => {
@@ -334,7 +386,7 @@ impl<T: TreeElement, S: AsHasher, C: Config> Tree<T, S, C> {
                 }
                 _ => {
                     // branch
-                    let nxidx = (hash >> level) & C::HASH_MASK;
+                    let nxidx = (hash >> level) & C::MASK;
                     level += C::BRANCH_LG;
                     parent = Some(current);
                     child = Some(node);
@@ -386,7 +438,7 @@ impl<T: TreeElement, S: AsHasher, C: Config> Tree<T, S, C> {
                 }
                 _ => {
                     // branch
-                    current = &unsafe { node.deref() }.branch[(hash & C::HASH_MASK) as usize];
+                    current = &unsafe { node.deref() }.branch[(hash & C::MASK) as usize];
                     hash >>= C::BRANCH_LG;
                 }
             }
@@ -512,7 +564,7 @@ impl<T: TreeElement, S: AsHasher, C: Config> Tree<T, S, C> {
                 _ => {
                     // branch
                     levels.push((current, node));
-                    let nxidx = (hash >> level) & C::HASH_MASK;
+                    let nxidx = (hash >> level) & C::MASK;
                     level += C::BRANCH_LG;
                     current = &unsafe { node.deref() }.branch[nxidx as usize];
                 }
@@ -522,7 +574,7 @@ impl<T: TreeElement, S: AsHasher, C: Config> Tree<T, S, C> {
 }
 
 // low-level methods
-impl<T, S, C: Config> Tree<T, S, C> {
+impl<T, C: Config> Tree<T, C> {
     // hilarious enough but true, l doesn't affect safety but only creates an incorrect state
     fn decr_len(&self) {
         self.l.fetch_sub(1, ORD_ACQ);
@@ -660,7 +712,7 @@ impl<T, S, C: Config> Tree<T, S, C> {
     }
 }
 
-impl<T, S, C: Config> Drop for Tree<T, S, C> {
+impl<T, C: Config> Drop for Tree<T, C> {
     fn drop(&mut self) {
         unsafe {
             // UNSAFE(@ohsayan): sole live owner
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index a65c7f84..ba58476a 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -98,3 +98,23 @@ macro_rules! union {
     (@defeat0 [$($decls:tt)*] [$($head:tt)*]) => (union!(@defeat1 $($decls)* { $($head)* }););
     (@defeat1 $i:item) => ($i);
 }
+
+macro_rules! dbgfn {
+    ($($vis:vis fn $fn:ident($($arg:ident: $argty:ty),* $(,)?) $(-> $ret:ty)? $block:block)*) => {
+        $(dbgfn!(@int $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block {panic!("called dbg symbol in non-dbg build")});)*
+    };
+    ($($vis:vis fn $fn:ident($($arg:ident: $argty:ty),* $(,)?) $(-> $ret:ty)? $block:block else $block_b:block)*) => {
+        $(dbgfn!(@int $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block $block_b);)*
+    };
+    (@int $vis:vis fn $fn:ident($($arg:ident: $argty:ty),* $(,)?) $(-> $ret:ty)? $block_a:block $block_b:block) => {
+        #[cfg(debug_assertions)]
+        $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block_a
+        #[cfg(not(debug_assertions))]
+        $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block_b
+    }
+}
+
+#[allow(unused_macros)]
+macro_rules! void {
+    () => {()};
+}

From bec5e10f7b7f33ae9079d27cc25aaf15c124d8a7 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 29 Jan 2023 06:58:53 -0800
Subject: [PATCH 106/310] Add mt idx empty tests

---
 server/src/engine/idx/mod.rs         |  8 +--
 server/src/engine/idx/mtchm/imp.rs   | 30 ++++-----
 server/src/engine/idx/mtchm/meta.rs  | 14 ++--
 server/src/engine/idx/mtchm/mod.rs   |  2 +
 server/src/engine/idx/mtchm/tests.rs | 99 ++++++++++++++++++++++++++++
 5 files changed, 121 insertions(+), 32 deletions(-)
 create mode 100644 server/src/engine/idx/mtchm/tests.rs

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 70b33e9c..6ad60a29 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -175,17 +175,15 @@ pub trait MTIndex<K, V>: IndexBaseSpec<K, V> {
         V: AsValueClone;
     // update
     /// Returns true if the entry is updated
-    fn mt_update<Q>(&self, key: K, val: V, g: &Guard) -> bool
+    fn mt_update(&self, key: K, val: V, g: &Guard) -> bool
     where
         K: AsKeyClone,
-        V: AsValue,
-        Q: ?Sized + AsKey;
+        V: AsValue;
     /// Updates the entry and returns the old value, if it exists
-    fn mt_update_return<'t, 'g, 'v, Q>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
+    fn mt_update_return<'t, 'g, 'v>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
     where
         K: AsKeyClone,
         V: AsValue,
-        Q: ?Sized + AsKey,
         't: 'v,
         'g: 't + 'v;
     // delete
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index 6d10e8f6..d6a0b6f8 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -25,10 +25,10 @@
 */
 
 use super::{
-    super::{super::sync::atm::Guard, AsKey, DummyMetrics, IndexBaseSpec, MTIndex},
+    super::{super::sync::atm::Guard, AsKey, IndexBaseSpec, MTIndex},
     iter::{IterKV, IterKey, IterVal},
     meta::{Config, Key, Value},
-    Tree,
+    CHTRuntimeLog, Tree,
 };
 use std::{borrow::Borrow, sync::Arc};
 
@@ -45,7 +45,7 @@ where
 {
     const PREALLOC: bool = false;
 
-    type Metrics = DummyMetrics;
+    type Metrics = CHTRuntimeLog;
 
     fn idx_init() -> Self {
         ChmArc::new()
@@ -55,8 +55,9 @@ where
         s
     }
 
+    #[cfg(debug_assertions)]
     fn idx_metrics(&self) -> &Self::Metrics {
-        &DummyMetrics
+        &self.m
     }
 }
 
@@ -127,16 +128,12 @@ where
         self.get(key, g).cloned()
     }
 
-    fn mt_update<Q>(&self, key: K, val: V, g: &Guard) -> bool
-    where
-        Q: ?Sized + AsKey,
-    {
+    fn mt_update(&self, key: K, val: V, g: &Guard) -> bool {
         self.update(arc(key, val), g)
     }
 
-    fn mt_update_return<'t, 'g, 'v, Q>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
+    fn mt_update_return<'t, 'g, 'v>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
     where
-        Q: ?Sized + AsKey,
         't: 'v,
         'g: 't + 'v,
     {
@@ -170,7 +167,7 @@ where
 {
     const PREALLOC: bool = false;
 
-    type Metrics = DummyMetrics;
+    type Metrics = CHTRuntimeLog;
 
     fn idx_init() -> Self {
         ChmCopy::new()
@@ -180,8 +177,9 @@ where
         s
     }
 
+    #[cfg(debug_assertions)]
     fn idx_metrics(&self) -> &Self::Metrics {
-        &DummyMetrics
+        &self.m
     }
 }
 
@@ -252,16 +250,12 @@ where
         self.get(key, g).cloned()
     }
 
-    fn mt_update<Q>(&self, key: K, val: V, g: &Guard) -> bool
-    where
-        Q: ?Sized + AsKey,
-    {
+    fn mt_update(&self, key: K, val: V, g: &Guard) -> bool {
         self.update((key, val), g)
     }
 
-    fn mt_update_return<'t, 'g, 'v, Q>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
+    fn mt_update_return<'t, 'g, 'v>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
     where
-        Q: ?Sized + AsKey,
         't: 'v,
         'g: 't + 'v,
     {
diff --git a/server/src/engine/idx/mtchm/meta.rs b/server/src/engine/idx/mtchm/meta.rs
index e0d7d699..8702d5fb 100644
--- a/server/src/engine/idx/mtchm/meta.rs
+++ b/server/src/engine/idx/mtchm/meta.rs
@@ -28,7 +28,7 @@ use super::super::{super::mem::VInline, AsKeyClone};
 use std::{collections::hash_map::RandomState, hash::BuildHasher, sync::Arc};
 
 const LNODE_STACK: usize = 2;
-pub type DefConfig = Config2BRandomState;
+pub type DefConfig = Config2B<RandomState>;
 pub type LNode<T> = VInline<LNODE_STACK, T>;
 
 pub trait PreConfig: Sized + 'static {
@@ -58,16 +58,12 @@ pub trait Config: PreConfig {
 
 impl<T: PreConfig> Config for T {}
 
-macro_rules! impl_config {
-    ($($vis:vis $name:ident: $state:ty = $ty:ty),*) => {
-        $($vis struct $name; impl $crate::engine::idx::mtchm::meta::PreConfig for $name {
-            type HState = $state; const BITS: u32 = <$ty>::BITS;
-        })*
-    }
+pub struct Config2B<T: AsHasher + 'static>(T);
+impl<T: AsHasher> PreConfig for Config2B<T> {
+    const BITS: u32 = u16::BITS;
+    type HState = T;
 }
 
-impl_config!(pub Config2BRandomState: RandomState = u16);
-
 pub trait Key: AsKeyClone + 'static {}
 impl<T> Key for T where T: AsKeyClone + 'static {}
 pub trait Value: Clone + 'static {}
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index 3c7a08e5..47a7ccfb 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -28,6 +28,8 @@ mod access;
 pub(super) mod imp;
 mod iter;
 pub(super) mod meta;
+#[cfg(test)]
+mod tests;
 
 use self::{
     access::{ReadMode, WriteMode},
diff --git a/server/src/engine/idx/mtchm/tests.rs b/server/src/engine/idx/mtchm/tests.rs
new file mode 100644
index 00000000..a58432cf
--- /dev/null
+++ b/server/src/engine/idx/mtchm/tests.rs
@@ -0,0 +1,99 @@
+/*
+ * Created on Sun Jan 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::{
+    super::{super::sync::atm::cpin, IndexBaseSpec, MTIndex},
+    imp::ChmCopy,
+    meta::DefConfig,
+};
+use std::hash::{BuildHasher, Hasher};
+
+type Chm<K, V> = ChmCopy<K, V, DefConfig>;
+
+struct LolHash {
+    seed: usize,
+}
+
+impl LolHash {
+    const fn with_seed(seed: usize) -> Self {
+        Self { seed }
+    }
+    const fn init_default_seed() -> Self {
+        Self::with_seed(0)
+    }
+}
+
+impl Default for LolHash {
+    fn default() -> Self {
+        Self::init_default_seed()
+    }
+}
+
+impl Hasher for LolHash {
+    fn finish(&self) -> u64 {
+        self.seed as _
+    }
+    fn write(&mut self, _: &[u8]) {}
+}
+
+struct LolState {
+    seed: usize,
+}
+
+impl BuildHasher for LolState {
+    type Hasher = LolHash;
+
+    fn build_hasher(&self) -> Self::Hasher {
+        LolHash::with_seed(self.seed)
+    }
+}
+
+impl Default for LolState {
+    fn default() -> Self {
+        Self { seed: 0 }
+    }
+}
+
+type ChmU8 = Chm<u8, u8>;
+
+// empty
+#[test]
+fn drop_empty() {
+    let idx = ChmU8::idx_init();
+    drop(idx);
+}
+
+#[test]
+fn get_empty() {
+    let idx = ChmU8::idx_init();
+    assert!(idx.mt_get(&10, &cpin()).is_none());
+}
+
+#[test]
+fn update_empty() {
+    let idx = ChmU8::idx_init();
+    assert!(!idx.mt_update(10, 20, &cpin()));
+}

From 3eb00f892fdb5f286c0456715afdb0021e5440c0 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 29 Jan 2023 07:54:54 -0800
Subject: [PATCH 107/310] Add insert test

---
 server/src/engine/idx/mtchm/mod.rs   |  4 +-
 server/src/engine/idx/mtchm/tests.rs | 65 ++++++++++++++++++++++++++--
 2 files changed, 64 insertions(+), 5 deletions(-)

diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index 47a7ccfb..bda35d0c 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -151,12 +151,12 @@ impl<T, C: Config> CTFlagAlign for Tree<T, C> {
     const FL_B: bool = atm::ensure_flag_align::<Node<C>, { NodeFlag::bits() }>();
 }
 
-pub struct Tree<T, C: Config> {
+pub struct Tree<T, C: Config = DefConfig> {
     root: Atomic<Node<C>>,
     h: C::HState,
     l: AtomicUsize,
-    _m: PhantomData<T>,
     m: CHTRuntimeLog,
+    _m: PhantomData<T>,
 }
 
 impl<T, C: Config> Tree<T, C> {
diff --git a/server/src/engine/idx/mtchm/tests.rs b/server/src/engine/idx/mtchm/tests.rs
index a58432cf..52eeb938 100644
--- a/server/src/engine/idx/mtchm/tests.rs
+++ b/server/src/engine/idx/mtchm/tests.rs
@@ -26,12 +26,17 @@
 
 use super::{
     super::{super::sync::atm::cpin, IndexBaseSpec, MTIndex},
-    imp::ChmCopy,
+    imp::ChmCopy as _ChmCopy,
     meta::DefConfig,
 };
-use std::hash::{BuildHasher, Hasher};
+use std::{
+    hash::{BuildHasher, Hasher},
+    sync::{Arc, RwLock},
+    thread::{self, JoinHandle},
+};
 
-type Chm<K, V> = ChmCopy<K, V, DefConfig>;
+type Chm<K, V> = ChmCopy<K, V>;
+type ChmCopy<K, V> = _ChmCopy<K, V, DefConfig>;
 
 struct LolHash {
     seed: usize,
@@ -97,3 +102,57 @@ fn update_empty() {
     let idx = ChmU8::idx_init();
     assert!(!idx.mt_update(10, 20, &cpin()));
 }
+
+const SPAM_INSERT: usize = 16_384;
+const SPAM_TENANTS: usize = 32;
+
+#[test]
+fn multispam_insert() {
+    let idx = Arc::new(ChmCopy::new());
+    let token = Arc::new(RwLock::new(()));
+    let hold = token.write();
+    let data: Vec<(Arc<String>, Arc<String>)> = (0..SPAM_INSERT)
+        .into_iter()
+        .map(|int| (format!("{int}"), format!("x-{int}-{}", int + 1)))
+        .map(|(k, v)| (Arc::new(k), Arc::new(v)))
+        .collect();
+    let distr_data: Vec<Vec<(Arc<String>, Arc<String>)>> = data
+        .chunks(SPAM_INSERT / SPAM_TENANTS)
+        .map(|chunk| {
+            chunk
+                .iter()
+                .map(|(k, v)| (Arc::clone(k), Arc::clone(v)))
+                .collect()
+        })
+        .collect();
+    let threads: Vec<JoinHandle<_>> = distr_data
+        .into_iter()
+        .enumerate()
+        .map(|(tid, this_data)| {
+            let this_token = token.clone();
+            let this_idx = idx.clone();
+            thread::Builder::new()
+                .name(tid.to_string())
+                .spawn(move || {
+                    let _token = this_token.read();
+                    let g = cpin();
+                    this_data.into_iter().for_each(|(k, v)| {
+                        assert!(this_idx.insert((k, v), &g));
+                    })
+                })
+                .unwrap()
+        })
+        .collect();
+    // rush everyone to insert; superb intercore traffic
+    drop(hold);
+    let _x: Box<[()]> = threads
+        .into_iter()
+        .map(JoinHandle::join)
+        .map(Result::unwrap)
+        .collect();
+    let pin = cpin();
+    assert_eq!(idx.len(), SPAM_INSERT);
+    data.into_iter().for_each(|(k, v)| {
+        assert_eq!(idx.mt_get(&k, &pin).unwrap().as_str(), &*v);
+    });
+}

From b21162c4dcc14abed6b3daae58a165ed8caaa78c Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 29 Jan 2023 22:02:17 -0800
Subject: [PATCH 108/310] Make st idx generic on alloc strategy

---
 server/src/engine/idx/meta.rs                 |  30 ++++
 server/src/engine/idx/mod.rs                  |   6 +-
 server/src/engine/idx/mtchm/meta.rs           |   6 +-
 server/src/engine/idx/stord/config.rs         | 125 +++++++++++++++
 .../src/engine/idx/{stord.rs => stord/mod.rs} | 147 +++++++-----------
 server/src/engine/idx/tests.rs                |  25 +--
 6 files changed, 236 insertions(+), 103 deletions(-)
 create mode 100644 server/src/engine/idx/meta.rs
 create mode 100644 server/src/engine/idx/stord/config.rs
 rename server/src/engine/idx/{stord.rs => stord/mod.rs} (89%)

diff --git a/server/src/engine/idx/meta.rs b/server/src/engine/idx/meta.rs
new file mode 100644
index 00000000..c486e87b
--- /dev/null
+++ b/server/src/engine/idx/meta.rs
@@ -0,0 +1,30 @@
+/*
+ * Created on Sun Jan 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use std::hash::BuildHasher;
+
+pub trait AsHasher: BuildHasher + Default {}
+impl<T> AsHasher for T where T: BuildHasher + Default {}
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 6ad60a29..e484015d 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -24,6 +24,7 @@
  *
 */
 
+mod meta;
 mod mtchm;
 mod stdhm;
 mod stord;
@@ -34,9 +35,8 @@ use super::sync::atm::Guard;
 use core::{borrow::Borrow, hash::Hash};
 
 // re-exports
-pub type IndexSTSeq<K, V, S> = stord::IndexSTSeqDll<K, V, S>;
-pub type IndexSTSeqDef<K, V> = IndexSTSeq<K, V, IndexSTSeqHasher>;
-pub type IndexSTSeqHasher = stord::IndexSTSeqDllHasher;
+pub type IndexSTSeqCns<K, V> = stord::IndexSTSeqDll<K, V, stord::config::ConservativeConfig<K, V>>;
+pub type IndexSTSeqLib<K, V> = stord::IndexSTSeqDll<K, V, stord::config::LiberalConfig<K, V>>;
 pub type IndexMTRC<K, V> = mtchm::imp::ChmArc<K, V, mtchm::meta::DefConfig>;
 pub type IndexMTCp<K, V> = mtchm::imp::ChmCopy<K, V, mtchm::meta::DefConfig>;
 
diff --git a/server/src/engine/idx/mtchm/meta.rs b/server/src/engine/idx/mtchm/meta.rs
index 8702d5fb..04c43bec 100644
--- a/server/src/engine/idx/mtchm/meta.rs
+++ b/server/src/engine/idx/mtchm/meta.rs
@@ -24,8 +24,8 @@
  *
 */
 
-use super::super::{super::mem::VInline, AsKeyClone};
-use std::{collections::hash_map::RandomState, hash::BuildHasher, sync::Arc};
+use super::super::{super::mem::VInline, meta::AsHasher, AsKeyClone};
+use std::{collections::hash_map::RandomState, sync::Arc};
 
 const LNODE_STACK: usize = 2;
 pub type DefConfig = Config2B<RandomState>;
@@ -68,8 +68,6 @@ pub trait Key: AsKeyClone + 'static {}
 impl<T> Key for T where T: AsKeyClone + 'static {}
 pub trait Value: Clone + 'static {}
 impl<T> Value for T where T: Clone + 'static {}
-pub trait AsHasher: BuildHasher + Default {}
-impl<T> AsHasher for T where T: BuildHasher + Default {}
 
 pub trait TreeElement: Clone + 'static {
     type Key: Key;
diff --git a/server/src/engine/idx/stord/config.rs b/server/src/engine/idx/stord/config.rs
new file mode 100644
index 00000000..8a8b38c6
--- /dev/null
+++ b/server/src/engine/idx/stord/config.rs
@@ -0,0 +1,125 @@
+/*
+ * Created on Sun Jan 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::{super::meta::AsHasher, IndexSTSeqDllNode, IndexSTSeqDllNodePtr};
+use std::{collections::hash_map::RandomState, marker::PhantomData, ptr};
+
+pub struct LiberalStrategy<K, V> {
+    f: *mut IndexSTSeqDllNode<K, V>,
+}
+
+impl<K, V> AllocStrategy<K, V> for LiberalStrategy<K, V> {
+    const NEW: Self = Self { f: ptr::null_mut() };
+    const METRIC_REFRESH: bool = true;
+    #[inline(always)]
+    unsafe fn free(&mut self, n: *mut IndexSTSeqDllNode<K, V>) {
+        (*n).n = self.f;
+        self.f = n;
+    }
+    #[inline(always)]
+    fn alloc(
+        &mut self,
+        node: IndexSTSeqDllNode<K, V>,
+        refresh_metric: &mut bool,
+    ) -> IndexSTSeqDllNodePtr<K, V> {
+        if self.f.is_null() {
+            IndexSTSeqDllNode::alloc_box(node)
+        } else {
+            *refresh_metric = true;
+            unsafe {
+                // UNSAFE(@ohsayan): Safe because we already did a nullptr check
+                let f = self.f;
+                self.f = (*self.f).n;
+                ptr::write(f, node);
+                IndexSTSeqDllNodePtr::new_unchecked(f)
+            }
+        }
+    }
+    #[inline(always)]
+    fn cleanup(&mut self) {
+        unsafe {
+            // UNSAFE(@ohsayan): All nullck
+            let mut c = self.f;
+            while !c.is_null() {
+                let nx = (*c).n;
+                IndexSTSeqDllNode::dealloc_headless(c);
+                c = nx;
+            }
+        }
+        self.f = ptr::null_mut();
+    }
+}
+
+pub struct ConservativeStrategy<K, V> {
+    _d: PhantomData<IndexSTSeqDllNodePtr<K, V>>,
+}
+
+impl<K, V> AllocStrategy<K, V> for ConservativeStrategy<K, V> {
+    const NEW: Self = Self { _d: PhantomData };
+    const METRIC_REFRESH: bool = false;
+    #[inline(always)]
+    unsafe fn free(&mut self, n: *mut IndexSTSeqDllNode<K, V>) {
+        IndexSTSeqDllNode::dealloc_headless(n)
+    }
+    #[inline(always)]
+    fn alloc(&mut self, node: IndexSTSeqDllNode<K, V>, _: &mut bool) -> IndexSTSeqDllNodePtr<K, V> {
+        IndexSTSeqDllNode::alloc_box(node)
+    }
+    #[inline(always)]
+    fn cleanup(&mut self) {}
+}
+
+pub trait AllocStrategy<K, V>: Sized {
+    // HACK(@ohsayan): I trust the optimizer, but not so much
+    const METRIC_REFRESH: bool;
+    const NEW: Self;
+    fn alloc(
+        &mut self,
+        node: IndexSTSeqDllNode<K, V>,
+        refresh_metric: &mut bool,
+    ) -> IndexSTSeqDllNodePtr<K, V>;
+    unsafe fn free(&mut self, f: *mut IndexSTSeqDllNode<K, V>);
+    fn cleanup(&mut self);
+}
+
+pub trait Config<K, V> {
+    type Hasher: AsHasher;
+    type AllocStrategy: AllocStrategy<K, V>;
+}
+
+pub struct ConservativeConfig<K, V>(PhantomData<super::IndexSTSeqDll<K, V, Self>>);
+
+impl<K, V> Config<K, V> for ConservativeConfig<K, V> {
+    type Hasher = RandomState;
+    type AllocStrategy = ConservativeStrategy<K, V>;
+}
+
+pub struct LiberalConfig<K, V>(PhantomData<super::IndexSTSeqDll<K, V, Self>>);
+
+impl<K, V> Config<K, V> for LiberalConfig<K, V> {
+    type Hasher = RandomState;
+    type AllocStrategy = LiberalStrategy<K, V>;
+}
diff --git a/server/src/engine/idx/stord.rs b/server/src/engine/idx/stord/mod.rs
similarity index 89%
rename from server/src/engine/idx/stord.rs
rename to server/src/engine/idx/stord/mod.rs
index ffd2e50e..5af20388 100644
--- a/server/src/engine/idx/stord.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -24,25 +24,25 @@
  *
 */
 
+pub(super) mod config;
+
+use self::config::{AllocStrategy, Config};
 use super::{AsKey, AsKeyClone, AsValue, AsValueClone, IndexBaseSpec, STIndex, STIndexSeq};
 use std::{
     alloc::{alloc as std_alloc, dealloc as std_dealloc, Layout},
     borrow::Borrow,
     collections::{
-        hash_map::{Iter, Keys as StdMapIterKey, RandomState, Values as StdMapIterVal},
+        hash_map::{Iter, Keys as StdMapIterKey, Values as StdMapIterVal},
         HashMap as StdMap,
     },
     fmt::{self, Debug},
-    hash::{BuildHasher, Hash, Hasher},
+    hash::{Hash, Hasher},
     iter::FusedIterator,
     marker::PhantomData,
     mem,
     ptr::{self, NonNull},
 };
 
-pub type IndexSTSeqDllDef<K, V> = IndexSTSeqDll<K, V, IndexSTSeqDllHasher>;
-pub type IndexSTSeqDllHasher = RandomState;
-
 /*
     For the ordered index impl, we resort to some crazy unsafe code, especially because there's no other way to
     deal with non-primitive Ks. That's why we'll ENTIRELY AVOID exporting any structures; if we end up using a node
@@ -130,7 +130,7 @@ where
 }
 
 #[derive(Debug)]
-struct IndexSTSeqDllNode<K, V> {
+pub struct IndexSTSeqDllNode<K, V> {
     k: K,
     v: V,
     n: *mut Self,
@@ -210,7 +210,7 @@ impl<K, V> IndexSTSeqDllNode<K, V> {
     }
 }
 
-type IndexSTSeqDllNodePtr<K, V> = NonNull<IndexSTSeqDllNode<K, V>>;
+pub type IndexSTSeqDllNodePtr<K, V> = NonNull<IndexSTSeqDllNode<K, V>>;
 
 #[cfg(debug_assertions)]
 pub struct IndexSTSeqDllMetrics {
@@ -219,65 +219,59 @@ pub struct IndexSTSeqDllMetrics {
 
 #[cfg(debug_assertions)]
 impl IndexSTSeqDllMetrics {
-    pub fn raw_f(&self) -> usize {
+    pub const fn raw_f(&self) -> usize {
         self.stat_f
     }
-    fn new() -> IndexSTSeqDllMetrics {
+    const fn new() -> IndexSTSeqDllMetrics {
         Self { stat_f: 0 }
     }
 }
 
 /// An ST-index with ordering. Inefficient ordered scanning since not in block
-pub struct IndexSTSeqDll<K, V, S> {
-    m: StdMap<IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>, S>,
+pub struct IndexSTSeqDll<K, V, C: Config<K, V>> {
+    m: StdMap<IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>, C::Hasher>,
     h: *mut IndexSTSeqDllNode<K, V>,
-    f: *mut IndexSTSeqDllNode<K, V>,
+    a: C::AllocStrategy,
     #[cfg(debug_assertions)]
     metrics: IndexSTSeqDllMetrics,
 }
 
-impl<K, V, S: BuildHasher> IndexSTSeqDll<K, V, S> {
+impl<K, V, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
     const DEF_CAP: usize = 0;
     #[inline(always)]
-    fn _new(
-        m: StdMap<IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>, S>,
+    const fn _new(
+        m: StdMap<IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>, C::Hasher>,
         h: *mut IndexSTSeqDllNode<K, V>,
-        f: *mut IndexSTSeqDllNode<K, V>,
-    ) -> IndexSTSeqDll<K, V, S> {
+    ) -> IndexSTSeqDll<K, V, C> {
         Self {
             m,
             h,
-            f,
+            a: C::AllocStrategy::NEW,
             #[cfg(debug_assertions)]
             metrics: IndexSTSeqDllMetrics::new(),
         }
     }
     #[inline(always)]
-    fn _new_map(m: StdMap<IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>, S>) -> Self {
-        Self::_new(m, ptr::null_mut(), ptr::null_mut())
+    fn _new_map(m: StdMap<IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>, C::Hasher>) -> Self {
+        Self::_new(m, ptr::null_mut())
     }
     #[inline(always)]
-    pub fn with_hasher(hasher: S) -> Self {
+    pub fn with_hasher(hasher: C::Hasher) -> Self {
         Self::with_capacity_and_hasher(Self::DEF_CAP, hasher)
     }
     #[inline(always)]
-    pub fn with_capacity_and_hasher(cap: usize, hasher: S) -> Self {
+    pub fn with_capacity_and_hasher(cap: usize, hasher: C::Hasher) -> Self {
         Self::_new_map(StdMap::with_capacity_and_hasher(cap, hasher))
     }
-}
-
-impl<K, V> IndexSTSeqDll<K, V, RandomState> {
-    #[inline(always)]
-    pub fn new() -> Self {
-        Self::with_capacity(Self::DEF_CAP)
-    }
-    #[inline(always)]
-    pub fn with_capacity(cap: usize) -> Self {
-        Self::with_capacity_and_hasher(cap, RandomState::default())
+    fn metrics_update_f_empty(&mut self) {
+        #[cfg(debug_assertions)]
+        {
+            self.metrics.stat_f = 0;
+        }
     }
 }
 
-impl<K, V, S> IndexSTSeqDll<K, V, S> {
+impl<K, V, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
     #[inline(always)]
     fn metrics_update_f_decr(&mut self) {
         #[cfg(debug_assertions)]
@@ -318,35 +312,6 @@ impl<K, V, S> IndexSTSeqDll<K, V, S> {
         }
     }
     #[inline(always)]
-    fn vacuum_free(&mut self) {
-        unsafe {
-            // UNSAFE(@ohsayan): All nullck
-            let mut c = self.f;
-            while !c.is_null() {
-                let nx = (*c).n;
-                IndexSTSeqDllNode::dealloc_headless(c);
-                c = nx;
-                self.metrics_update_f_decr();
-            }
-        }
-        self.f = ptr::null_mut();
-    }
-    #[inline(always)]
-    fn recycle_or_alloc(&mut self, node: IndexSTSeqDllNode<K, V>) -> IndexSTSeqDllNodePtr<K, V> {
-        if self.f.is_null() {
-            IndexSTSeqDllNode::alloc_box(node)
-        } else {
-            self.metrics_update_f_decr();
-            unsafe {
-                // UNSAFE(@ohsayan): Safe because we already did a nullptr check
-                let f = self.f;
-                self.f = (*self.f).n;
-                ptr::write(f, node);
-                IndexSTSeqDllNodePtr::new_unchecked(f)
-            }
-        }
-    }
-    #[inline(always)]
     /// NOTE: `&mut Self` for aliasing
     /// ## Safety
     /// Ensure head is non null
@@ -358,7 +323,7 @@ impl<K, V, S> IndexSTSeqDll<K, V, S> {
     }
 }
 
-impl<K, V, S> IndexSTSeqDll<K, V, S> {
+impl<K, V, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
     #[inline(always)]
     fn _iter_unord_kv<'a>(&'a self) -> IndexSTSeqDllIterUnordKV<'a, K, V> {
         IndexSTSeqDllIterUnordKV::new(&self.m)
@@ -391,16 +356,19 @@ impl<K, V, S> IndexSTSeqDll<K, V, S> {
     }
 }
 
-impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
+impl<K: AsKey, V: AsValue, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
     #[inline(always)]
     /// Clean up unused and cached memory
     fn vacuum_full(&mut self) {
         self.m.shrink_to_fit();
-        self.vacuum_free();
+        self.a.cleanup();
+        if C::AllocStrategy::METRIC_REFRESH {
+            self.metrics_update_f_empty();
+        }
     }
 }
 
-impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
+impl<K: AsKey, V: AsValue, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
     const GET_REFRESH: bool = true;
     const GET_BYPASS: bool = false;
     #[inline(always)]
@@ -474,16 +442,23 @@ impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
                 let v = ptr::read(&(*n).v);
                 // UNSAFE(@ohsayan): non-null guaranteed by as_ptr
                 IndexSTSeqDllNode::unlink(n);
-                (*n).n = self.f;
-                self.f = n;
-                self.metrics_update_f_incr();
+                self.a.free(n);
+                if C::AllocStrategy::METRIC_REFRESH {
+                    self.metrics_update_f_incr();
+                }
                 v
             })
     }
     #[inline(always)]
     fn __insert(&mut self, k: K, v: V) -> bool {
         self.ensure_sentinel();
-        let node = self.recycle_or_alloc(IndexSTSeqDllNode::new_null(k, v));
+        let mut refresh = false;
+        let node = self
+            .a
+            .alloc(IndexSTSeqDllNode::new_null(k, v), &mut refresh);
+        if C::AllocStrategy::METRIC_REFRESH & cfg!(debug_assertions) & refresh {
+            self.metrics_update_f_decr();
+        }
         let kptr = unsafe {
             // UNSAFE(@ohsayan): All g, we allocated it rn
             IndexSTSeqDllKeyptr::new(&node.as_ref().k)
@@ -532,7 +507,7 @@ impl<K: AsKey, V: AsValue, S: BuildHasher> IndexSTSeqDll<K, V, S> {
     }
 }
 
-impl<K, V, S> Drop for IndexSTSeqDll<K, V, S> {
+impl<K, V, C: Config<K, V>> Drop for IndexSTSeqDll<K, V, C> {
     fn drop(&mut self) {
         if !self.h.is_null() {
             unsafe {
@@ -542,29 +517,27 @@ impl<K, V, S> Drop for IndexSTSeqDll<K, V, S> {
                 IndexSTSeqDllNode::dealloc_headless(self.h);
             }
         }
-        self.vacuum_free();
+        self.a.cleanup();
     }
 }
 
-impl<K: AsKey, V: AsValue, S: BuildHasher + Default> FromIterator<(K, V)>
-    for IndexSTSeqDll<K, V, S>
-{
+impl<K: AsKey, V: AsValue, C: Config<K, V>> FromIterator<(K, V)> for IndexSTSeqDll<K, V, C> {
     fn from_iter<T: IntoIterator<Item = (K, V)>>(iter: T) -> Self {
-        let mut slf = Self::with_hasher(S::default());
+        let mut slf = Self::with_hasher(C::Hasher::default());
         iter.into_iter()
             .for_each(|(k, v)| assert!(slf._insert(k, v)));
         slf
     }
 }
 
-impl<K, V, S: BuildHasher + Default> IndexBaseSpec<K, V> for IndexSTSeqDll<K, V, S> {
+impl<K, V, C: Config<K, V>> IndexBaseSpec<K, V> for IndexSTSeqDll<K, V, C> {
     const PREALLOC: bool = true;
 
     #[cfg(debug_assertions)]
     type Metrics = IndexSTSeqDllMetrics;
 
     fn idx_init() -> Self {
-        Self::with_hasher(S::default())
+        Self::with_hasher(C::Hasher::default())
     }
 
     fn idx_init_with(s: Self) -> Self {
@@ -572,7 +545,7 @@ impl<K, V, S: BuildHasher + Default> IndexBaseSpec<K, V> for IndexSTSeqDll<K, V,
     }
 
     fn idx_init_cap(cap: usize) -> Self {
-        Self::with_capacity_and_hasher(cap, S::default())
+        Self::with_capacity_and_hasher(cap, C::Hasher::default())
     }
 
     #[cfg(debug_assertions)]
@@ -581,7 +554,7 @@ impl<K, V, S: BuildHasher + Default> IndexBaseSpec<K, V> for IndexSTSeqDll<K, V,
     }
 }
 
-impl<K, V, S: BuildHasher + Default> STIndex<K, V> for IndexSTSeqDll<K, V, S>
+impl<K, V, C: Config<K, V>> STIndex<K, V> for IndexSTSeqDll<K, V, C>
 where
     K: AsKey,
     V: AsValue,
@@ -691,11 +664,11 @@ where
     }
 }
 
-impl<K, V, S> STIndexSeq<K, V> for IndexSTSeqDll<K, V, S>
+impl<K, V, C> STIndexSeq<K, V> for IndexSTSeqDll<K, V, C>
 where
     K: AsKey,
     V: AsValue,
-    S: BuildHasher + Default,
+    C: Config<K, V>,
 {
     type IterOrdKV<'a> = IndexSTSeqDllIterOrdKV<'a, K, V>
     where
@@ -726,9 +699,9 @@ where
     }
 }
 
-impl<K: AsKeyClone, V: AsValueClone, S: BuildHasher + Default> Clone for IndexSTSeqDll<K, V, S> {
+impl<K: AsKeyClone, V: AsValueClone, C: Config<K, V>> Clone for IndexSTSeqDll<K, V, C> {
     fn clone(&self) -> Self {
-        let mut slf = Self::with_capacity_and_hasher(self.len(), S::default());
+        let mut slf = Self::with_capacity_and_hasher(self.len(), C::Hasher::default());
         self._iter_ord_kv()
             .map(|(k, v)| (k.clone(), v.clone()))
             .for_each(|(k, v)| {
@@ -738,8 +711,8 @@ impl<K: AsKeyClone, V: AsValueClone, S: BuildHasher + Default> Clone for IndexST
     }
 }
 
-unsafe impl<K: Send, V: Send, S: Send> Send for IndexSTSeqDll<K, V, S> {}
-unsafe impl<K: Sync, V: Sync, S: Sync> Sync for IndexSTSeqDll<K, V, S> {}
+unsafe impl<K: Send, V: Send, C: Config<K, V> + Send> Send for IndexSTSeqDll<K, V, C> {}
+unsafe impl<K: Sync, V: Sync, C: Config<K, V> + Sync> Sync for IndexSTSeqDll<K, V, C> {}
 
 macro_rules! unsafe_marker_impl {
     (unsafe impl for $ty:ty) => {
@@ -967,7 +940,7 @@ struct IndexSTSeqDllIterOrdBase<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<
 
 impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> IndexSTSeqDllIterOrdBase<'a, K, V, C> {
     #[inline(always)]
-    fn new<S>(idx: &'a IndexSTSeqDll<K, V, S>) -> Self {
+    fn new<Mc: Config<K, V>>(idx: &'a IndexSTSeqDll<K, V, Mc>) -> Self {
         Self {
             h: if idx.h.is_null() {
                 ptr::null_mut()
diff --git a/server/src/engine/idx/tests.rs b/server/src/engine/idx/tests.rs
index 3cf57c03..658b10e3 100644
--- a/server/src/engine/idx/tests.rs
+++ b/server/src/engine/idx/tests.rs
@@ -27,16 +27,23 @@
 use super::*;
 
 mod idx_st_seq_dll {
-    use super::{stord::IndexSTSeqDllDef, IndexBaseSpec, STIndex, STIndexSeq};
+    use super::{IndexBaseSpec, IndexSTSeqLib, STIndex, STIndexSeq};
     use rand::{distributions::Alphanumeric, Rng};
 
+    #[cfg(not(miri))]
     const SPAM_CNT: usize = 131_072;
+    #[cfg(miri)]
+    const SPAM_CNT: usize = 128;
+    #[cfg(not(miri))]
     const SPAM_SIZE: usize = 128;
-    type Index = IndexSTSeqDllDef<String, String>;
+    #[cfg(miri)]
+    const SPAM_SIZE: usize = 4;
+
+    type Index = IndexSTSeqLib<String, String>;
 
     /// Returns an index with: `i -> "{i+1}"` starting from 0 upto the value of [`SPAM_CNT`]
-    fn mkidx() -> IndexSTSeqDllDef<usize, String> {
-        let mut idx = IndexSTSeqDllDef::idx_init();
+    fn mkidx() -> IndexSTSeqLib<usize, String> {
+        let mut idx = IndexSTSeqLib::idx_init();
         for int in 0..SPAM_CNT {
             assert!(idx.st_insert(int, (int + 1).to_string()));
         }
@@ -66,7 +73,7 @@ mod idx_st_seq_dll {
     }
     #[test]
     fn spam_read_nx() {
-        let idx = IndexSTSeqDllDef::<usize, String>::new();
+        let idx = IndexSTSeqLib::<usize, String>::idx_init();
         for int in SPAM_CNT..SPAM_CNT * 2 {
             assert!(idx.st_get(&int).is_none());
         }
@@ -80,14 +87,14 @@ mod idx_st_seq_dll {
     }
     #[test]
     fn spam_update_nx() {
-        let mut idx = IndexSTSeqDllDef::<usize, String>::new();
+        let mut idx = IndexSTSeqLib::<usize, String>::idx_init();
         for int in 0..SPAM_CNT {
             assert!(!idx.st_update(&int, (int + 2).to_string()));
         }
     }
     #[test]
     fn spam_delete_nx() {
-        let mut idx = IndexSTSeqDllDef::<usize, String>::new();
+        let mut idx = IndexSTSeqLib::<usize, String>::idx_init();
         for int in 0..SPAM_CNT {
             assert!(!idx.st_delete(&int));
         }
@@ -104,7 +111,7 @@ mod idx_st_seq_dll {
     }
     #[test]
     fn spam_crud() {
-        let mut idx = IndexSTSeqDllDef::idx_init();
+        let mut idx = IndexSTSeqLib::idx_init();
         for int in 0..SPAM_CNT {
             assert!(idx.st_insert(int, int + 1));
             assert_eq!(*idx.st_get(&int).unwrap(), int + 1);
@@ -116,7 +123,7 @@ mod idx_st_seq_dll {
     }
     #[test]
     fn spam_read() {
-        let mut idx = IndexSTSeqDllDef::idx_init();
+        let mut idx = IndexSTSeqLib::idx_init();
         for int in 0..SPAM_CNT {
             let v = (int + 1).to_string();
             assert!(idx.st_insert(int, v.clone()));

From 427ff72e42c45ab1fa7b0d9ddfeacc045d5fa579 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 29 Jan 2023 22:39:49 -0800
Subject: [PATCH 109/310] Move iter into module for st seq idx

---
 server/src/engine/idx/stord/iter.rs | 491 ++++++++++++++++++++++++++++
 server/src/engine/idx/stord/mod.rs  | 457 +-------------------------
 2 files changed, 504 insertions(+), 444 deletions(-)
 create mode 100644 server/src/engine/idx/stord/iter.rs

diff --git a/server/src/engine/idx/stord/iter.rs b/server/src/engine/idx/stord/iter.rs
new file mode 100644
index 00000000..237d07c5
--- /dev/null
+++ b/server/src/engine/idx/stord/iter.rs
@@ -0,0 +1,491 @@
+/*
+ * Created on Sun Jan 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::{
+    config::Config, IndexSTSeqDll, IndexSTSeqDllKeyptr, IndexSTSeqDllNode, IndexSTSeqDllNodePtr,
+};
+use std::{
+    collections::{
+        hash_map::{Iter, Keys as StdMapIterKey, Values as StdMapIterVal},
+        HashMap as StdMap,
+    },
+    fmt::{self, Debug},
+    iter::FusedIterator,
+    marker::PhantomData,
+    ptr::{self, NonNull},
+};
+
+macro_rules! unsafe_marker_impl {
+    (unsafe impl for $ty:ty) => {
+        unsafe impl<'a, K: Send, V: Send> Send for $ty {}
+        unsafe impl<'a, K: Sync, V: Sync> Sync for $ty {}
+    };
+}
+
+pub struct IndexSTSeqDllIterUnordKV<'a, K: 'a, V: 'a> {
+    i: Iter<'a, IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>>,
+}
+
+// UNSAFE(@ohsayan): aliasing guarantees correctness
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordKV<'a, K, V>);
+
+impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordKV<'a, K, V> {
+    #[inline(always)]
+    pub(super) fn new<S>(
+        m: &'a StdMap<IndexSTSeqDllKeyptr<K>, NonNull<IndexSTSeqDllNode<K, V>>, S>,
+    ) -> Self {
+        Self { i: m.iter() }
+    }
+}
+
+impl<'a, K, V> Clone for IndexSTSeqDllIterUnordKV<'a, K, V> {
+    fn clone(&self) -> Self {
+        Self { i: self.i.clone() }
+    }
+}
+
+impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordKV<'a, K, V> {
+    type Item = (&'a K, &'a V);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.i.next().map(|(_, n)| {
+            let n = n.as_ptr();
+            unsafe {
+                // UNSAFE(@ohsayan): nullck
+                (&(*n).k, &(*n).v)
+            }
+        })
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        <_ as Iterator>::size_hint(&self.i)
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordKV<'a, K, V> {
+    fn len(&self) -> usize {
+        self.i.len()
+    }
+}
+
+impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordKV<'a, K, V> {}
+
+impl<'a, K: 'a + Debug, V: 'a + Debug> Debug for IndexSTSeqDllIterUnordKV<'a, K, V> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.clone()).finish()
+    }
+}
+
+pub struct IndexSTSeqDllIterUnordKey<'a, K: 'a, V: 'a> {
+    k: StdMapIterKey<'a, IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>>,
+}
+
+// UNSAFE(@ohsayan): aliasing guarantees correctness
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordKey<'a, K, V>);
+
+impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordKey<'a, K, V> {
+    #[inline(always)]
+    pub(super) fn new<S>(
+        m: &'a StdMap<IndexSTSeqDllKeyptr<K>, NonNull<IndexSTSeqDllNode<K, V>>, S>,
+    ) -> Self {
+        Self { k: m.keys() }
+    }
+}
+
+impl<'a, K, V> Clone for IndexSTSeqDllIterUnordKey<'a, K, V> {
+    fn clone(&self) -> Self {
+        Self { k: self.k.clone() }
+    }
+}
+
+impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordKey<'a, K, V> {
+    type Item = &'a K;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.k.next().map(|k| {
+            unsafe {
+                // UNSAFE(@ohsayan): nullck
+                &*(*k).p
+            }
+        })
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        <_ as Iterator>::size_hint(&self.k)
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordKey<'a, K, V> {
+    fn len(&self) -> usize {
+        self.k.len()
+    }
+}
+
+impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordKey<'a, K, V> {}
+
+impl<'a, K: Debug, V> Debug for IndexSTSeqDllIterUnordKey<'a, K, V> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.clone()).finish()
+    }
+}
+
+pub struct IndexSTSeqDllIterUnordValue<'a, K: 'a, V: 'a> {
+    v: StdMapIterVal<'a, IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>>,
+}
+
+// UNSAFE(@ohsayan): aliasing guarantees correctness
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordValue<'a, K, V>);
+
+impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordValue<'a, K, V> {
+    #[inline(always)]
+    pub(super) fn new<S>(
+        m: &'a StdMap<IndexSTSeqDllKeyptr<K>, NonNull<IndexSTSeqDllNode<K, V>>, S>,
+    ) -> Self {
+        Self { v: m.values() }
+    }
+}
+
+impl<'a, K, V> Clone for IndexSTSeqDllIterUnordValue<'a, K, V> {
+    fn clone(&self) -> Self {
+        Self { v: self.v.clone() }
+    }
+}
+
+impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordValue<'a, K, V> {
+    type Item = &'a V;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.v.next().map(|n| {
+            unsafe {
+                // UNSAFE(@ohsayan): nullck
+                &(*n.as_ptr()).v
+            }
+        })
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        <_ as Iterator>::size_hint(&self.v)
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordValue<'a, K, V> {
+    fn len(&self) -> usize {
+        self.v.len()
+    }
+}
+
+impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordValue<'a, K, V> {}
+
+impl<'a, K, V: Debug> Debug for IndexSTSeqDllIterUnordValue<'a, K, V> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.clone()).finish()
+    }
+}
+
+trait IndexSTSeqDllIterOrdConfig<K, V> {
+    type Ret<'a>
+    where
+        K: 'a,
+        V: 'a;
+    /// ## Safety
+    /// Ptr must be non-null
+    unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode<K, V>) -> Option<Self::Ret<'a>>
+    where
+        K: 'a,
+        V: 'a;
+}
+
+struct IndexSTSeqDllIterOrdConfigFull;
+
+impl<K, V> IndexSTSeqDllIterOrdConfig<K, V> for IndexSTSeqDllIterOrdConfigFull {
+    type Ret<'a> = (&'a K, &'a V) where K: 'a, V: 'a;
+    #[inline(always)]
+    unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode<K, V>) -> Option<Self::Ret<'a>>
+    where
+        K: 'a,
+        V: 'a,
+    {
+        Some((&(*ptr).k, &(*ptr).v))
+    }
+}
+
+struct IndexSTSeqDllIterOrdConfigKey;
+
+impl<K, V> IndexSTSeqDllIterOrdConfig<K, V> for IndexSTSeqDllIterOrdConfigKey {
+    type Ret<'a> = &'a K
+    where
+        K: 'a,
+        V: 'a;
+    #[inline(always)]
+    unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode<K, V>) -> Option<&'a K>
+    where
+        K: 'a,
+        V: 'a,
+    {
+        Some(&(*ptr).k)
+    }
+}
+
+struct IndexSTSeqDllIterOrdConfigValue;
+
+impl<K, V> IndexSTSeqDllIterOrdConfig<K, V> for IndexSTSeqDllIterOrdConfigValue {
+    type Ret<'a> = &'a V
+    where
+        K: 'a,
+        V: 'a;
+    #[inline(always)]
+    unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode<K, V>) -> Option<&'a V>
+    where
+        K: 'a,
+        V: 'a,
+    {
+        Some(&(*ptr).v)
+    }
+}
+
+struct IndexSTSeqDllIterOrdBase<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> {
+    h: *const IndexSTSeqDllNode<K, V>,
+    t: *const IndexSTSeqDllNode<K, V>,
+    r: usize,
+    _l: PhantomData<(&'a K, &'a V, C)>,
+}
+
+impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> IndexSTSeqDllIterOrdBase<'a, K, V, C> {
+    #[inline(always)]
+    fn new<Mc: Config<K, V>>(idx: &'a IndexSTSeqDll<K, V, Mc>) -> Self {
+        Self {
+            h: if idx.h.is_null() {
+                ptr::null_mut()
+            } else {
+                unsafe {
+                    // UNSAFE(@ohsayan): nullck
+                    (*idx.h).p
+                }
+            },
+            t: idx.h,
+            r: idx.len(),
+            _l: PhantomData,
+        }
+    }
+    #[inline(always)]
+    fn _next(&mut self) -> Option<C::Ret<'a>> {
+        if self.h == self.t {
+            None
+        } else {
+            self.r -= 1;
+            unsafe {
+                // UNSAFE(@ohsayan): Assuming we had a legal init, this should be fine
+                let this = C::read_ret(self.h);
+                self.h = (*self.h).p;
+                this
+            }
+        }
+    }
+    #[inline(always)]
+    fn _next_back(&mut self) -> Option<C::Ret<'a>> {
+        if self.h == self.t {
+            None
+        } else {
+            self.r -= 1;
+            unsafe {
+                // UNSAFE(@ohsayan): legal init, then ok
+                self.t = (*self.t).n;
+                // UNSAFE(@ohsayan): non-null (sentinel)
+                C::read_ret(self.t)
+            }
+        }
+    }
+}
+
+impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> Debug
+    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
+where
+    C::Ret<'a>: Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.clone()).finish()
+    }
+}
+
+impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> Iterator
+    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
+{
+    type Item = C::Ret<'a>;
+    fn next(&mut self) -> Option<Self::Item> {
+        self._next()
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (self.r, Some(self.r))
+    }
+}
+
+impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> DoubleEndedIterator
+    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
+{
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self._next_back()
+    }
+}
+
+impl<'a, K, V, C: IndexSTSeqDllIterOrdConfig<K, V>> ExactSizeIterator
+    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
+{
+    fn len(&self) -> usize {
+        self.r
+    }
+}
+
+impl<'a, K, V, C: IndexSTSeqDllIterOrdConfig<K, V>> Clone
+    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
+{
+    fn clone(&self) -> Self {
+        Self { ..*self }
+    }
+}
+
+#[derive(Debug)]
+pub struct IndexSTSeqDllIterOrdKV<'a, K: 'a, V: 'a> {
+    i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigFull>,
+}
+impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterOrdKV<'a, K, V> {
+    pub(super) fn new<C: Config<K, V>>(arg: &'a IndexSTSeqDll<K, V, C>) -> Self {
+        Self {
+            i: IndexSTSeqDllIterOrdBase::new(arg),
+        }
+    }
+}
+
+// UNSAFE(@ohsayan): aliasing guarantees correctness
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdKV<'a, K, V>);
+
+impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdKV<'a, K, V> {
+    type Item = (&'a K, &'a V);
+    fn next(&mut self) -> Option<Self::Item> {
+        self.i.next()
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.i.size_hint()
+    }
+}
+
+impl<'a, K: 'a, V: 'a> DoubleEndedIterator for IndexSTSeqDllIterOrdKV<'a, K, V> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.i.next_back()
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdKV<'a, K, V> {
+    fn len(&self) -> usize {
+        self.i.len()
+    }
+}
+
+impl<'a, K, V> Clone for IndexSTSeqDllIterOrdKV<'a, K, V> {
+    fn clone(&self) -> Self {
+        Self { i: self.i.clone() }
+    }
+}
+
+#[derive(Debug)]
+pub struct IndexSTSeqDllIterOrdKey<'a, K: 'a, V: 'a> {
+    i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigKey>,
+}
+impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterOrdKey<'a, K, V> {
+    pub(super) fn new<C: Config<K, V>>(arg: &'a IndexSTSeqDll<K, V, C>) -> Self {
+        Self {
+            i: IndexSTSeqDllIterOrdBase::new(arg),
+        }
+    }
+}
+
+// UNSAFE(@ohsayan): aliasing guarantees correctness
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdKey<'a, K, V>);
+
+impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdKey<'a, K, V> {
+    type Item = &'a K;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.i.next()
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.i.size_hint()
+    }
+}
+
+impl<'a, K: 'a, V: 'a> DoubleEndedIterator for IndexSTSeqDllIterOrdKey<'a, K, V> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.i.next_back()
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdKey<'a, K, V> {
+    fn len(&self) -> usize {
+        self.i.len()
+    }
+}
+
+impl<'a, K, V> Clone for IndexSTSeqDllIterOrdKey<'a, K, V> {
+    fn clone(&self) -> Self {
+        Self { i: self.i.clone() }
+    }
+}
+
+#[derive(Debug)]
+pub struct IndexSTSeqDllIterOrdValue<'a, K: 'a, V: 'a> {
+    i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigValue>,
+}
+impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterOrdValue<'a, K, V> {
+    pub(super) fn new<C: Config<K, V>>(arg: &'a IndexSTSeqDll<K, V, C>) -> Self {
+        Self {
+            i: IndexSTSeqDllIterOrdBase::new(arg),
+        }
+    }
+}
+
+// UNSAFE(@ohsayan): aliasing guarantees correctness
+unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdValue<'a, K, V>);
+
+impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdValue<'a, K, V> {
+    type Item = &'a V;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.i.next()
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.i.size_hint()
+    }
+}
+
+impl<'a, K: 'a, V: 'a> DoubleEndedIterator for IndexSTSeqDllIterOrdValue<'a, K, V> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.i.next_back()
+    }
+}
+
+impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdValue<'a, K, V> {
+    fn len(&self) -> usize {
+        self.i.len()
+    }
+}
+
+impl<'a, K, V> Clone for IndexSTSeqDllIterOrdValue<'a, K, V> {
+    fn clone(&self) -> Self {
+        Self { i: self.i.clone() }
+    }
+}
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index 5af20388..a5aecb97 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -25,20 +25,22 @@
 */
 
 pub(super) mod config;
+mod iter;
 
-use self::config::{AllocStrategy, Config};
+use self::{
+    config::{AllocStrategy, Config},
+    iter::{
+        IndexSTSeqDllIterOrdKV, IndexSTSeqDllIterOrdKey, IndexSTSeqDllIterOrdValue,
+        IndexSTSeqDllIterUnordKV, IndexSTSeqDllIterUnordKey, IndexSTSeqDllIterUnordValue,
+    },
+};
 use super::{AsKey, AsKeyClone, AsValue, AsValueClone, IndexBaseSpec, STIndex, STIndexSeq};
 use std::{
     alloc::{alloc as std_alloc, dealloc as std_dealloc, Layout},
     borrow::Borrow,
-    collections::{
-        hash_map::{Iter, Keys as StdMapIterKey, Values as StdMapIterVal},
-        HashMap as StdMap,
-    },
-    fmt::{self, Debug},
+    collections::HashMap as StdMap,
+    fmt::Debug,
     hash::{Hash, Hasher},
-    iter::FusedIterator,
-    marker::PhantomData,
     mem,
     ptr::{self, NonNull},
 };
@@ -338,21 +340,15 @@ impl<K, V, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
     }
     #[inline(always)]
     fn _iter_ord_kv<'a>(&'a self) -> IndexSTSeqDllIterOrdKV<'a, K, V> {
-        IndexSTSeqDllIterOrdKV {
-            i: IndexSTSeqDllIterOrdBase::new(self),
-        }
+        IndexSTSeqDllIterOrdKV::new(self)
     }
     #[inline(always)]
     fn _iter_ord_k<'a>(&'a self) -> IndexSTSeqDllIterOrdKey<'a, K, V> {
-        IndexSTSeqDllIterOrdKey {
-            i: IndexSTSeqDllIterOrdBase::new(self),
-        }
+        IndexSTSeqDllIterOrdKey::new(self)
     }
     #[inline(always)]
     fn _iter_ord_v<'a>(&'a self) -> IndexSTSeqDllIterOrdValue<'a, K, V> {
-        IndexSTSeqDllIterOrdValue {
-            i: IndexSTSeqDllIterOrdBase::new(self),
-        }
+        IndexSTSeqDllIterOrdValue::new(self)
     }
 }
 
@@ -369,8 +365,6 @@ impl<K: AsKey, V: AsValue, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
 }
 
 impl<K: AsKey, V: AsValue, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
-    const GET_REFRESH: bool = true;
-    const GET_BYPASS: bool = false;
     #[inline(always)]
     fn _insert(&mut self, k: K, v: V) -> bool {
         if self.m.contains_key(&IndexSTSeqDllKeyptr::new(&k)) {
@@ -713,428 +707,3 @@ impl<K: AsKeyClone, V: AsValueClone, C: Config<K, V>> Clone for IndexSTSeqDll<K,
 
 unsafe impl<K: Send, V: Send, C: Config<K, V> + Send> Send for IndexSTSeqDll<K, V, C> {}
 unsafe impl<K: Sync, V: Sync, C: Config<K, V> + Sync> Sync for IndexSTSeqDll<K, V, C> {}
-
-macro_rules! unsafe_marker_impl {
-    (unsafe impl for $ty:ty) => {
-        unsafe impl<'a, K: Send, V: Send> Send for $ty {}
-        unsafe impl<'a, K: Sync, V: Sync> Sync for $ty {}
-    };
-}
-
-pub struct IndexSTSeqDllIterUnordKV<'a, K: 'a, V: 'a> {
-    i: Iter<'a, IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>>,
-}
-
-// UNSAFE(@ohsayan): aliasing guarantees correctness
-unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordKV<'a, K, V>);
-
-impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordKV<'a, K, V> {
-    #[inline(always)]
-    fn new<S>(m: &'a StdMap<IndexSTSeqDllKeyptr<K>, NonNull<IndexSTSeqDllNode<K, V>>, S>) -> Self {
-        Self { i: m.iter() }
-    }
-}
-
-impl<'a, K, V> Clone for IndexSTSeqDllIterUnordKV<'a, K, V> {
-    fn clone(&self) -> Self {
-        Self { i: self.i.clone() }
-    }
-}
-
-impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordKV<'a, K, V> {
-    type Item = (&'a K, &'a V);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.i.next().map(|(_, n)| {
-            let n = n.as_ptr();
-            unsafe {
-                // UNSAFE(@ohsayan): nullck
-                (&(*n).k, &(*n).v)
-            }
-        })
-    }
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        <_ as Iterator>::size_hint(&self.i)
-    }
-}
-
-impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordKV<'a, K, V> {
-    fn len(&self) -> usize {
-        self.i.len()
-    }
-}
-
-impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordKV<'a, K, V> {}
-
-impl<'a, K: 'a + Debug, V: 'a + Debug> Debug for IndexSTSeqDllIterUnordKV<'a, K, V> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_list().entries(self.clone()).finish()
-    }
-}
-
-pub struct IndexSTSeqDllIterUnordKey<'a, K: 'a, V: 'a> {
-    k: StdMapIterKey<'a, IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>>,
-}
-
-// UNSAFE(@ohsayan): aliasing guarantees correctness
-unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordKey<'a, K, V>);
-
-impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordKey<'a, K, V> {
-    #[inline(always)]
-    fn new<S>(m: &'a StdMap<IndexSTSeqDllKeyptr<K>, NonNull<IndexSTSeqDllNode<K, V>>, S>) -> Self {
-        Self { k: m.keys() }
-    }
-}
-
-impl<'a, K, V> Clone for IndexSTSeqDllIterUnordKey<'a, K, V> {
-    fn clone(&self) -> Self {
-        Self { k: self.k.clone() }
-    }
-}
-
-impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordKey<'a, K, V> {
-    type Item = &'a K;
-    fn next(&mut self) -> Option<Self::Item> {
-        self.k.next().map(|k| {
-            unsafe {
-                // UNSAFE(@ohsayan): nullck
-                &*(*k).p
-            }
-        })
-    }
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        <_ as Iterator>::size_hint(&self.k)
-    }
-}
-
-impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordKey<'a, K, V> {
-    fn len(&self) -> usize {
-        self.k.len()
-    }
-}
-
-impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordKey<'a, K, V> {}
-
-impl<'a, K: Debug, V> Debug for IndexSTSeqDllIterUnordKey<'a, K, V> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_list().entries(self.clone()).finish()
-    }
-}
-
-pub struct IndexSTSeqDllIterUnordValue<'a, K: 'a, V: 'a> {
-    v: StdMapIterVal<'a, IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>>,
-}
-
-// UNSAFE(@ohsayan): aliasing guarantees correctness
-unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterUnordValue<'a, K, V>);
-
-impl<'a, K: 'a, V: 'a> IndexSTSeqDllIterUnordValue<'a, K, V> {
-    #[inline(always)]
-    fn new<S>(m: &'a StdMap<IndexSTSeqDllKeyptr<K>, NonNull<IndexSTSeqDllNode<K, V>>, S>) -> Self {
-        Self { v: m.values() }
-    }
-}
-
-impl<'a, K, V> Clone for IndexSTSeqDllIterUnordValue<'a, K, V> {
-    fn clone(&self) -> Self {
-        Self { v: self.v.clone() }
-    }
-}
-
-impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordValue<'a, K, V> {
-    type Item = &'a V;
-    fn next(&mut self) -> Option<Self::Item> {
-        self.v.next().map(|n| {
-            unsafe {
-                // UNSAFE(@ohsayan): nullck
-                &(*n.as_ptr()).v
-            }
-        })
-    }
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        <_ as Iterator>::size_hint(&self.v)
-    }
-}
-
-impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterUnordValue<'a, K, V> {
-    fn len(&self) -> usize {
-        self.v.len()
-    }
-}
-
-impl<'a, K, V> FusedIterator for IndexSTSeqDllIterUnordValue<'a, K, V> {}
-
-impl<'a, K, V: Debug> Debug for IndexSTSeqDllIterUnordValue<'a, K, V> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_list().entries(self.clone()).finish()
-    }
-}
-
-trait IndexSTSeqDllIterOrdConfig<K, V> {
-    type Ret<'a>
-    where
-        K: 'a,
-        V: 'a;
-    /// ## Safety
-    /// Ptr must be non-null
-    unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode<K, V>) -> Option<Self::Ret<'a>>
-    where
-        K: 'a,
-        V: 'a;
-}
-
-struct IndexSTSeqDllIterOrdConfigFull;
-
-impl<K, V> IndexSTSeqDllIterOrdConfig<K, V> for IndexSTSeqDllIterOrdConfigFull {
-    type Ret<'a> = (&'a K, &'a V) where K: 'a, V: 'a;
-    #[inline(always)]
-    unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode<K, V>) -> Option<Self::Ret<'a>>
-    where
-        K: 'a,
-        V: 'a,
-    {
-        Some((&(*ptr).k, &(*ptr).v))
-    }
-}
-
-struct IndexSTSeqDllIterOrdConfigKey;
-
-impl<K, V> IndexSTSeqDllIterOrdConfig<K, V> for IndexSTSeqDllIterOrdConfigKey {
-    type Ret<'a> = &'a K
-    where
-        K: 'a,
-        V: 'a;
-    #[inline(always)]
-    unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode<K, V>) -> Option<&'a K>
-    where
-        K: 'a,
-        V: 'a,
-    {
-        Some(&(*ptr).k)
-    }
-}
-
-struct IndexSTSeqDllIterOrdConfigValue;
-
-impl<K, V> IndexSTSeqDllIterOrdConfig<K, V> for IndexSTSeqDllIterOrdConfigValue {
-    type Ret<'a> = &'a V
-    where
-        K: 'a,
-        V: 'a;
-    #[inline(always)]
-    unsafe fn read_ret<'a>(ptr: *const IndexSTSeqDllNode<K, V>) -> Option<&'a V>
-    where
-        K: 'a,
-        V: 'a,
-    {
-        Some(&(*ptr).v)
-    }
-}
-
-struct IndexSTSeqDllIterOrdBase<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> {
-    h: *const IndexSTSeqDllNode<K, V>,
-    t: *const IndexSTSeqDllNode<K, V>,
-    r: usize,
-    _l: PhantomData<(&'a K, &'a V, C)>,
-}
-
-impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> IndexSTSeqDllIterOrdBase<'a, K, V, C> {
-    #[inline(always)]
-    fn new<Mc: Config<K, V>>(idx: &'a IndexSTSeqDll<K, V, Mc>) -> Self {
-        Self {
-            h: if idx.h.is_null() {
-                ptr::null_mut()
-            } else {
-                unsafe {
-                    // UNSAFE(@ohsayan): nullck
-                    (*idx.h).p
-                }
-            },
-            t: idx.h,
-            r: idx.len(),
-            _l: PhantomData,
-        }
-    }
-    #[inline(always)]
-    fn _next(&mut self) -> Option<C::Ret<'a>> {
-        if self.h == self.t {
-            None
-        } else {
-            self.r -= 1;
-            unsafe {
-                // UNSAFE(@ohsayan): Assuming we had a legal init, this should be fine
-                let this = C::read_ret(self.h);
-                self.h = (*self.h).p;
-                this
-            }
-        }
-    }
-    #[inline(always)]
-    fn _next_back(&mut self) -> Option<C::Ret<'a>> {
-        if self.h == self.t {
-            None
-        } else {
-            self.r -= 1;
-            unsafe {
-                // UNSAFE(@ohsayan): legal init, then ok
-                self.t = (*self.t).n;
-                // UNSAFE(@ohsayan): non-null (sentinel)
-                C::read_ret(self.t)
-            }
-        }
-    }
-}
-
-impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> Debug
-    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
-where
-    C::Ret<'a>: Debug,
-{
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_list().entries(self.clone()).finish()
-    }
-}
-
-impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> Iterator
-    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
-{
-    type Item = C::Ret<'a>;
-    fn next(&mut self) -> Option<Self::Item> {
-        self._next()
-    }
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        (self.r, Some(self.r))
-    }
-}
-
-impl<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> DoubleEndedIterator
-    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
-{
-    fn next_back(&mut self) -> Option<Self::Item> {
-        self._next_back()
-    }
-}
-
-impl<'a, K, V, C: IndexSTSeqDllIterOrdConfig<K, V>> ExactSizeIterator
-    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
-{
-    fn len(&self) -> usize {
-        self.r
-    }
-}
-
-impl<'a, K, V, C: IndexSTSeqDllIterOrdConfig<K, V>> Clone
-    for IndexSTSeqDllIterOrdBase<'a, K, V, C>
-{
-    fn clone(&self) -> Self {
-        Self { ..*self }
-    }
-}
-
-#[derive(Debug)]
-pub struct IndexSTSeqDllIterOrdKV<'a, K: 'a, V: 'a> {
-    i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigFull>,
-}
-
-// UNSAFE(@ohsayan): aliasing guarantees correctness
-unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdKV<'a, K, V>);
-
-impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdKV<'a, K, V> {
-    type Item = (&'a K, &'a V);
-    fn next(&mut self) -> Option<Self::Item> {
-        self.i.next()
-    }
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        self.i.size_hint()
-    }
-}
-
-impl<'a, K: 'a, V: 'a> DoubleEndedIterator for IndexSTSeqDllIterOrdKV<'a, K, V> {
-    fn next_back(&mut self) -> Option<Self::Item> {
-        self.i.next_back()
-    }
-}
-
-impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdKV<'a, K, V> {
-    fn len(&self) -> usize {
-        self.i.len()
-    }
-}
-
-impl<'a, K, V> Clone for IndexSTSeqDllIterOrdKV<'a, K, V> {
-    fn clone(&self) -> Self {
-        Self { i: self.i.clone() }
-    }
-}
-
-#[derive(Debug)]
-pub struct IndexSTSeqDllIterOrdKey<'a, K: 'a, V: 'a> {
-    i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigKey>,
-}
-
-// UNSAFE(@ohsayan): aliasing guarantees correctness
-unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdKey<'a, K, V>);
-
-impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdKey<'a, K, V> {
-    type Item = &'a K;
-    fn next(&mut self) -> Option<Self::Item> {
-        self.i.next()
-    }
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        self.i.size_hint()
-    }
-}
-
-impl<'a, K: 'a, V: 'a> DoubleEndedIterator for IndexSTSeqDllIterOrdKey<'a, K, V> {
-    fn next_back(&mut self) -> Option<Self::Item> {
-        self.i.next_back()
-    }
-}
-
-impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdKey<'a, K, V> {
-    fn len(&self) -> usize {
-        self.i.len()
-    }
-}
-
-impl<'a, K, V> Clone for IndexSTSeqDllIterOrdKey<'a, K, V> {
-    fn clone(&self) -> Self {
-        Self { i: self.i.clone() }
-    }
-}
-
-#[derive(Debug)]
-pub struct IndexSTSeqDllIterOrdValue<'a, K: 'a, V: 'a> {
-    i: IndexSTSeqDllIterOrdBase<'a, K, V, IndexSTSeqDllIterOrdConfigValue>,
-}
-
-// UNSAFE(@ohsayan): aliasing guarantees correctness
-unsafe_marker_impl!(unsafe impl for IndexSTSeqDllIterOrdValue<'a, K, V>);
-
-impl<'a, K: 'a, V: 'a> Iterator for IndexSTSeqDllIterOrdValue<'a, K, V> {
-    type Item = &'a V;
-    fn next(&mut self) -> Option<Self::Item> {
-        self.i.next()
-    }
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        self.i.size_hint()
-    }
-}
-
-impl<'a, K: 'a, V: 'a> DoubleEndedIterator for IndexSTSeqDllIterOrdValue<'a, K, V> {
-    fn next_back(&mut self) -> Option<Self::Item> {
-        self.i.next_back()
-    }
-}
-
-impl<'a, K, V> ExactSizeIterator for IndexSTSeqDllIterOrdValue<'a, K, V> {
-    fn len(&self) -> usize {
-        self.i.len()
-    }
-}
-
-impl<'a, K, V> Clone for IndexSTSeqDllIterOrdValue<'a, K, V> {
-    fn clone(&self) -> Self {
-        Self { i: self.i.clone() }
-    }
-}

From 51622ba4d69d1232ea83f794e2f64fd56fe1d6dc Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 30 Jan 2023 10:58:45 -0800
Subject: [PATCH 110/310] Add some smart pointers

---
 server/src/engine/sync/mod.rs   |   1 +
 server/src/engine/sync/smart.rs | 215 ++++++++++++++++++++++++++++++++
 2 files changed, 216 insertions(+)
 create mode 100644 server/src/engine/sync/smart.rs

diff --git a/server/src/engine/sync/mod.rs b/server/src/engine/sync/mod.rs
index a5f67e40..7670d162 100644
--- a/server/src/engine/sync/mod.rs
+++ b/server/src/engine/sync/mod.rs
@@ -26,3 +26,4 @@
 
 pub(super) mod atm;
 pub(super) mod cell;
+pub(super) mod smart;
diff --git a/server/src/engine/sync/smart.rs b/server/src/engine/sync/smart.rs
new file mode 100644
index 00000000..60179843
--- /dev/null
+++ b/server/src/engine/sync/smart.rs
@@ -0,0 +1,215 @@
+/*
+ * Created on Sun Jan 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::atm::{ORD_ACQ, ORD_REL, ORD_RLX};
+use std::{
+    alloc::{dealloc, Layout},
+    borrow::Borrow,
+    fmt,
+    hash::{Hash, Hasher},
+    mem::{self, ManuallyDrop},
+    ops::Deref,
+    process,
+    ptr::{self, NonNull},
+    slice, str,
+    sync::atomic::{self, AtomicUsize},
+};
+
+pub type BytesRC = SliceRC<u8>;
+
+#[derive(Debug, Clone, Hash)]
+pub struct StrRC {
+    base: SliceRC<u8>,
+}
+
+impl StrRC {
+    fn new(base: SliceRC<u8>) -> Self {
+        Self { base }
+    }
+    pub fn from_bx(b: Box<str>) -> Self {
+        let mut md = ManuallyDrop::new(b);
+        Self::new(SliceRC::new(
+            unsafe {
+                // UNSAFE(@ohsayan): nullck + always aligned
+                NonNull::new_unchecked(md.as_mut_ptr())
+            },
+            md.len(),
+        ))
+    }
+    pub fn as_str(&self) -> &str {
+        unsafe {
+            // UNSAFE(@ohsayan): Ctor guarantees correctness
+            str::from_utf8_unchecked(self.base.as_slice())
+        }
+    }
+}
+
+impl PartialEq for StrRC {
+    fn eq(&self, other: &Self) -> bool {
+        self.as_str() == other.as_str()
+    }
+}
+
+impl PartialEq<str> for StrRC {
+    fn eq(&self, other: &str) -> bool {
+        self.as_str() == other
+    }
+}
+
+impl Deref for StrRC {
+    type Target = str;
+    fn deref(&self) -> &Self::Target {
+        self.as_str()
+    }
+}
+
+impl Eq for StrRC {}
+
+pub struct SliceRC<T> {
+    ptr: NonNull<T>,
+    len: usize,
+    rc: NonNull<AtomicUsize>,
+}
+
+impl<T> SliceRC<T> {
+    #[inline(always)]
+    fn new(ptr: NonNull<T>, len: usize) -> Self {
+        Self {
+            ptr,
+            len,
+            rc: unsafe {
+                // UNSAFE(@ohsayan): box would either fail or return a collect alloc
+                NonNull::new_unchecked(Box::into_raw(Box::new(AtomicUsize::new(0))))
+            },
+        }
+    }
+    #[inline(always)]
+    pub fn from_bx(b: Box<[T]>) -> Self {
+        let mut b = ManuallyDrop::new(b);
+        unsafe {
+            // UNSAFE(@ohsayan): non-null from the slice as usual
+            Self::new(NonNull::new_unchecked(b.as_mut_ptr()), b.len())
+        }
+    }
+    #[inline(always)]
+    pub fn as_slice(&self) -> &[T] {
+        unsafe {
+            // UNSAFE(@ohsayan): rc guard + ctor
+            slice::from_raw_parts(self.ptr.as_ptr(), self.len)
+        }
+    }
+    #[inline(always)]
+    fn _rc(&self) -> &AtomicUsize {
+        unsafe {
+            // UNSAFE(@ohsayan): rc + ctor
+            self.rc.as_ref()
+        }
+    }
+    /// SAFETY: Synchronize last man alive
+    #[inline(never)]
+    unsafe fn drop_slow(&self) {
+        // dtor
+        if mem::needs_drop::<T>() {
+            // UNSAFE(@ohsayan): dtor through, the ctor guarantees correct alignment and len
+            ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.ptr.as_ptr(), self.len));
+        }
+        // dealloc
+        // UNSAFE(@ohsayan): we allocated it
+        let layout = Layout::array::<T>(self.len).unwrap_unchecked();
+        // UNSAFE(@ohsayan): layout structure guaranteed by ctor
+        dealloc(self.ptr.as_ptr() as *mut u8, layout);
+        // UNSAFE(@ohsayan): well cmon, look for yourself
+        drop(Box::from_raw(self.rc.as_ptr()));
+    }
+}
+
+impl<T> Drop for SliceRC<T> {
+    fn drop(&mut self) {
+        if self._rc().fetch_sub(1, ORD_REL) != 1 {
+            // not the last man alive
+            return;
+        }
+        // emit a fence for sync with stores
+        atomic::fence(ORD_ACQ);
+        unsafe {
+            // UNSAFE(@ohsayan): Confirmed, we're the last one alive
+            self.drop_slow();
+        }
+    }
+}
+
+impl<T> Clone for SliceRC<T> {
+    #[inline(always)]
+    fn clone(&self) -> Self {
+        let new_rc = self._rc().fetch_add(1, ORD_RLX);
+        if new_rc > (isize::MAX) as usize {
+            // some incredibly degenerate case; this won't ever happen but who knows if some fella decided to have atomic overflow fun?
+            process::abort();
+        }
+        Self { ..*self }
+    }
+}
+
+impl<T: fmt::Debug> fmt::Debug for SliceRC<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.as_slice()).finish()
+    }
+}
+
+impl<T: Hash> Hash for SliceRC<T> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.as_slice().hash(state)
+    }
+}
+
+impl<T: PartialEq> PartialEq for SliceRC<T> {
+    fn eq(&self, other: &Self) -> bool {
+        self.as_slice() == other.as_slice()
+    }
+}
+
+impl<T: PartialEq> PartialEq<[T]> for SliceRC<T> {
+    fn eq(&self, other: &[T]) -> bool {
+        self.as_slice() == other
+    }
+}
+
+impl<T: Eq> Eq for SliceRC<T> {}
+impl<T> Borrow<[T]> for SliceRC<T> {
+    fn borrow(&self) -> &[T] {
+        self.as_slice()
+    }
+}
+
+impl<T> Deref for SliceRC<T> {
+    type Target = [T];
+    fn deref(&self) -> &Self::Target {
+        self.as_slice()
+    }
+}
+
+unsafe impl<T: Send> Send for SliceRC<T> {}
+unsafe impl<T: Sync> Sync for SliceRC<T> {}

From 19206642739e59cf21a4e0536672aa751c741b45 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 1 Feb 2023 08:45:25 -0800
Subject: [PATCH 111/310] Fix use-after-free in mtchm impl, add tests and add
 trait impls

---
 server/src/engine/idx/mtchm/imp.rs   |  19 ++-
 server/src/engine/idx/mtchm/meta.rs  |   7 +
 server/src/engine/idx/mtchm/mod.rs   |  15 +-
 server/src/engine/idx/mtchm/tests.rs | 227 ++++++++++++++++++++++-----
 4 files changed, 224 insertions(+), 44 deletions(-)

diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index d6a0b6f8..64212e66 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -25,9 +25,12 @@
 */
 
 use super::{
-    super::{super::sync::atm::Guard, AsKey, IndexBaseSpec, MTIndex},
+    super::{
+        super::sync::atm::{upin, Guard},
+        AsKey, IndexBaseSpec, MTIndex,
+    },
     iter::{IterKV, IterKey, IterVal},
-    meta::{Config, Key, Value},
+    meta::{Config, Key, TreeElement, Value},
     CHTRuntimeLog, Tree,
 };
 use std::{borrow::Borrow, sync::Arc};
@@ -280,3 +283,15 @@ where
         self.remove_return(key, g)
     }
 }
+
+impl<T: TreeElement, C: Config> FromIterator<T> for Tree<T, C> {
+    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
+        let g = unsafe {
+            // UNSAFE(@ohsayan): it's me, hi, I'm the problem, it's me. yeah, Taylor knows it too. it's just us
+            upin()
+        };
+        let t = Tree::new();
+        iter.into_iter().for_each(|te| assert!(t.insert(te, &g)));
+        t
+    }
+}
diff --git a/server/src/engine/idx/mtchm/meta.rs b/server/src/engine/idx/mtchm/meta.rs
index 04c43bec..72b2cc8f 100644
--- a/server/src/engine/idx/mtchm/meta.rs
+++ b/server/src/engine/idx/mtchm/meta.rs
@@ -74,6 +74,7 @@ pub trait TreeElement: Clone + 'static {
     type Value: Value;
     fn key(&self) -> &Self::Key;
     fn val(&self) -> &Self::Value;
+    fn new(k: Self::Key, v: Self::Value) -> Self;
 }
 
 impl<K: Key, V: Value> TreeElement for (K, V) {
@@ -87,6 +88,9 @@ impl<K: Key, V: Value> TreeElement for (K, V) {
     fn val(&self) -> &V {
         &self.1
     }
+    fn new(k: Self::Key, v: Self::Value) -> Self {
+        (k, v)
+    }
 }
 
 impl<K: Key, V: Value> TreeElement for Arc<(K, V)> {
@@ -100,6 +104,9 @@ impl<K: Key, V: Value> TreeElement for Arc<(K, V)> {
     fn val(&self) -> &V {
         &self.1
     }
+    fn new(k: Self::Key, v: Self::Value) -> Self {
+        Arc::new((k, v))
+    }
 }
 
 flags! {
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index bda35d0c..7c20a729 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -151,6 +151,12 @@ impl<T, C: Config> CTFlagAlign for Tree<T, C> {
     const FL_B: bool = atm::ensure_flag_align::<Node<C>, { NodeFlag::bits() }>();
 }
 
+impl<T, C: Config> Default for Tree<T, C> {
+    fn default() -> Self {
+        Self::_new(C::HState::default())
+    }
+}
+
 pub struct Tree<T, C: Config = DefConfig> {
     root: Atomic<Node<C>>,
     h: C::HState,
@@ -303,7 +309,6 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                         debug_assert_eq!(data.len(), 1, "logic,lnode before height ub");
                         if W::WMODE == access::WRITEMODE_REFRESH {
                             // another job well done; an snode with the wrong key; so basically it's missing
-                            return W::nx();
                         }
                         let next_chunk = (self.hash(data[0].key()) >> level) & C::MASK;
                         let mut new_branch = Node::null();
@@ -327,8 +332,10 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                                     || W::WMODE == access::WRITEMODE_ANY =>
                             {
                                 // update the entry and create a new node
-                                let mut new_ln = data.clone();
-                                new_ln[i] = elem.clone();
+                                let mut new_ln = LNode::new();
+                                new_ln.extend(data[..i].iter().cloned());
+                                new_ln.extend(data[i + 1..].iter().cloned());
+                                new_ln.push(elem.clone());
                                 match current.cx_rel(
                                     node,
                                     Self::new_lnode(new_ln).into_shared(g),
@@ -381,7 +388,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                             }
                             None if W::WMODE == access::WRITEMODE_REFRESH => return W::nx(),
                             _ => {
-                                unreachable!("logic,W::WMODE mismatch: `{}`", W::WMODE);
+                                unreachable!("logic, WMODE mismatch: `{}`", W::WMODE);
                             }
                         }
                     }
diff --git a/server/src/engine/idx/mtchm/tests.rs b/server/src/engine/idx/mtchm/tests.rs
index 52eeb938..fe4e5778 100644
--- a/server/src/engine/idx/mtchm/tests.rs
+++ b/server/src/engine/idx/mtchm/tests.rs
@@ -25,14 +25,18 @@
 */
 
 use super::{
-    super::{super::sync::atm::cpin, IndexBaseSpec, MTIndex},
+    super::{
+        super::sync::atm::{cpin, Guard},
+        IndexBaseSpec, MTIndex,
+    },
     imp::ChmCopy as _ChmCopy,
     meta::DefConfig,
 };
 use std::{
     hash::{BuildHasher, Hasher},
-    sync::{Arc, RwLock},
+    sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},
     thread::{self, JoinHandle},
+    time::Duration,
 };
 
 type Chm<K, V> = ChmCopy<K, V>;
@@ -103,56 +107,203 @@ fn update_empty() {
     assert!(!idx.mt_update(10, 20, &cpin()));
 }
 
-const SPAM_INSERT: usize = 16_384;
+#[cfg(not(miri))]
+const SPAM_QCOUNT: usize = 16_384;
+#[cfg(miri)]
+const SPAM_QCOUNT: usize = 128;
+#[cfg(not(miri))]
 const SPAM_TENANTS: usize = 32;
+#[cfg(miri)]
+const SPAM_TENANTS: usize = 1;
 
-#[test]
-fn multispam_insert() {
-    let idx = Arc::new(ChmCopy::new());
-    let token = Arc::new(RwLock::new(()));
-    let hold = token.write();
-    let data: Vec<(Arc<String>, Arc<String>)> = (0..SPAM_INSERT)
-        .into_iter()
-        .map(|int| (format!("{int}"), format!("x-{int}-{}", int + 1)))
-        .map(|(k, v)| (Arc::new(k), Arc::new(v)))
-        .collect();
-    let distr_data: Vec<Vec<(Arc<String>, Arc<String>)>> = data
-        .chunks(SPAM_INSERT / SPAM_TENANTS)
-        .map(|chunk| {
-            chunk
-                .iter()
-                .map(|(k, v)| (Arc::clone(k), Arc::clone(v)))
-                .collect()
-        })
-        .collect();
-    let threads: Vec<JoinHandle<_>> = distr_data
+#[derive(Clone, Debug)]
+struct ControlToken(Arc<RwLock<()>>);
+impl ControlToken {
+    fn new() -> Self {
+        Self(Arc::new(RwLock::new(())))
+    }
+    fn acquire_hold(&self) -> RwLockWriteGuard<'_, ()> {
+        self.0.write().unwrap()
+    }
+    fn acquire_permit(&self) -> RwLockReadGuard<'_, ()> {
+        self.0.read().unwrap()
+    }
+}
+
+const TUP_IDENTITY: fn(usize) -> (usize, usize) = |x| (x, x);
+const TUP_INCR: fn(usize) -> (usize, usize) = |x| (x, x + 1);
+const TUP_INCR_TWICE: fn(usize) -> (usize, usize) = |x| (x, x + 2);
+
+fn prepare_distr_data(source_buf: &[StringTup], distr_buf: &mut Vec<Vec<StringTup>>) {
+    distr_buf.try_reserve(SPAM_TENANTS).unwrap();
+    distr_buf.extend(source_buf.chunks(SPAM_QCOUNT / SPAM_TENANTS).map(|chunk| {
+        chunk
+            .iter()
+            .map(|(k, v)| (Arc::clone(k), Arc::clone(v)))
+            .collect()
+    }));
+}
+
+fn prepare_data<X, Y, F>(source_buf: &mut Vec<StringTup>, f: F)
+where
+    F: Fn(usize) -> (X, Y),
+    X: ToString,
+    Y: ToString,
+{
+    source_buf.try_reserve(SPAM_QCOUNT).unwrap();
+    source_buf.extend(
+        (0..SPAM_QCOUNT)
+            .into_iter()
+            .map(f)
+            .map(|(k, v)| (Arc::new(k.to_string()), Arc::new(v.to_string()))),
+    );
+}
+
+type StringTup = (Arc<String>, Arc<String>);
+
+fn tdistribute_jobs<K, V, F>(
+    token: &ControlToken,
+    tree: &Arc<ChmCopy<K, V>>,
+    distr_data: Vec<Vec<StringTup>>,
+    f: F,
+) -> Vec<JoinHandle<()>>
+where
+    F: FnOnce(ControlToken, Arc<ChmCopy<K, V>>, Vec<StringTup>) + Send + 'static + Copy,
+    K: Send + Sync + 'static,
+    V: Send + Sync + 'static,
+{
+    let r = distr_data
         .into_iter()
         .enumerate()
         .map(|(tid, this_data)| {
             let this_token = token.clone();
-            let this_idx = idx.clone();
+            let this_idx = tree.clone();
             thread::Builder::new()
                 .name(tid.to_string())
-                .spawn(move || {
-                    let _token = this_token.read();
-                    let g = cpin();
-                    this_data.into_iter().for_each(|(k, v)| {
-                        assert!(this_idx.insert((k, v), &g));
-                    })
-                })
+                .spawn(move || f(this_token, this_idx, this_data))
                 .unwrap()
         })
         .collect();
-    // rush everyone to insert; superb intercore traffic
-    drop(hold);
-    let _x: Box<[()]> = threads
+    thread::sleep(Duration::from_millis(1 * SPAM_TENANTS as u64));
+    r
+}
+
+fn tjoin_all<T>(handles: Vec<JoinHandle<T>>) -> Box<[T]> {
+    handles
         .into_iter()
         .map(JoinHandle::join)
-        .map(Result::unwrap)
-        .collect();
+        .map(|h| match h {
+            Ok(v) => v,
+            Err(e) => {
+                panic!("thread died with: {:?}", e.downcast_ref::<&str>())
+            }
+        })
+        .collect()
+}
+
+fn store_and_verify_integrity<K, V>(
+    token: &ControlToken,
+    tree: &Arc<ChmCopy<K, V>>,
+    source_buf: &[StringTup],
+    action: fn(token: ControlToken, tree: Arc<ChmCopy<K, V>>, thread_chunk: Vec<StringTup>),
+    verify: fn(g: &Guard, tree: &ChmCopy<K, V>, src: &[StringTup]),
+) where
+    K: Send + Sync + 'static,
+    V: Send + Sync + 'static,
+{
+    let mut distr_data = Vec::new();
+    prepare_distr_data(source_buf, &mut distr_data);
+    let hold = token.acquire_hold();
+    let threads = tdistribute_jobs(token, tree, distr_data, action);
+    // BLOW THAT INTERCORE TRAFFIC
+    drop(hold);
+    let _x: Box<[()]> = tjoin_all(threads);
     let pin = cpin();
-    assert_eq!(idx.len(), SPAM_INSERT);
+    verify(&pin, tree, source_buf);
+}
+
+fn _action_put(
+    token: ControlToken,
+    idx: Arc<ChmCopy<Arc<String>, Arc<String>>>,
+    data: Vec<StringTup>,
+) {
+    let _token = token.acquire_permit();
+    let g = cpin();
     data.into_iter().for_each(|(k, v)| {
-        assert_eq!(idx.mt_get(&k, &pin).unwrap().as_str(), &*v);
+        assert!(idx.insert((k, v), &g));
     });
 }
+fn _verify_eq(
+    pin: &Guard,
+    idx: &ChmCopy<Arc<String>, Arc<String>>,
+    source: &[(Arc<String>, Arc<String>)],
+) {
+    assert_eq!(idx.len(), SPAM_QCOUNT);
+    source.into_iter().for_each(|(k, v)| {
+        assert_eq!(
+            idx.mt_get(k, &pin)
+                .expect(&format!("failed to find key: {}", k))
+                .as_str(),
+            v.as_str()
+        );
+    });
+}
+
+#[test]
+fn multispam_insert() {
+    let idx = Arc::default();
+    let token = ControlToken::new();
+    let mut data = Vec::new();
+    prepare_data(&mut data, TUP_INCR);
+    store_and_verify_integrity(&token, &idx, &data, _action_put, _verify_eq);
+}
+
+#[test]
+fn multispam_update() {
+    let idx = Arc::default();
+    let token = ControlToken::new();
+    let mut data = Vec::new();
+    prepare_data(&mut data, TUP_INCR);
+    assert!(data
+        .iter()
+        .enumerate()
+        .all(|(i, (k, v))| { i == k.parse().unwrap() && i + 1 == v.parse().unwrap() }));
+    store_and_verify_integrity(&token, &idx, &data, _action_put, _verify_eq);
+    // update our data set
+    data.iter_mut().enumerate().for_each(|(i, (_, v))| {
+        *v = Arc::new((i + 2).to_string());
+    });
+    assert!(data
+        .iter()
+        .enumerate()
+        .all(|(i, (k, v))| { i == k.parse().unwrap() && i + 2 == v.parse().unwrap() }));
+    // store and verify integrity
+    store_and_verify_integrity(
+        &token,
+        &idx,
+        &data,
+        |tok, idx, chunk| {
+            let g = cpin();
+            let _permit = tok.acquire_permit();
+            chunk.into_iter().for_each(|(k, v)| {
+                let ret = idx
+                    .mt_update_return(k.clone(), v, &g)
+                    .expect(&format!("couldn't find key: {}", k));
+                assert_eq!(
+                    ret.as_str().parse::<usize>().unwrap(),
+                    (k.parse::<usize>().unwrap() + 1)
+                );
+            });
+            // hmm
+        },
+        |pin, idx, source| {
+            assert_eq!(idx.len(), SPAM_QCOUNT);
+            source.into_iter().for_each(|(k, v)| {
+                let ret = idx
+                    .mt_get(k, &pin)
+                    .expect(&format!("couldn't find key: {}", k));
+                assert_eq!(ret.as_str(), v.as_str());
+            });
+        },
+    );
+}

From 932bb4f58b1235707db90e40f8903f0e4dc9e34a Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 1 Feb 2023 09:00:54 -0800
Subject: [PATCH 112/310] Fix remove len impl, use relaxed and add tests

---
 server/src/engine/idx/mtchm/mod.rs   | 10 +++++--
 server/src/engine/idx/mtchm/tests.rs | 44 ++++++++++++++++++++--------
 2 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index 7c20a729..4603189d 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -503,6 +503,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                         Self::read_data(node)
                     };
                     let mut ret = W::nx();
+                    let mut rem = false;
                     // this node shouldn't be empty
                     debug_assert!(!data.is_empty(), "logic,empty node not collected");
                     // build new lnode
@@ -511,6 +512,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                         .filter_map(|this_elem| {
                             if this_elem.key().borrow() == k {
                                 ret = W::ex(this_elem);
+                                rem = true;
                                 None
                             } else {
                                 Some(this_elem.clone())
@@ -567,6 +569,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                             break;
                         }
                     }
+                    self.decr_len_by(rem as _);
                     gc(g);
                     return ret;
                 }
@@ -586,10 +589,13 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
 impl<T, C: Config> Tree<T, C> {
     // hilarious enough but true, l doesn't affect safety but only creates an incorrect state
     fn decr_len(&self) {
-        self.l.fetch_sub(1, ORD_ACQ);
+        self.decr_len_by(1)
+    }
+    fn decr_len_by(&self, by: usize) {
+        self.l.fetch_sub(by, ORD_RLX);
     }
     fn incr_len(&self) {
-        self.l.fetch_add(1, ORD_ACQ);
+        self.l.fetch_add(1, ORD_RLX);
     }
     #[inline(always)]
     fn new_lnode(node: LNode<T>) -> Owned<Node<C>> {
diff --git a/server/src/engine/idx/mtchm/tests.rs b/server/src/engine/idx/mtchm/tests.rs
index fe4e5778..43ec8be3 100644
--- a/server/src/engine/idx/mtchm/tests.rs
+++ b/server/src/engine/idx/mtchm/tests.rs
@@ -201,7 +201,7 @@ fn tjoin_all<T>(handles: Vec<JoinHandle<T>>) -> Box<[T]> {
         .collect()
 }
 
-fn store_and_verify_integrity<K, V>(
+fn modify_and_verify_integrity<K, V>(
     token: &ControlToken,
     tree: &Arc<ChmCopy<K, V>>,
     source_buf: &[StringTup],
@@ -230,7 +230,7 @@ fn _action_put(
     let _token = token.acquire_permit();
     let g = cpin();
     data.into_iter().for_each(|(k, v)| {
-        assert!(idx.insert((k, v), &g));
+        assert!(idx.mt_insert(k, v, &g));
     });
 }
 fn _verify_eq(
@@ -255,7 +255,7 @@ fn multispam_insert() {
     let token = ControlToken::new();
     let mut data = Vec::new();
     prepare_data(&mut data, TUP_INCR);
-    store_and_verify_integrity(&token, &idx, &data, _action_put, _verify_eq);
+    modify_and_verify_integrity(&token, &idx, &data, _action_put, _verify_eq);
 }
 
 #[test]
@@ -264,21 +264,13 @@ fn multispam_update() {
     let token = ControlToken::new();
     let mut data = Vec::new();
     prepare_data(&mut data, TUP_INCR);
-    assert!(data
-        .iter()
-        .enumerate()
-        .all(|(i, (k, v))| { i == k.parse().unwrap() && i + 1 == v.parse().unwrap() }));
-    store_and_verify_integrity(&token, &idx, &data, _action_put, _verify_eq);
+    modify_and_verify_integrity(&token, &idx, &data, _action_put, _verify_eq);
     // update our data set
     data.iter_mut().enumerate().for_each(|(i, (_, v))| {
         *v = Arc::new((i + 2).to_string());
     });
-    assert!(data
-        .iter()
-        .enumerate()
-        .all(|(i, (k, v))| { i == k.parse().unwrap() && i + 2 == v.parse().unwrap() }));
     // store and verify integrity
-    store_and_verify_integrity(
+    modify_and_verify_integrity(
         &token,
         &idx,
         &data,
@@ -307,3 +299,29 @@ fn multispam_update() {
         },
     );
 }
+
+#[test]
+fn multispam_delete() {
+    let idx = Arc::default();
+    let token = ControlToken::new();
+    let mut data = Vec::new();
+    prepare_data(&mut data, TUP_INCR);
+    modify_and_verify_integrity(&token, &idx, &data, _action_put, _verify_eq);
+    // now expunge
+    modify_and_verify_integrity(
+        &token,
+        &idx,
+        &data,
+        |tok, idx, chunk| {
+            let g = cpin();
+            let _permit = tok.acquire_permit();
+            chunk.into_iter().for_each(|(k, v)| {
+                assert_eq!(idx.mt_delete_return(&k, &g).unwrap().as_str(), v.as_str());
+            });
+        },
+        |g, idx, orig| {
+            assert!(orig.into_iter().all(|(k, _)| idx.mt_get(k, &g).is_none()));
+            assert!(idx.is_empty(), "expected empty, found {} elements instead", idx.len());
+        },
+    );
+}

From 263a287edda9e7e05fd543951c3bb835631cd4d5 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 1 Feb 2023 09:56:02 -0800
Subject: [PATCH 113/310] Fix double-free, use-after-free and other bugs in
 mtchm

The bug was actually because of my oversight in the VInline clone
impl for non-Copy Ts. Also added tests for worst case O(n)
performance.
---
 server/src/engine/idx/mod.rs         |  2 ++
 server/src/engine/idx/mtchm/mod.rs   | 45 ++++++++++++++++--------
 server/src/engine/idx/mtchm/tests.rs | 52 ++++++++++++++++++----------
 server/src/engine/mem/vinline.rs     | 24 +------------
 4 files changed, 68 insertions(+), 55 deletions(-)

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index e484015d..64b43288 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -24,6 +24,8 @@
  *
 */
 
+#![deny(unreachable_patterns)]
+
 mod meta;
 mod mtchm;
 mod stdhm;
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index 4603189d..5cdd02e6 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -96,6 +96,16 @@ impl CHTRuntimeLog {
         } else {
             void!()
         }
+        fn repsplit(self: &Self) -> usize {
+            self.data.split.load(ORD_RLX)
+        } else {
+            void!()
+        }
+        fn replnode(self: &Self) -> usize {
+            self.data.hln.load(ORD_RLX)
+        } else {
+            void!()
+        }
     }
 }
 
@@ -321,7 +331,6 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                             in this case we either have the same key or we found an lnode. resolve any conflicts and attempt
                             to update
                         */
-                        self.m.hlnode();
                         let p = data.iter().position(|e| e.key() == elem.key());
                         match p {
                             Some(v) if W::WMODE == access::WRITEMODE_FRESH => {
@@ -336,12 +345,13 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                                 new_ln.extend(data[..i].iter().cloned());
                                 new_ln.extend(data[i + 1..].iter().cloned());
                                 new_ln.push(elem.clone());
-                                match current.cx_rel(
-                                    node,
-                                    Self::new_lnode(new_ln).into_shared(g),
-                                    g,
-                                ) {
-                                    Ok(_) => {
+                                match current.cx_rel(node, Self::new_lnode(new_ln), g) {
+                                    Ok(new) => {
+                                        if cfg!(debug_assertions)
+                                            && unsafe { Self::read_data(new) }.len() > 1
+                                        {
+                                            self.m.hlnode();
+                                        }
                                         unsafe {
                                             /*
                                                 UNSAFE(@ohsayan): swapped out, and we'll be the last thread to see this once the epoch proceeds
@@ -356,17 +366,24 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                                     Err(CompareExchangeError { new, .. }) => {
                                         // failed to swap it in
                                         unsafe {
-                                            // UNSAFE(@ohsayan): Failed to swap this in, and no one else saw it (well)
-                                            Self::ldrop(new)
+                                            Self::ldrop(new.into_shared(g));
                                         }
                                     }
                                 }
                             }
-                            None if W::WMODE == access::WRITEMODE_ANY => {
+                            None if W::WMODE == access::WRITEMODE_ANY
+                                || W::WMODE == access::WRITEMODE_FRESH =>
+                            {
                                 // no funk here
-                                let new_node = Self::new_data(elem.clone());
-                                match current.cx_rel(node, new_node.into_shared(g), g) {
-                                    Ok(_) => {
+                                let mut new_node = data.clone();
+                                new_node.push(elem.clone());
+                                match current.cx_rel(node, Self::new_lnode(new_node), g) {
+                                    Ok(new) => {
+                                        if cfg!(debug_assertions)
+                                            && unsafe { Self::read_data(new) }.len() > 1
+                                        {
+                                            self.m.hlnode();
+                                        }
                                         // swapped out
                                         unsafe {
                                             // UNSAFE(@ohsayan): last thread to see this (well, sorta)
@@ -381,7 +398,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                                         // failed to swap it
                                         unsafe {
                                             // UNSAFE(@ohsayan): never published this, so we're the last one
-                                            Self::ldrop(new)
+                                            Self::ldrop(new.into_shared(g))
                                         }
                                     }
                                 }
diff --git a/server/src/engine/idx/mtchm/tests.rs b/server/src/engine/idx/mtchm/tests.rs
index 43ec8be3..01782df9 100644
--- a/server/src/engine/idx/mtchm/tests.rs
+++ b/server/src/engine/idx/mtchm/tests.rs
@@ -30,7 +30,7 @@ use super::{
         IndexBaseSpec, MTIndex,
     },
     imp::ChmCopy as _ChmCopy,
-    meta::DefConfig,
+    meta::{Config, DefConfig},
 };
 use std::{
     hash::{BuildHasher, Hasher},
@@ -110,11 +110,11 @@ fn update_empty() {
 #[cfg(not(miri))]
 const SPAM_QCOUNT: usize = 16_384;
 #[cfg(miri)]
-const SPAM_QCOUNT: usize = 128;
+const SPAM_QCOUNT: usize = 32;
 #[cfg(not(miri))]
 const SPAM_TENANTS: usize = 32;
 #[cfg(miri)]
-const SPAM_TENANTS: usize = 1;
+const SPAM_TENANTS: usize = 2;
 
 #[derive(Clone, Debug)]
 struct ControlToken(Arc<RwLock<()>>);
@@ -161,16 +161,17 @@ where
 
 type StringTup = (Arc<String>, Arc<String>);
 
-fn tdistribute_jobs<K, V, F>(
+fn tdistribute_jobs<K, V, C: Config, F>(
     token: &ControlToken,
-    tree: &Arc<ChmCopy<K, V>>,
+    tree: &Arc<_ChmCopy<K, V, C>>,
     distr_data: Vec<Vec<StringTup>>,
     f: F,
 ) -> Vec<JoinHandle<()>>
 where
-    F: FnOnce(ControlToken, Arc<ChmCopy<K, V>>, Vec<StringTup>) + Send + 'static + Copy,
+    F: FnOnce(ControlToken, Arc<_ChmCopy<K, V, C>>, Vec<StringTup>) + Send + 'static + Copy,
     K: Send + Sync + 'static,
     V: Send + Sync + 'static,
+    C::HState: Send + Sync,
 {
     let r = distr_data
         .into_iter()
@@ -201,15 +202,16 @@ fn tjoin_all<T>(handles: Vec<JoinHandle<T>>) -> Box<[T]> {
         .collect()
 }
 
-fn modify_and_verify_integrity<K, V>(
+fn modify_and_verify_integrity<K, V, C: Config>(
     token: &ControlToken,
-    tree: &Arc<ChmCopy<K, V>>,
+    tree: &Arc<_ChmCopy<K, V, C>>,
     source_buf: &[StringTup],
-    action: fn(token: ControlToken, tree: Arc<ChmCopy<K, V>>, thread_chunk: Vec<StringTup>),
-    verify: fn(g: &Guard, tree: &ChmCopy<K, V>, src: &[StringTup]),
+    action: fn(token: ControlToken, tree: Arc<_ChmCopy<K, V, C>>, thread_chunk: Vec<StringTup>),
+    verify: fn(g: &Guard, tree: &_ChmCopy<K, V, C>, src: &[StringTup]),
 ) where
     K: Send + Sync + 'static,
     V: Send + Sync + 'static,
+    C::HState: Send + Sync,
 {
     let mut distr_data = Vec::new();
     prepare_distr_data(source_buf, &mut distr_data);
@@ -222,9 +224,9 @@ fn modify_and_verify_integrity<K, V>(
     verify(&pin, tree, source_buf);
 }
 
-fn _action_put(
+fn _action_put<C: Config>(
     token: ControlToken,
-    idx: Arc<ChmCopy<Arc<String>, Arc<String>>>,
+    idx: Arc<_ChmCopy<Arc<String>, Arc<String>, C>>,
     data: Vec<StringTup>,
 ) {
     let _token = token.acquire_permit();
@@ -233,9 +235,9 @@ fn _action_put(
         assert!(idx.mt_insert(k, v, &g));
     });
 }
-fn _verify_eq(
+fn _verify_eq<C: Config>(
     pin: &Guard,
-    idx: &ChmCopy<Arc<String>, Arc<String>>,
+    idx: &_ChmCopy<Arc<String>, Arc<String>, C>,
     source: &[(Arc<String>, Arc<String>)],
 ) {
     assert_eq!(idx.len(), SPAM_QCOUNT);
@@ -251,7 +253,7 @@ fn _verify_eq(
 
 #[test]
 fn multispam_insert() {
-    let idx = Arc::default();
+    let idx = Arc::new(ChmCopy::default());
     let token = ControlToken::new();
     let mut data = Vec::new();
     prepare_data(&mut data, TUP_INCR);
@@ -260,7 +262,7 @@ fn multispam_insert() {
 
 #[test]
 fn multispam_update() {
-    let idx = Arc::default();
+    let idx = Arc::new(ChmCopy::default());
     let token = ControlToken::new();
     let mut data = Vec::new();
     prepare_data(&mut data, TUP_INCR);
@@ -302,7 +304,7 @@ fn multispam_update() {
 
 #[test]
 fn multispam_delete() {
-    let idx = Arc::default();
+    let idx = Arc::new(ChmCopy::default());
     let token = ControlToken::new();
     let mut data = Vec::new();
     prepare_data(&mut data, TUP_INCR);
@@ -321,7 +323,21 @@ fn multispam_delete() {
         },
         |g, idx, orig| {
             assert!(orig.into_iter().all(|(k, _)| idx.mt_get(k, &g).is_none()));
-            assert!(idx.is_empty(), "expected empty, found {} elements instead", idx.len());
+            assert!(
+                idx.is_empty(),
+                "expected empty, found {} elements instead",
+                idx.len()
+            );
         },
     );
 }
+
+#[test]
+fn multispam_lol() {
+    let idx = Arc::new(super::Tree::<StringTup, super::meta::Config2B<LolState>>::new());
+    let token = ControlToken::new();
+    let mut data = Vec::new();
+    prepare_data(&mut data, TUP_INCR);
+    modify_and_verify_integrity(&token, &idx, &data, _action_put, _verify_eq);
+    assert_eq!(idx.idx_metrics().replnode(), SPAM_QCOUNT - 1);
+}
diff --git a/server/src/engine/mem/vinline.rs b/server/src/engine/mem/vinline.rs
index 663ddbef..22db1bcd 100644
--- a/server/src/engine/mem/vinline.rs
+++ b/server/src/engine/mem/vinline.rs
@@ -293,29 +293,7 @@ impl<T, const N: usize> Extend<T> for VInline<N, T> {
 
 impl<T: Clone, const N: usize> Clone for VInline<N, T> {
     fn clone(&self) -> Self {
-        unsafe {
-            if self.on_stack() {
-                // simple stack copy
-                let mut new_stack = Self::INLINE_NULL_STACK;
-                ptr::copy_nonoverlapping(self.d.s.as_ptr(), new_stack.as_mut_ptr(), self.l);
-                Self {
-                    d: VData {
-                        s: ManuallyDrop::new(new_stack),
-                    },
-                    l: self.l,
-                    c: N,
-                }
-            } else {
-                // new allocation
-                let nb = Self::alloc_block(self.len());
-                ptr::copy_nonoverlapping(self._as_ptr(), nb, self.l);
-                Self {
-                    d: VData { h: nb },
-                    l: self.l,
-                    c: self.l,
-                }
-            }
-        }
+        self.iter().cloned().collect()
     }
 }
 

From e404a273ef104ec1fc4986327766d297348d6ae9 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 1 Feb 2023 21:16:45 -0800
Subject: [PATCH 114/310] Simplify kw and sym lookup

---
 server/src/engine/ql/{lex.rs => lex/mod.rs} | 549 +-------------------
 server/src/engine/ql/lex/raw.rs             | 506 ++++++++++++++++++
 server/src/engine/ql/macros.rs              |  19 +
 server/src/engine/ql/tests.rs               |  47 +-
 server/src/engine/ql/tests/schema_tests.rs  |  72 +--
 server/src/engine/ql/tests/structure_syn.rs |  42 +-
 6 files changed, 605 insertions(+), 630 deletions(-)
 rename server/src/engine/ql/{lex.rs => lex/mod.rs} (54%)
 create mode 100644 server/src/engine/ql/lex/raw.rs

diff --git a/server/src/engine/ql/lex.rs b/server/src/engine/ql/lex/mod.rs
similarity index 54%
rename from server/src/engine/ql/lex.rs
rename to server/src/engine/ql/lex/mod.rs
index f949e6bb..142bbc36 100644
--- a/server/src/engine/ql/lex.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -24,527 +24,22 @@
  *
 */
 
+mod raw;
+
+use self::raw::RawLexer;
 use {
     super::{LangError, LangResult},
     crate::util::compiler,
     core::{cmp, fmt, ops::BitOr, slice, str},
 };
 
+pub use self::raw::{Keyword, Lit, LitIR, LitIROwned, Symbol, Token};
 pub type Slice<'a> = &'a [u8];
 
 /*
-    Lex meta
+    Lexer impls
 */
 
-#[derive(Debug, PartialEq, Clone)]
-pub enum Token<'a> {
-    Symbol(Symbol),
-    Keyword(Keyword),
-    Ident(Slice<'a>),
-    #[cfg(test)]
-    /// A comma that can be ignored (used for fuzzing)
-    IgnorableComma,
-    Lit(Lit<'a>), // literal
-}
-
-impl<'a> PartialEq<Symbol> for Token<'a> {
-    fn eq(&self, other: &Symbol) -> bool {
-        match self {
-            Self::Symbol(s) => s == other,
-            _ => false,
-        }
-    }
-}
-
-enum_impls! {
-    Token<'a> => {
-        Keyword as Keyword,
-        Symbol as Symbol,
-        Lit<'a> as Lit,
-    }
-}
-
-#[derive(Debug, PartialEq, Clone)]
-#[repr(u8)]
-/// A [`Lit`] as represented by an insecure token stream
-pub enum Lit<'a> {
-    Str(Box<str>),
-    Bool(bool),
-    UnsignedInt(u64),
-    SignedInt(i64),
-    Bin(Slice<'a>),
-}
-
-impl<'a> Lit<'a> {
-    pub(super) fn as_ir(&'a self) -> LitIR<'a> {
-        match self {
-            Self::Str(s) => LitIR::Str(s.as_ref()),
-            Self::Bool(b) => LitIR::Bool(*b),
-            Self::UnsignedInt(u) => LitIR::UInt(*u),
-            Self::SignedInt(s) => LitIR::SInt(*s),
-            Self::Bin(b) => LitIR::Bin(b),
-        }
-    }
-}
-
-impl<'a> From<&'static str> for Lit<'a> {
-    fn from(s: &'static str) -> Self {
-        Self::Str(s.into())
-    }
-}
-
-enum_impls! {
-    Lit<'a> => {
-        Box<str> as Str,
-        String as Str,
-        bool as Bool,
-        u64 as UnsignedInt,
-    }
-}
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-#[repr(u8)]
-pub enum Symbol {
-    OpArithmeticAdd,  // +
-    OpArithmeticSub,  // -
-    OpArithmeticMul,  // *
-    OpArithmeticDiv,  // /
-    OpLogicalNot,     // !
-    OpLogicalAnd,     // &
-    OpLogicalXor,     // ^
-    OpLogicalOr,      // |
-    OpAssign,         // =
-    TtOpenParen,      // (
-    TtCloseParen,     // )
-    TtOpenSqBracket,  // [
-    TtCloseSqBracket, // ]
-    TtOpenBrace,      // {
-    TtCloseBrace,     // }
-    OpComparatorLt,   // <
-    OpComparatorGt,   // >
-    QuoteS,           // '
-    QuoteD,           // "
-    SymAt,            // @
-    SymHash,          // #
-    SymDollar,        // $
-    SymPercent,       // %
-    SymUnderscore,    // _
-    SymBackslash,     // \
-    SymColon,         // :
-    SymSemicolon,     // ;
-    SymComma,         // ,
-    SymPeriod,        // .
-    SymQuestion,      // ?
-    SymTilde,         // ~
-    SymAccent,        // `
-}
-
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
-#[repr(u8)]
-pub enum Keyword {
-    Table,
-    Model,
-    Space,
-    Index,
-    Type,
-    Function,
-    Use,
-    Create,
-    Alter,
-    Drop,
-    Describe,
-    Truncate,
-    Rename,
-    Add,
-    Remove,
-    Transform,
-    Order,
-    By,
-    Primary,
-    Key,
-    Value,
-    With,
-    On,
-    Lock,
-    All,
-    Insert,
-    Select,
-    Exists,
-    Update,
-    Delete,
-    Into,
-    From,
-    As,
-    Return,
-    Sort,
-    Group,
-    Limit,
-    Asc,
-    Desc,
-    To,
-    Set,
-    Auto,
-    Default,
-    In,
-    Of,
-    Transaction,
-    Batch,
-    Read,
-    Write,
-    Begin,
-    End,
-    Where,
-    If,
-    And,
-    Or,
-    Not,
-    Null,
-}
-
-/*
-    This section implements LUTs constructed using DAGs, as described by Czech et al in their paper. I wrote these pretty much by
-    brute-force using a byte-level multiplicative function (inside a script). This unfortunately implies that every time we *do*
-    need to add a new keyword, I will need to recompute and rewrite the vertices. I don't plan to use any codegen, so I think
-    this is good as-is. The real challenge here is to keep the graph small, and I couldn't do that for the symbols table even with
-    multiple trials. Please see if you can improve them.
-
-    Also the functions are unique to every graph, and every input set, so BE WARNED!
-
-    -- Sayan (@ohsayan)
-    Sept. 18, 2022
-*/
-
-const SYM_MAGIC_A: u8 = b'w';
-const SYM_MAGIC_B: u8 = b'E';
-
-static SYM_GRAPH: [u8; 69] = [
-    0, 0, 25, 0, 3, 0, 21, 0, 6, 13, 0, 0, 0, 0, 8, 0, 0, 0, 17, 0, 0, 30, 0, 28, 0, 20, 19, 12, 0,
-    0, 2, 0, 0, 15, 0, 0, 0, 5, 0, 31, 14, 0, 1, 0, 18, 29, 24, 0, 0, 10, 0, 0, 26, 0, 0, 0, 22, 0,
-    23, 7, 0, 27, 0, 4, 16, 11, 0, 0, 9,
-];
-
-static SYM_LUT: [(u8, Symbol); 32] = [
-    (b'+', Symbol::OpArithmeticAdd),
-    (b'-', Symbol::OpArithmeticSub),
-    (b'*', Symbol::OpArithmeticMul),
-    (b'/', Symbol::OpArithmeticDiv),
-    (b'!', Symbol::OpLogicalNot),
-    (b'&', Symbol::OpLogicalAnd),
-    (b'^', Symbol::OpLogicalXor),
-    (b'|', Symbol::OpLogicalOr),
-    (b'=', Symbol::OpAssign),
-    (b'(', Symbol::TtOpenParen),
-    (b')', Symbol::TtCloseParen),
-    (b'[', Symbol::TtOpenSqBracket),
-    (b']', Symbol::TtCloseSqBracket),
-    (b'{', Symbol::TtOpenBrace),
-    (b'}', Symbol::TtCloseBrace),
-    (b'<', Symbol::OpComparatorLt),
-    (b'>', Symbol::OpComparatorGt),
-    (b'\'', Symbol::QuoteS),
-    (b'"', Symbol::QuoteD),
-    (b'@', Symbol::SymAt),
-    (b'#', Symbol::SymHash),
-    (b'$', Symbol::SymDollar),
-    (b'%', Symbol::SymPercent),
-    (b'_', Symbol::SymUnderscore),
-    (b'\\', Symbol::SymBackslash),
-    (b':', Symbol::SymColon),
-    (b';', Symbol::SymSemicolon),
-    (b',', Symbol::SymComma),
-    (b'.', Symbol::SymPeriod),
-    (b'?', Symbol::SymQuestion),
-    (b'~', Symbol::SymTilde),
-    (b'`', Symbol::SymAccent),
-];
-
-#[inline(always)]
-fn symfh(k: u8, magic: u8) -> u16 {
-    (magic as u16 * k as u16) % SYM_GRAPH.len() as u16
-}
-
-#[inline(always)]
-fn symph(k: u8) -> u8 {
-    (SYM_GRAPH[symfh(k, SYM_MAGIC_A) as usize] + SYM_GRAPH[symfh(k, SYM_MAGIC_B) as usize])
-        % SYM_GRAPH.len() as u8
-}
-
-#[inline(always)]
-fn symof(sym: u8) -> Option<Symbol> {
-    let hf = symph(sym);
-    if hf < SYM_LUT.len() as u8 && SYM_LUT[hf as usize].0 == sym {
-        Some(SYM_LUT[hf as usize].1)
-    } else {
-        None
-    }
-}
-
-static KW_LUT: [(&[u8], Keyword); 57] = [
-    (b"table", Keyword::Table),
-    (b"model", Keyword::Model),
-    (b"space", Keyword::Space),
-    (b"index", Keyword::Index),
-    (b"type", Keyword::Type),
-    (b"function", Keyword::Function),
-    (b"use", Keyword::Use),
-    (b"create", Keyword::Create),
-    (b"alter", Keyword::Alter),
-    (b"drop", Keyword::Drop),
-    (b"describe", Keyword::Describe),
-    (b"truncate", Keyword::Truncate),
-    (b"rename", Keyword::Rename),
-    (b"add", Keyword::Add),
-    (b"remove", Keyword::Remove),
-    (b"transform", Keyword::Transform),
-    (b"order", Keyword::Order),
-    (b"by", Keyword::By),
-    (b"primary", Keyword::Primary),
-    (b"key", Keyword::Key),
-    (b"value", Keyword::Value),
-    (b"with", Keyword::With),
-    (b"on", Keyword::On),
-    (b"lock", Keyword::Lock),
-    (b"all", Keyword::All),
-    (b"insert", Keyword::Insert),
-    (b"select", Keyword::Select),
-    (b"exists", Keyword::Exists),
-    (b"update", Keyword::Update),
-    (b"delete", Keyword::Delete),
-    (b"into", Keyword::Into),
-    (b"from", Keyword::From),
-    (b"as", Keyword::As),
-    (b"return", Keyword::Return),
-    (b"sort", Keyword::Sort),
-    (b"group", Keyword::Group),
-    (b"limit", Keyword::Limit),
-    (b"asc", Keyword::Asc),
-    (b"desc", Keyword::Desc),
-    (b"to", Keyword::To),
-    (b"set", Keyword::Set),
-    (b"auto", Keyword::Auto),
-    (b"default", Keyword::Default),
-    (b"in", Keyword::In),
-    (b"of", Keyword::Of),
-    (b"transaction", Keyword::Transaction),
-    (b"batch", Keyword::Batch),
-    (b"read", Keyword::Read),
-    (b"write", Keyword::Write),
-    (b"begin", Keyword::Begin),
-    (b"end", Keyword::End),
-    (b"where", Keyword::Where),
-    (b"if", Keyword::If),
-    (b"and", Keyword::And),
-    (b"or", Keyword::Or),
-    (b"not", Keyword::Not),
-    (b"null", Keyword::Null),
-];
-
-static KWG: [u8; 63] = [
-    0, 24, 15, 29, 51, 53, 44, 38, 43, 4, 27, 1, 37, 57, 32, 0, 46, 24, 59, 45, 32, 52, 8, 0, 23,
-    19, 33, 48, 56, 60, 33, 53, 18, 47, 49, 53, 2, 19, 1, 34, 19, 58, 11, 5, 0, 41, 27, 24, 20, 2,
-    0, 0, 48, 2, 42, 46, 43, 0, 18, 33, 21, 12, 41,
-];
-
-const KWMG_1: [u8; 11] = *b"MpVBwC1vsCy";
-const KWMG_2: [u8; 11] = *b"m7sNd9mtGzC";
-const KWMG_S: usize = KWMG_1.len();
-
-#[inline(always)]
-fn kwhf(k: &[u8], mg: &[u8]) -> u32 {
-    let mut i = 0;
-    let mut s = 0;
-    while i < k.len() {
-        s += mg[(i % KWMG_S) as usize] as u32 * k[i] as u32;
-        i += 1;
-    }
-    s % KWG.len() as u32
-}
-
-#[inline(always)]
-fn kwph(k: &[u8]) -> u8 {
-    (KWG[kwhf(k, &KWMG_1) as usize] + KWG[kwhf(k, &KWMG_2) as usize]) % KWG.len() as u8
-}
-
-#[inline(always)]
-fn kwof(key: &[u8]) -> Option<Keyword> {
-    let ph = kwph(key);
-    if ph < KW_LUT.len() as u8 && KW_LUT[ph as usize].0 == key {
-        Some(KW_LUT[ph as usize].1)
-    } else {
-        None
-    }
-}
-
-impl<'a> Token<'a> {
-    #[inline(always)]
-    pub(crate) const fn is_ident(&self) -> bool {
-        matches!(self, Token::Ident(_))
-    }
-    #[inline(always)]
-    pub(super) const fn is_lit(&self) -> bool {
-        matches!(self, Self::Lit(_))
-    }
-}
-
-impl<'a> AsRef<Token<'a>> for Token<'a> {
-    #[inline(always)]
-    fn as_ref(&self) -> &Token<'a> {
-        self
-    }
-}
-
-/*
-    Lexer impl
-*/
-
-#[derive(Debug)]
-pub struct RawLexer<'a> {
-    c: *const u8,
-    e: *const u8,
-    tokens: Vec<Token<'a>>,
-    last_error: Option<LangError>,
-}
-
-// ctor
-impl<'a> RawLexer<'a> {
-    #[inline(always)]
-    pub const fn new(src: Slice<'a>) -> Self {
-        Self {
-            c: src.as_ptr(),
-            e: unsafe {
-                // UNSAFE(@ohsayan): Always safe (<= EOA)
-                src.as_ptr().add(src.len())
-            },
-            last_error: None,
-            tokens: Vec::new(),
-        }
-    }
-}
-
-// meta
-impl<'a> RawLexer<'a> {
-    #[inline(always)]
-    const fn cursor(&self) -> *const u8 {
-        self.c
-    }
-    #[inline(always)]
-    const fn data_end_ptr(&self) -> *const u8 {
-        self.e
-    }
-    #[inline(always)]
-    fn not_exhausted(&self) -> bool {
-        self.data_end_ptr() > self.cursor()
-    }
-    #[inline(always)]
-    fn exhausted(&self) -> bool {
-        self.cursor() == self.data_end_ptr()
-    }
-    #[inline(always)]
-    fn remaining(&self) -> usize {
-        unsafe { self.e.offset_from(self.c) as usize }
-    }
-    #[inline(always)]
-    unsafe fn deref_cursor(&self) -> u8 {
-        *self.cursor()
-    }
-    #[inline(always)]
-    unsafe fn incr_cursor_by(&mut self, by: usize) {
-        debug_assert!(self.remaining() >= by);
-        self.c = self.cursor().add(by)
-    }
-    #[inline(always)]
-    unsafe fn incr_cursor(&mut self) {
-        self.incr_cursor_by(1)
-    }
-    #[inline(always)]
-    unsafe fn incr_cursor_if(&mut self, iff: bool) {
-        self.incr_cursor_by(iff as usize)
-    }
-    #[inline(always)]
-    fn push_token(&mut self, token: impl Into<Token<'a>>) {
-        self.tokens.push(token.into())
-    }
-    #[inline(always)]
-    fn peek_is(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
-        self.not_exhausted() && unsafe { f(self.deref_cursor()) }
-    }
-    #[inline(always)]
-    fn peek_is_and_forward(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
-        let did_fw = self.not_exhausted() && unsafe { f(self.deref_cursor()) };
-        unsafe {
-            self.incr_cursor_if(did_fw);
-        }
-        did_fw
-    }
-    #[inline(always)]
-    fn peek_eq_and_forward_or_eof(&mut self, eq: u8) -> bool {
-        unsafe {
-            let eq = self.not_exhausted() && self.deref_cursor() == eq;
-            self.incr_cursor_if(eq);
-            eq | self.exhausted()
-        }
-    }
-    #[inline(always)]
-    fn peek_neq(&self, b: u8) -> bool {
-        self.not_exhausted() && unsafe { self.deref_cursor() != b }
-    }
-    #[inline(always)]
-    fn peek_eq_and_forward(&mut self, b: u8) -> bool {
-        unsafe {
-            let r = self.not_exhausted() && self.deref_cursor() == b;
-            self.incr_cursor_if(r);
-            r
-        }
-    }
-    #[inline(always)]
-    fn trim_ahead(&mut self) {
-        while self.peek_is_and_forward(|b| b == b' ' || b == b'\t' || b == b'\n') {}
-    }
-    #[inline(always)]
-    fn set_error(&mut self, e: LangError) {
-        self.last_error = Some(e);
-    }
-    #[inline(always)]
-    fn no_error(&self) -> bool {
-        self.last_error.is_none()
-    }
-}
-
-// high level methods
-impl<'a> RawLexer<'a> {
-    #[inline(always)]
-    fn scan_ident(&mut self) -> Slice<'a> {
-        let s = self.cursor();
-        unsafe {
-            while self.peek_is(|b| b.is_ascii_alphanumeric() || b == b'_') {
-                self.incr_cursor();
-            }
-            slice::from_raw_parts(s, self.cursor().offset_from(s) as usize)
-        }
-    }
-    #[inline(always)]
-    fn scan_ident_or_keyword(&mut self) {
-        let s = self.scan_ident();
-        let st = s.to_ascii_lowercase();
-        match kwof(&st) {
-            Some(kw) => self.tokens.push(kw.into()),
-            // FIXME(@ohsayan): Uh, mind fixing this? The only advantage is that I can keep the graph *memory* footprint small
-            None if st == b"true" || st == b"false" => self.push_token(Lit::Bool(st == b"true")),
-            None => self.tokens.push(Token::Ident(s)),
-        }
-    }
-    #[inline(always)]
-    fn scan_byte(&mut self, byte: u8) {
-        match symof(byte) {
-            Some(tok) => self.push_token(tok),
-            None => return self.set_error(LangError::UnexpectedChar),
-        }
-        unsafe {
-            self.incr_cursor();
-        }
-    }
-}
-
 #[derive(Debug)]
 /// This implements the `opmode-dev` for BlueQL
 pub struct InsecureLexer<'a> {
@@ -894,40 +389,6 @@ where
     number
 }
 
-#[derive(PartialEq, Debug, Clone, Copy)]
-/// Intermediate literal repr
-pub enum LitIR<'a> {
-    Str(&'a str),
-    Bin(Slice<'a>),
-    UInt(u64),
-    SInt(i64),
-    Bool(bool),
-    Float(f64),
-}
-
-impl<'a> LitIR<'a> {
-    pub fn to_litir_owned(&self) -> LitIROwned {
-        match self {
-            Self::Str(s) => LitIROwned::Str(s.to_string().into_boxed_str()),
-            Self::Bin(b) => LitIROwned::Bin(b.to_vec().into_boxed_slice()),
-            Self::UInt(u) => LitIROwned::UInt(*u),
-            Self::SInt(s) => LitIROwned::SInt(*s),
-            Self::Bool(b) => LitIROwned::Bool(*b),
-            Self::Float(f) => LitIROwned::Float(*f),
-        }
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub enum LitIROwned {
-    Str(Box<str>),
-    Bin(Box<[u8]>),
-    UInt(u64),
-    SInt(i64),
-    Bool(bool),
-    Float(f64),
-}
-
 #[derive(Debug, PartialEq)]
 /// Data constructed from `opmode-safe`
 pub struct SafeQueryData<'a> {
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
new file mode 100644
index 00000000..407d4439
--- /dev/null
+++ b/server/src/engine/ql/lex/raw.rs
@@ -0,0 +1,506 @@
+/*
+ * Created on Wed Feb 01 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::{super::LangError, Slice};
+use core::slice;
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum Token<'a> {
+    Symbol(Symbol),
+    Keyword(Keyword),
+    Ident(Slice<'a>),
+    #[cfg(test)]
+    /// A comma that can be ignored (used for fuzzing)
+    IgnorableComma,
+    Lit(Lit<'a>), // literal
+}
+
+impl<'a> ToString for Token<'a> {
+    fn to_string(&self) -> String {
+        match self {
+            Self::Symbol(s) => s.to_string(),
+            Self::Keyword(k) => k.to_string(),
+            Self::Ident(id) => String::from_utf8_lossy(id).to_string(),
+            Self::Lit(l) => l.to_string(),
+            #[cfg(test)]
+            Self::IgnorableComma => "[IGNORE_COMMA]".to_owned(),
+        }
+    }
+}
+
+impl<'a> PartialEq<Symbol> for Token<'a> {
+    fn eq(&self, other: &Symbol) -> bool {
+        match self {
+            Self::Symbol(s) => s == other,
+            _ => false,
+        }
+    }
+}
+
+enum_impls! {
+    Token<'a> => {
+        Keyword as Keyword,
+        Symbol as Symbol,
+        Lit<'a> as Lit,
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
+#[repr(u8)]
+/// A [`Lit`] as represented by an insecure token stream
+pub enum Lit<'a> {
+    Str(Box<str>),
+    Bool(bool),
+    UnsignedInt(u64),
+    SignedInt(i64),
+    Bin(Slice<'a>),
+}
+
+impl<'a> ToString for Lit<'a> {
+    fn to_string(&self) -> String {
+        match self {
+            Self::Str(s) => format!("{:?}", s),
+            Self::Bool(b) => b.to_string(),
+            Self::UnsignedInt(u) => u.to_string(),
+            Self::SignedInt(s) => s.to_string(),
+            Self::Bin(b) => format!("{:?}", b),
+        }
+    }
+}
+
+impl<'a> Lit<'a> {
+    pub fn as_ir(&'a self) -> LitIR<'a> {
+        match self {
+            Self::Str(s) => LitIR::Str(s.as_ref()),
+            Self::Bool(b) => LitIR::Bool(*b),
+            Self::UnsignedInt(u) => LitIR::UInt(*u),
+            Self::SignedInt(s) => LitIR::SInt(*s),
+            Self::Bin(b) => LitIR::Bin(b),
+        }
+    }
+}
+
+impl<'a> From<&'static str> for Lit<'a> {
+    fn from(s: &'static str) -> Self {
+        Self::Str(s.into())
+    }
+}
+
+enum_impls! {
+    Lit<'a> => {
+        Box<str> as Str,
+        String as Str,
+        bool as Bool,
+        u64 as UnsignedInt,
+    }
+}
+
+build_lut!(
+    static KW_LUT in kwlut;
+    #[derive(Debug, PartialEq, Eq, Clone, Copy)]
+    pub enum Keyword {
+        Table = "table",
+        Model = "model",
+        Space = "space",
+        Index = "index",
+        Type = "type",
+        Function = "function",
+        Use = "use",
+        Create = "create",
+        Alter = "alter",
+        Drop = "drop",
+        Describe = "describe",
+        Truncate = "truncate",
+        Rename = "rename",
+        Add = "add",
+        Remove = "remove",
+        Transform = "transform",
+        Order = "order",
+        By = "by",
+        Primary = "primary",
+        Key = "key",
+        Value = "value",
+        With = "with",
+        On = "on",
+        Lock = "lock",
+        All = "all",
+        Insert = "insert",
+        Select = "select",
+        Exists = "exists",
+        Update = "update",
+        Delete = "delete",
+        Into = "into",
+        From = "from",
+        As = "as",
+        Return = "return",
+        Sort = "sort",
+        Group = "group",
+        Limit = "limit",
+        Asc = "asc",
+        Desc = "desc",
+        To = "to",
+        Set = "set",
+        Auto = "auto",
+        Default = "default",
+        In = "in",
+        Of = "of",
+        Transaction = "transaction",
+        Batch = "batch",
+        Read = "read",
+        Write = "write",
+        Begin = "begin",
+        End = "end",
+        Where = "where",
+        If = "if",
+        And = "and",
+        Or = "or",
+        Not = "not",
+        Null = "null",
+    }
+    |b: &str| -> &[u8] { b.as_bytes() },
+    |b: &str| -> String { b.to_ascii_uppercase() }
+);
+
+build_lut!(
+    static SYM_LUT in symlut;
+    #[derive(Debug, Copy, Clone, PartialEq, Eq)]
+    #[repr(u8)]
+    pub enum Symbol {
+        OpArithmeticAdd = b'+',
+        OpArithmeticSub = b'-',
+        OpArithmeticMul = b'*',
+        OpArithmeticDiv = b'/',
+        OpLogicalNot = b'!',
+        OpLogicalAnd = b'&',
+        OpLogicalXor = b'^',
+        OpLogicalOr = b'|',
+        OpAssign = b'=',
+        TtOpenParen = b'(',
+        TtCloseParen = b')',
+        TtOpenSqBracket = b'[',
+        TtCloseSqBracket = b']',
+        TtOpenBrace = b'{',
+        TtCloseBrace = b'}',
+        OpComparatorLt = b'<',
+        OpComparatorGt = b'>',
+        QuoteS = b'\'',
+        QuoteD = b'"',
+        SymAt = b'@',
+        SymHash = b'#',
+        SymDollar = b'$',
+        SymPercent = b'%',
+        SymUnderscore = b'_',
+        SymBackslash = b'\\',
+        SymColon = b':',
+        SymSemicolon = b';',
+        SymComma = b',',
+        SymPeriod = b'.',
+        SymQuestion = b'?',
+        SymTilde = b'~',
+        SymAccent = b'`',
+    }
+    |s: u8| -> u8 { s },
+    |c: u8| -> String { char::from(c).to_string() }
+);
+
+/*
+    This section implements LUTs constructed using DAGs, as described by Czech et al in their paper. I wrote these pretty much by
+    brute-force using a byte-level multiplicative function (inside a script). This unfortunately implies that every time we *do*
+    need to add a new keyword, I will need to recompute and rewrite the vertices. I don't plan to use any codegen, so I think
+    this is good as-is. The real challenge here is to keep the graph small, and I couldn't do that for the symbols table even with
+    multiple trials. Please see if you can improve them.
+
+    Also the functions are unique to every graph, and every input set, so BE WARNED!
+
+    -- Sayan (@ohsayan)
+    Sept. 18, 2022
+*/
+
+const SYM_MAGIC_A: u8 = b'w';
+const SYM_MAGIC_B: u8 = b'E';
+
+static SYM_GRAPH: [u8; 69] = [
+    0, 0, 25, 0, 3, 0, 21, 0, 6, 13, 0, 0, 0, 0, 8, 0, 0, 0, 17, 0, 0, 30, 0, 28, 0, 20, 19, 12, 0,
+    0, 2, 0, 0, 15, 0, 0, 0, 5, 0, 31, 14, 0, 1, 0, 18, 29, 24, 0, 0, 10, 0, 0, 26, 0, 0, 0, 22, 0,
+    23, 7, 0, 27, 0, 4, 16, 11, 0, 0, 9,
+];
+
+#[inline(always)]
+fn symfh(k: u8, magic: u8) -> u16 {
+    (magic as u16 * k as u16) % SYM_GRAPH.len() as u16
+}
+
+#[inline(always)]
+fn symph(k: u8) -> u8 {
+    (SYM_GRAPH[symfh(k, SYM_MAGIC_A) as usize] + SYM_GRAPH[symfh(k, SYM_MAGIC_B) as usize])
+        % SYM_GRAPH.len() as u8
+}
+
+#[inline(always)]
+pub(super) fn symof(sym: u8) -> Option<Symbol> {
+    let hf = symph(sym);
+    if hf < SYM_LUT.len() as u8 && SYM_LUT[hf as usize].0 == sym {
+        Some(SYM_LUT[hf as usize].1)
+    } else {
+        None
+    }
+}
+
+static KWG: [u8; 63] = [
+    0, 24, 15, 29, 51, 53, 44, 38, 43, 4, 27, 1, 37, 57, 32, 0, 46, 24, 59, 45, 32, 52, 8, 0, 23,
+    19, 33, 48, 56, 60, 33, 53, 18, 47, 49, 53, 2, 19, 1, 34, 19, 58, 11, 5, 0, 41, 27, 24, 20, 2,
+    0, 0, 48, 2, 42, 46, 43, 0, 18, 33, 21, 12, 41,
+];
+
+const KWMG_1: [u8; 11] = *b"MpVBwC1vsCy";
+const KWMG_2: [u8; 11] = *b"m7sNd9mtGzC";
+const KWMG_S: usize = KWMG_1.len();
+
+#[inline(always)]
+fn kwhf(k: &[u8], mg: &[u8]) -> u32 {
+    let mut i = 0;
+    let mut s = 0;
+    while i < k.len() {
+        s += mg[(i % KWMG_S) as usize] as u32 * k[i] as u32;
+        i += 1;
+    }
+    s % KWG.len() as u32
+}
+
+#[inline(always)]
+fn kwph(k: &[u8]) -> u8 {
+    (KWG[kwhf(k, &KWMG_1) as usize] + KWG[kwhf(k, &KWMG_2) as usize]) % KWG.len() as u8
+}
+
+#[inline(always)]
+pub(super) fn kwof(key: &[u8]) -> Option<Keyword> {
+    let ph = kwph(key);
+    if ph < KW_LUT.len() as u8 && KW_LUT[ph as usize].0 == key {
+        Some(KW_LUT[ph as usize].1)
+    } else {
+        None
+    }
+}
+
+impl<'a> Token<'a> {
+    #[inline(always)]
+    pub(crate) const fn is_ident(&self) -> bool {
+        matches!(self, Token::Ident(_))
+    }
+    #[inline(always)]
+    pub const fn is_lit(&self) -> bool {
+        matches!(self, Self::Lit(_))
+    }
+}
+
+impl<'a> AsRef<Token<'a>> for Token<'a> {
+    #[inline(always)]
+    fn as_ref(&self) -> &Token<'a> {
+        self
+    }
+}
+
+#[derive(PartialEq, Debug, Clone, Copy)]
+/// Intermediate literal repr
+pub enum LitIR<'a> {
+    Str(&'a str),
+    Bin(Slice<'a>),
+    UInt(u64),
+    SInt(i64),
+    Bool(bool),
+    Float(f64),
+}
+
+impl<'a> LitIR<'a> {
+    pub fn to_litir_owned(&self) -> LitIROwned {
+        match self {
+            Self::Str(s) => LitIROwned::Str(s.to_string().into_boxed_str()),
+            Self::Bin(b) => LitIROwned::Bin(b.to_vec().into_boxed_slice()),
+            Self::UInt(u) => LitIROwned::UInt(*u),
+            Self::SInt(s) => LitIROwned::SInt(*s),
+            Self::Bool(b) => LitIROwned::Bool(*b),
+            Self::Float(f) => LitIROwned::Float(*f),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum LitIROwned {
+    Str(Box<str>),
+    Bin(Box<[u8]>),
+    UInt(u64),
+    SInt(i64),
+    Bool(bool),
+    Float(f64),
+}
+
+#[derive(Debug)]
+pub struct RawLexer<'a> {
+    c: *const u8,
+    e: *const u8,
+    pub(super) tokens: Vec<Token<'a>>,
+    pub(super) last_error: Option<LangError>,
+}
+
+// ctor
+impl<'a> RawLexer<'a> {
+    #[inline(always)]
+    pub(super) const fn new(src: Slice<'a>) -> Self {
+        Self {
+            c: src.as_ptr(),
+            e: unsafe {
+                // UNSAFE(@ohsayan): Always safe (<= EOA)
+                src.as_ptr().add(src.len())
+            },
+            last_error: None,
+            tokens: Vec::new(),
+        }
+    }
+}
+
+// meta
+impl<'a> RawLexer<'a> {
+    #[inline(always)]
+    pub(super) const fn cursor(&self) -> *const u8 {
+        self.c
+    }
+    #[inline(always)]
+    pub(super) const fn data_end_ptr(&self) -> *const u8 {
+        self.e
+    }
+    #[inline(always)]
+    pub(super) fn not_exhausted(&self) -> bool {
+        self.data_end_ptr() > self.cursor()
+    }
+    #[inline(always)]
+    pub(super) fn exhausted(&self) -> bool {
+        self.cursor() == self.data_end_ptr()
+    }
+    #[inline(always)]
+    pub(super) fn remaining(&self) -> usize {
+        unsafe { self.e.offset_from(self.c) as usize }
+    }
+    #[inline(always)]
+    pub(super) unsafe fn deref_cursor(&self) -> u8 {
+        *self.cursor()
+    }
+    #[inline(always)]
+    pub(super) unsafe fn incr_cursor_by(&mut self, by: usize) {
+        debug_assert!(self.remaining() >= by);
+        self.c = self.cursor().add(by)
+    }
+    #[inline(always)]
+    pub(super) unsafe fn incr_cursor(&mut self) {
+        self.incr_cursor_by(1)
+    }
+    #[inline(always)]
+    unsafe fn incr_cursor_if(&mut self, iff: bool) {
+        self.incr_cursor_by(iff as usize)
+    }
+    #[inline(always)]
+    pub(super) fn push_token(&mut self, token: impl Into<Token<'a>>) {
+        self.tokens.push(token.into())
+    }
+    #[inline(always)]
+    pub(super) fn peek_is(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
+        self.not_exhausted() && unsafe { f(self.deref_cursor()) }
+    }
+    #[inline(always)]
+    pub(super) fn peek_is_and_forward(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
+        let did_fw = self.not_exhausted() && unsafe { f(self.deref_cursor()) };
+        unsafe {
+            self.incr_cursor_if(did_fw);
+        }
+        did_fw
+    }
+    #[inline(always)]
+    fn peek_eq_and_forward_or_eof(&mut self, eq: u8) -> bool {
+        unsafe {
+            let eq = self.not_exhausted() && self.deref_cursor() == eq;
+            self.incr_cursor_if(eq);
+            eq | self.exhausted()
+        }
+    }
+    #[inline(always)]
+    pub(super) fn peek_neq(&self, b: u8) -> bool {
+        self.not_exhausted() && unsafe { self.deref_cursor() != b }
+    }
+    #[inline(always)]
+    pub(super) fn peek_eq_and_forward(&mut self, b: u8) -> bool {
+        unsafe {
+            let r = self.not_exhausted() && self.deref_cursor() == b;
+            self.incr_cursor_if(r);
+            r
+        }
+    }
+    #[inline(always)]
+    pub(super) fn trim_ahead(&mut self) {
+        while self.peek_is_and_forward(|b| b == b' ' || b == b'\t' || b == b'\n') {}
+    }
+    #[inline(always)]
+    pub(super) fn set_error(&mut self, e: LangError) {
+        self.last_error = Some(e);
+    }
+    #[inline(always)]
+    pub(super) fn no_error(&self) -> bool {
+        self.last_error.is_none()
+    }
+}
+
+// high level methods
+impl<'a> RawLexer<'a> {
+    #[inline(always)]
+    pub(super) fn scan_ident(&mut self) -> Slice<'a> {
+        let s = self.cursor();
+        unsafe {
+            while self.peek_is(|b| b.is_ascii_alphanumeric() || b == b'_') {
+                self.incr_cursor();
+            }
+            slice::from_raw_parts(s, self.cursor().offset_from(s) as usize)
+        }
+    }
+    #[inline(always)]
+    pub(super) fn scan_ident_or_keyword(&mut self) {
+        let s = self.scan_ident();
+        let st = s.to_ascii_lowercase();
+        match kwof(&st) {
+            Some(kw) => self.tokens.push(kw.into()),
+            // FIXME(@ohsayan): Uh, mind fixing this? The only advantage is that I can keep the graph *memory* footprint small
+            None if st == b"true" || st == b"false" => self.push_token(Lit::Bool(st == b"true")),
+            None => self.tokens.push(Token::Ident(s)),
+        }
+    }
+    #[inline(always)]
+    pub(super) fn scan_byte(&mut self, byte: u8) {
+        match symof(byte) {
+            Some(tok) => self.push_token(tok),
+            None => return self.set_error(LangError::UnexpectedChar),
+        }
+        unsafe {
+            self.incr_cursor();
+        }
+    }
+}
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 4b60718a..1049657f 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -338,3 +338,22 @@ macro_rules! statictbl {
         &'static $name
     }};
 }
+
+macro_rules! build_lut {
+    (
+        $(#[$attr_s:meta])* $vis_s:vis static $LUT:ident in $lut:ident; $(#[$attr_e:meta])* $vis_e:vis enum $SYM:ident {$($variant:ident = $match:literal),*$(,)?}
+        |$arg:ident: $inp:ty| -> $ret:ty $block:block,
+        |$arg2:ident: $inp2:ty| -> String $block2:block
+    ) => {
+        mod $lut {
+            pub const L: usize = { let mut i = 0; $(let _ = $match;i += 1;)*i };
+            pub const fn f($arg: $inp) -> $ret $block
+            pub fn s($arg2: $inp2) -> String $block2
+        }
+        $(#[$attr_e])* $vis_e enum $SYM {$($variant),*}
+        $(#[$attr_s])* $vis_s static $LUT: [($ret, $SYM); $lut::L] = {[$(($lut::f($match), $SYM::$variant)),*]};
+        impl ::std::string::ToString for $SYM {
+            fn to_string(&self) -> ::std::string::String {match self {$(Self::$variant => {$lut::s($match)},)*}}
+        }
+    }
+}
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index a9ed24e3..cd56f4e3 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -104,7 +104,8 @@ impl NullableMapEntry for super::schema::Dict {
 }
 
 /// A very "basic" fuzzer that will randomly inject tokens wherever applicable
-fn fuzz_tokens(src: &[Token], fuzzwith: impl Fn(bool, &[Token])) {
+fn fuzz_tokens(src: &[u8], fuzzverify: impl Fn(bool, &[Token]) -> bool) {
+    let src_tokens = lex_insecure(src).unwrap();
     static FUZZ_TARGETS: [Token; 2] = [Token::Symbol(Symbol::SymComma), Token::IgnorableComma];
     let mut rng = rand::thread_rng();
     #[inline(always)]
@@ -114,25 +115,37 @@ fn fuzz_tokens(src: &[Token], fuzzwith: impl Fn(bool, &[Token])) {
             .for_each(|_| new_src.push(Token::Symbol(Symbol::SymComma)));
         new_src.len() - start
     }
-    let fuzz_amount = src.iter().filter(|tok| FUZZ_TARGETS.contains(tok)).count();
+    let fuzz_amount = src_tokens
+        .iter()
+        .filter(|tok| FUZZ_TARGETS.contains(tok))
+        .count();
     for _ in 0..(fuzz_amount.pow(2)) {
-        let mut new_src = Vec::with_capacity(src.len());
+        let mut new_src = Vec::with_capacity(src_tokens.len());
         let mut should_pass = true;
-        src.iter().for_each(|tok| match tok {
-            Token::IgnorableComma => {
-                let added = inject(&mut new_src, &mut rng);
-                should_pass &= added <= 1;
+        src_tokens.iter().for_each(|tok| {
+            println!("fuse: {should_pass}");
+            match tok {
+                Token::IgnorableComma => {
+                    let added = inject(&mut new_src, &mut rng);
+                    should_pass &= added <= 1;
+                }
+                Token::Symbol(Symbol::SymComma) => {
+                    let added = inject(&mut new_src, &mut rng);
+                    should_pass &= added == 1;
+                }
+                tok => new_src.push(tok.clone()),
             }
-            Token::Symbol(Symbol::SymComma) => {
-                let added = inject(&mut new_src, &mut rng);
-                should_pass &= added == 1;
-            }
-            tok => new_src.push(tok.clone()),
         });
-        assert!(
-            new_src.iter().all(|tok| tok != &Token::IgnorableComma),
-            "found ignorable comma in rectified source"
-        );
-        fuzzwith(should_pass, &new_src);
+        if fuzzverify(should_pass, &new_src) && !should_pass {
+            panic!(
+                "expected failure for `{}`, but it passed",
+                new_src
+                    .iter()
+                    .map(|tok| format!("{} ", tok.to_string()).into_bytes())
+                    .flatten()
+                    .map(char::from)
+                    .collect::<String>()
+            )
+        }
     }
 }
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 2a27f5c4..d0129088 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -207,18 +207,15 @@ mod tymeta {
     fn fuzz_tymeta_normal() {
         // { maxlen: 10, unique: true, users: "sayan" }
         //   ^start
-        let tok = lex_insecure(
-            b"
-                    maxlen: 10,
-                    unique: true,
-                    auth: {
-                        maybe: true\x01
-                    },
-                    users: \"sayan\"\x01
-                }
-            ",
-        )
-        .unwrap();
+        let tok = b"
+                maxlen: 10,
+                unique: true,
+                auth: {
+                    maybe: true\x01
+                },
+                users: \"sayan\"\x01
+            }
+        ";
         let expected = nullable_dict! {
             "maxlen" => Lit::UnsignedInt(10),
             "unique" => Lit::Bool(true),
@@ -227,38 +224,31 @@ mod tymeta {
             },
             "users" => Lit::Str("sayan".into())
         };
-        fuzz_tokens(&tok, |should_pass, new_src| {
+        fuzz_tokens(tok.as_slice(), |should_pass, new_src| {
             let (tymeta, okay, cursor, data) = schema::fold_tymeta(&new_src);
             if should_pass {
                 assert!(okay, "{:?}", &new_src);
                 assert!(!tymeta.has_more());
                 assert_eq!(cursor, new_src.len());
                 assert_eq!(data, expected);
-            } else if okay {
-                panic!(
-                    "Expected failure but passed for token stream: `{:?}`",
-                    new_src
-                );
             }
+            okay
         });
     }
     #[test]
     fn fuzz_tymeta_with_ty() {
         // list { maxlen: 10, unique: true, type string, users: "sayan" }
         //   ^start
-        let tok = lex_insecure(
-            b"
-                    maxlen: 10,
-                    unique: true,
-                    auth: {
-                        maybe: true\x01
-                    },
-                    type string,
-                    users: \"sayan\"\x01
-                }
-            ",
-        )
-        .unwrap();
+        let tok = b"
+                maxlen: 10,
+                unique: true,
+                auth: {
+                    maybe: true\x01
+                },
+                type string,
+                users: \"sayan\"\x01
+            }
+        ";
         let expected = nullable_dict! {
             "maxlen" => Lit::UnsignedInt(10),
             "unique" => Lit::Bool(true),
@@ -266,16 +256,15 @@ mod tymeta {
                 "maybe" => Lit::Bool(true),
             },
         };
-        fuzz_tokens(&tok, |should_pass, new_src| {
+        fuzz_tokens(tok.as_slice(), |should_pass, new_src| {
             let (tymeta, okay, cursor, data) = schema::fold_tymeta(&new_src);
             if should_pass {
                 assert!(okay);
                 assert!(tymeta.has_more());
                 assert!(new_src[cursor] == Token::Ident(b"string"));
                 assert_eq!(data, expected);
-            } else if okay {
-                panic!("Expected failure but passed for token stream: `{:?}`", tok);
             }
+            okay
         });
     }
 }
@@ -375,8 +364,7 @@ mod layer {
 
     #[test]
     fn fuzz_layer() {
-        let tok = lex_insecure(
-            b"
+        let tok = b"
             list {
                 type list {
                     maxlen: 100,
@@ -384,9 +372,7 @@ mod layer {
                 },
                 unique: true\x01
             }
-        ",
-        )
-        .unwrap();
+        ";
         let expected = vec![
             Layer::new_noreset(b"string", nullable_dict!()),
             Layer::new_noreset(
@@ -397,18 +383,14 @@ mod layer {
             ),
             Layer::new_noreset(b"list", nullable_dict!("unique" => Lit::Bool(true))),
         ];
-        fuzz_tokens(&tok, |should_pass, new_tok| {
+        fuzz_tokens(tok.as_slice(), |should_pass, new_tok| {
             let (layers, c, okay) = schema::fold_layers(&new_tok);
             if should_pass {
                 assert!(okay);
                 assert_eq!(c, new_tok.len());
                 assert_eq!(layers, expected);
-            } else if okay {
-                panic!(
-                    "expected failure but passed for token stream: `{:?}`",
-                    new_tok
-                );
             }
+            okay
         });
     }
 }
diff --git a/server/src/engine/ql/tests/structure_syn.rs b/server/src/engine/ql/tests/structure_syn.rs
index fd78499b..e8065ad9 100644
--- a/server/src/engine/ql/tests/structure_syn.rs
+++ b/server/src/engine/ql/tests/structure_syn.rs
@@ -170,25 +170,22 @@ mod dict {
 
     #[test]
     fn fuzz_dict() {
-        let ret = lex_insecure(
-            b"
-                {
-                    the_tradition_is: \"hello, world\",
-                    could_have_been: {
-                        this: true,
-                        or_maybe_this: 100,
-                        even_this: \"hello, universe!\"\x01
-                    },
-                    but_oh_well: \"it continues to be the 'annoying' phrase\",
-                    lorem: {
-                        ipsum: {
-                            dolor: \"sit amet\"\x01
-                        }\x01
+        let tok = b"
+            {
+                the_tradition_is: \"hello, world\",
+                could_have_been: {
+                    this: true,
+                    or_maybe_this: 100,
+                    even_this: \"hello, universe!\"\x01
+                },
+                but_oh_well: \"it continues to be the 'annoying' phrase\",
+                lorem: {
+                    ipsum: {
+                        dolor: \"sit amet\"\x01
                     }\x01
-                }
-            ",
-        )
-        .unwrap();
+                }\x01
+            }
+        ";
         let ret_dict = nullable_dict! {
             "the_tradition_is" => Lit::Str("hello, world".into()),
             "could_have_been" => nullable_dict! {
@@ -203,16 +200,13 @@ mod dict {
                 }
             }
         };
-        fuzz_tokens(&ret, |should_pass, new_src| {
+        fuzz_tokens(&tok[..], |should_pass, new_src| {
             let r = schema::fold_dict(&new_src);
+            let okay = r.is_some();
             if should_pass {
                 assert_eq!(r.unwrap(), ret_dict)
-            } else if r.is_some() {
-                panic!(
-                    "expected failure, but passed for token stream: `{:?}`",
-                    new_src
-                );
             }
+            okay
         });
     }
 }

From 64d87fe089b15527fdd05d852a4376c863009b06 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 1 Feb 2023 21:34:54 -0800
Subject: [PATCH 115/310] Refactor ddl module

---
 server/src/engine/ql/ast.rs                  | 10 ++-
 server/src/engine/ql/benches.rs              |  2 +-
 server/src/engine/ql/{ddl.rs => ddl/drop.rs} | 66 +++--------------
 server/src/engine/ql/ddl/ins.rs              | 77 ++++++++++++++++++++
 server/src/engine/ql/ddl/mod.rs              | 28 +++++++
 server/src/engine/ql/tests.rs                |  1 -
 server/src/engine/ql/tests/schema_tests.rs   | 18 ++---
 7 files changed, 131 insertions(+), 71 deletions(-)
 rename server/src/engine/ql/{ddl.rs => ddl/drop.rs} (62%)
 create mode 100644 server/src/engine/ql/ddl/ins.rs
 create mode 100644 server/src/engine/ql/ddl/mod.rs

diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast.rs
index c1a00b7a..a92dd3c0 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast.rs
@@ -524,13 +524,13 @@ pub enum Statement<'a> {
     /// Conditions:
     /// - Model view is empty
     /// - Model is not in active use
-    DropModel(ddl::DropModel<'a>),
+    DropModel(ddl::drop::DropModel<'a>),
     /// DDL query to drop a space
     ///
     /// Conditions:
     /// - Space doesn't have any other structures
     /// - Space is not in active use
-    DropSpace(ddl::DropSpace<'a>),
+    DropSpace(ddl::drop::DropSpace<'a>),
     /// DDL query to inspect a space (returns a list of models in the space)
     InspectSpace(Slice<'a>),
     /// DDL query to inspect a model (returns the model definition)
@@ -574,8 +574,10 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], d: Qd) -> LangResult<Sta
             }
             _ => compiler::cold_rerr(LangError::UnknownAlterStatement),
         },
-        Token![drop] if state.remaining() >= 2 => ddl::parse_drop(&mut state),
-        Token::Ident(id) if id.eq_ignore_ascii_case(b"inspect") => ddl::parse_inspect(&mut state),
+        Token![drop] if state.remaining() >= 2 => ddl::drop::parse_drop(&mut state),
+        Token::Ident(id) if id.eq_ignore_ascii_case(b"inspect") => {
+            ddl::ins::parse_inspect(&mut state)
+        }
         // DML
         Token![insert] => {
             dml::ins::InsertStatement::parse_insert(&mut state).map(Statement::Insert)
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index 62f560ab..b4f0bbeb 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -188,7 +188,7 @@ mod ddl_queries {
     mod drop_stmt {
         use {
             super::*,
-            crate::engine::ql::ddl::{DropModel, DropSpace},
+            crate::engine::ql::ddl::drop::{DropModel, DropSpace},
         };
         #[bench]
         fn drop_space(b: &mut Bencher) {
diff --git a/server/src/engine/ql/ddl.rs b/server/src/engine/ql/ddl/drop.rs
similarity index 62%
rename from server/src/engine/ql/ddl.rs
rename to server/src/engine/ql/ddl/drop.rs
index bb49e8b7..8c67e84b 100644
--- a/server/src/engine/ql/ddl.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -1,5 +1,5 @@
 /*
- * Created on Wed Nov 16 2022
+ * Created on Wed Feb 01 2023
  *
  * This file is a part of Skytable
  * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@@ -7,7 +7,7 @@
  * vision to provide flexibility in data modelling without compromising
  * on performance, queryability or scalability.
  *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published by
@@ -25,14 +25,11 @@
 */
 
 #[cfg(test)]
-use super::ast::InplaceData;
-use {
-    super::{
-        ast::{Entity, QueryData, State, Statement},
-        lex::{Slice, Token},
-        LangError, LangResult,
-    },
-    crate::util::compiler,
+use crate::engine::ql::ast::InplaceData;
+use crate::engine::ql::{
+    ast::{Entity, QueryData, State, Statement},
+    lex::{Slice, Token},
+    LangError, LangResult,
 };
 
 #[derive(Debug, PartialEq)]
@@ -45,7 +42,7 @@ pub struct DropSpace<'a> {
 impl<'a> DropSpace<'a> {
     #[inline(always)]
     /// Instantiate
-    pub(super) const fn new(space: Slice<'a>, force: bool) -> Self {
+    pub const fn new(space: Slice<'a>, force: bool) -> Self {
         Self { space, force }
     }
 }
@@ -67,9 +64,7 @@ impl<'a> DropModel<'a> {
 /// ## Panic
 ///
 /// If token stream length is < 2
-pub(super) fn parse_drop<'a, Qd: QueryData<'a>>(
-    state: &mut State<'a, Qd>,
-) -> LangResult<Statement<'a>> {
+pub fn parse_drop<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Statement<'a>> {
     match state.fw_read() {
         Token![model] => {
             // we have a model. now parse entity and see if we should force deletion
@@ -103,50 +98,9 @@ pub(super) fn parse_drop<'a, Qd: QueryData<'a>>(
 }
 
 #[cfg(test)]
-pub(super) fn parse_drop_full<'a>(tok: &'a [Token]) -> LangResult<Statement<'a>> {
+pub fn parse_drop_full<'a>(tok: &'a [Token]) -> LangResult<Statement<'a>> {
     let mut state = State::new(tok, InplaceData::new());
     let r = self::parse_drop(&mut state);
     assert_full_tt!(state);
     r
 }
-
-pub(super) fn parse_inspect<'a, Qd: QueryData<'a>>(
-    state: &mut State<'a, Qd>,
-) -> LangResult<Statement<'a>> {
-    /*
-        inpsect model <entity>
-        inspect space <entity>
-        inspect spaces
-
-        min length -> (<model> | <space>) <model> = 2
-    */
-
-    if compiler::unlikely(state.remaining() < 1) {
-        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
-    }
-
-    match state.fw_read() {
-        Token![model] => Entity::attempt_process_entity_result(state).map(Statement::InspectModel),
-        Token![space] if state.cursor_has_ident_rounded() => {
-            Ok(Statement::InspectSpace(unsafe {
-                // UNSAFE(@ohsayan): Safe because of the match predicate
-                extract!(state.fw_read(), Token::Ident(ref space) => space)
-            }))
-        }
-        Token::Ident(id) if id.eq_ignore_ascii_case(b"spaces") && state.exhausted() => {
-            Ok(Statement::InspectSpaces)
-        }
-        _ => {
-            state.cursor_back();
-            Err(LangError::ExpectedStatement)
-        }
-    }
-}
-
-#[cfg(test)]
-pub(super) fn parse_inspect_full<'a>(tok: &'a [Token]) -> LangResult<Statement<'a>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let r = self::parse_inspect(&mut state);
-    assert_full_tt!(state);
-    r
-}
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
new file mode 100644
index 00000000..1ee7ce04
--- /dev/null
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -0,0 +1,77 @@
+/*
+ * Created on Wed Feb 01 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[cfg(test)]
+use crate::engine::ql::ast::InplaceData;
+use crate::{
+    engine::ql::{
+        ast::{Entity, QueryData, State, Statement},
+        lex::Token,
+        LangError, LangResult,
+    },
+    util::compiler,
+};
+
+pub fn parse_inspect<'a, Qd: QueryData<'a>>(
+    state: &mut State<'a, Qd>,
+) -> LangResult<Statement<'a>> {
+    /*
+        inpsect model <entity>
+        inspect space <entity>
+        inspect spaces
+
+        min length -> (<model> | <space>) <model> = 2
+    */
+
+    if compiler::unlikely(state.remaining() < 1) {
+        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+    }
+
+    match state.fw_read() {
+        Token![model] => Entity::attempt_process_entity_result(state).map(Statement::InspectModel),
+        Token![space] if state.cursor_has_ident_rounded() => {
+            Ok(Statement::InspectSpace(unsafe {
+                // UNSAFE(@ohsayan): Safe because of the match predicate
+                extract!(state.fw_read(), Token::Ident(ref space) => space)
+            }))
+        }
+        Token::Ident(id) if id.eq_ignore_ascii_case(b"spaces") && state.exhausted() => {
+            Ok(Statement::InspectSpaces)
+        }
+        _ => {
+            state.cursor_back();
+            Err(LangError::ExpectedStatement)
+        }
+    }
+}
+
+#[cfg(test)]
+pub fn parse_inspect_full<'a>(tok: &'a [Token]) -> LangResult<Statement<'a>> {
+    let mut state = State::new(tok, InplaceData::new());
+    let r = self::parse_inspect(&mut state);
+    assert_full_tt!(state);
+    r
+}
diff --git a/server/src/engine/ql/ddl/mod.rs b/server/src/engine/ql/ddl/mod.rs
new file mode 100644
index 00000000..2e04ebf5
--- /dev/null
+++ b/server/src/engine/ql/ddl/mod.rs
@@ -0,0 +1,28 @@
+/*
+ * Created on Wed Nov 16 2022
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+pub mod drop;
+pub mod ins;
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index cd56f4e3..7696b76a 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -123,7 +123,6 @@ fn fuzz_tokens(src: &[u8], fuzzverify: impl Fn(bool, &[Token]) -> bool) {
         let mut new_src = Vec::with_capacity(src_tokens.len());
         let mut should_pass = true;
         src_tokens.iter().for_each(|tok| {
-            println!("fuse: {should_pass}");
             match tok {
                 Token::IgnorableComma => {
                     let added = inject(&mut new_src, &mut rng);
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index d0129088..735968bc 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -43,7 +43,7 @@ mod inspect {
     fn inspect_space() {
         let tok = lex_insecure(b"inspect space myspace").unwrap();
         assert_eq!(
-            ddl::parse_inspect_full(&tok[1..]).unwrap(),
+            ddl::ins::parse_inspect_full(&tok[1..]).unwrap(),
             Statement::InspectSpace(b"myspace")
         );
     }
@@ -51,12 +51,12 @@ mod inspect {
     fn inspect_model() {
         let tok = lex_insecure(b"inspect model users").unwrap();
         assert_eq!(
-            ddl::parse_inspect_full(&tok[1..]).unwrap(),
+            ddl::ins::parse_inspect_full(&tok[1..]).unwrap(),
             Statement::InspectModel(Entity::Single(b"users"))
         );
         let tok = lex_insecure(b"inspect model tweeter.users").unwrap();
         assert_eq!(
-            ddl::parse_inspect_full(&tok[1..]).unwrap(),
+            ddl::ins::parse_inspect_full(&tok[1..]).unwrap(),
             Statement::InspectModel(Entity::Full(b"tweeter", b"users"))
         );
     }
@@ -64,7 +64,7 @@ mod inspect {
     fn inspect_spaces() {
         let tok = lex_insecure(b"inspect spaces").unwrap();
         assert_eq!(
-            ddl::parse_inspect_full(&tok[1..]).unwrap(),
+            ddl::ins::parse_inspect_full(&tok[1..]).unwrap(),
             Statement::InspectSpaces
         );
     }
@@ -1212,14 +1212,14 @@ mod ddl_other_query_tests {
         super::*,
         crate::engine::ql::{
             ast::{Entity, Statement},
-            ddl::{self, DropModel, DropSpace},
+            ddl::drop::{self, DropModel, DropSpace},
         },
     };
     #[test]
     fn drop_space() {
         let src = lex_insecure(br"drop space myspace").unwrap();
         assert_eq!(
-            ddl::parse_drop_full(&src[1..]).unwrap(),
+            drop::parse_drop_full(&src[1..]).unwrap(),
             Statement::DropSpace(DropSpace::new(b"myspace", false))
         );
     }
@@ -1227,7 +1227,7 @@ mod ddl_other_query_tests {
     fn drop_space_force() {
         let src = lex_insecure(br"drop space myspace force").unwrap();
         assert_eq!(
-            ddl::parse_drop_full(&src[1..]).unwrap(),
+            drop::parse_drop_full(&src[1..]).unwrap(),
             Statement::DropSpace(DropSpace::new(b"myspace", true))
         );
     }
@@ -1235,7 +1235,7 @@ mod ddl_other_query_tests {
     fn drop_model() {
         let src = lex_insecure(br"drop model mymodel").unwrap();
         assert_eq!(
-            ddl::parse_drop_full(&src[1..]).unwrap(),
+            drop::parse_drop_full(&src[1..]).unwrap(),
             Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), false))
         );
     }
@@ -1243,7 +1243,7 @@ mod ddl_other_query_tests {
     fn drop_model_force() {
         let src = lex_insecure(br"drop model mymodel force").unwrap();
         assert_eq!(
-            ddl::parse_drop_full(&src[1..]).unwrap(),
+            drop::parse_drop_full(&src[1..]).unwrap(),
             Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), true))
         );
     }

From f98478b2742a827f2c9f4f8a923bd450b3fd018b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 2 Feb 2023 22:56:38 -0800
Subject: [PATCH 116/310] Fix state machines, refactor schema and add `ASTNode`
 def and impls

This is one huge commit that fixes a bunch of issues we've had with
the schema and surrounding impls. Here are some of them
- The type metadata syn parse was incredibly broken. More than
one trailing comma or other punctuation following a trailing symbol
(as in a comma) is illegal
- The `*_full` set of methods were repetitive and annoying. That has
been fixed (not the most elegant solution, but it works and fits
well)
- As a result of the first issue, fuzz targets kept blowing up,
rightfully. This has been fixed too. The fuzz needs to be more
deterministic covering all permutations, and this is something I'll
fix up ahead.

Finally, I can now happily say that the QL impl is rock solid. There is
another possible bug that I'm tracking down, but that will be fixed.
---
 server/src/engine/idx/mtchm/imp.rs          |    6 +-
 server/src/engine/idx/mtchm/mod.rs          |    8 +-
 server/src/engine/ql/{ast.rs => ast/mod.rs} |   53 +-
 server/src/engine/ql/ast/traits.rs          |   68 ++
 server/src/engine/ql/ddl/alt.rs             |  196 ++++
 server/src/engine/ql/ddl/crt.rs             |  160 +++
 server/src/engine/ql/ddl/drop.rs            |   91 +-
 server/src/engine/ql/ddl/ins.rs             |   19 +-
 server/src/engine/ql/ddl/mod.rs             |    8 +-
 server/src/engine/ql/ddl/syn.rs             |  614 ++++++++++
 server/src/engine/ql/dml/del.rs             |   19 +-
 server/src/engine/ql/dml/ins.rs             |   82 +-
 server/src/engine/ql/dml/mod.rs             |   35 +-
 server/src/engine/ql/dml/sel.rs             |   19 +-
 server/src/engine/ql/dml/upd.rs             |   12 +-
 server/src/engine/ql/macros.rs              |   16 +-
 server/src/engine/ql/mod.rs                 |    1 -
 server/src/engine/ql/schema.rs              | 1135 ------------------
 server/src/engine/ql/tests.rs               |   50 +-
 server/src/engine/ql/tests/dml_tests.rs     |  133 ++-
 server/src/engine/ql/tests/schema_tests.rs  | 1148 +++++++++----------
 server/src/engine/ql/tests/structure_syn.rs |   68 +-
 server/src/util/mod.rs                      |   41 +-
 23 files changed, 1936 insertions(+), 2046 deletions(-)
 rename server/src/engine/ql/{ast.rs => ast/mod.rs} (93%)
 create mode 100644 server/src/engine/ql/ast/traits.rs
 create mode 100644 server/src/engine/ql/ddl/alt.rs
 create mode 100644 server/src/engine/ql/ddl/crt.rs
 create mode 100644 server/src/engine/ql/ddl/syn.rs
 delete mode 100644 server/src/engine/ql/schema.rs

diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index 64212e66..92261a2c 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -24,6 +24,8 @@
  *
 */
 
+#[cfg(debug_assertions)]
+use super::CHTRuntimeLog;
 use super::{
     super::{
         super::sync::atm::{upin, Guard},
@@ -31,7 +33,7 @@ use super::{
     },
     iter::{IterKV, IterKey, IterVal},
     meta::{Config, Key, TreeElement, Value},
-    CHTRuntimeLog, Tree,
+    Tree,
 };
 use std::{borrow::Borrow, sync::Arc};
 
@@ -48,6 +50,7 @@ where
 {
     const PREALLOC: bool = false;
 
+    #[cfg(debug_assertions)]
     type Metrics = CHTRuntimeLog;
 
     fn idx_init() -> Self {
@@ -170,6 +173,7 @@ where
 {
     const PREALLOC: bool = false;
 
+    #[cfg(debug_assertions)]
     type Metrics = CHTRuntimeLog;
 
     fn idx_init() -> Self {
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index 5cdd02e6..074a7095 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -36,10 +36,12 @@ use self::{
     iter::{IterKV, IterKey, IterVal},
     meta::{CompressState, Config, DefConfig, LNode, NodeFlag, TreeElement},
 };
+#[cfg(debug_assertions)]
+use super::super::sync::atm::ORD_ACQ;
 use super::{
     super::{
         mem::UArray,
-        sync::atm::{self, cpin, upin, Atomic, Guard, Owned, Shared, ORD_ACQ, ORD_ACR, ORD_RLX},
+        sync::atm::{self, cpin, upin, Atomic, Guard, Owned, Shared, ORD_ACR, ORD_RLX},
     },
     AsKey,
 };
@@ -99,12 +101,12 @@ impl CHTRuntimeLog {
         fn repsplit(self: &Self) -> usize {
             self.data.split.load(ORD_RLX)
         } else {
-            void!()
+            0
         }
         fn replnode(self: &Self) -> usize {
             self.data.hln.load(ORD_RLX)
         } else {
-            void!()
+            0
         }
     }
 }
diff --git a/server/src/engine/ql/ast.rs b/server/src/engine/ql/ast/mod.rs
similarity index 93%
rename from server/src/engine/ql/ast.rs
rename to server/src/engine/ql/ast/mod.rs
index a92dd3c0..5976cfcc 100644
--- a/server/src/engine/ql/ast.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -24,11 +24,16 @@
  *
 */
 
+pub mod traits;
+
+#[cfg(test)]
+pub use traits::{parse_ast_node_full, parse_ast_node_multiple_full};
 use {
+    self::traits::ASTNode,
     super::{
         ddl, dml,
         lex::{LitIR, Slice, Token},
-        schema, LangError, LangResult,
+        LangError, LangResult,
     },
     crate::{
         engine::core::DataType,
@@ -51,6 +56,12 @@ pub struct State<'a, Qd> {
     f: bool,
 }
 
+impl<'a> State<'a, InplaceData> {
+    pub const fn new_inplace(tok: &'a [Token<'a>]) -> Self {
+        Self::new(tok, InplaceData::new())
+    }
+}
+
 impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     #[inline(always)]
     /// Create a new [`State`] instance using the given tokens and data
@@ -512,13 +523,13 @@ pub enum Statement<'a> {
     /// DDL query to switch between spaces and models
     Use(Entity<'a>),
     /// DDL query to create a model
-    CreateModel(schema::Model<'a>),
+    CreateModel(ddl::crt::Model<'a>),
     /// DDL query to create a space
-    CreateSpace(schema::Space<'a>),
+    CreateSpace(ddl::crt::Space<'a>),
     /// DDL query to alter a space (properties)
-    AlterSpace(schema::AlterSpace<'a>),
+    AlterSpace(ddl::alt::AlterSpace<'a>),
     /// DDL query to alter a model (properties, field types, etc)
-    AlterModel(schema::Alter<'a>),
+    AlterModel(ddl::alt::AlterModel<'a>),
     /// DDL query to drop a model
     ///
     /// Conditions:
@@ -557,21 +568,13 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], d: Qd) -> LangResult<Sta
         // DDL
         Token![use] => Entity::attempt_process_entity_result(&mut state).map(Statement::Use),
         Token![create] => match state.fw_read() {
-            Token![model] => {
-                schema::parse_model_from_tokens(&mut state).map(Statement::CreateModel)
-            }
-            Token![space] => {
-                schema::parse_space_from_tokens(&mut state).map(Statement::CreateSpace)
-            }
+            Token![model] => ASTNode::from_state(&mut state).map(Statement::CreateModel),
+            Token![space] => ASTNode::from_state(&mut state).map(Statement::CreateSpace),
             _ => compiler::cold_rerr(LangError::UnknownCreateStatement),
         },
         Token![alter] => match state.fw_read() {
-            Token![model] => {
-                schema::parse_alter_kind_from_tokens(&mut state).map(Statement::AlterModel)
-            }
-            Token![space] => {
-                schema::parse_alter_space_from_tokens(&mut state).map(Statement::AlterSpace)
-            }
+            Token![model] => ASTNode::from_state(&mut state).map(Statement::AlterModel),
+            Token![space] => ASTNode::from_state(&mut state).map(Statement::AlterSpace),
             _ => compiler::cold_rerr(LangError::UnknownAlterStatement),
         },
         Token![drop] if state.remaining() >= 2 => ddl::drop::parse_drop(&mut state),
@@ -579,18 +582,10 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], d: Qd) -> LangResult<Sta
             ddl::ins::parse_inspect(&mut state)
         }
         // DML
-        Token![insert] => {
-            dml::ins::InsertStatement::parse_insert(&mut state).map(Statement::Insert)
-        }
-        Token![select] => {
-            dml::sel::SelectStatement::parse_select(&mut state).map(Statement::Select)
-        }
-        Token![update] => {
-            dml::upd::UpdateStatement::parse_update(&mut state).map(Statement::Update)
-        }
-        Token![delete] => {
-            dml::del::DeleteStatement::parse_delete(&mut state).map(Statement::Delete)
-        }
+        Token![insert] => ASTNode::from_state(&mut state).map(Statement::Insert),
+        Token![select] => ASTNode::from_state(&mut state).map(Statement::Select),
+        Token![update] => ASTNode::from_state(&mut state).map(Statement::Update),
+        Token![delete] => ASTNode::from_state(&mut state).map(Statement::Delete),
         _ => compiler::cold_rerr(LangError::ExpectedStatement),
     }
 }
diff --git a/server/src/engine/ql/ast/traits.rs b/server/src/engine/ql/ast/traits.rs
new file mode 100644
index 00000000..7d891ec2
--- /dev/null
+++ b/server/src/engine/ql/ast/traits.rs
@@ -0,0 +1,68 @@
+/*
+ * Created on Thu Feb 02 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[cfg(test)]
+use crate::engine::ql::{ast::InplaceData, lex::Token};
+use crate::engine::ql::{
+    ast::{QueryData, State},
+    LangResult,
+};
+
+pub trait ASTNode<'a>: Sized {
+    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self>;
+    #[cfg(test)]
+    fn from_insecure_tokens_full(tok: &'a [Token<'a>]) -> LangResult<Self> {
+        let mut state = State::new(tok, InplaceData::new());
+        let r = <Self as ASTNode>::from_state(&mut state)?;
+        assert!(state.exhausted());
+        Ok(r)
+    }
+    #[cfg(test)]
+    fn multiple_from_state<Qd: QueryData<'a>>(_: &mut State<'a, Qd>) -> LangResult<Vec<Self>> {
+        unimplemented!()
+    }
+    #[cfg(test)]
+    fn multiple_from_insecure_tokens_full(tok: &'a [Token<'a>]) -> LangResult<Vec<Self>> {
+        let mut state = State::new(tok, InplaceData::new());
+        let r = Self::multiple_from_state(&mut state);
+        if state.exhausted() && state.okay() {
+            r
+        } else {
+            Err(super::super::LangError::UnexpectedToken)
+        }
+    }
+}
+
+#[cfg(test)]
+pub fn parse_ast_node_full<'a, N: ASTNode<'a>>(tok: &'a [Token<'a>]) -> LangResult<N> {
+    N::from_insecure_tokens_full(tok)
+}
+#[cfg(test)]
+pub fn parse_ast_node_multiple_full<'a, N: ASTNode<'a>>(
+    tok: &'a [Token<'a>],
+) -> LangResult<Vec<N>> {
+    N::multiple_from_insecure_tokens_full(tok)
+}
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
new file mode 100644
index 00000000..2c2e838f
--- /dev/null
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -0,0 +1,196 @@
+/*
+ * Created on Thu Feb 02 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::syn::{self, Dict, DictFoldState, ExpandedField};
+use crate::{
+    engine::ql::{
+        ast::{QueryData, State},
+        lex::{Slice, Token},
+        LangError, LangResult,
+    },
+    util::compiler,
+};
+
+#[derive(Debug, PartialEq)]
+/// An alter space query with corresponding data
+pub struct AlterSpace<'a> {
+    space_name: Slice<'a>,
+    updated_props: Dict,
+}
+
+impl<'a> AlterSpace<'a> {
+    pub fn new(space_name: Slice<'a>, updated_props: Dict) -> Self {
+        Self {
+            space_name,
+            updated_props,
+        }
+    }
+    #[inline(always)]
+    /// Parse alter space from tokens
+    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        if compiler::unlikely(state.remaining() <= 3) {
+            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+        }
+        let space_name = state.fw_read();
+        state.poison_if_not(state.cursor_eq(Token![with]));
+        state.cursor_ahead(); // ignore errors
+        state.poison_if_not(state.cursor_eq(Token![open {}]));
+        state.cursor_ahead(); // ignore errors
+
+        if compiler::unlikely(!state.okay()) {
+            return Err(LangError::UnexpectedToken);
+        }
+
+        let space_name = unsafe { extract!(space_name, Token::Ident(ref space) => space.clone()) };
+        let mut d = Dict::new();
+        syn::rfold_dict(DictFoldState::CB_OR_IDENT, state, &mut d);
+        if state.okay() {
+            Ok(AlterSpace {
+                space_name,
+                updated_props: d,
+            })
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    }
+}
+
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+pub struct AlterModel<'a> {
+    model: Slice<'a>,
+    kind: AlterKind<'a>,
+}
+
+impl<'a> AlterModel<'a> {
+    #[inline(always)]
+    pub fn new(model: Slice<'a>, kind: AlterKind<'a>) -> Self {
+        Self { model, kind }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+/// The alter operation kind
+pub enum AlterKind<'a> {
+    Add(Box<[ExpandedField<'a>]>),
+    Remove(Box<[Slice<'a>]>),
+    Update(Box<[ExpandedField<'a>]>),
+}
+
+impl<'a> AlterModel<'a> {
+    #[inline(always)]
+    /// Parse an [`AlterKind`] from the given token stream
+    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        // alter model mymodel remove x
+        if state.remaining() <= 2 || !state.cursor_has_ident_rounded() {
+            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+        }
+        let model_name = unsafe { extract!(state.fw_read(), Token::Ident(ref l) => l.clone()) };
+        let kind = match state.fw_read() {
+            Token![add] => AlterKind::alter_add(state),
+            Token![remove] => AlterKind::alter_remove(state),
+            Token![update] => AlterKind::alter_update(state),
+            _ => Err(LangError::ExpectedStatement),
+        };
+        kind.map(|kind| AlterModel::new(model_name, kind))
+    }
+}
+
+impl<'a> AlterKind<'a> {
+    #[inline(always)]
+    /// Parse the expression for `alter model <> add (..)`
+    fn alter_add<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        ExpandedField::parse_multiple(state).map(Self::Add)
+    }
+    #[inline(always)]
+    /// Parse the expression for `alter model <> add (..)`
+    fn alter_update<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        ExpandedField::parse_multiple(state).map(Self::Update)
+    }
+    #[inline(always)]
+    /// Parse the expression for `alter model <> remove (..)`
+    fn alter_remove<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        const DEFAULT_REMOVE_COL_CNT: usize = 4;
+        /*
+            WARNING: No trailing commas allowed
+            <remove> ::= <ident> | <openparen> (<ident> <comma>)*<closeparen>
+        */
+        if compiler::unlikely(state.exhausted()) {
+            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+        }
+
+        let r = match state.fw_read() {
+            Token::Ident(id) => Box::new([id.clone()]),
+            Token![() open] => {
+                let mut stop = false;
+                let mut cols = Vec::with_capacity(DEFAULT_REMOVE_COL_CNT);
+                while state.loop_tt() && !stop {
+                    match state.fw_read() {
+                        Token::Ident(ref ident) => {
+                            cols.push(ident.clone());
+                            let nx_comma = state.cursor_rounded_eq(Token![,]);
+                            let nx_close = state.cursor_rounded_eq(Token![() close]);
+                            state.poison_if_not(nx_comma | nx_close);
+                            stop = nx_close;
+                            state.cursor_ahead_if(state.okay());
+                        }
+                        _ => {
+                            state.cursor_back();
+                            state.poison();
+                            break;
+                        }
+                    }
+                }
+                state.poison_if_not(stop);
+                if state.okay() {
+                    cols.into_boxed_slice()
+                } else {
+                    return Err(LangError::UnexpectedToken);
+                }
+            }
+            _ => return Err(LangError::ExpectedStatement),
+        };
+        Ok(Self::Remove(r))
+    }
+}
+
+mod impls {
+    use super::{AlterModel, AlterSpace};
+    use crate::engine::ql::{
+        ast::{traits::ASTNode, QueryData, State},
+        LangResult,
+    };
+    impl<'a> ASTNode<'a> for AlterModel<'a> {
+        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            Self::parse(state)
+        }
+    }
+    impl<'a> ASTNode<'a> for AlterSpace<'a> {
+        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            Self::parse(state)
+        }
+    }
+}
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
new file mode 100644
index 00000000..b57cf93f
--- /dev/null
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -0,0 +1,160 @@
+/*
+ * Created on Thu Feb 02 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::syn::{self, Dict, DictFoldState, Field};
+use crate::{
+    engine::ql::{
+        ast::{QueryData, State},
+        lex::{Slice, Token},
+        LangError, LangResult,
+    },
+    util::compiler,
+};
+
+#[derive(Debug, PartialEq)]
+/// A space
+pub struct Space<'a> {
+    /// the space name
+    pub(super) space_name: Slice<'a>,
+    /// properties
+    pub(super) props: Dict,
+}
+
+impl<'a> Space<'a> {
+    #[inline(always)]
+    /// Parse space data from the given tokens
+    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        // smallest declaration: `create space myspace` -> >= 1 token
+        if compiler::unlikely(state.remaining() < 1) {
+            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+        }
+        let space_name = state.fw_read();
+        state.poison_if_not(space_name.is_ident());
+        // either we have `with` or nothing. don't be stupid
+        let has_more_properties = state.cursor_rounded_eq(Token![with]);
+        state.poison_if_not(has_more_properties | state.exhausted());
+        state.cursor_ahead_if(has_more_properties); // +WITH
+        let mut d = Dict::new();
+        // properties
+        if has_more_properties && state.okay() {
+            syn::rfold_dict(DictFoldState::OB, state, &mut d);
+        }
+        if state.okay() {
+            Ok(Space {
+                space_name: unsafe { extract!(space_name, Token::Ident(ref id) => id.clone()) },
+                props: d,
+            })
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+/// A model definition
+pub struct Model<'a> {
+    /// the model name
+    model_name: Slice<'a>,
+    /// the fields
+    fields: Vec<Field<'a>>,
+    /// properties
+    props: Dict,
+}
+
+/*
+    model definition:
+    create model mymodel(
+        [primary|null] ident: type,
+    )
+*/
+
+impl<'a> Model<'a> {
+    pub fn new(model_name: Slice<'a>, fields: Vec<Field<'a>>, props: Dict) -> Self {
+        Self {
+            model_name,
+            fields,
+            props,
+        }
+    }
+
+    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        if compiler::unlikely(state.remaining() < 10) {
+            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+        }
+        // field name; ignore errors
+        let field_name = state.fw_read();
+        state.poison_if_not(field_name.is_ident());
+        state.poison_if_not(state.cursor_eq(Token![() open]));
+        state.cursor_ahead();
+        // fields
+        let mut stop = false;
+        let mut fields = Vec::with_capacity(2);
+        while state.loop_tt() && !stop {
+            fields.push(Field::parse(state)?);
+            let nx_close = state.cursor_rounded_eq(Token![() close]);
+            let nx_comma = state.cursor_rounded_eq(Token![,]);
+            state.poison_if_not(nx_close | nx_comma);
+            state.cursor_ahead_if(nx_close | nx_comma);
+            stop = nx_close;
+        }
+        state.poison_if_not(stop);
+        // check props
+        let mut props = Dict::new();
+        if state.cursor_rounded_eq(Token![with]) {
+            state.cursor_ahead();
+            // parse props
+            syn::rfold_dict(DictFoldState::OB, state, &mut props);
+        }
+        // we're done
+        if state.okay() {
+            Ok(Self {
+                model_name: unsafe { extract!(field_name, Token::Ident(id) => *id) },
+                fields,
+                props,
+            })
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    }
+}
+
+mod impls {
+    use super::{Model, Space};
+    use crate::engine::ql::{
+        ast::{traits::ASTNode, QueryData, State},
+        LangResult,
+    };
+    impl<'a> ASTNode<'a> for Space<'a> {
+        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            Self::parse(state)
+        }
+    }
+    impl<'a> ASTNode<'a> for Model<'a> {
+        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            Self::parse(state)
+        }
+    }
+}
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index 8c67e84b..c50bfc9f 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -24,8 +24,6 @@
  *
 */
 
-#[cfg(test)]
-use crate::engine::ql::ast::InplaceData;
 use crate::engine::ql::{
     ast::{Entity, QueryData, State, Statement},
     lex::{Slice, Token},
@@ -45,6 +43,25 @@ impl<'a> DropSpace<'a> {
     pub const fn new(space: Slice<'a>, force: bool) -> Self {
         Self { space, force }
     }
+    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<DropSpace<'a>> {
+        if state.cursor_is_ident() {
+            let ident = state.fw_read();
+            // should we force drop?
+            let force = state.cursor_rounded_eq(Token::Ident(b"force"));
+            state.cursor_ahead_if(force);
+            // either `force` or nothing
+            if state.exhausted() {
+                return Ok(DropSpace::new(
+                    unsafe {
+                        // UNSAFE(@ohsayan): Safe because the match predicate ensures that tok[1] is indeed an ident
+                        extract!(ident, Token::Ident(ref space) => *space)
+                    },
+                    force,
+                ));
+            }
+        }
+        Err(LangError::UnexpectedToken)
+    }
 }
 
 #[derive(Debug, PartialEq)]
@@ -58,6 +75,16 @@ impl<'a> DropModel<'a> {
     pub fn new(entity: Entity<'a>, force: bool) -> Self {
         Self { entity, force }
     }
+    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        let e = Entity::attempt_process_entity_result(state)?;
+        let force = state.cursor_rounded_eq(Token::Ident(b"force"));
+        state.cursor_ahead_if(force);
+        if state.exhausted() {
+            return Ok(DropModel::new(e, force));
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    }
 }
 
 // drop (<space> | <model>) <ident> [<force>]
@@ -66,41 +93,33 @@ impl<'a> DropModel<'a> {
 /// If token stream length is < 2
 pub fn parse_drop<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Statement<'a>> {
     match state.fw_read() {
-        Token![model] => {
-            // we have a model. now parse entity and see if we should force deletion
-            let e = Entity::attempt_process_entity_result(state)?;
-            let force = state.cursor_rounded_eq(Token::Ident(b"force"));
-            state.cursor_ahead_if(force);
-            // if we've exhausted the stream, we're good to go (either `force`, or nothing)
-            if state.exhausted() {
-                return Ok(Statement::DropModel(DropModel::new(e, force)));
-            }
-        }
-        Token![space] if state.cursor_is_ident() => {
-            let ident = state.fw_read();
-            // should we force drop?
-            let force = state.cursor_rounded_eq(Token::Ident(b"force"));
-            state.cursor_ahead_if(force);
-            // either `force` or nothing
-            if state.exhausted() {
-                return Ok(Statement::DropSpace(DropSpace::new(
-                    unsafe {
-                        // UNSAFE(@ohsayan): Safe because the match predicate ensures that tok[1] is indeed an ident
-                        extract!(ident, Token::Ident(ref space) => *space)
-                    },
-                    force,
-                )));
-            }
-        }
-        _ => {}
+        Token![model] => DropModel::parse(state).map(Statement::DropModel),
+        Token![space] => return DropSpace::parse(state).map(Statement::DropSpace),
+        _ => Err(LangError::UnexpectedToken),
     }
-    Err(LangError::UnexpectedToken)
 }
 
-#[cfg(test)]
-pub fn parse_drop_full<'a>(tok: &'a [Token]) -> LangResult<Statement<'a>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let r = self::parse_drop(&mut state);
-    assert_full_tt!(state);
-    r
+pub use impls::DropStatementAST;
+mod impls {
+    use super::{DropModel, DropSpace};
+    use crate::engine::ql::{
+        ast::{traits::ASTNode, QueryData, State, Statement},
+        LangResult,
+    };
+    impl<'a> ASTNode<'a> for DropModel<'a> {
+        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            Self::parse(state)
+        }
+    }
+    impl<'a> ASTNode<'a> for DropSpace<'a> {
+        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            Self::parse(state)
+        }
+    }
+    pub struct DropStatementAST<'a>(pub Statement<'a>);
+    impl<'a> ASTNode<'a> for DropStatementAST<'a> {
+        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            super::parse_drop(state).map(Self)
+        }
+    }
 }
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
index 1ee7ce04..68ea9314 100644
--- a/server/src/engine/ql/ddl/ins.rs
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -24,8 +24,6 @@
  *
 */
 
-#[cfg(test)]
-use crate::engine::ql::ast::InplaceData;
 use crate::{
     engine::ql::{
         ast::{Entity, QueryData, State, Statement},
@@ -68,10 +66,15 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
     }
 }
 
-#[cfg(test)]
-pub fn parse_inspect_full<'a>(tok: &'a [Token]) -> LangResult<Statement<'a>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let r = self::parse_inspect(&mut state);
-    assert_full_tt!(state);
-    r
+pub use impls::InspectStatementAST;
+mod impls {
+    use crate::engine::ql::ast::{traits::ASTNode, QueryData, State, Statement};
+    pub struct InspectStatementAST<'a>(pub Statement<'a>);
+    impl<'a> ASTNode<'a> for InspectStatementAST<'a> {
+        fn from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> crate::engine::ql::LangResult<Self> {
+            super::parse_inspect(state).map(Self)
+        }
+    }
 }
diff --git a/server/src/engine/ql/ddl/mod.rs b/server/src/engine/ql/ddl/mod.rs
index 2e04ebf5..2d84e344 100644
--- a/server/src/engine/ql/ddl/mod.rs
+++ b/server/src/engine/ql/ddl/mod.rs
@@ -24,5 +24,9 @@
  *
 */
 
-pub mod drop;
-pub mod ins;
+#[macro_use]
+pub(super) mod syn;
+pub(super) mod alt;
+pub(super) mod crt;
+pub(super) mod drop;
+pub(super) mod ins;
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
new file mode 100644
index 00000000..e4cd4346
--- /dev/null
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -0,0 +1,614 @@
+/*
+ * Created on Wed Feb 01 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/*
+    Most grammar tools are pretty much "off the shelf" which makes some things incredibly hard to achieve (such
+    as custom error injection logic). To make things icier, Rust's integration with these tools (like lex) is not
+    very "refined." Hence, it is best for us to implement our own parsers. In the future, I plan to optimize our
+    rule checkers but that's not a concern at the moment.
+
+    This module makes use of DFAs with additional flags, accepting a token stream as input to generate appropriate
+    structures, and have provable correctness. Hence, the unsafe code used here is correct, because the states are
+    only transitioned to if the input is accepted. If you do find otherwise, please file a bug report. The
+    transitions are currently very inefficient but can be made much faster.
+
+    TODO: The SMs can be reduced greatly, enocded to fixed-sized structures even, so do that
+    NOTE: The `ASTNode` impls are test-only. most of the time they do stupid things. we should never rely on `ASTNode`
+    impls for `syn` elements
+
+    --
+    Sayan (@ohsayan)
+    Feb. 2, 2023
+*/
+
+use crate::{
+    engine::ql::{
+        ast::{QueryData, State},
+        lex::{LitIR, LitIROwned, Slice, Token},
+        LangError, LangResult,
+    },
+    util::{compiler, MaybeInit},
+};
+use std::{collections::HashMap, str};
+
+#[derive(Debug, PartialEq)]
+/// A dictionary entry type. Either a literal or another dictionary
+pub enum DictEntry {
+    Lit(LitIROwned),
+    Map(Dict),
+}
+
+impl<'a> From<LitIR<'a>> for DictEntry {
+    fn from(l: LitIR<'a>) -> Self {
+        Self::Lit(l.to_litir_owned())
+    }
+}
+
+impl From<Dict> for DictEntry {
+    fn from(d: Dict) -> Self {
+        Self::Map(d)
+    }
+}
+
+/// A metadata dictionary
+pub type Dict = HashMap<String, Option<DictEntry>>;
+
+/// This macro constructs states for our machine
+///
+/// **DO NOT** construct states manually
+macro_rules! states {
+    ($(#[$attr:meta])+$vis:vis struct $stateid:ident: $statebase:ty {$($(#[$tyattr:meta])*$v:vis$state:ident = $statexp:expr),+ $(,)?}) => {
+        #[::core::prelude::v1::derive(::core::cmp::PartialEq,::core::cmp::Eq,::core::clone::Clone,::core::marker::Copy)]
+        $(#[$attr])+$vis struct $stateid {__base: $statebase}
+        impl $stateid {$($(#[$tyattr])*$v const $state:Self=$stateid{__base: $statexp,};)*}
+        impl ::core::fmt::Debug for $stateid {
+            fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
+                let r = match self.__base {$($statexp => ::core::stringify!($state),)* _ => panic!("invalid state"),};
+                ::core::write!(f, "{}::{}", ::core::stringify!($stateid), r)
+            }
+        }
+    }
+}
+
+/*
+    Context-free dict
+*/
+
+states! {
+    /// The dict fold state
+    pub struct DictFoldState: u8 {
+        FINAL = 0xFF,
+        pub(super) OB = 0x00,
+        pub(super) CB_OR_IDENT = 0x01,
+        COLON = 0x02,
+        LIT_OR_OB = 0x03,
+        COMMA_OR_CB = 0x04,
+    }
+}
+
+trait Breakpoint<'a> {
+    const HAS_BREAKPOINT: bool;
+    fn check_breakpoint(state: DictFoldState, tok: &'a Token<'a>) -> bool;
+}
+
+struct NoBreakpoint;
+impl<'a> Breakpoint<'a> for NoBreakpoint {
+    const HAS_BREAKPOINT: bool = false;
+    fn check_breakpoint(_: DictFoldState, _: &'a Token<'a>) -> bool {
+        false
+    }
+}
+struct TypeBreakpoint;
+impl<'a> Breakpoint<'a> for TypeBreakpoint {
+    const HAS_BREAKPOINT: bool = true;
+    fn check_breakpoint(state: DictFoldState, tok: &'a Token<'a>) -> bool {
+        (state == DictFoldState::CB_OR_IDENT) & matches!(tok, Token![type])
+    }
+}
+
+/// Fold a dictionary
+fn _rfold_dict<'a, Qd, Bp>(
+    mut mstate: DictFoldState,
+    state: &mut State<'a, Qd>,
+    dict: &mut Dict,
+) -> bool
+where
+    Qd: QueryData<'a>,
+    Bp: Breakpoint<'a>,
+{
+    /*
+        NOTE: Assume rules wherever applicable
+
+        <openbrace> ::= "{"
+        <closebrace> ::= "}"
+        <comma> ::= ","
+        <colon> ::= ":"
+        <dict> ::= <openbrace> (<ident> <colon> (<lit> | <dict>) <comma>)* <comma>* <closebrace>
+    */
+    let mut key = MaybeInit::uninit();
+    while state.loop_tt() {
+        match (state.fw_read(), mstate) {
+            (Token![open {}], DictFoldState::OB) => {
+                // open
+                mstate = DictFoldState::CB_OR_IDENT;
+            }
+            (Token![close {}], DictFoldState::CB_OR_IDENT | DictFoldState::COMMA_OR_CB) => {
+                // well, that's the end of the dict
+                mstate = DictFoldState::FINAL;
+                break;
+            }
+            (Token::Ident(id), DictFoldState::CB_OR_IDENT) => {
+                key = MaybeInit::new(unsafe { str::from_utf8_unchecked(id) });
+                // found a key, now expect colon
+                mstate = DictFoldState::COLON;
+            }
+            (Token![:], DictFoldState::COLON) => {
+                // found colon, now lit or ob
+                mstate = DictFoldState::LIT_OR_OB;
+            }
+            (tok, DictFoldState::LIT_OR_OB) if state.can_read_lit_from(tok) => {
+                // found lit
+                unsafe {
+                    let v = Some(state.read_lit_unchecked_from(tok).into());
+                    state.poison_if_not(dict.insert(key.take().to_string(), v).is_none());
+                }
+                // after lit we're either done or expect something else
+                mstate = DictFoldState::COMMA_OR_CB;
+            }
+            (Token![null], DictFoldState::LIT_OR_OB) => {
+                // found a null
+                unsafe {
+                    state.poison_if_not(dict.insert(key.take().to_string(), None).is_none());
+                }
+                // after a null (essentially counts as a lit) we're either done or expect something else
+                mstate = DictFoldState::COMMA_OR_CB;
+            }
+            (Token![open {}], DictFoldState::LIT_OR_OB) => {
+                // found a nested dict
+                let mut ndict = Dict::new();
+                _rfold_dict::<Qd, NoBreakpoint>(DictFoldState::CB_OR_IDENT, state, &mut ndict);
+                unsafe {
+                    state.poison_if_not(
+                        dict.insert(key.take().to_string(), Some(ndict.into()))
+                            .is_none(),
+                    );
+                }
+                mstate = DictFoldState::COMMA_OR_CB;
+            }
+            (Token![,], DictFoldState::COMMA_OR_CB) => {
+                // expecting a comma, found it. now expect a close brace or an ident
+                mstate = DictFoldState::CB_OR_IDENT;
+            }
+            (this_tok, this_key)
+                if Bp::HAS_BREAKPOINT && Bp::check_breakpoint(this_key, this_tok) =>
+            {
+                // reached custom breakpoint
+                return true;
+            }
+            x => {
+                dbg!(x);
+                state.cursor_back();
+                state.poison();
+                break;
+            }
+        }
+    }
+    state.poison_if_not(mstate == DictFoldState::FINAL);
+    false
+}
+
+pub(super) fn rfold_dict<'a, Qd: QueryData<'a>>(
+    mstate: DictFoldState,
+    state: &mut State<'a, Qd>,
+    dict: &mut Dict,
+) {
+    _rfold_dict::<Qd, NoBreakpoint>(mstate, state, dict);
+}
+
+pub(super) fn rfold_tymeta<'a, Qd: QueryData<'a>>(
+    mstate: DictFoldState,
+    state: &mut State<'a, Qd>,
+    dict: &mut Dict,
+) -> bool {
+    _rfold_dict::<Qd, TypeBreakpoint>(mstate, state, dict)
+}
+
+#[derive(Debug, PartialEq)]
+/// A layer contains a type and corresponding metadata
+pub struct Layer<'a> {
+    ty: Slice<'a>,
+    props: Dict,
+}
+
+impl<'a> Layer<'a> {
+    //// Create a new layer
+    pub const fn new(ty: Slice<'a>, props: Dict) -> Self {
+        Self { ty, props }
+    }
+}
+
+states! {
+    /// Layer fold state
+    pub struct LayerFoldState: u8 {
+        BEGIN_IDENT = 0x01,
+        IDENT_OR_CB = 0x02,
+        FOLD_INCOMPLETE = 0x03,
+        FINAL_OR_OB = 0x04,
+        FINAL = 0xFF,
+    }
+}
+
+fn rfold_layers<'a, Qd: QueryData<'a>>(
+    mut mstate: LayerFoldState,
+    state: &mut State<'a, Qd>,
+    layers: &mut Vec<Layer<'a>>,
+) {
+    let mut ty = MaybeInit::uninit();
+    let mut props = dict!();
+    while state.loop_tt() {
+        match (state.fw_read(), mstate) {
+            (Token::Ident(id), LayerFoldState::BEGIN_IDENT) => {
+                ty = MaybeInit::new(*id);
+                mstate = LayerFoldState::FINAL_OR_OB;
+            }
+            (Token![open {}], LayerFoldState::FINAL_OR_OB) => {
+                // we were done ... but we found some props
+                if rfold_tymeta(DictFoldState::CB_OR_IDENT, state, &mut props) {
+                    // we have more layers
+                    // but we first need a colon
+                    state.poison_if_not(state.cursor_rounded_eq(Token![:]));
+                    state.cursor_ahead_if(state.okay());
+                    rfold_layers(LayerFoldState::BEGIN_IDENT, state, layers);
+                    // we are yet to parse the remaining props
+                    mstate = LayerFoldState::FOLD_INCOMPLETE;
+                } else {
+                    // didn't hit bp; so we should be done here
+                    mstate = LayerFoldState::FINAL;
+                    break;
+                }
+            }
+            (Token![close {}], LayerFoldState::FOLD_INCOMPLETE) => {
+                // found end of the dict. roger the terminal!
+                mstate = LayerFoldState::FINAL;
+                break;
+            }
+            (Token![,], LayerFoldState::FOLD_INCOMPLETE) => {
+                // we found a comma, but we should finish parsing the dict
+                rfold_dict(DictFoldState::CB_OR_IDENT, state, &mut props);
+                // we're done parsing
+                mstate = LayerFoldState::FINAL;
+                break;
+            }
+            // FIXME(@ohsayan): if something falls apart, it's the arm below
+            (_, LayerFoldState::FINAL_OR_OB) => {
+                state.cursor_back();
+                mstate = LayerFoldState::FINAL;
+                break;
+            }
+            _ => {
+                state.cursor_back();
+                state.poison();
+                break;
+            }
+        }
+    }
+    if ((mstate == LayerFoldState::FINAL) | (mstate == LayerFoldState::FINAL_OR_OB)) & state.okay()
+    {
+        layers.push(Layer {
+            ty: unsafe { ty.take() },
+            props,
+        });
+    } else {
+        state.poison();
+    }
+}
+
+#[derive(Debug, PartialEq)]
+/// A field definition
+pub struct Field<'a> {
+    /// the field name
+    field_name: Slice<'a>,
+    /// layers
+    layers: Vec<Layer<'a>>,
+    /// is null
+    null: bool,
+    /// is primary
+    primary: bool,
+}
+
+impl<'a> Field<'a> {
+    pub fn new(field_name: Slice<'a>, layers: Vec<Layer<'a>>, null: bool, primary: bool) -> Self {
+        Self {
+            field_name,
+            layers,
+            null,
+            primary,
+        }
+    }
+    pub fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        if compiler::unlikely(state.remaining() < 2) {
+            // smallest field: `ident: type`
+            return Err(LangError::UnexpectedEndofStatement);
+        }
+        // check if primary or null
+        let is_primary = state.cursor_eq(Token![primary]);
+        state.cursor_ahead_if(is_primary);
+        let is_null = state.cursor_eq(Token![null]);
+        state.cursor_ahead_if(is_null);
+        state.poison_if(is_primary & is_null);
+        // parse layers
+        // field name
+        let field_name = match (state.fw_read(), state.fw_read()) {
+            (Token::Ident(id), Token![:]) => id,
+            _ => return Err(LangError::UnexpectedToken),
+        };
+        // layers
+        let mut layers = Vec::new();
+        rfold_layers(LayerFoldState::BEGIN_IDENT, state, &mut layers);
+        if state.okay() {
+            Ok(Field {
+                field_name: field_name.clone(),
+                layers,
+                null: is_null,
+                primary: is_primary,
+            })
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+/// An [`ExpandedField`] is a full field definition with advanced metadata
+pub struct ExpandedField<'a> {
+    field_name: Slice<'a>,
+    layers: Vec<Layer<'a>>,
+    props: Dict,
+}
+
+impl<'a> ExpandedField<'a> {
+    pub fn new(field_name: Slice<'a>, layers: Vec<Layer<'a>>, props: Dict) -> Self {
+        Self {
+            field_name,
+            layers,
+            props,
+        }
+    }
+    #[inline(always)]
+    /// Parse a field declared using the field syntax
+    pub(super) fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        if compiler::unlikely(state.remaining() < 6) {
+            // smallest: fieldname { type: ident }
+            return Err(LangError::UnexpectedEndofStatement);
+        }
+        let field_name = state.fw_read();
+        state.poison_if_not(field_name.is_ident());
+        state.poison_if_not(state.cursor_eq(Token![open {}]));
+        state.cursor_ahead();
+        // ignore errors; now attempt a tymeta-like parse
+        let mut props = Dict::new();
+        let mut layers = Vec::new();
+        if rfold_tymeta(DictFoldState::CB_OR_IDENT, state, &mut props) {
+            // this has layers. fold them; but don't forget the colon
+            if compiler::unlikely(state.exhausted()) {
+                // we need more tokens
+                return Err(LangError::UnexpectedEndofStatement);
+            }
+            state.poison_if_not(state.cursor_eq(Token![:]));
+            state.cursor_ahead();
+            rfold_layers(LayerFoldState::BEGIN_IDENT, state, &mut layers);
+            match state.fw_read() {
+                Token![,] => {
+                    rfold_dict(DictFoldState::CB_OR_IDENT, state, &mut props);
+                }
+                Token![close {}] => {
+                    // hit end
+                }
+                _ => {
+                    state.poison();
+                }
+            }
+        }
+        if state.okay() {
+            Ok(Self {
+                field_name: unsafe { extract!(field_name, Token::Ident(id) => *id) },
+                props,
+                layers,
+            })
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    }
+    #[inline(always)]
+    /// Parse multiple fields declared using the field syntax. Flag setting allows or disallows reset syntax
+    pub fn parse_multiple<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Box<[Self]>> {
+        const DEFAULT_ADD_COL_CNT: usize = 4;
+        /*
+            WARNING: No trailing commas allowed
+
+            <add> ::= (<field_syntax> <comma>)*
+
+            Smallest length:
+            alter model add myfield { type string }
+        */
+        if compiler::unlikely(state.remaining() < 5) {
+            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+        }
+        match state.read() {
+            Token::Ident(_) => {
+                let ef = Self::parse(state)?;
+                Ok([ef].into())
+            }
+            Token![() open] => {
+                state.cursor_ahead();
+                let mut stop = false;
+                let mut cols = Vec::with_capacity(DEFAULT_ADD_COL_CNT);
+                while state.loop_tt() && !stop {
+                    match state.read() {
+                        Token::Ident(_) => {
+                            let ef = Self::parse(state)?;
+                            cols.push(ef);
+                            let nx_comma = state.cursor_rounded_eq(Token![,]);
+                            let nx_close = state.cursor_rounded_eq(Token![() close]);
+                            stop = nx_close;
+                            state.poison_if_not(nx_comma | nx_close);
+                            state.cursor_ahead_if(state.okay());
+                        }
+                        _ => {
+                            state.poison();
+                            break;
+                        }
+                    }
+                }
+                state.poison_if_not(stop);
+                if state.okay() {
+                    Ok(cols.into_boxed_slice())
+                } else {
+                    Err(LangError::UnexpectedToken)
+                }
+            }
+            _ => Err(LangError::ExpectedStatement),
+        }
+    }
+}
+
+#[cfg(test)]
+pub use impls::{DictBasic, DictTypeMeta, DictTypeMetaSplit};
+#[cfg(test)]
+mod impls {
+    use super::{
+        rfold_dict, rfold_layers, rfold_tymeta, Dict, DictFoldState, ExpandedField, Field, Layer,
+        LayerFoldState,
+    };
+    use crate::engine::ql::{
+        ast::{traits::ASTNode, QueryData, State},
+        LangError, LangResult,
+    };
+    use std::ops::{Deref, DerefMut};
+    impl<'a> ASTNode<'a> for ExpandedField<'a> {
+        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            Self::parse(state)
+        }
+        fn multiple_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> LangResult<Vec<Self>> {
+            Self::parse_multiple(state).map(Vec::from)
+        }
+    }
+    impl<'a> ASTNode<'a> for Layer<'a> {
+        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            let mut layers = Vec::new();
+            rfold_layers(LayerFoldState::BEGIN_IDENT, state, &mut layers);
+            assert!(layers.len() == 1);
+            Ok(layers.swap_remove(0))
+        }
+        fn multiple_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> LangResult<Vec<Self>> {
+            let mut l = Vec::new();
+            rfold_layers(LayerFoldState::BEGIN_IDENT, state, &mut l);
+            if state.okay() {
+                Ok(l)
+            } else {
+                Err(LangError::UnexpectedToken)
+            }
+        }
+    }
+    pub struct DictBasic(pub Dict);
+    impl<'a> ASTNode<'a> for DictBasic {
+        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            let mut dict = Dict::new();
+            rfold_dict(DictFoldState::OB, state, &mut dict);
+            if state.okay() {
+                Ok(Self(dict))
+            } else {
+                Err(LangError::UnexpectedToken)
+            }
+        }
+    }
+    impl Deref for DictBasic {
+        type Target = Dict;
+        fn deref(&self) -> &Self::Target {
+            &self.0
+        }
+    }
+    impl DerefMut for DictBasic {
+        fn deref_mut(&mut self) -> &mut Self::Target {
+            &mut self.0
+        }
+    }
+    pub struct DictTypeMetaSplit(pub Dict);
+    impl<'a> ASTNode<'a> for DictTypeMetaSplit {
+        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            let mut dict = Dict::new();
+            rfold_tymeta(DictFoldState::CB_OR_IDENT, state, &mut dict);
+            if state.okay() {
+                Ok(Self(dict))
+            } else {
+                Err(LangError::UnexpectedToken)
+            }
+        }
+    }
+    impl Deref for DictTypeMetaSplit {
+        type Target = Dict;
+        fn deref(&self) -> &Self::Target {
+            &self.0
+        }
+    }
+    impl DerefMut for DictTypeMetaSplit {
+        fn deref_mut(&mut self) -> &mut Self::Target {
+            &mut self.0
+        }
+    }
+    pub struct DictTypeMeta(pub Dict);
+    impl<'a> ASTNode<'a> for DictTypeMeta {
+        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            let mut dict = Dict::new();
+            rfold_tymeta(DictFoldState::OB, state, &mut dict);
+            if state.okay() {
+                Ok(Self(dict))
+            } else {
+                Err(LangError::UnexpectedToken)
+            }
+        }
+    }
+    impl Deref for DictTypeMeta {
+        type Target = Dict;
+        fn deref(&self) -> &Self::Target {
+            &self.0
+        }
+    }
+    impl DerefMut for DictTypeMeta {
+        fn deref_mut(&mut self) -> &mut Self::Target {
+            &mut self.0
+        }
+    }
+    impl<'a> ASTNode<'a> for Field<'a> {
+        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            Self::parse(state)
+        }
+    }
+}
diff --git a/server/src/engine/ql/dml/del.rs b/server/src/engine/ql/dml/del.rs
index 54eea275..95f6ed0b 100644
--- a/server/src/engine/ql/dml/del.rs
+++ b/server/src/engine/ql/dml/del.rs
@@ -24,21 +24,18 @@
  *
 */
 
+#[cfg(test)]
+use super::WhereClauseCollection;
 use {
     super::WhereClause,
     crate::{
         engine::ql::{
-            ast::{Entity, QueryData, State},
+            ast::{traits::ASTNode, Entity, QueryData, State},
             LangError, LangResult,
         },
         util::{compiler, MaybeInit},
     },
 };
-#[cfg(test)]
-use {
-    super::WhereClauseCollection,
-    crate::engine::ql::{ast::InplaceData, lex::Token},
-};
 
 /*
     Impls for delete
@@ -97,10 +94,8 @@ impl<'a> DeleteStatement<'a> {
     }
 }
 
-#[cfg(test)]
-pub fn parse_delete_full<'a>(tok: &'a [Token]) -> LangResult<DeleteStatement<'a>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let ret = DeleteStatement::parse_delete(&mut state);
-    assert_full_tt!(state);
-    ret
+impl<'a> ASTNode<'a> for DeleteStatement<'a> {
+    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        Self::parse_delete(state)
+    }
 }
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index 8de9360d..1ddaf42b 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -24,15 +24,13 @@
  *
 */
 
-#[cfg(test)]
-use crate::engine::ql::ast::InplaceData;
 use {
     super::read_ident,
     crate::{
         engine::{
             core::DataType,
             ql::{
-                ast::{Entity, QueryData, State},
+                ast::{traits::ASTNode, Entity, QueryData, State},
                 lex::Token,
                 LangError, LangResult,
             },
@@ -210,12 +208,18 @@ unsafe fn handle_func_sub<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> O
 }
 
 #[cfg(test)]
-pub fn parse_list_full<'a>(tok: &'a [Token], qd: impl QueryData<'a>) -> Option<Vec<DataType>> {
-    let mut l = Vec::new();
-    let mut state = State::new(tok, qd);
-    parse_list(&mut state, &mut l);
-    assert_full_tt!(state);
-    state.okay().then_some(l)
+pub struct List(pub Vec<DataType>);
+#[cfg(test)]
+impl<'a> ASTNode<'a> for List {
+    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        let mut l = Vec::new();
+        parse_list(state, &mut l);
+        if state.okay() {
+            Ok(List(l))
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    }
 }
 
 /// ## Panics
@@ -266,11 +270,17 @@ pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
 }
 
 #[cfg(test)]
-pub fn parse_data_tuple_syntax_full(tok: &[Token]) -> Option<Vec<Option<DataType>>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let ret = parse_data_tuple_syntax(&mut state);
-    assert_full_tt!(state);
-    state.okay().then_some(ret)
+pub struct DataTuple(pub Vec<Option<DataType>>);
+#[cfg(test)]
+impl<'a> ASTNode<'a> for DataTuple {
+    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        let r = parse_data_tuple_syntax(state);
+        if state.okay() {
+            Ok(Self(r))
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    }
 }
 
 /// ## Panics
@@ -330,20 +340,26 @@ pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
 }
 
 #[cfg(test)]
-pub fn parse_data_map_syntax_full(tok: &[Token]) -> Option<HashMap<Box<str>, Option<DataType>>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let r = parse_data_map_syntax(&mut state);
-    assert_full_tt!(state);
-    state.okay().then_some(
-        r.into_iter()
-            .map(|(ident, val)| {
-                (
-                    String::from_utf8_lossy(ident).to_string().into_boxed_str(),
-                    val,
-                )
-            })
-            .collect(),
-    )
+pub struct DataMap(pub HashMap<Box<str>, Option<DataType>>);
+#[cfg(test)]
+impl<'a> ASTNode<'a> for DataMap {
+    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        let r = parse_data_map_syntax(state);
+        if state.okay() {
+            Ok(Self(
+                r.into_iter()
+                    .map(|(ident, val)| {
+                        (
+                            String::from_utf8_lossy(ident).to_string().into_boxed_str(),
+                            val,
+                        )
+                    })
+                    .collect(),
+            ))
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    }
 }
 
 #[derive(Debug, PartialEq)]
@@ -425,10 +441,8 @@ impl<'a> InsertStatement<'a> {
     }
 }
 
-#[cfg(test)]
-pub fn parse_insert_full<'a>(tok: &'a [Token]) -> Option<InsertStatement<'a>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let ret = InsertStatement::parse_insert(&mut state);
-    assert_full_tt!(state);
-    ret.ok()
+impl<'a> ASTNode<'a> for InsertStatement<'a> {
+    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        Self::parse_insert(state)
+    }
 }
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index 90ec4969..cf7ba7d9 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -34,12 +34,11 @@ pub mod ins;
 pub mod sel;
 pub mod upd;
 
-#[cfg(test)]
-use super::ast::InplaceData;
 use {
     super::{
-        ast::{QueryData, State},
+        ast::{traits::ASTNode, QueryData, State},
         lex::{LitIR, Token},
+        LangError, LangResult,
     },
     crate::util::compiler,
     std::collections::HashMap,
@@ -120,6 +119,12 @@ impl<'a> RelationalExpr<'a> {
     }
 }
 
+impl<'a> ASTNode<'a> for RelationalExpr<'a> {
+    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        Self::try_parse(state).ok_or(LangError::UnexpectedToken)
+    }
+}
+
 #[derive(Debug, PartialEq)]
 pub struct WhereClause<'a> {
     c: WhereClauseCollection<'a>,
@@ -159,19 +164,13 @@ impl<'a> WhereClause<'a> {
     }
 }
 
-#[cfg(test)]
-pub(super) fn parse_where_clause_full<'a>(tok: &'a [Token]) -> Option<WhereClause<'a>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let ret = WhereClause::parse_where(&mut state);
-    assert_full_tt!(state);
-    state.okay().then_some(ret)
-}
-
-#[cfg(test)]
-#[inline(always)]
-pub(super) fn parse_relexpr_full<'a>(tok: &'a [Token]) -> Option<RelationalExpr<'a>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let ret = RelationalExpr::try_parse(&mut state);
-    assert_full_tt!(state);
-    ret
+impl<'a> ASTNode<'a> for WhereClause<'a> {
+    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        let wh = Self::parse_where(state);
+        if state.okay() {
+            Ok(wh)
+        } else {
+            Err(LangError::UnexpectedToken)
+        }
+    }
 }
diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs
index 0242f910..88e78a99 100644
--- a/server/src/engine/ql/dml/sel.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -24,13 +24,11 @@
  *
 */
 
-#[cfg(test)]
-use crate::engine::ql::ast::InplaceData;
 use {
     super::{WhereClause, WhereClauseCollection},
     crate::{
         engine::ql::{
-            ast::{Entity, QueryData, State},
+            ast::{traits::ASTNode, Entity, QueryData, State},
             lex::Token,
             LangError, LangResult,
         },
@@ -80,15 +78,6 @@ impl<'a> SelectStatement<'a> {
     }
 }
 
-#[cfg(test)]
-/// **test-mode only** parse for a `select` where the full token stream is exhausted
-pub fn parse_select_full<'a>(tok: &'a [Token]) -> Option<SelectStatement<'a>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let r = SelectStatement::parse_select(&mut state);
-    assert_full_tt!(state);
-    r.ok()
-}
-
 impl<'a> SelectStatement<'a> {
     pub fn parse_select<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
         /*
@@ -144,3 +133,9 @@ impl<'a> SelectStatement<'a> {
         }
     }
 }
+
+impl<'a> ASTNode<'a> for SelectStatement<'a> {
+    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        Self::parse_select(state)
+    }
+}
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index 9245eed8..2713b150 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -33,7 +33,7 @@ use {
     super::{read_ident, u, WhereClause},
     crate::{
         engine::ql::{
-            ast::{Entity, QueryData, State},
+            ast::{traits::ASTNode, Entity, QueryData, State},
             lex::LitIR,
             LangError, LangResult,
         },
@@ -227,10 +227,8 @@ impl<'a> UpdateStatement<'a> {
     }
 }
 
-#[cfg(test)]
-pub fn parse_update_full<'a>(tok: &'a [Token]) -> LangResult<UpdateStatement<'a>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let r = UpdateStatement::parse_update(&mut state);
-    assert_full_tt!(state);
-    r
+impl<'a> ASTNode<'a> for UpdateStatement<'a> {
+    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        Self::parse_update(state)
+    }
 }
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 1049657f..9adc304e 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -283,14 +283,14 @@ macro_rules! dict {
 }
 
 #[cfg(test)]
-macro_rules! nullable_dict {
+macro_rules! null_dict {
     () => {
         dict! {}
     };
     ($($key:expr => $value:expr),* $(,)?) => {
         dict! {
             $(
-                $key => $crate::engine::ql::tests::NullableMapEntry::data($value),
+                $key => $crate::engine::ql::tests::NullableDictEntry::data($value),
             )*
         }
     };
@@ -308,18 +308,6 @@ macro_rules! dict_nullable {
     }};
 }
 
-#[cfg(test)]
-macro_rules! set {
-    () => {
-        <::std::collections::HashSet<_> as ::core::default::Default>::default()
-    };
-    ($($key:expr),* $(,)?) => {{
-        let mut hs: ::std::collections::HashSet<_> = ::core::default::Default::default();
-        $(hs.insert($key.into());)*
-        hs
-    }};
-}
-
 #[cfg(test)]
 macro_rules! into_array {
     ($($e:expr),* $(,)?) => { [$($e.into()),*] };
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 6d34fce3..e364dde5 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -33,7 +33,6 @@ mod benches;
 pub(super) mod ddl;
 pub(super) mod dml;
 pub(super) mod lex;
-pub(super) mod schema;
 #[cfg(test)]
 mod tests;
 
diff --git a/server/src/engine/ql/schema.rs b/server/src/engine/ql/schema.rs
deleted file mode 100644
index 844c669b..00000000
--- a/server/src/engine/ql/schema.rs
+++ /dev/null
@@ -1,1135 +0,0 @@
-/*
- * Created on Fri Sep 16 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
-    Most grammar tools are pretty much "off the shelf" which makes some things incredibly hard to achieve (such
-    as custom error injection logic). To make things icier, Rust's integration with these tools (like lex) is not
-    very "refined." Hence, it is best for us to implement our own parsers. In the future, I plan to optimize our
-    rule checkers but that's not a concern at the moment.
-
-    This module makes use of DFAs with additional flags, accepting a token stream as input to generate appropriate
-    structures, and have provable correctness. Hence, the unsafe code used here is correct, because the states are
-    only transitioned to if the input is accepted. If you do find otherwise, please file a bug report. The
-    transitions are currently very inefficient but can be made much faster.
-
-    TODO: The SMs can be reduced greatly, enocded to fixed-sized structures even, so do that
-    FIXME: For now, try and reduce reliance on additional flags (encoded into state?)
-    FIXME: The returns are awfully large right now. Do something about it
-
-    --
-    Sayan (@ohsayan)
-    Sept. 15, 2022
-*/
-
-#[cfg(test)]
-use crate::engine::ql::ast::InplaceData;
-use {
-    super::{
-        ast::{QueryData, State},
-        lex::{LitIR, LitIROwned, Slice, Symbol, Token},
-        LangError, LangResult,
-    },
-    crate::util::{compiler, MaybeInit},
-    core::str,
-    std::collections::{HashMap, HashSet},
-};
-
-/*
-    Meta
-*/
-
-/// This macro constructs states for our machine
-///
-/// **DO NOT** construct states manually
-macro_rules! states {
-    ($(#[$attr:meta])+$vis:vis struct $stateid:ident: $statebase:ty {$($(#[$tyattr:meta])*$v:vis$state:ident = $statexp:expr),+ $(,)?}) => {
-        #[::core::prelude::v1::derive(::core::cmp::PartialEq,::core::cmp::Eq,::core::clone::Clone,::core::marker::Copy)]
-        $(#[$attr])+$vis struct $stateid {__base: $statebase}
-        impl $stateid {$($(#[$tyattr])*$v const $state:Self=$stateid{__base: $statexp,};)*}
-        impl ::core::fmt::Debug for $stateid {
-            fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
-                let r = match self.__base {$($statexp => ::core::stringify!($state),)* _ => panic!("invalid state"),};
-                ::core::write!(f, "{}::{}", ::core::stringify!($stateid), r)
-            }
-        }
-    }
-}
-
-/// A static string slice
-type StaticStr = &'static str;
-
-const HIBIT: u64 = 1 << 63;
-/// Flag for disallowing the `..` syntax
-const DISALLOW_RESET_SYNTAX: bool = false;
-/// Flag for allowing the `..` syntax
-const ALLOW_RESET_SYNTAX: bool = true;
-
-#[derive(Debug, PartialEq)]
-/// A dictionary entry type. Either a literal or another dictionary
-pub enum DictEntry {
-    Lit(LitIROwned),
-    Map(Dict),
-}
-
-impl<'a> From<LitIR<'a>> for DictEntry {
-    fn from(l: LitIR<'a>) -> Self {
-        Self::Lit(l.to_litir_owned())
-    }
-}
-
-impl From<Dict> for DictEntry {
-    fn from(d: Dict) -> Self {
-        Self::Map(d)
-    }
-}
-
-/// A metadata dictionary
-pub type Dict = HashMap<String, Option<DictEntry>>;
-
-#[derive(Debug, PartialEq)]
-/// A layer contains a type and corresponding metadata
-pub struct Layer<'a> {
-    ty: Slice<'a>,
-    props: Dict,
-    reset: bool,
-}
-
-impl<'a> Layer<'a> {
-    //// Create a new layer
-    pub(super) const fn new(ty: Slice<'a>, props: Dict, reset: bool) -> Self {
-        Self { ty, props, reset }
-    }
-    /// Create a new layer that doesn't have any reset
-    pub(super) const fn new_noreset(ty: Slice<'a>, props: Dict) -> Self {
-        Self::new(ty, props, false)
-    }
-    /// Create a new layer that adds a reset
-    pub(super) const fn new_reset(ty: Slice<'a>, props: Dict) -> Self {
-        Self::new(ty, props, true)
-    }
-}
-
-#[derive(Debug, Default, PartialEq, Eq)]
-/// Field properties
-pub struct FieldProperties {
-    pub(super) properties: HashSet<StaticStr>,
-}
-
-impl FieldProperties {
-    const NULL: StaticStr = "null";
-    const PRIMARY: StaticStr = "primary";
-    pub fn new() -> Self {
-        Self {
-            properties: HashSet::new(),
-        }
-    }
-}
-
-#[derive(Debug, PartialEq)]
-/// A field definition
-pub struct Field<'a> {
-    /// the field name
-    pub(super) field_name: Slice<'a>,
-    /// layers
-    pub(super) layers: Vec<Layer<'a>>,
-    /// properties
-    pub(super) props: HashSet<StaticStr>,
-}
-
-impl<'a> Field<'a> {
-    #[inline(always)]
-    pub fn new(field_name: Slice<'a>, layers: Vec<Layer<'a>>, props: HashSet<StaticStr>) -> Self {
-        Self {
-            field_name,
-            layers,
-            props,
-        }
-    }
-}
-
-#[derive(Debug, PartialEq)]
-/// A model definition
-pub struct Model<'a> {
-    /// the model name
-    pub(super) model_name: Slice<'a>,
-    /// the fields
-    pub(super) fields: Vec<Field<'a>>,
-    /// properties
-    pub(super) props: Dict,
-}
-
-impl<'a> Model<'a> {
-    #[inline(always)]
-    pub fn new(model_name: Slice<'a>, fields: Vec<Field<'a>>, props: Dict) -> Self {
-        Self {
-            model_name,
-            fields,
-            props,
-        }
-    }
-}
-
-#[derive(Debug, PartialEq)]
-/// A space
-pub struct Space<'a> {
-    /// the space name
-    pub(super) space_name: Slice<'a>,
-    /// properties
-    pub(super) props: Dict,
-}
-
-#[derive(Debug, PartialEq)]
-/// An alter space query with corresponding data
-pub struct AlterSpace<'a> {
-    pub(super) space_name: Slice<'a>,
-    pub(super) updated_props: Dict,
-}
-
-/*
-    Context-free dict
-*/
-
-states! {
-    /// The dict fold state
-    pub struct DictFoldState: u8 {
-        FINAL = 0xFF,
-        OB = 0x00,
-        CB_OR_IDENT = 0x01,
-        COLON = 0x02,
-        LIT_OR_OB = 0x03,
-        COMMA_OR_CB = 0x04,
-    }
-}
-
-/// Fold a dictionary
-pub(super) fn rfold_dict<'a, Qd: QueryData<'a>>(
-    mut mstate: DictFoldState,
-    state: &mut State<'a, Qd>,
-    dict: &mut Dict,
-) {
-    /*
-        NOTE: Assume rules wherever applicable
-
-        <openbrace> ::= "{"
-        <closebrace> ::= "}"
-        <comma> ::= ","
-        <colon> ::= ":"
-        <dict> ::= <openbrace> (<ident> <colon> (<lit> | <dict>) <comma>)* <comma>* <closebrace>
-    */
-    let mut tmp = MaybeInit::uninit();
-
-    while state.loop_tt() {
-        match (state.fw_read(), mstate) {
-            (Token::Symbol(Symbol::TtOpenBrace), DictFoldState::OB) => {
-                // we found a brace, expect a close brace or an ident
-                mstate = DictFoldState::CB_OR_IDENT;
-            }
-            (
-                Token::Symbol(Symbol::TtCloseBrace),
-                DictFoldState::CB_OR_IDENT | DictFoldState::COMMA_OR_CB,
-            ) => {
-                // end of stream
-                mstate = DictFoldState::FINAL;
-                break;
-            }
-            (Token::Ident(id), DictFoldState::CB_OR_IDENT) => {
-                // found ident, so expect colon
-                tmp = MaybeInit::new(unsafe { str::from_utf8_unchecked(id) });
-                mstate = DictFoldState::COLON;
-            }
-            (Token::Symbol(Symbol::SymColon), DictFoldState::COLON) => {
-                // found colon, expect literal or openbrace
-                mstate = DictFoldState::LIT_OR_OB;
-            }
-            (tok, DictFoldState::LIT_OR_OB) if state.can_read_lit_from(tok) => {
-                // found literal; so push in k/v pair and then expect a comma or close brace
-                unsafe {
-                    let v = Some(state.read_lit_unchecked_from(tok).into());
-                    state
-                        .poison_if_not(dict.insert(tmp.assume_init_ref().to_string(), v).is_none());
-                }
-                mstate = DictFoldState::COMMA_OR_CB;
-            }
-            (Token![null], DictFoldState::LIT_OR_OB) => {
-                // null
-                state.poison_if_not(
-                    dict.insert(unsafe { tmp.assume_init_ref() }.to_string(), None)
-                        .is_none(),
-                );
-                mstate = DictFoldState::COMMA_OR_CB;
-            }
-            // ONLY COMMA CAPTURE
-            (Token::Symbol(Symbol::SymComma), DictFoldState::COMMA_OR_CB) => {
-                // we found a comma, expect a *strict* brace close or ident
-                mstate = DictFoldState::CB_OR_IDENT;
-            }
-            (Token::Symbol(Symbol::TtOpenBrace), DictFoldState::LIT_OR_OB) => {
-                // we found an open brace, so this is a dict
-                let mut new_dict = Dict::new();
-                rfold_dict(DictFoldState::CB_OR_IDENT, state, &mut new_dict);
-                state.poison_if_not(
-                    dict.insert(
-                        unsafe { tmp.assume_init_ref() }.to_string(),
-                        Some(new_dict.into()),
-                    )
-                    .is_none(),
-                );
-                // at the end of a dict we either expect a comma or close brace
-                mstate = DictFoldState::COMMA_OR_CB;
-            }
-            _ => {
-                state.poison();
-                state.cursor_back();
-                break;
-            }
-        }
-    }
-    state.poison_if_not(mstate == DictFoldState::FINAL);
-}
-
-#[cfg(test)]
-/// Fold a dictionary (**test-only**)
-pub fn fold_dict(tok: &[Token]) -> Option<Dict> {
-    let mut d = Dict::new();
-    let mut state = State::new(tok, InplaceData::new());
-    rfold_dict(DictFoldState::OB, &mut state, &mut d);
-    state.okay().then_some(d)
-}
-
-/*
-    Contextual dict (tymeta)
-*/
-
-states! {
-    /// Type metadata fold state
-    pub struct TyMetaFoldState: u8 {
-        IDENT_OR_CB = 0x00,
-        COLON = 0x01,
-        LIT_OR_OB = 0x02,
-        COMMA_OR_CB = 0x03,
-        CB = 0x04,
-        FINAL = 0xFF,
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub struct TyMetaReturn {
-    more: bool,
-    reset: bool,
-}
-
-impl TyMetaReturn {
-    #[inline(always)]
-    pub const fn new() -> Self {
-        Self {
-            more: false,
-            reset: false,
-        }
-    }
-    #[inline(always)]
-    pub const fn has_more(&self) -> bool {
-        self.more
-    }
-    #[inline(always)]
-    pub const fn has_reset(&self) -> bool {
-        self.reset
-    }
-    #[inline(always)]
-    pub fn set_has_more(&mut self) {
-        self.more = true;
-    }
-    #[inline(always)]
-    pub fn set_has_reset(&mut self) {
-        self.reset = true;
-    }
-}
-
-/// Fold type metadata (flag setup dependent on caller)
-pub(super) fn rfold_tymeta<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
-    mut mstate: TyMetaFoldState,
-    state: &mut State<'a, Qd>,
-    dict: &mut Dict,
-) -> TyMetaReturn {
-    let mut tmp = MaybeInit::uninit();
-    let mut tymr = TyMetaReturn::new();
-    while state.loop_tt() {
-        match (state.fw_read(), mstate) {
-            (Token![type], TyMetaFoldState::IDENT_OR_CB) => {
-                // we were expecting an ident but found the type keyword! increase depth
-                tymr.set_has_more();
-                mstate = TyMetaFoldState::FINAL;
-                break;
-            }
-            (Token![.], TyMetaFoldState::IDENT_OR_CB) if ALLOW_RESET => {
-                let reset = state.cursor_rounded_eq(Token![.]);
-                state.cursor_ahead_if(reset);
-                tymr.set_has_reset();
-                state.poison_if_not(reset);
-                mstate = TyMetaFoldState::CB;
-            }
-            (
-                Token::Symbol(Symbol::TtCloseBrace),
-                TyMetaFoldState::IDENT_OR_CB | TyMetaFoldState::COMMA_OR_CB | TyMetaFoldState::CB,
-            ) => {
-                // found close brace. end of stream
-                mstate = TyMetaFoldState::FINAL;
-                break;
-            }
-            (Token::Ident(ident), TyMetaFoldState::IDENT_OR_CB) => {
-                tmp = MaybeInit::new(unsafe { str::from_utf8_unchecked(ident) });
-                // we just saw an ident, so we expect to see a colon
-                mstate = TyMetaFoldState::COLON;
-            }
-            (Token::Symbol(Symbol::SymColon), TyMetaFoldState::COLON) => {
-                // we just saw a colon. now we want a literal or openbrace
-                mstate = TyMetaFoldState::LIT_OR_OB;
-            }
-            (tok, TyMetaFoldState::LIT_OR_OB) if state.can_read_lit_from(tok) => {
-                unsafe {
-                    let v = Some(state.read_lit_unchecked_from(tok).into());
-                    state
-                        .poison_if_not(dict.insert(tmp.assume_init_ref().to_string(), v).is_none());
-                }
-                // saw a literal. next is either comma or close brace
-                mstate = TyMetaFoldState::COMMA_OR_CB;
-            }
-            (Token![null], TyMetaFoldState::LIT_OR_OB) => {
-                state.poison_if_not(
-                    dict.insert(unsafe { tmp.assume_init_ref() }.to_string(), None)
-                        .is_none(),
-                );
-                // saw null, start parsing another entry
-                mstate = TyMetaFoldState::COMMA_OR_CB;
-            }
-            (Token::Symbol(Symbol::SymComma), TyMetaFoldState::COMMA_OR_CB) => {
-                // next is strictly a close brace or ident
-                mstate = TyMetaFoldState::IDENT_OR_CB;
-            }
-            (Token::Symbol(Symbol::TtOpenBrace), TyMetaFoldState::LIT_OR_OB) => {
-                // another dict in here
-                let mut nd = Dict::new();
-                let ret =
-                    rfold_tymeta::<Qd, ALLOW_RESET>(TyMetaFoldState::IDENT_OR_CB, state, &mut nd);
-                // L2 cannot have type definitions
-                state.poison_if(ret.has_more());
-                // end of definition or comma followed by something
-                state.poison_if_not(
-                    dict.insert(
-                        unsafe { tmp.assume_init_ref() }.to_string(),
-                        Some(nd.into()),
-                    )
-                    .is_none(),
-                );
-                mstate = TyMetaFoldState::COMMA_OR_CB;
-            }
-            _ => {
-                state.cursor_back();
-                state.poison();
-                break;
-            }
-        }
-    }
-    state.poison_if_not(mstate == TyMetaFoldState::FINAL);
-    tymr
-}
-
-#[cfg(test)]
-/// (**test-only**) fold type metadata
-pub(super) fn fold_tymeta(tok: &[Token]) -> (TyMetaReturn, bool, usize, Dict) {
-    let mut state = State::new(tok, InplaceData::new());
-    let mut d = Dict::new();
-    let ret = rfold_tymeta::<InplaceData, DISALLOW_RESET_SYNTAX>(
-        TyMetaFoldState::IDENT_OR_CB,
-        &mut state,
-        &mut d,
-    );
-    (ret, state.okay(), state.cursor(), d)
-}
-
-/*
-    Layer
-*/
-
-states! {
-    /// Layer fold state
-    pub struct LayerFoldState: u8 {
-        TY = 0x00,
-        END_OR_OB = 0x01,
-        FOLD_DICT_INCOMPLETE = 0x02,
-        FOLD_COMPLETED = 0xFF
-    }
-}
-
-/// Fold layers
-pub(super) fn rfold_layers<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
-    start: LayerFoldState,
-    state: &mut State<'a, Qd>,
-    layers: &mut Vec<Layer<'a>>,
-) {
-    /*
-        NOTE: Assume rules wherever applicable
-
-        <openbrace> ::= "{"
-        <closebrace> ::= "}"
-        <comma> ::= ","
-        <colon> ::= ":"
-        <kw_type> ::= "type"
-        <layer> ::= <openbrace>
-            (<kw_type> <kw_typeid> <layer> <comma>)*1
-            (<ident> <colon> (<lit> | <dict>) <comma>)*
-            <comma>* <closebrace>
-    */
-    let mut mstate = start;
-    let mut tmp = MaybeInit::uninit();
-    let mut dict = Dict::new();
-    while state.loop_tt() {
-        match (state.fw_read(), mstate) {
-            (Token::Ident(ty), LayerFoldState::TY) => {
-                // expecting type, and found type. next is either end or an open brace or some arbitrary token
-                tmp = MaybeInit::new(ty.clone());
-                mstate = LayerFoldState::END_OR_OB;
-            }
-            (Token::Symbol(Symbol::TtOpenBrace), LayerFoldState::END_OR_OB) => {
-                // since we found an open brace, this type has some meta
-                let ret =
-                    rfold_tymeta::<Qd, ALLOW_RESET>(TyMetaFoldState::IDENT_OR_CB, state, &mut dict);
-                if ret.has_more() {
-                    // more layers
-                    rfold_layers::<Qd, ALLOW_RESET>(LayerFoldState::TY, state, layers);
-                    mstate = LayerFoldState::FOLD_DICT_INCOMPLETE;
-                } else if state.okay() {
-                    // done folding dictionary. nothing more expected. break
-                    mstate = LayerFoldState::FOLD_COMPLETED;
-                    layers.push(Layer {
-                        ty: unsafe { tmp.assume_init() }.clone(),
-                        props: dict,
-                        reset: ret.has_reset(),
-                    });
-                    break;
-                }
-            }
-            (Token::Symbol(Symbol::SymComma), LayerFoldState::FOLD_DICT_INCOMPLETE) => {
-                // there is a comma at the end of this
-                let ret =
-                    rfold_tymeta::<Qd, ALLOW_RESET>(TyMetaFoldState::IDENT_OR_CB, state, &mut dict);
-                state.poison_if(ret.has_more()); // not more than one type depth
-                if state.okay() {
-                    // done folding dict successfully. nothing more expected. break.
-                    mstate = LayerFoldState::FOLD_COMPLETED;
-                    layers.push(Layer {
-                        ty: unsafe { tmp.assume_init() }.clone(),
-                        props: dict,
-                        reset: ret.has_reset(),
-                    });
-                    break;
-                }
-            }
-            (Token::Symbol(Symbol::TtCloseBrace), LayerFoldState::FOLD_DICT_INCOMPLETE) => {
-                // end of stream
-                mstate = LayerFoldState::FOLD_COMPLETED;
-                layers.push(Layer {
-                    ty: unsafe { tmp.assume_init() }.clone(),
-                    props: dict,
-                    reset: false,
-                });
-                break;
-            }
-            (_, LayerFoldState::END_OR_OB) => {
-                state.cursor_back();
-                // random arbitrary byte. finish append
-                mstate = LayerFoldState::FOLD_COMPLETED;
-                layers.push(Layer {
-                    ty: unsafe { tmp.assume_init() }.clone(),
-                    props: dict,
-                    reset: false,
-                });
-                break;
-            }
-            _ => {
-                state.cursor_back();
-                state.poison();
-                break;
-            }
-        }
-    }
-    state.poison_if_not(mstate == LayerFoldState::FOLD_COMPLETED);
-}
-
-#[cfg(test)]
-#[inline(always)]
-/// (**test-only**) fold layers
-pub(super) fn fold_layers<'a>(tok: &'a [Token]) -> (Vec<Layer<'a>>, usize, bool) {
-    let mut state = State::new(tok, InplaceData::new());
-    let mut l = Vec::new();
-    rfold_layers::<InplaceData, DISALLOW_RESET_SYNTAX>(LayerFoldState::TY, &mut state, &mut l);
-    (l, state.consumed(), state.okay())
-}
-
-#[inline(always)]
-/// Collect field properties
-pub(super) fn collect_field_properties<'a, Qd: QueryData<'a>>(
-    state: &mut State<'a, Qd>,
-) -> FieldProperties {
-    let mut props = FieldProperties::default();
-    while state.loop_tt() {
-        match state.fw_read() {
-            Token![primary] => {
-                state.poison_if_not(props.properties.insert(FieldProperties::PRIMARY))
-            }
-            Token![null] => state.poison_if_not(props.properties.insert(FieldProperties::NULL)),
-            Token::Ident(_) => {
-                state.cursor_back();
-                break;
-            }
-            _ => {
-                // we could pass this over to the caller, but it's better if we do it since we're doing
-                // a linear scan anyways
-                state.cursor_back();
-                state.poison();
-                break;
-            }
-        }
-    }
-    props
-}
-
-#[cfg(test)]
-#[inline(always)]
-/// (**test-only**) parse field properties
-pub(super) fn parse_field_properties(tok: &[Token]) -> (FieldProperties, usize, bool) {
-    let mut state = State::new(tok, InplaceData::new());
-    let p = collect_field_properties(&mut state);
-    (p, state.cursor(), state.okay())
-}
-
-#[cfg(test)]
-pub(super) fn parse_field_full<'a>(tok: &'a [Token]) -> LangResult<(usize, Field<'a>)> {
-    let mut state = State::new(tok, InplaceData::new());
-    self::parse_field(&mut state).map(|field| (state.cursor(), field))
-}
-
-#[inline(always)]
-/// Parse a field using the declaration-syntax (not field syntax)
-///
-/// Expected start token: field name (ident)
-pub(super) fn parse_field<'a, Qd: QueryData<'a>>(
-    state: &mut State<'a, Qd>,
-) -> LangResult<Field<'a>> {
-    // parse field properties
-    let props = collect_field_properties(state);
-    // if exhauted or broken, simply return
-    if compiler::unlikely(state.exhausted() | !state.okay() || state.remaining() == 1) {
-        return Err(LangError::UnexpectedEndofStatement);
-    }
-    // field name
-    let field_name = match (state.fw_read(), state.fw_read()) {
-        (Token::Ident(id), Token![:]) => id,
-        _ => return Err(LangError::UnexpectedToken),
-    };
-
-    // layers
-    let mut layers = Vec::new();
-    rfold_layers::<Qd, DISALLOW_RESET_SYNTAX>(LayerFoldState::TY, state, &mut layers);
-    if state.okay() {
-        Ok(Field {
-            field_name: field_name.clone(),
-            layers,
-            props: props.properties,
-        })
-    } else {
-        Err(LangError::UnexpectedToken)
-    }
-}
-
-states! {
-    /// Accept state for a schema parse
-    pub struct SchemaParseState: u8 {
-        OPEN_PAREN = 0x00,
-        FIELD = 0x01,
-        COMMA_OR_END = 0x02,
-        END_OR_FIELD = 0x03,
-    }
-}
-
-#[cfg(test)]
-pub(super) fn parse_schema_from_tokens_full<'a>(
-    tok: &'a [Token],
-) -> LangResult<(Model<'a>, usize)> {
-    let mut state = State::new(tok, InplaceData::new());
-    self::parse_model_from_tokens::<InplaceData>(&mut state).map(|model| (model, state.cursor()))
-}
-
-#[inline(always)]
-/// Parse a fresh schema with declaration-syntax fields
-pub(super) fn parse_model_from_tokens<'a, Qd: QueryData<'a>>(
-    state: &mut State<'a, Qd>,
-) -> LangResult<Model<'a>> {
-    // parse fields
-    // check if we have our model name
-    // smallest model declaration: create model mymodel(username: string, password: binary) -> 10 tokens
-    if compiler::unlikely(state.remaining() < 10) {
-        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
-    }
-    let model_name = state.fw_read();
-    state.poison_if_not(model_name.is_ident());
-    let mut fields = Vec::with_capacity(2);
-    let mut mstate = SchemaParseState::OPEN_PAREN;
-
-    while state.loop_tt() {
-        match (state.fw_read(), mstate) {
-            (Token::Symbol(Symbol::TtOpenParen), SchemaParseState::OPEN_PAREN) => {
-                mstate = SchemaParseState::FIELD;
-            }
-            (
-                Token![primary] | Token![null] | Token::Ident(_),
-                SchemaParseState::FIELD | SchemaParseState::END_OR_FIELD,
-            ) => {
-                state.cursor_back();
-                // fine, we found a field. let's see what we've got
-                let f = self::parse_field(state)?;
-                fields.push(f);
-                mstate = SchemaParseState::COMMA_OR_END;
-            }
-            (Token::Symbol(Symbol::SymComma), SchemaParseState::COMMA_OR_END) => {
-                // expect a field or close paren
-                mstate = SchemaParseState::END_OR_FIELD;
-            }
-            (
-                Token::Symbol(Symbol::TtCloseParen),
-                SchemaParseState::COMMA_OR_END | SchemaParseState::END_OR_FIELD,
-            ) => {
-                // end of stream
-                break;
-            }
-            _ => {
-                state.cursor_back();
-                state.poison();
-                break;
-            }
-        }
-    }
-
-    // model properties
-    if !state.okay() {
-        return Err(LangError::UnexpectedToken);
-    }
-
-    let model_name = unsafe {
-        // UNSAFE(@ohsayan): Now that we're sure that we have the model name ident, get it
-        extract!(model_name, Token::Ident(ref model_name) => model_name.clone())
-    };
-
-    if state.cursor_rounded_eq(Token![with]) {
-        // we have some more input, and it should be a dict of properties
-        state.cursor_ahead(); // +WITH
-
-        // great, parse the dict
-        let mut dict = Dict::new();
-        self::rfold_dict(DictFoldState::OB, state, &mut dict);
-
-        if state.okay() {
-            // sweet, so we got our dict
-            Ok(Model {
-                model_name,
-                props: dict,
-                fields,
-            })
-        } else {
-            Err(LangError::UnexpectedToken)
-        }
-    } else {
-        // we've reached end of stream, so there's nothing more to parse
-        Ok(Model {
-            model_name,
-            props: dict! {},
-            fields,
-        })
-    }
-}
-
-#[inline(always)]
-/// Parse space data from the given tokens
-pub(super) fn parse_space_from_tokens<'a, Qd: QueryData<'a>>(
-    state: &mut State<'a, Qd>,
-) -> LangResult<Space<'a>> {
-    // smallest declaration: `create space myspace` -> >= 1 token
-    if compiler::unlikely(state.remaining() < 1) {
-        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
-    }
-    let space_name = state.fw_read();
-    state.poison_if_not(space_name.is_ident());
-    // either we have `with` or nothing. don't be stupid
-    let has_more_properties = state.cursor_rounded_eq(Token![with]);
-    state.poison_if_not(has_more_properties | state.exhausted());
-    state.cursor_ahead_if(has_more_properties); // +WITH
-    let mut d = Dict::new();
-    // properties
-    if has_more_properties && state.okay() {
-        self::rfold_dict(DictFoldState::OB, state, &mut d);
-    }
-    if state.okay() {
-        Ok(Space {
-            space_name: unsafe { extract!(space_name, Token::Ident(ref id) => id.clone()) },
-            props: d,
-        })
-    } else {
-        Err(LangError::UnexpectedToken)
-    }
-}
-
-#[inline(always)]
-/// Parse alter space from tokens
-pub(super) fn parse_alter_space_from_tokens<'a, Qd: QueryData<'a>>(
-    state: &mut State<'a, Qd>,
-) -> LangResult<AlterSpace<'a>> {
-    if compiler::unlikely(state.remaining() <= 3) {
-        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
-    }
-    let space_name = state.fw_read();
-    state.poison_if_not(state.cursor_eq(Token![with]));
-    state.cursor_ahead(); // ignore errors
-    state.poison_if_not(state.cursor_eq(Token![open {}]));
-    state.cursor_ahead(); // ignore errors
-
-    if compiler::unlikely(!state.okay()) {
-        return Err(LangError::UnexpectedToken);
-    }
-
-    let space_name = unsafe { extract!(space_name, Token::Ident(ref space) => space.clone()) };
-    let mut d = Dict::new();
-    rfold_dict(DictFoldState::CB_OR_IDENT, state, &mut d);
-    if state.okay() {
-        Ok(AlterSpace {
-            space_name,
-            updated_props: d,
-        })
-    } else {
-        Err(LangError::UnexpectedToken)
-    }
-}
-
-#[cfg(test)]
-pub(super) fn alter_space_full<'a>(tok: &'a [Token]) -> LangResult<AlterSpace<'a>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let a = self::parse_alter_space_from_tokens(&mut state)?;
-    assert_full_tt!(state);
-    Ok(a)
-}
-
-states! {
-    /// The field syntax parse state
-    pub struct FieldSyntaxParseState: u8 {
-        IDENT = 0x00,
-        OB = 0x01,
-        FOLD_DICT_INCOMPLETE = 0x02,
-        COMPLETED = 0xFF,
-    }
-}
-
-#[derive(Debug, PartialEq)]
-/// An [`ExpandedField`] is a full field definition with advanced metadata
-pub struct ExpandedField<'a> {
-    pub(super) field_name: Slice<'a>,
-    pub(super) props: Dict,
-    pub(super) layers: Vec<Layer<'a>>,
-    pub(super) reset: bool,
-}
-
-#[cfg(test)]
-pub fn parse_field_syntax_full<'a, const ALLOW_RESET: bool>(
-    tok: &'a [Token],
-) -> LangResult<(ExpandedField<'a>, usize)> {
-    let mut state = State::new(tok, InplaceData::new());
-    self::parse_field_syntax::<InplaceData, ALLOW_RESET>(&mut state)
-        .map(|efield| (efield, state.cursor()))
-}
-
-#[inline(always)]
-/// Parse a field declared using the field syntax
-pub(super) fn parse_field_syntax<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
-    state: &mut State<'a, Qd>,
-) -> LangResult<ExpandedField<'a>> {
-    let mut mstate = FieldSyntaxParseState::IDENT;
-    let mut tmp = MaybeInit::uninit();
-    let mut props = Dict::new();
-    let mut layers = vec![];
-    let mut reset = false;
-    while state.loop_tt() {
-        match (state.fw_read(), mstate) {
-            (Token::Ident(field), FieldSyntaxParseState::IDENT) => {
-                tmp = MaybeInit::new(field.clone());
-                // expect open brace
-                mstate = FieldSyntaxParseState::OB;
-            }
-            (Token::Symbol(Symbol::TtOpenBrace), FieldSyntaxParseState::OB) => {
-                let r = self::rfold_tymeta::<Qd, ALLOW_RESET>(
-                    TyMetaFoldState::IDENT_OR_CB,
-                    state,
-                    &mut props,
-                );
-                if r.has_more() && state.not_exhausted() {
-                    // now parse layers
-                    self::rfold_layers::<Qd, ALLOW_RESET>(LayerFoldState::TY, state, &mut layers);
-                    mstate = FieldSyntaxParseState::FOLD_DICT_INCOMPLETE;
-                } else {
-                    state.poison();
-                    break;
-                }
-            }
-            (Token::Symbol(Symbol::SymComma), FieldSyntaxParseState::FOLD_DICT_INCOMPLETE) => {
-                let r = self::rfold_tymeta::<Qd, ALLOW_RESET>(
-                    TyMetaFoldState::IDENT_OR_CB,
-                    state,
-                    &mut props,
-                );
-                reset = ALLOW_RESET && r.has_reset();
-                if state.okay() {
-                    mstate = FieldSyntaxParseState::COMPLETED;
-                    break;
-                }
-            }
-            (Token::Symbol(Symbol::TtCloseBrace), FieldSyntaxParseState::FOLD_DICT_INCOMPLETE) => {
-                // great, were done
-                mstate = FieldSyntaxParseState::COMPLETED;
-                break;
-            }
-            _ => {
-                state.cursor_back();
-                state.poison();
-                break;
-            }
-        }
-    }
-    state.poison_if_not(mstate == FieldSyntaxParseState::COMPLETED);
-    if state.okay() {
-        Ok(ExpandedField {
-            field_name: unsafe { tmp.assume_init() },
-            layers,
-            props,
-            reset,
-        })
-    } else {
-        Err(LangError::UnexpectedToken)
-    }
-}
-
-#[derive(Debug)]
-#[cfg_attr(test, derive(PartialEq))]
-pub struct Alter<'a> {
-    model: Slice<'a>,
-    kind: AlterKind<'a>,
-}
-
-impl<'a> Alter<'a> {
-    #[inline(always)]
-    pub(super) fn new(model: Slice<'a>, kind: AlterKind<'a>) -> Self {
-        Self { model, kind }
-    }
-}
-
-#[derive(Debug)]
-#[cfg_attr(test, derive(PartialEq))]
-/// The alter operation kind
-pub enum AlterKind<'a> {
-    Add(Box<[ExpandedField<'a>]>),
-    Remove(Box<[Slice<'a>]>),
-    Update(Box<[ExpandedField<'a>]>),
-}
-
-#[inline(always)]
-/// Parse an [`AlterKind`] from the given token stream
-pub(super) fn parse_alter_kind_from_tokens<'a, Qd: QueryData<'a>>(
-    state: &mut State<'a, Qd>,
-) -> LangResult<Alter<'a>> {
-    // alter model mymodel remove x
-    if state.remaining() <= 2 || !state.cursor_has_ident_rounded() {
-        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
-    }
-    let model_name = unsafe { extract!(state.fw_read(), Token::Ident(ref l) => l.clone()) };
-    match state.fw_read() {
-        Token![add] => alter_add(state)
-            .map(AlterKind::Add)
-            .map(|kind| Alter::new(model_name, kind)),
-        Token![remove] => alter_remove(state)
-            .map(AlterKind::Remove)
-            .map(|kind| Alter::new(model_name, kind)),
-        Token![update] => alter_update(state)
-            .map(AlterKind::Update)
-            .map(|kind| Alter::new(model_name, kind)),
-        _ => return Err(LangError::ExpectedStatement),
-    }
-}
-
-#[inline(always)]
-/// Parse multiple fields declared using the field syntax. Flag setting allows or disallows reset syntax
-pub(super) fn parse_multiple_field_syntax<'a, Qd: QueryData<'a>, const ALLOW_RESET: bool>(
-    state: &mut State<'a, Qd>,
-) -> LangResult<Box<[ExpandedField<'a>]>> {
-    const DEFAULT_ADD_COL_CNT: usize = 4;
-    /*
-        WARNING: No trailing commas allowed
-
-        <add> ::= (<field_syntax> <comma>)*
-
-        Smallest length:
-        alter model add myfield { type string }
-    */
-    if compiler::unlikely(state.remaining() < 5) {
-        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
-    }
-    match state.read() {
-        Token::Ident(_) => {
-            let ef = parse_field_syntax::<Qd, ALLOW_RESET>(state)?;
-            Ok([ef].into())
-        }
-        Token::Symbol(Symbol::TtOpenParen) => {
-            state.cursor_ahead();
-            let mut stop = false;
-            let mut cols = Vec::with_capacity(DEFAULT_ADD_COL_CNT);
-            while state.loop_tt() && !stop {
-                match state.read() {
-                    Token::Ident(_) => {
-                        let ef = parse_field_syntax::<Qd, ALLOW_RESET>(state)?;
-                        cols.push(ef);
-                        let nx_comma = state.cursor_rounded_eq(Token![,]);
-                        let nx_close = state.cursor_rounded_eq(Token![() close]);
-                        stop = nx_close;
-                        state.poison_if_not(nx_comma | nx_close);
-                        state.cursor_ahead_if(state.okay());
-                    }
-                    _ => {
-                        state.poison();
-                        break;
-                    }
-                }
-            }
-            state.poison_if_not(stop);
-            if state.okay() {
-                Ok(cols.into_boxed_slice())
-            } else {
-                Err(LangError::UnexpectedToken)
-            }
-        }
-        _ => Err(LangError::ExpectedStatement),
-    }
-}
-
-#[inline(always)]
-/// Parse the expression for `alter model <> add (..)`
-pub(super) fn alter_add<'a, Qd: QueryData<'a>>(
-    state: &mut State<'a, Qd>,
-) -> LangResult<Box<[ExpandedField<'a>]>> {
-    self::parse_multiple_field_syntax::<Qd, DISALLOW_RESET_SYNTAX>(state)
-}
-
-#[cfg(test)]
-pub(super) fn alter_add_full<'a>(
-    tok: &'a [Token],
-    current: &mut usize,
-) -> LangResult<Box<[ExpandedField<'a>]>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let r = self::alter_add(&mut state);
-    *current += state.consumed();
-    r
-}
-
-#[inline(always)]
-/// Parse the expression for `alter model <> remove (..)`
-pub(super) fn alter_remove<'a, Qd: QueryData<'a>>(
-    state: &mut State<'a, Qd>,
-) -> LangResult<Box<[Slice<'a>]>> {
-    const DEFAULT_REMOVE_COL_CNT: usize = 4;
-    /*
-        WARNING: No trailing commas allowed
-        <remove> ::= <ident> | <openparen> (<ident> <comma>)*<closeparen>
-    */
-    if compiler::unlikely(state.exhausted()) {
-        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
-    }
-
-    let r = match state.fw_read() {
-        Token::Ident(id) => Box::new([id.clone()]),
-        Token::Symbol(Symbol::TtOpenParen) => {
-            let mut stop = false;
-            let mut cols = Vec::with_capacity(DEFAULT_REMOVE_COL_CNT);
-            while state.loop_tt() && !stop {
-                match state.fw_read() {
-                    Token::Ident(ref ident) => {
-                        cols.push(ident.clone());
-                        let nx_comma = state.cursor_rounded_eq(Token![,]);
-                        let nx_close = state.cursor_rounded_eq(Token![() close]);
-                        state.poison_if_not(nx_comma | nx_close);
-                        stop = nx_close;
-                        state.cursor_ahead_if(state.okay());
-                    }
-                    _ => {
-                        state.cursor_back();
-                        state.poison();
-                        break;
-                    }
-                }
-            }
-            state.poison_if_not(stop);
-            if state.okay() {
-                cols.into_boxed_slice()
-            } else {
-                return Err(LangError::UnexpectedToken);
-            }
-        }
-        _ => return Err(LangError::ExpectedStatement),
-    };
-    Ok(r)
-}
-
-#[cfg(test)]
-pub(super) fn alter_remove_full<'a>(
-    tok: &'a [Token<'a>],
-    i: &mut usize,
-) -> LangResult<Box<[Slice<'a>]>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let r = self::alter_remove(&mut state);
-    *i += state.consumed();
-    r
-}
-
-#[inline(always)]
-/// Parse the expression for `alter model <> update (..)`
-pub(super) fn alter_update<'a, Qd: QueryData<'a>>(
-    state: &mut State<'a, Qd>,
-) -> LangResult<Box<[ExpandedField<'a>]>> {
-    self::parse_multiple_field_syntax::<Qd, ALLOW_RESET_SYNTAX>(state)
-}
-
-#[cfg(test)]
-pub(super) fn alter_update_full<'a>(
-    tok: &'a [Token],
-    i: &mut usize,
-) -> LangResult<Box<[ExpandedField<'a>]>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let r = self::alter_update(&mut state);
-    *i += state.consumed();
-    r
-}
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 7696b76a..a87c538b 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -33,12 +33,6 @@ use {
     rand::{self, Rng},
 };
 
-macro_rules! fold_dict {
-    ($($input:expr),* $(,)?) => {
-        ($({$crate::engine::ql::schema::fold_dict(&super::lex_insecure($input).unwrap()).unwrap()}),*)
-    }
-}
-
 mod dml_tests;
 mod entity;
 mod lexer_tests;
@@ -81,25 +75,27 @@ fn nullable_datatype(v: impl NullableData<DataType>) -> Option<DataType> {
     v.data()
 }
 
-pub trait NullableMapEntry {
-    fn data(self) -> Option<super::schema::DictEntry>;
+pub trait NullableDictEntry {
+    fn data(self) -> Option<super::ddl::syn::DictEntry>;
 }
 
-impl NullableMapEntry for Null {
-    fn data(self) -> Option<super::schema::DictEntry> {
+impl NullableDictEntry for Null {
+    fn data(self) -> Option<super::ddl::syn::DictEntry> {
         None
     }
 }
 
-impl<'a> NullableMapEntry for super::lex::Lit<'a> {
-    fn data(self) -> Option<super::schema::DictEntry> {
-        Some(super::schema::DictEntry::Lit(self.as_ir().to_litir_owned()))
+impl<'a> NullableDictEntry for super::lex::Lit<'a> {
+    fn data(self) -> Option<super::ddl::syn::DictEntry> {
+        Some(super::ddl::syn::DictEntry::Lit(
+            self.as_ir().to_litir_owned(),
+        ))
     }
 }
 
-impl NullableMapEntry for super::schema::Dict {
-    fn data(self) -> Option<super::schema::DictEntry> {
-        Some(super::schema::DictEntry::Map(self))
+impl NullableDictEntry for super::ddl::syn::Dict {
+    fn data(self) -> Option<super::ddl::syn::DictEntry> {
+        Some(super::ddl::syn::DictEntry::Map(self))
     }
 }
 
@@ -119,21 +115,19 @@ fn fuzz_tokens(src: &[u8], fuzzverify: impl Fn(bool, &[Token]) -> bool) {
         .iter()
         .filter(|tok| FUZZ_TARGETS.contains(tok))
         .count();
-    for _ in 0..(fuzz_amount.pow(2)) {
+    for _ in 0..(fuzz_amount.pow(3)) {
         let mut new_src = Vec::with_capacity(src_tokens.len());
         let mut should_pass = true;
-        src_tokens.iter().for_each(|tok| {
-            match tok {
-                Token::IgnorableComma => {
-                    let added = inject(&mut new_src, &mut rng);
-                    should_pass &= added <= 1;
-                }
-                Token::Symbol(Symbol::SymComma) => {
-                    let added = inject(&mut new_src, &mut rng);
-                    should_pass &= added == 1;
-                }
-                tok => new_src.push(tok.clone()),
+        src_tokens.iter().for_each(|tok| match tok {
+            Token::IgnorableComma => {
+                let added = inject(&mut new_src, &mut rng);
+                should_pass &= added <= 1;
+            }
+            Token::Symbol(Symbol::SymComma) => {
+                let added = inject(&mut new_src, &mut rng);
+                should_pass &= added == 1;
             }
+            tok => new_src.push(tok.clone()),
         });
         if fuzzverify(should_pass, &new_src) && !should_pass {
             panic!(
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index f6a8d90c..680c836b 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -28,8 +28,8 @@ use super::*;
 mod list_parse {
     use super::*;
     use crate::engine::ql::{
-        ast::{InplaceData, SubstitutedData},
-        dml::ins::parse_list_full,
+        ast::{parse_ast_node_full, traits::ASTNode, State, SubstitutedData},
+        dml::ins::List,
         lex::LitIR,
     };
 
@@ -41,7 +41,7 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_list_full(&tok[1..], InplaceData::new()).unwrap();
+        let r = parse_ast_node_full::<List>(&tok[1..]).unwrap().0;
         assert_eq!(r, vec![])
     }
     #[test]
@@ -52,7 +52,7 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_list_full(&tok[1..], InplaceData::new()).unwrap();
+        let r = parse_ast_node_full::<List>(&tok[1..]).unwrap().0;
         assert_eq!(r.as_slice(), into_array![1, 2, 3, 4])
     }
     #[test]
@@ -69,8 +69,11 @@ mod list_parse {
             LitIR::UInt(3),
             LitIR::UInt(4),
         ];
-        let r = parse_list_full(&tok[1..], SubstitutedData::new(&data)).unwrap();
-        assert_eq!(r.as_slice(), into_array![1, 2, 3, 4])
+        let mut state = State::new(&tok[1..], SubstitutedData::new(&data));
+        assert_eq!(
+            <List as ASTNode>::from_state(&mut state).unwrap().0,
+            into_array![1, 2, 3, 4]
+        )
     }
     #[test]
     fn list_pro() {
@@ -85,7 +88,7 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_list_full(&tok[1..], InplaceData::new()).unwrap();
+        let r = parse_ast_node_full::<List>(&tok[1..]).unwrap().0;
         assert_eq!(
             r.as_slice(),
             into_array![
@@ -117,9 +120,9 @@ mod list_parse {
             LitIR::UInt(5),
             LitIR::UInt(6),
         ];
-        let r = parse_list_full(&tok[1..], SubstitutedData::new(&data)).unwrap();
+        let mut state = State::new(&tok[1..], SubstitutedData::new(&data));
         assert_eq!(
-            r.as_slice(),
+            <List as ASTNode>::from_state(&mut state).unwrap().0,
             into_array![
                 into_array![1, 2],
                 into_array![3, 4],
@@ -141,7 +144,7 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_list_full(&tok[1..], InplaceData::new()).unwrap();
+        let r = parse_ast_node_full::<List>(&tok[1..]).unwrap().0;
         assert_eq!(
             r.as_slice(),
             into_array![
@@ -179,9 +182,9 @@ mod list_parse {
             LitIR::UInt(7),
             LitIR::UInt(7),
         ];
-        let r = parse_list_full(&tok[1..], SubstitutedData::new(&data)).unwrap();
+        let mut state = State::new(&tok[1..], SubstitutedData::new(&data));
         assert_eq!(
-            r.as_slice(),
+            <List as ASTNode>::from_state(&mut state).unwrap().0,
             into_array![
                 into_array![into_array![1, 1], into_array![2, 2]],
                 into_array![into_array![], into_array![4, 4]],
@@ -194,12 +197,12 @@ mod list_parse {
 
 mod tuple_syntax {
     use super::*;
-    use crate::engine::ql::dml::ins::parse_data_tuple_syntax_full;
+    use crate::engine::ql::{ast::parse_ast_node_full, dml::ins::DataTuple};
 
     #[test]
     fn tuple_mini() {
         let tok = lex_insecure(b"()").unwrap();
-        let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<DataTuple>(&tok[1..]).unwrap().0;
         assert_eq!(r, vec![]);
     }
 
@@ -211,7 +214,7 @@ mod tuple_syntax {
             "#,
         )
         .unwrap();
-        let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<DataTuple>(&tok[1..]).unwrap().0;
         assert_eq!(
             r.as_slice(),
             into_array_nullable![1234, "email@example.com", true]
@@ -231,7 +234,7 @@ mod tuple_syntax {
             "#,
         )
         .unwrap();
-        let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<DataTuple>(&tok[1..]).unwrap().0;
         assert_eq!(
             r.as_slice(),
             into_array_nullable![
@@ -263,7 +266,7 @@ mod tuple_syntax {
             "#,
         )
         .unwrap();
-        let r = parse_data_tuple_syntax_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<DataTuple>(&tok[1..]).unwrap().0;
         assert_eq!(
             r.as_slice(),
             into_array_nullable![
@@ -284,13 +287,13 @@ mod tuple_syntax {
 }
 mod map_syntax {
     use super::*;
-    use crate::engine::ql::dml::ins::parse_data_map_syntax_full;
+    use crate::engine::ql::{ast::parse_ast_node_full, dml::ins::DataMap};
 
     #[test]
     fn map_mini() {
         let tok = lex_insecure(b"{}").unwrap();
-        let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
-        assert_eq!(r, nullable_dict! {})
+        let r = parse_ast_node_full::<DataMap>(&tok[1..]).unwrap().0;
+        assert_eq!(r, null_dict! {})
     }
 
     #[test]
@@ -306,7 +309,7 @@ mod map_syntax {
             "#,
         )
         .unwrap();
-        let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<DataMap>(&tok[1..]).unwrap().0;
         assert_eq!(
             r,
             dict_nullable! {
@@ -332,7 +335,7 @@ mod map_syntax {
             "#,
         )
         .unwrap();
-        let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<DataMap>(&tok[1..]).unwrap().0;
         assert_eq!(
             r,
             dict_nullable! {
@@ -361,7 +364,7 @@ mod map_syntax {
                 }
             "#)
         .unwrap();
-        let r = parse_data_map_syntax_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<DataMap>(&tok[1..]).unwrap().0;
         assert_eq!(
             r,
             dict_nullable! {
@@ -386,7 +389,7 @@ mod stmt_insert {
     use {
         super::*,
         crate::engine::ql::{
-            ast::Entity,
+            ast::{parse_ast_node_full, Entity},
             dml::{self, ins::InsertStatement},
         },
     };
@@ -399,7 +402,7 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::ins::parse_insert_full(&x[1..]).unwrap();
+        let r = parse_ast_node_full::<InsertStatement>(&x[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(b"twitter", b"users"),
             into_array_nullable!["sayan"].to_vec().into(),
@@ -421,7 +424,7 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::ins::parse_insert_full(&x[1..]).unwrap();
+        let r = parse_ast_node_full::<InsertStatement>(&x[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(b"twitter", b"users"),
             into_array_nullable!["sayan", "Sayan", "sayan@example.com", true, 12345, 67890]
@@ -448,7 +451,7 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::ins::parse_insert_full(&x[1..]).unwrap();
+        let r = parse_ast_node_full::<InsertStatement>(&x[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(b"twitter", b"users"),
             into_array_nullable![
@@ -475,7 +478,7 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::ins::parse_insert_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(b"jotsy", b"app"),
             dict_nullable! {
@@ -500,7 +503,7 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::ins::parse_insert_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(b"jotsy", b"app"),
             dict_nullable! {
@@ -533,7 +536,7 @@ mod stmt_insert {
             "#,
         )
         .unwrap();
-        let r = dml::ins::parse_insert_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(b"jotsy", b"app"),
             dict_nullable! {
@@ -555,7 +558,7 @@ mod stmt_insert {
     fn insert_tuple_fnsub() {
         let tok =
             lex_insecure(br#"insert into jotsy.app(@uuidstr(), "sayan", @timesec())"#).unwrap();
-        let ret = dml::ins::parse_insert_full(&tok[1..]).unwrap();
+        let ret = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let expected = InsertStatement::new(
             Entity::Full(b"jotsy", b"app"),
             into_array_nullable![dml::ins::T_UUIDSTR, "sayan", dml::ins::T_TIMESEC]
@@ -569,7 +572,7 @@ mod stmt_insert {
         let tok = lex_insecure(
             br#"insert into jotsy.app { uuid: @uuidstr(), username: "sayan", signup_time: @timesec() }"#
         ).unwrap();
-        let ret = dml::ins::parse_insert_full(&tok[1..]).unwrap();
+        let ret = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let expected = InsertStatement::new(
             Entity::Full(b"jotsy", b"app"),
             dict_nullable! {
@@ -584,13 +587,11 @@ mod stmt_insert {
 }
 
 mod stmt_select {
-    use crate::engine::ql::dml::RelationalExpr;
-
     use {
         super::*,
         crate::engine::ql::{
-            ast::Entity,
-            dml::{self, sel::SelectStatement},
+            ast::{parse_ast_node_full, Entity},
+            dml::{sel::SelectStatement, RelationalExpr},
             lex::LitIR,
         },
     };
@@ -602,7 +603,7 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = dml::sel::parse_select_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<SelectStatement>(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
             Entity::Single(b"users"),
             [].to_vec(),
@@ -623,7 +624,7 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = dml::sel::parse_select_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<SelectStatement>(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
             Entity::Single(b"users"),
             [b"field1".as_slice()].to_vec(),
@@ -644,7 +645,7 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = dml::sel::parse_select_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<SelectStatement>(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
             Entity::Full(b"twitter", b"users"),
             [b"field1".as_slice()].to_vec(),
@@ -665,7 +666,7 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = dml::sel::parse_select_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<SelectStatement>(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
             Entity::Full(b"twitter", b"users"),
             [b"field1".as_slice(), b"field2".as_slice()].to_vec(),
@@ -740,9 +741,8 @@ mod update_statement {
     use {
         super::*,
         crate::engine::ql::{
-            ast::Entity,
+            ast::{parse_ast_node_full, Entity},
             dml::{
-                self,
                 upd::{AssignmentExpression, Operator, UpdateStatement},
                 RelationalExpr, WhereClause,
             },
@@ -757,7 +757,7 @@ mod update_statement {
             "#,
         )
         .unwrap();
-        let r = dml::upd::parse_update_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<UpdateStatement>(&tok[1..]).unwrap();
         let e = UpdateStatement::new(
             Entity::Single(b"app"),
             vec![AssignmentExpression::new(
@@ -789,7 +789,7 @@ mod update_statement {
             "#,
         )
         .unwrap();
-        let r = dml::upd::parse_update_full(&tok[1..]).unwrap();
+        let r = parse_ast_node_full::<UpdateStatement>(&tok[1..]).unwrap();
         let e = UpdateStatement::new(
             Entity::Full(b"jotsy", b"app"),
             vec![
@@ -819,11 +819,8 @@ mod delete_stmt {
     use {
         super::*,
         crate::engine::ql::{
-            ast::Entity,
-            dml::{
-                del::{self, DeleteStatement},
-                RelationalExpr,
-            },
+            ast::{parse_ast_node_full, Entity},
+            dml::{del::DeleteStatement, RelationalExpr},
             lex::LitIR,
         },
     };
@@ -846,8 +843,10 @@ mod delete_stmt {
                 )
             },
         );
-        let r = del::parse_delete_full(&tok[1..]).unwrap();
-        assert_eq!(r, e);
+        assert_eq!(
+            parse_ast_node_full::<DeleteStatement>(&tok[1..]).unwrap(),
+            e
+        );
     }
     #[test]
     fn delete() {
@@ -867,23 +866,22 @@ mod delete_stmt {
                 )
             },
         );
-        let r = del::parse_delete_full(&tok[1..]).unwrap();
-        assert_eq!(r, e);
+        assert_eq!(
+            parse_ast_node_full::<DeleteStatement>(&tok[1..]).unwrap(),
+            e
+        );
     }
 }
 mod relational_expr {
     use {
         super::*,
-        crate::engine::ql::{
-            dml::{self, RelationalExpr},
-            lex::LitIR,
-        },
+        crate::engine::ql::{ast::parse_ast_node_full, dml::RelationalExpr, lex::LitIR},
     };
 
     #[test]
     fn expr_eq() {
         let expr = lex_insecure(b"primary_key = 10").unwrap();
-        let r = dml::parse_relexpr_full(&expr).unwrap();
+        let r = parse_ast_node_full::<RelationalExpr>(&expr).unwrap();
         assert_eq!(
             r,
             RelationalExpr {
@@ -896,7 +894,7 @@ mod relational_expr {
     #[test]
     fn expr_ne() {
         let expr = lex_insecure(b"primary_key != 10").unwrap();
-        let r = dml::parse_relexpr_full(&expr).unwrap();
+        let r = parse_ast_node_full::<RelationalExpr>(&expr).unwrap();
         assert_eq!(
             r,
             RelationalExpr {
@@ -909,7 +907,7 @@ mod relational_expr {
     #[test]
     fn expr_gt() {
         let expr = lex_insecure(b"primary_key > 10").unwrap();
-        let r = dml::parse_relexpr_full(&expr).unwrap();
+        let r = parse_ast_node_full::<RelationalExpr>(&expr).unwrap();
         assert_eq!(
             r,
             RelationalExpr {
@@ -922,7 +920,7 @@ mod relational_expr {
     #[test]
     fn expr_ge() {
         let expr = lex_insecure(b"primary_key >= 10").unwrap();
-        let r = dml::parse_relexpr_full(&expr).unwrap();
+        let r = parse_ast_node_full::<RelationalExpr>(&expr).unwrap();
         assert_eq!(
             r,
             RelationalExpr {
@@ -935,7 +933,7 @@ mod relational_expr {
     #[test]
     fn expr_lt() {
         let expr = lex_insecure(b"primary_key < 10").unwrap();
-        let r = dml::parse_relexpr_full(&expr).unwrap();
+        let r = parse_ast_node_full::<RelationalExpr>(&expr).unwrap();
         assert_eq!(
             r,
             RelationalExpr {
@@ -948,7 +946,7 @@ mod relational_expr {
     #[test]
     fn expr_le() {
         let expr = lex_insecure(b"primary_key <= 10").unwrap();
-        let r = dml::parse_relexpr_full(&expr).unwrap();
+        let r = parse_ast_node_full::<RelationalExpr>(&expr).unwrap();
         assert_eq!(
             r,
             RelationalExpr::new(
@@ -963,7 +961,8 @@ mod where_clause {
     use {
         super::*,
         crate::engine::ql::{
-            dml::{self, RelationalExpr, WhereClause},
+            ast::parse_ast_node_full,
+            dml::{RelationalExpr, WhereClause},
             lex::LitIR,
         },
     };
@@ -982,7 +981,7 @@ mod where_clause {
                 RelationalExpr::OP_EQ
             )
         });
-        assert_eq!(expected, dml::parse_where_clause_full(&tok).unwrap());
+        assert_eq!(expected, parse_ast_node_full::<WhereClause>(&tok).unwrap());
     }
     #[test]
     fn where_double() {
@@ -1004,7 +1003,7 @@ mod where_clause {
                 RelationalExpr::OP_EQ
             )
         });
-        assert_eq!(expected, dml::parse_where_clause_full(&tok).unwrap());
+        assert_eq!(expected, parse_ast_node_full::<WhereClause>(&tok).unwrap());
     }
     #[test]
     fn where_duplicate_condition() {
@@ -1014,6 +1013,6 @@ mod where_clause {
             "#,
         )
         .unwrap();
-        assert!(dml::parse_where_clause_full(&tok).is_none());
+        assert!(parse_ast_node_full::<WhereClause>(&tok).is_err());
     }
 }
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 735968bc..18710609 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -25,25 +25,24 @@
 */
 
 use super::{
-    super::{
-        lex::{Lit, Token},
-        schema,
-    },
+    super::lex::{Lit, Token},
     lex_insecure, *,
 };
 mod inspect {
     use {
         super::*,
         crate::engine::ql::{
-            ast::{Entity, Statement},
-            ddl,
+            ast::{parse_ast_node_full, Entity, Statement},
+            ddl::ins::InspectStatementAST,
         },
     };
     #[test]
     fn inspect_space() {
         let tok = lex_insecure(b"inspect space myspace").unwrap();
         assert_eq!(
-            ddl::ins::parse_inspect_full(&tok[1..]).unwrap(),
+            parse_ast_node_full::<InspectStatementAST>(&tok[1..])
+                .unwrap()
+                .0,
             Statement::InspectSpace(b"myspace")
         );
     }
@@ -51,12 +50,16 @@ mod inspect {
     fn inspect_model() {
         let tok = lex_insecure(b"inspect model users").unwrap();
         assert_eq!(
-            ddl::ins::parse_inspect_full(&tok[1..]).unwrap(),
+            parse_ast_node_full::<InspectStatementAST>(&tok[1..])
+                .unwrap()
+                .0,
             Statement::InspectModel(Entity::Single(b"users"))
         );
         let tok = lex_insecure(b"inspect model tweeter.users").unwrap();
         assert_eq!(
-            ddl::ins::parse_inspect_full(&tok[1..]).unwrap(),
+            parse_ast_node_full::<InspectStatementAST>(&tok[1..])
+                .unwrap()
+                .0,
             Statement::InspectModel(Entity::Full(b"tweeter", b"users"))
         );
     }
@@ -64,7 +67,9 @@ mod inspect {
     fn inspect_spaces() {
         let tok = lex_insecure(b"inspect spaces").unwrap();
         assert_eq!(
-            ddl::ins::parse_inspect_full(&tok[1..]).unwrap(),
+            parse_ast_node_full::<InspectStatementAST>(&tok[1..])
+                .unwrap()
+                .0,
             Statement::InspectSpaces
         );
     }
@@ -73,22 +78,13 @@ mod inspect {
 mod alter_space {
     use {
         super::*,
-        crate::engine::ql::{
-            lex::Lit,
-            schema::{self, AlterSpace},
-        },
+        crate::engine::ql::{ast::parse_ast_node_full, ddl::alt::AlterSpace, lex::Lit},
     };
     #[test]
     fn alter_space_mini() {
         let tok = lex_insecure(b"alter model mymodel with {}").unwrap();
-        let r = schema::alter_space_full(&tok[2..]).unwrap();
-        assert_eq!(
-            r,
-            AlterSpace {
-                space_name: b"mymodel",
-                updated_props: nullable_dict! {}
-            }
-        );
+        let r = parse_ast_node_full::<AlterSpace>(&tok[2..]).unwrap();
+        assert_eq!(r, AlterSpace::new(b"mymodel", null_dict! {}));
     }
     #[test]
     fn alter_space() {
@@ -101,49 +97,44 @@ mod alter_space {
             "#,
         )
         .unwrap();
-        let r = schema::alter_space_full(&tok[2..]).unwrap();
+        let r = parse_ast_node_full::<AlterSpace>(&tok[2..]).unwrap();
         assert_eq!(
             r,
-            AlterSpace {
-                space_name: b"mymodel",
-                updated_props: nullable_dict! {
+            AlterSpace::new(
+                b"mymodel",
+                null_dict! {
                     "max_entry" => Lit::UnsignedInt(1000),
                     "driver" => Lit::Str("ts-0.8".into())
                 }
-            }
+            )
         );
     }
 }
 mod tymeta {
     use super::*;
+    use crate::engine::ql::{
+        ast::{parse_ast_node_full, traits::ASTNode, State},
+        ddl::syn::{DictTypeMeta, DictTypeMetaSplit},
+    };
     #[test]
     fn tymeta_mini() {
-        let tok = lex_insecure(b"}").unwrap();
-        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
-        assert!(okay);
-        assert!(!tymeta.has_more());
-        assert_eq!(cursor, 1);
-        assert_eq!(data, nullable_dict!());
+        let tok = lex_insecure(b"{}").unwrap();
+        let tymeta = parse_ast_node_full::<DictTypeMeta>(&tok).unwrap();
+        assert_eq!(tymeta.0, null_dict!());
     }
     #[test]
+    #[should_panic]
     fn tymeta_mini_fail() {
-        let tok = lex_insecure(b",}").unwrap();
-        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
-        assert!(!okay);
-        assert!(!tymeta.has_more());
-        assert_eq!(cursor, 0);
-        assert_eq!(data, nullable_dict!());
+        let tok = lex_insecure(b"{,}").unwrap();
+        parse_ast_node_full::<DictTypeMeta>(&tok).unwrap();
     }
     #[test]
     fn tymeta() {
-        let tok = lex_insecure(br#"hello: "world", loading: true, size: 100 }"#).unwrap();
-        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
-        assert!(okay);
-        assert!(!tymeta.has_more());
-        assert_eq!(cursor, tok.len());
+        let tok = lex_insecure(br#"{hello: "world", loading: true, size: 100 }"#).unwrap();
+        let tymeta = parse_ast_node_full::<DictTypeMeta>(&tok).unwrap();
         assert_eq!(
-            data,
-            nullable_dict! {
+            tymeta.0,
+            null_dict! {
                 "hello" => Lit::Str("world".into()),
                 "loading" => Lit::Bool(true),
                 "size" => Lit::UnsignedInt(100)
@@ -152,23 +143,22 @@ mod tymeta {
     }
     #[test]
     fn tymeta_pro() {
-        // list { maxlen: 100, type string, unique: true }
+        // list { maxlen: 100, type: string, unique: true }
         //        ^^^^^^^^^^^^^^^^^^ cursor should be at string
-        let tok = lex_insecure(br#"maxlen: 100, type string, unique: true }"#).unwrap();
-        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
-        assert!(okay);
-        assert!(tymeta.has_more());
-        assert_eq!(cursor, 5);
-        let remslice = &tok[cursor + 2..];
-        let (tymeta2, okay2, cursor2, data2) = schema::fold_tymeta(remslice);
-        assert!(okay2);
-        assert!(!tymeta2.has_more());
-        assert_eq!(cursor2 + cursor + 2, tok.len());
-        let mut final_ret = data;
-        final_ret.extend(data2);
+        let tok = lex_insecure(br#"{maxlen: 100, type: string, unique: true }"#).unwrap();
+        let mut state = State::new_inplace(&tok);
+        let tymeta: DictTypeMeta = ASTNode::from_state(&mut state).unwrap();
+        assert_eq!(state.cursor(), 6);
+        assert!(Token![:].eq(state.fw_read()));
+        assert!(Token::Ident(b"string").eq(state.fw_read()));
+        assert!(Token![,].eq(state.fw_read()));
+        let tymeta2: DictTypeMetaSplit = ASTNode::from_state(&mut state).unwrap();
+        assert!(state.exhausted());
+        let mut final_ret = tymeta;
+        final_ret.0.extend(tymeta2.0);
         assert_eq!(
-            final_ret,
-            nullable_dict! {
+            final_ret.0,
+            null_dict! {
                 "maxlen" => Lit::UnsignedInt(100),
                 "unique" => Lit::Bool(true)
             }
@@ -176,123 +166,51 @@ mod tymeta {
     }
     #[test]
     fn tymeta_pro_max() {
-        // list { maxlen: 100, this: { is: "cool" }, type string, unique: true }
+        // list { maxlen: 100, this: { is: "cool" }, type: string, unique: true }
         //        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cursor should be at string
         let tok =
-            lex_insecure(br#"maxlen: 100, this: { is: "cool" }, type string, unique: true }"#)
+            lex_insecure(br#"{maxlen: 100, this: { is: "cool" }, type: string, unique: true }"#)
                 .unwrap();
-        let (tymeta, okay, cursor, data) = schema::fold_tymeta(&tok);
-        assert!(okay);
-        assert!(tymeta.has_more());
-        assert_eq!(cursor, 13);
-        let remslice = &tok[cursor + 2..];
-        let (tymeta2, okay2, cursor2, data2) = schema::fold_tymeta(remslice);
-        assert!(okay2);
-        assert!(!tymeta2.has_more());
-        assert_eq!(cursor2 + cursor + 2, tok.len());
-        let mut final_ret = data;
-        final_ret.extend(data2);
+        let mut state = State::new_inplace(&tok);
+        let tymeta: DictTypeMeta = ASTNode::from_state(&mut state).unwrap();
+        assert_eq!(state.cursor(), 14);
+        assert!(Token![:].eq(state.fw_read()));
+        assert!(Token::Ident(b"string").eq(state.fw_read()));
+        assert!(Token![,].eq(state.fw_read()));
+        let tymeta2: DictTypeMetaSplit = ASTNode::from_state(&mut state).unwrap();
+        assert!(state.exhausted());
+        let mut final_ret = tymeta;
+        final_ret.0.extend(tymeta2.0);
         assert_eq!(
-            final_ret,
-            nullable_dict! {
+            final_ret.0,
+            null_dict! {
                 "maxlen" => Lit::UnsignedInt(100),
                 "unique" => Lit::Bool(true),
-                "this" => nullable_dict! {
+                "this" => null_dict! {
                     "is" => Lit::Str("cool".into())
                 }
             }
         )
     }
-    #[test]
-    fn fuzz_tymeta_normal() {
-        // { maxlen: 10, unique: true, users: "sayan" }
-        //   ^start
-        let tok = b"
-                maxlen: 10,
-                unique: true,
-                auth: {
-                    maybe: true\x01
-                },
-                users: \"sayan\"\x01
-            }
-        ";
-        let expected = nullable_dict! {
-            "maxlen" => Lit::UnsignedInt(10),
-            "unique" => Lit::Bool(true),
-            "auth" => nullable_dict! {
-                "maybe" => Lit::Bool(true),
-            },
-            "users" => Lit::Str("sayan".into())
-        };
-        fuzz_tokens(tok.as_slice(), |should_pass, new_src| {
-            let (tymeta, okay, cursor, data) = schema::fold_tymeta(&new_src);
-            if should_pass {
-                assert!(okay, "{:?}", &new_src);
-                assert!(!tymeta.has_more());
-                assert_eq!(cursor, new_src.len());
-                assert_eq!(data, expected);
-            }
-            okay
-        });
-    }
-    #[test]
-    fn fuzz_tymeta_with_ty() {
-        // list { maxlen: 10, unique: true, type string, users: "sayan" }
-        //   ^start
-        let tok = b"
-                maxlen: 10,
-                unique: true,
-                auth: {
-                    maybe: true\x01
-                },
-                type string,
-                users: \"sayan\"\x01
-            }
-        ";
-        let expected = nullable_dict! {
-            "maxlen" => Lit::UnsignedInt(10),
-            "unique" => Lit::Bool(true),
-            "auth" => nullable_dict! {
-                "maybe" => Lit::Bool(true),
-            },
-        };
-        fuzz_tokens(tok.as_slice(), |should_pass, new_src| {
-            let (tymeta, okay, cursor, data) = schema::fold_tymeta(&new_src);
-            if should_pass {
-                assert!(okay);
-                assert!(tymeta.has_more());
-                assert!(new_src[cursor] == Token::Ident(b"string"));
-                assert_eq!(data, expected);
-            }
-            okay
-        });
-    }
 }
 mod layer {
     use super::*;
-    use crate::engine::ql::schema::Layer;
+    use crate::engine::ql::{ast::parse_ast_node_multiple_full, ddl::syn::Layer};
     #[test]
     fn layer_mini() {
-        let tok = lex_insecure(b"string)").unwrap();
-        let (layers, c, okay) = schema::fold_layers(&tok);
-        assert_eq!(c, tok.len() - 1);
-        assert!(okay);
-        assert_eq!(
-            layers,
-            vec![Layer::new_noreset(b"string", nullable_dict! {})]
-        );
+        let tok = lex_insecure(b"string").unwrap();
+        let layers = parse_ast_node_multiple_full::<Layer>(&tok).unwrap();
+        assert_eq!(layers, vec![Layer::new(b"string", null_dict! {})]);
     }
     #[test]
     fn layer() {
         let tok = lex_insecure(b"string { maxlen: 100 }").unwrap();
-        let (layers, c, okay) = schema::fold_layers(&tok);
-        assert_eq!(c, tok.len());
-        assert!(okay);
+        let layers = parse_ast_node_multiple_full::<Layer>(&tok).unwrap();
         assert_eq!(
             layers,
-            vec![Layer::new_noreset(
+            vec![Layer::new(
                 b"string",
-                nullable_dict! {
+                null_dict! {
                     "maxlen" => Lit::UnsignedInt(100)
                 }
             )]
@@ -300,31 +218,27 @@ mod layer {
     }
     #[test]
     fn layer_plus() {
-        let tok = lex_insecure(b"list { type string }").unwrap();
-        let (layers, c, okay) = schema::fold_layers(&tok);
-        assert_eq!(c, tok.len());
-        assert!(okay);
+        let tok = lex_insecure(b"list { type: string }").unwrap();
+        let layers = parse_ast_node_multiple_full::<Layer>(&tok).unwrap();
         assert_eq!(
             layers,
             vec![
-                Layer::new_noreset(b"string", nullable_dict! {}),
-                Layer::new_noreset(b"list", nullable_dict! {})
+                Layer::new(b"string", null_dict! {}),
+                Layer::new(b"list", null_dict! {})
             ]
         );
     }
     #[test]
     fn layer_pro() {
-        let tok = lex_insecure(b"list { unique: true, type string, maxlen: 10 }").unwrap();
-        let (layers, c, okay) = schema::fold_layers(&tok);
-        assert_eq!(c, tok.len());
-        assert!(okay);
+        let tok = lex_insecure(b"list { unique: true, type: string, maxlen: 10 }").unwrap();
+        let layers = parse_ast_node_multiple_full::<Layer>(&tok).unwrap();
         assert_eq!(
             layers,
             vec![
-                Layer::new_noreset(b"string", nullable_dict! {}),
-                Layer::new_noreset(
+                Layer::new(b"string", null_dict! {}),
+                Layer::new(
                     b"list",
-                    nullable_dict! {
+                    null_dict! {
                         "unique" => Lit::Bool(true),
                         "maxlen" => Lit::UnsignedInt(10),
                     }
@@ -335,25 +249,23 @@ mod layer {
     #[test]
     fn layer_pro_max() {
         let tok = lex_insecure(
-            b"list { unique: true, type string { ascii_only: true, maxlen: 255 }, maxlen: 10 }",
+            b"list { unique: true, type: string { ascii_only: true, maxlen: 255 }, maxlen: 10 }",
         )
         .unwrap();
-        let (layers, c, okay) = schema::fold_layers(&tok);
-        assert_eq!(c, tok.len());
-        assert!(okay);
+        let layers = parse_ast_node_multiple_full::<Layer>(&tok).unwrap();
         assert_eq!(
             layers,
             vec![
-                Layer::new_noreset(
+                Layer::new(
                     b"string",
-                    nullable_dict! {
+                    null_dict! {
                         "ascii_only" => Lit::Bool(true),
                         "maxlen" => Lit::UnsignedInt(255)
                     }
                 ),
-                Layer::new_noreset(
+                Layer::new(
                     b"list",
-                    nullable_dict! {
+                    null_dict! {
                         "unique" => Lit::Bool(true),
                         "maxlen" => Lit::UnsignedInt(10),
                     }
@@ -366,101 +278,67 @@ mod layer {
     fn fuzz_layer() {
         let tok = b"
             list {
-                type list {
+                type: list {
                     maxlen: 100,
-                    type string\x01
+                    type: string\x01
                 },
                 unique: true\x01
             }
         ";
         let expected = vec![
-            Layer::new_noreset(b"string", nullable_dict!()),
-            Layer::new_noreset(
+            Layer::new(b"string", null_dict!()),
+            Layer::new(
                 b"list",
-                nullable_dict! {
+                null_dict! {
                     "maxlen" => Lit::UnsignedInt(100),
                 },
             ),
-            Layer::new_noreset(b"list", nullable_dict!("unique" => Lit::Bool(true))),
+            Layer::new(b"list", null_dict!("unique" => Lit::Bool(true))),
         ];
         fuzz_tokens(tok.as_slice(), |should_pass, new_tok| {
-            let (layers, c, okay) = schema::fold_layers(&new_tok);
+            let layers = parse_ast_node_multiple_full::<Layer>(&new_tok);
+            let ok = layers.is_ok();
             if should_pass {
-                assert!(okay);
-                assert_eq!(c, new_tok.len());
-                assert_eq!(layers, expected);
+                assert_eq!(layers.unwrap(), expected);
             }
-            okay
+            ok
         });
     }
 }
-mod field_properties {
-    use {super::*, crate::engine::ql::schema::FieldProperties};
-
-    #[test]
-    fn field_properties_empty() {
-        let tok = lex_insecure(b"myfield:").unwrap();
-        let (props, c, okay) = schema::parse_field_properties(&tok);
-        assert!(okay);
-        assert_eq!(c, 0);
-        assert_eq!(props, FieldProperties::default());
-    }
-    #[test]
-    fn field_properties_full() {
-        let tok = lex_insecure(b"primary null myfield:").unwrap();
-        let (props, c, okay) = schema::parse_field_properties(&tok);
-        assert_eq!(c, 2);
-        assert_eq!(tok[c], Token::Ident(b"myfield"));
-        assert!(okay);
-        assert_eq!(
-            props,
-            FieldProperties {
-                properties: set!["primary", "null"],
-            }
-        )
-    }
-}
 mod fields {
     use {
         super::*,
-        crate::engine::ql::schema::{Field, Layer},
+        crate::engine::ql::{
+            ast::parse_ast_node_full,
+            ddl::syn::{Field, Layer},
+        },
     };
     #[test]
     fn field_mini() {
-        let tok = lex_insecure(
-            b"
-                username: string,
-            ",
-        )
-        .unwrap();
-        let (c, f) = schema::parse_field_full(&tok).unwrap();
-        assert_eq!(c, tok.len() - 1);
+        let tok = lex_insecure(b"username: string").unwrap();
+        let f = parse_ast_node_full::<Field>(&tok).unwrap();
         assert_eq!(
             f,
-            Field {
-                field_name: b"username",
-                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
-                props: set![],
-            }
+            Field::new(
+                b"username",
+                [Layer::new(b"string", null_dict! {})].into(),
+                false,
+                false
+            )
         )
     }
     #[test]
     fn field() {
-        let tok = lex_insecure(
-            b"
-                primary username: string,    
-            ",
-        )
-        .unwrap();
-        let (c, f) = schema::parse_field_full(&tok).unwrap();
-        assert_eq!(c, tok.len() - 1);
+        let tok = lex_insecure(b"primary username: string").unwrap();
+        let f = parse_ast_node_full::<Field>(&tok).unwrap();
         assert_eq!(
             f,
-            Field {
-                field_name: b"username",
-                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
-                props: set!["primary"],
-            }
+            Field::new(
+                b"username",
+                [Layer::new(b"string", null_dict! {})].into(),
+                false,
+                true
+            )
         )
     }
     #[test]
@@ -474,22 +352,22 @@ mod fields {
             ",
         )
         .unwrap();
-        let (c, f) = schema::parse_field_full(&tok).unwrap();
-        assert_eq!(c, tok.len());
+        let f = parse_ast_node_full::<Field>(&tok).unwrap();
         assert_eq!(
             f,
-            Field {
-                field_name: b"username",
-                layers: [Layer::new_noreset(
+            Field::new(
+                b"username",
+                [Layer::new(
                     b"string",
-                    nullable_dict! {
+                    null_dict! {
                         "maxlen" => Lit::UnsignedInt(10),
                         "ascii_only" => Lit::Bool(true),
                     }
                 )]
                 .into(),
-                props: set!["primary"],
-            }
+                false,
+                true,
+            )
         )
     }
     #[test]
@@ -497,7 +375,7 @@ mod fields {
         let tok = lex_insecure(
             b"
                 null notes: list {
-                    type string {
+                    type: string {
                         maxlen: 255,
                         ascii_only: true,
                     },
@@ -506,44 +384,49 @@ mod fields {
             ",
         )
         .unwrap();
-        let (c, f) = schema::parse_field_full(&tok).unwrap();
-        assert_eq!(c, tok.len());
+        let f = parse_ast_node_full::<Field>(&tok).unwrap();
         assert_eq!(
             f,
-            Field {
-                field_name: b"notes",
-                layers: [
-                    Layer::new_noreset(
+            Field::new(
+                b"notes",
+                [
+                    Layer::new(
                         b"string",
-                        nullable_dict! {
+                        null_dict! {
                             "maxlen" => Lit::UnsignedInt(255),
                             "ascii_only" => Lit::Bool(true),
                         }
                     ),
-                    Layer::new_noreset(
+                    Layer::new(
                         b"list",
-                        nullable_dict! {
+                        null_dict! {
                             "unique" => Lit::Bool(true)
                         }
                     ),
                 ]
                 .into(),
-                props: set!["null"],
-            }
+                true,
+                false,
+            )
         )
     }
 }
 mod schemas {
-    use crate::engine::ql::schema::{Field, Layer, Model};
-
     use super::*;
+    use crate::engine::ql::{
+        ast::parse_ast_node_full,
+        ddl::{
+            crt::Model,
+            syn::{Field, Layer},
+        },
+    };
     #[test]
     fn schema_mini() {
         let tok = lex_insecure(
             b"
                 create model mymodel(
                     primary username: string,
-                    password: binary,
+                    password: binary
                 )
             ",
         )
@@ -551,26 +434,28 @@ mod schemas {
         let tok = &tok[2..];
 
         // parse model
-        let (model, c) = schema::parse_schema_from_tokens_full(tok).unwrap();
-        assert_eq!(c, tok.len());
+        let model = parse_ast_node_full::<Model>(tok).unwrap();
+
         assert_eq!(
             model,
-            Model {
-                model_name: b"mymodel",
-                fields: vec![
-                    Field {
-                        field_name: b"username",
-                        layers: vec![Layer::new_noreset(b"string", nullable_dict! {})],
-                        props: set!["primary"]
-                    },
-                    Field {
-                        field_name: b"password",
-                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
-                        props: set![]
-                    }
+            Model::new(
+                b"mymodel",
+                vec![
+                    Field::new(
+                        b"username",
+                        vec![Layer::new(b"string", null_dict! {})],
+                        false,
+                        true,
+                    ),
+                    Field::new(
+                        b"password",
+                        vec![Layer::new(b"binary", null_dict! {})],
+                        false,
+                        false,
+                    )
                 ],
-                props: nullable_dict! {}
-            }
+                null_dict! {}
+            )
         )
     }
     #[test]
@@ -580,7 +465,7 @@ mod schemas {
                 create model mymodel(
                     primary username: string,
                     password: binary,
-                    null profile_pic: binary,
+                    null profile_pic: binary
                 )
             ",
         )
@@ -588,31 +473,34 @@ mod schemas {
         let tok = &tok[2..];
 
         // parse model
-        let (model, c) = schema::parse_schema_from_tokens_full(tok).unwrap();
-        assert_eq!(c, tok.len());
+        let model = parse_ast_node_full::<Model>(tok).unwrap();
+
         assert_eq!(
             model,
-            Model {
-                model_name: b"mymodel",
-                fields: vec![
-                    Field {
-                        field_name: b"username",
-                        layers: vec![Layer::new_noreset(b"string", nullable_dict! {})],
-                        props: set!["primary"]
-                    },
-                    Field {
-                        field_name: b"password",
-                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
-                        props: set![]
-                    },
-                    Field {
-                        field_name: b"profile_pic",
-                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
-                        props: set!["null"]
-                    }
+            Model::new(
+                b"mymodel",
+                vec![
+                    Field::new(
+                        b"username",
+                        vec![Layer::new(b"string", null_dict! {})],
+                        false,
+                        true,
+                    ),
+                    Field::new(
+                        b"password",
+                        vec![Layer::new(b"binary", null_dict! {})],
+                        false,
+                        false,
+                    ),
+                    Field::new(
+                        b"profile_pic",
+                        vec![Layer::new(b"binary", null_dict! {})],
+                        true,
+                        false,
+                    )
                 ],
-                props: nullable_dict! {}
-            }
+                null_dict! {}
+            )
         )
     }
 
@@ -625,9 +513,9 @@ mod schemas {
                     password: binary,
                     null profile_pic: binary,
                     null notes: list {
-                        type string,
+                        type: string,
                         unique: true,
-                    },
+                    }
                 )
             ",
         )
@@ -635,44 +523,48 @@ mod schemas {
         let tok = &tok[2..];
 
         // parse model
-        let (model, c) = schema::parse_schema_from_tokens_full(tok).unwrap();
-        assert_eq!(c, tok.len());
+        let model = parse_ast_node_full::<Model>(tok).unwrap();
+
         assert_eq!(
             model,
-            Model {
-                model_name: b"mymodel",
-                fields: vec![
-                    Field {
-                        field_name: b"username",
-                        layers: vec![Layer::new_noreset(b"string", nullable_dict! {})],
-                        props: set!["primary"]
-                    },
-                    Field {
-                        field_name: b"password",
-                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
-                        props: set![]
-                    },
-                    Field {
-                        field_name: b"profile_pic",
-                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
-                        props: set!["null"]
-                    },
-                    Field {
-                        field_name: b"notes",
-                        layers: vec![
-                            Layer::new_noreset(b"string", nullable_dict! {}),
-                            Layer::new_noreset(
+            Model::new(
+                b"mymodel",
+                vec![
+                    Field::new(
+                        b"username",
+                        vec![Layer::new(b"string", null_dict! {})],
+                        false,
+                        true
+                    ),
+                    Field::new(
+                        b"password",
+                        vec![Layer::new(b"binary", null_dict! {})],
+                        false,
+                        false
+                    ),
+                    Field::new(
+                        b"profile_pic",
+                        vec![Layer::new(b"binary", null_dict! {})],
+                        true,
+                        false
+                    ),
+                    Field::new(
+                        b"notes",
+                        vec![
+                            Layer::new(b"string", null_dict! {}),
+                            Layer::new(
                                 b"list",
-                                nullable_dict! {
+                                null_dict! {
                                     "unique" => Lit::Bool(true)
                                 }
                             )
                         ],
-                        props: set!["null"]
-                    }
+                        true,
+                        false
+                    )
                 ],
-                props: nullable_dict! {}
-            }
+                null_dict! {}
+            )
         )
     }
 
@@ -685,9 +577,9 @@ mod schemas {
                     password: binary,
                     null profile_pic: binary,
                     null notes: list {
-                        type string,
+                        type: string,
                         unique: true,
-                    },
+                    }
                 ) with {
                     env: {
                         free_user_limit: 100,
@@ -700,68 +592,73 @@ mod schemas {
         let tok = &tok[2..];
 
         // parse model
-        let (model, c) = schema::parse_schema_from_tokens_full(tok).unwrap();
-        assert_eq!(c, tok.len());
+        let model = parse_ast_node_full::<Model>(tok).unwrap();
+
         assert_eq!(
             model,
-            Model {
-                model_name: b"mymodel",
-                fields: vec![
-                    Field {
-                        field_name: b"username",
-                        layers: vec![Layer::new_noreset(b"string", nullable_dict! {})],
-                        props: set!["primary"]
-                    },
-                    Field {
-                        field_name: b"password",
-                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
-                        props: set![]
-                    },
-                    Field {
-                        field_name: b"profile_pic",
-                        layers: vec![Layer::new_noreset(b"binary", nullable_dict! {})],
-                        props: set!["null"]
-                    },
-                    Field {
-                        field_name: b"notes",
-                        layers: vec![
-                            Layer::new_noreset(b"string", nullable_dict! {}),
-                            Layer::new_noreset(
+            Model::new(
+                b"mymodel",
+                vec![
+                    Field::new(
+                        b"username",
+                        vec![Layer::new(b"string", null_dict! {})],
+                        false,
+                        true
+                    ),
+                    Field::new(
+                        b"password",
+                        vec![Layer::new(b"binary", null_dict! {})],
+                        false,
+                        false
+                    ),
+                    Field::new(
+                        b"profile_pic",
+                        vec![Layer::new(b"binary", null_dict! {})],
+                        true,
+                        false
+                    ),
+                    Field::new(
+                        b"notes",
+                        vec![
+                            Layer::new(b"string", null_dict! {}),
+                            Layer::new(
                                 b"list",
-                                nullable_dict! {
+                                null_dict! {
                                     "unique" => Lit::Bool(true)
                                 }
                             )
                         ],
-                        props: set!["null"]
-                    }
+                        true,
+                        false
+                    )
                 ],
-                props: nullable_dict! {
-                    "env" => nullable_dict! {
+                null_dict! {
+                    "env" => null_dict! {
                         "free_user_limit" => Lit::UnsignedInt(100),
                     },
                     "storage_driver" => Lit::Str("skyheap".into()),
                 }
-            }
+            )
         )
     }
 }
 mod dict_field_syntax {
     use super::*;
-    use crate::engine::ql::schema::{ExpandedField, Layer};
+    use crate::engine::ql::{
+        ast::parse_ast_node_full,
+        ddl::syn::{ExpandedField, Layer},
+    };
     #[test]
     fn field_syn_mini() {
-        let tok = lex_insecure(b"username { type string }").unwrap();
-        let (ef, i) = schema::parse_field_syntax_full::<true>(&tok).unwrap();
-        assert_eq!(i, tok.len());
+        let tok = lex_insecure(b"username { type: string }").unwrap();
+        let ef = parse_ast_node_full::<ExpandedField>(&tok).unwrap();
         assert_eq!(
             ef,
-            ExpandedField {
-                field_name: b"username",
-                layers: vec![Layer::new_noreset(b"string", nullable_dict! {})],
-                props: nullable_dict! {},
-                reset: false
-            }
+            ExpandedField::new(
+                b"username",
+                vec![Layer::new(b"string", null_dict! {})],
+                null_dict! {}
+            )
         )
     }
     #[test]
@@ -770,23 +667,21 @@ mod dict_field_syntax {
             b"
                 username {
                     nullable: false,
-                    type string,
+                    type: string,
                 }
             ",
         )
         .unwrap();
-        let (ef, i) = schema::parse_field_syntax_full::<true>(&tok).unwrap();
-        assert_eq!(i, tok.len());
+        let ef = parse_ast_node_full::<ExpandedField>(&tok).unwrap();
         assert_eq!(
             ef,
-            ExpandedField {
-                field_name: b"username",
-                props: nullable_dict! {
+            ExpandedField::new(
+                b"username",
+                vec![Layer::new(b"string", null_dict! {})],
+                null_dict! {
                     "nullable" => Lit::Bool(false),
                 },
-                layers: vec![Layer::new_noreset(b"string", nullable_dict! {})],
-                reset: false
-            }
+            )
         );
     }
     #[test]
@@ -795,7 +690,7 @@ mod dict_field_syntax {
             b"
                 username {
                     nullable: false,
-                    type string {
+                    type: string {
                         minlen: 6,
                         maxlen: 255,
                     },
@@ -804,25 +699,23 @@ mod dict_field_syntax {
             ",
         )
         .unwrap();
-        let (ef, i) = schema::parse_field_syntax_full::<true>(&tok).unwrap();
-        assert_eq!(i, tok.len());
+        let ef = parse_ast_node_full::<ExpandedField>(&tok).unwrap();
         assert_eq!(
             ef,
-            ExpandedField {
-                field_name: b"username",
-                props: nullable_dict! {
-                    "nullable" => Lit::Bool(false),
-                    "jingle_bells" => Lit::Str("snow".into()),
-                },
-                layers: vec![Layer::new_noreset(
+            ExpandedField::new(
+                b"username",
+                vec![Layer::new(
                     b"string",
-                    nullable_dict! {
+                    null_dict! {
                         "minlen" => Lit::UnsignedInt(6),
                         "maxlen" => Lit::UnsignedInt(255),
                     }
                 )],
-                reset: false
-            }
+                null_dict! {
+                    "nullable" => Lit::Bool(false),
+                    "jingle_bells" => Lit::Str("snow".into()),
+                },
+            )
         );
     }
     #[test]
@@ -831,8 +724,8 @@ mod dict_field_syntax {
             b"
                 notes {
                     nullable: true,
-                    type list {
-                        type string {
+                    type: list {
+                        type: string {
                             ascii_only: true,
                         },
                         unique: true,
@@ -842,141 +735,165 @@ mod dict_field_syntax {
             ",
         )
         .unwrap();
-        let (ef, i) = schema::parse_field_syntax_full::<true>(&tok).unwrap();
-        assert_eq!(i, tok.len());
+        let ef = parse_ast_node_full::<ExpandedField>(&tok).unwrap();
         assert_eq!(
             ef,
-            ExpandedField {
-                field_name: b"notes",
-                props: nullable_dict! {
-                    "nullable" => Lit::Bool(true),
-                    "jingle_bells" => Lit::Str("snow".into()),
-                },
-                layers: vec![
-                    Layer::new_noreset(
+            ExpandedField::new(
+                b"notes",
+                vec![
+                    Layer::new(
                         b"string",
-                        nullable_dict! {
+                        null_dict! {
                             "ascii_only" => Lit::Bool(true),
                         }
                     ),
-                    Layer::new_noreset(
+                    Layer::new(
                         b"list",
-                        nullable_dict! {
+                        null_dict! {
                             "unique" => Lit::Bool(true),
                         }
                     )
                 ],
-                reset: false
-            }
+                null_dict! {
+                    "nullable" => Lit::Bool(true),
+                    "jingle_bells" => Lit::Str("snow".into()),
+                },
+            )
         );
     }
 }
 mod alter_model_remove {
     use super::*;
+    use crate::engine::ql::{
+        ast::parse_ast_node_full,
+        ddl::alt::{AlterKind, AlterModel},
+    };
     #[test]
     fn alter_mini() {
         let tok = lex_insecure(b"alter model mymodel remove myfield").unwrap();
-        let mut i = 4;
-        let remove = schema::alter_remove_full(&tok[i..], &mut i).unwrap();
-        assert_eq!(i, tok.len());
-        assert_eq!(remove, [b"myfield".as_slice()].into());
+        let remove = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+        assert_eq!(
+            remove,
+            AlterModel::new(
+                b"mymodel",
+                AlterKind::Remove(Box::from([b"myfield".as_slice()]))
+            )
+        );
     }
     #[test]
     fn alter_mini_2() {
         let tok = lex_insecure(b"alter model mymodel remove (myfield)").unwrap();
-        let mut i = 4;
-        let remove = schema::alter_remove_full(&tok[i..], &mut i).unwrap();
-        assert_eq!(i, tok.len());
-        assert_eq!(remove, [b"myfield".as_slice()].into());
+        let remove = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+        assert_eq!(
+            remove,
+            AlterModel::new(
+                b"mymodel",
+                AlterKind::Remove(Box::from([b"myfield".as_slice()]))
+            )
+        );
     }
     #[test]
     fn alter() {
         let tok =
             lex_insecure(b"alter model mymodel remove (myfield1, myfield2, myfield3, myfield4)")
                 .unwrap();
-        let mut i = 4;
-        let remove = schema::alter_remove_full(&tok[i..], &mut i).unwrap();
-        assert_eq!(i, tok.len());
+        let remove = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
         assert_eq!(
             remove,
-            [
-                b"myfield1".as_slice(),
-                b"myfield2".as_slice(),
-                b"myfield3".as_slice(),
-                b"myfield4".as_slice(),
-            ]
-            .into()
+            AlterModel::new(
+                b"mymodel",
+                AlterKind::Remove(Box::from([
+                    b"myfield1".as_slice(),
+                    b"myfield2".as_slice(),
+                    b"myfield3".as_slice(),
+                    b"myfield4".as_slice(),
+                ]))
+            )
         );
     }
 }
 mod alter_model_add {
     use super::*;
-    use crate::engine::ql::schema::{ExpandedField, Layer};
+    use crate::engine::ql::{
+        ast::parse_ast_node_full,
+        ddl::{
+            alt::{AlterKind, AlterModel},
+            syn::{ExpandedField, Layer},
+        },
+    };
     #[test]
     fn add_mini() {
         let tok = lex_insecure(
             b"
-                alter model mymodel add myfield { type string }
+                alter model mymodel add myfield { type: string }
             ",
         )
         .unwrap();
-        let mut i = 4;
-        let r = schema::alter_add_full(&tok[i..], &mut i).unwrap();
-        assert_eq!(i, tok.len());
         assert_eq!(
-            r.as_ref(),
-            [ExpandedField {
-                field_name: b"myfield",
-                props: nullable_dict! {},
-                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
-                reset: false
-            }]
+            parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap(),
+            AlterModel::new(
+                b"mymodel",
+                AlterKind::Add(
+                    [ExpandedField::new(
+                        b"myfield",
+                        [Layer::new(b"string", null_dict! {})].into(),
+                        null_dict! {},
+                    )]
+                    .into()
+                )
+            )
         );
     }
     #[test]
     fn add() {
         let tok = lex_insecure(
             b"
-                alter model mymodel add myfield { type string, nullable: true }
+                alter model mymodel add myfield { type: string, nullable: true }
             ",
         )
         .unwrap();
-        let mut i = 4;
-        let r = schema::alter_add_full(&tok[i..], &mut i).unwrap();
-        assert_eq!(i, tok.len());
+        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
         assert_eq!(
-            r.as_ref(),
-            [ExpandedField {
-                field_name: b"myfield",
-                props: nullable_dict! {
-                    "nullable" => Lit::Bool(true)
-                },
-                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
-                reset: false
-            }]
+            r,
+            AlterModel::new(
+                b"mymodel",
+                AlterKind::Add(
+                    [ExpandedField::new(
+                        b"myfield",
+                        [Layer::new(b"string", null_dict! {})].into(),
+                        null_dict! {
+                            "nullable" => Lit::Bool(true)
+                        },
+                    )]
+                    .into()
+                )
+            )
         );
     }
     #[test]
     fn add_pro() {
         let tok = lex_insecure(
             b"
-                alter model mymodel add (myfield { type string, nullable: true })
+                alter model mymodel add (myfield { type: string, nullable: true })
             ",
         )
         .unwrap();
-        let mut i = 4;
-        let r = schema::alter_add_full(&tok[i..], &mut i).unwrap();
-        assert_eq!(i, tok.len());
+        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
         assert_eq!(
-            r.as_ref(),
-            [ExpandedField {
-                field_name: b"myfield",
-                props: nullable_dict! {
-                    "nullable" => Lit::Bool(true)
-                },
-                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
-                reset: false
-            }]
+            r,
+            AlterModel::new(
+                b"mymodel",
+                AlterKind::Add(
+                    [ExpandedField::new(
+                        b"myfield",
+                        [Layer::new(b"string", null_dict! {})].into(),
+                        null_dict! {
+                            "nullable" => Lit::Bool(true)
+                        },
+                    )]
+                    .into()
+                )
+            )
         );
     }
     #[test]
@@ -985,12 +902,12 @@ mod alter_model_add {
             b"
                 alter model mymodel add (
                     myfield {
-                        type string,
+                        type: string,
                         nullable: true
                     },
                     another {
-                        type list {
-                            type string {
+                        type: list {
+                            type: string {
                                 maxlen: 255
                             },
                             unique: true
@@ -1001,90 +918,104 @@ mod alter_model_add {
             ",
         )
         .unwrap();
-        let mut i = 4;
-        let r = schema::alter_add_full(&tok[i..], &mut i).unwrap();
-        assert_eq!(i, tok.len());
+        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
         assert_eq!(
-            r.as_ref(),
-            [
-                ExpandedField {
-                    field_name: b"myfield",
-                    props: nullable_dict! {
-                        "nullable" => Lit::Bool(true)
-                    },
-                    layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
-                    reset: false
-                },
-                ExpandedField {
-                    field_name: b"another",
-                    props: nullable_dict! {
-                        "nullable" => Lit::Bool(false)
-                    },
-                    layers: [
-                        Layer::new_noreset(
-                            b"string",
-                            nullable_dict! {
-                                "maxlen" => Lit::UnsignedInt(255)
-                            }
+            r,
+            AlterModel::new(
+                b"mymodel",
+                AlterKind::Add(
+                    [
+                        ExpandedField::new(
+                            b"myfield",
+                            [Layer::new(b"string", null_dict! {})].into(),
+                            null_dict! {
+                                "nullable" => Lit::Bool(true)
+                            },
                         ),
-                        Layer::new_noreset(
-                            b"list",
-                            nullable_dict! {
-                               "unique" => Lit::Bool(true)
+                        ExpandedField::new(
+                            b"another",
+                            [
+                                Layer::new(
+                                    b"string",
+                                    null_dict! {
+                                        "maxlen" => Lit::UnsignedInt(255)
+                                    }
+                                ),
+                                Layer::new(
+                                    b"list",
+                                    null_dict! {
+                                       "unique" => Lit::Bool(true)
+                                    },
+                                )
+                            ]
+                            .into(),
+                            null_dict! {
+                                "nullable" => Lit::Bool(false)
                             },
                         )
                     ]
-                    .into(),
-                    reset: false
-                }
-            ]
+                    .into()
+                )
+            )
         );
     }
 }
 mod alter_model_update {
     use super::*;
-    use crate::engine::ql::schema::{ExpandedField, Layer};
+    use crate::engine::ql::{
+        ast::parse_ast_node_full,
+        ddl::{
+            alt::{AlterKind, AlterModel},
+            syn::{ExpandedField, Layer},
+        },
+    };
 
     #[test]
     fn alter_mini() {
         let tok = lex_insecure(
             b"
-                alter model mymodel update myfield { type string, .. }
+                alter model mymodel update myfield { type: string }
             ",
         )
         .unwrap();
-        let mut i = 4;
-        let r = schema::alter_update_full(&tok[i..], &mut i).unwrap();
-        assert_eq!(i, tok.len());
+        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
         assert_eq!(
-            r.as_ref(),
-            [ExpandedField {
-                field_name: b"myfield",
-                props: nullable_dict! {},
-                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
-                reset: true
-            }]
+            r,
+            AlterModel::new(
+                b"mymodel",
+                AlterKind::Update(
+                    [ExpandedField::new(
+                        b"myfield",
+                        [Layer::new(b"string", null_dict! {})].into(),
+                        null_dict! {},
+                    )]
+                    .into()
+                )
+            )
         );
     }
     #[test]
     fn alter_mini_2() {
         let tok = lex_insecure(
             b"
-                alter model mymodel update (myfield { type string, .. })
+                alter model mymodel update (myfield { type: string })
             ",
         )
         .unwrap();
-        let mut i = 4;
-        let r = schema::alter_update_full(&tok[i..], &mut i).unwrap();
-        assert_eq!(i, tok.len());
+        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
         assert_eq!(
-            r.as_ref(),
-            [ExpandedField {
-                field_name: b"myfield",
-                props: nullable_dict! {},
-                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
-                reset: true
-            }]
+            r,
+            AlterModel::new(
+                b"mymodel",
+                AlterKind::Update(
+                    [ExpandedField::new(
+                        b"myfield",
+                        [Layer::new(b"string", null_dict! {})].into(),
+                        null_dict! {},
+                    )]
+                    .into()
+                )
+            )
         );
     }
     #[test]
@@ -1093,27 +1024,29 @@ mod alter_model_update {
             b"
                 alter model mymodel update (
                     myfield {
-                        type string,
+                        type: string,
                         nullable: true,
-                        ..
                     }
                 )
             ",
         )
         .unwrap();
-        let mut i = 4;
-        let r = schema::alter_update_full(&tok[i..], &mut i).unwrap();
-        assert_eq!(i, tok.len());
+        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
         assert_eq!(
-            r.as_ref(),
-            [ExpandedField {
-                field_name: b"myfield",
-                props: nullable_dict! {
-                    "nullable" => Lit::Bool(true)
-                },
-                layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
-                reset: true
-            }]
+            r,
+            AlterModel::new(
+                b"mymodel",
+                AlterKind::Update(
+                    [ExpandedField::new(
+                        b"myfield",
+                        [Layer::new(b"string", null_dict! {})].into(),
+                        null_dict! {
+                            "nullable" => Lit::Bool(true)
+                        },
+                    )]
+                    .into()
+                )
+            )
         );
     }
     #[test]
@@ -1122,39 +1055,39 @@ mod alter_model_update {
             b"
                 alter model mymodel update (
                     myfield {
-                        type string,
+                        type: string,
                         nullable: true,
-                        ..
                     },
                     myfield2 {
-                        type string,
-                        ..
+                        type: string,
                     }
                 )
             ",
         )
         .unwrap();
-        let mut i = 4;
-        let r = schema::alter_update_full(&tok[i..], &mut i).unwrap();
-        assert_eq!(i, tok.len());
+        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
         assert_eq!(
-            r.as_ref(),
-            [
-                ExpandedField {
-                    field_name: b"myfield",
-                    props: nullable_dict! {
-                        "nullable" => Lit::Bool(true)
-                    },
-                    layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
-                    reset: true
-                },
-                ExpandedField {
-                    field_name: b"myfield2",
-                    props: nullable_dict! {},
-                    layers: [Layer::new_noreset(b"string", nullable_dict! {})].into(),
-                    reset: true
-                }
-            ]
+            r,
+            AlterModel::new(
+                b"mymodel",
+                AlterKind::Update(
+                    [
+                        ExpandedField::new(
+                            b"myfield",
+                            [Layer::new(b"string", null_dict! {})].into(),
+                            null_dict! {
+                                "nullable" => Lit::Bool(true)
+                            },
+                        ),
+                        ExpandedField::new(
+                            b"myfield2",
+                            [Layer::new(b"string", null_dict! {})].into(),
+                            null_dict! {},
+                        )
+                    ]
+                    .into()
+                )
+            )
         );
     }
     #[test]
@@ -1163,46 +1096,45 @@ mod alter_model_update {
             b"
                 alter model mymodel update (
                     myfield {
-                        type string {..},
+                        type: string {},
                         nullable: true,
-                        ..
                     },
                     myfield2 {
-                        type string {
+                        type: string {
                             maxlen: 255,
-                            ..
                         },
-                        ..
                     }
                 )
             ",
         )
         .unwrap();
-        let mut i = 4;
-        let r = schema::alter_update_full(&tok[i..], &mut i).unwrap();
-        assert_eq!(i, tok.len());
+        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
         assert_eq!(
-            r.as_ref(),
-            [
-                ExpandedField {
-                    field_name: b"myfield",
-                    props: nullable_dict! {
-                        "nullable" => Lit::Bool(true)
-                    },
-                    layers: [Layer::new_reset(b"string", nullable_dict! {})].into(),
-                    reset: true
-                },
-                ExpandedField {
-                    field_name: b"myfield2",
-                    props: nullable_dict! {},
-                    layers: [Layer::new_reset(
-                        b"string",
-                        nullable_dict! {"maxlen" => Lit::UnsignedInt(255)}
-                    )]
-                    .into(),
-                    reset: true
-                }
-            ]
+            r,
+            AlterModel::new(
+                b"mymodel",
+                AlterKind::Update(
+                    [
+                        ExpandedField::new(
+                            b"myfield",
+                            [Layer::new(b"string", null_dict! {})].into(),
+                            null_dict! {
+                                "nullable" => Lit::Bool(true)
+                            },
+                        ),
+                        ExpandedField::new(
+                            b"myfield2",
+                            [Layer::new(
+                                b"string",
+                                null_dict! {"maxlen" => Lit::UnsignedInt(255)}
+                            )]
+                            .into(),
+                            null_dict! {},
+                        )
+                    ]
+                    .into()
+                )
+            )
         );
     }
 }
@@ -1211,15 +1143,17 @@ mod ddl_other_query_tests {
     use {
         super::*,
         crate::engine::ql::{
-            ast::{Entity, Statement},
-            ddl::drop::{self, DropModel, DropSpace},
+            ast::{parse_ast_node_full, Entity, Statement},
+            ddl::drop::{DropModel, DropSpace, DropStatementAST},
         },
     };
     #[test]
     fn drop_space() {
         let src = lex_insecure(br"drop space myspace").unwrap();
         assert_eq!(
-            drop::parse_drop_full(&src[1..]).unwrap(),
+            parse_ast_node_full::<DropStatementAST>(&src[1..])
+                .unwrap()
+                .0,
             Statement::DropSpace(DropSpace::new(b"myspace", false))
         );
     }
@@ -1227,7 +1161,9 @@ mod ddl_other_query_tests {
     fn drop_space_force() {
         let src = lex_insecure(br"drop space myspace force").unwrap();
         assert_eq!(
-            drop::parse_drop_full(&src[1..]).unwrap(),
+            parse_ast_node_full::<DropStatementAST>(&src[1..])
+                .unwrap()
+                .0,
             Statement::DropSpace(DropSpace::new(b"myspace", true))
         );
     }
@@ -1235,7 +1171,9 @@ mod ddl_other_query_tests {
     fn drop_model() {
         let src = lex_insecure(br"drop model mymodel").unwrap();
         assert_eq!(
-            drop::parse_drop_full(&src[1..]).unwrap(),
+            parse_ast_node_full::<DropStatementAST>(&src[1..])
+                .unwrap()
+                .0,
             Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), false))
         );
     }
@@ -1243,7 +1181,9 @@ mod ddl_other_query_tests {
     fn drop_model_force() {
         let src = lex_insecure(br"drop model mymodel force").unwrap();
         assert_eq!(
-            drop::parse_drop_full(&src[1..]).unwrap(),
+            parse_ast_node_full::<DropStatementAST>(&src[1..])
+                .unwrap()
+                .0,
             Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), true))
         );
     }
diff --git a/server/src/engine/ql/tests/structure_syn.rs b/server/src/engine/ql/tests/structure_syn.rs
index e8065ad9..fc8542da 100644
--- a/server/src/engine/ql/tests/structure_syn.rs
+++ b/server/src/engine/ql/tests/structure_syn.rs
@@ -26,8 +26,26 @@
 
 use {
     super::*,
-    crate::engine::ql::{lex::Lit, schema},
+    crate::engine::ql::{
+        ast::parse_ast_node_full,
+        ddl::syn::{Dict, DictBasic},
+        lex::Lit,
+    },
 };
+
+macro_rules! fold_dict {
+    ($($dict:expr),+ $(,)?) => {
+        ($(
+            fold_dict($dict).unwrap()
+        ),+)
+    }
+}
+
+fn fold_dict(raw: &[u8]) -> Option<Dict> {
+    let lexed = lex_insecure(raw).unwrap();
+    parse_ast_node_full::<DictBasic>(&lexed).map(|v| v.0).ok()
+}
+
 mod dict {
     use super::*;
 
@@ -37,7 +55,7 @@ mod dict {
             br#"{name: "sayan"}"#,
             br#"{name: "sayan",}"#,
         };
-        let r = nullable_dict!("name" => Lit::Str("sayan".into()));
+        let r = null_dict!("name" => Lit::Str("sayan".into()));
         multi_assert_eq!(d1, d2 => r);
     }
     #[test]
@@ -58,7 +76,7 @@ mod dict {
                 }
             "#,
         };
-        let r = nullable_dict! (
+        let r = null_dict! (
             "name" => Lit::Str("sayan".into()),
             "verified" => Lit::Bool(true),
             "burgers" => Lit::UnsignedInt(152),
@@ -99,9 +117,9 @@ mod dict {
             }"#
         };
         multi_assert_eq!(
-            d1, d2, d3 => nullable_dict! {
+            d1, d2, d3 => null_dict! {
                 "name" => Lit::Str("sayan".into()),
-                "notes" => nullable_dict! {
+                "notes" => null_dict! {
                     "burgers" => Lit::Str("all the time, extra mayo".into()),
                     "taco" => Lit::Bool(true),
                     "pretzels" => Lit::UnsignedInt(1),
@@ -151,14 +169,14 @@ mod dict {
                         },
                     },
                 }
-            }"#
+            "#
         };
         multi_assert_eq!(
-            d1, d2, d3 => nullable_dict! {
-                "well" => nullable_dict! {
-                    "now" => nullable_dict! {
-                        "this" => nullable_dict! {
-                            "is" => nullable_dict! {
+            d1, d2, d3 => null_dict! {
+                "well" => null_dict! {
+                    "now" => null_dict! {
+                        "this" => null_dict! {
+                            "is" => null_dict! {
                                 "ridiculous" => Lit::Bool(true),
                             }
                         }
@@ -186,31 +204,31 @@ mod dict {
                 }\x01
             }
         ";
-        let ret_dict = nullable_dict! {
+        let ret_dict = null_dict! {
             "the_tradition_is" => Lit::Str("hello, world".into()),
-            "could_have_been" => nullable_dict! {
+            "could_have_been" => null_dict! {
                 "this" => Lit::Bool(true),
                 "or_maybe_this" => Lit::UnsignedInt(100),
                 "even_this" => Lit::Str("hello, universe!".into()),
             },
             "but_oh_well" => Lit::Str("it continues to be the 'annoying' phrase".into()),
-            "lorem" => nullable_dict! {
-                "ipsum" => nullable_dict! {
+            "lorem" => null_dict! {
+                "ipsum" => null_dict! {
                     "dolor" => Lit::Str("sit amet".into())
                 }
             }
         };
         fuzz_tokens(&tok[..], |should_pass, new_src| {
-            let r = schema::fold_dict(&new_src);
-            let okay = r.is_some();
+            let r = parse_ast_node_full::<DictBasic>(new_src);
+            let okay = r.is_ok();
             if should_pass {
-                assert_eq!(r.unwrap(), ret_dict)
+                assert_eq!(r.unwrap().0, ret_dict)
             }
             okay
         });
     }
 }
-mod nullable_dict_tests {
+mod null_dict_tests {
     use super::*;
     mod dict {
         use {super::*, crate::engine::ql::lex::Lit};
@@ -220,7 +238,7 @@ mod nullable_dict_tests {
             let d = fold_dict!(br"{ x: null }");
             assert_eq!(
                 d,
-                nullable_dict! {
+                null_dict! {
                     "x" => Null,
                 }
             );
@@ -237,7 +255,7 @@ mod nullable_dict_tests {
             };
             assert_eq!(
                 d,
-                nullable_dict! {
+                null_dict! {
                     "this_is_non_null" => Lit::Str("hello".into()),
                     "but_this_is_null" => Null,
                 }
@@ -258,10 +276,10 @@ mod nullable_dict_tests {
             };
             assert_eq!(
                 d,
-                nullable_dict! {
+                null_dict! {
                     "a_string" => Lit::Str("this is a string".into()),
                     "num" => Lit::UnsignedInt(1234),
-                    "a_dict" => nullable_dict! {
+                    "a_dict" => null_dict! {
                         "a_null" => Null,
                     }
                 }
@@ -283,10 +301,10 @@ mod nullable_dict_tests {
             };
             assert_eq!(
                 d,
-                nullable_dict! {
+                null_dict! {
                     "a_string" => Lit::Str("this is a string".into()),
                     "num" => Lit::UnsignedInt(1234),
-                    "a_dict" => nullable_dict! {
+                    "a_dict" => null_dict! {
                         "a_null" => Null,
                     },
                     "another_null" => Null,
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index c5f899f9..23757475 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use std::mem;
+
 #[macro_use]
 mod macros;
 pub mod compiler;
@@ -228,6 +230,8 @@ unsafe impl<'a, T: Sync> Sync for Life<'a, T> {}
 pub struct MaybeInit<T> {
     #[cfg(test)]
     is_init: bool,
+    #[cfg(not(test))]
+    is_init: (),
     base: MaybeUninit<T>,
 }
 
@@ -238,6 +242,8 @@ impl<T> MaybeInit<T> {
         Self {
             #[cfg(test)]
             is_init: false,
+            #[cfg(not(test))]
+            is_init: (),
             base: MaybeUninit::uninit(),
         }
     }
@@ -247,9 +253,20 @@ impl<T> MaybeInit<T> {
         Self {
             #[cfg(test)]
             is_init: true,
+            #[cfg(not(test))]
+            is_init: (),
             base: MaybeUninit::new(val),
         }
     }
+    const fn ensure_init(#[cfg(test)] is_init: bool, #[cfg(not(test))] is_init: ()) {
+        #[cfg(test)]
+        {
+            if !is_init {
+                panic!("Tried to `assume_init` on uninitialized data");
+            }
+        }
+        let _ = is_init;
+    }
     /// Assume that `self` is initialized and return the inner value
     ///
     /// ## Safety
@@ -257,12 +274,7 @@ impl<T> MaybeInit<T> {
     /// Caller needs to ensure that the data is actually initialized
     #[inline(always)]
     pub const unsafe fn assume_init(self) -> T {
-        #[cfg(test)]
-        {
-            if !self.is_init {
-                panic!("Tried to `assume_init` on uninitialized data");
-            }
-        }
+        Self::ensure_init(self.is_init);
         self.base.assume_init()
     }
     /// Assume that `self` is initialized and return a reference
@@ -272,13 +284,22 @@ impl<T> MaybeInit<T> {
     /// Caller needs to ensure that the data is actually initialized
     #[inline(always)]
     pub const unsafe fn assume_init_ref(&self) -> &T {
+        Self::ensure_init(self.is_init);
+        self.base.assume_init_ref()
+    }
+    /// Assumes `self` is initialized, replaces `self` with an uninit state, returning
+    /// the older value
+    ///
+    /// ## Safety
+    pub unsafe fn take(&mut self) -> T {
+        Self::ensure_init(self.is_init);
+        let mut r = MaybeUninit::uninit();
+        mem::swap(&mut r, &mut self.base);
         #[cfg(test)]
         {
-            if !self.is_init {
-                panic!("Tried to `assume_init_ref` on uninitialized data");
-            }
+            self.is_init = false;
         }
-        self.base.assume_init_ref()
+        r.assume_init()
     }
 }
 

From 798871bdd6976f8853c4e9e0ecd925c17a25332c Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 3 Feb 2023 01:38:33 -0800
Subject: [PATCH 117/310] Simplify wrapper based access

---
 server/src/engine/ql/ast/mod.rs             |  2 +-
 server/src/engine/ql/ddl/alt.rs             |  3 +-
 server/src/engine/ql/ddl/drop.rs            |  3 +-
 server/src/engine/ql/ddl/ins.rs             |  3 +-
 server/src/engine/ql/ddl/syn.rs             | 43 +++--------------
 server/src/engine/ql/dml/ins.rs             |  9 ++--
 server/src/engine/ql/tests/dml_tests.rs     | 36 +++++++++------
 server/src/engine/ql/tests/schema_tests.rs  | 48 +++++++------------
 server/src/engine/ql/tests/structure_syn.rs |  6 ++-
 sky-macros/src/lib.rs                       | 51 ++++++++++++++++++++-
 10 files changed, 109 insertions(+), 95 deletions(-)

diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 5976cfcc..b8b45902 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -516,7 +516,7 @@ impl<'a> Entity<'a> {
     }
 }
 
-#[cfg_attr(test, derive(Debug, PartialEq))]
+#[derive(Debug, PartialEq)]
 /// A [`Statement`] is a fully BlueQL statement that can be executed by the query engine
 // TODO(@ohsayan): Determine whether we actually need this
 pub enum Statement<'a> {
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 2c2e838f..0aa6e68d 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -78,8 +78,7 @@ impl<'a> AlterSpace<'a> {
     }
 }
 
-#[derive(Debug)]
-#[cfg_attr(test, derive(PartialEq))]
+#[derive(Debug, PartialEq)]
 pub struct AlterModel<'a> {
     model: Slice<'a>,
     kind: AlterKind<'a>,
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index c50bfc9f..7b57e499 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -116,7 +116,8 @@ mod impls {
             Self::parse(state)
         }
     }
-    pub struct DropStatementAST<'a>(pub Statement<'a>);
+    #[derive(sky_macros::Wrapper, Debug)]
+    pub struct DropStatementAST<'a>(Statement<'a>);
     impl<'a> ASTNode<'a> for DropStatementAST<'a> {
         fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             super::parse_drop(state).map(Self)
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
index 68ea9314..3eae00e1 100644
--- a/server/src/engine/ql/ddl/ins.rs
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -69,7 +69,8 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
 pub use impls::InspectStatementAST;
 mod impls {
     use crate::engine::ql::ast::{traits::ASTNode, QueryData, State, Statement};
-    pub struct InspectStatementAST<'a>(pub Statement<'a>);
+    #[derive(sky_macros::Wrapper, Debug)]
+    pub struct InspectStatementAST<'a>(Statement<'a>);
     impl<'a> ASTNode<'a> for InspectStatementAST<'a> {
         fn from_state<Qd: QueryData<'a>>(
             state: &mut State<'a, Qd>,
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index e4cd4346..1f056be0 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -507,7 +507,6 @@ mod impls {
         ast::{traits::ASTNode, QueryData, State},
         LangError, LangResult,
     };
-    use std::ops::{Deref, DerefMut};
     impl<'a> ASTNode<'a> for ExpandedField<'a> {
         fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             Self::parse(state)
@@ -537,7 +536,8 @@ mod impls {
             }
         }
     }
-    pub struct DictBasic(pub Dict);
+    #[derive(sky_macros::Wrapper, Debug)]
+    pub struct DictBasic(Dict);
     impl<'a> ASTNode<'a> for DictBasic {
         fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             let mut dict = Dict::new();
@@ -549,18 +549,8 @@ mod impls {
             }
         }
     }
-    impl Deref for DictBasic {
-        type Target = Dict;
-        fn deref(&self) -> &Self::Target {
-            &self.0
-        }
-    }
-    impl DerefMut for DictBasic {
-        fn deref_mut(&mut self) -> &mut Self::Target {
-            &mut self.0
-        }
-    }
-    pub struct DictTypeMetaSplit(pub Dict);
+    #[derive(sky_macros::Wrapper, Debug)]
+    pub struct DictTypeMetaSplit(Dict);
     impl<'a> ASTNode<'a> for DictTypeMetaSplit {
         fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             let mut dict = Dict::new();
@@ -572,18 +562,8 @@ mod impls {
             }
         }
     }
-    impl Deref for DictTypeMetaSplit {
-        type Target = Dict;
-        fn deref(&self) -> &Self::Target {
-            &self.0
-        }
-    }
-    impl DerefMut for DictTypeMetaSplit {
-        fn deref_mut(&mut self) -> &mut Self::Target {
-            &mut self.0
-        }
-    }
-    pub struct DictTypeMeta(pub Dict);
+    #[derive(sky_macros::Wrapper, Debug)]
+    pub struct DictTypeMeta(Dict);
     impl<'a> ASTNode<'a> for DictTypeMeta {
         fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             let mut dict = Dict::new();
@@ -595,17 +575,6 @@ mod impls {
             }
         }
     }
-    impl Deref for DictTypeMeta {
-        type Target = Dict;
-        fn deref(&self) -> &Self::Target {
-            &self.0
-        }
-    }
-    impl DerefMut for DictTypeMeta {
-        fn deref_mut(&mut self) -> &mut Self::Target {
-            &mut self.0
-        }
-    }
     impl<'a> ASTNode<'a> for Field<'a> {
         fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             Self::parse(state)
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index 1ddaf42b..d911f348 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -208,7 +208,8 @@ unsafe fn handle_func_sub<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> O
 }
 
 #[cfg(test)]
-pub struct List(pub Vec<DataType>);
+#[derive(sky_macros::Wrapper, Debug)]
+pub struct List(Vec<DataType>);
 #[cfg(test)]
 impl<'a> ASTNode<'a> for List {
     fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
@@ -270,7 +271,8 @@ pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
 }
 
 #[cfg(test)]
-pub struct DataTuple(pub Vec<Option<DataType>>);
+#[derive(sky_macros::Wrapper, Debug)]
+pub struct DataTuple(Vec<Option<DataType>>);
 #[cfg(test)]
 impl<'a> ASTNode<'a> for DataTuple {
     fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
@@ -340,7 +342,8 @@ pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
 }
 
 #[cfg(test)]
-pub struct DataMap(pub HashMap<Box<str>, Option<DataType>>);
+#[derive(sky_macros::Wrapper, Debug)]
+pub struct DataMap(HashMap<Box<str>, Option<DataType>>);
 #[cfg(test)]
 impl<'a> ASTNode<'a> for DataMap {
     fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 680c836b..d3ada309 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -41,7 +41,7 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_ast_node_full::<List>(&tok[1..]).unwrap().0;
+        let r = parse_ast_node_full::<List>(&tok[1..]).unwrap();
         assert_eq!(r, vec![])
     }
     #[test]
@@ -52,7 +52,7 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_ast_node_full::<List>(&tok[1..]).unwrap().0;
+        let r = parse_ast_node_full::<List>(&tok[1..]).unwrap();
         assert_eq!(r.as_slice(), into_array![1, 2, 3, 4])
     }
     #[test]
@@ -71,7 +71,9 @@ mod list_parse {
         ];
         let mut state = State::new(&tok[1..], SubstitutedData::new(&data));
         assert_eq!(
-            <List as ASTNode>::from_state(&mut state).unwrap().0,
+            <List as ASTNode>::from_state(&mut state)
+                .unwrap()
+                .into_inner(),
             into_array![1, 2, 3, 4]
         )
     }
@@ -88,7 +90,7 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_ast_node_full::<List>(&tok[1..]).unwrap().0;
+        let r = parse_ast_node_full::<List>(&tok[1..]).unwrap();
         assert_eq!(
             r.as_slice(),
             into_array![
@@ -122,7 +124,9 @@ mod list_parse {
         ];
         let mut state = State::new(&tok[1..], SubstitutedData::new(&data));
         assert_eq!(
-            <List as ASTNode>::from_state(&mut state).unwrap().0,
+            <List as ASTNode>::from_state(&mut state)
+                .unwrap()
+                .into_inner(),
             into_array![
                 into_array![1, 2],
                 into_array![3, 4],
@@ -144,7 +148,7 @@ mod list_parse {
             ",
         )
         .unwrap();
-        let r = parse_ast_node_full::<List>(&tok[1..]).unwrap().0;
+        let r = parse_ast_node_full::<List>(&tok[1..]).unwrap();
         assert_eq!(
             r.as_slice(),
             into_array![
@@ -184,7 +188,9 @@ mod list_parse {
         ];
         let mut state = State::new(&tok[1..], SubstitutedData::new(&data));
         assert_eq!(
-            <List as ASTNode>::from_state(&mut state).unwrap().0,
+            <List as ASTNode>::from_state(&mut state)
+                .unwrap()
+                .into_inner(),
             into_array![
                 into_array![into_array![1, 1], into_array![2, 2]],
                 into_array![into_array![], into_array![4, 4]],
@@ -202,7 +208,7 @@ mod tuple_syntax {
     #[test]
     fn tuple_mini() {
         let tok = lex_insecure(b"()").unwrap();
-        let r = parse_ast_node_full::<DataTuple>(&tok[1..]).unwrap().0;
+        let r = parse_ast_node_full::<DataTuple>(&tok[1..]).unwrap();
         assert_eq!(r, vec![]);
     }
 
@@ -214,7 +220,7 @@ mod tuple_syntax {
             "#,
         )
         .unwrap();
-        let r = parse_ast_node_full::<DataTuple>(&tok[1..]).unwrap().0;
+        let r = parse_ast_node_full::<DataTuple>(&tok[1..]).unwrap();
         assert_eq!(
             r.as_slice(),
             into_array_nullable![1234, "email@example.com", true]
@@ -234,7 +240,7 @@ mod tuple_syntax {
             "#,
         )
         .unwrap();
-        let r = parse_ast_node_full::<DataTuple>(&tok[1..]).unwrap().0;
+        let r = parse_ast_node_full::<DataTuple>(&tok[1..]).unwrap();
         assert_eq!(
             r.as_slice(),
             into_array_nullable![
@@ -266,7 +272,7 @@ mod tuple_syntax {
             "#,
         )
         .unwrap();
-        let r = parse_ast_node_full::<DataTuple>(&tok[1..]).unwrap().0;
+        let r = parse_ast_node_full::<DataTuple>(&tok[1..]).unwrap();
         assert_eq!(
             r.as_slice(),
             into_array_nullable![
@@ -292,7 +298,7 @@ mod map_syntax {
     #[test]
     fn map_mini() {
         let tok = lex_insecure(b"{}").unwrap();
-        let r = parse_ast_node_full::<DataMap>(&tok[1..]).unwrap().0;
+        let r = parse_ast_node_full::<DataMap>(&tok[1..]).unwrap();
         assert_eq!(r, null_dict! {})
     }
 
@@ -309,7 +315,7 @@ mod map_syntax {
             "#,
         )
         .unwrap();
-        let r = parse_ast_node_full::<DataMap>(&tok[1..]).unwrap().0;
+        let r = parse_ast_node_full::<DataMap>(&tok[1..]).unwrap();
         assert_eq!(
             r,
             dict_nullable! {
@@ -335,7 +341,7 @@ mod map_syntax {
             "#,
         )
         .unwrap();
-        let r = parse_ast_node_full::<DataMap>(&tok[1..]).unwrap().0;
+        let r = parse_ast_node_full::<DataMap>(&tok[1..]).unwrap();
         assert_eq!(
             r,
             dict_nullable! {
@@ -364,7 +370,7 @@ mod map_syntax {
                 }
             "#)
         .unwrap();
-        let r = parse_ast_node_full::<DataMap>(&tok[1..]).unwrap().0;
+        let r = parse_ast_node_full::<DataMap>(&tok[1..]).unwrap();
         assert_eq!(
             r,
             dict_nullable! {
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 18710609..0ae72e8f 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -40,9 +40,7 @@ mod inspect {
     fn inspect_space() {
         let tok = lex_insecure(b"inspect space myspace").unwrap();
         assert_eq!(
-            parse_ast_node_full::<InspectStatementAST>(&tok[1..])
-                .unwrap()
-                .0,
+            parse_ast_node_full::<InspectStatementAST>(&tok[1..]).unwrap(),
             Statement::InspectSpace(b"myspace")
         );
     }
@@ -50,16 +48,12 @@ mod inspect {
     fn inspect_model() {
         let tok = lex_insecure(b"inspect model users").unwrap();
         assert_eq!(
-            parse_ast_node_full::<InspectStatementAST>(&tok[1..])
-                .unwrap()
-                .0,
+            parse_ast_node_full::<InspectStatementAST>(&tok[1..]).unwrap(),
             Statement::InspectModel(Entity::Single(b"users"))
         );
         let tok = lex_insecure(b"inspect model tweeter.users").unwrap();
         assert_eq!(
-            parse_ast_node_full::<InspectStatementAST>(&tok[1..])
-                .unwrap()
-                .0,
+            parse_ast_node_full::<InspectStatementAST>(&tok[1..]).unwrap(),
             Statement::InspectModel(Entity::Full(b"tweeter", b"users"))
         );
     }
@@ -67,9 +61,7 @@ mod inspect {
     fn inspect_spaces() {
         let tok = lex_insecure(b"inspect spaces").unwrap();
         assert_eq!(
-            parse_ast_node_full::<InspectStatementAST>(&tok[1..])
-                .unwrap()
-                .0,
+            parse_ast_node_full::<InspectStatementAST>(&tok[1..]).unwrap(),
             Statement::InspectSpaces
         );
     }
@@ -120,7 +112,7 @@ mod tymeta {
     fn tymeta_mini() {
         let tok = lex_insecure(b"{}").unwrap();
         let tymeta = parse_ast_node_full::<DictTypeMeta>(&tok).unwrap();
-        assert_eq!(tymeta.0, null_dict!());
+        assert_eq!(tymeta, null_dict!());
     }
     #[test]
     #[should_panic]
@@ -133,7 +125,7 @@ mod tymeta {
         let tok = lex_insecure(br#"{hello: "world", loading: true, size: 100 }"#).unwrap();
         let tymeta = parse_ast_node_full::<DictTypeMeta>(&tok).unwrap();
         assert_eq!(
-            tymeta.0,
+            tymeta,
             null_dict! {
                 "hello" => Lit::Str("world".into()),
                 "loading" => Lit::Bool(true),
@@ -154,10 +146,10 @@ mod tymeta {
         assert!(Token![,].eq(state.fw_read()));
         let tymeta2: DictTypeMetaSplit = ASTNode::from_state(&mut state).unwrap();
         assert!(state.exhausted());
-        let mut final_ret = tymeta;
-        final_ret.0.extend(tymeta2.0);
+        let mut final_ret = tymeta.into_inner();
+        final_ret.extend(tymeta2.into_inner());
         assert_eq!(
-            final_ret.0,
+            final_ret,
             null_dict! {
                 "maxlen" => Lit::UnsignedInt(100),
                 "unique" => Lit::Bool(true)
@@ -179,10 +171,10 @@ mod tymeta {
         assert!(Token![,].eq(state.fw_read()));
         let tymeta2: DictTypeMetaSplit = ASTNode::from_state(&mut state).unwrap();
         assert!(state.exhausted());
-        let mut final_ret = tymeta;
-        final_ret.0.extend(tymeta2.0);
+        let mut final_ret = tymeta.into_inner();
+        final_ret.extend(tymeta2.into_inner());
         assert_eq!(
-            final_ret.0,
+            final_ret,
             null_dict! {
                 "maxlen" => Lit::UnsignedInt(100),
                 "unique" => Lit::Bool(true),
@@ -1151,9 +1143,7 @@ mod ddl_other_query_tests {
     fn drop_space() {
         let src = lex_insecure(br"drop space myspace").unwrap();
         assert_eq!(
-            parse_ast_node_full::<DropStatementAST>(&src[1..])
-                .unwrap()
-                .0,
+            parse_ast_node_full::<DropStatementAST>(&src[1..]).unwrap(),
             Statement::DropSpace(DropSpace::new(b"myspace", false))
         );
     }
@@ -1161,9 +1151,7 @@ mod ddl_other_query_tests {
     fn drop_space_force() {
         let src = lex_insecure(br"drop space myspace force").unwrap();
         assert_eq!(
-            parse_ast_node_full::<DropStatementAST>(&src[1..])
-                .unwrap()
-                .0,
+            parse_ast_node_full::<DropStatementAST>(&src[1..]).unwrap(),
             Statement::DropSpace(DropSpace::new(b"myspace", true))
         );
     }
@@ -1171,9 +1159,7 @@ mod ddl_other_query_tests {
     fn drop_model() {
         let src = lex_insecure(br"drop model mymodel").unwrap();
         assert_eq!(
-            parse_ast_node_full::<DropStatementAST>(&src[1..])
-                .unwrap()
-                .0,
+            parse_ast_node_full::<DropStatementAST>(&src[1..]).unwrap(),
             Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), false))
         );
     }
@@ -1181,9 +1167,7 @@ mod ddl_other_query_tests {
     fn drop_model_force() {
         let src = lex_insecure(br"drop model mymodel force").unwrap();
         assert_eq!(
-            parse_ast_node_full::<DropStatementAST>(&src[1..])
-                .unwrap()
-                .0,
+            parse_ast_node_full::<DropStatementAST>(&src[1..]).unwrap(),
             Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), true))
         );
     }
diff --git a/server/src/engine/ql/tests/structure_syn.rs b/server/src/engine/ql/tests/structure_syn.rs
index fc8542da..86d0ad95 100644
--- a/server/src/engine/ql/tests/structure_syn.rs
+++ b/server/src/engine/ql/tests/structure_syn.rs
@@ -43,7 +43,9 @@ macro_rules! fold_dict {
 
 fn fold_dict(raw: &[u8]) -> Option<Dict> {
     let lexed = lex_insecure(raw).unwrap();
-    parse_ast_node_full::<DictBasic>(&lexed).map(|v| v.0).ok()
+    parse_ast_node_full::<DictBasic>(&lexed)
+        .map(|v| v.into_inner())
+        .ok()
 }
 
 mod dict {
@@ -222,7 +224,7 @@ mod dict {
             let r = parse_ast_node_full::<DictBasic>(new_src);
             let okay = r.is_ok();
             if should_pass {
-                assert_eq!(r.unwrap().0, ret_dict)
+                assert_eq!(r.unwrap(), ret_dict)
             }
             okay
         });
diff --git a/sky-macros/src/lib.rs b/sky-macros/src/lib.rs
index 13b0dedd..a51ace61 100644
--- a/sky-macros/src/lib.rs
+++ b/sky-macros/src/lib.rs
@@ -43,7 +43,12 @@
 //!     - `__MYENTITY__` - `String` with entity
 //!
 
-use {proc_macro::TokenStream, quote::quote, syn::Lit};
+use {
+    proc_macro::TokenStream,
+    proc_macro2::TokenStream as TokenStream2,
+    quote::quote,
+    syn::{Data, DataStruct, DeriveInput, Fields, Lit},
+};
 
 mod dbtest_fn;
 mod dbtest_mod;
@@ -177,3 +182,47 @@ pub fn compiled_eresp_bytes_v1(tokens: TokenStream) -> TokenStream {
     }
     .into()
 }
+
+#[proc_macro_derive(Wrapper)]
+/// Implements necessary traits for some type `T` to make it identify as a different type but mimic the functionality
+/// as the inner type it wraps around
+pub fn derive_wrapper(t: TokenStream) -> TokenStream {
+    let item = syn::parse_macro_input!(t as DeriveInput);
+    let r = wrapper(item);
+    r.into()
+}
+
+fn wrapper(item: DeriveInput) -> TokenStream2 {
+    let st_name = &item.ident;
+    let fields = match item.data {
+        Data::Struct(DataStruct {
+            fields: Fields::Unnamed(ref f),
+            ..
+        }) if f.unnamed.len() == 1 => f,
+        _ => panic!("only works on tuple structs with one field"),
+    };
+    let field = &fields.unnamed[0];
+    let ty = &field.ty;
+    let (impl_generics, ty_generics, where_clause) = item.generics.split_for_impl();
+    quote! {
+        #[automatically_derived]
+        impl #impl_generics #st_name #ty_generics #where_clause { pub fn into_inner(self) -> #ty { self.0 } }
+        #[automatically_derived]
+        impl #impl_generics ::core::ops::Deref for #st_name #ty_generics #where_clause {
+            type Target = #ty;
+            fn deref(&self) -> &Self::Target { &self.0 }
+        }
+        #[automatically_derived]
+        impl #impl_generics ::core::ops::DerefMut for #st_name #ty_generics #where_clause {
+            fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }
+        }
+        #[automatically_derived]
+        impl #impl_generics ::core::cmp::PartialEq<#ty> for #st_name #ty_generics #where_clause {
+            fn eq(&self, other: &#ty) -> bool { ::core::cmp::PartialEq::eq(&self.0, other) }
+        }
+        #[automatically_derived]
+        impl #impl_generics ::core::cmp::PartialEq<#st_name #ty_generics> for #ty #where_clause {
+            fn eq(&self, other: &#st_name #ty_generics) -> bool { ::core::cmp::PartialEq::eq(self, &other.0) }
+        }
+    }
+}

From 35143b796001df92bc7f9b1f82ba2f89419ef477 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 3 Feb 2023 04:30:31 -0800
Subject: [PATCH 118/310] Ensure we don't crash the CI runner

Too many input keys might consume far too much memory crashing the runner
---
 server/src/engine/idx/mtchm/tests.rs | 16 ++++++++--------
 server/src/util/mod.rs               |  2 ++
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/server/src/engine/idx/mtchm/tests.rs b/server/src/engine/idx/mtchm/tests.rs
index 01782df9..bc55f153 100644
--- a/server/src/engine/idx/mtchm/tests.rs
+++ b/server/src/engine/idx/mtchm/tests.rs
@@ -107,14 +107,14 @@ fn update_empty() {
     assert!(!idx.mt_update(10, 20, &cpin()));
 }
 
-#[cfg(not(miri))]
-const SPAM_QCOUNT: usize = 16_384;
-#[cfg(miri)]
-const SPAM_QCOUNT: usize = 32;
-#[cfg(not(miri))]
-const SPAM_TENANTS: usize = 32;
-#[cfg(miri)]
-const SPAM_TENANTS: usize = 2;
+const SPAM_QCOUNT: usize = if crate::util::IS_ON_CI {
+    1_024
+} else if cfg!(miri) {
+    32
+} else {
+    16_384
+};
+const SPAM_TENANTS: usize = if cfg!(miri) { 2 } else { 32 };
 
 #[derive(Clone, Debug)]
 struct ControlToken(Arc<RwLock<()>>);
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index 23757475..7895f971 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -47,6 +47,8 @@ use {
     std::process,
 };
 
+pub const IS_ON_CI: bool = option_env!("CI").is_some();
+
 const EXITCODE_ONE: i32 = 0x01;
 
 /// # Unsafe unwrapping

From e4836fd7b31327c3016ab5446b2815700c213128 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 3 Feb 2023 05:30:15 -0800
Subject: [PATCH 119/310] Skip saving artifacts [skip ci]

This wastes unnecessary space on our current plan
---
 .github/workflows/test-push.yml | 58 ---------------------------------
 1 file changed, 58 deletions(-)

diff --git a/.github/workflows/test-push.yml b/.github/workflows/test-push.yml
index 45e6ef51..7ddac1f9 100644
--- a/.github/workflows/test-push.yml
+++ b/.github/workflows/test-push.yml
@@ -72,29 +72,6 @@ jobs:
           RUST_BACKTRACE: 1
           TARGET: ${{ matrix.rust }}
 
-      - name: Archive artifacts
-        run: |
-          mkdir dist
-          cargo build --target ${{ matrix.rust }}
-          mv target/${{ matrix.rust }}/debug/skyd target/${{ matrix.rust }}/debug/skysh target/${{ matrix.rust }}/debug/sky-bench dist
-        if: runner.os != 'Windows'
-
-      - name: Archive artifacts
-        shell: cmd
-        run: |
-          cargo build --target ${{ matrix.rust }}
-          rm -rf dist
-          mkdir dist
-          move target\${{ matrix.rust }}\debug\*.exe dist
-        env:
-          RUSTFLAGS: -Ctarget-feature=+crt-static
-        if: runner.os == 'Windows'
-
-      - name: Save artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ github.sha }}-${{ matrix.rust }}-builds.zip
-          path: dist
   test_32bit:
     name: Test (32-bit)
     runs-on: ${{ matrix.os }}
@@ -141,29 +118,6 @@ jobs:
           RUST_BACKTRACE: 1
           TARGET: ${{ matrix.rust }}
 
-      - name: Archive artifacts
-        run: |
-          mkdir dist
-          cargo build --target ${{ matrix.rust }}
-          mv target/${{ matrix.rust }}/debug/skyd target/${{ matrix.rust }}/debug/skysh target/${{ matrix.rust }}/debug/sky-bench dist
-        if: runner.os == 'Linux'
-
-      - name: Archive artifacts
-        shell: cmd
-        run: |
-          cargo build --target ${{ matrix.rust }}
-          rm -rf dist
-          mkdir dist
-          move target\${{ matrix.rust }}\debug\*.exe dist
-        env:
-          RUSTFLAGS: -Ctarget-feature=+crt-static
-        if: runner.os == 'Windows'
-
-      - name: Save artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ github.sha }}-${{ matrix.rust }}-builds.zip
-          path: dist
   test_musl64:
     name: Test MUSL x86_64 (Tier 2)
     runs-on: ${{ matrix.os }}
@@ -207,15 +161,3 @@ jobs:
         env:
           RUST_BACKTRACE: 1
           TARGET: ${{ matrix.rust }}
-
-      - name: Archive artifacts
-        run: |
-          mkdir dist
-          cargo build --target ${{ matrix.rust }}
-          mv target/${{ matrix.rust }}/debug/skyd target/${{ matrix.rust }}/debug/skysh target/${{ matrix.rust }}/debug/sky-bench dist
-
-      - name: Save artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ github.sha }}-${{ matrix.rust }}-builds.zip
-          path: dist

From 25ef03221b4c6051b0d6d064144e196c49e2c7f0 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 3 Feb 2023 07:06:23 -0800
Subject: [PATCH 120/310] Make `ASTNode` sensitive to COW style parsing

---
 server/src/engine/ql/ast/mod.rs            |   4 +-
 server/src/engine/ql/ast/traits.rs         |  47 +++++++-
 server/src/engine/ql/ddl/alt.rs            |   4 +-
 server/src/engine/ql/ddl/crt.rs            |  20 ++--
 server/src/engine/ql/ddl/drop.rs           |   6 +-
 server/src/engine/ql/ddl/ins.rs            |   2 +-
 server/src/engine/ql/ddl/syn.rs            |  50 ++++-----
 server/src/engine/ql/dml/del.rs            |  15 ++-
 server/src/engine/ql/dml/ins.rs            | 122 +++++++++++----------
 server/src/engine/ql/dml/mod.rs            |  34 +++---
 server/src/engine/ql/dml/sel.rs            |  15 ++-
 server/src/engine/ql/dml/upd.rs            |  55 ++++++----
 server/src/engine/ql/macros.rs             |  15 ---
 server/src/engine/ql/tests/dml_tests.rs    |  16 ++-
 server/src/engine/ql/tests/schema_tests.rs |  18 +--
 15 files changed, 234 insertions(+), 189 deletions(-)

diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index b8b45902..89f91324 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -523,9 +523,9 @@ pub enum Statement<'a> {
     /// DDL query to switch between spaces and models
     Use(Entity<'a>),
     /// DDL query to create a model
-    CreateModel(ddl::crt::Model<'a>),
+    CreateModel(ddl::crt::CreateModel<'a>),
     /// DDL query to create a space
-    CreateSpace(ddl::crt::Space<'a>),
+    CreateSpace(ddl::crt::CreateSpace<'a>),
     /// DDL query to alter a space (properties)
     AlterSpace(ddl::alt::AlterSpace<'a>),
     /// DDL query to alter a model (properties, field types, etc)
diff --git a/server/src/engine/ql/ast/traits.rs b/server/src/engine/ql/ast/traits.rs
index 7d891ec2..d787f6aa 100644
--- a/server/src/engine/ql/ast/traits.rs
+++ b/server/src/engine/ql/ast/traits.rs
@@ -28,12 +28,49 @@
 use crate::engine::ql::{ast::InplaceData, lex::Token};
 use crate::engine::ql::{
     ast::{QueryData, State},
-    LangResult,
+    LangError, LangResult,
 };
 
+/// An AST node
 pub trait ASTNode<'a>: Sized {
-    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self>;
+    const VERIFY: bool = false;
+    /// Parse this AST node from the given state
+    ///
+    /// Note to implementors:
+    /// - If the implementor uses a cow style parse, then set [`ASTNode::VERIFY`] to
+    /// true
+    /// - Try to propagate errors via [`State`] if possible
+    fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self>;
+    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        let r = <Self as ASTNode>::_from_state(state);
+        if Self::VERIFY {
+            return if state.okay() {
+                r
+            } else {
+                Err(LangError::UnexpectedToken)
+            };
+        }
+        r
+    }
+    #[cfg(test)]
+    /// Parse multiple nodes of this AST node type. Intended for the test suite.
+    fn _multiple_from_state<Qd: QueryData<'a>>(_: &mut State<'a, Qd>) -> LangResult<Vec<Self>> {
+        unimplemented!()
+    }
     #[cfg(test)]
+    fn multiple_from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Vec<Self>> {
+        let r = <Self as ASTNode>::_multiple_from_state(state);
+        if Self::VERIFY {
+            return if state.okay() {
+                r
+            } else {
+                Err(LangError::UnexpectedToken)
+            };
+        }
+        r
+    }
+    #[cfg(test)]
+    /// Parse this AST node utilizing the full token-stream. Intended for the test suite.
     fn from_insecure_tokens_full(tok: &'a [Token<'a>]) -> LangResult<Self> {
         let mut state = State::new(tok, InplaceData::new());
         let r = <Self as ASTNode>::from_state(&mut state)?;
@@ -41,10 +78,8 @@ pub trait ASTNode<'a>: Sized {
         Ok(r)
     }
     #[cfg(test)]
-    fn multiple_from_state<Qd: QueryData<'a>>(_: &mut State<'a, Qd>) -> LangResult<Vec<Self>> {
-        unimplemented!()
-    }
-    #[cfg(test)]
+    /// Parse multiple nodes of this AST node type, utilizing the full token stream.
+    /// Intended for the test suite.
     fn multiple_from_insecure_tokens_full(tok: &'a [Token<'a>]) -> LangResult<Vec<Self>> {
         let mut state = State::new(tok, InplaceData::new());
         let r = Self::multiple_from_state(&mut state);
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 0aa6e68d..940bf496 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -183,12 +183,12 @@ mod impls {
         LangResult,
     };
     impl<'a> ASTNode<'a> for AlterModel<'a> {
-        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             Self::parse(state)
         }
     }
     impl<'a> ASTNode<'a> for AlterSpace<'a> {
-        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             Self::parse(state)
         }
     }
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index b57cf93f..c90b24fb 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -36,14 +36,14 @@ use crate::{
 
 #[derive(Debug, PartialEq)]
 /// A space
-pub struct Space<'a> {
+pub struct CreateSpace<'a> {
     /// the space name
     pub(super) space_name: Slice<'a>,
     /// properties
     pub(super) props: Dict,
 }
 
-impl<'a> Space<'a> {
+impl<'a> CreateSpace<'a> {
     #[inline(always)]
     /// Parse space data from the given tokens
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
@@ -63,7 +63,7 @@ impl<'a> Space<'a> {
             syn::rfold_dict(DictFoldState::OB, state, &mut d);
         }
         if state.okay() {
-            Ok(Space {
+            Ok(CreateSpace {
                 space_name: unsafe { extract!(space_name, Token::Ident(ref id) => id.clone()) },
                 props: d,
             })
@@ -75,7 +75,7 @@ impl<'a> Space<'a> {
 
 #[derive(Debug, PartialEq)]
 /// A model definition
-pub struct Model<'a> {
+pub struct CreateModel<'a> {
     /// the model name
     model_name: Slice<'a>,
     /// the fields
@@ -91,7 +91,7 @@ pub struct Model<'a> {
     )
 */
 
-impl<'a> Model<'a> {
+impl<'a> CreateModel<'a> {
     pub fn new(model_name: Slice<'a>, fields: Vec<Field<'a>>, props: Dict) -> Self {
         Self {
             model_name,
@@ -142,18 +142,18 @@ impl<'a> Model<'a> {
 }
 
 mod impls {
-    use super::{Model, Space};
+    use super::{CreateModel, CreateSpace};
     use crate::engine::ql::{
         ast::{traits::ASTNode, QueryData, State},
         LangResult,
     };
-    impl<'a> ASTNode<'a> for Space<'a> {
-        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    impl<'a> ASTNode<'a> for CreateSpace<'a> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             Self::parse(state)
         }
     }
-    impl<'a> ASTNode<'a> for Model<'a> {
-        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    impl<'a> ASTNode<'a> for CreateModel<'a> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             Self::parse(state)
         }
     }
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index 7b57e499..a1dba5ae 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -107,19 +107,19 @@ mod impls {
         LangResult,
     };
     impl<'a> ASTNode<'a> for DropModel<'a> {
-        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             Self::parse(state)
         }
     }
     impl<'a> ASTNode<'a> for DropSpace<'a> {
-        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             Self::parse(state)
         }
     }
     #[derive(sky_macros::Wrapper, Debug)]
     pub struct DropStatementAST<'a>(Statement<'a>);
     impl<'a> ASTNode<'a> for DropStatementAST<'a> {
-        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             super::parse_drop(state).map(Self)
         }
     }
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
index 3eae00e1..b49c45b9 100644
--- a/server/src/engine/ql/ddl/ins.rs
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -72,7 +72,7 @@ mod impls {
     #[derive(sky_macros::Wrapper, Debug)]
     pub struct InspectStatementAST<'a>(Statement<'a>);
     impl<'a> ASTNode<'a> for InspectStatementAST<'a> {
-        fn from_state<Qd: QueryData<'a>>(
+        fn _from_state<Qd: QueryData<'a>>(
             state: &mut State<'a, Qd>,
         ) -> crate::engine::ql::LangResult<Self> {
             super::parse_inspect(state).map(Self)
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 1f056be0..9387949d 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -505,78 +505,70 @@ mod impls {
     };
     use crate::engine::ql::{
         ast::{traits::ASTNode, QueryData, State},
-        LangError, LangResult,
+        LangResult,
     };
     impl<'a> ASTNode<'a> for ExpandedField<'a> {
-        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             Self::parse(state)
         }
-        fn multiple_from_state<Qd: QueryData<'a>>(
+        fn _multiple_from_state<Qd: QueryData<'a>>(
             state: &mut State<'a, Qd>,
         ) -> LangResult<Vec<Self>> {
             Self::parse_multiple(state).map(Vec::from)
         }
     }
     impl<'a> ASTNode<'a> for Layer<'a> {
-        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        // important: upstream must verify this
+        const VERIFY: bool = true;
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             let mut layers = Vec::new();
             rfold_layers(LayerFoldState::BEGIN_IDENT, state, &mut layers);
             assert!(layers.len() == 1);
             Ok(layers.swap_remove(0))
         }
-        fn multiple_from_state<Qd: QueryData<'a>>(
+        fn _multiple_from_state<Qd: QueryData<'a>>(
             state: &mut State<'a, Qd>,
         ) -> LangResult<Vec<Self>> {
             let mut l = Vec::new();
             rfold_layers(LayerFoldState::BEGIN_IDENT, state, &mut l);
-            if state.okay() {
-                Ok(l)
-            } else {
-                Err(LangError::UnexpectedToken)
-            }
+            Ok(l)
         }
     }
     #[derive(sky_macros::Wrapper, Debug)]
     pub struct DictBasic(Dict);
     impl<'a> ASTNode<'a> for DictBasic {
-        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        // important: upstream must verify this
+        const VERIFY: bool = true;
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             let mut dict = Dict::new();
             rfold_dict(DictFoldState::OB, state, &mut dict);
-            if state.okay() {
-                Ok(Self(dict))
-            } else {
-                Err(LangError::UnexpectedToken)
-            }
+            Ok(Self(dict))
         }
     }
     #[derive(sky_macros::Wrapper, Debug)]
     pub struct DictTypeMetaSplit(Dict);
     impl<'a> ASTNode<'a> for DictTypeMetaSplit {
-        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        // important: upstream must verify this
+        const VERIFY: bool = true;
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             let mut dict = Dict::new();
             rfold_tymeta(DictFoldState::CB_OR_IDENT, state, &mut dict);
-            if state.okay() {
-                Ok(Self(dict))
-            } else {
-                Err(LangError::UnexpectedToken)
-            }
+            Ok(Self(dict))
         }
     }
     #[derive(sky_macros::Wrapper, Debug)]
     pub struct DictTypeMeta(Dict);
     impl<'a> ASTNode<'a> for DictTypeMeta {
-        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        // important: upstream must verify this
+        const VERIFY: bool = true;
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             let mut dict = Dict::new();
             rfold_tymeta(DictFoldState::OB, state, &mut dict);
-            if state.okay() {
-                Ok(Self(dict))
-            } else {
-                Err(LangError::UnexpectedToken)
-            }
+            Ok(Self(dict))
         }
     }
     impl<'a> ASTNode<'a> for Field<'a> {
-        fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             Self::parse(state)
         }
     }
diff --git a/server/src/engine/ql/dml/del.rs b/server/src/engine/ql/dml/del.rs
index 95f6ed0b..70c955e2 100644
--- a/server/src/engine/ql/dml/del.rs
+++ b/server/src/engine/ql/dml/del.rs
@@ -30,7 +30,7 @@ use {
     super::WhereClause,
     crate::{
         engine::ql::{
-            ast::{traits::ASTNode, Entity, QueryData, State},
+            ast::{Entity, QueryData, State},
             LangError, LangResult,
         },
         util::{compiler, MaybeInit},
@@ -94,8 +94,15 @@ impl<'a> DeleteStatement<'a> {
     }
 }
 
-impl<'a> ASTNode<'a> for DeleteStatement<'a> {
-    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-        Self::parse_delete(state)
+mod impls {
+    use super::DeleteStatement;
+    use crate::engine::ql::{
+        ast::{traits::ASTNode, QueryData, State},
+        LangResult,
+    };
+    impl<'a> ASTNode<'a> for DeleteStatement<'a> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            Self::parse_delete(state)
+        }
     }
 }
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index d911f348..ff586b40 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -30,7 +30,7 @@ use {
         engine::{
             core::DataType,
             ql::{
-                ast::{traits::ASTNode, Entity, QueryData, State},
+                ast::{Entity, QueryData, State},
                 lex::Token,
                 LangError, LangResult,
             },
@@ -207,22 +207,6 @@ unsafe fn handle_func_sub<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> O
     ldfunc(func).map(move |f| f())
 }
 
-#[cfg(test)]
-#[derive(sky_macros::Wrapper, Debug)]
-pub struct List(Vec<DataType>);
-#[cfg(test)]
-impl<'a> ASTNode<'a> for List {
-    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-        let mut l = Vec::new();
-        parse_list(state, &mut l);
-        if state.okay() {
-            Ok(List(l))
-        } else {
-            Err(LangError::UnexpectedToken)
-        }
-    }
-}
-
 /// ## Panics
 /// - If tt is empty
 pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
@@ -270,21 +254,6 @@ pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
     data
 }
 
-#[cfg(test)]
-#[derive(sky_macros::Wrapper, Debug)]
-pub struct DataTuple(Vec<Option<DataType>>);
-#[cfg(test)]
-impl<'a> ASTNode<'a> for DataTuple {
-    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-        let r = parse_data_tuple_syntax(state);
-        if state.okay() {
-            Ok(Self(r))
-        } else {
-            Err(LangError::UnexpectedToken)
-        }
-    }
-}
-
 /// ## Panics
 /// Panics if tt is empty
 pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
@@ -341,30 +310,6 @@ pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
     data
 }
 
-#[cfg(test)]
-#[derive(sky_macros::Wrapper, Debug)]
-pub struct DataMap(HashMap<Box<str>, Option<DataType>>);
-#[cfg(test)]
-impl<'a> ASTNode<'a> for DataMap {
-    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-        let r = parse_data_map_syntax(state);
-        if state.okay() {
-            Ok(Self(
-                r.into_iter()
-                    .map(|(ident, val)| {
-                        (
-                            String::from_utf8_lossy(ident).to_string().into_boxed_str(),
-                            val,
-                        )
-                    })
-                    .collect(),
-            ))
-        } else {
-            Err(LangError::UnexpectedToken)
-        }
-    }
-}
-
 #[derive(Debug, PartialEq)]
 pub enum InsertData<'a> {
     Ordered(Vec<Option<DataType>>),
@@ -444,8 +389,67 @@ impl<'a> InsertStatement<'a> {
     }
 }
 
-impl<'a> ASTNode<'a> for InsertStatement<'a> {
-    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-        Self::parse_insert(state)
+#[cfg(test)]
+pub use impls::test::{DataMap, DataTuple, List};
+mod impls {
+    use super::InsertStatement;
+    use crate::engine::ql::{
+        ast::{traits::ASTNode, QueryData, State},
+        LangResult,
+    };
+    impl<'a> ASTNode<'a> for InsertStatement<'a> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            Self::parse_insert(state)
+        }
+    }
+    #[cfg(test)]
+    pub mod test {
+        use super::super::{
+            parse_data_map_syntax, parse_data_tuple_syntax, parse_list, DataType, HashMap,
+        };
+        use crate::engine::ql::{
+            ast::{traits::ASTNode, QueryData, State},
+            LangResult,
+        };
+        #[derive(sky_macros::Wrapper, Debug)]
+        pub struct List(Vec<DataType>);
+        impl<'a> ASTNode<'a> for List {
+            // important: upstream must verify this
+            const VERIFY: bool = true;
+            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+                let mut l = Vec::new();
+                parse_list(state, &mut l);
+                Ok(List(l))
+            }
+        }
+        #[derive(sky_macros::Wrapper, Debug)]
+        pub struct DataTuple(Vec<Option<DataType>>);
+        impl<'a> ASTNode<'a> for DataTuple {
+            // important: upstream must verify this
+            const VERIFY: bool = true;
+            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+                let r = parse_data_tuple_syntax(state);
+                Ok(Self(r))
+            }
+        }
+        #[derive(sky_macros::Wrapper, Debug)]
+        pub struct DataMap(HashMap<Box<str>, Option<DataType>>);
+        impl<'a> ASTNode<'a> for DataMap {
+            // important: upstream must verify this
+            const VERIFY: bool = true;
+            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+                let r = parse_data_map_syntax(state);
+                Ok(Self(
+                    r.into_iter()
+                        .map(|(ident, val)| {
+                            (
+                                String::from_utf8_lossy(ident).to_string().into_boxed_str(),
+                                val,
+                            )
+                        })
+                        .collect(),
+                ))
+            }
+        }
     }
 }
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index cf7ba7d9..ce0c5725 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -36,9 +36,8 @@ pub mod upd;
 
 use {
     super::{
-        ast::{traits::ASTNode, QueryData, State},
+        ast::{QueryData, State},
         lex::{LitIR, Token},
-        LangError, LangResult,
     },
     crate::util::compiler,
     std::collections::HashMap,
@@ -119,12 +118,6 @@ impl<'a> RelationalExpr<'a> {
     }
 }
 
-impl<'a> ASTNode<'a> for RelationalExpr<'a> {
-    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-        Self::try_parse(state).ok_or(LangError::UnexpectedToken)
-    }
-}
-
 #[derive(Debug, PartialEq)]
 pub struct WhereClause<'a> {
     c: WhereClauseCollection<'a>,
@@ -164,13 +157,26 @@ impl<'a> WhereClause<'a> {
     }
 }
 
-impl<'a> ASTNode<'a> for WhereClause<'a> {
-    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-        let wh = Self::parse_where(state);
-        if state.okay() {
+#[cfg(test)]
+mod impls {
+    use super::{
+        super::{
+            ast::{traits::ASTNode, QueryData, State},
+            LangError, LangResult,
+        },
+        RelationalExpr, WhereClause,
+    };
+    impl<'a> ASTNode<'a> for WhereClause<'a> {
+        // important: upstream must verify this
+        const VERIFY: bool = true;
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            let wh = Self::parse_where(state);
             Ok(wh)
-        } else {
-            Err(LangError::UnexpectedToken)
+        }
+    }
+    impl<'a> ASTNode<'a> for RelationalExpr<'a> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            Self::try_parse(state).ok_or(LangError::UnexpectedToken)
         }
     }
 }
diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs
index 88e78a99..5ad5c687 100644
--- a/server/src/engine/ql/dml/sel.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -28,7 +28,7 @@ use {
     super::{WhereClause, WhereClauseCollection},
     crate::{
         engine::ql::{
-            ast::{traits::ASTNode, Entity, QueryData, State},
+            ast::{Entity, QueryData, State},
             lex::Token,
             LangError, LangResult,
         },
@@ -134,8 +134,15 @@ impl<'a> SelectStatement<'a> {
     }
 }
 
-impl<'a> ASTNode<'a> for SelectStatement<'a> {
-    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-        Self::parse_select(state)
+mod impls {
+    use super::SelectStatement;
+    use crate::engine::ql::{
+        ast::{traits::ASTNode, QueryData, State},
+        LangResult,
+    };
+    impl<'a> ASTNode<'a> for SelectStatement<'a> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            Self::parse_select(state)
+        }
     }
 }
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index 2713b150..10d4702b 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -25,15 +25,12 @@
 */
 
 #[cfg(test)]
-use {
-    super::WhereClauseCollection,
-    crate::engine::ql::{ast::InplaceData, lex::Token},
-};
+use super::WhereClauseCollection;
 use {
     super::{read_ident, u, WhereClause},
     crate::{
         engine::ql::{
-            ast::{traits::ASTNode, Entity, QueryData, State},
+            ast::{Entity, QueryData, State},
             lex::LitIR,
             LangError, LangResult,
         },
@@ -132,20 +129,6 @@ impl<'a> AssignmentExpression<'a> {
     }
 }
 
-#[cfg(test)]
-pub fn parse_assn_expression_full<'a>(tok: &'a [Token]) -> Option<AssignmentExpression<'a>> {
-    let mut state = State::new(tok, InplaceData::new());
-    let mut exprs = Vec::new();
-    AssignmentExpression::parse_and_append_expression(&mut state, &mut exprs);
-    assert_full_tt!(state);
-    if state.okay() {
-        assert_eq!(exprs.len(), 1, "expected one expression, found multiple");
-        Some(exprs.remove(0))
-    } else {
-        None
-    }
-}
-
 #[derive(Debug, PartialEq)]
 pub struct UpdateStatement<'a> {
     pub(super) entity: Entity<'a>,
@@ -227,8 +210,36 @@ impl<'a> UpdateStatement<'a> {
     }
 }
 
-impl<'a> ASTNode<'a> for UpdateStatement<'a> {
-    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-        Self::parse_update(state)
+mod impls {
+    use super::UpdateStatement;
+    use crate::engine::ql::{
+        ast::{traits::ASTNode, QueryData, State},
+        LangResult,
+    };
+    impl<'a> ASTNode<'a> for UpdateStatement<'a> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            Self::parse_update(state)
+        }
+    }
+    #[cfg(test)]
+    mod test {
+        use super::{super::AssignmentExpression, *};
+        impl<'a> ASTNode<'a> for AssignmentExpression<'a> {
+            // important: upstream must verify this
+            const VERIFY: bool = true;
+            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+                let mut expr = Vec::new();
+                AssignmentExpression::parse_and_append_expression(state, &mut expr);
+                state.poison_if_not(expr.len() == 1);
+                Ok(expr.remove(0))
+            }
+            fn _multiple_from_state<Qd: QueryData<'a>>(
+                state: &mut State<'a, Qd>,
+            ) -> LangResult<Vec<Self>> {
+                let mut expr = Vec::new();
+                AssignmentExpression::parse_and_append_expression(state, &mut expr);
+                Ok(expr)
+            }
+        }
     }
 }
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 9adc304e..a158a339 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -24,21 +24,6 @@
  *
 */
 
-#[cfg(test)]
-macro_rules! assert_full_tt {
-    ($a:expr, $b:expr) => {
-        assert_eq!($a, $b, "full token stream not utilized")
-    };
-    ($a:expr) => {
-        assert!(
-            crate::engine::ql::ast::State::exhausted(&$a),
-            "full tt not utilized at: {}:{}",
-            ::core::file!(),
-            ::core::line!()
-        )
-    };
-}
-
 macro_rules! __sym_token {
     ($ident:ident) => {
         $crate::engine::ql::lex::Token::Symbol($crate::engine::ql::lex::Symbol::$ident)
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index d3ada309..6c62fd32 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -690,17 +690,15 @@ mod expression_tests {
     use {
         super::*,
         crate::engine::ql::{
-            dml::{
-                self,
-                upd::{AssignmentExpression, Operator},
-            },
+            ast::parse_ast_node_full,
+            dml::upd::{AssignmentExpression, Operator},
             lex::LitIR,
         },
     };
     #[test]
     fn expr_assign() {
         let src = lex_insecure(b"username = 'sayan'").unwrap();
-        let r = dml::upd::parse_assn_expression_full(&src).unwrap();
+        let r = parse_ast_node_full::<AssignmentExpression>(&src).unwrap();
         assert_eq!(
             r,
             AssignmentExpression::new(b"username", LitIR::Str("sayan"), Operator::Assign)
@@ -709,7 +707,7 @@ mod expression_tests {
     #[test]
     fn expr_add_assign() {
         let src = lex_insecure(b"followers += 100").unwrap();
-        let r = dml::upd::parse_assn_expression_full(&src).unwrap();
+        let r = parse_ast_node_full::<AssignmentExpression>(&src).unwrap();
         assert_eq!(
             r,
             AssignmentExpression::new(b"followers", LitIR::UInt(100), Operator::AddAssign)
@@ -718,7 +716,7 @@ mod expression_tests {
     #[test]
     fn expr_sub_assign() {
         let src = lex_insecure(b"following -= 150").unwrap();
-        let r = dml::upd::parse_assn_expression_full(&src).unwrap();
+        let r = parse_ast_node_full::<AssignmentExpression>(&src).unwrap();
         assert_eq!(
             r,
             AssignmentExpression::new(b"following", LitIR::UInt(150), Operator::SubAssign)
@@ -727,7 +725,7 @@ mod expression_tests {
     #[test]
     fn expr_mul_assign() {
         let src = lex_insecure(b"product_qty *= 2").unwrap();
-        let r = dml::upd::parse_assn_expression_full(&src).unwrap();
+        let r = parse_ast_node_full::<AssignmentExpression>(&src).unwrap();
         assert_eq!(
             r,
             AssignmentExpression::new(b"product_qty", LitIR::UInt(2), Operator::MulAssign)
@@ -736,7 +734,7 @@ mod expression_tests {
     #[test]
     fn expr_div_assign() {
         let src = lex_insecure(b"image_crop_factor /= 2").unwrap();
-        let r = dml::upd::parse_assn_expression_full(&src).unwrap();
+        let r = parse_ast_node_full::<AssignmentExpression>(&src).unwrap();
         assert_eq!(
             r,
             AssignmentExpression::new(b"image_crop_factor", LitIR::UInt(2), Operator::DivAssign)
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 0ae72e8f..73535c11 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -408,7 +408,7 @@ mod schemas {
     use crate::engine::ql::{
         ast::parse_ast_node_full,
         ddl::{
-            crt::Model,
+            crt::CreateModel,
             syn::{Field, Layer},
         },
     };
@@ -426,11 +426,11 @@ mod schemas {
         let tok = &tok[2..];
 
         // parse model
-        let model = parse_ast_node_full::<Model>(tok).unwrap();
+        let model = parse_ast_node_full::<CreateModel>(tok).unwrap();
 
         assert_eq!(
             model,
-            Model::new(
+            CreateModel::new(
                 b"mymodel",
                 vec![
                     Field::new(
@@ -465,11 +465,11 @@ mod schemas {
         let tok = &tok[2..];
 
         // parse model
-        let model = parse_ast_node_full::<Model>(tok).unwrap();
+        let model = parse_ast_node_full::<CreateModel>(tok).unwrap();
 
         assert_eq!(
             model,
-            Model::new(
+            CreateModel::new(
                 b"mymodel",
                 vec![
                     Field::new(
@@ -515,11 +515,11 @@ mod schemas {
         let tok = &tok[2..];
 
         // parse model
-        let model = parse_ast_node_full::<Model>(tok).unwrap();
+        let model = parse_ast_node_full::<CreateModel>(tok).unwrap();
 
         assert_eq!(
             model,
-            Model::new(
+            CreateModel::new(
                 b"mymodel",
                 vec![
                     Field::new(
@@ -584,11 +584,11 @@ mod schemas {
         let tok = &tok[2..];
 
         // parse model
-        let model = parse_ast_node_full::<Model>(tok).unwrap();
+        let model = parse_ast_node_full::<CreateModel>(tok).unwrap();
 
         assert_eq!(
             model,
-            Model::new(
+            CreateModel::new(
                 b"mymodel",
                 vec![
                     Field::new(

From 080563cc2466610aec7762731eee2ecda987e013 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 3 Feb 2023 10:01:52 -0800
Subject: [PATCH 121/310] Re-organize imports to match import style [no ci]

---
 server/src/engine/idx/mod.rs          |  6 +++--
 server/src/engine/idx/mtchm/imp.rs    | 16 +++++++------
 server/src/engine/idx/mtchm/iter.rs   | 12 ++++++----
 server/src/engine/idx/mtchm/meta.rs   |  9 +++++--
 server/src/engine/idx/mtchm/mod.rs    | 34 +++++++++++++--------------
 server/src/engine/idx/mtchm/tests.rs  | 24 ++++++++++---------
 server/src/engine/idx/stdhm.rs        | 20 +++++++++-------
 server/src/engine/idx/stord/config.rs |  7 ++++--
 server/src/engine/idx/stord/iter.rs   | 24 ++++++++++---------
 server/src/engine/idx/stord/mod.rs    | 32 +++++++++++++------------
 server/src/engine/ql/ddl/alt.rs       | 26 +++++++++++---------
 server/src/engine/ql/ddl/crt.rs       | 26 +++++++++++---------
 server/src/engine/ql/ddl/drop.rs      | 10 ++++----
 server/src/engine/ql/ddl/syn.rs       | 32 ++++++++++++++-----------
 server/src/engine/ql/dml/del.rs       | 10 ++++----
 server/src/engine/ql/dml/ins.rs       | 24 +++++++++++--------
 server/src/engine/ql/dml/mod.rs       |  6 ++---
 server/src/engine/ql/dml/sel.rs       | 10 ++++----
 server/src/engine/ql/dml/upd.rs       | 12 ++++++----
 server/src/engine/ql/lex/mod.rs       |  2 +-
 server/src/engine/ql/lex/raw.rs       |  3 +--
 server/src/engine/sync/atm.rs         |  6 +++--
 server/src/engine/sync/cell.rs        |  9 +++----
 server/src/engine/sync/smart.rs       | 26 ++++++++++----------
 24 files changed, 219 insertions(+), 167 deletions(-)

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 64b43288..4d1cf095 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -33,8 +33,10 @@ mod stord;
 #[cfg(test)]
 mod tests;
 
-use super::sync::atm::Guard;
-use core::{borrow::Borrow, hash::Hash};
+use {
+    crate::engine::sync::atm::Guard,
+    core::{borrow::Borrow, hash::Hash},
+};
 
 // re-exports
 pub type IndexSTSeqCns<K, V> = stord::IndexSTSeqDll<K, V, stord::config::ConservativeConfig<K, V>>;
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index 92261a2c..01f0151f 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -26,16 +26,18 @@
 
 #[cfg(debug_assertions)]
 use super::CHTRuntimeLog;
-use super::{
+use {
     super::{
-        super::sync::atm::{upin, Guard},
-        AsKey, IndexBaseSpec, MTIndex,
+        iter::{IterKV, IterKey, IterVal},
+        meta::{Config, Key, TreeElement, Value},
+        Tree,
     },
-    iter::{IterKV, IterKey, IterVal},
-    meta::{Config, Key, TreeElement, Value},
-    Tree,
+    crate::engine::{
+        idx::{AsKey, IndexBaseSpec, MTIndex},
+        sync::atm::{upin, Guard},
+    },
+    std::{borrow::Borrow, sync::Arc},
 };
-use std::{borrow::Borrow, sync::Arc};
 
 #[inline(always)]
 fn arc<K, V>(k: K, v: V) -> Arc<(K, V)> {
diff --git a/server/src/engine/idx/mtchm/iter.rs b/server/src/engine/idx/mtchm/iter.rs
index 203d4637..432b42e2 100644
--- a/server/src/engine/idx/mtchm/iter.rs
+++ b/server/src/engine/idx/mtchm/iter.rs
@@ -24,15 +24,17 @@
  *
 */
 
-use super::{
-    super::super::{
+use {
+    super::{
+        meta::{Config, DefConfig, NodeFlag, TreeElement},
+        Node, Tree,
+    },
+    crate::engine::{
         mem::UArray,
         sync::atm::{Guard, Shared},
     },
-    meta::{Config, DefConfig, NodeFlag, TreeElement},
-    Node, Tree,
+    std::marker::PhantomData,
 };
-use std::marker::PhantomData;
 
 pub struct IterKV<'t, 'g, 'v, T, C>
 where
diff --git a/server/src/engine/idx/mtchm/meta.rs b/server/src/engine/idx/mtchm/meta.rs
index 72b2cc8f..14d46568 100644
--- a/server/src/engine/idx/mtchm/meta.rs
+++ b/server/src/engine/idx/mtchm/meta.rs
@@ -24,8 +24,13 @@
  *
 */
 
-use super::super::{super::mem::VInline, meta::AsHasher, AsKeyClone};
-use std::{collections::hash_map::RandomState, sync::Arc};
+use {
+    crate::engine::{
+        idx::{meta::AsHasher, AsKeyClone},
+        mem::VInline,
+    },
+    std::{collections::hash_map::RandomState, sync::Arc},
+};
 
 const LNODE_STACK: usize = 2;
 pub type DefConfig = Config2B<RandomState>;
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index 074a7095..b10d0c7b 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -31,27 +31,27 @@ pub(super) mod meta;
 #[cfg(test)]
 mod tests;
 
-use self::{
-    access::{ReadMode, WriteMode},
-    iter::{IterKV, IterKey, IterVal},
-    meta::{CompressState, Config, DefConfig, LNode, NodeFlag, TreeElement},
-};
 #[cfg(debug_assertions)]
-use super::super::sync::atm::ORD_ACQ;
-use super::{
-    super::{
+use crate::engine::sync::atm::ORD_ACQ;
+use {
+    self::{
+        access::{ReadMode, WriteMode},
+        iter::{IterKV, IterKey, IterVal},
+        meta::{CompressState, Config, DefConfig, LNode, NodeFlag, TreeElement},
+    },
+    crate::engine::{
+        idx::AsKey,
         mem::UArray,
         sync::atm::{self, cpin, upin, Atomic, Guard, Owned, Shared, ORD_ACR, ORD_RLX},
     },
-    AsKey,
-};
-use crossbeam_epoch::CompareExchangeError;
-use std::{
-    borrow::Borrow,
-    hash::{BuildHasher, Hasher},
-    marker::PhantomData,
-    mem,
-    sync::atomic::AtomicUsize,
+    crossbeam_epoch::CompareExchangeError,
+    std::{
+        borrow::Borrow,
+        hash::{BuildHasher, Hasher},
+        marker::PhantomData,
+        mem,
+        sync::atomic::AtomicUsize,
+    },
 };
 
 /*
diff --git a/server/src/engine/idx/mtchm/tests.rs b/server/src/engine/idx/mtchm/tests.rs
index bc55f153..00e24292 100644
--- a/server/src/engine/idx/mtchm/tests.rs
+++ b/server/src/engine/idx/mtchm/tests.rs
@@ -24,19 +24,21 @@
  *
 */
 
-use super::{
+use {
     super::{
-        super::sync::atm::{cpin, Guard},
-        IndexBaseSpec, MTIndex,
+        imp::ChmCopy as _ChmCopy,
+        meta::{Config, DefConfig},
+    },
+    crate::engine::{
+        idx::{IndexBaseSpec, MTIndex},
+        sync::atm::{cpin, Guard},
+    },
+    std::{
+        hash::{BuildHasher, Hasher},
+        sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},
+        thread::{self, JoinHandle},
+        time::Duration,
     },
-    imp::ChmCopy as _ChmCopy,
-    meta::{Config, DefConfig},
-};
-use std::{
-    hash::{BuildHasher, Hasher},
-    sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},
-    thread::{self, JoinHandle},
-    time::Duration,
 };
 
 type Chm<K, V> = ChmCopy<K, V>;
diff --git a/server/src/engine/idx/stdhm.rs b/server/src/engine/idx/stdhm.rs
index 72cee783..a78fcee3 100644
--- a/server/src/engine/idx/stdhm.rs
+++ b/server/src/engine/idx/stdhm.rs
@@ -26,15 +26,19 @@
 
 #[cfg(debug_assertions)]
 use super::DummyMetrics;
-use super::{AsKey, AsValue, AsValueClone, IndexBaseSpec, STIndex};
-use std::{
-    borrow::Borrow,
-    collections::{
-        hash_map::{Entry, Iter as StdMapIterKV, Keys as StdMapIterKey, Values as StdMapIterVal},
-        HashMap as StdMap,
+use {
+    super::{AsKey, AsValue, AsValueClone, IndexBaseSpec, STIndex},
+    std::{
+        borrow::Borrow,
+        collections::{
+            hash_map::{
+                Entry, Iter as StdMapIterKV, Keys as StdMapIterKey, Values as StdMapIterVal,
+            },
+            HashMap as StdMap,
+        },
+        hash::BuildHasher,
+        mem,
     },
-    hash::BuildHasher,
-    mem,
 };
 
 impl<K, V, S> IndexBaseSpec<K, V> for StdMap<K, V, S>
diff --git a/server/src/engine/idx/stord/config.rs b/server/src/engine/idx/stord/config.rs
index 8a8b38c6..241e487a 100644
--- a/server/src/engine/idx/stord/config.rs
+++ b/server/src/engine/idx/stord/config.rs
@@ -24,8 +24,11 @@
  *
 */
 
-use super::{super::meta::AsHasher, IndexSTSeqDllNode, IndexSTSeqDllNodePtr};
-use std::{collections::hash_map::RandomState, marker::PhantomData, ptr};
+use {
+    super::{IndexSTSeqDllNode, IndexSTSeqDllNodePtr},
+    crate::engine::idx::meta::AsHasher,
+    std::{collections::hash_map::RandomState, marker::PhantomData, ptr},
+};
 
 pub struct LiberalStrategy<K, V> {
     f: *mut IndexSTSeqDllNode<K, V>,
diff --git a/server/src/engine/idx/stord/iter.rs b/server/src/engine/idx/stord/iter.rs
index 237d07c5..fd887c7e 100644
--- a/server/src/engine/idx/stord/iter.rs
+++ b/server/src/engine/idx/stord/iter.rs
@@ -24,18 +24,20 @@
  *
 */
 
-use super::{
-    config::Config, IndexSTSeqDll, IndexSTSeqDllKeyptr, IndexSTSeqDllNode, IndexSTSeqDllNodePtr,
-};
-use std::{
-    collections::{
-        hash_map::{Iter, Keys as StdMapIterKey, Values as StdMapIterVal},
-        HashMap as StdMap,
+use {
+    super::{
+        config::Config, IndexSTSeqDll, IndexSTSeqDllKeyptr, IndexSTSeqDllNode, IndexSTSeqDllNodePtr,
+    },
+    std::{
+        collections::{
+            hash_map::{Iter, Keys as StdMapIterKey, Values as StdMapIterVal},
+            HashMap as StdMap,
+        },
+        fmt::{self, Debug},
+        iter::FusedIterator,
+        marker::PhantomData,
+        ptr::{self, NonNull},
     },
-    fmt::{self, Debug},
-    iter::FusedIterator,
-    marker::PhantomData,
-    ptr::{self, NonNull},
 };
 
 macro_rules! unsafe_marker_impl {
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index a5aecb97..237a817f 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -27,22 +27,24 @@
 pub(super) mod config;
 mod iter;
 
-use self::{
-    config::{AllocStrategy, Config},
-    iter::{
-        IndexSTSeqDllIterOrdKV, IndexSTSeqDllIterOrdKey, IndexSTSeqDllIterOrdValue,
-        IndexSTSeqDllIterUnordKV, IndexSTSeqDllIterUnordKey, IndexSTSeqDllIterUnordValue,
+use {
+    self::{
+        config::{AllocStrategy, Config},
+        iter::{
+            IndexSTSeqDllIterOrdKV, IndexSTSeqDllIterOrdKey, IndexSTSeqDllIterOrdValue,
+            IndexSTSeqDllIterUnordKV, IndexSTSeqDllIterUnordKey, IndexSTSeqDllIterUnordValue,
+        },
+    },
+    super::{AsKey, AsKeyClone, AsValue, AsValueClone, IndexBaseSpec, STIndex, STIndexSeq},
+    std::{
+        alloc::{alloc as std_alloc, dealloc as std_dealloc, Layout},
+        borrow::Borrow,
+        collections::HashMap as StdMap,
+        fmt::Debug,
+        hash::{Hash, Hasher},
+        mem,
+        ptr::{self, NonNull},
     },
-};
-use super::{AsKey, AsKeyClone, AsValue, AsValueClone, IndexBaseSpec, STIndex, STIndexSeq};
-use std::{
-    alloc::{alloc as std_alloc, dealloc as std_dealloc, Layout},
-    borrow::Borrow,
-    collections::HashMap as StdMap,
-    fmt::Debug,
-    hash::{Hash, Hasher},
-    mem,
-    ptr::{self, NonNull},
 };
 
 /*
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 940bf496..ddc30327 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -24,14 +24,16 @@
  *
 */
 
-use super::syn::{self, Dict, DictFoldState, ExpandedField};
-use crate::{
-    engine::ql::{
-        ast::{QueryData, State},
-        lex::{Slice, Token},
-        LangError, LangResult,
+use {
+    super::syn::{self, Dict, DictFoldState, ExpandedField},
+    crate::{
+        engine::ql::{
+            ast::{QueryData, State},
+            lex::{Slice, Token},
+            LangError, LangResult,
+        },
+        util::compiler,
     },
-    util::compiler,
 };
 
 #[derive(Debug, PartialEq)]
@@ -177,10 +179,12 @@ impl<'a> AlterKind<'a> {
 }
 
 mod impls {
-    use super::{AlterModel, AlterSpace};
-    use crate::engine::ql::{
-        ast::{traits::ASTNode, QueryData, State},
-        LangResult,
+    use {
+        super::{AlterModel, AlterSpace},
+        crate::engine::ql::{
+            ast::{traits::ASTNode, QueryData, State},
+            LangResult,
+        },
     };
     impl<'a> ASTNode<'a> for AlterModel<'a> {
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index c90b24fb..8cacd4b1 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -24,14 +24,16 @@
  *
 */
 
-use super::syn::{self, Dict, DictFoldState, Field};
-use crate::{
-    engine::ql::{
-        ast::{QueryData, State},
-        lex::{Slice, Token},
-        LangError, LangResult,
+use {
+    super::syn::{self, Dict, DictFoldState, Field},
+    crate::{
+        engine::ql::{
+            ast::{QueryData, State},
+            lex::{Slice, Token},
+            LangError, LangResult,
+        },
+        util::compiler,
     },
-    util::compiler,
 };
 
 #[derive(Debug, PartialEq)]
@@ -142,10 +144,12 @@ impl<'a> CreateModel<'a> {
 }
 
 mod impls {
-    use super::{CreateModel, CreateSpace};
-    use crate::engine::ql::{
-        ast::{traits::ASTNode, QueryData, State},
-        LangResult,
+    use {
+        super::{CreateModel, CreateSpace},
+        crate::engine::ql::{
+            ast::{traits::ASTNode, QueryData, State},
+            LangResult,
+        },
     };
     impl<'a> ASTNode<'a> for CreateSpace<'a> {
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index a1dba5ae..d7d401c2 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -101,10 +101,12 @@ pub fn parse_drop<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResul
 
 pub use impls::DropStatementAST;
 mod impls {
-    use super::{DropModel, DropSpace};
-    use crate::engine::ql::{
-        ast::{traits::ASTNode, QueryData, State, Statement},
-        LangResult,
+    use {
+        super::{DropModel, DropSpace},
+        crate::engine::ql::{
+            ast::{traits::ASTNode, QueryData, State, Statement},
+            LangResult,
+        },
     };
     impl<'a> ASTNode<'a> for DropModel<'a> {
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 9387949d..b821bc5e 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -44,15 +44,17 @@
     Feb. 2, 2023
 */
 
-use crate::{
-    engine::ql::{
-        ast::{QueryData, State},
-        lex::{LitIR, LitIROwned, Slice, Token},
-        LangError, LangResult,
+use {
+    crate::{
+        engine::ql::{
+            ast::{QueryData, State},
+            lex::{LitIR, LitIROwned, Slice, Token},
+            LangError, LangResult,
+        },
+        util::{compiler, MaybeInit},
     },
-    util::{compiler, MaybeInit},
+    std::{collections::HashMap, str},
 };
-use std::{collections::HashMap, str};
 
 #[derive(Debug, PartialEq)]
 /// A dictionary entry type. Either a literal or another dictionary
@@ -499,13 +501,15 @@ impl<'a> ExpandedField<'a> {
 pub use impls::{DictBasic, DictTypeMeta, DictTypeMetaSplit};
 #[cfg(test)]
 mod impls {
-    use super::{
-        rfold_dict, rfold_layers, rfold_tymeta, Dict, DictFoldState, ExpandedField, Field, Layer,
-        LayerFoldState,
-    };
-    use crate::engine::ql::{
-        ast::{traits::ASTNode, QueryData, State},
-        LangResult,
+    use {
+        super::{
+            rfold_dict, rfold_layers, rfold_tymeta, Dict, DictFoldState, ExpandedField, Field,
+            Layer, LayerFoldState,
+        },
+        crate::engine::ql::{
+            ast::{traits::ASTNode, QueryData, State},
+            LangResult,
+        },
     };
     impl<'a> ASTNode<'a> for ExpandedField<'a> {
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
diff --git a/server/src/engine/ql/dml/del.rs b/server/src/engine/ql/dml/del.rs
index 70c955e2..5f52bb5a 100644
--- a/server/src/engine/ql/dml/del.rs
+++ b/server/src/engine/ql/dml/del.rs
@@ -95,10 +95,12 @@ impl<'a> DeleteStatement<'a> {
 }
 
 mod impls {
-    use super::DeleteStatement;
-    use crate::engine::ql::{
-        ast::{traits::ASTNode, QueryData, State},
-        LangResult,
+    use {
+        super::DeleteStatement,
+        crate::engine::ql::{
+            ast::{traits::ASTNode, QueryData, State},
+            LangResult,
+        },
     };
     impl<'a> ASTNode<'a> for DeleteStatement<'a> {
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index ff586b40..65084f38 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -392,10 +392,12 @@ impl<'a> InsertStatement<'a> {
 #[cfg(test)]
 pub use impls::test::{DataMap, DataTuple, List};
 mod impls {
-    use super::InsertStatement;
-    use crate::engine::ql::{
-        ast::{traits::ASTNode, QueryData, State},
-        LangResult,
+    use {
+        super::InsertStatement,
+        crate::engine::ql::{
+            ast::{traits::ASTNode, QueryData, State},
+            LangResult,
+        },
     };
     impl<'a> ASTNode<'a> for InsertStatement<'a> {
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
@@ -404,12 +406,14 @@ mod impls {
     }
     #[cfg(test)]
     pub mod test {
-        use super::super::{
-            parse_data_map_syntax, parse_data_tuple_syntax, parse_list, DataType, HashMap,
-        };
-        use crate::engine::ql::{
-            ast::{traits::ASTNode, QueryData, State},
-            LangResult,
+        use {
+            super::super::{
+                parse_data_map_syntax, parse_data_tuple_syntax, parse_list, DataType, HashMap,
+            },
+            crate::engine::ql::{
+                ast::{traits::ASTNode, QueryData, State},
+                LangResult,
+            },
         };
         #[derive(sky_macros::Wrapper, Debug)]
         pub struct List(Vec<DataType>);
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index ce0c5725..c0e7cce5 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -159,12 +159,12 @@ impl<'a> WhereClause<'a> {
 
 #[cfg(test)]
 mod impls {
-    use super::{
-        super::{
+    use {
+        super::{RelationalExpr, WhereClause},
+        crate::engine::ql::{
             ast::{traits::ASTNode, QueryData, State},
             LangError, LangResult,
         },
-        RelationalExpr, WhereClause,
     };
     impl<'a> ASTNode<'a> for WhereClause<'a> {
         // important: upstream must verify this
diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs
index 5ad5c687..ecf6fdd3 100644
--- a/server/src/engine/ql/dml/sel.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -135,10 +135,12 @@ impl<'a> SelectStatement<'a> {
 }
 
 mod impls {
-    use super::SelectStatement;
-    use crate::engine::ql::{
-        ast::{traits::ASTNode, QueryData, State},
-        LangResult,
+    use {
+        super::SelectStatement,
+        crate::engine::ql::{
+            ast::{traits::ASTNode, QueryData, State},
+            LangResult,
+        },
     };
     impl<'a> ASTNode<'a> for SelectStatement<'a> {
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index 10d4702b..c41f6d5b 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -211,10 +211,12 @@ impl<'a> UpdateStatement<'a> {
 }
 
 mod impls {
-    use super::UpdateStatement;
-    use crate::engine::ql::{
-        ast::{traits::ASTNode, QueryData, State},
-        LangResult,
+    use {
+        super::UpdateStatement,
+        crate::engine::ql::{
+            ast::{traits::ASTNode, QueryData, State},
+            LangResult,
+        },
     };
     impl<'a> ASTNode<'a> for UpdateStatement<'a> {
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
@@ -223,7 +225,7 @@ mod impls {
     }
     #[cfg(test)]
     mod test {
-        use super::{super::AssignmentExpression, *};
+        use super::{super::AssignmentExpression, ASTNode, LangResult, QueryData, State};
         impl<'a> ASTNode<'a> for AssignmentExpression<'a> {
             // important: upstream must verify this
             const VERIFY: bool = true;
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index 142bbc36..ebf417a9 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -26,8 +26,8 @@
 
 mod raw;
 
-use self::raw::RawLexer;
 use {
+    self::raw::RawLexer,
     super::{LangError, LangResult},
     crate::util::compiler,
     core::{cmp, fmt, ops::BitOr, slice, str},
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index 407d4439..cd5911be 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -24,8 +24,7 @@
  *
 */
 
-use super::{super::LangError, Slice};
-use core::slice;
+use {super::Slice, crate::engine::ql::LangError, core::slice};
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum Token<'a> {
diff --git a/server/src/engine/sync/atm.rs b/server/src/engine/sync/atm.rs
index 57fa3142..bffae52a 100644
--- a/server/src/engine/sync/atm.rs
+++ b/server/src/engine/sync/atm.rs
@@ -24,8 +24,10 @@
  *
 */
 
-use core::{fmt, mem, ops::Deref, sync::atomic::Ordering};
-use crossbeam_epoch::{Atomic as CBAtomic, CompareExchangeError, Pointable, Pointer};
+use {
+    core::{fmt, mem, ops::Deref, sync::atomic::Ordering},
+    crossbeam_epoch::{Atomic as CBAtomic, CompareExchangeError, Pointable, Pointer},
+};
 // re-export here because we have some future plans ;) (@ohsayan)
 pub use crossbeam_epoch::{pin as cpin, unprotected as upin, Guard, Owned, Shared};
 
diff --git a/server/src/engine/sync/cell.rs b/server/src/engine/sync/cell.rs
index ac11a629..d34e1d65 100644
--- a/server/src/engine/sync/cell.rs
+++ b/server/src/engine/sync/cell.rs
@@ -24,10 +24,11 @@
  *
 */
 
-use super::atm::{upin, Atomic, Guard, Owned, Shared, ORD_REL};
-use core::ops::Deref;
-use parking_lot::{Mutex, MutexGuard};
-use std::marker::PhantomData;
+use {
+    super::atm::{upin, Atomic, Guard, Owned, Shared, ORD_REL},
+    core::{marker::PhantomData, ops::Deref},
+    parking_lot::{Mutex, MutexGuard},
+};
 
 /// A [`TMCell`] provides atomic reads and serialized writes; the `static` is a CB hack
 #[derive(Debug)]
diff --git a/server/src/engine/sync/smart.rs b/server/src/engine/sync/smart.rs
index 60179843..7302cc58 100644
--- a/server/src/engine/sync/smart.rs
+++ b/server/src/engine/sync/smart.rs
@@ -24,18 +24,20 @@
  *
 */
 
-use super::atm::{ORD_ACQ, ORD_REL, ORD_RLX};
-use std::{
-    alloc::{dealloc, Layout},
-    borrow::Borrow,
-    fmt,
-    hash::{Hash, Hasher},
-    mem::{self, ManuallyDrop},
-    ops::Deref,
-    process,
-    ptr::{self, NonNull},
-    slice, str,
-    sync::atomic::{self, AtomicUsize},
+use {
+    super::atm::{ORD_ACQ, ORD_REL, ORD_RLX},
+    std::{
+        alloc::{dealloc, Layout},
+        borrow::Borrow,
+        fmt,
+        hash::{Hash, Hasher},
+        mem::{self, ManuallyDrop},
+        ops::Deref,
+        process,
+        ptr::{self, NonNull},
+        slice, str,
+        sync::atomic::{self, AtomicUsize},
+    },
 };
 
 pub type BytesRC = SliceRC<u8>;

From 79f8630424763bb9cf42f55d7babf78d06a777f9 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 4 Feb 2023 07:18:01 -0800
Subject: [PATCH 122/310] Use `Ident` to avoid major future refactors

---
 server/src/actions/del.rs                  |   4 +-
 server/src/actions/exists.rs               |   4 +-
 server/src/actions/keylen.rs               |   6 +-
 server/src/actions/lskeys.rs               |   6 +-
 server/src/actions/mpop.rs                 |   4 +-
 server/src/actions/mset.rs                 |  10 +-
 server/src/actions/mupdate.rs              |  10 +-
 server/src/actions/set.rs                  |   6 +-
 server/src/actions/strong/sset.rs          |   4 +-
 server/src/actions/uset.rs                 |  10 +-
 server/src/admin/mksnap.rs                 |  11 +-
 server/src/admin/sys.rs                    |   5 +-
 server/src/engine/macros.rs                |   4 +-
 server/src/engine/ql/ast/mod.rs            |  19 +-
 server/src/engine/ql/benches.rs            |  49 +++--
 server/src/engine/ql/ddl/alt.rs            |  12 +-
 server/src/engine/ql/ddl/crt.rs            |   8 +-
 server/src/engine/ql/ddl/drop.rs           |  10 +-
 server/src/engine/ql/ddl/ins.rs            |   4 +-
 server/src/engine/ql/ddl/syn.rs            |  18 +-
 server/src/engine/ql/dml/ins.rs            |  22 +-
 server/src/engine/ql/dml/mod.rs            |  12 +-
 server/src/engine/ql/dml/sel.rs            |   8 +-
 server/src/engine/ql/dml/upd.rs            |   6 +-
 server/src/engine/ql/lex/mod.rs            |   2 +-
 server/src/engine/ql/lex/raw.rs            |  94 +++++++-
 server/src/engine/ql/tests/dml_tests.rs    | 185 +++++++++-------
 server/src/engine/ql/tests/entity.rs       |   8 +-
 server/src/engine/ql/tests/lexer_tests.rs  |  27 ++-
 server/src/engine/ql/tests/schema_tests.rs | 241 +++++++++++----------
 server/src/tests/mod.rs                    |   2 +-
 sky-bench/src/bench/mod.rs                 |   4 +-
 32 files changed, 481 insertions(+), 334 deletions(-)

diff --git a/server/src/actions/del.rs b/server/src/actions/del.rs
index 617c8152..7019944f 100644
--- a/server/src/actions/del.rs
+++ b/server/src/actions/del.rs
@@ -28,8 +28,8 @@
 //! This module provides functions to work with `DEL` queries
 
 use crate::{
-    corestore::table::DataModel, dbnet::prelude::*,
-    kvengine::encoding::ENCODING_LUT_ITER, util::compiler,
+    corestore::table::DataModel, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER,
+    util::compiler,
 };
 
 action!(
diff --git a/server/src/actions/exists.rs b/server/src/actions/exists.rs
index cf974fab..93238da3 100644
--- a/server/src/actions/exists.rs
+++ b/server/src/actions/exists.rs
@@ -28,8 +28,8 @@
 //! This module provides functions to work with `EXISTS` queries
 
 use crate::{
-    corestore::table::DataModel, dbnet::prelude::*,
-    kvengine::encoding::ENCODING_LUT_ITER, queryengine::ActionIter, util::compiler,
+    corestore::table::DataModel, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER,
+    queryengine::ActionIter, util::compiler,
 };
 
 action!(
diff --git a/server/src/actions/keylen.rs b/server/src/actions/keylen.rs
index d5b87170..d32a5ad3 100644
--- a/server/src/actions/keylen.rs
+++ b/server/src/actions/keylen.rs
@@ -30,7 +30,11 @@ action!(
     /// Run a `KEYLEN` query
     ///
     /// At this moment, `keylen` only supports a single key
-    fn keylen(handle: &crate::corestore::Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
+    fn keylen(
+        handle: &crate::corestore::Corestore,
+        con: &mut Connection<C, P>,
+        mut act: ActionIter<'a>,
+    ) {
         ensure_length::<P>(act.len(), |len| len == 1)?;
         let res: Option<usize> = {
             let reader = handle.get_table_with::<P, KVEBlob>()?;
diff --git a/server/src/actions/lskeys.rs b/server/src/actions/lskeys.rs
index e13c8d6a..647f5256 100644
--- a/server/src/actions/lskeys.rs
+++ b/server/src/actions/lskeys.rs
@@ -33,7 +33,11 @@ const DEFAULT_COUNT: usize = 10;
 
 action!(
     /// Run an `LSKEYS` query
-    fn lskeys(handle: &crate::corestore::Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
+    fn lskeys(
+        handle: &crate::corestore::Corestore,
+        con: &mut Connection<C, P>,
+        mut act: ActionIter<'a>,
+    ) {
         ensure_length::<P>(act.len(), |size| size < 4)?;
         let (table, count) = if act.is_empty() {
             (get_tbl!(handle, con), DEFAULT_COUNT)
diff --git a/server/src/actions/mpop.rs b/server/src/actions/mpop.rs
index 6bd988ad..1f931905 100644
--- a/server/src/actions/mpop.rs
+++ b/server/src/actions/mpop.rs
@@ -25,8 +25,8 @@
 */
 
 use crate::{
-    corestore, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER,
-    queryengine::ActionIter, util::compiler,
+    corestore, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER, queryengine::ActionIter,
+    util::compiler,
 };
 
 action!(
diff --git a/server/src/actions/mset.rs b/server/src/actions/mset.rs
index d4666bdd..36382969 100644
--- a/server/src/actions/mset.rs
+++ b/server/src/actions/mset.rs
@@ -25,13 +25,17 @@
 */
 
 use crate::{
-    corestore::SharedSlice, dbnet::prelude::*,
-    kvengine::encoding::ENCODING_LUT_ITER_PAIR, util::compiler,
+    corestore::SharedSlice, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER_PAIR,
+    util::compiler,
 };
 
 action!(
     /// Run an `MSET` query
-    fn mset(handle: &crate::corestore::Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
+    fn mset(
+        handle: &crate::corestore::Corestore,
+        con: &mut Connection<C, P>,
+        mut act: ActionIter<'a>,
+    ) {
         let howmany = act.len();
         ensure_length::<P>(howmany, |size| size & 1 == 0 && size != 0)?;
         let kve = handle.get_table_with::<P, KVEBlob>()?;
diff --git a/server/src/actions/mupdate.rs b/server/src/actions/mupdate.rs
index bc97b421..9e8472fb 100644
--- a/server/src/actions/mupdate.rs
+++ b/server/src/actions/mupdate.rs
@@ -25,13 +25,17 @@
 */
 
 use crate::{
-    corestore::SharedSlice, dbnet::prelude::*,
-    kvengine::encoding::ENCODING_LUT_ITER_PAIR, util::compiler,
+    corestore::SharedSlice, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER_PAIR,
+    util::compiler,
 };
 
 action!(
     /// Run an `MUPDATE` query
-    fn mupdate(handle: &crate::corestore::Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
+    fn mupdate(
+        handle: &crate::corestore::Corestore,
+        con: &mut Connection<C, P>,
+        mut act: ActionIter<'a>,
+    ) {
         let howmany = act.len();
         ensure_length::<P>(howmany, |size| size & 1 == 0 && size != 0)?;
         let kve = handle.get_table_with::<P, KVEBlob>()?;
diff --git a/server/src/actions/set.rs b/server/src/actions/set.rs
index d4f58cc5..1c1c0c30 100644
--- a/server/src/actions/set.rs
+++ b/server/src/actions/set.rs
@@ -31,7 +31,11 @@ use crate::{corestore::SharedSlice, dbnet::prelude::*, queryengine::ActionIter};
 
 action!(
     /// Run a `SET` query
-    fn set(handle: &crate::corestore::Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
+    fn set(
+        handle: &crate::corestore::Corestore,
+        con: &mut Connection<C, P>,
+        mut act: ActionIter<'a>,
+    ) {
         ensure_length::<P>(act.len(), |len| len == 2)?;
         if registry::state_okay() {
             let did_we = {
diff --git a/server/src/actions/strong/sset.rs b/server/src/actions/strong/sset.rs
index dca70564..825f5679 100644
--- a/server/src/actions/strong/sset.rs
+++ b/server/src/actions/strong/sset.rs
@@ -108,9 +108,7 @@ pub(super) fn snapshot_and_insert<'a, T: 'a + DerefUnsafeSlice>(
             // fine, the keys were non-existent when we looked at them
             while let (Some(key), Some(value)) = (act.next(), act.next()) {
                 unsafe {
-                    if let Some(fresh) =
-                        lowtable.fresh_entry(SharedSlice::new(key.deref_slice()))
-                    {
+                    if let Some(fresh) = lowtable.fresh_entry(SharedSlice::new(key.deref_slice())) {
                         fresh.insert(SharedSlice::new(value.deref_slice()));
                     }
                     // we don't care if some other thread initialized the value we checked
diff --git a/server/src/actions/uset.rs b/server/src/actions/uset.rs
index d881e435..87893f7f 100644
--- a/server/src/actions/uset.rs
+++ b/server/src/actions/uset.rs
@@ -25,15 +25,19 @@
 */
 
 use crate::{
-    corestore::SharedSlice, dbnet::prelude::*,
-    kvengine::encoding::ENCODING_LUT_ITER_PAIR, queryengine::ActionIter, util::compiler,
+    corestore::SharedSlice, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER_PAIR,
+    queryengine::ActionIter, util::compiler,
 };
 
 action!(
     /// Run an `USET` query
     ///
     /// This is like "INSERT or UPDATE"
-    fn uset(handle: &crate::corestore::Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
+    fn uset(
+        handle: &crate::corestore::Corestore,
+        con: &mut Connection<C, P>,
+        mut act: ActionIter<'a>,
+    ) {
         let howmany = act.len();
         ensure_length::<P>(howmany, |size| size & 1 == 0 && size != 0)?;
         let kve = handle.get_table_with::<P, KVEBlob>()?;
diff --git a/server/src/admin/mksnap.rs b/server/src/admin/mksnap.rs
index c4521d24..b0031ad6 100644
--- a/server/src/admin/mksnap.rs
+++ b/server/src/admin/mksnap.rs
@@ -25,10 +25,7 @@
 */
 
 use {
-    crate::{
-        dbnet::prelude::*, kvengine::encoding,
-        storage::v1::sengine::SnapshotActionResult,
-    },
+    crate::{dbnet::prelude::*, kvengine::encoding, storage::v1::sengine::SnapshotActionResult},
     core::str,
     std::path::{Component, PathBuf},
 };
@@ -36,7 +33,11 @@ use {
 action!(
     /// Create a snapshot
     ///
-    fn mksnap(handle: &crate::corestore::Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
+    fn mksnap(
+        handle: &crate::corestore::Corestore,
+        con: &mut Connection<C, P>,
+        mut act: ActionIter<'a>,
+    ) {
         let engine = handle.get_engine();
         if act.is_empty() {
             // traditional mksnap
diff --git a/server/src/admin/sys.rs b/server/src/admin/sys.rs
index d35d0135..d517a120 100644
--- a/server/src/admin/sys.rs
+++ b/server/src/admin/sys.rs
@@ -25,10 +25,7 @@
 */
 
 use {
-    crate::{
-        corestore::booltable::BoolTable, dbnet::prelude::*,
-        storage::v1::interface::DIR_ROOT,
-    },
+    crate::{corestore::booltable::BoolTable, dbnet::prelude::*, storage::v1::interface::DIR_ROOT},
     libsky::VERSION,
 };
 
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index ba58476a..6163e5af 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -116,5 +116,7 @@ macro_rules! dbgfn {
 
 #[allow(unused_macros)]
 macro_rules! void {
-    () => {()};
+    () => {
+        ()
+    };
 }
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 89f91324..994aef2f 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -32,7 +32,7 @@ use {
     self::traits::ASTNode,
     super::{
         ddl, dml,
-        lex::{LitIR, Slice, Token},
+        lex::{Ident, LitIR, Token},
         LangError, LangResult,
     },
     crate::{
@@ -361,7 +361,7 @@ pub enum Entity<'a> {
     /// ```sql
     /// :model
     /// ```
-    Partial(Slice<'a>),
+    Partial(Ident<'a>),
     /// A single entity is used when switching to a model wrt the currently set space (commonly used
     /// when running DML queries)
     ///
@@ -369,7 +369,7 @@ pub enum Entity<'a> {
     /// ```sql
     /// model
     /// ```
-    Single(Slice<'a>),
+    Single(Ident<'a>),
     /// A full entity is a complete definition to a model wrt to the given space (commonly used with
     /// DML queries)
     ///
@@ -377,14 +377,7 @@ pub enum Entity<'a> {
     /// ```sql
     /// space.model
     /// ```
-    Full(Slice<'a>, Slice<'a>),
-}
-
-impl<'a> From<(Slice<'a>, Slice<'a>)> for Entity<'a> {
-    #[inline(always)]
-    fn from((space, model): (Slice<'a>, Slice<'a>)) -> Self {
-        Self::Full(space, model)
-    }
+    Full(Ident<'a>, Ident<'a>),
 }
 
 impl<'a> Entity<'a> {
@@ -543,7 +536,7 @@ pub enum Statement<'a> {
     /// - Space is not in active use
     DropSpace(ddl::drop::DropSpace<'a>),
     /// DDL query to inspect a space (returns a list of models in the space)
-    InspectSpace(Slice<'a>),
+    InspectSpace(Ident<'a>),
     /// DDL query to inspect a model (returns the model definition)
     InspectModel(Entity<'a>),
     /// DDL query to inspect all spaces (returns a list of spaces in the database)
@@ -578,7 +571,7 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], d: Qd) -> LangResult<Sta
             _ => compiler::cold_rerr(LangError::UnknownAlterStatement),
         },
         Token![drop] if state.remaining() >= 2 => ddl::drop::parse_drop(&mut state),
-        Token::Ident(id) if id.eq_ignore_ascii_case(b"inspect") => {
+        Token::Ident(id) if id.eq_ignore_ascii_case("inspect") => {
             ddl::ins::parse_inspect(&mut state)
         }
         // DML
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index b4f0bbeb..ebbe1e24 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -37,7 +37,10 @@
 
 extern crate test;
 
-use {crate::engine::ql::tests::lex_insecure, test::Bencher};
+use {
+    crate::engine::ql::{lex::Ident, tests::lex_insecure},
+    test::Bencher,
+};
 
 mod lexer {
     use {
@@ -85,7 +88,7 @@ mod ast {
     };
     #[bench]
     fn parse_entity_single(b: &mut Bencher) {
-        let e = Entity::Single(b"user");
+        let e = Entity::Single(Ident::from("user"));
         b.iter(|| {
             let src = lex_insecure(b"user").unwrap();
             let mut state = State::new(&src, InplaceData::new());
@@ -96,7 +99,7 @@ mod ast {
     }
     #[bench]
     fn parse_entity_double(b: &mut Bencher) {
-        let e = Entity::Full(b"tweeter", b"user");
+        let e = Entity::Full(Ident::from("tweeter"), Ident::from("user"));
         b.iter(|| {
             let src = lex_insecure(b"tweeter.user").unwrap();
             let mut state = State::new(&src, InplaceData::new());
@@ -107,7 +110,7 @@ mod ast {
     }
     #[bench]
     fn parse_entity_partial(b: &mut Bencher) {
-        let e = Entity::Partial(b"user");
+        let e = Entity::Partial(Ident::from("user"));
         b.iter(|| {
             let src = lex_insecure(b":user").unwrap();
             let mut i = 0;
@@ -130,7 +133,7 @@ mod ddl_queries {
         #[bench]
         fn use_space(b: &mut Bencher) {
             let src = b"use myspace";
-            let expected = Statement::Use(Entity::Single(b"myspace"));
+            let expected = Statement::Use(Entity::Single(Ident::from("myspace")));
             b.iter(|| {
                 let lexed = InsecureLexer::lex(src).unwrap();
                 assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
@@ -139,7 +142,8 @@ mod ddl_queries {
         #[bench]
         fn use_model(b: &mut Bencher) {
             let src = b"use myspace.mymodel";
-            let expected = Statement::Use(Entity::Full(b"myspace", b"mymodel"));
+            let expected =
+                Statement::Use(Entity::Full(Ident::from("myspace"), Ident::from("mymodel")));
             b.iter(|| {
                 let lexed = InsecureLexer::lex(src).unwrap();
                 assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
@@ -151,7 +155,7 @@ mod ddl_queries {
         #[bench]
         fn inspect_space(b: &mut Bencher) {
             let src = b"inspect space myspace";
-            let expected = Statement::InspectSpace(b"myspace");
+            let expected = Statement::InspectSpace(Ident::from("myspace"));
             b.iter(|| {
                 let lexed = InsecureLexer::lex(src).unwrap();
                 assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
@@ -160,7 +164,7 @@ mod ddl_queries {
         #[bench]
         fn inspect_model_single_entity(b: &mut Bencher) {
             let src = b"inspect model mymodel";
-            let expected = Statement::InspectModel(Entity::Single(b"mymodel"));
+            let expected = Statement::InspectModel(Entity::Single(Ident::from("mymodel")));
             b.iter(|| {
                 let lexed = InsecureLexer::lex(src).unwrap();
                 assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
@@ -169,7 +173,10 @@ mod ddl_queries {
         #[bench]
         fn inspect_model_full_entity(b: &mut Bencher) {
             let src = b"inspect model myspace.mymodel";
-            let expected = Statement::InspectModel(Entity::Full(b"myspace", b"mymodel"));
+            let expected = Statement::InspectModel(Entity::Full(
+                Ident::from("myspace"),
+                Ident::from("mymodel"),
+            ));
             b.iter(|| {
                 let lexed = InsecureLexer::lex(src).unwrap();
                 assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
@@ -193,7 +200,7 @@ mod ddl_queries {
         #[bench]
         fn drop_space(b: &mut Bencher) {
             let src = b"drop space myspace";
-            let expected = Statement::DropSpace(DropSpace::new(b"myspace", false));
+            let expected = Statement::DropSpace(DropSpace::new(Ident::from("myspace"), false));
             b.iter(|| {
                 let lexed = InsecureLexer::lex(src).unwrap();
                 assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
@@ -202,7 +209,7 @@ mod ddl_queries {
         #[bench]
         fn drop_space_force(b: &mut Bencher) {
             let src = b"drop space myspace force";
-            let expected = Statement::DropSpace(DropSpace::new(b"myspace", true));
+            let expected = Statement::DropSpace(DropSpace::new(Ident::from("myspace"), true));
             b.iter(|| {
                 let lexed = InsecureLexer::lex(src).unwrap();
                 assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
@@ -211,7 +218,10 @@ mod ddl_queries {
         #[bench]
         fn drop_model_single(b: &mut Bencher) {
             let src = b"drop model mymodel";
-            let expected = Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), false));
+            let expected = Statement::DropModel(DropModel::new(
+                Entity::Single(Ident::from("mymodel")),
+                false,
+            ));
             b.iter(|| {
                 let lexed = InsecureLexer::lex(src).unwrap();
                 assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
@@ -220,7 +230,8 @@ mod ddl_queries {
         #[bench]
         fn drop_model_single_force(b: &mut Bencher) {
             let src = b"drop model mymodel force";
-            let expected = Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), true));
+            let expected =
+                Statement::DropModel(DropModel::new(Entity::Single(Ident::from("mymodel")), true));
             b.iter(|| {
                 let lexed = InsecureLexer::lex(src).unwrap();
                 assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
@@ -229,8 +240,10 @@ mod ddl_queries {
         #[bench]
         fn drop_model_full(b: &mut Bencher) {
             let src = b"drop model myspace.mymodel";
-            let expected =
-                Statement::DropModel(DropModel::new(Entity::Full(b"myspace", b"mymodel"), false));
+            let expected = Statement::DropModel(DropModel::new(
+                Entity::Full(Ident::from("myspace"), Ident::from("mymodel")),
+                false,
+            ));
             b.iter(|| {
                 let lexed = InsecureLexer::lex(src).unwrap();
                 assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
@@ -239,8 +252,10 @@ mod ddl_queries {
         #[bench]
         fn drop_model_full_force(b: &mut Bencher) {
             let src = b"drop model myspace.mymodel force";
-            let expected =
-                Statement::DropModel(DropModel::new(Entity::Full(b"myspace", b"mymodel"), true));
+            let expected = Statement::DropModel(DropModel::new(
+                Entity::Full(Ident::from("myspace"), Ident::from("mymodel")),
+                true,
+            ));
             b.iter(|| {
                 let lexed = InsecureLexer::lex(src).unwrap();
                 assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index ddc30327..8076c975 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -29,7 +29,7 @@ use {
     crate::{
         engine::ql::{
             ast::{QueryData, State},
-            lex::{Slice, Token},
+            lex::{Ident, Token},
             LangError, LangResult,
         },
         util::compiler,
@@ -39,12 +39,12 @@ use {
 #[derive(Debug, PartialEq)]
 /// An alter space query with corresponding data
 pub struct AlterSpace<'a> {
-    space_name: Slice<'a>,
+    space_name: Ident<'a>,
     updated_props: Dict,
 }
 
 impl<'a> AlterSpace<'a> {
-    pub fn new(space_name: Slice<'a>, updated_props: Dict) -> Self {
+    pub fn new(space_name: Ident<'a>, updated_props: Dict) -> Self {
         Self {
             space_name,
             updated_props,
@@ -82,13 +82,13 @@ impl<'a> AlterSpace<'a> {
 
 #[derive(Debug, PartialEq)]
 pub struct AlterModel<'a> {
-    model: Slice<'a>,
+    model: Ident<'a>,
     kind: AlterKind<'a>,
 }
 
 impl<'a> AlterModel<'a> {
     #[inline(always)]
-    pub fn new(model: Slice<'a>, kind: AlterKind<'a>) -> Self {
+    pub fn new(model: Ident<'a>, kind: AlterKind<'a>) -> Self {
         Self { model, kind }
     }
 }
@@ -97,7 +97,7 @@ impl<'a> AlterModel<'a> {
 /// The alter operation kind
 pub enum AlterKind<'a> {
     Add(Box<[ExpandedField<'a>]>),
-    Remove(Box<[Slice<'a>]>),
+    Remove(Box<[Ident<'a>]>),
     Update(Box<[ExpandedField<'a>]>),
 }
 
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index 8cacd4b1..ef13556a 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -29,7 +29,7 @@ use {
     crate::{
         engine::ql::{
             ast::{QueryData, State},
-            lex::{Slice, Token},
+            lex::{Ident, Token},
             LangError, LangResult,
         },
         util::compiler,
@@ -40,7 +40,7 @@ use {
 /// A space
 pub struct CreateSpace<'a> {
     /// the space name
-    pub(super) space_name: Slice<'a>,
+    pub(super) space_name: Ident<'a>,
     /// properties
     pub(super) props: Dict,
 }
@@ -79,7 +79,7 @@ impl<'a> CreateSpace<'a> {
 /// A model definition
 pub struct CreateModel<'a> {
     /// the model name
-    model_name: Slice<'a>,
+    model_name: Ident<'a>,
     /// the fields
     fields: Vec<Field<'a>>,
     /// properties
@@ -94,7 +94,7 @@ pub struct CreateModel<'a> {
 */
 
 impl<'a> CreateModel<'a> {
-    pub fn new(model_name: Slice<'a>, fields: Vec<Field<'a>>, props: Dict) -> Self {
+    pub fn new(model_name: Ident<'a>, fields: Vec<Field<'a>>, props: Dict) -> Self {
         Self {
             model_name,
             fields,
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index d7d401c2..e81f4f79 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -26,28 +26,28 @@
 
 use crate::engine::ql::{
     ast::{Entity, QueryData, State, Statement},
-    lex::{Slice, Token},
+    lex::{Ident, Token},
     LangError, LangResult,
 };
 
 #[derive(Debug, PartialEq)]
 /// A generic representation of `drop` query
 pub struct DropSpace<'a> {
-    pub(super) space: Slice<'a>,
+    pub(super) space: Ident<'a>,
     pub(super) force: bool,
 }
 
 impl<'a> DropSpace<'a> {
     #[inline(always)]
     /// Instantiate
-    pub const fn new(space: Slice<'a>, force: bool) -> Self {
+    pub const fn new(space: Ident<'a>, force: bool) -> Self {
         Self { space, force }
     }
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<DropSpace<'a>> {
         if state.cursor_is_ident() {
             let ident = state.fw_read();
             // should we force drop?
-            let force = state.cursor_rounded_eq(Token::Ident(b"force"));
+            let force = state.cursor_rounded_eq(Token::Ident(Ident::from("force")));
             state.cursor_ahead_if(force);
             // either `force` or nothing
             if state.exhausted() {
@@ -77,7 +77,7 @@ impl<'a> DropModel<'a> {
     }
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
         let e = Entity::attempt_process_entity_result(state)?;
-        let force = state.cursor_rounded_eq(Token::Ident(b"force"));
+        let force = state.cursor_rounded_eq(Token::Ident(Ident::from("force")));
         state.cursor_ahead_if(force);
         if state.exhausted() {
             return Ok(DropModel::new(e, force));
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
index b49c45b9..47b02075 100644
--- a/server/src/engine/ql/ddl/ins.rs
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -53,10 +53,10 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
         Token![space] if state.cursor_has_ident_rounded() => {
             Ok(Statement::InspectSpace(unsafe {
                 // UNSAFE(@ohsayan): Safe because of the match predicate
-                extract!(state.fw_read(), Token::Ident(ref space) => space)
+                extract!(state.fw_read(), Token::Ident(ref space) => *space)
             }))
         }
-        Token::Ident(id) if id.eq_ignore_ascii_case(b"spaces") && state.exhausted() => {
+        Token::Ident(id) if id.eq_ignore_ascii_case("spaces") && state.exhausted() => {
             Ok(Statement::InspectSpaces)
         }
         _ => {
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index b821bc5e..911c303c 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -48,12 +48,12 @@ use {
     crate::{
         engine::ql::{
             ast::{QueryData, State},
-            lex::{LitIR, LitIROwned, Slice, Token},
+            lex::{Ident, LitIR, LitIROwned, Token},
             LangError, LangResult,
         },
         util::{compiler, MaybeInit},
     },
-    std::{collections::HashMap, str},
+    std::collections::HashMap,
 };
 
 #[derive(Debug, PartialEq)]
@@ -163,7 +163,7 @@ where
                 break;
             }
             (Token::Ident(id), DictFoldState::CB_OR_IDENT) => {
-                key = MaybeInit::new(unsafe { str::from_utf8_unchecked(id) });
+                key = MaybeInit::new(*id);
                 // found a key, now expect colon
                 mstate = DictFoldState::COLON;
             }
@@ -241,13 +241,13 @@ pub(super) fn rfold_tymeta<'a, Qd: QueryData<'a>>(
 #[derive(Debug, PartialEq)]
 /// A layer contains a type and corresponding metadata
 pub struct Layer<'a> {
-    ty: Slice<'a>,
+    ty: Ident<'a>,
     props: Dict,
 }
 
 impl<'a> Layer<'a> {
     //// Create a new layer
-    pub const fn new(ty: Slice<'a>, props: Dict) -> Self {
+    pub const fn new(ty: Ident<'a>, props: Dict) -> Self {
         Self { ty, props }
     }
 }
@@ -332,7 +332,7 @@ fn rfold_layers<'a, Qd: QueryData<'a>>(
 /// A field definition
 pub struct Field<'a> {
     /// the field name
-    field_name: Slice<'a>,
+    field_name: Ident<'a>,
     /// layers
     layers: Vec<Layer<'a>>,
     /// is null
@@ -342,7 +342,7 @@ pub struct Field<'a> {
 }
 
 impl<'a> Field<'a> {
-    pub fn new(field_name: Slice<'a>, layers: Vec<Layer<'a>>, null: bool, primary: bool) -> Self {
+    pub fn new(field_name: Ident<'a>, layers: Vec<Layer<'a>>, null: bool, primary: bool) -> Self {
         Self {
             field_name,
             layers,
@@ -386,13 +386,13 @@ impl<'a> Field<'a> {
 #[derive(Debug, PartialEq)]
 /// An [`ExpandedField`] is a full field definition with advanced metadata
 pub struct ExpandedField<'a> {
-    field_name: Slice<'a>,
+    field_name: Ident<'a>,
     layers: Vec<Layer<'a>>,
     props: Dict,
 }
 
 impl<'a> ExpandedField<'a> {
-    pub fn new(field_name: Slice<'a>, layers: Vec<Layer<'a>>, props: Dict) -> Self {
+    pub fn new(field_name: Ident<'a>, layers: Vec<Layer<'a>>, props: Dict) -> Self {
         Self {
             field_name,
             layers,
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index 65084f38..f25ea5e4 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -31,7 +31,7 @@ use {
             core::DataType,
             ql::{
                 ast::{Entity, QueryData, State},
-                lex::Token,
+                lex::{Ident, Token},
                 LangError, LangResult,
             },
         },
@@ -115,7 +115,8 @@ fn hashp(key: &[u8]) -> u32 {
         % PRODUCER_G.len() as u32
 }
 #[inline(always)]
-fn ldfunc(func: &[u8]) -> Option<ProducerFn> {
+fn ldfunc(func: Ident<'_>) -> Option<ProducerFn> {
+    let func = func.as_bytes();
     let ph = hashp(func) as usize;
     let min = cmp::min(ph, PRODUCER_F.len() - 1);
     let data = PRODUCER_F[min as usize];
@@ -126,7 +127,7 @@ fn ldfunc(func: &[u8]) -> Option<ProducerFn> {
     }
 }
 #[inline(always)]
-fn ldfunc_exists(func: &[u8]) -> bool {
+fn ldfunc_exists(func: Ident<'_>) -> bool {
     ldfunc(func).is_some()
 }
 #[inline(always)]
@@ -258,7 +259,7 @@ pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
 /// Panics if tt is empty
 pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
     state: &mut State<'a, Qd>,
-) -> HashMap<&'a [u8], Option<DataType>> {
+) -> HashMap<Ident<'a>, Option<DataType>> {
     let mut stop = state.cursor_eq(Token![close {}]);
     state.cursor_ahead_if(stop);
     let mut data = HashMap::with_capacity(2);
@@ -313,7 +314,7 @@ pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
 #[derive(Debug, PartialEq)]
 pub enum InsertData<'a> {
     Ordered(Vec<Option<DataType>>),
-    Map(HashMap<&'a [u8], Option<DataType>>),
+    Map(HashMap<Ident<'a>, Option<DataType>>),
 }
 
 impl<'a> From<Vec<Option<DataType>>> for InsertData<'a> {
@@ -322,8 +323,8 @@ impl<'a> From<Vec<Option<DataType>>> for InsertData<'a> {
     }
 }
 
-impl<'a> From<HashMap<&'static [u8], Option<DataType>>> for InsertData<'a> {
-    fn from(m: HashMap<&'static [u8], Option<DataType>>) -> Self {
+impl<'a> From<HashMap<Ident<'static>, Option<DataType>>> for InsertData<'a> {
+    fn from(m: HashMap<Ident<'static>, Option<DataType>>) -> Self {
         Self::Map(m)
     }
 }
@@ -445,12 +446,7 @@ mod impls {
                 let r = parse_data_map_syntax(state);
                 Ok(Self(
                     r.into_iter()
-                        .map(|(ident, val)| {
-                            (
-                                String::from_utf8_lossy(ident).to_string().into_boxed_str(),
-                                val,
-                            )
-                        })
+                        .map(|(ident, val)| (ident.to_string().into_boxed_str(), val))
                         .collect(),
                 ))
             }
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index c0e7cce5..5fd7996d 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -37,15 +37,15 @@ pub mod upd;
 use {
     super::{
         ast::{QueryData, State},
-        lex::{LitIR, Token},
+        lex::{Ident, LitIR, Token},
     },
     crate::util::compiler,
     std::collections::HashMap,
 };
 
 #[inline(always)]
-unsafe fn read_ident<'a>(tok: &'a Token<'a>) -> &'a [u8] {
-    extract!(tok, Token::Ident(ref tok) => tok)
+unsafe fn read_ident<'a>(tok: &'a Token<'a>) -> Ident<'a> {
+    extract!(tok, Token::Ident(ref tok) => *tok)
 }
 
 #[inline(always)]
@@ -63,14 +63,14 @@ fn u(b: bool) -> u8 {
 
 #[derive(Debug, PartialEq)]
 pub struct RelationalExpr<'a> {
-    pub(super) lhs: &'a [u8],
+    pub(super) lhs: Ident<'a>,
     pub(super) rhs: LitIR<'a>,
     pub(super) opc: u8,
 }
 
 impl<'a> RelationalExpr<'a> {
     #[inline(always)]
-    pub(super) fn new(lhs: &'a [u8], rhs: LitIR<'a>, opc: u8) -> RelationalExpr<'a> {
+    pub(super) fn new(lhs: Ident<'a>, rhs: LitIR<'a>, opc: u8) -> RelationalExpr<'a> {
         Self { lhs, rhs, opc }
     }
     pub(super) const OP_EQ: u8 = 1;
@@ -123,7 +123,7 @@ pub struct WhereClause<'a> {
     c: WhereClauseCollection<'a>,
 }
 
-type WhereClauseCollection<'a> = HashMap<&'a [u8], RelationalExpr<'a>>;
+type WhereClauseCollection<'a> = HashMap<Ident<'a>, RelationalExpr<'a>>;
 
 impl<'a> WhereClause<'a> {
     #[inline(always)]
diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs
index ecf6fdd3..21da799a 100644
--- a/server/src/engine/ql/dml/sel.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -29,7 +29,7 @@ use {
     crate::{
         engine::ql::{
             ast::{Entity, QueryData, State},
-            lex::Token,
+            lex::{Ident, Token},
             LangError, LangResult,
         },
         util::{compiler, MaybeInit},
@@ -45,7 +45,7 @@ pub struct SelectStatement<'a> {
     /// the entity
     pub(super) entity: Entity<'a>,
     /// fields in order of querying. will be zero when wildcard is set
-    pub(super) fields: Vec<&'a [u8]>,
+    pub(super) fields: Vec<Ident<'a>>,
     /// whether a wildcard was passed
     pub(super) wildcard: bool,
     /// where clause
@@ -56,7 +56,7 @@ impl<'a> SelectStatement<'a> {
     #[inline(always)]
     pub(crate) fn new_test(
         entity: Entity<'a>,
-        fields: Vec<&'a [u8]>,
+        fields: Vec<Ident<'a>>,
         wildcard: bool,
         clauses: WhereClauseCollection<'a>,
     ) -> SelectStatement<'a> {
@@ -65,7 +65,7 @@ impl<'a> SelectStatement<'a> {
     #[inline(always)]
     fn new(
         entity: Entity<'a>,
-        fields: Vec<&'a [u8]>,
+        fields: Vec<Ident<'a>>,
         wildcard: bool,
         clauses: WhereClauseCollection<'a>,
     ) -> SelectStatement<'a> {
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index c41f6d5b..c2d71396 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -31,7 +31,7 @@ use {
     crate::{
         engine::ql::{
             ast::{Entity, QueryData, State},
-            lex::LitIR,
+            lex::{Ident, LitIR},
             LangError, LangResult,
         },
         util::{compiler, MaybeInit},
@@ -64,7 +64,7 @@ static OPERATOR: [Operator; 6] = [
 #[derive(Debug, PartialEq)]
 pub struct AssignmentExpression<'a> {
     /// the LHS ident
-    pub(super) lhs: &'a [u8],
+    pub(super) lhs: Ident<'a>,
     /// the RHS lit
     pub(super) rhs: LitIR<'a>,
     /// operator
@@ -72,7 +72,7 @@ pub struct AssignmentExpression<'a> {
 }
 
 impl<'a> AssignmentExpression<'a> {
-    pub fn new(lhs: &'a [u8], rhs: LitIR<'a>, operator_fn: Operator) -> Self {
+    pub fn new(lhs: Ident<'a>, rhs: LitIR<'a>, operator_fn: Operator) -> Self {
         Self {
             lhs,
             rhs,
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index ebf417a9..2786c254 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -33,7 +33,7 @@ use {
     core::{cmp, fmt, ops::BitOr, slice, str},
 };
 
-pub use self::raw::{Keyword, Lit, LitIR, LitIROwned, Symbol, Token};
+pub use self::raw::{Ident, Keyword, Lit, LitIR, LitIROwned, Symbol, Token};
 pub type Slice<'a> = &'a [u8];
 
 /*
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index cd5911be..2e0c1648 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -24,13 +24,96 @@
  *
 */
 
-use {super::Slice, crate::engine::ql::LangError, core::slice};
+use {
+    super::Slice,
+    crate::engine::ql::LangError,
+    core::{borrow::Borrow, fmt, ops::Deref, slice, str},
+};
+
+#[repr(transparent)]
+#[derive(PartialEq, Eq, Clone, Copy, Hash)]
+pub struct Ident<'a>(&'a [u8]);
+impl<'a> Ident<'a> {
+    pub const unsafe fn new(v: &'a [u8]) -> Self {
+        Self(v)
+    }
+    pub const fn new_str(v: &'a str) -> Self {
+        Self(v.as_bytes())
+    }
+    pub fn as_slice(&self) -> &'a [u8] {
+        self.0
+    }
+    pub fn as_str(&self) -> &'a str {
+        unsafe { str::from_utf8_unchecked(self.0) }
+    }
+}
+impl<'a> fmt::Debug for Ident<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(self.as_str())
+    }
+}
+impl<'a> Deref for Ident<'a> {
+    type Target = str;
+    fn deref(&self) -> &Self::Target {
+        self.as_str()
+    }
+}
+impl<'a> PartialEq<[u8]> for Ident<'a> {
+    fn eq(&self, other: &[u8]) -> bool {
+        self.0 == other
+    }
+}
+impl<'a> PartialEq<Ident<'a>> for [u8] {
+    fn eq(&self, other: &Ident<'a>) -> bool {
+        self == other.as_bytes()
+    }
+}
+impl<'a> PartialEq<str> for Ident<'a> {
+    fn eq(&self, other: &str) -> bool {
+        self.0 == other.as_bytes()
+    }
+}
+impl<'a> PartialEq<Ident<'a>> for str {
+    fn eq(&self, other: &Ident<'a>) -> bool {
+        self == other.as_str()
+    }
+}
+impl<'a> From<&'a str> for Ident<'a> {
+    fn from(s: &'a str) -> Self {
+        Self::new_str(s)
+    }
+}
+impl<'a> AsRef<[u8]> for Ident<'a> {
+    fn as_ref(&self) -> &'a [u8] {
+        self.0
+    }
+}
+impl<'a> AsRef<str> for Ident<'a> {
+    fn as_ref(&self) -> &'a str {
+        self.as_str()
+    }
+}
+impl<'a> Default for Ident<'a> {
+    fn default() -> Self {
+        Self::new_str("")
+    }
+}
+impl<'a> Borrow<[u8]> for Ident<'a> {
+    fn borrow(&self) -> &[u8] {
+        self.0
+    }
+}
+impl<'a> Borrow<str> for Ident<'a> {
+    fn borrow(&self) -> &'a str {
+        self.as_str()
+    }
+}
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum Token<'a> {
     Symbol(Symbol),
     Keyword(Keyword),
-    Ident(Slice<'a>),
+    Ident(Ident<'a>),
     #[cfg(test)]
     /// A comma that can be ignored (used for fuzzing)
     IgnorableComma,
@@ -42,7 +125,7 @@ impl<'a> ToString for Token<'a> {
         match self {
             Self::Symbol(s) => s.to_string(),
             Self::Keyword(k) => k.to_string(),
-            Self::Ident(id) => String::from_utf8_lossy(id).to_string(),
+            Self::Ident(id) => id.to_string(),
             Self::Lit(l) => l.to_string(),
             #[cfg(test)]
             Self::IgnorableComma => "[IGNORE_COMMA]".to_owned(),
@@ -489,7 +572,10 @@ impl<'a> RawLexer<'a> {
             Some(kw) => self.tokens.push(kw.into()),
             // FIXME(@ohsayan): Uh, mind fixing this? The only advantage is that I can keep the graph *memory* footprint small
             None if st == b"true" || st == b"false" => self.push_token(Lit::Bool(st == b"true")),
-            None => self.tokens.push(Token::Ident(s)),
+            None => self.tokens.push(unsafe {
+                // UNSAFE(@ohsayan): scan_ident only returns a valid ident which is always a string
+                Token::Ident(Ident::new(s))
+            }),
         }
     }
     #[inline(always)]
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 6c62fd32..eb654de0 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -397,6 +397,7 @@ mod stmt_insert {
         crate::engine::ql::{
             ast::{parse_ast_node_full, Entity},
             dml::{self, ins::InsertStatement},
+            lex::Ident,
         },
     };
 
@@ -410,7 +411,7 @@ mod stmt_insert {
         .unwrap();
         let r = parse_ast_node_full::<InsertStatement>(&x[1..]).unwrap();
         let e = InsertStatement::new(
-            Entity::Full(b"twitter", b"users"),
+            Entity::Full(Ident::from("twitter"), Ident::from("users")),
             into_array_nullable!["sayan"].to_vec().into(),
         );
         assert_eq!(e, r);
@@ -432,7 +433,7 @@ mod stmt_insert {
         .unwrap();
         let r = parse_ast_node_full::<InsertStatement>(&x[1..]).unwrap();
         let e = InsertStatement::new(
-            Entity::Full(b"twitter", b"users"),
+            Entity::Full(Ident::from("twitter"), Ident::from("users")),
             into_array_nullable!["sayan", "Sayan", "sayan@example.com", true, 12345, 67890]
                 .to_vec()
                 .into(),
@@ -459,7 +460,7 @@ mod stmt_insert {
         .unwrap();
         let r = parse_ast_node_full::<InsertStatement>(&x[1..]).unwrap();
         let e = InsertStatement::new(
-            Entity::Full(b"twitter", b"users"),
+            Entity::Full(Ident::from("twitter"), Ident::from("users")),
             into_array_nullable![
                 "sayan",
                 "Sayan",
@@ -486,9 +487,9 @@ mod stmt_insert {
         .unwrap();
         let r = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let e = InsertStatement::new(
-            Entity::Full(b"jotsy", b"app"),
+            Entity::Full(Ident::from("jotsy"), Ident::from("app")),
             dict_nullable! {
-                "username".as_bytes() => "sayan"
+                Ident::from("username") => "sayan"
             }
             .into(),
         );
@@ -511,14 +512,14 @@ mod stmt_insert {
         .unwrap();
         let r = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let e = InsertStatement::new(
-            Entity::Full(b"jotsy", b"app"),
+            Entity::Full(Ident::from("jotsy"), Ident::from("app")),
             dict_nullable! {
-                "username".as_bytes() => "sayan",
-                "name".as_bytes() => "Sayan",
-                "email".as_bytes() => "sayan@example.com",
-                "verified".as_bytes() => true,
-                "following".as_bytes() => 12345,
-                "followers".as_bytes() => 67890
+                Ident::from("username") => "sayan",
+                Ident::from("name") => "Sayan",
+                Ident::from("email") => "sayan@example.com",
+                Ident::from("verified") => true,
+                Ident::from("following") => 12345,
+                Ident::from("followers") => 67890
             }
             .into(),
         );
@@ -544,17 +545,17 @@ mod stmt_insert {
         .unwrap();
         let r = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let e = InsertStatement::new(
-            Entity::Full(b"jotsy", b"app"),
+            Entity::Full(Ident::from("jotsy"), Ident::from("app")),
             dict_nullable! {
-                "username".as_bytes() => "sayan",
-                "password".as_bytes() => "pass123",
-                "email".as_bytes() => "sayan@example.com",
-                "verified".as_bytes() => true,
-                "following".as_bytes() => 12345,
-                "followers".as_bytes() => 67890,
-                "linked_smart_devices".as_bytes() => Null,
-                "bookmarks".as_bytes() => 12345,
-                "other_linked_accounts".as_bytes() => Null
+                Ident::from("username") => "sayan",
+                "password" => "pass123",
+                "email" => "sayan@example.com",
+                "verified" => true,
+                "following" => 12345,
+                "followers" => 67890,
+                "linked_smart_devices" => Null,
+                "bookmarks" => 12345,
+                "other_linked_accounts" => Null
             }
             .into(),
         );
@@ -566,7 +567,7 @@ mod stmt_insert {
             lex_insecure(br#"insert into jotsy.app(@uuidstr(), "sayan", @timesec())"#).unwrap();
         let ret = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let expected = InsertStatement::new(
-            Entity::Full(b"jotsy", b"app"),
+            Entity::Full(Ident::from("jotsy"), Ident::from("app")),
             into_array_nullable![dml::ins::T_UUIDSTR, "sayan", dml::ins::T_TIMESEC]
                 .to_vec()
                 .into(),
@@ -580,11 +581,11 @@ mod stmt_insert {
         ).unwrap();
         let ret = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let expected = InsertStatement::new(
-            Entity::Full(b"jotsy", b"app"),
+            Entity::Full(Ident::from("jotsy"), Ident::from("app")),
             dict_nullable! {
-                "uuid".as_bytes() => dml::ins::T_UUIDSTR,
-                "username".as_bytes() => "sayan",
-                "signup_time".as_bytes() => dml::ins::T_TIMESEC,
+                "uuid" => dml::ins::T_UUIDSTR,
+                Ident::from("username") => "sayan",
+                "signup_time" => dml::ins::T_TIMESEC,
             }
             .into(),
         );
@@ -598,7 +599,7 @@ mod stmt_select {
         crate::engine::ql::{
             ast::{parse_ast_node_full, Entity},
             dml::{sel::SelectStatement, RelationalExpr},
-            lex::LitIR,
+            lex::{Ident, LitIR},
         },
     };
     #[test]
@@ -611,12 +612,12 @@ mod stmt_select {
         .unwrap();
         let r = parse_ast_node_full::<SelectStatement>(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
-            Entity::Single(b"users"),
+            Entity::Single(Ident::from("users")),
             [].to_vec(),
             true,
             dict! {
-                "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(), LitIR::Str("sayan"), RelationalExpr::OP_EQ
+                Ident::from("username") => RelationalExpr::new(
+                    Ident::from("username"), LitIR::Str("sayan"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -632,12 +633,12 @@ mod stmt_select {
         .unwrap();
         let r = parse_ast_node_full::<SelectStatement>(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
-            Entity::Single(b"users"),
-            [b"field1".as_slice()].to_vec(),
+            Entity::Single(Ident::from("users")),
+            [Ident::from("field1")].to_vec(),
             false,
             dict! {
-                "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(), LitIR::Str("sayan"), RelationalExpr::OP_EQ
+                Ident::from("username") => RelationalExpr::new(
+                    Ident::from("username"), LitIR::Str("sayan"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -653,12 +654,12 @@ mod stmt_select {
         .unwrap();
         let r = parse_ast_node_full::<SelectStatement>(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
-            Entity::Full(b"twitter", b"users"),
-            [b"field1".as_slice()].to_vec(),
+            Entity::Full(Ident::from("twitter"), Ident::from("users")),
+            [Ident::from("field1")].to_vec(),
             false,
             dict! {
-                "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(), LitIR::Str("sayan"), RelationalExpr::OP_EQ
+                Ident::from("username") => RelationalExpr::new(
+                    Ident::from("username"), LitIR::Str("sayan"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -674,12 +675,12 @@ mod stmt_select {
         .unwrap();
         let r = parse_ast_node_full::<SelectStatement>(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
-            Entity::Full(b"twitter", b"users"),
-            [b"field1".as_slice(), b"field2".as_slice()].to_vec(),
+            Entity::Full(Ident::from("twitter"), Ident::from("users")),
+            [Ident::from("field1"), Ident::from("field2")].to_vec(),
             false,
             dict! {
-                "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(), LitIR::Str("sayan"), RelationalExpr::OP_EQ
+                Ident::from("username") => RelationalExpr::new(
+                    Ident::from("username"), LitIR::Str("sayan"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -692,7 +693,7 @@ mod expression_tests {
         crate::engine::ql::{
             ast::parse_ast_node_full,
             dml::upd::{AssignmentExpression, Operator},
-            lex::LitIR,
+            lex::{Ident, LitIR},
         },
     };
     #[test]
@@ -701,7 +702,11 @@ mod expression_tests {
         let r = parse_ast_node_full::<AssignmentExpression>(&src).unwrap();
         assert_eq!(
             r,
-            AssignmentExpression::new(b"username", LitIR::Str("sayan"), Operator::Assign)
+            AssignmentExpression::new(
+                Ident::from("username"),
+                LitIR::Str("sayan"),
+                Operator::Assign
+            )
         );
     }
     #[test]
@@ -710,7 +715,11 @@ mod expression_tests {
         let r = parse_ast_node_full::<AssignmentExpression>(&src).unwrap();
         assert_eq!(
             r,
-            AssignmentExpression::new(b"followers", LitIR::UInt(100), Operator::AddAssign)
+            AssignmentExpression::new(
+                Ident::from("followers"),
+                LitIR::UInt(100),
+                Operator::AddAssign
+            )
         );
     }
     #[test]
@@ -719,7 +728,11 @@ mod expression_tests {
         let r = parse_ast_node_full::<AssignmentExpression>(&src).unwrap();
         assert_eq!(
             r,
-            AssignmentExpression::new(b"following", LitIR::UInt(150), Operator::SubAssign)
+            AssignmentExpression::new(
+                Ident::from("following"),
+                LitIR::UInt(150),
+                Operator::SubAssign
+            )
         );
     }
     #[test]
@@ -728,7 +741,11 @@ mod expression_tests {
         let r = parse_ast_node_full::<AssignmentExpression>(&src).unwrap();
         assert_eq!(
             r,
-            AssignmentExpression::new(b"product_qty", LitIR::UInt(2), Operator::MulAssign)
+            AssignmentExpression::new(
+                Ident::from("product_qty"),
+                LitIR::UInt(2),
+                Operator::MulAssign
+            )
         );
     }
     #[test]
@@ -737,7 +754,11 @@ mod expression_tests {
         let r = parse_ast_node_full::<AssignmentExpression>(&src).unwrap();
         assert_eq!(
             r,
-            AssignmentExpression::new(b"image_crop_factor", LitIR::UInt(2), Operator::DivAssign)
+            AssignmentExpression::new(
+                Ident::from("image_crop_factor"),
+                LitIR::UInt(2),
+                Operator::DivAssign
+            )
         );
     }
 }
@@ -750,7 +771,7 @@ mod update_statement {
                 upd::{AssignmentExpression, Operator, UpdateStatement},
                 RelationalExpr, WhereClause,
             },
-            lex::LitIR,
+            lex::{Ident, LitIR},
         },
     };
     #[test]
@@ -763,15 +784,15 @@ mod update_statement {
         .unwrap();
         let r = parse_ast_node_full::<UpdateStatement>(&tok[1..]).unwrap();
         let e = UpdateStatement::new(
-            Entity::Single(b"app"),
+            Entity::Single(Ident::from("app")),
             vec![AssignmentExpression::new(
-                b"notes",
+                Ident::from("notes"),
                 LitIR::Str("this is my new note"),
                 Operator::AddAssign,
             )],
             WhereClause::new(dict! {
-                "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(),
+                Ident::from("username") => RelationalExpr::new(
+                    Ident::from("username"),
                     LitIR::Str("sayan"),
                     RelationalExpr::OP_EQ
                 )
@@ -795,22 +816,22 @@ mod update_statement {
         .unwrap();
         let r = parse_ast_node_full::<UpdateStatement>(&tok[1..]).unwrap();
         let e = UpdateStatement::new(
-            Entity::Full(b"jotsy", b"app"),
+            Entity::Full(Ident::from("jotsy"), Ident::from("app")),
             vec![
                 AssignmentExpression::new(
-                    b"notes",
+                    Ident::from("notes"),
                     LitIR::Str("this is my new note"),
                     Operator::AddAssign,
                 ),
                 AssignmentExpression::new(
-                    b"email",
+                    Ident::from("email"),
                     LitIR::Str("sayan@example.com"),
                     Operator::Assign,
                 ),
             ],
             WhereClause::new(dict! {
-                "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(),
+                Ident::from("username") => RelationalExpr::new(
+                    Ident::from("username"),
                     LitIR::Str("sayan"),
                     RelationalExpr::OP_EQ
                 )
@@ -825,7 +846,7 @@ mod delete_stmt {
         crate::engine::ql::{
             ast::{parse_ast_node_full, Entity},
             dml::{del::DeleteStatement, RelationalExpr},
-            lex::LitIR,
+            lex::{Ident, LitIR},
         },
     };
 
@@ -838,10 +859,10 @@ mod delete_stmt {
         )
         .unwrap();
         let e = DeleteStatement::new_test(
-            Entity::Single(b"users"),
+            Entity::Single(Ident::from("users")),
             dict! {
-                "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(),
+                Ident::from("username") => RelationalExpr::new(
+                    Ident::from("username"),
                     LitIR::Str("sayan"),
                     RelationalExpr::OP_EQ
                 )
@@ -861,10 +882,10 @@ mod delete_stmt {
         )
         .unwrap();
         let e = DeleteStatement::new_test(
-            Entity::Full(b"twitter", b"users"),
+            Entity::Full(Ident::from("twitter"), Ident::from("users")),
             dict! {
-                "username".as_bytes() => RelationalExpr::new(
-                    "username".as_bytes(),
+                Ident::from("username") => RelationalExpr::new(
+                    Ident::from("username"),
                     LitIR::Str("sayan"),
                     RelationalExpr::OP_EQ
                 )
@@ -879,7 +900,11 @@ mod delete_stmt {
 mod relational_expr {
     use {
         super::*,
-        crate::engine::ql::{ast::parse_ast_node_full, dml::RelationalExpr, lex::LitIR},
+        crate::engine::ql::{
+            ast::parse_ast_node_full,
+            dml::RelationalExpr,
+            lex::{Ident, LitIR},
+        },
     };
 
     #[test]
@@ -890,7 +915,7 @@ mod relational_expr {
             r,
             RelationalExpr {
                 rhs: LitIR::UInt(10),
-                lhs: "primary_key".as_bytes(),
+                lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_EQ
             }
         );
@@ -903,7 +928,7 @@ mod relational_expr {
             r,
             RelationalExpr {
                 rhs: LitIR::UInt(10),
-                lhs: "primary_key".as_bytes(),
+                lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_NE
             }
         );
@@ -916,7 +941,7 @@ mod relational_expr {
             r,
             RelationalExpr {
                 rhs: LitIR::UInt(10),
-                lhs: "primary_key".as_bytes(),
+                lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_GT
             }
         );
@@ -929,7 +954,7 @@ mod relational_expr {
             r,
             RelationalExpr {
                 rhs: LitIR::UInt(10),
-                lhs: "primary_key".as_bytes(),
+                lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_GE
             }
         );
@@ -942,7 +967,7 @@ mod relational_expr {
             r,
             RelationalExpr {
                 rhs: LitIR::UInt(10),
-                lhs: "primary_key".as_bytes(),
+                lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_LT
             }
         );
@@ -954,7 +979,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr::new(
-                "primary_key".as_bytes(),
+                Ident::from("primary_key"),
                 LitIR::UInt(10),
                 RelationalExpr::OP_LE
             )
@@ -967,7 +992,7 @@ mod where_clause {
         crate::engine::ql::{
             ast::parse_ast_node_full,
             dml::{RelationalExpr, WhereClause},
-            lex::LitIR,
+            lex::{Ident, LitIR},
         },
     };
     #[test]
@@ -979,8 +1004,8 @@ mod where_clause {
         )
         .unwrap();
         let expected = WhereClause::new(dict! {
-            "x".as_bytes() => RelationalExpr::new(
-                "x".as_bytes(),
+            Ident::from("x") => RelationalExpr::new(
+                Ident::from("x"),
                 LitIR::UInt(100),
                 RelationalExpr::OP_EQ
             )
@@ -996,13 +1021,13 @@ mod where_clause {
         )
         .unwrap();
         let expected = WhereClause::new(dict! {
-            "userid".as_bytes() => RelationalExpr::new(
-                "userid".as_bytes(),
+            Ident::from("userid") => RelationalExpr::new(
+                Ident::from("userid"),
                 LitIR::UInt(100),
                 RelationalExpr::OP_EQ
             ),
-            "pass".as_bytes() => RelationalExpr::new(
-                "pass".as_bytes(),
+            Ident::from("pass") => RelationalExpr::new(
+                Ident::from("pass"),
                 LitIR::Str("password"),
                 RelationalExpr::OP_EQ
             )
diff --git a/server/src/engine/ql/tests/entity.rs b/server/src/engine/ql/tests/entity.rs
index 4b8699e6..4efbcfa5 100644
--- a/server/src/engine/ql/tests/entity.rs
+++ b/server/src/engine/ql/tests/entity.rs
@@ -25,22 +25,22 @@
 */
 
 use super::*;
-use crate::engine::ql::ast::Entity;
+use crate::engine::ql::{ast::Entity, lex::Ident};
 #[test]
 fn entity_current() {
     let t = lex_insecure(b"hello").unwrap();
     let r = Entity::parse_from_tokens(&t, &mut 0).unwrap();
-    assert_eq!(r, Entity::Single(b"hello"))
+    assert_eq!(r, Entity::Single(Ident::from("hello")))
 }
 #[test]
 fn entity_partial() {
     let t = lex_insecure(b":hello").unwrap();
     let r = Entity::parse_from_tokens(&t, &mut 0).unwrap();
-    assert_eq!(r, Entity::Partial(b"hello"))
+    assert_eq!(r, Entity::Partial(Ident::from("hello")))
 }
 #[test]
 fn entity_full() {
     let t = lex_insecure(b"hello.world").unwrap();
     let r = Entity::parse_from_tokens(&t, &mut 0).unwrap();
-    assert_eq!(r, Entity::Full(b"hello", b"world"))
+    assert_eq!(r, Entity::Full(Ident::from("hello"), Ident::from("world")))
 }
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index 247cf5ad..3ed17504 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -26,7 +26,7 @@
 
 use {
     super::{
-        super::lex::{Lit, Token},
+        super::lex::{Ident, Lit, Token},
         lex_insecure,
     },
     crate::engine::ql::LangError,
@@ -44,7 +44,10 @@ macro_rules! v(
 #[test]
 fn lex_ident() {
     let src = v!("hello");
-    assert_eq!(lex_insecure(&src).unwrap(), vec![Token::Ident(b"hello")]);
+    assert_eq!(
+        lex_insecure(&src).unwrap(),
+        vec![Token::Ident(Ident::from("hello"))]
+    );
 }
 
 // literals
@@ -416,7 +419,7 @@ mod safequery_params {
 }
 
 mod safequery_full_param {
-    use crate::engine::ql::lex::{LitIR, SafeQueryData, Token};
+    use crate::engine::ql::lex::{Ident, LitIR, SafeQueryData, Token};
     #[test]
     fn p_mini() {
         let query = b"select * from myapp where username = ?";
@@ -430,9 +433,9 @@ mod safequery_full_param {
                     Token![select],
                     Token![*],
                     Token![from],
-                    Token::Ident(b"myapp"),
+                    Token::Ident(Ident::from("myapp")),
                     Token![where],
-                    Token::Ident(b"username"),
+                    Token::Ident(Ident::from("username")),
                     Token![=],
                     Token![?]
                 ]
@@ -452,13 +455,13 @@ mod safequery_full_param {
                     Token![select],
                     Token![*],
                     Token![from],
-                    Token::Ident(b"myapp"),
+                    Token::Ident(Ident::from("myapp")),
                     Token![where],
-                    Token::Ident(b"username"),
+                    Token::Ident(Ident::from("username")),
                     Token![=],
                     Token![?],
                     Token![and],
-                    Token::Ident(b"pass"),
+                    Token::Ident(Ident::from("pass")),
                     Token![=],
                     Token![?]
                 ]
@@ -482,19 +485,19 @@ mod safequery_full_param {
                 vec![
                     Token![select],
                     Token![$],
-                    Token::Ident(b"notes"),
+                    Token::Ident(Ident::from("notes")),
                     Token![open []],
                     Token![~],
                     Token![?],
                     Token![close []],
                     Token![from],
-                    Token::Ident(b"myapp"),
+                    Token::Ident(Ident::from("myapp")),
                     Token![where],
-                    Token::Ident(b"username"),
+                    Token::Ident(Ident::from("username")),
                     Token![=],
                     Token![?],
                     Token![and],
-                    Token::Ident(b"pass"),
+                    Token::Ident(Ident::from("pass")),
                     Token![=],
                     Token![?]
                 ]
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 73535c11..4a18fdb7 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -25,7 +25,7 @@
 */
 
 use super::{
-    super::lex::{Lit, Token},
+    super::lex::{Ident, Lit},
     lex_insecure, *,
 };
 mod inspect {
@@ -41,7 +41,7 @@ mod inspect {
         let tok = lex_insecure(b"inspect space myspace").unwrap();
         assert_eq!(
             parse_ast_node_full::<InspectStatementAST>(&tok[1..]).unwrap(),
-            Statement::InspectSpace(b"myspace")
+            Statement::InspectSpace(Ident::from("myspace"))
         );
     }
     #[test]
@@ -49,12 +49,12 @@ mod inspect {
         let tok = lex_insecure(b"inspect model users").unwrap();
         assert_eq!(
             parse_ast_node_full::<InspectStatementAST>(&tok[1..]).unwrap(),
-            Statement::InspectModel(Entity::Single(b"users"))
+            Statement::InspectModel(Entity::Single(Ident::from("users")))
         );
         let tok = lex_insecure(b"inspect model tweeter.users").unwrap();
         assert_eq!(
             parse_ast_node_full::<InspectStatementAST>(&tok[1..]).unwrap(),
-            Statement::InspectModel(Entity::Full(b"tweeter", b"users"))
+            Statement::InspectModel(Entity::Full(Ident::from("tweeter"), Ident::from("users")))
         );
     }
     #[test]
@@ -76,7 +76,7 @@ mod alter_space {
     fn alter_space_mini() {
         let tok = lex_insecure(b"alter model mymodel with {}").unwrap();
         let r = parse_ast_node_full::<AlterSpace>(&tok[2..]).unwrap();
-        assert_eq!(r, AlterSpace::new(b"mymodel", null_dict! {}));
+        assert_eq!(r, AlterSpace::new(Ident::from("mymodel"), null_dict! {}));
     }
     #[test]
     fn alter_space() {
@@ -93,7 +93,7 @@ mod alter_space {
         assert_eq!(
             r,
             AlterSpace::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 null_dict! {
                     "max_entry" => Lit::UnsignedInt(1000),
                     "driver" => Lit::Str("ts-0.8".into())
@@ -142,7 +142,7 @@ mod tymeta {
         let tymeta: DictTypeMeta = ASTNode::from_state(&mut state).unwrap();
         assert_eq!(state.cursor(), 6);
         assert!(Token![:].eq(state.fw_read()));
-        assert!(Token::Ident(b"string").eq(state.fw_read()));
+        assert!(Token::Ident(Ident::from("string")).eq(state.fw_read()));
         assert!(Token![,].eq(state.fw_read()));
         let tymeta2: DictTypeMetaSplit = ASTNode::from_state(&mut state).unwrap();
         assert!(state.exhausted());
@@ -167,7 +167,7 @@ mod tymeta {
         let tymeta: DictTypeMeta = ASTNode::from_state(&mut state).unwrap();
         assert_eq!(state.cursor(), 14);
         assert!(Token![:].eq(state.fw_read()));
-        assert!(Token::Ident(b"string").eq(state.fw_read()));
+        assert!(Token::Ident(Ident::from("string")).eq(state.fw_read()));
         assert!(Token![,].eq(state.fw_read()));
         let tymeta2: DictTypeMetaSplit = ASTNode::from_state(&mut state).unwrap();
         assert!(state.exhausted());
@@ -192,7 +192,10 @@ mod layer {
     fn layer_mini() {
         let tok = lex_insecure(b"string").unwrap();
         let layers = parse_ast_node_multiple_full::<Layer>(&tok).unwrap();
-        assert_eq!(layers, vec![Layer::new(b"string", null_dict! {})]);
+        assert_eq!(
+            layers,
+            vec![Layer::new(Ident::from("string"), null_dict! {})]
+        );
     }
     #[test]
     fn layer() {
@@ -201,7 +204,7 @@ mod layer {
         assert_eq!(
             layers,
             vec![Layer::new(
-                b"string",
+                Ident::from("string"),
                 null_dict! {
                     "maxlen" => Lit::UnsignedInt(100)
                 }
@@ -215,8 +218,8 @@ mod layer {
         assert_eq!(
             layers,
             vec![
-                Layer::new(b"string", null_dict! {}),
-                Layer::new(b"list", null_dict! {})
+                Layer::new(Ident::from("string"), null_dict! {}),
+                Layer::new(Ident::from("list"), null_dict! {})
             ]
         );
     }
@@ -227,9 +230,9 @@ mod layer {
         assert_eq!(
             layers,
             vec![
-                Layer::new(b"string", null_dict! {}),
+                Layer::new(Ident::from("string"), null_dict! {}),
                 Layer::new(
-                    b"list",
+                    Ident::from("list"),
                     null_dict! {
                         "unique" => Lit::Bool(true),
                         "maxlen" => Lit::UnsignedInt(10),
@@ -249,14 +252,14 @@ mod layer {
             layers,
             vec![
                 Layer::new(
-                    b"string",
+                    Ident::from("string"),
                     null_dict! {
                         "ascii_only" => Lit::Bool(true),
                         "maxlen" => Lit::UnsignedInt(255)
                     }
                 ),
                 Layer::new(
-                    b"list",
+                    Ident::from("list"),
                     null_dict! {
                         "unique" => Lit::Bool(true),
                         "maxlen" => Lit::UnsignedInt(10),
@@ -278,14 +281,14 @@ mod layer {
             }
         ";
         let expected = vec![
-            Layer::new(b"string", null_dict!()),
+            Layer::new(Ident::from("string"), null_dict!()),
             Layer::new(
-                b"list",
+                Ident::from("list"),
                 null_dict! {
                     "maxlen" => Lit::UnsignedInt(100),
                 },
             ),
-            Layer::new(b"list", null_dict!("unique" => Lit::Bool(true))),
+            Layer::new(Ident::from("list"), null_dict!("unique" => Lit::Bool(true))),
         ];
         fuzz_tokens(tok.as_slice(), |should_pass, new_tok| {
             let layers = parse_ast_node_multiple_full::<Layer>(&new_tok);
@@ -303,6 +306,7 @@ mod fields {
         crate::engine::ql::{
             ast::parse_ast_node_full,
             ddl::syn::{Field, Layer},
+            lex::Ident,
         },
     };
     #[test]
@@ -312,8 +316,8 @@ mod fields {
         assert_eq!(
             f,
             Field::new(
-                b"username",
-                [Layer::new(b"string", null_dict! {})].into(),
+                Ident::from("username"),
+                [Layer::new(Ident::from("string"), null_dict! {})].into(),
                 false,
                 false
             )
@@ -326,8 +330,8 @@ mod fields {
         assert_eq!(
             f,
             Field::new(
-                b"username",
-                [Layer::new(b"string", null_dict! {})].into(),
+                Ident::from("username"),
+                [Layer::new(Ident::from("string"), null_dict! {})].into(),
                 false,
                 true
             )
@@ -348,9 +352,9 @@ mod fields {
         assert_eq!(
             f,
             Field::new(
-                b"username",
+                Ident::from("username"),
                 [Layer::new(
-                    b"string",
+                    Ident::from("string"),
                     null_dict! {
                         "maxlen" => Lit::UnsignedInt(10),
                         "ascii_only" => Lit::Bool(true),
@@ -380,17 +384,17 @@ mod fields {
         assert_eq!(
             f,
             Field::new(
-                b"notes",
+                Ident::from("notes"),
                 [
                     Layer::new(
-                        b"string",
+                        Ident::from("string"),
                         null_dict! {
                             "maxlen" => Lit::UnsignedInt(255),
                             "ascii_only" => Lit::Bool(true),
                         }
                     ),
                     Layer::new(
-                        b"list",
+                        Ident::from("list"),
                         null_dict! {
                             "unique" => Lit::Bool(true)
                         }
@@ -431,17 +435,17 @@ mod schemas {
         assert_eq!(
             model,
             CreateModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 vec![
                     Field::new(
-                        b"username",
-                        vec![Layer::new(b"string", null_dict! {})],
+                        Ident::from("username"),
+                        vec![Layer::new(Ident::from("string"), null_dict! {})],
                         false,
                         true,
                     ),
                     Field::new(
-                        b"password",
-                        vec![Layer::new(b"binary", null_dict! {})],
+                        Ident::from("password"),
+                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
                         false,
                         false,
                     )
@@ -470,23 +474,23 @@ mod schemas {
         assert_eq!(
             model,
             CreateModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 vec![
                     Field::new(
-                        b"username",
-                        vec![Layer::new(b"string", null_dict! {})],
+                        Ident::from("username"),
+                        vec![Layer::new(Ident::from("string"), null_dict! {})],
                         false,
                         true,
                     ),
                     Field::new(
-                        b"password",
-                        vec![Layer::new(b"binary", null_dict! {})],
+                        Ident::from("password"),
+                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
                         false,
                         false,
                     ),
                     Field::new(
-                        b"profile_pic",
-                        vec![Layer::new(b"binary", null_dict! {})],
+                        Ident::from("profile_pic"),
+                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
                         true,
                         false,
                     )
@@ -520,32 +524,32 @@ mod schemas {
         assert_eq!(
             model,
             CreateModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 vec![
                     Field::new(
-                        b"username",
-                        vec![Layer::new(b"string", null_dict! {})],
+                        Ident::from("username"),
+                        vec![Layer::new(Ident::from("string"), null_dict! {})],
                         false,
                         true
                     ),
                     Field::new(
-                        b"password",
-                        vec![Layer::new(b"binary", null_dict! {})],
+                        Ident::from("password"),
+                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
                         false,
                         false
                     ),
                     Field::new(
-                        b"profile_pic",
-                        vec![Layer::new(b"binary", null_dict! {})],
+                        Ident::from("profile_pic"),
+                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
                         true,
                         false
                     ),
                     Field::new(
-                        b"notes",
+                        Ident::from("notes"),
                         vec![
-                            Layer::new(b"string", null_dict! {}),
+                            Layer::new(Ident::from("string"), null_dict! {}),
                             Layer::new(
-                                b"list",
+                                Ident::from("list"),
                                 null_dict! {
                                     "unique" => Lit::Bool(true)
                                 }
@@ -589,32 +593,32 @@ mod schemas {
         assert_eq!(
             model,
             CreateModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 vec![
                     Field::new(
-                        b"username",
-                        vec![Layer::new(b"string", null_dict! {})],
+                        Ident::from("username"),
+                        vec![Layer::new(Ident::from("string"), null_dict! {})],
                         false,
                         true
                     ),
                     Field::new(
-                        b"password",
-                        vec![Layer::new(b"binary", null_dict! {})],
+                        Ident::from("password"),
+                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
                         false,
                         false
                     ),
                     Field::new(
-                        b"profile_pic",
-                        vec![Layer::new(b"binary", null_dict! {})],
+                        Ident::from("profile_pic"),
+                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
                         true,
                         false
                     ),
                     Field::new(
-                        b"notes",
+                        Ident::from("notes"),
                         vec![
-                            Layer::new(b"string", null_dict! {}),
+                            Layer::new(Ident::from("string"), null_dict! {}),
                             Layer::new(
-                                b"list",
+                                Ident::from("list"),
                                 null_dict! {
                                     "unique" => Lit::Bool(true)
                                 }
@@ -647,8 +651,8 @@ mod dict_field_syntax {
         assert_eq!(
             ef,
             ExpandedField::new(
-                b"username",
-                vec![Layer::new(b"string", null_dict! {})],
+                Ident::from("username"),
+                vec![Layer::new(Ident::from("string"), null_dict! {})],
                 null_dict! {}
             )
         )
@@ -668,8 +672,8 @@ mod dict_field_syntax {
         assert_eq!(
             ef,
             ExpandedField::new(
-                b"username",
-                vec![Layer::new(b"string", null_dict! {})],
+                Ident::from("username"),
+                vec![Layer::new(Ident::from("string"), null_dict! {})],
                 null_dict! {
                     "nullable" => Lit::Bool(false),
                 },
@@ -695,9 +699,9 @@ mod dict_field_syntax {
         assert_eq!(
             ef,
             ExpandedField::new(
-                b"username",
+                Ident::from("username"),
                 vec![Layer::new(
-                    b"string",
+                    Ident::from("string"),
                     null_dict! {
                         "minlen" => Lit::UnsignedInt(6),
                         "maxlen" => Lit::UnsignedInt(255),
@@ -731,16 +735,16 @@ mod dict_field_syntax {
         assert_eq!(
             ef,
             ExpandedField::new(
-                b"notes",
+                Ident::from("notes"),
                 vec![
                     Layer::new(
-                        b"string",
+                        Ident::from("string"),
                         null_dict! {
                             "ascii_only" => Lit::Bool(true),
                         }
                     ),
                     Layer::new(
-                        b"list",
+                        Ident::from("list"),
                         null_dict! {
                             "unique" => Lit::Bool(true),
                         }
@@ -759,6 +763,7 @@ mod alter_model_remove {
     use crate::engine::ql::{
         ast::parse_ast_node_full,
         ddl::alt::{AlterKind, AlterModel},
+        lex::Ident,
     };
     #[test]
     fn alter_mini() {
@@ -767,8 +772,8 @@ mod alter_model_remove {
         assert_eq!(
             remove,
             AlterModel::new(
-                b"mymodel",
-                AlterKind::Remove(Box::from([b"myfield".as_slice()]))
+                Ident::from("mymodel"),
+                AlterKind::Remove(Box::from([Ident::from("myfield")]))
             )
         );
     }
@@ -779,8 +784,8 @@ mod alter_model_remove {
         assert_eq!(
             remove,
             AlterModel::new(
-                b"mymodel",
-                AlterKind::Remove(Box::from([b"myfield".as_slice()]))
+                Ident::from("mymodel"),
+                AlterKind::Remove(Box::from([Ident::from("myfield")]))
             )
         );
     }
@@ -793,12 +798,12 @@ mod alter_model_remove {
         assert_eq!(
             remove,
             AlterModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 AlterKind::Remove(Box::from([
-                    b"myfield1".as_slice(),
-                    b"myfield2".as_slice(),
-                    b"myfield3".as_slice(),
-                    b"myfield4".as_slice(),
+                    Ident::from("myfield1"),
+                    Ident::from("myfield2"),
+                    Ident::from("myfield3"),
+                    Ident::from("myfield4"),
                 ]))
             )
         );
@@ -824,11 +829,11 @@ mod alter_model_add {
         assert_eq!(
             parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap(),
             AlterModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 AlterKind::Add(
                     [ExpandedField::new(
-                        b"myfield",
-                        [Layer::new(b"string", null_dict! {})].into(),
+                        Ident::from("myfield"),
+                        [Layer::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {},
                     )]
                     .into()
@@ -848,11 +853,11 @@ mod alter_model_add {
         assert_eq!(
             r,
             AlterModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 AlterKind::Add(
                     [ExpandedField::new(
-                        b"myfield",
-                        [Layer::new(b"string", null_dict! {})].into(),
+                        Ident::from("myfield"),
+                        [Layer::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {
                             "nullable" => Lit::Bool(true)
                         },
@@ -874,11 +879,11 @@ mod alter_model_add {
         assert_eq!(
             r,
             AlterModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 AlterKind::Add(
                     [ExpandedField::new(
-                        b"myfield",
-                        [Layer::new(b"string", null_dict! {})].into(),
+                        Ident::from("myfield"),
+                        [Layer::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {
                             "nullable" => Lit::Bool(true)
                         },
@@ -914,27 +919,27 @@ mod alter_model_add {
         assert_eq!(
             r,
             AlterModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 AlterKind::Add(
                     [
                         ExpandedField::new(
-                            b"myfield",
-                            [Layer::new(b"string", null_dict! {})].into(),
+                            Ident::from("myfield"),
+                            [Layer::new(Ident::from("string"), null_dict! {})].into(),
                             null_dict! {
                                 "nullable" => Lit::Bool(true)
                             },
                         ),
                         ExpandedField::new(
-                            b"another",
+                            Ident::from("another"),
                             [
                                 Layer::new(
-                                    b"string",
+                                    Ident::from("string"),
                                     null_dict! {
                                         "maxlen" => Lit::UnsignedInt(255)
                                     }
                                 ),
                                 Layer::new(
-                                    b"list",
+                                    Ident::from("list"),
                                     null_dict! {
                                        "unique" => Lit::Bool(true)
                                     },
@@ -974,11 +979,11 @@ mod alter_model_update {
         assert_eq!(
             r,
             AlterModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 AlterKind::Update(
                     [ExpandedField::new(
-                        b"myfield",
-                        [Layer::new(b"string", null_dict! {})].into(),
+                        Ident::from("myfield"),
+                        [Layer::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {},
                     )]
                     .into()
@@ -998,11 +1003,11 @@ mod alter_model_update {
         assert_eq!(
             r,
             AlterModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 AlterKind::Update(
                     [ExpandedField::new(
-                        b"myfield",
-                        [Layer::new(b"string", null_dict! {})].into(),
+                        Ident::from("myfield"),
+                        [Layer::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {},
                     )]
                     .into()
@@ -1027,11 +1032,11 @@ mod alter_model_update {
         assert_eq!(
             r,
             AlterModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 AlterKind::Update(
                     [ExpandedField::new(
-                        b"myfield",
-                        [Layer::new(b"string", null_dict! {})].into(),
+                        Ident::from("myfield"),
+                        [Layer::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {
                             "nullable" => Lit::Bool(true)
                         },
@@ -1061,19 +1066,19 @@ mod alter_model_update {
         assert_eq!(
             r,
             AlterModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 AlterKind::Update(
                     [
                         ExpandedField::new(
-                            b"myfield",
-                            [Layer::new(b"string", null_dict! {})].into(),
+                            Ident::from("myfield"),
+                            [Layer::new(Ident::from("string"), null_dict! {})].into(),
                             null_dict! {
                                 "nullable" => Lit::Bool(true)
                             },
                         ),
                         ExpandedField::new(
-                            b"myfield2",
-                            [Layer::new(b"string", null_dict! {})].into(),
+                            Ident::from("myfield2"),
+                            [Layer::new(Ident::from("string"), null_dict! {})].into(),
                             null_dict! {},
                         )
                     ]
@@ -1104,20 +1109,20 @@ mod alter_model_update {
         assert_eq!(
             r,
             AlterModel::new(
-                b"mymodel",
+                Ident::from("mymodel"),
                 AlterKind::Update(
                     [
                         ExpandedField::new(
-                            b"myfield",
-                            [Layer::new(b"string", null_dict! {})].into(),
+                            Ident::from("myfield"),
+                            [Layer::new(Ident::from("string"), null_dict! {})].into(),
                             null_dict! {
                                 "nullable" => Lit::Bool(true)
                             },
                         ),
                         ExpandedField::new(
-                            b"myfield2",
+                            Ident::from("myfield2"),
                             [Layer::new(
-                                b"string",
+                                Ident::from("string"),
                                 null_dict! {"maxlen" => Lit::UnsignedInt(255)}
                             )]
                             .into(),
@@ -1137,6 +1142,7 @@ mod ddl_other_query_tests {
         crate::engine::ql::{
             ast::{parse_ast_node_full, Entity, Statement},
             ddl::drop::{DropModel, DropSpace, DropStatementAST},
+            lex::Ident,
         },
     };
     #[test]
@@ -1144,7 +1150,7 @@ mod ddl_other_query_tests {
         let src = lex_insecure(br"drop space myspace").unwrap();
         assert_eq!(
             parse_ast_node_full::<DropStatementAST>(&src[1..]).unwrap(),
-            Statement::DropSpace(DropSpace::new(b"myspace", false))
+            Statement::DropSpace(DropSpace::new(Ident::from("myspace"), false))
         );
     }
     #[test]
@@ -1152,7 +1158,7 @@ mod ddl_other_query_tests {
         let src = lex_insecure(br"drop space myspace force").unwrap();
         assert_eq!(
             parse_ast_node_full::<DropStatementAST>(&src[1..]).unwrap(),
-            Statement::DropSpace(DropSpace::new(b"myspace", true))
+            Statement::DropSpace(DropSpace::new(Ident::from("myspace"), true))
         );
     }
     #[test]
@@ -1160,7 +1166,10 @@ mod ddl_other_query_tests {
         let src = lex_insecure(br"drop model mymodel").unwrap();
         assert_eq!(
             parse_ast_node_full::<DropStatementAST>(&src[1..]).unwrap(),
-            Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), false))
+            Statement::DropModel(DropModel::new(
+                Entity::Single(Ident::from("mymodel")),
+                false
+            ))
         );
     }
     #[test]
@@ -1168,7 +1177,7 @@ mod ddl_other_query_tests {
         let src = lex_insecure(br"drop model mymodel force").unwrap();
         assert_eq!(
             parse_ast_node_full::<DropStatementAST>(&src[1..]).unwrap(),
-            Statement::DropModel(DropModel::new(Entity::Single(b"mymodel"), true))
+            Statement::DropModel(DropModel::new(Entity::Single(Ident::from("mymodel")), true))
         );
     }
 }
diff --git a/server/src/tests/mod.rs b/server/src/tests/mod.rs
index b40608df..306bd731 100644
--- a/server/src/tests/mod.rs
+++ b/server/src/tests/mod.rs
@@ -32,13 +32,13 @@ mod macros;
 mod auth;
 mod ddl_tests;
 mod inspect_tests;
+mod issue_tests;
 mod kvengine;
 mod kvengine_encoding;
 mod kvengine_list;
 mod persist;
 mod pipeline;
 mod snapshot;
-mod issue_tests;
 
 mod tls {
     use skytable::{query, Element};
diff --git a/sky-bench/src/bench/mod.rs b/sky-bench/src/bench/mod.rs
index 31fa6f54..77cb2789 100644
--- a/sky-bench/src/bench/mod.rs
+++ b/sky-bench/src/bench/mod.rs
@@ -251,9 +251,7 @@ pub fn run_bench(servercfg: &ServerConfig, matches: ArgMatches) -> BResult<()> {
     binfo!("Finished benchmarks. Cleaning up ...");
     let r: Element = misc_connection.run_query(Query::from("drop model default.tmpbench force"))?;
     if r != Element::RespCode(RespCode::Okay) {
-        return Err(Error::Runtime(
-            "failed to clean up after benchmarks".into(),
-        ));
+        return Err(Error::Runtime("failed to clean up after benchmarks".into()));
     }
 
     if config::should_output_messages() {

From fb2a218ead4608d8cc772b27d53e4eb6fe2d5173 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 4 Feb 2023 07:56:31 -0800
Subject: [PATCH 123/310] Add `UArray` based str

---
 server/src/engine/mem/mod.rs    | 118 ++++++++++++++++++++++++++++++++
 server/src/engine/mem/uarray.rs |  21 ++++++
 server/src/engine/ql/lex/raw.rs |   5 +-
 3 files changed, 143 insertions(+), 1 deletion(-)

diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index d4c545dd..ec8dfbd6 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -29,5 +29,123 @@ mod tests;
 mod uarray;
 mod vinline;
 
+use {
+    crate::engine::ql::lex::Ident,
+    std::{
+        borrow::Borrow,
+        fmt, mem,
+        ops::{Deref, DerefMut},
+    },
+};
+
 pub use uarray::UArray;
 pub use vinline::VInline;
+
+#[derive(PartialEq, Eq, Hash)]
+#[repr(transparent)]
+pub struct AStr<const N: usize> {
+    base: UArray<N, u8>,
+}
+impl<const N: usize> AStr<N> {
+    pub fn try_new(s: &str) -> Option<Self> {
+        if s.len() <= N {
+            Some(unsafe {
+                // UNSAFE(@ohsayan): verified len
+                Self::from_len_unchecked(s)
+            })
+        } else {
+            None
+        }
+    }
+    pub fn new(s: &str) -> Self {
+        Self::try_new(s).expect("length overflow")
+    }
+    pub unsafe fn from_len_unchecked_ident(i: Ident<'_>) -> Self {
+        Self::from_len_unchecked(i.as_str())
+    }
+    pub unsafe fn from_len_unchecked(s: &str) -> Self {
+        Self {
+            base: UArray::from_slice(s.as_bytes()),
+        }
+    }
+    pub unsafe fn from_len_unchecked_bytes(b: &[u8]) -> Self {
+        Self::from_len_unchecked(mem::transmute(b))
+    }
+    pub fn as_str(&self) -> &str {
+        unsafe { mem::transmute(self.base.as_slice()) }
+    }
+    pub fn as_mut_str(&mut self) -> &mut str {
+        unsafe { mem::transmute(self.base.as_slice_mut()) }
+    }
+}
+impl<const N: usize> fmt::Debug for AStr<N> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(self.as_str())
+    }
+}
+impl<const N: usize> Deref for AStr<N> {
+    type Target = str;
+    fn deref(&self) -> &Self::Target {
+        self.as_str()
+    }
+}
+impl<const N: usize> DerefMut for AStr<N> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.as_mut_str()
+    }
+}
+impl<'a, const N: usize> From<Ident<'a>> for AStr<N> {
+    fn from(value: Ident<'a>) -> Self {
+        Self::new(value.as_str())
+    }
+}
+impl<'a, const N: usize> From<&'a str> for AStr<N> {
+    fn from(s: &str) -> Self {
+        Self::new(s)
+    }
+}
+impl<const N: usize> PartialEq<str> for AStr<N> {
+    fn eq(&self, other: &str) -> bool {
+        self.as_str() == other
+    }
+}
+impl<const N: usize> PartialEq<AStr<N>> for str {
+    fn eq(&self, other: &AStr<N>) -> bool {
+        self == other.as_str()
+    }
+}
+impl<const N: usize> PartialEq<[u8]> for AStr<N> {
+    fn eq(&self, other: &[u8]) -> bool {
+        self.as_bytes() == other
+    }
+}
+impl<const N: usize> PartialEq<AStr<N>> for [u8] {
+    fn eq(&self, other: &AStr<N>) -> bool {
+        self == other.as_bytes()
+    }
+}
+impl<const N: usize> AsRef<[u8]> for AStr<N> {
+    fn as_ref(&self) -> &[u8] {
+        self.as_bytes()
+    }
+}
+impl<const N: usize> AsRef<str> for AStr<N> {
+    fn as_ref(&self) -> &str {
+        self.as_str()
+    }
+}
+impl<const N: usize> Default for AStr<N> {
+    fn default() -> Self {
+        Self::new("")
+    }
+}
+impl<const N: usize> Borrow<[u8]> for AStr<N> {
+    fn borrow(&self) -> &[u8] {
+        self.as_bytes()
+    }
+}
+impl<const N: usize> Borrow<str> for AStr<N> {
+    fn borrow(&self) -> &str {
+        self.as_str()
+    }
+}
diff --git a/server/src/engine/mem/uarray.rs b/server/src/engine/mem/uarray.rs
index c4e6933e..0a0ed0a9 100644
--- a/server/src/engine/mem/uarray.rs
+++ b/server/src/engine/mem/uarray.rs
@@ -26,6 +26,7 @@
 
 use core::{
     fmt,
+    hash::{Hash, Hasher},
     iter::FusedIterator,
     mem::MaybeUninit,
     ops::{Deref, DerefMut},
@@ -145,12 +146,26 @@ impl<const N: usize, T> UArray<N, T> {
     }
 }
 
+impl<const N: usize, T: Copy> UArray<N, T> {
+    pub unsafe fn from_slice(s: &[T]) -> Self {
+        debug_assert!(s.len() <= N);
+        let mut new = Self::new();
+        unsafe {
+            ptr::copy_nonoverlapping(s.as_ptr(), new.a.as_mut_ptr() as *mut T, s.len());
+            new.set_len(s.len());
+        }
+        new
+    }
+}
+
 impl<const N: usize, T: Clone> Clone for UArray<N, T> {
     fn clone(&self) -> Self {
         self.iter().cloned().collect()
     }
 }
 
+impl<const N: usize, T: Eq> Eq for UArray<N, T> {}
+
 impl<const M: usize, const N: usize, T: PartialEq> PartialEq<UArray<M, T>> for UArray<N, T> {
     fn eq(&self, other: &UArray<M, T>) -> bool {
         self.as_slice() == other.as_slice()
@@ -201,6 +216,12 @@ impl<const N: usize, T: fmt::Debug> fmt::Debug for UArray<N, T> {
     }
 }
 
+impl<const N: usize, T: Hash> Hash for UArray<N, T> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.as_slice().hash(state)
+    }
+}
+
 pub struct IntoIter<const N: usize, T> {
     i: usize,
     l: usize,
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index 2e0c1648..f151bec6 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -44,7 +44,10 @@ impl<'a> Ident<'a> {
         self.0
     }
     pub fn as_str(&self) -> &'a str {
-        unsafe { str::from_utf8_unchecked(self.0) }
+        unsafe {
+            // UNSAFE(@ohsayan): it's the ctor
+            str::from_utf8_unchecked(self.0)
+        }
     }
 }
 impl<'a> fmt::Debug for Ident<'a> {

From 2f128c2cf2d497c2c099d2eb39e9ddc9a050e173 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 4 Feb 2023 09:21:33 -0800
Subject: [PATCH 124/310] Use `HSData` for dict entries for simplicity

---
 server/src/engine/core/data.rs  | 140 ++++++++++++++++++++++++++++++++
 server/src/engine/core/mod.rs   | 106 ++----------------------
 server/src/engine/ql/ast/mod.rs |  14 ++--
 server/src/engine/ql/ddl/syn.rs |  21 +++--
 server/src/engine/ql/dml/ins.rs |  48 +++++------
 server/src/engine/ql/lex/mod.rs |   2 +-
 server/src/engine/ql/lex/raw.rs |  23 ------
 server/src/engine/ql/tests.rs   |  18 ++--
 8 files changed, 200 insertions(+), 172 deletions(-)
 create mode 100644 server/src/engine/core/data.rs

diff --git a/server/src/engine/core/data.rs b/server/src/engine/core/data.rs
new file mode 100644
index 00000000..247eca24
--- /dev/null
+++ b/server/src/engine/core/data.rs
@@ -0,0 +1,140 @@
+/*
+ * Created on Sat Feb 04 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::ql::lex::{Lit, LitIR};
+
+/// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
+/// be of one type.
+// TODO(@ohsayan): Change the underlying structures, there are just rudimentary ones used during integration with the QL
+#[derive(Debug, PartialEq, Clone)]
+#[repr(u8)]
+pub enum HSData {
+    /// An UTF-8 string
+    String(Box<str>) = DataKind::STR_BX.d(),
+    /// Bytes
+    Binary(Box<[u8]>) = DataKind::BIN_BX.d(),
+    /// An unsigned integer
+    ///
+    /// **NOTE:** This is the default evaluated type for unsigned integers by the query processor. It is the
+    /// responsibility of the executor to ensure integrity checks depending on actual type width in the declared
+    /// schema (if any)
+    UnsignedInt(u64) = DataKind::UINT64.d(),
+    /// A signed integer
+    ///
+    /// **NOTE:** This is the default evaluated type for signed integers by the query processor. It is the
+    /// responsibility of the executor to ensure integrity checks depending on actual type width in the declared
+    /// schema (if any)
+    SignedInt(i64) = DataKind::SINT64.d(),
+    /// A boolean
+    Boolean(bool) = DataKind::BOOL.d(),
+    /// A float (64-bit)
+    Float(f64) = DataKind::FLOAT64.d(),
+    /// A single-type list. Note, you **need** to keep up the invariant that the [`DataType`] disc. remains the same for all
+    /// elements to ensure correctness in this specific context
+    /// FIXME(@ohsayan): Try enforcing this somehow
+    List(Vec<Self>) = DataKind::LIST.d(),
+}
+
+enum_impls! {
+    HSData => {
+        String as String,
+        Vec<u8> as Binary,
+        u64 as UnsignedInt,
+        bool as Boolean,
+        Vec<Self> as List,
+        &'static str as String,
+    }
+}
+
+impl HSData {
+    #[inline(always)]
+    pub(super) fn clone_from_lit(lit: Lit) -> Self {
+        match lit {
+            Lit::Str(s) => HSData::String(s.clone()),
+            Lit::Bool(b) => HSData::Boolean(b),
+            Lit::UnsignedInt(u) => HSData::UnsignedInt(u),
+            Lit::SignedInt(i) => HSData::SignedInt(i),
+            Lit::Bin(l) => HSData::Binary(l.to_vec().into_boxed_slice()),
+        }
+    }
+    #[inline(always)]
+    pub(super) fn clone_from_litir<'a>(lit: LitIR<'a>) -> Self {
+        match lit {
+            LitIR::Str(s) => Self::String(s.to_owned().into_boxed_str()),
+            LitIR::Bin(b) => Self::Binary(b.to_owned().into_boxed_slice()),
+            LitIR::Float(f) => Self::Float(f),
+            LitIR::SInt(s) => Self::SignedInt(s),
+            LitIR::UInt(u) => Self::UnsignedInt(u),
+            LitIR::Bool(b) => Self::Boolean(b),
+        }
+    }
+}
+
+impl<'a> From<Lit<'a>> for HSData {
+    fn from(l: Lit<'a>) -> Self {
+        Self::clone_from_lit(l)
+    }
+}
+
+impl<'a> From<LitIR<'a>> for HSData {
+    fn from(l: LitIR<'a>) -> Self {
+        Self::clone_from_litir(l)
+    }
+}
+
+impl<const N: usize> From<[HSData; N]> for HSData {
+    fn from(f: [HSData; N]) -> Self {
+        Self::List(f.into())
+    }
+}
+
+flags! {
+    #[derive(PartialEq, Eq, Clone, Copy)]
+    pub struct DataKind: u8 {
+        // primitive: integer unsigned
+        UINT8 = 0,
+        UINT16 = 1,
+        UINT32 = 2,
+        UINT64 = 3,
+        // primitive: integer unsigned
+        SINT8 = 4,
+        SINT16 = 5,
+        SINT32 = 6,
+        SINT64 = 7,
+        // primitive: misc
+        BOOL = 8,
+        // primitive: floating point
+        FLOAT32 = 9,
+        FLOAT64 = 10,
+        // compound: flat
+        STR = 11,
+        STR_BX = DataKind::_BASE_HB | DataKind::STR.d(),
+        BIN = 12,
+        BIN_BX = DataKind::_BASE_HB | DataKind::BIN.d(),
+        // compound: recursive
+        LIST = 13,
+    }
+}
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index fae09acf..bce453b0 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -24,106 +24,10 @@
  *
 */
 
-// TODO(@ohsayan): Change the underlying structures, there are just rudimentary ones used during integration with the QL
+mod data;
 
-use super::ql::lex::{Lit, LitIR};
+use super::mem::AStr;
+pub use data::HSData;
 
-/// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
-/// be of one type.
-#[derive(Debug, PartialEq, Clone)]
-#[repr(u8)]
-pub enum DataType {
-    /// An UTF-8 string
-    String(Box<str>) = DataKind::STR_BX.d(),
-    /// Bytes
-    Binary(Box<[u8]>) = DataKind::BIN_BX.d(),
-    /// An unsigned integer
-    ///
-    /// **NOTE:** This is the default evaluated type for unsigned integers by the query processor. It is the
-    /// responsibility of the executor to ensure integrity checks depending on actual type width in the declared
-    /// schema (if any)
-    UnsignedInt(u64) = DataKind::UINT64.d(),
-    /// A signed integer
-    ///
-    /// **NOTE:** This is the default evaluated type for signed integers by the query processor. It is the
-    /// responsibility of the executor to ensure integrity checks depending on actual type width in the declared
-    /// schema (if any)
-    SignedInt(i64) = DataKind::SINT64.d(),
-    /// A boolean
-    Boolean(bool) = DataKind::BOOL.d(),
-    /// A float (64-bit)
-    Float(f64) = DataKind::FLOAT64.d(),
-    /// A single-type list. Note, you **need** to keep up the invariant that the [`DataType`] disc. remains the same for all
-    /// elements to ensure correctness in this specific context
-    /// FIXME(@ohsayan): Try enforcing this somehow
-    List(Vec<Self>) = DataKind::LIST.d(),
-}
-
-enum_impls! {
-    DataType => {
-        String as String,
-        Vec<u8> as Binary,
-        u64 as UnsignedInt,
-        bool as Boolean,
-        Vec<Self> as List,
-        &'static str as String,
-    }
-}
-
-impl DataType {
-    #[inline(always)]
-    pub(super) fn clone_from_lit(lit: &Lit) -> Self {
-        match lit {
-            Lit::Str(s) => DataType::String(s.clone()),
-            Lit::Bool(b) => DataType::Boolean(*b),
-            Lit::UnsignedInt(u) => DataType::UnsignedInt(*u),
-            Lit::SignedInt(i) => DataType::SignedInt(*i),
-            Lit::Bin(l) => DataType::Binary(l.to_vec().into_boxed_slice()),
-        }
-    }
-    #[inline(always)]
-    pub(super) fn clone_from_litir<'a>(lit: LitIR<'a>) -> Self {
-        match lit {
-            LitIR::Str(s) => Self::String(s.to_owned().into_boxed_str()),
-            LitIR::Bin(b) => Self::Binary(b.to_owned().into_boxed_slice()),
-            LitIR::Float(f) => Self::Float(f),
-            LitIR::SInt(s) => Self::SignedInt(s),
-            LitIR::UInt(u) => Self::UnsignedInt(u),
-            LitIR::Bool(b) => Self::Boolean(b),
-        }
-    }
-}
-
-impl<const N: usize> From<[DataType; N]> for DataType {
-    fn from(f: [DataType; N]) -> Self {
-        Self::List(f.into())
-    }
-}
-
-flags! {
-    #[derive(PartialEq, Eq, Clone, Copy)]
-    pub struct DataKind: u8 {
-        // primitive: integer unsigned
-        UINT8 = 0,
-        UINT16 = 1,
-        UINT32 = 2,
-        UINT64 = 3,
-        // primitive: integer unsigned
-        SINT8 = 4,
-        SINT16 = 5,
-        SINT32 = 6,
-        SINT64 = 7,
-        // primitive: misc
-        BOOL = 8,
-        // primitive: floating point
-        FLOAT32 = 9,
-        FLOAT64 = 10,
-        // compound: flat
-        STR = 11,
-        STR_BX = DataKind::_BASE_HB | DataKind::STR.d(),
-        BIN = 12,
-        BIN_BX = DataKind::_BASE_HB | DataKind::BIN.d(),
-        // compound: recursive
-        LIST = 13,
-    }
-}
+const IDENT_MX: usize = 64;
+type ItemID = AStr<IDENT_MX>;
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 994aef2f..c421faf1 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -36,7 +36,7 @@ use {
         LangError, LangResult,
     },
     crate::{
-        engine::core::DataType,
+        engine::core::HSData,
         util::{compiler, MaybeInit},
     },
     core::cmp,
@@ -236,7 +236,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     ///
     /// Caller should have checked that the token matches a lit signature and that enough data is available
     /// in the data source. (ideally should run `can_read_lit_from` or `can_read_lit_rounded`)
-    pub unsafe fn read_lit_into_data_type_unchecked_from(&mut self, tok: &'a Token) -> DataType {
+    pub unsafe fn read_lit_into_data_type_unchecked_from(&mut self, tok: &'a Token) -> HSData {
         self.d.read_data_type(tok)
     }
     #[inline(always)]
@@ -278,7 +278,7 @@ pub trait QueryData<'a> {
     ///
     /// ## Safety
     /// The current token must match the signature of a lit
-    unsafe fn read_data_type(&mut self, tok: &'a Token) -> DataType;
+    unsafe fn read_data_type(&mut self, tok: &'a Token) -> HSData;
     /// Returns true if the data source has enough data
     fn nonzero(&self) -> bool;
 }
@@ -302,8 +302,8 @@ impl<'a> QueryData<'a> for InplaceData {
         extract!(tok, Token::Lit(l) => l.as_ir())
     }
     #[inline(always)]
-    unsafe fn read_data_type(&mut self, tok: &'a Token) -> DataType {
-        DataType::clone_from_lit(extract!(tok, Token::Lit(ref l) => l))
+    unsafe fn read_data_type(&mut self, tok: &'a Token) -> HSData {
+        HSData::from(extract!(tok, Token::Lit(ref l) => l.to_owned()))
     }
     #[inline(always)]
     fn nonzero(&self) -> bool {
@@ -335,11 +335,11 @@ impl<'a> QueryData<'a> for SubstitutedData<'a> {
         ret
     }
     #[inline(always)]
-    unsafe fn read_data_type(&mut self, tok: &'a Token) -> DataType {
+    unsafe fn read_data_type(&mut self, tok: &'a Token) -> HSData {
         debug_assert!(Token![?].eq(tok));
         let ret = self.data[0];
         self.data = &self.data[1..];
-        DataType::clone_from_litir(ret)
+        HSData::from(ret)
     }
     #[inline(always)]
     fn nonzero(&self) -> bool {
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 911c303c..44ebd679 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -46,10 +46,13 @@
 
 use {
     crate::{
-        engine::ql::{
-            ast::{QueryData, State},
-            lex::{Ident, LitIR, LitIROwned, Token},
-            LangError, LangResult,
+        engine::{
+            core::HSData,
+            ql::{
+                ast::{QueryData, State},
+                lex::{Ident, Lit, LitIR, Token},
+                LangError, LangResult,
+            },
         },
         util::{compiler, MaybeInit},
     },
@@ -59,13 +62,19 @@ use {
 #[derive(Debug, PartialEq)]
 /// A dictionary entry type. Either a literal or another dictionary
 pub enum DictEntry {
-    Lit(LitIROwned),
+    Lit(HSData),
     Map(Dict),
 }
 
 impl<'a> From<LitIR<'a>> for DictEntry {
     fn from(l: LitIR<'a>) -> Self {
-        Self::Lit(l.to_litir_owned())
+        Self::Lit(HSData::from(l))
+    }
+}
+
+impl<'a> From<Lit<'a>> for DictEntry {
+    fn from(value: Lit<'a>) -> Self {
+        Self::Lit(HSData::from(value))
     }
 }
 
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index f25ea5e4..8e0a2ac1 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -28,7 +28,7 @@ use {
     super::read_ident,
     crate::{
         engine::{
-            core::DataType,
+            core::HSData,
             ql::{
                 ast::{Entity, QueryData, State},
                 lex::{Ident, Token},
@@ -56,7 +56,7 @@ pub const T_UUIDSTR: &str = "4593264b-0231-43e9-b0aa-50784f14e204";
 pub const T_UUIDBIN: &[u8] = T_UUIDSTR.as_bytes();
 pub const T_TIMESEC: u64 = 1673187839_u64;
 
-type ProducerFn = fn() -> DataType;
+type ProducerFn = fn() -> HSData;
 
 // base
 #[inline(always)]
@@ -77,16 +77,16 @@ fn pfnbase_uuid() -> Uuid {
 }
 // impl
 #[inline(always)]
-fn pfn_timesec() -> DataType {
-    DataType::UnsignedInt(pfnbase_time().as_secs())
+fn pfn_timesec() -> HSData {
+    HSData::UnsignedInt(pfnbase_time().as_secs())
 }
 #[inline(always)]
-fn pfn_uuidstr() -> DataType {
-    DataType::String(pfnbase_uuid().to_string().into_boxed_str())
+fn pfn_uuidstr() -> HSData {
+    HSData::String(pfnbase_uuid().to_string().into_boxed_str())
 }
 #[inline(always)]
-fn pfn_uuidbin() -> DataType {
-    DataType::Binary(pfnbase_uuid().as_bytes().to_vec().into_boxed_slice())
+fn pfn_uuidbin() -> HSData {
+    HSData::Binary(pfnbase_uuid().as_bytes().to_vec().into_boxed_slice())
 }
 
 static PRODUCER_G: [u8; 4] = [0, 2, 3, 0];
@@ -141,8 +141,8 @@ unsafe fn ldfunc_unchecked(func: &[u8]) -> ProducerFn {
 /// - If tt length is less than 1
 pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
     state: &mut State<'a, Qd>,
-    list: &mut Vec<DataType>,
-) -> Option<Discriminant<DataType>> {
+    list: &mut Vec<HSData>,
+) -> Option<Discriminant<HSData>> {
     let mut stop = state.cursor_eq(Token![close []]);
     state.cursor_ahead_if(stop);
     let mut overall_dscr = None;
@@ -169,7 +169,7 @@ pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
                 if prev_nlist_dscr.is_none() && nlist_dscr.is_some() {
                     prev_nlist_dscr = nlist_dscr;
                 }
-                DataType::List(nested_list)
+                HSData::List(nested_list)
             }
             Token![@] if state.cursor_signature_match_fn_arity0_rounded() => match unsafe {
                 // UNSAFE(@ohsayan): Just verified at guard
@@ -202,7 +202,7 @@ pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
 #[inline(always)]
 /// ## Safety
 /// - Cursor must match arity(0) function signature
-unsafe fn handle_func_sub<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> Option<DataType> {
+unsafe fn handle_func_sub<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> Option<HSData> {
     let func = read_ident(state.fw_read());
     state.cursor_ahead_by(2); // skip tt:paren
     ldfunc(func).map(move |f| f())
@@ -212,7 +212,7 @@ unsafe fn handle_func_sub<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> O
 /// - If tt is empty
 pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
     state: &mut State<'a, Qd>,
-) -> Vec<Option<DataType>> {
+) -> Vec<Option<HSData>> {
     let mut stop = state.cursor_eq(Token![() close]);
     state.cursor_ahead_if(stop);
     let mut data = Vec::new();
@@ -259,7 +259,7 @@ pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
 /// Panics if tt is empty
 pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
     state: &mut State<'a, Qd>,
-) -> HashMap<Ident<'a>, Option<DataType>> {
+) -> HashMap<Ident<'a>, Option<HSData>> {
     let mut stop = state.cursor_eq(Token![close {}]);
     state.cursor_ahead_if(stop);
     let mut data = HashMap::with_capacity(2);
@@ -313,18 +313,18 @@ pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
 
 #[derive(Debug, PartialEq)]
 pub enum InsertData<'a> {
-    Ordered(Vec<Option<DataType>>),
-    Map(HashMap<Ident<'a>, Option<DataType>>),
+    Ordered(Vec<Option<HSData>>),
+    Map(HashMap<Ident<'a>, Option<HSData>>),
 }
 
-impl<'a> From<Vec<Option<DataType>>> for InsertData<'a> {
-    fn from(v: Vec<Option<DataType>>) -> Self {
+impl<'a> From<Vec<Option<HSData>>> for InsertData<'a> {
+    fn from(v: Vec<Option<HSData>>) -> Self {
         Self::Ordered(v)
     }
 }
 
-impl<'a> From<HashMap<Ident<'static>, Option<DataType>>> for InsertData<'a> {
-    fn from(m: HashMap<Ident<'static>, Option<DataType>>) -> Self {
+impl<'a> From<HashMap<Ident<'static>, Option<HSData>>> for InsertData<'a> {
+    fn from(m: HashMap<Ident<'static>, Option<HSData>>) -> Self {
         Self::Map(m)
     }
 }
@@ -409,7 +409,7 @@ mod impls {
     pub mod test {
         use {
             super::super::{
-                parse_data_map_syntax, parse_data_tuple_syntax, parse_list, DataType, HashMap,
+                parse_data_map_syntax, parse_data_tuple_syntax, parse_list, HSData, HashMap,
             },
             crate::engine::ql::{
                 ast::{traits::ASTNode, QueryData, State},
@@ -417,7 +417,7 @@ mod impls {
             },
         };
         #[derive(sky_macros::Wrapper, Debug)]
-        pub struct List(Vec<DataType>);
+        pub struct List(Vec<HSData>);
         impl<'a> ASTNode<'a> for List {
             // important: upstream must verify this
             const VERIFY: bool = true;
@@ -428,7 +428,7 @@ mod impls {
             }
         }
         #[derive(sky_macros::Wrapper, Debug)]
-        pub struct DataTuple(Vec<Option<DataType>>);
+        pub struct DataTuple(Vec<Option<HSData>>);
         impl<'a> ASTNode<'a> for DataTuple {
             // important: upstream must verify this
             const VERIFY: bool = true;
@@ -438,7 +438,7 @@ mod impls {
             }
         }
         #[derive(sky_macros::Wrapper, Debug)]
-        pub struct DataMap(HashMap<Box<str>, Option<DataType>>);
+        pub struct DataMap(HashMap<Box<str>, Option<HSData>>);
         impl<'a> ASTNode<'a> for DataMap {
             // important: upstream must verify this
             const VERIFY: bool = true;
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index 2786c254..4daa44c3 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -33,7 +33,7 @@ use {
     core::{cmp, fmt, ops::BitOr, slice, str},
 };
 
-pub use self::raw::{Ident, Keyword, Lit, LitIR, LitIROwned, Symbol, Token};
+pub use self::raw::{Ident, Keyword, Lit, LitIR, Symbol, Token};
 pub type Slice<'a> = &'a [u8];
 
 /*
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index f151bec6..82a897ae 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -419,29 +419,6 @@ pub enum LitIR<'a> {
     Float(f64),
 }
 
-impl<'a> LitIR<'a> {
-    pub fn to_litir_owned(&self) -> LitIROwned {
-        match self {
-            Self::Str(s) => LitIROwned::Str(s.to_string().into_boxed_str()),
-            Self::Bin(b) => LitIROwned::Bin(b.to_vec().into_boxed_slice()),
-            Self::UInt(u) => LitIROwned::UInt(*u),
-            Self::SInt(s) => LitIROwned::SInt(*s),
-            Self::Bool(b) => LitIROwned::Bool(*b),
-            Self::Float(f) => LitIROwned::Float(*f),
-        }
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub enum LitIROwned {
-    Str(Box<str>),
-    Bin(Box<[u8]>),
-    UInt(u64),
-    SInt(i64),
-    Bool(bool),
-    Float(f64),
-}
-
 #[derive(Debug)]
 pub struct RawLexer<'a> {
     c: *const u8,
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index a87c538b..808c7788 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -29,7 +29,7 @@ use {
         lex::{InsecureLexer, SafeLexer, Symbol, Token},
         LangResult,
     },
-    crate::{engine::core::DataType, util::test_utils},
+    crate::{engine::core::HSData, util::test_utils},
     rand::{self, Rng},
 };
 
@@ -54,24 +54,24 @@ pub trait NullableData<T> {
     fn data(self) -> Option<T>;
 }
 
-impl<T> NullableData<DataType> for T
+impl<T> NullableData<HSData> for T
 where
-    T: Into<DataType>,
+    T: Into<HSData>,
 {
-    fn data(self) -> Option<DataType> {
+    fn data(self) -> Option<HSData> {
         Some(self.into())
     }
 }
 
 struct Null;
 
-impl NullableData<DataType> for Null {
-    fn data(self) -> Option<DataType> {
+impl NullableData<HSData> for Null {
+    fn data(self) -> Option<HSData> {
         None
     }
 }
 
-fn nullable_datatype(v: impl NullableData<DataType>) -> Option<DataType> {
+fn nullable_datatype(v: impl NullableData<HSData>) -> Option<HSData> {
     v.data()
 }
 
@@ -87,9 +87,7 @@ impl NullableDictEntry for Null {
 
 impl<'a> NullableDictEntry for super::lex::Lit<'a> {
     fn data(self) -> Option<super::ddl::syn::DictEntry> {
-        Some(super::ddl::syn::DictEntry::Lit(
-            self.as_ir().to_litir_owned(),
-        ))
+        Some(super::ddl::syn::DictEntry::from(self.as_ir()))
     }
 }
 

From 6899a3587047abbb3b938606f35770940603a7a1 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 5 Feb 2023 21:13:46 -0800
Subject: [PATCH 125/310] Use specific error kind for lex

---
 server/src/engine/error.rs                | 49 +++++++++++++++++++++++
 server/src/engine/mod.rs                  |  1 +
 server/src/engine/ql/lex/mod.rs           | 37 +++++++++--------
 server/src/engine/ql/lex/raw.rs           |  8 ++--
 server/src/engine/ql/tests.rs             | 12 +++---
 server/src/engine/ql/tests/lexer_tests.rs |  8 ++--
 6 files changed, 84 insertions(+), 31 deletions(-)
 create mode 100644 server/src/engine/error.rs

diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
new file mode 100644
index 00000000..1c8ace2e
--- /dev/null
+++ b/server/src/engine/error.rs
@@ -0,0 +1,49 @@
+/*
+ * Created on Sat Feb 04 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+pub type LexResult<T> = Result<T, LexError>;
+pub type LangResult<T> = Result<T, LangError>;
+
+#[derive(Debug, PartialEq)]
+pub enum LexError {
+    // insecure lex
+    /// Invalid signed numeric literal
+    InvalidSignedNumericLit,
+    /// Invalid unsigned literal
+    InvalidUnsignedLiteral,
+    /// Invaid binary literal
+    InvalidBinaryLiteral,
+    /// Invalid string literal
+    InvalidStringLiteral,
+    // secure lex
+    /// Dataframe params are invalid
+    BadPframe,
+    // generic
+    /// Unrecognized byte in stream   
+    UnexpectedByte,
+}
+#[derive(Debug, PartialEq)]
+pub enum LangError {}
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index b367bf89..0af63b07 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -29,6 +29,7 @@
 #[macro_use]
 mod macros;
 mod core;
+mod error;
 mod idx;
 mod mem;
 mod ql;
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index 4daa44c3..6cad4d92 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -28,8 +28,10 @@ mod raw;
 
 use {
     self::raw::RawLexer,
-    super::{LangError, LangResult},
-    crate::util::compiler,
+    crate::{
+        engine::error::{LexError, LexResult},
+        util::compiler,
+    },
     core::{cmp, fmt, ops::BitOr, slice, str},
 };
 
@@ -54,7 +56,7 @@ impl<'a> InsecureLexer<'a> {
         }
     }
     #[inline(always)]
-    pub fn lex(src: Slice<'a>) -> LangResult<Vec<Token<'a>>> {
+    pub fn lex(src: Slice<'a>) -> LexResult<Vec<Token<'a>>> {
         let mut slf = Self::new(src);
         slf._lex();
         let RawLexer {
@@ -119,7 +121,7 @@ impl<'a> InsecureLexer<'a> {
                     slf.push_token(Lit::SignedInt(num));
                 }
                 _ => {
-                    compiler::cold_val(slf.set_error(LangError::InvalidNumericLiteral));
+                    compiler::cold_val(slf.set_error(LexError::InvalidSignedNumericLit));
                 }
             }
         } else {
@@ -150,7 +152,7 @@ impl<'a> InsecureLexer<'a> {
                 Ok(num) if compiler::likely(wseof) => {
                     slf.tokens.push(Token::Lit(Lit::UnsignedInt(num)))
                 }
-                _ => slf.set_error(LangError::InvalidNumericLiteral),
+                _ => slf.set_error(LexError::InvalidUnsignedLiteral),
             }
         }
     }
@@ -188,7 +190,7 @@ impl<'a> InsecureLexer<'a> {
                 slf.incr_cursor_by(size);
             }
         } else {
-            slf.set_error(LangError::InvalidSafeLiteral);
+            slf.set_error(LexError::InvalidBinaryLiteral);
         }
     }
     #[inline(always)]
@@ -225,7 +227,7 @@ impl<'a> InsecureLexer<'a> {
             let terminated = slf.peek_eq_and_forward(quote_style);
             match String::from_utf8(buf) {
                 Ok(st) if terminated => slf.tokens.push(Token::Lit(st.into_boxed_str().into())),
-                _ => slf.set_error(LangError::InvalidStringLiteral),
+                _ => slf.set_error(LexError::InvalidStringLiteral),
             }
         }
     }
@@ -245,11 +247,11 @@ impl<'a> SafeLexer<'a> {
         }
     }
     #[inline(always)]
-    pub fn lex(src: Slice<'a>) -> LangResult<Vec<Token>> {
+    pub fn lex(src: Slice<'a>) -> LexResult<Vec<Token>> {
         Self::new(src)._lex()
     }
     #[inline(always)]
-    fn _lex(self) -> LangResult<Vec<Token<'a>>> {
+    fn _lex(self) -> LexResult<Vec<Token<'a>>> {
         let Self { base: mut l } = self;
         while l.not_exhausted() && l.no_error() {
             let b = unsafe { l.deref_cursor() };
@@ -402,20 +404,21 @@ impl<'a> SafeQueryData<'a> {
         Self { p, t }
     }
     #[inline(always)]
-    pub fn parse_data(pf: Slice<'a>, pf_sz: usize) -> LangResult<Box<[LitIR<'a>]>> {
+    pub fn parse_data(pf: Slice<'a>, pf_sz: usize) -> LexResult<Box<[LitIR<'a>]>> {
         Self::p_revloop(pf, pf_sz)
     }
     #[inline(always)]
-    pub fn parse(qf: Slice<'a>, pf: Slice<'a>, pf_sz: usize) -> LangResult<Self> {
+    pub fn parse(qf: Slice<'a>, pf: Slice<'a>, pf_sz: usize) -> LexResult<Self> {
         let q = SafeLexer::lex(qf);
         let p = Self::p_revloop(pf, pf_sz);
-        let (Ok(t), Ok(p)) = (q, p) else {
-            return Err(LangError::UnexpectedChar)
-        };
-        Ok(Self { p, t })
+        match (q, p) {
+            (Ok(q), Ok(p)) => Ok(Self { t: q, p }),
+            // first error
+            (Err(e), _) | (_, Err(e)) => Err(e),
+        }
     }
     #[inline]
-    pub(super) fn p_revloop(mut src: Slice<'a>, size: usize) -> LangResult<Box<[LitIR<'a>]>> {
+    pub(super) fn p_revloop(mut src: Slice<'a>, size: usize) -> LexResult<Box<[LitIR<'a>]>> {
         static LITIR_TF: [for<'a> fn(Slice<'a>, &mut usize, &mut Vec<LitIR<'a>>) -> bool; 7] = [
             SafeQueryData::uint,  // tc: 0
             SafeQueryData::sint,  // tc: 1
@@ -441,7 +444,7 @@ impl<'a> SafeQueryData<'a> {
         if compiler::likely(okay) {
             Ok(data.into_boxed_slice())
         } else {
-            Err(LangError::BadPframe)
+            Err(LexError::BadPframe)
         }
     }
 }
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index 82a897ae..b918145c 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -26,7 +26,7 @@
 
 use {
     super::Slice,
-    crate::engine::ql::LangError,
+    crate::engine::error::LexError,
     core::{borrow::Borrow, fmt, ops::Deref, slice, str},
 };
 
@@ -424,7 +424,7 @@ pub struct RawLexer<'a> {
     c: *const u8,
     e: *const u8,
     pub(super) tokens: Vec<Token<'a>>,
-    pub(super) last_error: Option<LangError>,
+    pub(super) last_error: Option<LexError>,
 }
 
 // ctor
@@ -523,7 +523,7 @@ impl<'a> RawLexer<'a> {
         while self.peek_is_and_forward(|b| b == b' ' || b == b'\t' || b == b'\n') {}
     }
     #[inline(always)]
-    pub(super) fn set_error(&mut self, e: LangError) {
+    pub(super) fn set_error(&mut self, e: LexError) {
         self.last_error = Some(e);
     }
     #[inline(always)]
@@ -562,7 +562,7 @@ impl<'a> RawLexer<'a> {
     pub(super) fn scan_byte(&mut self, byte: u8) {
         match symof(byte) {
             Some(tok) => self.push_token(tok),
-            None => return self.set_error(LangError::UnexpectedChar),
+            None => return self.set_error(LexError::UnexpectedByte),
         }
         unsafe {
             self.incr_cursor();
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 808c7788..27947f4e 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -25,11 +25,11 @@
 */
 
 use {
-    super::{
-        lex::{InsecureLexer, SafeLexer, Symbol, Token},
-        LangResult,
+    super::lex::{InsecureLexer, SafeLexer, Symbol, Token},
+    crate::{
+        engine::{core::HSData, error::LexResult},
+        util::test_utils,
     },
-    crate::{engine::core::HSData, util::test_utils},
     rand::{self, Rng},
 };
 
@@ -41,12 +41,12 @@ mod structure_syn;
 
 #[inline(always)]
 /// Uses the [`InsecureLexer`] to lex the given input
-pub(super) fn lex_insecure(src: &[u8]) -> LangResult<Vec<Token>> {
+pub(super) fn lex_insecure(src: &[u8]) -> LexResult<Vec<Token>> {
     InsecureLexer::lex(src)
 }
 #[inline(always)]
 /// Uses the [`SafeLexer`] to lex the given input
-pub(super) fn lex_secure(src: &[u8]) -> LangResult<Vec<Token>> {
+pub(super) fn lex_secure(src: &[u8]) -> LexResult<Vec<Token>> {
     SafeLexer::lex(src)
 }
 
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index 3ed17504..ad4e5e21 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -29,7 +29,7 @@ use {
         super::lex::{Ident, Lit, Token},
         lex_insecure,
     },
-    crate::engine::ql::LangError,
+    crate::engine::error::LexError,
 };
 
 macro_rules! v(
@@ -141,7 +141,7 @@ fn lex_string_bad_escape() {
     let wth = br#" '\a should be an alert on windows apparently' "#;
     assert_eq!(
         lex_insecure(wth).unwrap_err(),
-        LangError::InvalidStringLiteral
+        LexError::InvalidStringLiteral
     );
 }
 #[test]
@@ -149,12 +149,12 @@ fn lex_string_unclosed() {
     let wth = br#" 'omg where did the end go "#;
     assert_eq!(
         lex_insecure(wth).unwrap_err(),
-        LangError::InvalidStringLiteral
+        LexError::InvalidStringLiteral
     );
     let wth = br#" 'see, we escaped the end\' "#;
     assert_eq!(
         lex_insecure(wth).unwrap_err(),
-        LangError::InvalidStringLiteral
+        LexError::InvalidStringLiteral
     );
 }
 #[test]

From 260dff954bbf6721222a17a5631bbdf53da7619d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 5 Feb 2023 21:46:53 -0800
Subject: [PATCH 126/310] Use specific errors for AST nodes

---
 server/src/engine/error.rs         | 27 +++++++++++++++++++++++++-
 server/src/engine/ql/ast/mod.rs    | 16 ++++++++-------
 server/src/engine/ql/ast/traits.rs | 12 ++++++------
 server/src/engine/ql/ddl/alt.rs    | 31 ++++++++++++++++--------------
 server/src/engine/ql/ddl/crt.rs    | 24 ++++++++++++-----------
 server/src/engine/ql/ddl/drop.rs   | 22 +++++++++++----------
 server/src/engine/ql/ddl/ins.rs    | 21 +++++++++++---------
 server/src/engine/ql/ddl/syn.rs    | 24 +++++++++++------------
 server/src/engine/ql/dml/del.rs    | 16 +++++++--------
 server/src/engine/ql/dml/ins.rs    | 18 ++++++++---------
 server/src/engine/ql/dml/mod.rs    |  8 ++++----
 server/src/engine/ql/dml/sel.rs    | 22 +++++++++++----------
 server/src/engine/ql/dml/upd.rs    | 20 ++++++++++---------
 server/src/engine/ql/mod.rs        | 24 -----------------------
 14 files changed, 151 insertions(+), 134 deletions(-)

diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 1c8ace2e..760eaf3c 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -46,4 +46,29 @@ pub enum LexError {
     UnexpectedByte,
 }
 #[derive(Debug, PartialEq)]
-pub enum LangError {}
+pub enum LangError {
+    // generic
+    /// Unexpected end of syntax
+    UnexpectedEOS,
+    /// Last resort error kind when error specificity is hard to trace
+    BadSyntax,
+    /// Expected a token that defines a statement, found something else
+    ExpectedStatement,
+    // ast nodes: usually parents at heigher hights
+    /// Expected an entity, but found invalid tokens
+    ExpectedEntity,
+    // ast nodes: usually children wrt height
+    /// Bad syn tymeta element
+    SynBadTyMeta,
+    /// Bad syn map element
+    SynBadMap,
+    /// Bad expr: relational
+    ExprBadRel,
+    // ast nodes: usually the root
+    /// Unknown `create` statement
+    StmtUnknownCreate,
+    /// Unknown `alter` statement
+    StmtUnknownAlter,
+    /// unknown `drop` statement
+    StmtUnknownDrop,
+}
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index c421faf1..ade99c9f 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -33,10 +33,12 @@ use {
     super::{
         ddl, dml,
         lex::{Ident, LitIR, Token},
-        LangError, LangResult,
     },
     crate::{
-        engine::core::HSData,
+        engine::{
+            core::HSData,
+            error::{LangError, LangResult},
+        },
         util::{compiler, MaybeInit},
     },
     core::cmp,
@@ -451,7 +453,7 @@ impl<'a> Entity<'a> {
                 *c += 2;
                 Self::partial_entity_from_slice(tok)
             },
-            _ => return Err(LangError::UnexpectedToken),
+            _ => return Err(LangError::ExpectedEntity),
         };
         Ok(r)
     }
@@ -467,7 +469,7 @@ impl<'a> Entity<'a> {
                 Ok(e.assume_init())
             }
         } else {
-            Err(LangError::UnexpectedToken)
+            Err(LangError::ExpectedEntity)
         }
     }
     #[inline(always)]
@@ -554,7 +556,7 @@ pub enum Statement<'a> {
 #[inline(always)]
 pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], d: Qd) -> LangResult<Statement<'a>> {
     if compiler::unlikely(tok.len() < 2) {
-        return Err(LangError::UnexpectedEndofStatement);
+        return Err(LangError::UnexpectedEOS);
     }
     let mut state = State::new(tok, d);
     match state.fw_read() {
@@ -563,12 +565,12 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], d: Qd) -> LangResult<Sta
         Token![create] => match state.fw_read() {
             Token![model] => ASTNode::from_state(&mut state).map(Statement::CreateModel),
             Token![space] => ASTNode::from_state(&mut state).map(Statement::CreateSpace),
-            _ => compiler::cold_rerr(LangError::UnknownCreateStatement),
+            _ => compiler::cold_rerr(LangError::StmtUnknownCreate),
         },
         Token![alter] => match state.fw_read() {
             Token![model] => ASTNode::from_state(&mut state).map(Statement::AlterModel),
             Token![space] => ASTNode::from_state(&mut state).map(Statement::AlterSpace),
-            _ => compiler::cold_rerr(LangError::UnknownAlterStatement),
+            _ => compiler::cold_rerr(LangError::StmtUnknownAlter),
         },
         Token![drop] if state.remaining() >= 2 => ddl::drop::parse_drop(&mut state),
         Token::Ident(id) if id.eq_ignore_ascii_case("inspect") => {
diff --git a/server/src/engine/ql/ast/traits.rs b/server/src/engine/ql/ast/traits.rs
index d787f6aa..b3ba812c 100644
--- a/server/src/engine/ql/ast/traits.rs
+++ b/server/src/engine/ql/ast/traits.rs
@@ -26,9 +26,9 @@
 
 #[cfg(test)]
 use crate::engine::ql::{ast::InplaceData, lex::Token};
-use crate::engine::ql::{
-    ast::{QueryData, State},
-    LangError, LangResult,
+use crate::engine::{
+    error::{LangError, LangResult},
+    ql::ast::{QueryData, State},
 };
 
 /// An AST node
@@ -47,7 +47,7 @@ pub trait ASTNode<'a>: Sized {
             return if state.okay() {
                 r
             } else {
-                Err(LangError::UnexpectedToken)
+                Err(LangError::BadSyntax)
             };
         }
         r
@@ -64,7 +64,7 @@ pub trait ASTNode<'a>: Sized {
             return if state.okay() {
                 r
             } else {
-                Err(LangError::UnexpectedToken)
+                Err(LangError::BadSyntax)
             };
         }
         r
@@ -86,7 +86,7 @@ pub trait ASTNode<'a>: Sized {
         if state.exhausted() && state.okay() {
             r
         } else {
-            Err(super::super::LangError::UnexpectedToken)
+            Err(LangError::BadSyntax)
         }
     }
 }
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 8076c975..0f37e02a 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -27,10 +27,12 @@
 use {
     super::syn::{self, Dict, DictFoldState, ExpandedField},
     crate::{
-        engine::ql::{
-            ast::{QueryData, State},
-            lex::{Ident, Token},
-            LangError, LangResult,
+        engine::{
+            error::{LangError, LangResult},
+            ql::{
+                ast::{QueryData, State},
+                lex::{Ident, Token},
+            },
         },
         util::compiler,
     },
@@ -54,7 +56,7 @@ impl<'a> AlterSpace<'a> {
     /// Parse alter space from tokens
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
         if compiler::unlikely(state.remaining() <= 3) {
-            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+            return compiler::cold_rerr(LangError::UnexpectedEOS);
         }
         let space_name = state.fw_read();
         state.poison_if_not(state.cursor_eq(Token![with]));
@@ -63,7 +65,7 @@ impl<'a> AlterSpace<'a> {
         state.cursor_ahead(); // ignore errors
 
         if compiler::unlikely(!state.okay()) {
-            return Err(LangError::UnexpectedToken);
+            return Err(LangError::BadSyntax);
         }
 
         let space_name = unsafe { extract!(space_name, Token::Ident(ref space) => space.clone()) };
@@ -75,7 +77,7 @@ impl<'a> AlterSpace<'a> {
                 updated_props: d,
             })
         } else {
-            Err(LangError::UnexpectedToken)
+            Err(LangError::SynBadMap)
         }
     }
 }
@@ -107,7 +109,8 @@ impl<'a> AlterModel<'a> {
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
         // alter model mymodel remove x
         if state.remaining() <= 2 || !state.cursor_has_ident_rounded() {
-            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+            return compiler::cold_rerr(LangError::BadSyntax);
+            // FIXME(@ohsayan): bad because no specificity
         }
         let model_name = unsafe { extract!(state.fw_read(), Token::Ident(ref l) => l.clone()) };
         let kind = match state.fw_read() {
@@ -140,7 +143,7 @@ impl<'a> AlterKind<'a> {
             <remove> ::= <ident> | <openparen> (<ident> <comma>)*<closeparen>
         */
         if compiler::unlikely(state.exhausted()) {
-            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+            return compiler::cold_rerr(LangError::UnexpectedEOS);
         }
 
         let r = match state.fw_read() {
@@ -169,10 +172,10 @@ impl<'a> AlterKind<'a> {
                 if state.okay() {
                     cols.into_boxed_slice()
                 } else {
-                    return Err(LangError::UnexpectedToken);
+                    return Err(LangError::BadSyntax);
                 }
             }
-            _ => return Err(LangError::ExpectedStatement),
+            _ => return Err(LangError::BadSyntax),
         };
         Ok(Self::Remove(r))
     }
@@ -181,9 +184,9 @@ impl<'a> AlterKind<'a> {
 mod impls {
     use {
         super::{AlterModel, AlterSpace},
-        crate::engine::ql::{
-            ast::{traits::ASTNode, QueryData, State},
-            LangResult,
+        crate::engine::{
+            error::LangResult,
+            ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for AlterModel<'a> {
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index ef13556a..df5b9584 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -27,10 +27,12 @@
 use {
     super::syn::{self, Dict, DictFoldState, Field},
     crate::{
-        engine::ql::{
-            ast::{QueryData, State},
-            lex::{Ident, Token},
-            LangError, LangResult,
+        engine::{
+            error::{LangError, LangResult},
+            ql::{
+                ast::{QueryData, State},
+                lex::{Ident, Token},
+            },
         },
         util::compiler,
     },
@@ -51,7 +53,7 @@ impl<'a> CreateSpace<'a> {
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
         // smallest declaration: `create space myspace` -> >= 1 token
         if compiler::unlikely(state.remaining() < 1) {
-            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+            return compiler::cold_rerr(LangError::UnexpectedEOS);
         }
         let space_name = state.fw_read();
         state.poison_if_not(space_name.is_ident());
@@ -70,7 +72,7 @@ impl<'a> CreateSpace<'a> {
                 props: d,
             })
         } else {
-            Err(LangError::UnexpectedToken)
+            Err(LangError::BadSyntax)
         }
     }
 }
@@ -104,7 +106,7 @@ impl<'a> CreateModel<'a> {
 
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
         if compiler::unlikely(state.remaining() < 10) {
-            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+            return compiler::cold_rerr(LangError::UnexpectedEOS);
         }
         // field name; ignore errors
         let field_name = state.fw_read();
@@ -138,7 +140,7 @@ impl<'a> CreateModel<'a> {
                 props,
             })
         } else {
-            Err(LangError::UnexpectedToken)
+            Err(LangError::BadSyntax)
         }
     }
 }
@@ -146,9 +148,9 @@ impl<'a> CreateModel<'a> {
 mod impls {
     use {
         super::{CreateModel, CreateSpace},
-        crate::engine::ql::{
-            ast::{traits::ASTNode, QueryData, State},
-            LangResult,
+        crate::engine::{
+            error::LangResult,
+            ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for CreateSpace<'a> {
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index e81f4f79..9f18efd3 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -24,10 +24,12 @@
  *
 */
 
-use crate::engine::ql::{
-    ast::{Entity, QueryData, State, Statement},
-    lex::{Ident, Token},
-    LangError, LangResult,
+use crate::engine::{
+    error::{LangError, LangResult},
+    ql::{
+        ast::{Entity, QueryData, State, Statement},
+        lex::{Ident, Token},
+    },
 };
 
 #[derive(Debug, PartialEq)]
@@ -60,7 +62,7 @@ impl<'a> DropSpace<'a> {
                 ));
             }
         }
-        Err(LangError::UnexpectedToken)
+        Err(LangError::BadSyntax)
     }
 }
 
@@ -82,7 +84,7 @@ impl<'a> DropModel<'a> {
         if state.exhausted() {
             return Ok(DropModel::new(e, force));
         } else {
-            Err(LangError::UnexpectedToken)
+            Err(LangError::BadSyntax)
         }
     }
 }
@@ -95,7 +97,7 @@ pub fn parse_drop<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResul
     match state.fw_read() {
         Token![model] => DropModel::parse(state).map(Statement::DropModel),
         Token![space] => return DropSpace::parse(state).map(Statement::DropSpace),
-        _ => Err(LangError::UnexpectedToken),
+        _ => Err(LangError::StmtUnknownDrop),
     }
 }
 
@@ -103,9 +105,9 @@ pub use impls::DropStatementAST;
 mod impls {
     use {
         super::{DropModel, DropSpace},
-        crate::engine::ql::{
-            ast::{traits::ASTNode, QueryData, State, Statement},
-            LangResult,
+        crate::engine::{
+            error::LangResult,
+            ql::ast::{traits::ASTNode, QueryData, State, Statement},
         },
     };
     impl<'a> ASTNode<'a> for DropModel<'a> {
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
index 47b02075..33e7cc19 100644
--- a/server/src/engine/ql/ddl/ins.rs
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -25,10 +25,12 @@
 */
 
 use crate::{
-    engine::ql::{
-        ast::{Entity, QueryData, State, Statement},
-        lex::Token,
-        LangError, LangResult,
+    engine::{
+        error::{LangError, LangResult},
+        ql::{
+            ast::{Entity, QueryData, State, Statement},
+            lex::Token,
+        },
     },
     util::compiler,
 };
@@ -45,7 +47,7 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
     */
 
     if compiler::unlikely(state.remaining() < 1) {
-        return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+        return compiler::cold_rerr(LangError::UnexpectedEOS);
     }
 
     match state.fw_read() {
@@ -68,13 +70,14 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
 
 pub use impls::InspectStatementAST;
 mod impls {
-    use crate::engine::ql::ast::{traits::ASTNode, QueryData, State, Statement};
+    use crate::engine::{
+        error::LangResult,
+        ql::ast::{traits::ASTNode, QueryData, State, Statement},
+    };
     #[derive(sky_macros::Wrapper, Debug)]
     pub struct InspectStatementAST<'a>(Statement<'a>);
     impl<'a> ASTNode<'a> for InspectStatementAST<'a> {
-        fn _from_state<Qd: QueryData<'a>>(
-            state: &mut State<'a, Qd>,
-        ) -> crate::engine::ql::LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             super::parse_inspect(state).map(Self)
         }
     }
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 44ebd679..65c7ceb6 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -48,10 +48,10 @@ use {
     crate::{
         engine::{
             core::HSData,
+            error::{LangError, LangResult},
             ql::{
                 ast::{QueryData, State},
                 lex::{Ident, Lit, LitIR, Token},
-                LangError, LangResult,
             },
         },
         util::{compiler, MaybeInit},
@@ -362,7 +362,7 @@ impl<'a> Field<'a> {
     pub fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
         if compiler::unlikely(state.remaining() < 2) {
             // smallest field: `ident: type`
-            return Err(LangError::UnexpectedEndofStatement);
+            return Err(LangError::UnexpectedEOS);
         }
         // check if primary or null
         let is_primary = state.cursor_eq(Token![primary]);
@@ -374,7 +374,7 @@ impl<'a> Field<'a> {
         // field name
         let field_name = match (state.fw_read(), state.fw_read()) {
             (Token::Ident(id), Token![:]) => id,
-            _ => return Err(LangError::UnexpectedToken),
+            _ => return Err(LangError::BadSyntax),
         };
         // layers
         let mut layers = Vec::new();
@@ -387,7 +387,7 @@ impl<'a> Field<'a> {
                 primary: is_primary,
             })
         } else {
-            Err(LangError::UnexpectedToken)
+            Err(LangError::SynBadTyMeta)
         }
     }
 }
@@ -413,7 +413,7 @@ impl<'a> ExpandedField<'a> {
     pub(super) fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
         if compiler::unlikely(state.remaining() < 6) {
             // smallest: fieldname { type: ident }
-            return Err(LangError::UnexpectedEndofStatement);
+            return Err(LangError::UnexpectedEOS);
         }
         let field_name = state.fw_read();
         state.poison_if_not(field_name.is_ident());
@@ -426,7 +426,7 @@ impl<'a> ExpandedField<'a> {
             // this has layers. fold them; but don't forget the colon
             if compiler::unlikely(state.exhausted()) {
                 // we need more tokens
-                return Err(LangError::UnexpectedEndofStatement);
+                return Err(LangError::UnexpectedEOS);
             }
             state.poison_if_not(state.cursor_eq(Token![:]));
             state.cursor_ahead();
@@ -450,7 +450,7 @@ impl<'a> ExpandedField<'a> {
                 layers,
             })
         } else {
-            Err(LangError::UnexpectedToken)
+            Err(LangError::BadSyntax)
         }
     }
     #[inline(always)]
@@ -466,7 +466,7 @@ impl<'a> ExpandedField<'a> {
             alter model add myfield { type string }
         */
         if compiler::unlikely(state.remaining() < 5) {
-            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+            return compiler::cold_rerr(LangError::UnexpectedEOS);
         }
         match state.read() {
             Token::Ident(_) => {
@@ -498,7 +498,7 @@ impl<'a> ExpandedField<'a> {
                 if state.okay() {
                     Ok(cols.into_boxed_slice())
                 } else {
-                    Err(LangError::UnexpectedToken)
+                    Err(LangError::BadSyntax)
                 }
             }
             _ => Err(LangError::ExpectedStatement),
@@ -515,9 +515,9 @@ mod impls {
             rfold_dict, rfold_layers, rfold_tymeta, Dict, DictFoldState, ExpandedField, Field,
             Layer, LayerFoldState,
         },
-        crate::engine::ql::{
-            ast::{traits::ASTNode, QueryData, State},
-            LangResult,
+        crate::engine::{
+            error::LangResult,
+            ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for ExpandedField<'a> {
diff --git a/server/src/engine/ql/dml/del.rs b/server/src/engine/ql/dml/del.rs
index 5f52bb5a..48f73778 100644
--- a/server/src/engine/ql/dml/del.rs
+++ b/server/src/engine/ql/dml/del.rs
@@ -29,9 +29,9 @@ use super::WhereClauseCollection;
 use {
     super::WhereClause,
     crate::{
-        engine::ql::{
-            ast::{Entity, QueryData, State},
-            LangError, LangResult,
+        engine::{
+            error::{LangError, LangResult},
+            ql::ast::{Entity, QueryData, State},
         },
         util::{compiler, MaybeInit},
     },
@@ -69,7 +69,7 @@ impl<'a> DeleteStatement<'a> {
                    ^1   ^2    ^3    ^4  ^5
         */
         if compiler::unlikely(state.remaining() < 5) {
-            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+            return compiler::cold_rerr(LangError::UnexpectedEOS);
         }
         // from + entity
         state.poison_if_not(state.cursor_eq(Token![from]));
@@ -89,7 +89,7 @@ impl<'a> DeleteStatement<'a> {
                 wc,
             })
         } else {
-            compiler::cold_rerr(LangError::UnexpectedToken)
+            compiler::cold_rerr(LangError::BadSyntax)
         }
     }
 }
@@ -97,9 +97,9 @@ impl<'a> DeleteStatement<'a> {
 mod impls {
     use {
         super::DeleteStatement,
-        crate::engine::ql::{
-            ast::{traits::ASTNode, QueryData, State},
-            LangResult,
+        crate::engine::{
+            error::LangResult,
+            ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for DeleteStatement<'a> {
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index 8e0a2ac1..5f75b563 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -29,10 +29,10 @@ use {
     crate::{
         engine::{
             core::HSData,
+            error::{LangError, LangResult},
             ql::{
                 ast::{Entity, QueryData, State},
                 lex::{Ident, Token},
-                LangError, LangResult,
             },
         },
         util::{compiler, MaybeInit},
@@ -350,7 +350,7 @@ impl<'a> InsertStatement<'a> {
                    ^1    ^2   ^3      ^4 ^5
         */
         if compiler::unlikely(state.remaining() < 5) {
-            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+            return compiler::cold_rerr(LangError::UnexpectedEOS);
         }
         state.poison_if_not(state.cursor_eq(Token![into]));
         state.cursor_ahead(); // ignore errors
@@ -385,7 +385,7 @@ impl<'a> InsertStatement<'a> {
                 data,
             })
         } else {
-            compiler::cold_rerr(LangError::UnexpectedToken)
+            compiler::cold_rerr(LangError::BadSyntax)
         }
     }
 }
@@ -395,9 +395,9 @@ pub use impls::test::{DataMap, DataTuple, List};
 mod impls {
     use {
         super::InsertStatement,
-        crate::engine::ql::{
-            ast::{traits::ASTNode, QueryData, State},
-            LangResult,
+        crate::engine::{
+            error::LangResult,
+            ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for InsertStatement<'a> {
@@ -411,9 +411,9 @@ mod impls {
             super::super::{
                 parse_data_map_syntax, parse_data_tuple_syntax, parse_list, HSData, HashMap,
             },
-            crate::engine::ql::{
-                ast::{traits::ASTNode, QueryData, State},
-                LangResult,
+            crate::engine::{
+                error::LangResult,
+                ql::ast::{traits::ASTNode, QueryData, State},
             },
         };
         #[derive(sky_macros::Wrapper, Debug)]
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index 5fd7996d..09ba4218 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -161,9 +161,9 @@ impl<'a> WhereClause<'a> {
 mod impls {
     use {
         super::{RelationalExpr, WhereClause},
-        crate::engine::ql::{
-            ast::{traits::ASTNode, QueryData, State},
-            LangError, LangResult,
+        crate::engine::{
+            error::{LangError, LangResult},
+            ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for WhereClause<'a> {
@@ -176,7 +176,7 @@ mod impls {
     }
     impl<'a> ASTNode<'a> for RelationalExpr<'a> {
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-            Self::try_parse(state).ok_or(LangError::UnexpectedToken)
+            Self::try_parse(state).ok_or(LangError::ExprBadRel)
         }
     }
 }
diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs
index 21da799a..729d148a 100644
--- a/server/src/engine/ql/dml/sel.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -27,10 +27,12 @@
 use {
     super::{WhereClause, WhereClauseCollection},
     crate::{
-        engine::ql::{
-            ast::{Entity, QueryData, State},
-            lex::{Ident, Token},
-            LangError, LangResult,
+        engine::{
+            error::{LangError, LangResult},
+            ql::{
+                ast::{Entity, QueryData, State},
+                lex::{Ident, Token},
+            },
         },
         util::{compiler, MaybeInit},
     },
@@ -87,7 +89,7 @@ impl<'a> SelectStatement<'a> {
                    1 2    3
         */
         if compiler::unlikely(state.remaining() < 3) {
-            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+            return compiler::cold_rerr(LangError::UnexpectedEOS);
         }
         let mut select_fields = Vec::new();
         let is_wildcard = state.cursor_eq(Token![*]);
@@ -106,7 +108,7 @@ impl<'a> SelectStatement<'a> {
         state.poison_if_not(is_wildcard | !select_fields.is_empty());
         // we should have from + model
         if compiler::unlikely(state.remaining() < 2 || !state.okay()) {
-            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+            return compiler::cold_rerr(LangError::BadSyntax);
         }
         state.poison_if_not(state.cursor_eq(Token![from]));
         state.cursor_ahead(); // ignore errors
@@ -129,7 +131,7 @@ impl<'a> SelectStatement<'a> {
                 clause: WhereClause::new(clauses),
             })
         } else {
-            compiler::cold_rerr(LangError::UnexpectedToken)
+            compiler::cold_rerr(LangError::BadSyntax)
         }
     }
 }
@@ -137,9 +139,9 @@ impl<'a> SelectStatement<'a> {
 mod impls {
     use {
         super::SelectStatement,
-        crate::engine::ql::{
-            ast::{traits::ASTNode, QueryData, State},
-            LangResult,
+        crate::engine::{
+            error::LangResult,
+            ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for SelectStatement<'a> {
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index c2d71396..8f92f873 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -29,10 +29,12 @@ use super::WhereClauseCollection;
 use {
     super::{read_ident, u, WhereClause},
     crate::{
-        engine::ql::{
-            ast::{Entity, QueryData, State},
-            lex::{Ident, LitIR},
-            LangError, LangResult,
+        engine::{
+            error::{LangError, LangResult},
+            ql::{
+                ast::{Entity, QueryData, State},
+                lex::{Ident, LitIR},
+            },
         },
         util::{compiler, MaybeInit},
     },
@@ -167,7 +169,7 @@ impl<'a> UpdateStatement<'a> {
                    ^1    ^2  ^3 ^4 ^5^6    ^7^8^9
         */
         if compiler::unlikely(state.remaining() < 9) {
-            return compiler::cold_rerr(LangError::UnexpectedEndofStatement);
+            return compiler::cold_rerr(LangError::UnexpectedEOS);
         }
         // parse entity
         let mut entity = MaybeInit::uninit();
@@ -205,7 +207,7 @@ impl<'a> UpdateStatement<'a> {
                 wc: WhereClause::new(clauses),
             })
         } else {
-            compiler::cold_rerr(LangError::UnexpectedToken)
+            compiler::cold_rerr(LangError::BadSyntax)
         }
     }
 }
@@ -213,9 +215,9 @@ impl<'a> UpdateStatement<'a> {
 mod impls {
     use {
         super::UpdateStatement,
-        crate::engine::ql::{
-            ast::{traits::ASTNode, QueryData, State},
-            LangResult,
+        crate::engine::{
+            error::LangResult,
+            ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for UpdateStatement<'a> {
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index e364dde5..674a5757 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -35,27 +35,3 @@ pub(super) mod dml;
 pub(super) mod lex;
 #[cfg(test)]
 mod tests;
-
-/*
-    Lang errors
-*/
-
-pub type LangResult<T> = Result<T, LangError>;
-
-#[derive(Debug, PartialEq)]
-#[repr(u8)]
-pub enum LangError {
-    InvalidNumericLiteral,
-    InvalidStringLiteral,
-    UnexpectedChar,
-    InvalidTypeExpression,
-    ExpectedStatement,
-    UnexpectedEndofStatement,
-    UnexpectedToken,
-    InvalidDictionaryExpression,
-    InvalidTypeDefinition,
-    InvalidSafeLiteral,
-    BadPframe,
-    UnknownCreateStatement,
-    UnknownAlterStatement,
-}

From a33721e0a4992c4bd9c3ed6d9c3ea5a757bebb15 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 8 Feb 2023 06:30:55 -0800
Subject: [PATCH 127/310] Implement basic executor for `create space`

Also:

- Added tests for `ItemID`
- Added tests for `create space` executor
---
 server/src/engine/core/mod.rs                |  26 +++-
 server/src/engine/core/model.rs              |  37 +++++
 server/src/engine/core/space.rs              | 136 +++++++++++++++++++
 server/src/engine/core/tests.rs              |  49 +++++++
 server/src/engine/core/tests/create_space.rs |  99 ++++++++++++++
 server/src/engine/error.rs                   |  14 +-
 server/src/engine/idx/mod.rs                 |  20 +--
 server/src/engine/macros.rs                  |  26 ++++
 server/src/engine/mem/mod.rs                 |  26 +++-
 server/src/engine/mod.rs                     |  12 ++
 server/src/engine/ql/ast/mod.rs              |   7 +-
 server/src/engine/ql/ddl/crt.rs              |   4 +-
 server/src/engine/ql/ddl/mod.rs              |  10 +-
 server/src/engine/ql/ddl/syn.rs              |  16 ++-
 server/src/engine/ql/macros.rs               |   1 +
 server/src/engine/ql/mod.rs                  |   2 +-
 server/src/engine/ql/tests.rs                |   4 +-
 server/src/util/test_utils.rs                |  21 ++-
 18 files changed, 481 insertions(+), 29 deletions(-)
 create mode 100644 server/src/engine/core/model.rs
 create mode 100644 server/src/engine/core/space.rs
 create mode 100644 server/src/engine/core/tests.rs
 create mode 100644 server/src/engine/core/tests/create_space.rs

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index bce453b0..e781e8dd 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -25,9 +25,33 @@
 */
 
 mod data;
+mod model;
+mod space;
+#[cfg(test)]
+mod tests;
 
-use super::mem::AStr;
 pub use data::HSData;
+use {
+    self::space::Space,
+    super::{idx::IndexST, mem::AStr},
+    parking_lot::RwLock,
+    std::sync::Arc,
+};
 
 const IDENT_MX: usize = 64;
 type ItemID = AStr<IDENT_MX>;
+/// Use this for now since it substitutes for a file lock (and those syscalls are expensive),
+/// but something better is in the offing
+type RWLIdx<K, V> = RwLock<IndexST<K, V>>;
+
+// FIXME(@ohsayan): Make sure we update what all structures we're making use of here
+
+struct GlobalNS {
+    spaces: RWLIdx<ItemID, Arc<Space>>,
+}
+
+impl GlobalNS {
+    pub(self) fn _spaces(&self) -> &RWLIdx<ItemID, Arc<Space>> {
+        &self.spaces
+    }
+}
diff --git a/server/src/engine/core/model.rs b/server/src/engine/core/model.rs
new file mode 100644
index 00000000..c0bed173
--- /dev/null
+++ b/server/src/engine/core/model.rs
@@ -0,0 +1,37 @@
+/*
+ * Created on Mon Feb 06 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+// FIXME(@ohsayan): update this!
+
+#[derive(Debug)]
+pub struct ModelNS {}
+
+#[cfg(test)]
+impl PartialEq for ModelNS {
+    fn eq(&self, _: &Self) -> bool {
+        true
+    }
+}
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
new file mode 100644
index 00000000..67db94e7
--- /dev/null
+++ b/server/src/engine/core/space.rs
@@ -0,0 +1,136 @@
+/*
+ * Created on Mon Feb 06 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::engine::{
+        core::{model::ModelNS, ItemID, RWLIdx},
+        error::{DatabaseError, DatabaseResult},
+        idx::{IndexST, STIndex},
+        ql::ddl::{crt::CreateSpace, syn::DictEntry},
+    },
+    std::sync::Arc,
+};
+
+#[derive(Debug)]
+pub struct Space {
+    mns: RWLIdx<ItemID, Arc<ModelNS>>,
+    meta: SpaceMeta,
+}
+
+#[derive(Debug, Default)]
+pub struct SpaceMeta {
+    env: RWLIdx<Box<str>, DictEntry>,
+}
+
+impl SpaceMeta {
+    pub const KEY_ENV: &str = "env";
+    pub fn with_env(env: IndexST<Box<str>, DictEntry>) -> Self {
+        Self {
+            env: RWLIdx::new(env),
+        }
+    }
+}
+
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+pub(super) struct Procedure {
+    space_name: ItemID,
+    space: Space,
+}
+
+impl Procedure {
+    #[inline(always)]
+    pub(super) fn new(space_name: ItemID, space: Space) -> Self {
+        Self { space_name, space }
+    }
+}
+
+impl Space {
+    #[inline(always)]
+    pub fn new(mns: IndexST<ItemID, Arc<ModelNS>>, meta: SpaceMeta) -> Self {
+        Self {
+            mns: RWLIdx::new(mns),
+            meta,
+        }
+    }
+    #[inline]
+    pub(super) fn validate(
+        CreateSpace {
+            space_name,
+            mut props,
+        }: CreateSpace,
+    ) -> DatabaseResult<Procedure> {
+        let space_name = ItemID::try_new(&space_name).ok_or(DatabaseError::SysBadItemID)?;
+        // check env
+        let env;
+        match props.remove(SpaceMeta::KEY_ENV) {
+            Some(Some(DictEntry::Map(m))) if props.is_empty() => env = m,
+            None | Some(None) if props.is_empty() => env = IndexST::default(),
+            _ => {
+                return Err(DatabaseError::DdlCreateSpaceBadProperty);
+            }
+        }
+        Ok(Procedure {
+            space_name,
+            space: Self::new(
+                IndexST::default(),
+                SpaceMeta::with_env(
+                    // FIXME(@ohsayan): see this is bad. attempt to do it at AST build time
+                    env.into_iter()
+                        .filter_map(|(k, v)| v.map(move |v| (k, v)))
+                        .collect(),
+                ),
+            ),
+        })
+    }
+    fn validate_and_apply(gns: &super::GlobalNS, space: CreateSpace) -> DatabaseResult<()> {
+        let Procedure { space_name, space } = Self::validate(space)?;
+        let mut wl = gns._spaces().write();
+        if wl.st_insert(space_name, Arc::new(space)) {
+            Ok(())
+        } else {
+            Err(DatabaseError::DdlCreateSpaceAlreadyExists)
+        }
+    }
+}
+
+#[cfg(test)]
+impl PartialEq for SpaceMeta {
+    fn eq(&self, other: &Self) -> bool {
+        let x = self.env.read();
+        let y = other.env.read();
+        &*x == &*y
+    }
+}
+
+#[cfg(test)]
+impl PartialEq for Space {
+    fn eq(&self, other: &Self) -> bool {
+        let self_mns = self.mns.read();
+        let other_mns = other.mns.read();
+        self.meta == other.meta && &*self_mns == &*other_mns
+    }
+}
diff --git a/server/src/engine/core/tests.rs b/server/src/engine/core/tests.rs
new file mode 100644
index 00000000..91aafaf8
--- /dev/null
+++ b/server/src/engine/core/tests.rs
@@ -0,0 +1,49 @@
+/*
+ * Created on Wed Feb 08 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod create_space;
+
+use {super::ItemID, crate::engine::ql::lex::Ident};
+
+#[test]
+fn item_id_okay() {
+    let _ = ItemID::from(Ident::from("hello"));
+}
+
+#[test]
+fn test_item_id_exact() {
+    let _ = ItemID::from(Ident::from(
+        "Abe76d912c6e205aa05edf974cd21cd48061d86d12d92ac1028e5b90f3132f4e",
+    ));
+}
+
+#[test]
+#[should_panic(expected = "length overflow")]
+fn item_id_too_long() {
+    let _ = ItemID::from(Ident::from(
+        "Abe76d912c6e205aa05edf974cd21cd48061d86d12d92ac1028e5b90f3132f4e_",
+    ));
+}
diff --git a/server/src/engine/core/tests/create_space.rs b/server/src/engine/core/tests/create_space.rs
new file mode 100644
index 00000000..67fca7e6
--- /dev/null
+++ b/server/src/engine/core/tests/create_space.rs
@@ -0,0 +1,99 @@
+/*
+ * Created on Wed Feb 08 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    core::{
+        space::{Procedure, Space, SpaceMeta},
+        HSData,
+    },
+    error::DatabaseError,
+    ql::{
+        ast::{compile_test, Statement},
+        tests::lex_insecure as lex,
+    },
+};
+
+#[test]
+fn create_space_simple() {
+    let tok = lex("create space myspace".as_bytes()).unwrap();
+    let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
+    let procedure = Space::validate(space).unwrap();
+    assert_eq!(
+        procedure,
+        Procedure::new(
+            "myspace".into(),
+            Space::new(Default::default(), SpaceMeta::with_env(into_dict! {}))
+        )
+    );
+}
+
+#[test]
+fn create_space_with_env() {
+    let tok = lex(br#"
+        create space myspace with {
+            env: {
+                MAX_MODELS: 100
+            }
+        }
+    "#)
+    .unwrap();
+    let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
+    let procedure = Space::validate(space).unwrap();
+    assert_eq!(
+        procedure,
+        Procedure::new(
+            "myspace".into(),
+            Space::new(
+                into_dict! {},
+                SpaceMeta::with_env(into_dict! {
+                    "MAX_MODELS" => HSData::UnsignedInt(100)
+                })
+            )
+        )
+    );
+}
+
+#[test]
+fn create_space_with_bad_env_type() {
+    let tok = lex(br#"create space myspace with { env: 100 }"#).unwrap();
+    let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
+    assert_eq!(
+        Space::validate(space).unwrap_err(),
+        DatabaseError::DdlCreateSpaceBadProperty
+    );
+}
+
+#[test]
+fn create_space_with_random_property() {
+    let random_property = "i_am_blue_da_ba_dee";
+    let query = format!("create space myspace with {{ {random_property}: 100 }}").into_bytes();
+    let tok = lex(&query).unwrap();
+    let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
+    assert_eq!(
+        Space::validate(space).unwrap_err(),
+        DatabaseError::DdlCreateSpaceBadProperty
+    );
+}
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 760eaf3c..421994da 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -24,8 +24,9 @@
  *
 */
 
-pub type LexResult<T> = Result<T, LexError>;
 pub type LangResult<T> = Result<T, LangError>;
+pub type LexResult<T> = Result<T, LexError>;
+pub type DatabaseResult<T> = Result<T, DatabaseError>;
 
 #[derive(Debug, PartialEq)]
 pub enum LexError {
@@ -72,3 +73,14 @@ pub enum LangError {
     /// unknown `drop` statement
     StmtUnknownDrop,
 }
+
+#[derive(Debug, PartialEq)]
+pub enum DatabaseError {
+    // sys
+    SysBadItemID,
+    // ddl: create space
+    /// unknown property or bad type for property
+    DdlCreateSpaceBadProperty,
+    /// the space already exists
+    DdlCreateSpaceAlreadyExists,
+}
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 4d1cf095..9b371761 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -43,6 +43,8 @@ pub type IndexSTSeqCns<K, V> = stord::IndexSTSeqDll<K, V, stord::config::Conserv
 pub type IndexSTSeqLib<K, V> = stord::IndexSTSeqDll<K, V, stord::config::LiberalConfig<K, V>>;
 pub type IndexMTRC<K, V> = mtchm::imp::ChmArc<K, V, mtchm::meta::DefConfig>;
 pub type IndexMTCp<K, V> = mtchm::imp::ChmCopy<K, V, mtchm::meta::DefConfig>;
+pub type IndexST<K, V, S = std::collections::hash_map::RandomState> =
+    std::collections::hash_map::HashMap<K, V, S>;
 
 /// Any type implementing this trait can be used as a key inside memory engine structures
 pub trait AsKey: Hash + Eq {
@@ -232,52 +234,52 @@ pub trait STIndex<K, V>: IndexBaseSpec<K, V> {
     /// violated
     fn st_insert(&mut self, key: K, val: V) -> bool
     where
-        K: AsKeyClone,
+        K: AsKey,
         V: AsValue;
     /// Updates or inserts the given value
     fn st_upsert(&mut self, key: K, val: V)
     where
-        K: AsKeyClone,
+        K: AsKey,
         V: AsValue;
     // read
     fn st_contains<Q>(&self, key: &Q) -> bool
     where
-        K: Borrow<Q> + AsKeyClone,
+        K: Borrow<Q> + AsKey,
         Q: ?Sized + AsKey;
     /// Returns a reference to the value corresponding to the key, if it exists
     fn st_get<Q>(&self, key: &Q) -> Option<&V>
     where
-        K: AsKeyClone + Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKey;
     /// Returns a clone of the value corresponding to the key, if it exists
     fn st_get_cloned<Q>(&self, key: &Q) -> Option<V>
     where
-        K: AsKeyClone + Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKey,
         V: AsValueClone;
     // update
     /// Returns true if the entry is updated
     fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
     where
-        K: AsKeyClone + Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         V: AsValue,
         Q: ?Sized + AsKey;
     /// Updates the entry and returns the old value, if it exists
     fn st_update_return<Q>(&mut self, key: &Q, val: V) -> Option<V>
     where
-        K: AsKeyClone + Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         V: AsValue,
         Q: ?Sized + AsKey;
     // delete
     /// Returns true if the entry was deleted
     fn st_delete<Q>(&mut self, key: &Q) -> bool
     where
-        K: AsKeyClone + Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKey;
     /// Removes the entry and returns it, if it exists
     fn st_delete_return<Q>(&mut self, key: &Q) -> Option<V>
     where
-        K: AsKeyClone + Borrow<Q>,
+        K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKey;
     // iter
     /// Returns an iterator over a tuple of keys and values
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index 6163e5af..7e60f251 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -34,6 +34,17 @@ macro_rules! extract {
     };
 }
 
+#[cfg(test)]
+macro_rules! extract_safe {
+    ($src:expr, $what:pat => $ret:expr) => {
+        if let $what = $src {
+            $ret
+        } else {
+            panic!("expected one {}, found {:?}", stringify!($what), $src);
+        }
+    };
+}
+
 #[cfg(test)]
 macro_rules! multi_assert_eq {
     ($($lhs:expr),* => $rhs:expr) => {
@@ -120,3 +131,18 @@ macro_rules! void {
         ()
     };
 }
+
+#[cfg(test)]
+/// Convert all the KV pairs into an iterator and then turn it into an appropriate collection
+/// (inferred).
+/// 
+/// **Warning: This is going to be potentially slow due to the iterator creation**
+macro_rules! into_dict {
+    () => { ::core::default::Default::default() };
+    ($($key:expr => $value:expr),+ $(,)?) => {{
+        [$(($key.into(), $value.into())),+]
+        .map(|(k, v)| (k, v))
+        .into_iter()
+        .collect()
+    }};
+}
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index ec8dfbd6..e5d371fb 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -47,8 +47,13 @@ pub struct AStr<const N: usize> {
     base: UArray<N, u8>,
 }
 impl<const N: usize> AStr<N> {
+    #[inline(always)]
+    pub fn check(v: &str) -> bool {
+        v.len() <= N
+    }
+    #[inline(always)]
     pub fn try_new(s: &str) -> Option<Self> {
-        if s.len() <= N {
+        if Self::check(s) {
             Some(unsafe {
                 // UNSAFE(@ohsayan): verified len
                 Self::from_len_unchecked(s)
@@ -57,23 +62,29 @@ impl<const N: usize> AStr<N> {
             None
         }
     }
+    #[inline(always)]
     pub fn new(s: &str) -> Self {
         Self::try_new(s).expect("length overflow")
     }
+    #[inline(always)]
     pub unsafe fn from_len_unchecked_ident(i: Ident<'_>) -> Self {
         Self::from_len_unchecked(i.as_str())
     }
+    #[inline(always)]
     pub unsafe fn from_len_unchecked(s: &str) -> Self {
         Self {
             base: UArray::from_slice(s.as_bytes()),
         }
     }
+    #[inline(always)]
     pub unsafe fn from_len_unchecked_bytes(b: &[u8]) -> Self {
         Self::from_len_unchecked(mem::transmute(b))
     }
+    #[inline(always)]
     pub fn as_str(&self) -> &str {
         unsafe { mem::transmute(self.base.as_slice()) }
     }
+    #[inline(always)]
     pub fn as_mut_str(&mut self) -> &mut str {
         unsafe { mem::transmute(self.base.as_slice_mut()) }
     }
@@ -85,66 +96,79 @@ impl<const N: usize> fmt::Debug for AStr<N> {
 }
 impl<const N: usize> Deref for AStr<N> {
     type Target = str;
+    #[inline(always)]
     fn deref(&self) -> &Self::Target {
         self.as_str()
     }
 }
 impl<const N: usize> DerefMut for AStr<N> {
+    #[inline(always)]
     fn deref_mut(&mut self) -> &mut Self::Target {
         self.as_mut_str()
     }
 }
 impl<'a, const N: usize> From<Ident<'a>> for AStr<N> {
+    #[inline(always)]
     fn from(value: Ident<'a>) -> Self {
         Self::new(value.as_str())
     }
 }
 impl<'a, const N: usize> From<&'a str> for AStr<N> {
+    #[inline(always)]
     fn from(s: &str) -> Self {
         Self::new(s)
     }
 }
 impl<const N: usize> PartialEq<str> for AStr<N> {
+    #[inline(always)]
     fn eq(&self, other: &str) -> bool {
         self.as_str() == other
     }
 }
 impl<const N: usize> PartialEq<AStr<N>> for str {
+    #[inline(always)]
     fn eq(&self, other: &AStr<N>) -> bool {
         self == other.as_str()
     }
 }
 impl<const N: usize> PartialEq<[u8]> for AStr<N> {
+    #[inline(always)]
     fn eq(&self, other: &[u8]) -> bool {
         self.as_bytes() == other
     }
 }
 impl<const N: usize> PartialEq<AStr<N>> for [u8] {
+    #[inline(always)]
     fn eq(&self, other: &AStr<N>) -> bool {
         self == other.as_bytes()
     }
 }
 impl<const N: usize> AsRef<[u8]> for AStr<N> {
+    #[inline(always)]
     fn as_ref(&self) -> &[u8] {
         self.as_bytes()
     }
 }
 impl<const N: usize> AsRef<str> for AStr<N> {
+    #[inline(always)]
     fn as_ref(&self) -> &str {
         self.as_str()
     }
 }
 impl<const N: usize> Default for AStr<N> {
+    #[inline(always)]
     fn default() -> Self {
         Self::new("")
     }
 }
 impl<const N: usize> Borrow<[u8]> for AStr<N> {
+    #[inline(always)]
     fn borrow(&self) -> &[u8] {
         self.as_bytes()
     }
 }
 impl<const N: usize> Borrow<str> for AStr<N> {
+    #[inline(always)]
     fn borrow(&self) -> &str {
         self.as_str()
     }
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 0af63b07..5159d7aa 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -34,3 +34,15 @@ mod idx;
 mod mem;
 mod ql;
 mod sync;
+
+/*
+
+    A word on tests:
+
+    "Nature is not equal. That's the whole problem." - Freeman Dyson
+
+    Well, that applies to us for atleast all test cases since most of them are based on a quiescent
+    state than a chaotic state as in runtime. We do emulate such cases, but remember most assertions
+    that you'll make on most structures are just illusionary, and are only atomically correct at point
+    in time.
+*/
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index ade99c9f..b055b4a6 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -553,8 +553,13 @@ pub enum Statement<'a> {
     Delete(dml::del::DeleteStatement<'a>),
 }
 
+#[cfg(test)]
+pub fn compile_test<'a>(tok: &'a [Token<'a>]) -> LangResult<Statement<'a>> {
+    self::compile(tok, InplaceData::new())
+}
+
 #[inline(always)]
-pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token], d: Qd) -> LangResult<Statement<'a>> {
+pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> LangResult<Statement<'a>> {
     if compiler::unlikely(tok.len() < 2) {
         return Err(LangError::UnexpectedEOS);
     }
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index df5b9584..a3d89b2d 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -42,9 +42,9 @@ use {
 /// A space
 pub struct CreateSpace<'a> {
     /// the space name
-    pub(super) space_name: Ident<'a>,
+    pub space_name: Ident<'a>,
     /// properties
-    pub(super) props: Dict,
+    pub props: Dict,
 }
 
 impl<'a> CreateSpace<'a> {
diff --git a/server/src/engine/ql/ddl/mod.rs b/server/src/engine/ql/ddl/mod.rs
index 2d84e344..639106d0 100644
--- a/server/src/engine/ql/ddl/mod.rs
+++ b/server/src/engine/ql/ddl/mod.rs
@@ -25,8 +25,8 @@
 */
 
 #[macro_use]
-pub(super) mod syn;
-pub(super) mod alt;
-pub(super) mod crt;
-pub(super) mod drop;
-pub(super) mod ins;
+pub(in crate::engine) mod syn;
+pub(in crate::engine) mod alt;
+pub(in crate::engine) mod crt;
+pub(in crate::engine) mod drop;
+pub(in crate::engine) mod ins;
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 65c7ceb6..95f0f02b 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -78,6 +78,12 @@ impl<'a> From<Lit<'a>> for DictEntry {
     }
 }
 
+impl From<HSData> for DictEntry {
+    fn from(hsd: HSData) -> Self {
+        Self::Lit(hsd)
+    }
+}
+
 impl From<Dict> for DictEntry {
     fn from(d: Dict) -> Self {
         Self::Map(d)
@@ -85,7 +91,7 @@ impl From<Dict> for DictEntry {
 }
 
 /// A metadata dictionary
-pub type Dict = HashMap<String, Option<DictEntry>>;
+pub type Dict = HashMap<Box<str>, Option<DictEntry>>;
 
 /// This macro constructs states for our machine
 ///
@@ -184,7 +190,7 @@ where
                 // found lit
                 unsafe {
                     let v = Some(state.read_lit_unchecked_from(tok).into());
-                    state.poison_if_not(dict.insert(key.take().to_string(), v).is_none());
+                    state.poison_if_not(dict.insert(key.take().as_str().into(), v).is_none());
                 }
                 // after lit we're either done or expect something else
                 mstate = DictFoldState::COMMA_OR_CB;
@@ -192,7 +198,7 @@ where
             (Token![null], DictFoldState::LIT_OR_OB) => {
                 // found a null
                 unsafe {
-                    state.poison_if_not(dict.insert(key.take().to_string(), None).is_none());
+                    state.poison_if_not(dict.insert(key.take().as_str().into(), None).is_none());
                 }
                 // after a null (essentially counts as a lit) we're either done or expect something else
                 mstate = DictFoldState::COMMA_OR_CB;
@@ -203,7 +209,7 @@ where
                 _rfold_dict::<Qd, NoBreakpoint>(DictFoldState::CB_OR_IDENT, state, &mut ndict);
                 unsafe {
                     state.poison_if_not(
-                        dict.insert(key.take().to_string(), Some(ndict.into()))
+                        dict.insert(key.take().as_str().into(), Some(ndict.into()))
                             .is_none(),
                     );
                 }
@@ -278,7 +284,7 @@ fn rfold_layers<'a, Qd: QueryData<'a>>(
     layers: &mut Vec<Layer<'a>>,
 ) {
     let mut ty = MaybeInit::uninit();
-    let mut props = dict!();
+    let mut props = Default::default();
     while state.loop_tt() {
         match (state.fw_read(), mstate) {
             (Token::Ident(id), LayerFoldState::BEGIN_IDENT) => {
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index a158a339..7daa66ac 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -256,6 +256,7 @@ macro_rules! Token {
     };
 }
 
+#[cfg(test)]
 macro_rules! dict {
     () => {
         <::std::collections::HashMap<_, _> as ::core::default::Default>::default()
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 674a5757..c5a14b7a 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -34,4 +34,4 @@ pub(super) mod ddl;
 pub(super) mod dml;
 pub(super) mod lex;
 #[cfg(test)]
-mod tests;
+pub(in crate::engine) mod tests;
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 27947f4e..b5a99082 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -41,12 +41,12 @@ mod structure_syn;
 
 #[inline(always)]
 /// Uses the [`InsecureLexer`] to lex the given input
-pub(super) fn lex_insecure(src: &[u8]) -> LexResult<Vec<Token>> {
+pub fn lex_insecure(src: &[u8]) -> LexResult<Vec<Token>> {
     InsecureLexer::lex(src)
 }
 #[inline(always)]
 /// Uses the [`SafeLexer`] to lex the given input
-pub(super) fn lex_secure(src: &[u8]) -> LexResult<Vec<Token>> {
+pub fn lex_secure(src: &[u8]) -> LexResult<Vec<Token>> {
     SafeLexer::lex(src)
 }
 
diff --git a/server/src/util/test_utils.rs b/server/src/util/test_utils.rs
index 62801faf..299bd9c8 100644
--- a/server/src/util/test_utils.rs
+++ b/server/src/util/test_utils.rs
@@ -24,7 +24,10 @@
  *
 */
 
-use rand::{distributions::uniform::SampleUniform, Rng};
+use rand::{
+    distributions::{uniform::SampleUniform, Alphanumeric},
+    Rng,
+};
 
 // TODO(@ohsayan): Use my own PRNG algo here. Maybe my quadratic one?
 
@@ -36,3 +39,19 @@ pub fn random_bool(rng: &mut impl Rng) -> bool {
 pub fn random_number<T: SampleUniform + PartialOrd>(max: T, min: T, rng: &mut impl Rng) -> T {
     rng.gen_range(max..min)
 }
+
+pub fn random_string(rng: &mut impl Rng, l: usize) -> String {
+    rng.sample_iter(Alphanumeric)
+        .take(l)
+        .map(char::from)
+        .collect()
+}
+
+pub fn random_string_checked(rng: &mut impl Rng, l: usize, ck: impl Fn(&str) -> bool) -> String {
+    loop {
+        let r = random_string(rng, l);
+        if ck(&r) {
+            break r;
+        }
+    }
+}

From 997205a327d9a8dd3c15a2b58726115d9d76aedc Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 8 Feb 2023 10:19:50 -0800
Subject: [PATCH 128/310] Exec queries to test

---
 server/src/engine/core/mod.rs                |  7 ++-
 server/src/engine/core/space.rs              |  2 +-
 server/src/engine/core/tests/create_space.rs | 55 +++++++++++---------
 server/src/engine/mem/mod.rs                 | 40 +++++++-------
 server/src/engine/ql/lex/raw.rs              |  5 --
 5 files changed, 58 insertions(+), 51 deletions(-)

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index e781e8dd..457b64c1 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -46,7 +46,7 @@ type RWLIdx<K, V> = RwLock<IndexST<K, V>>;
 
 // FIXME(@ohsayan): Make sure we update what all structures we're making use of here
 
-struct GlobalNS {
+pub struct GlobalNS {
     spaces: RWLIdx<ItemID, Arc<Space>>,
 }
 
@@ -54,4 +54,9 @@ impl GlobalNS {
     pub(self) fn _spaces(&self) -> &RWLIdx<ItemID, Arc<Space>> {
         &self.spaces
     }
+    pub fn empty() -> Self {
+        Self {
+            spaces: RWLIdx::default(),
+        }
+    }
 }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 67db94e7..cbe701be 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -106,7 +106,7 @@ impl Space {
             ),
         })
     }
-    fn validate_and_apply(gns: &super::GlobalNS, space: CreateSpace) -> DatabaseResult<()> {
+    pub fn validate_apply(gns: &super::GlobalNS, space: CreateSpace) -> DatabaseResult<()> {
         let Procedure { space_name, space } = Self::validate(space)?;
         let mut wl = gns._spaces().write();
         if wl.st_insert(space_name, Arc::new(space)) {
diff --git a/server/src/engine/core/tests/create_space.rs b/server/src/engine/core/tests/create_space.rs
index 67fca7e6..da162501 100644
--- a/server/src/engine/core/tests/create_space.rs
+++ b/server/src/engine/core/tests/create_space.rs
@@ -26,10 +26,11 @@
 
 use crate::engine::{
     core::{
-        space::{Procedure, Space, SpaceMeta},
-        HSData,
+        space::{Space, SpaceMeta},
+        GlobalNS, HSData,
     },
     error::DatabaseError,
+    idx::STIndex,
     ql::{
         ast::{compile_test, Statement},
         tests::lex_insecure as lex,
@@ -37,21 +38,24 @@ use crate::engine::{
 };
 
 #[test]
-fn create_space_simple() {
+fn exec_create_space_simple() {
+    let gns = GlobalNS::empty();
     let tok = lex("create space myspace".as_bytes()).unwrap();
     let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
-    let procedure = Space::validate(space).unwrap();
+    Space::validate_apply(&gns, space).unwrap();
     assert_eq!(
-        procedure,
-        Procedure::new(
-            "myspace".into(),
-            Space::new(Default::default(), SpaceMeta::with_env(into_dict! {}))
-        )
+        gns._spaces()
+            .read()
+            .st_get_cloned("myspace".as_bytes())
+            .unwrap()
+            .as_ref(),
+        &Space::new(Default::default(), SpaceMeta::with_env(into_dict! {}))
     );
 }
 
 #[test]
-fn create_space_with_env() {
+fn exec_create_space_with_env() {
+    let gns = GlobalNS::empty();
     let tok = lex(br#"
         create space myspace with {
             env: {
@@ -61,39 +65,42 @@ fn create_space_with_env() {
     "#)
     .unwrap();
     let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
-    let procedure = Space::validate(space).unwrap();
+    Space::validate_apply(&gns, space).unwrap();
     assert_eq!(
-        procedure,
-        Procedure::new(
-            "myspace".into(),
-            Space::new(
-                into_dict! {},
-                SpaceMeta::with_env(into_dict! {
-                    "MAX_MODELS" => HSData::UnsignedInt(100)
-                })
-            )
+        gns._spaces()
+            .read()
+            .st_get_cloned("myspace".as_bytes())
+            .unwrap()
+            .as_ref(),
+        &Space::new(
+            into_dict! {},
+            SpaceMeta::with_env(into_dict! {
+                "MAX_MODELS" => HSData::UnsignedInt(100)
+            })
         )
     );
 }
 
 #[test]
-fn create_space_with_bad_env_type() {
+fn exec_create_space_with_bad_env_type() {
+    let gns = GlobalNS::empty();
     let tok = lex(br#"create space myspace with { env: 100 }"#).unwrap();
     let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
     assert_eq!(
-        Space::validate(space).unwrap_err(),
+        Space::validate_apply(&gns, space).unwrap_err(),
         DatabaseError::DdlCreateSpaceBadProperty
     );
 }
 
 #[test]
-fn create_space_with_random_property() {
+fn exec_create_space_with_random_property() {
+    let gns = GlobalNS::empty();
     let random_property = "i_am_blue_da_ba_dee";
     let query = format!("create space myspace with {{ {random_property}: 100 }}").into_bytes();
     let tok = lex(&query).unwrap();
     let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
     assert_eq!(
-        Space::validate(space).unwrap_err(),
+        Space::validate_apply(&gns, space).unwrap_err(),
         DatabaseError::DdlCreateSpaceBadProperty
     );
 }
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index e5d371fb..149d3a7d 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -41,7 +41,7 @@ use {
 pub use uarray::UArray;
 pub use vinline::VInline;
 
-#[derive(PartialEq, Eq, Hash)]
+#[derive(PartialEq, Eq, Hash, Clone)]
 #[repr(transparent)]
 pub struct AStr<const N: usize> {
     base: UArray<N, u8>,
@@ -81,30 +81,36 @@ impl<const N: usize> AStr<N> {
         Self::from_len_unchecked(mem::transmute(b))
     }
     #[inline(always)]
-    pub fn as_str(&self) -> &str {
-        unsafe { mem::transmute(self.base.as_slice()) }
+    pub fn _as_str(&self) -> &str {
+        unsafe { mem::transmute(self._as_bytes()) }
     }
     #[inline(always)]
-    pub fn as_mut_str(&mut self) -> &mut str {
-        unsafe { mem::transmute(self.base.as_slice_mut()) }
+    pub fn _as_mut_str(&mut self) -> &mut str {
+        unsafe { mem::transmute(self._as_bytes_mut()) }
+    }
+    pub fn _as_bytes(&self) -> &[u8] {
+        self.base.as_slice()
+    }
+    pub fn _as_bytes_mut(&mut self) -> &mut [u8] {
+        self.base.as_slice_mut()
     }
 }
 impl<const N: usize> fmt::Debug for AStr<N> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str(self.as_str())
+        f.write_str(self._as_str())
     }
 }
 impl<const N: usize> Deref for AStr<N> {
     type Target = str;
     #[inline(always)]
     fn deref(&self) -> &Self::Target {
-        self.as_str()
+        self._as_str()
     }
 }
 impl<const N: usize> DerefMut for AStr<N> {
     #[inline(always)]
     fn deref_mut(&mut self) -> &mut Self::Target {
-        self.as_mut_str()
+        self._as_mut_str()
     }
 }
 impl<'a, const N: usize> From<Ident<'a>> for AStr<N> {
@@ -122,19 +128,19 @@ impl<'a, const N: usize> From<&'a str> for AStr<N> {
 impl<const N: usize> PartialEq<str> for AStr<N> {
     #[inline(always)]
     fn eq(&self, other: &str) -> bool {
-        self.as_str() == other
+        self._as_bytes() == other.as_bytes()
     }
 }
 impl<const N: usize> PartialEq<AStr<N>> for str {
     #[inline(always)]
     fn eq(&self, other: &AStr<N>) -> bool {
-        self == other.as_str()
+        other._as_bytes() == self.as_bytes()
     }
 }
 impl<const N: usize> PartialEq<[u8]> for AStr<N> {
     #[inline(always)]
     fn eq(&self, other: &[u8]) -> bool {
-        self.as_bytes() == other
+        self._as_bytes() == other
     }
 }
 impl<const N: usize> PartialEq<AStr<N>> for [u8] {
@@ -146,13 +152,13 @@ impl<const N: usize> PartialEq<AStr<N>> for [u8] {
 impl<const N: usize> AsRef<[u8]> for AStr<N> {
     #[inline(always)]
     fn as_ref(&self) -> &[u8] {
-        self.as_bytes()
+        self._as_bytes()
     }
 }
 impl<const N: usize> AsRef<str> for AStr<N> {
     #[inline(always)]
     fn as_ref(&self) -> &str {
-        self.as_str()
+        self._as_str()
     }
 }
 impl<const N: usize> Default for AStr<N> {
@@ -164,12 +170,6 @@ impl<const N: usize> Default for AStr<N> {
 impl<const N: usize> Borrow<[u8]> for AStr<N> {
     #[inline(always)]
     fn borrow(&self) -> &[u8] {
-        self.as_bytes()
-    }
-}
-impl<const N: usize> Borrow<str> for AStr<N> {
-    #[inline(always)]
-    fn borrow(&self) -> &str {
-        self.as_str()
+        self._as_bytes()
     }
 }
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index b918145c..75d603d5 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -106,11 +106,6 @@ impl<'a> Borrow<[u8]> for Ident<'a> {
         self.0
     }
 }
-impl<'a> Borrow<str> for Ident<'a> {
-    fn borrow(&self) -> &'a str {
-        self.as_str()
-    }
-}
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum Token<'a> {

From f669cc7b00b9b6f41105638b2ca0c13dc81bef70 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 9 Feb 2023 09:50:29 -0800
Subject: [PATCH 129/310] Add patch based metadata merge algorithm

---
 .gitignore                                    |   3 +-
 server/src/engine/core/data.rs                |  11 +-
 server/src/engine/core/data/md_dict.rs        | 230 ++++++++++++++++++
 server/src/engine/core/mod.rs                 |   2 +-
 server/src/engine/core/model.rs               |   4 +-
 server/src/engine/core/space.rs               |  27 +-
 server/src/engine/core/tests.rs               |   3 +-
 server/src/engine/core/tests/create/mod.rs    |  27 ++
 .../{create_space.rs => create/space.rs}      |   0
 server/src/engine/core/tests/data_tests.rs    |  80 ++++++
 server/src/engine/macros.rs                   |   3 +-
 server/src/engine/ql/ddl/alt.rs               |   9 +-
 server/src/engine/ql/ddl/crt.rs               |  13 +-
 server/src/engine/ql/ddl/syn.rs               |  87 ++-----
 server/src/engine/ql/tests.rs                 |  14 +-
 server/src/engine/ql/tests/structure_syn.rs   |   9 +-
 16 files changed, 418 insertions(+), 104 deletions(-)
 create mode 100644 server/src/engine/core/data/md_dict.rs
 create mode 100644 server/src/engine/core/tests/create/mod.rs
 rename server/src/engine/core/tests/{create_space.rs => create/space.rs} (100%)
 create mode 100644 server/src/engine/core/tests/data_tests.rs

diff --git a/.gitignore b/.gitignore
index 1fc42b43..79cf9637 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,8 @@
 /target
 /.vscode
 *.bin
-data
+/data/
+/server/data
 /server/snapshots
 snapstore.bin
 snapstore.partmap
diff --git a/server/src/engine/core/data.rs b/server/src/engine/core/data.rs
index 247eca24..d4a4c4c3 100644
--- a/server/src/engine/core/data.rs
+++ b/server/src/engine/core/data.rs
@@ -24,7 +24,13 @@
  *
 */
 
-use crate::engine::ql::lex::{Lit, LitIR};
+pub mod md_dict;
+
+pub use md_dict::{DictEntryGeneric, DictGeneric, MetaDict};
+use {
+    crate::engine::ql::lex::{Lit, LitIR},
+    std::mem::{self, Discriminant},
+};
 
 /// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
 /// be of one type.
@@ -91,6 +97,9 @@ impl HSData {
             LitIR::Bool(b) => Self::Boolean(b),
         }
     }
+    fn kind(&self) -> Discriminant<Self> {
+        mem::discriminant(&self)
+    }
 }
 
 impl<'a> From<Lit<'a>> for HSData {
diff --git a/server/src/engine/core/data/md_dict.rs b/server/src/engine/core/data/md_dict.rs
new file mode 100644
index 00000000..685318be
--- /dev/null
+++ b/server/src/engine/core/data/md_dict.rs
@@ -0,0 +1,230 @@
+/*
+ * Created on Thu Feb 09 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::engine::{
+        core::data::HSData,
+        idx::STIndex,
+        ql::lex::{Lit, LitIR},
+    },
+    std::collections::HashMap,
+};
+
+/*
+    dict kinds: one is from a generic parse while the other one is the stored metadata
+*/
+
+/// A metadata dictionary
+pub type MetaDict = HashMap<Box<str>, MetaDictEntry>;
+/// A generic dictionary built from scratch from syntactical elements
+pub type DictGeneric = HashMap<Box<str>, Option<DictEntryGeneric>>;
+
+#[derive(Debug, PartialEq)]
+/// A generic dict entry: either a literal or a recursive dictionary
+pub enum DictEntryGeneric {
+    Lit(HSData),
+    Map(DictGeneric),
+}
+
+#[derive(Debug, PartialEq)]
+#[cfg_attr(test, derive(Clone))]
+/// A metadata dictionary
+pub enum MetaDictEntry {
+    Data(HSData),
+    Map(MetaDict),
+}
+
+/*
+    patchsets
+*/
+#[derive(Debug, PartialEq, Default)]
+struct MetaDictPatch(HashMap<Box<str>, Option<MetaDictPatchEntry>>);
+#[derive(Debug, PartialEq)]
+enum MetaDictPatchEntry {
+    Data(HSData),
+    Map(MetaDictPatch),
+}
+
+/// Recursively flatten a [`DictGeneric`] into a [`MetaDict`]
+pub fn rflatten_metadata(new: DictGeneric) -> MetaDict {
+    let mut empty = MetaDict::new();
+    _rflatten_metadata(new, &mut empty);
+    empty
+}
+
+fn _rflatten_metadata(new: DictGeneric, empty: &mut MetaDict) {
+    for (key, val) in new {
+        match val {
+            Some(v) => match v {
+                DictEntryGeneric::Lit(l) => {
+                    empty.insert(key, MetaDictEntry::Data(l));
+                }
+                DictEntryGeneric::Map(m) => {
+                    let mut rnew = MetaDict::new();
+                    _rflatten_metadata(m, &mut rnew);
+                    empty.insert(key, MetaDictEntry::Map(rnew));
+                }
+            },
+            _ => {}
+        }
+    }
+}
+
+/// Recursively merge a [`DictGeneric`] into a [`MetaDict`] with the use of an intermediary
+/// patchset to avoid inconsistent states
+pub fn rmerge_metadata(current: &mut MetaDict, new: DictGeneric) -> bool {
+    let mut patch = MetaDictPatch::default();
+    let current_ref = current as &_;
+    let r = rmerge_metadata_prepare_patch(current_ref, new, &mut patch);
+    if r {
+        merge_data_with_patch(current, patch);
+    }
+    r
+}
+
+fn merge_data_with_patch(current: &mut MetaDict, patch: MetaDictPatch) {
+    for (key, patch) in patch.0 {
+        match patch {
+            Some(MetaDictPatchEntry::Data(d)) => {
+                current.st_upsert(key, MetaDictEntry::Data(d));
+            }
+            Some(MetaDictPatchEntry::Map(m)) => match current.get_mut(&key) {
+                Some(current_recursive) => match current_recursive {
+                    MetaDictEntry::Map(current_m) => {
+                        merge_data_with_patch(current_m, m);
+                    }
+                    _ => {
+                        // can never reach here since the patch is always correct
+                        unreachable!()
+                    }
+                },
+                None => {
+                    let mut new = MetaDict::new();
+                    merge_data_with_patch(&mut new, m);
+                }
+            },
+            None => {
+                let _ = current.remove(&key);
+            }
+        }
+    }
+}
+
+fn rmerge_metadata_prepare_patch(
+    current: &MetaDict,
+    new: DictGeneric,
+    patch: &mut MetaDictPatch,
+) -> bool {
+    let mut new = new.into_iter();
+    let mut okay = true;
+    while new.len() != 0 && okay {
+        let (key, new_entry) = new.next().unwrap();
+        match (current.get(&key), new_entry) {
+            // non-null -> non-null: merge flatten update
+            (Some(this_current), Some(new_entry)) => {
+                okay &= {
+                    match (this_current, new_entry) {
+                        (MetaDictEntry::Data(this_data), DictEntryGeneric::Lit(new_data))
+                            if this_data.kind() == new_data.kind() =>
+                        {
+                            patch
+                                .0
+                                .insert(key, Some(MetaDictPatchEntry::Data(new_data)));
+                            true
+                        }
+                        (
+                            MetaDictEntry::Map(this_recursive_data),
+                            DictEntryGeneric::Map(new_recursive_data),
+                        ) => {
+                            let mut this_patch = MetaDictPatch::default();
+                            let okay = rmerge_metadata_prepare_patch(
+                                this_recursive_data,
+                                new_recursive_data,
+                                &mut this_patch,
+                            );
+                            patch
+                                .0
+                                .insert(key, Some(MetaDictPatchEntry::Map(this_patch)));
+                            okay
+                        }
+                        _ => false,
+                    }
+                };
+            }
+            // null -> non-null: flatten insert
+            (None, Some(new_entry)) => match new_entry {
+                DictEntryGeneric::Lit(d) => {
+                    let _ = patch.0.insert(key, Some(MetaDictPatchEntry::Data(d)));
+                }
+                DictEntryGeneric::Map(m) => {
+                    let mut this_patch = MetaDictPatch::default();
+                    okay &= rmerge_metadata_prepare_patch(&into_dict!(), m, &mut this_patch);
+                    let _ = patch
+                        .0
+                        .insert(key, Some(MetaDictPatchEntry::Map(this_patch)));
+                }
+            },
+            // non-null -> null: remove
+            (Some(_), None) => {
+                patch.0.insert(key, None);
+            }
+            (None, None) => {
+                // ignore
+            }
+        }
+    }
+    okay
+}
+
+/*
+    impls
+*/
+
+impl<'a> From<LitIR<'a>> for DictEntryGeneric {
+    fn from(l: LitIR<'a>) -> Self {
+        Self::Lit(HSData::from(l))
+    }
+}
+
+impl<'a> From<Lit<'a>> for DictEntryGeneric {
+    fn from(value: Lit<'a>) -> Self {
+        Self::Lit(HSData::from(value))
+    }
+}
+
+enum_impls! {
+    DictEntryGeneric => {
+        HSData as Lit,
+        DictGeneric as Map,
+    }
+}
+
+enum_impls! {
+    MetaDictEntry => {
+        HSData as Data,
+        MetaDict as Map,
+    }
+}
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 457b64c1..284786f9 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -24,7 +24,7 @@
  *
 */
 
-mod data;
+pub(in crate::engine) mod data;
 mod model;
 mod space;
 #[cfg(test)]
diff --git a/server/src/engine/core/model.rs b/server/src/engine/core/model.rs
index c0bed173..d1d31acf 100644
--- a/server/src/engine/core/model.rs
+++ b/server/src/engine/core/model.rs
@@ -27,10 +27,10 @@
 // FIXME(@ohsayan): update this!
 
 #[derive(Debug)]
-pub struct ModelNS {}
+pub struct ModelView {}
 
 #[cfg(test)]
-impl PartialEq for ModelNS {
+impl PartialEq for ModelView {
     fn eq(&self, _: &Self) -> bool {
         true
     }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index cbe701be..564f49fe 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -26,28 +26,33 @@
 
 use {
     crate::engine::{
-        core::{model::ModelNS, ItemID, RWLIdx},
+        core::{
+            data::{md_dict, DictEntryGeneric, MetaDict},
+            model::ModelView,
+            ItemID, RWLIdx,
+        },
         error::{DatabaseError, DatabaseResult},
         idx::{IndexST, STIndex},
-        ql::ddl::{crt::CreateSpace, syn::DictEntry},
+        ql::ddl::crt::CreateSpace,
     },
+    parking_lot::RwLock,
     std::sync::Arc,
 };
 
 #[derive(Debug)]
 pub struct Space {
-    mns: RWLIdx<ItemID, Arc<ModelNS>>,
+    mns: RWLIdx<ItemID, Arc<ModelView>>,
     meta: SpaceMeta,
 }
 
 #[derive(Debug, Default)]
 pub struct SpaceMeta {
-    env: RWLIdx<Box<str>, DictEntry>,
+    env: RwLock<MetaDict>,
 }
 
 impl SpaceMeta {
     pub const KEY_ENV: &str = "env";
-    pub fn with_env(env: IndexST<Box<str>, DictEntry>) -> Self {
+    pub fn with_env(env: MetaDict) -> Self {
         Self {
             env: RWLIdx::new(env),
         }
@@ -56,7 +61,7 @@ impl SpaceMeta {
 
 #[derive(Debug)]
 #[cfg_attr(test, derive(PartialEq))]
-pub(super) struct Procedure {
+struct Procedure {
     space_name: ItemID,
     space: Space,
 }
@@ -70,14 +75,14 @@ impl Procedure {
 
 impl Space {
     #[inline(always)]
-    pub fn new(mns: IndexST<ItemID, Arc<ModelNS>>, meta: SpaceMeta) -> Self {
+    pub fn new(mns: IndexST<ItemID, Arc<ModelView>>, meta: SpaceMeta) -> Self {
         Self {
             mns: RWLIdx::new(mns),
             meta,
         }
     }
     #[inline]
-    pub(super) fn validate(
+    fn validate(
         CreateSpace {
             space_name,
             mut props,
@@ -87,7 +92,7 @@ impl Space {
         // check env
         let env;
         match props.remove(SpaceMeta::KEY_ENV) {
-            Some(Some(DictEntry::Map(m))) if props.is_empty() => env = m,
+            Some(Some(DictEntryGeneric::Map(m))) if props.is_empty() => env = m,
             None | Some(None) if props.is_empty() => env = IndexST::default(),
             _ => {
                 return Err(DatabaseError::DdlCreateSpaceBadProperty);
@@ -99,9 +104,7 @@ impl Space {
                 IndexST::default(),
                 SpaceMeta::with_env(
                     // FIXME(@ohsayan): see this is bad. attempt to do it at AST build time
-                    env.into_iter()
-                        .filter_map(|(k, v)| v.map(move |v| (k, v)))
-                        .collect(),
+                    md_dict::rflatten_metadata(env),
                 ),
             ),
         })
diff --git a/server/src/engine/core/tests.rs b/server/src/engine/core/tests.rs
index 91aafaf8..1c0b4965 100644
--- a/server/src/engine/core/tests.rs
+++ b/server/src/engine/core/tests.rs
@@ -24,7 +24,8 @@
  *
 */
 
-mod create_space;
+mod create;
+mod data_tests;
 
 use {super::ItemID, crate::engine::ql::lex::Ident};
 
diff --git a/server/src/engine/core/tests/create/mod.rs b/server/src/engine/core/tests/create/mod.rs
new file mode 100644
index 00000000..de68542b
--- /dev/null
+++ b/server/src/engine/core/tests/create/mod.rs
@@ -0,0 +1,27 @@
+/*
+ * Created on Thu Feb 09 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod space;
diff --git a/server/src/engine/core/tests/create_space.rs b/server/src/engine/core/tests/create/space.rs
similarity index 100%
rename from server/src/engine/core/tests/create_space.rs
rename to server/src/engine/core/tests/create/space.rs
diff --git a/server/src/engine/core/tests/data_tests.rs b/server/src/engine/core/tests/data_tests.rs
new file mode 100644
index 00000000..16b3756c
--- /dev/null
+++ b/server/src/engine/core/tests/data_tests.rs
@@ -0,0 +1,80 @@
+/*
+ * Created on Thu Feb 09 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::core::data::{
+    md_dict::{self, DictEntryGeneric, DictGeneric, MetaDict},
+    HSData,
+};
+
+#[test]
+fn t_simple_flatten() {
+    let generic_dict: DictGeneric = into_dict! {
+        "a_valid_key" => Some(DictEntryGeneric::Lit(100.into())),
+        "a_null_key" => None,
+    };
+    let expected: MetaDict = into_dict!(
+        "a_valid_key" => HSData::UnsignedInt(100)
+    );
+    let ret = md_dict::rflatten_metadata(generic_dict);
+    assert_eq!(ret, expected);
+}
+
+#[test]
+fn t_simple_patch() {
+    let mut current: MetaDict = into_dict! {
+        "a" => HSData::UnsignedInt(2),
+        "b" => HSData::UnsignedInt(3),
+        "z" => HSData::SignedInt(-100),
+    };
+    let new: DictGeneric = into_dict! {
+        "a" => Some(HSData::UnsignedInt(1).into()),
+        "b" => Some(HSData::UnsignedInt(2).into()),
+        "z" => None,
+    };
+    let expected: MetaDict = into_dict! {
+        "a" => HSData::UnsignedInt(1),
+        "b" => HSData::UnsignedInt(2),
+    };
+    assert!(md_dict::rmerge_metadata(&mut current, new));
+    assert_eq!(current, expected);
+}
+
+#[test]
+fn t_bad_patch() {
+    let mut current: MetaDict = into_dict! {
+        "a" => HSData::UnsignedInt(2),
+        "b" => HSData::UnsignedInt(3),
+        "z" => HSData::SignedInt(-100),
+    };
+    let backup = current.clone();
+    let new: DictGeneric = into_dict! {
+        "a" => Some(HSData::UnsignedInt(1).into()),
+        "b" => Some(HSData::UnsignedInt(2).into()),
+        "z" => Some(HSData::String("omg".into()).into()),
+    };
+    assert!(!md_dict::rmerge_metadata(&mut current, new));
+    assert_eq!(current, backup);
+}
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index 7e60f251..412b54cb 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -132,10 +132,9 @@ macro_rules! void {
     };
 }
 
-#[cfg(test)]
 /// Convert all the KV pairs into an iterator and then turn it into an appropriate collection
 /// (inferred).
-/// 
+///
 /// **Warning: This is going to be potentially slow due to the iterator creation**
 macro_rules! into_dict {
     () => { ::core::default::Default::default() };
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 0f37e02a..226f14cb 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -25,9 +25,10 @@
 */
 
 use {
-    super::syn::{self, Dict, DictFoldState, ExpandedField},
+    super::syn::{self, DictFoldState, ExpandedField},
     crate::{
         engine::{
+            core::data::DictGeneric,
             error::{LangError, LangResult},
             ql::{
                 ast::{QueryData, State},
@@ -42,11 +43,11 @@ use {
 /// An alter space query with corresponding data
 pub struct AlterSpace<'a> {
     space_name: Ident<'a>,
-    updated_props: Dict,
+    updated_props: DictGeneric,
 }
 
 impl<'a> AlterSpace<'a> {
-    pub fn new(space_name: Ident<'a>, updated_props: Dict) -> Self {
+    pub fn new(space_name: Ident<'a>, updated_props: DictGeneric) -> Self {
         Self {
             space_name,
             updated_props,
@@ -69,7 +70,7 @@ impl<'a> AlterSpace<'a> {
         }
 
         let space_name = unsafe { extract!(space_name, Token::Ident(ref space) => space.clone()) };
-        let mut d = Dict::new();
+        let mut d = DictGeneric::new();
         syn::rfold_dict(DictFoldState::CB_OR_IDENT, state, &mut d);
         if state.okay() {
             Ok(AlterSpace {
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index a3d89b2d..a642d8e6 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -25,9 +25,10 @@
 */
 
 use {
-    super::syn::{self, Dict, DictFoldState, Field},
+    super::syn::{self, DictFoldState, Field},
     crate::{
         engine::{
+            core::data::DictGeneric,
             error::{LangError, LangResult},
             ql::{
                 ast::{QueryData, State},
@@ -44,7 +45,7 @@ pub struct CreateSpace<'a> {
     /// the space name
     pub space_name: Ident<'a>,
     /// properties
-    pub props: Dict,
+    pub props: DictGeneric,
 }
 
 impl<'a> CreateSpace<'a> {
@@ -61,7 +62,7 @@ impl<'a> CreateSpace<'a> {
         let has_more_properties = state.cursor_rounded_eq(Token![with]);
         state.poison_if_not(has_more_properties | state.exhausted());
         state.cursor_ahead_if(has_more_properties); // +WITH
-        let mut d = Dict::new();
+        let mut d = DictGeneric::new();
         // properties
         if has_more_properties && state.okay() {
             syn::rfold_dict(DictFoldState::OB, state, &mut d);
@@ -85,7 +86,7 @@ pub struct CreateModel<'a> {
     /// the fields
     fields: Vec<Field<'a>>,
     /// properties
-    props: Dict,
+    props: DictGeneric,
 }
 
 /*
@@ -96,7 +97,7 @@ pub struct CreateModel<'a> {
 */
 
 impl<'a> CreateModel<'a> {
-    pub fn new(model_name: Ident<'a>, fields: Vec<Field<'a>>, props: Dict) -> Self {
+    pub fn new(model_name: Ident<'a>, fields: Vec<Field<'a>>, props: DictGeneric) -> Self {
         Self {
             model_name,
             fields,
@@ -126,7 +127,7 @@ impl<'a> CreateModel<'a> {
         }
         state.poison_if_not(stop);
         // check props
-        let mut props = Dict::new();
+        let mut props = DictGeneric::new();
         if state.cursor_rounded_eq(Token![with]) {
             state.cursor_ahead();
             // parse props
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 95f0f02b..a1aa513e 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -44,55 +44,18 @@
     Feb. 2, 2023
 */
 
-use {
-    crate::{
-        engine::{
-            core::HSData,
-            error::{LangError, LangResult},
-            ql::{
-                ast::{QueryData, State},
-                lex::{Ident, Lit, LitIR, Token},
-            },
+use crate::{
+    engine::{
+        core::data::DictGeneric,
+        error::{LangError, LangResult},
+        ql::{
+            ast::{QueryData, State},
+            lex::{Ident, Token},
         },
-        util::{compiler, MaybeInit},
     },
-    std::collections::HashMap,
+    util::{compiler, MaybeInit},
 };
 
-#[derive(Debug, PartialEq)]
-/// A dictionary entry type. Either a literal or another dictionary
-pub enum DictEntry {
-    Lit(HSData),
-    Map(Dict),
-}
-
-impl<'a> From<LitIR<'a>> for DictEntry {
-    fn from(l: LitIR<'a>) -> Self {
-        Self::Lit(HSData::from(l))
-    }
-}
-
-impl<'a> From<Lit<'a>> for DictEntry {
-    fn from(value: Lit<'a>) -> Self {
-        Self::Lit(HSData::from(value))
-    }
-}
-
-impl From<HSData> for DictEntry {
-    fn from(hsd: HSData) -> Self {
-        Self::Lit(hsd)
-    }
-}
-
-impl From<Dict> for DictEntry {
-    fn from(d: Dict) -> Self {
-        Self::Map(d)
-    }
-}
-
-/// A metadata dictionary
-pub type Dict = HashMap<Box<str>, Option<DictEntry>>;
-
 /// This macro constructs states for our machine
 ///
 /// **DO NOT** construct states manually
@@ -150,7 +113,7 @@ impl<'a> Breakpoint<'a> for TypeBreakpoint {
 fn _rfold_dict<'a, Qd, Bp>(
     mut mstate: DictFoldState,
     state: &mut State<'a, Qd>,
-    dict: &mut Dict,
+    dict: &mut DictGeneric,
 ) -> bool
 where
     Qd: QueryData<'a>,
@@ -205,7 +168,7 @@ where
             }
             (Token![open {}], DictFoldState::LIT_OR_OB) => {
                 // found a nested dict
-                let mut ndict = Dict::new();
+                let mut ndict = DictGeneric::new();
                 _rfold_dict::<Qd, NoBreakpoint>(DictFoldState::CB_OR_IDENT, state, &mut ndict);
                 unsafe {
                     state.poison_if_not(
@@ -240,7 +203,7 @@ where
 pub(super) fn rfold_dict<'a, Qd: QueryData<'a>>(
     mstate: DictFoldState,
     state: &mut State<'a, Qd>,
-    dict: &mut Dict,
+    dict: &mut DictGeneric,
 ) {
     _rfold_dict::<Qd, NoBreakpoint>(mstate, state, dict);
 }
@@ -248,7 +211,7 @@ pub(super) fn rfold_dict<'a, Qd: QueryData<'a>>(
 pub(super) fn rfold_tymeta<'a, Qd: QueryData<'a>>(
     mstate: DictFoldState,
     state: &mut State<'a, Qd>,
-    dict: &mut Dict,
+    dict: &mut DictGeneric,
 ) -> bool {
     _rfold_dict::<Qd, TypeBreakpoint>(mstate, state, dict)
 }
@@ -257,12 +220,12 @@ pub(super) fn rfold_tymeta<'a, Qd: QueryData<'a>>(
 /// A layer contains a type and corresponding metadata
 pub struct Layer<'a> {
     ty: Ident<'a>,
-    props: Dict,
+    props: DictGeneric,
 }
 
 impl<'a> Layer<'a> {
     //// Create a new layer
-    pub const fn new(ty: Ident<'a>, props: Dict) -> Self {
+    pub const fn new(ty: Ident<'a>, props: DictGeneric) -> Self {
         Self { ty, props }
     }
 }
@@ -403,11 +366,11 @@ impl<'a> Field<'a> {
 pub struct ExpandedField<'a> {
     field_name: Ident<'a>,
     layers: Vec<Layer<'a>>,
-    props: Dict,
+    props: DictGeneric,
 }
 
 impl<'a> ExpandedField<'a> {
-    pub fn new(field_name: Ident<'a>, layers: Vec<Layer<'a>>, props: Dict) -> Self {
+    pub fn new(field_name: Ident<'a>, layers: Vec<Layer<'a>>, props: DictGeneric) -> Self {
         Self {
             field_name,
             layers,
@@ -426,7 +389,7 @@ impl<'a> ExpandedField<'a> {
         state.poison_if_not(state.cursor_eq(Token![open {}]));
         state.cursor_ahead();
         // ignore errors; now attempt a tymeta-like parse
-        let mut props = Dict::new();
+        let mut props = DictGeneric::new();
         let mut layers = Vec::new();
         if rfold_tymeta(DictFoldState::CB_OR_IDENT, state, &mut props) {
             // this has layers. fold them; but don't forget the colon
@@ -518,8 +481,8 @@ pub use impls::{DictBasic, DictTypeMeta, DictTypeMetaSplit};
 mod impls {
     use {
         super::{
-            rfold_dict, rfold_layers, rfold_tymeta, Dict, DictFoldState, ExpandedField, Field,
-            Layer, LayerFoldState,
+            rfold_dict, rfold_layers, rfold_tymeta, DictFoldState, ExpandedField, Field, Layer,
+            LayerFoldState, DictGeneric,
         },
         crate::engine::{
             error::LangResult,
@@ -554,34 +517,34 @@ mod impls {
         }
     }
     #[derive(sky_macros::Wrapper, Debug)]
-    pub struct DictBasic(Dict);
+    pub struct DictBasic(DictGeneric);
     impl<'a> ASTNode<'a> for DictBasic {
         // important: upstream must verify this
         const VERIFY: bool = true;
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-            let mut dict = Dict::new();
+            let mut dict = DictGeneric::new();
             rfold_dict(DictFoldState::OB, state, &mut dict);
             Ok(Self(dict))
         }
     }
     #[derive(sky_macros::Wrapper, Debug)]
-    pub struct DictTypeMetaSplit(Dict);
+    pub struct DictTypeMetaSplit(DictGeneric);
     impl<'a> ASTNode<'a> for DictTypeMetaSplit {
         // important: upstream must verify this
         const VERIFY: bool = true;
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-            let mut dict = Dict::new();
+            let mut dict = DictGeneric::new();
             rfold_tymeta(DictFoldState::CB_OR_IDENT, state, &mut dict);
             Ok(Self(dict))
         }
     }
     #[derive(sky_macros::Wrapper, Debug)]
-    pub struct DictTypeMeta(Dict);
+    pub struct DictTypeMeta(DictGeneric);
     impl<'a> ASTNode<'a> for DictTypeMeta {
         // important: upstream must verify this
         const VERIFY: bool = true;
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-            let mut dict = Dict::new();
+            let mut dict = DictGeneric::new();
             rfold_tymeta(DictFoldState::OB, state, &mut dict);
             Ok(Self(dict))
         }
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index b5a99082..6c301fb9 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -76,24 +76,24 @@ fn nullable_datatype(v: impl NullableData<HSData>) -> Option<HSData> {
 }
 
 pub trait NullableDictEntry {
-    fn data(self) -> Option<super::ddl::syn::DictEntry>;
+    fn data(self) -> Option<crate::engine::core::data::DictEntryGeneric>;
 }
 
 impl NullableDictEntry for Null {
-    fn data(self) -> Option<super::ddl::syn::DictEntry> {
+    fn data(self) -> Option<crate::engine::core::data::DictEntryGeneric> {
         None
     }
 }
 
 impl<'a> NullableDictEntry for super::lex::Lit<'a> {
-    fn data(self) -> Option<super::ddl::syn::DictEntry> {
-        Some(super::ddl::syn::DictEntry::from(self.as_ir()))
+    fn data(self) -> Option<crate::engine::core::data::DictEntryGeneric> {
+        Some(crate::engine::core::data::DictEntryGeneric::from(self.as_ir()))
     }
 }
 
-impl NullableDictEntry for super::ddl::syn::Dict {
-    fn data(self) -> Option<super::ddl::syn::DictEntry> {
-        Some(super::ddl::syn::DictEntry::Map(self))
+impl NullableDictEntry for crate::engine::core::data::DictGeneric {
+    fn data(self) -> Option<crate::engine::core::data::DictEntryGeneric> {
+        Some(crate::engine::core::data::DictEntryGeneric::Map(self))
     }
 }
 
diff --git a/server/src/engine/ql/tests/structure_syn.rs b/server/src/engine/ql/tests/structure_syn.rs
index 86d0ad95..4f02ec5c 100644
--- a/server/src/engine/ql/tests/structure_syn.rs
+++ b/server/src/engine/ql/tests/structure_syn.rs
@@ -26,10 +26,9 @@
 
 use {
     super::*,
-    crate::engine::ql::{
-        ast::parse_ast_node_full,
-        ddl::syn::{Dict, DictBasic},
-        lex::Lit,
+    crate::engine::{
+        core::data::DictGeneric,
+        ql::{ast::parse_ast_node_full, ddl::syn::DictBasic, lex::Lit},
     },
 };
 
@@ -41,7 +40,7 @@ macro_rules! fold_dict {
     }
 }
 
-fn fold_dict(raw: &[u8]) -> Option<Dict> {
+fn fold_dict(raw: &[u8]) -> Option<DictGeneric> {
     let lexed = lex_insecure(raw).unwrap();
     parse_ast_node_full::<DictBasic>(&lexed)
         .map(|v| v.into_inner())

From 1b2c6923bccb3c5602e3887f5b0ba1c87cc058db Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 9 Feb 2023 20:45:37 -0800
Subject: [PATCH 130/310] Refactor engine modules

---
 server/src/engine/core/mod.rs                 |  7 +----
 server/src/engine/core/space.rs               |  7 ++---
 server/src/engine/core/tests/create/space.rs  |  3 ++-
 server/src/engine/core/tests/mod.rs           | 27 +++++++++++++++++++
 server/src/engine/{core => }/data/md_dict.rs  |  3 ++-
 .../src/engine/{core/data.rs => data/mod.rs}  | 10 ++++++-
 .../tests/md_dict_tests.rs}                   |  2 +-
 server/src/engine/data/tests/metadata_dict.rs |  0
 .../{core/tests.rs => data/tests/mod.rs}      |  7 +++--
 server/src/engine/mod.rs                      |  1 +
 server/src/engine/ql/ast/mod.rs               |  2 +-
 server/src/engine/ql/ddl/alt.rs               |  2 +-
 server/src/engine/ql/ddl/crt.rs               |  2 +-
 server/src/engine/ql/ddl/syn.rs               |  6 ++---
 server/src/engine/ql/dml/ins.rs               |  2 +-
 server/src/engine/ql/tests.rs                 | 16 +++++------
 server/src/engine/ql/tests/structure_syn.rs   |  2 +-
 17 files changed, 64 insertions(+), 35 deletions(-)
 create mode 100644 server/src/engine/core/tests/mod.rs
 rename server/src/engine/{core => }/data/md_dict.rs (99%)
 rename server/src/engine/{core/data.rs => data/mod.rs} (96%)
 rename server/src/engine/{core/tests/data_tests.rs => data/tests/md_dict_tests.rs} (98%)
 create mode 100644 server/src/engine/data/tests/metadata_dict.rs
 rename server/src/engine/{core/tests.rs => data/tests/mod.rs} (93%)

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 284786f9..599f1f9c 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -24,22 +24,17 @@
  *
 */
 
-pub(in crate::engine) mod data;
 mod model;
 mod space;
 #[cfg(test)]
 mod tests;
 
-pub use data::HSData;
 use {
-    self::space::Space,
-    super::{idx::IndexST, mem::AStr},
+    crate::engine::{core::space::Space, data::ItemID, idx::IndexST},
     parking_lot::RwLock,
     std::sync::Arc,
 };
 
-const IDENT_MX: usize = 64;
-type ItemID = AStr<IDENT_MX>;
 /// Use this for now since it substitutes for a file lock (and those syscalls are expensive),
 /// but something better is in the offing
 type RWLIdx<K, V> = RwLock<IndexST<K, V>>;
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 564f49fe..c36f5b1d 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -26,11 +26,8 @@
 
 use {
     crate::engine::{
-        core::{
-            data::{md_dict, DictEntryGeneric, MetaDict},
-            model::ModelView,
-            ItemID, RWLIdx,
-        },
+        core::{model::ModelView, ItemID, RWLIdx},
+        data::{md_dict, DictEntryGeneric, MetaDict},
         error::{DatabaseError, DatabaseResult},
         idx::{IndexST, STIndex},
         ql::ddl::crt::CreateSpace,
diff --git a/server/src/engine/core/tests/create/space.rs b/server/src/engine/core/tests/create/space.rs
index da162501..71340cab 100644
--- a/server/src/engine/core/tests/create/space.rs
+++ b/server/src/engine/core/tests/create/space.rs
@@ -27,8 +27,9 @@
 use crate::engine::{
     core::{
         space::{Space, SpaceMeta},
-        GlobalNS, HSData,
+        GlobalNS,
     },
+    data::HSData,
     error::DatabaseError,
     idx::STIndex,
     ql::{
diff --git a/server/src/engine/core/tests/mod.rs b/server/src/engine/core/tests/mod.rs
new file mode 100644
index 00000000..03b0da17
--- /dev/null
+++ b/server/src/engine/core/tests/mod.rs
@@ -0,0 +1,27 @@
+/*
+ * Created on Wed Feb 08 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod create;
diff --git a/server/src/engine/core/data/md_dict.rs b/server/src/engine/data/md_dict.rs
similarity index 99%
rename from server/src/engine/core/data/md_dict.rs
rename to server/src/engine/data/md_dict.rs
index 685318be..309d13fc 100644
--- a/server/src/engine/core/data/md_dict.rs
+++ b/server/src/engine/data/md_dict.rs
@@ -26,7 +26,7 @@
 
 use {
     crate::engine::{
-        core::data::HSData,
+        data::HSData,
         idx::STIndex,
         ql::lex::{Lit, LitIR},
     },
@@ -60,6 +60,7 @@ pub enum MetaDictEntry {
 /*
     patchsets
 */
+
 #[derive(Debug, PartialEq, Default)]
 struct MetaDictPatch(HashMap<Box<str>, Option<MetaDictPatchEntry>>);
 #[derive(Debug, PartialEq)]
diff --git a/server/src/engine/core/data.rs b/server/src/engine/data/mod.rs
similarity index 96%
rename from server/src/engine/core/data.rs
rename to server/src/engine/data/mod.rs
index d4a4c4c3..8fdce8f8 100644
--- a/server/src/engine/core/data.rs
+++ b/server/src/engine/data/mod.rs
@@ -25,13 +25,21 @@
 */
 
 pub mod md_dict;
+#[cfg(test)]
+mod tests;
 
 pub use md_dict::{DictEntryGeneric, DictGeneric, MetaDict};
 use {
-    crate::engine::ql::lex::{Lit, LitIR},
+    crate::engine::{
+        mem::AStr,
+        ql::lex::{Lit, LitIR},
+    },
     std::mem::{self, Discriminant},
 };
 
+const IDENT_MX: usize = 64;
+pub type ItemID = AStr<IDENT_MX>;
+
 /// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
 /// be of one type.
 // TODO(@ohsayan): Change the underlying structures, there are just rudimentary ones used during integration with the QL
diff --git a/server/src/engine/core/tests/data_tests.rs b/server/src/engine/data/tests/md_dict_tests.rs
similarity index 98%
rename from server/src/engine/core/tests/data_tests.rs
rename to server/src/engine/data/tests/md_dict_tests.rs
index 16b3756c..c1eff615 100644
--- a/server/src/engine/core/tests/data_tests.rs
+++ b/server/src/engine/data/tests/md_dict_tests.rs
@@ -24,7 +24,7 @@
  *
 */
 
-use crate::engine::core::data::{
+use crate::engine::data::{
     md_dict::{self, DictEntryGeneric, DictGeneric, MetaDict},
     HSData,
 };
diff --git a/server/src/engine/data/tests/metadata_dict.rs b/server/src/engine/data/tests/metadata_dict.rs
new file mode 100644
index 00000000..e69de29b
diff --git a/server/src/engine/core/tests.rs b/server/src/engine/data/tests/mod.rs
similarity index 93%
rename from server/src/engine/core/tests.rs
rename to server/src/engine/data/tests/mod.rs
index 1c0b4965..430bef15 100644
--- a/server/src/engine/core/tests.rs
+++ b/server/src/engine/data/tests/mod.rs
@@ -1,5 +1,5 @@
 /*
- * Created on Wed Feb 08 2023
+ * Created on Thu Feb 09 2023
  *
  * This file is a part of Skytable
  * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@@ -24,10 +24,9 @@
  *
 */
 
-mod create;
-mod data_tests;
+mod md_dict_tests;
 
-use {super::ItemID, crate::engine::ql::lex::Ident};
+use crate::engine::{data::ItemID, ql::lex::Ident};
 
 #[test]
 fn item_id_okay() {
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 5159d7aa..96b375fb 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -29,6 +29,7 @@
 #[macro_use]
 mod macros;
 mod core;
+mod data;
 mod error;
 mod idx;
 mod mem;
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index b055b4a6..55025110 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -36,7 +36,7 @@ use {
     },
     crate::{
         engine::{
-            core::HSData,
+            data::HSData,
             error::{LangError, LangResult},
         },
         util::{compiler, MaybeInit},
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 226f14cb..d9a64daf 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -28,7 +28,7 @@ use {
     super::syn::{self, DictFoldState, ExpandedField},
     crate::{
         engine::{
-            core::data::DictGeneric,
+            data::DictGeneric,
             error::{LangError, LangResult},
             ql::{
                 ast::{QueryData, State},
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index a642d8e6..5128eb50 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -28,7 +28,7 @@ use {
     super::syn::{self, DictFoldState, Field},
     crate::{
         engine::{
-            core::data::DictGeneric,
+            data::DictGeneric,
             error::{LangError, LangResult},
             ql::{
                 ast::{QueryData, State},
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index a1aa513e..410c8ee8 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -46,7 +46,7 @@
 
 use crate::{
     engine::{
-        core::data::DictGeneric,
+        data::DictGeneric,
         error::{LangError, LangResult},
         ql::{
             ast::{QueryData, State},
@@ -481,8 +481,8 @@ pub use impls::{DictBasic, DictTypeMeta, DictTypeMetaSplit};
 mod impls {
     use {
         super::{
-            rfold_dict, rfold_layers, rfold_tymeta, DictFoldState, ExpandedField, Field, Layer,
-            LayerFoldState, DictGeneric,
+            rfold_dict, rfold_layers, rfold_tymeta, DictFoldState, DictGeneric, ExpandedField,
+            Field, Layer, LayerFoldState,
         },
         crate::engine::{
             error::LangResult,
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index 5f75b563..c9242bb7 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -28,7 +28,7 @@ use {
     super::read_ident,
     crate::{
         engine::{
-            core::HSData,
+            data::HSData,
             error::{LangError, LangResult},
             ql::{
                 ast::{Entity, QueryData, State},
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 6c301fb9..18368796 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -27,7 +27,7 @@
 use {
     super::lex::{InsecureLexer, SafeLexer, Symbol, Token},
     crate::{
-        engine::{core::HSData, error::LexResult},
+        engine::{data::HSData, error::LexResult},
         util::test_utils,
     },
     rand::{self, Rng},
@@ -76,24 +76,24 @@ fn nullable_datatype(v: impl NullableData<HSData>) -> Option<HSData> {
 }
 
 pub trait NullableDictEntry {
-    fn data(self) -> Option<crate::engine::core::data::DictEntryGeneric>;
+    fn data(self) -> Option<crate::engine::data::DictEntryGeneric>;
 }
 
 impl NullableDictEntry for Null {
-    fn data(self) -> Option<crate::engine::core::data::DictEntryGeneric> {
+    fn data(self) -> Option<crate::engine::data::DictEntryGeneric> {
         None
     }
 }
 
 impl<'a> NullableDictEntry for super::lex::Lit<'a> {
-    fn data(self) -> Option<crate::engine::core::data::DictEntryGeneric> {
-        Some(crate::engine::core::data::DictEntryGeneric::from(self.as_ir()))
+    fn data(self) -> Option<crate::engine::data::DictEntryGeneric> {
+        Some(crate::engine::data::DictEntryGeneric::from(self.as_ir()))
     }
 }
 
-impl NullableDictEntry for crate::engine::core::data::DictGeneric {
-    fn data(self) -> Option<crate::engine::core::data::DictEntryGeneric> {
-        Some(crate::engine::core::data::DictEntryGeneric::Map(self))
+impl NullableDictEntry for crate::engine::data::DictGeneric {
+    fn data(self) -> Option<crate::engine::data::DictEntryGeneric> {
+        Some(crate::engine::data::DictEntryGeneric::Map(self))
     }
 }
 
diff --git a/server/src/engine/ql/tests/structure_syn.rs b/server/src/engine/ql/tests/structure_syn.rs
index 4f02ec5c..466ad38e 100644
--- a/server/src/engine/ql/tests/structure_syn.rs
+++ b/server/src/engine/ql/tests/structure_syn.rs
@@ -27,7 +27,7 @@
 use {
     super::*,
     crate::engine::{
-        core::data::DictGeneric,
+        data::DictGeneric,
         ql::{ast::parse_ast_node_full, ddl::syn::DictBasic, lex::Lit},
     },
 };

From bc7c7bf5955aff380f37712eac321246f90ccfd8 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 9 Feb 2023 21:30:26 -0800
Subject: [PATCH 131/310] Implement basic exec for `alter space`

---
 server/src/engine/core/space.rs               | 46 ++++++++++++----
 server/src/engine/core/tests/mod.rs           |  4 +-
 .../{create/space.rs => space/create.rs}      | 55 +++++++------------
 .../core/tests/{create => space}/mod.rs       | 39 ++++++++++++-
 server/src/engine/error.rs                    |  7 ++-
 server/src/engine/ql/ddl/alt.rs               |  4 +-
 6 files changed, 106 insertions(+), 49 deletions(-)
 rename server/src/engine/core/tests/{create/space.rs => space/create.rs} (65%)
 rename server/src/engine/core/tests/{create => space}/mod.rs (50%)

diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index c36f5b1d..32a64586 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -30,19 +30,21 @@ use {
         data::{md_dict, DictEntryGeneric, MetaDict},
         error::{DatabaseError, DatabaseResult},
         idx::{IndexST, STIndex},
-        ql::ddl::crt::CreateSpace,
+        ql::ddl::{alt::AlterSpace, crt::CreateSpace},
     },
     parking_lot::RwLock,
     std::sync::Arc,
 };
 
 #[derive(Debug)]
+/// A space with the model namespace
 pub struct Space {
     mns: RWLIdx<ItemID, Arc<ModelView>>,
     meta: SpaceMeta,
 }
 
 #[derive(Debug, Default)]
+/// Space metadata
 pub struct SpaceMeta {
     env: RwLock<MetaDict>,
 }
@@ -58,14 +60,16 @@ impl SpaceMeta {
 
 #[derive(Debug)]
 #[cfg_attr(test, derive(PartialEq))]
-struct Procedure {
+/// Procedure for `create space`
+struct ProcedureCreate {
     space_name: ItemID,
     space: Space,
 }
 
-impl Procedure {
+impl ProcedureCreate {
     #[inline(always)]
-    pub(super) fn new(space_name: ItemID, space: Space) -> Self {
+    /// Define the procedure
+    fn new(space_name: ItemID, space: Space) -> Self {
         Self { space_name, space }
     }
 }
@@ -79,12 +83,13 @@ impl Space {
         }
     }
     #[inline]
-    fn validate(
+    /// Validate a `create` stmt
+    fn validate_create(
         CreateSpace {
             space_name,
             mut props,
         }: CreateSpace,
-    ) -> DatabaseResult<Procedure> {
+    ) -> DatabaseResult<ProcedureCreate> {
         let space_name = ItemID::try_new(&space_name).ok_or(DatabaseError::SysBadItemID)?;
         // check env
         let env;
@@ -92,10 +97,10 @@ impl Space {
             Some(Some(DictEntryGeneric::Map(m))) if props.is_empty() => env = m,
             None | Some(None) if props.is_empty() => env = IndexST::default(),
             _ => {
-                return Err(DatabaseError::DdlCreateSpaceBadProperty);
+                return Err(DatabaseError::DdlSpaceBadProperty);
             }
         }
-        Ok(Procedure {
+        Ok(ProcedureCreate {
             space_name,
             space: Self::new(
                 IndexST::default(),
@@ -106,8 +111,9 @@ impl Space {
             ),
         })
     }
-    pub fn validate_apply(gns: &super::GlobalNS, space: CreateSpace) -> DatabaseResult<()> {
-        let Procedure { space_name, space } = Self::validate(space)?;
+    /// Execute a `create` stmt
+    pub fn exec_create(gns: &super::GlobalNS, space: CreateSpace) -> DatabaseResult<()> {
+        let ProcedureCreate { space_name, space } = Self::validate_create(space)?;
         let mut wl = gns._spaces().write();
         if wl.st_insert(space_name, Arc::new(space)) {
             Ok(())
@@ -115,6 +121,26 @@ impl Space {
             Err(DatabaseError::DdlCreateSpaceAlreadyExists)
         }
     }
+    /// Execute a `alter` stmt
+    pub fn exec_alter(
+        gns: &super::GlobalNS,
+        AlterSpace {
+            space_name,
+            updated_props,
+        }: AlterSpace,
+    ) -> DatabaseResult<()> {
+        match gns._spaces().read().st_get_cloned(space_name.as_bytes()) {
+            Some(space) => {
+                let mut space_meta = space.meta.env.write();
+                if md_dict::rmerge_metadata(&mut space_meta, updated_props) {
+                    Ok(())
+                } else {
+                    Err(DatabaseError::DdlSpaceBadProperty)
+                }
+            }
+            None => Err(DatabaseError::DdlAlterSpaceNotFound),
+        }
+    }
 }
 
 #[cfg(test)]
diff --git a/server/src/engine/core/tests/mod.rs b/server/src/engine/core/tests/mod.rs
index 03b0da17..af49ff11 100644
--- a/server/src/engine/core/tests/mod.rs
+++ b/server/src/engine/core/tests/mod.rs
@@ -24,4 +24,6 @@
  *
 */
 
-mod create;
+// ddl
+// space
+mod space;
diff --git a/server/src/engine/core/tests/create/space.rs b/server/src/engine/core/tests/space/create.rs
similarity index 65%
rename from server/src/engine/core/tests/create/space.rs
rename to server/src/engine/core/tests/space/create.rs
index 71340cab..6921b56e 100644
--- a/server/src/engine/core/tests/create/space.rs
+++ b/server/src/engine/core/tests/space/create.rs
@@ -31,7 +31,6 @@ use crate::engine::{
     },
     data::HSData,
     error::DatabaseError,
-    idx::STIndex,
     ql::{
         ast::{compile_test, Statement},
         tests::lex_insecure as lex,
@@ -41,45 +40,33 @@ use crate::engine::{
 #[test]
 fn exec_create_space_simple() {
     let gns = GlobalNS::empty();
-    let tok = lex("create space myspace".as_bytes()).unwrap();
-    let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
-    Space::validate_apply(&gns, space).unwrap();
-    assert_eq!(
-        gns._spaces()
-            .read()
-            .st_get_cloned("myspace".as_bytes())
-            .unwrap()
-            .as_ref(),
-        &Space::new(Default::default(), SpaceMeta::with_env(into_dict! {}))
-    );
+    super::exec_create_empty_verify(&gns, "create space myspace");
 }
 
 #[test]
 fn exec_create_space_with_env() {
     let gns = GlobalNS::empty();
-    let tok = lex(br#"
+    super::exec_create_and_verify(
+        &gns,
+        r#"
         create space myspace with {
             env: {
                 MAX_MODELS: 100
             }
         }
-    "#)
-    .unwrap();
-    let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
-    Space::validate_apply(&gns, space).unwrap();
-    assert_eq!(
-        gns._spaces()
-            .read()
-            .st_get_cloned("myspace".as_bytes())
-            .unwrap()
-            .as_ref(),
-        &Space::new(
-            into_dict! {},
-            SpaceMeta::with_env(into_dict! {
-                "MAX_MODELS" => HSData::UnsignedInt(100)
-            })
-        )
-    );
+    "#,
+        |space| {
+            assert_eq!(
+                space,
+                &Space::new(
+                    into_dict! {},
+                    SpaceMeta::with_env(into_dict! {
+                        "MAX_MODELS" => HSData::UnsignedInt(100)
+                    })
+                )
+            );
+        },
+    )
 }
 
 #[test]
@@ -88,8 +75,8 @@ fn exec_create_space_with_bad_env_type() {
     let tok = lex(br#"create space myspace with { env: 100 }"#).unwrap();
     let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
     assert_eq!(
-        Space::validate_apply(&gns, space).unwrap_err(),
-        DatabaseError::DdlCreateSpaceBadProperty
+        Space::exec_create(&gns, space).unwrap_err(),
+        DatabaseError::DdlSpaceBadProperty
     );
 }
 
@@ -101,7 +88,7 @@ fn exec_create_space_with_random_property() {
     let tok = lex(&query).unwrap();
     let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
     assert_eq!(
-        Space::validate_apply(&gns, space).unwrap_err(),
-        DatabaseError::DdlCreateSpaceBadProperty
+        Space::exec_create(&gns, space).unwrap_err(),
+        DatabaseError::DdlSpaceBadProperty
     );
 }
diff --git a/server/src/engine/core/tests/create/mod.rs b/server/src/engine/core/tests/space/mod.rs
similarity index 50%
rename from server/src/engine/core/tests/create/mod.rs
rename to server/src/engine/core/tests/space/mod.rs
index de68542b..c5e3b3a0 100644
--- a/server/src/engine/core/tests/create/mod.rs
+++ b/server/src/engine/core/tests/space/mod.rs
@@ -24,4 +24,41 @@
  *
 */
 
-mod space;
+mod create;
+
+use crate::engine::{
+    core::{
+        space::{Space, SpaceMeta},
+        GlobalNS,
+    },
+    idx::STIndex,
+    ql::{
+        ast::{compile_test, Statement},
+        tests::lex_insecure as lex,
+    },
+};
+
+/// Creates a space using the given tokens and allows the caller to verify it
+fn exec_create_and_verify(gns: &GlobalNS, tok: &str, verify: impl Fn(&Space)) {
+    let tok = lex(tok.as_bytes()).unwrap();
+    let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
+    let space_name = space.space_name;
+    Space::exec_create(&gns, space).unwrap();
+    verify(
+        gns._spaces()
+            .read()
+            .st_get_cloned(space_name.as_bytes())
+            .unwrap()
+            .as_ref(),
+    );
+}
+
+/// Creates an empty space with the given tokens
+fn exec_create_empty_verify(gns: &GlobalNS, tok: &str) {
+    self::exec_create_and_verify(gns, tok, |space| {
+        assert_eq!(
+            space,
+            &Space::new(Default::default(), SpaceMeta::with_env(into_dict! {}))
+        );
+    });
+}
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 421994da..ec16bb11 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -29,6 +29,7 @@ pub type LexResult<T> = Result<T, LexError>;
 pub type DatabaseResult<T> = Result<T, DatabaseError>;
 
 #[derive(Debug, PartialEq)]
+/// Lex phase errors
 pub enum LexError {
     // insecure lex
     /// Invalid signed numeric literal
@@ -47,6 +48,7 @@ pub enum LexError {
     UnexpectedByte,
 }
 #[derive(Debug, PartialEq)]
+/// AST errors
 pub enum LangError {
     // generic
     /// Unexpected end of syntax
@@ -75,12 +77,15 @@ pub enum LangError {
 }
 
 #[derive(Debug, PartialEq)]
+/// Executor errors
 pub enum DatabaseError {
     // sys
     SysBadItemID,
     // ddl: create space
     /// unknown property or bad type for property
-    DdlCreateSpaceBadProperty,
+    DdlSpaceBadProperty,
     /// the space already exists
     DdlCreateSpaceAlreadyExists,
+    /// the space doesn't exist
+    DdlAlterSpaceNotFound,
 }
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index d9a64daf..9890c93d 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -42,8 +42,8 @@ use {
 #[derive(Debug, PartialEq)]
 /// An alter space query with corresponding data
 pub struct AlterSpace<'a> {
-    space_name: Ident<'a>,
-    updated_props: DictGeneric,
+    pub space_name: Ident<'a>,
+    pub updated_props: DictGeneric,
 }
 
 impl<'a> AlterSpace<'a> {

From 3a992253c1caad900a626a3ccff7c7cbaf1110e9 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 10 Feb 2023 00:01:35 -0800
Subject: [PATCH 132/310] Fix alter space exec and add tests

---
 server/src/engine/core/space.rs              |  33 +++---
 server/src/engine/core/tests/space/alter.rs  | 116 +++++++++++++++++++
 server/src/engine/core/tests/space/create.rs |  28 ++---
 server/src/engine/core/tests/space/mod.rs    |  55 +++++++--
 server/src/engine/error.rs                   |   4 +-
 server/src/engine/ql/ddl/syn.rs              |   3 +-
 server/src/engine/ql/lex/raw.rs              |   3 +
 server/src/engine/ql/tests.rs                |   2 +-
 8 files changed, 195 insertions(+), 49 deletions(-)
 create mode 100644 server/src/engine/core/tests/space/alter.rs

diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 32a64586..df6a4118 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use crate::engine::data::md_dict::DictGeneric;
+
 use {
     crate::engine::{
         core::{model::ModelView, ItemID, RWLIdx},
@@ -40,13 +42,13 @@ use {
 /// A space with the model namespace
 pub struct Space {
     mns: RWLIdx<ItemID, Arc<ModelView>>,
-    meta: SpaceMeta,
+    pub(super) meta: SpaceMeta,
 }
 
 #[derive(Debug, Default)]
 /// Space metadata
 pub struct SpaceMeta {
-    env: RwLock<MetaDict>,
+    pub(super) env: RwLock<MetaDict>,
 }
 
 impl SpaceMeta {
@@ -56,6 +58,15 @@ impl SpaceMeta {
             env: RWLIdx::new(env),
         }
     }
+    fn read_from_props(props: &mut DictGeneric) -> DatabaseResult<DictGeneric> {
+        match props.remove(SpaceMeta::KEY_ENV) {
+            Some(Some(DictEntryGeneric::Map(m))) if props.is_empty() => Ok(m),
+            None | Some(None) if props.is_empty() => Ok(IndexST::default()),
+            _ => {
+                return Err(DatabaseError::DdlSpaceBadProperty);
+            }
+        }
+    }
 }
 
 #[derive(Debug)]
@@ -92,14 +103,7 @@ impl Space {
     ) -> DatabaseResult<ProcedureCreate> {
         let space_name = ItemID::try_new(&space_name).ok_or(DatabaseError::SysBadItemID)?;
         // check env
-        let env;
-        match props.remove(SpaceMeta::KEY_ENV) {
-            Some(Some(DictEntryGeneric::Map(m))) if props.is_empty() => env = m,
-            None | Some(None) if props.is_empty() => env = IndexST::default(),
-            _ => {
-                return Err(DatabaseError::DdlSpaceBadProperty);
-            }
-        }
+        let env = SpaceMeta::read_from_props(&mut props)?;
         Ok(ProcedureCreate {
             space_name,
             space: Self::new(
@@ -118,7 +122,7 @@ impl Space {
         if wl.st_insert(space_name, Arc::new(space)) {
             Ok(())
         } else {
-            Err(DatabaseError::DdlCreateSpaceAlreadyExists)
+            Err(DatabaseError::DdlSpaceAlreadyExists)
         }
     }
     /// Execute a `alter` stmt
@@ -126,19 +130,20 @@ impl Space {
         gns: &super::GlobalNS,
         AlterSpace {
             space_name,
-            updated_props,
+            mut updated_props,
         }: AlterSpace,
     ) -> DatabaseResult<()> {
+        let env = SpaceMeta::read_from_props(&mut updated_props)?;
         match gns._spaces().read().st_get_cloned(space_name.as_bytes()) {
             Some(space) => {
                 let mut space_meta = space.meta.env.write();
-                if md_dict::rmerge_metadata(&mut space_meta, updated_props) {
+                if md_dict::rmerge_metadata(&mut space_meta, env) {
                     Ok(())
                 } else {
                     Err(DatabaseError::DdlSpaceBadProperty)
                 }
             }
-            None => Err(DatabaseError::DdlAlterSpaceNotFound),
+            None => Err(DatabaseError::DdlSpaceNotFound),
         }
     }
 }
diff --git a/server/src/engine/core/tests/space/alter.rs b/server/src/engine/core/tests/space/alter.rs
new file mode 100644
index 00000000..b30284fb
--- /dev/null
+++ b/server/src/engine/core/tests/space/alter.rs
@@ -0,0 +1,116 @@
+/*
+ * Created on Thu Feb 09 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    core::{
+        space::{Space, SpaceMeta},
+        GlobalNS,
+    },
+    data::HSData,
+    error::DatabaseError,
+};
+
+#[test]
+fn alter_add_props() {
+    let gns = GlobalNS::empty();
+    super::exec_create_empty_verify(&gns, "create space myspace");
+    super::exec_alter_and_verify(
+        &gns,
+        "alter space myspace with { env: { MY_NEW_PROP: 100 } }",
+        |space| {
+            assert_eq!(
+                space.unwrap(),
+                &Space::new(
+                    into_dict!(),
+                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => HSData::UnsignedInt(100)))
+                )
+            )
+        },
+    )
+}
+
+#[test]
+fn alter_update_props() {
+    let gns = GlobalNS::empty();
+    super::exec_create_and_verify(
+        &gns,
+        "create space myspace with { env: { MY_NEW_PROP: 100 } }",
+        |space| {
+            assert_eq!(
+                space.unwrap().meta.env.read().get("MY_NEW_PROP").unwrap(),
+                &(HSData::UnsignedInt(100).into())
+            )
+        },
+    );
+    super::exec_alter_and_verify(
+        &gns,
+        "alter space myspace with { env: { MY_NEW_PROP: 200 } }",
+        |space| {
+            assert_eq!(
+                space.unwrap(),
+                &Space::new(
+                    into_dict!(),
+                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => HSData::UnsignedInt(200)))
+                )
+            )
+        },
+    )
+}
+
+#[test]
+fn alter_remove_props() {
+    let gns = GlobalNS::empty();
+    super::exec_create_and_verify(
+        &gns,
+        "create space myspace with { env: { MY_NEW_PROP: 100 } }",
+        |space| {
+            assert_eq!(
+                space.unwrap().meta.env.read().get("MY_NEW_PROP").unwrap(),
+                &(HSData::UnsignedInt(100).into())
+            )
+        },
+    );
+    super::exec_alter_and_verify(
+        &gns,
+        "alter space myspace with { env: { MY_NEW_PROP: null } }",
+        |space| {
+            assert_eq!(
+                space.unwrap(),
+                &Space::new(into_dict!(), SpaceMeta::with_env(into_dict!()))
+            )
+        },
+    )
+}
+
+#[test]
+fn alter_nx() {
+    let gns = GlobalNS::empty();
+    super::exec_alter_and_verify(
+        &gns,
+        "alter space myspace with { env: { MY_NEW_PROP: 100 } }",
+        |space| assert_eq!(space.unwrap_err(), DatabaseError::DdlSpaceNotFound),
+    )
+}
diff --git a/server/src/engine/core/tests/space/create.rs b/server/src/engine/core/tests/space/create.rs
index 6921b56e..d17b483a 100644
--- a/server/src/engine/core/tests/space/create.rs
+++ b/server/src/engine/core/tests/space/create.rs
@@ -31,10 +31,6 @@ use crate::engine::{
     },
     data::HSData,
     error::DatabaseError,
-    ql::{
-        ast::{compile_test, Statement},
-        tests::lex_insecure as lex,
-    },
 };
 
 #[test]
@@ -57,7 +53,7 @@ fn exec_create_space_with_env() {
     "#,
         |space| {
             assert_eq!(
-                space,
+                space.unwrap(),
                 &Space::new(
                     into_dict! {},
                     SpaceMeta::with_env(into_dict! {
@@ -72,23 +68,19 @@ fn exec_create_space_with_env() {
 #[test]
 fn exec_create_space_with_bad_env_type() {
     let gns = GlobalNS::empty();
-    let tok = lex(br#"create space myspace with { env: 100 }"#).unwrap();
-    let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
-    assert_eq!(
-        Space::exec_create(&gns, space).unwrap_err(),
-        DatabaseError::DdlSpaceBadProperty
-    );
+    super::exec_create_and_verify(&gns, "create space myspace with { env: 100 }", |space| {
+        assert_eq!(space.unwrap_err(), DatabaseError::DdlSpaceBadProperty);
+    });
 }
 
 #[test]
 fn exec_create_space_with_random_property() {
     let gns = GlobalNS::empty();
-    let random_property = "i_am_blue_da_ba_dee";
-    let query = format!("create space myspace with {{ {random_property}: 100 }}").into_bytes();
-    let tok = lex(&query).unwrap();
-    let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
-    assert_eq!(
-        Space::exec_create(&gns, space).unwrap_err(),
-        DatabaseError::DdlSpaceBadProperty
+    super::exec_create_and_verify(
+        &gns,
+        "create space myspace with { i_am_blue_da_ba_dee: 100 }",
+        |space| {
+            assert_eq!(space.unwrap_err(), DatabaseError::DdlSpaceBadProperty);
+        },
     );
 }
diff --git a/server/src/engine/core/tests/space/mod.rs b/server/src/engine/core/tests/space/mod.rs
index c5e3b3a0..b0075390 100644
--- a/server/src/engine/core/tests/space/mod.rs
+++ b/server/src/engine/core/tests/space/mod.rs
@@ -24,6 +24,7 @@
  *
 */
 
+mod alter;
 mod create;
 
 use crate::engine::{
@@ -31,6 +32,7 @@ use crate::engine::{
         space::{Space, SpaceMeta},
         GlobalNS,
     },
+    error::DatabaseResult,
     idx::STIndex,
     ql::{
         ast::{compile_test, Statement},
@@ -38,18 +40,47 @@ use crate::engine::{
     },
 };
 
+fn exec_verify(
+    gns: &GlobalNS,
+    query: &str,
+    exec: impl Fn(&GlobalNS, Statement<'_>) -> (DatabaseResult<()>, Box<str>),
+    verify: impl Fn(DatabaseResult<&Space>),
+) {
+    let tok = lex(query.as_bytes()).unwrap();
+    let ast_node = compile_test(&tok).unwrap();
+    let (res, space_name) = exec(gns, ast_node);
+    let space_arc = gns._spaces().read().st_get_cloned(space_name.as_bytes());
+    let r = res.map(|_| space_arc.as_deref().unwrap());
+    verify(r);
+}
+
+/// Creates a space using the given tokens and allows the caller to verify it
+fn exec_alter_and_verify(gns: &GlobalNS, tok: &str, verify: impl Fn(DatabaseResult<&Space>)) {
+    exec_verify(
+        gns,
+        tok,
+        |gns, stmt| {
+            let space = extract_safe!(stmt, Statement::AlterSpace(s) => s);
+            let space_name = space.space_name;
+            let r = Space::exec_alter(&gns, space);
+            (r, space_name.boxed_str())
+        },
+        verify,
+    );
+}
+
 /// Creates a space using the given tokens and allows the caller to verify it
-fn exec_create_and_verify(gns: &GlobalNS, tok: &str, verify: impl Fn(&Space)) {
-    let tok = lex(tok.as_bytes()).unwrap();
-    let space = extract_safe!(compile_test(&tok).unwrap(), Statement::CreateSpace(s) => s);
-    let space_name = space.space_name;
-    Space::exec_create(&gns, space).unwrap();
-    verify(
-        gns._spaces()
-            .read()
-            .st_get_cloned(space_name.as_bytes())
-            .unwrap()
-            .as_ref(),
+fn exec_create_and_verify(gns: &GlobalNS, tok: &str, verify: impl Fn(DatabaseResult<&Space>)) {
+    exec_verify(
+        gns,
+        tok,
+        |gns, stmt| {
+            let space = extract_safe!(stmt, Statement::CreateSpace(s) => s);
+            let space_name = space.space_name;
+            let r = Space::exec_create(&gns, space);
+            (r, space_name.boxed_str())
+        },
+        verify,
     );
 }
 
@@ -57,7 +88,7 @@ fn exec_create_and_verify(gns: &GlobalNS, tok: &str, verify: impl Fn(&Space)) {
 fn exec_create_empty_verify(gns: &GlobalNS, tok: &str) {
     self::exec_create_and_verify(gns, tok, |space| {
         assert_eq!(
-            space,
+            space.unwrap(),
             &Space::new(Default::default(), SpaceMeta::with_env(into_dict! {}))
         );
     });
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index ec16bb11..eee9f372 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -85,7 +85,7 @@ pub enum DatabaseError {
     /// unknown property or bad type for property
     DdlSpaceBadProperty,
     /// the space already exists
-    DdlCreateSpaceAlreadyExists,
+    DdlSpaceAlreadyExists,
     /// the space doesn't exist
-    DdlAlterSpaceNotFound,
+    DdlSpaceNotFound,
 }
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 410c8ee8..27e443da 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -188,8 +188,7 @@ where
                 // reached custom breakpoint
                 return true;
             }
-            x => {
-                dbg!(x);
+            _ => {
                 state.cursor_back();
                 state.poison();
                 break;
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index 75d603d5..76f3c137 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -49,6 +49,9 @@ impl<'a> Ident<'a> {
             str::from_utf8_unchecked(self.0)
         }
     }
+    pub fn boxed_str(&self) -> Box<str> {
+        self.as_str().to_string().into_boxed_str()
+    }
 }
 impl<'a> fmt::Debug for Ident<'a> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 18368796..6a678099 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -41,7 +41,7 @@ mod structure_syn;
 
 #[inline(always)]
 /// Uses the [`InsecureLexer`] to lex the given input
-pub fn lex_insecure(src: &[u8]) -> LexResult<Vec<Token>> {
+pub fn lex_insecure<'a>(src: &'a [u8]) -> LexResult<Vec<Token<'a>>> {
     InsecureLexer::lex(src)
 }
 #[inline(always)]

From 34990c3186339d3842732a84fc4c539b0f2bdff4 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 10 Feb 2023 00:56:03 -0800
Subject: [PATCH 133/310] Fix metadata merge in alter space exec for `null`
 prop cases

---
 server/src/engine/core/space.rs               | 39 ++++++++++---------
 server/src/engine/core/tests/space/alter.rs   | 24 ++++++++++--
 server/src/engine/core/tests/space/mod.rs     | 10 +----
 server/src/engine/data/tests/md_dict_tests.rs | 25 +++++++++++-
 server/src/engine/data/tests/metadata_dict.rs |  0
 5 files changed, 68 insertions(+), 30 deletions(-)
 delete mode 100644 server/src/engine/data/tests/metadata_dict.rs

diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index df6a4118..c538fba4 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -24,8 +24,6 @@
  *
 */
 
-use crate::engine::data::md_dict::DictGeneric;
-
 use {
     crate::engine::{
         core::{model::ModelView, ItemID, RWLIdx},
@@ -58,15 +56,6 @@ impl SpaceMeta {
             env: RWLIdx::new(env),
         }
     }
-    fn read_from_props(props: &mut DictGeneric) -> DatabaseResult<DictGeneric> {
-        match props.remove(SpaceMeta::KEY_ENV) {
-            Some(Some(DictEntryGeneric::Map(m))) if props.is_empty() => Ok(m),
-            None | Some(None) if props.is_empty() => Ok(IndexST::default()),
-            _ => {
-                return Err(DatabaseError::DdlSpaceBadProperty);
-            }
-        }
-    }
 }
 
 #[derive(Debug)]
@@ -86,6 +75,9 @@ impl ProcedureCreate {
 }
 
 impl Space {
+    pub fn empty() -> Self {
+        Space::new(Default::default(), SpaceMeta::with_env(into_dict! {}))
+    }
     #[inline(always)]
     pub fn new(mns: IndexST<ItemID, Arc<ModelView>>, meta: SpaceMeta) -> Self {
         Self {
@@ -103,7 +95,13 @@ impl Space {
     ) -> DatabaseResult<ProcedureCreate> {
         let space_name = ItemID::try_new(&space_name).ok_or(DatabaseError::SysBadItemID)?;
         // check env
-        let env = SpaceMeta::read_from_props(&mut props)?;
+        let env = match props.remove(SpaceMeta::KEY_ENV) {
+            Some(Some(DictEntryGeneric::Map(m))) if props.is_empty() => m,
+            Some(None) | None if props.is_empty() => IndexST::default(),
+            _ => {
+                return Err(DatabaseError::DdlSpaceBadProperty);
+            }
+        };
         Ok(ProcedureCreate {
             space_name,
             space: Self::new(
@@ -133,15 +131,20 @@ impl Space {
             mut updated_props,
         }: AlterSpace,
     ) -> DatabaseResult<()> {
-        let env = SpaceMeta::read_from_props(&mut updated_props)?;
         match gns._spaces().read().st_get_cloned(space_name.as_bytes()) {
             Some(space) => {
-                let mut space_meta = space.meta.env.write();
-                if md_dict::rmerge_metadata(&mut space_meta, env) {
-                    Ok(())
-                } else {
-                    Err(DatabaseError::DdlSpaceBadProperty)
+                let mut space_env = space.meta.env.write();
+                match updated_props.remove(SpaceMeta::KEY_ENV) {
+                    Some(Some(DictEntryGeneric::Map(env))) if updated_props.is_empty() => {
+                        if !md_dict::rmerge_metadata(&mut space_env, env) {
+                            return Err(DatabaseError::DdlSpaceBadProperty);
+                        }
+                    }
+                    Some(None) if updated_props.is_empty() => space_env.clear(),
+                    None => {}
+                    _ => return Err(DatabaseError::DdlSpaceBadProperty),
                 }
+                Ok(())
             }
             None => Err(DatabaseError::DdlSpaceNotFound),
         }
diff --git a/server/src/engine/core/tests/space/alter.rs b/server/src/engine/core/tests/space/alter.rs
index b30284fb..2509107a 100644
--- a/server/src/engine/core/tests/space/alter.rs
+++ b/server/src/engine/core/tests/space/alter.rs
@@ -34,7 +34,7 @@ use crate::engine::{
 };
 
 #[test]
-fn alter_add_props() {
+fn alter_add_prop_env_var() {
     let gns = GlobalNS::empty();
     super::exec_create_empty_verify(&gns, "create space myspace");
     super::exec_alter_and_verify(
@@ -53,7 +53,7 @@ fn alter_add_props() {
 }
 
 #[test]
-fn alter_update_props() {
+fn alter_update_prop_env_var() {
     let gns = GlobalNS::empty();
     super::exec_create_and_verify(
         &gns,
@@ -81,7 +81,7 @@ fn alter_update_props() {
 }
 
 #[test]
-fn alter_remove_props() {
+fn alter_remove_prop_env_var() {
     let gns = GlobalNS::empty();
     super::exec_create_and_verify(
         &gns,
@@ -114,3 +114,21 @@ fn alter_nx() {
         |space| assert_eq!(space.unwrap_err(), DatabaseError::DdlSpaceNotFound),
     )
 }
+
+#[test]
+fn alter_remove_all_env() {
+    let gns = GlobalNS::empty();
+    super::exec_create_and_verify(
+        &gns,
+        "create space myspace with { env: { MY_NEW_PROP: 100 } }",
+        |space| {
+            assert_eq!(
+                space.unwrap().meta.env.read().get("MY_NEW_PROP").unwrap(),
+                &(HSData::UnsignedInt(100).into())
+            )
+        },
+    );
+    super::exec_alter_and_verify(&gns, "alter space myspace with { env: null }", |space| {
+        assert_eq!(space.unwrap(), &Space::empty())
+    })
+}
diff --git a/server/src/engine/core/tests/space/mod.rs b/server/src/engine/core/tests/space/mod.rs
index b0075390..ede699f1 100644
--- a/server/src/engine/core/tests/space/mod.rs
+++ b/server/src/engine/core/tests/space/mod.rs
@@ -28,10 +28,7 @@ mod alter;
 mod create;
 
 use crate::engine::{
-    core::{
-        space::{Space, SpaceMeta},
-        GlobalNS,
-    },
+    core::{space::Space, GlobalNS},
     error::DatabaseResult,
     idx::STIndex,
     ql::{
@@ -87,9 +84,6 @@ fn exec_create_and_verify(gns: &GlobalNS, tok: &str, verify: impl Fn(DatabaseRes
 /// Creates an empty space with the given tokens
 fn exec_create_empty_verify(gns: &GlobalNS, tok: &str) {
     self::exec_create_and_verify(gns, tok, |space| {
-        assert_eq!(
-            space.unwrap(),
-            &Space::new(Default::default(), SpaceMeta::with_env(into_dict! {}))
-        );
+        assert_eq!(space.unwrap(), &Space::empty());
     });
 }
diff --git a/server/src/engine/data/tests/md_dict_tests.rs b/server/src/engine/data/tests/md_dict_tests.rs
index c1eff615..9f2577f0 100644
--- a/server/src/engine/data/tests/md_dict_tests.rs
+++ b/server/src/engine/data/tests/md_dict_tests.rs
@@ -25,7 +25,7 @@
 */
 
 use crate::engine::data::{
-    md_dict::{self, DictEntryGeneric, DictGeneric, MetaDict},
+    md_dict::{self, DictEntryGeneric, DictGeneric, MetaDict, MetaDictEntry},
     HSData,
 };
 
@@ -78,3 +78,26 @@ fn t_bad_patch() {
     assert!(!md_dict::rmerge_metadata(&mut current, new));
     assert_eq!(current, backup);
 }
+
+#[test]
+fn patch_null_out_dict() {
+    let mut current: MetaDict = into_dict! {
+        "a" => HSData::UnsignedInt(2),
+        "b" => HSData::UnsignedInt(3),
+        "z" => MetaDictEntry::Map(into_dict!(
+            "c" => HSData::UnsignedInt(1),
+            "d" => HSData::UnsignedInt(2)
+        )),
+    };
+    let expected: MetaDict = into_dict! {
+        "a" => HSData::UnsignedInt(2),
+        "b" => HSData::UnsignedInt(3),
+    };
+    let new: DictGeneric = into_dict! {
+        "a" => Some(HSData::UnsignedInt(2).into()),
+        "b" => Some(HSData::UnsignedInt(3).into()),
+        "z" => None,
+    };
+    assert!(md_dict::rmerge_metadata(&mut current, new));
+    assert_eq!(current, expected);
+}
diff --git a/server/src/engine/data/tests/metadata_dict.rs b/server/src/engine/data/tests/metadata_dict.rs
deleted file mode 100644
index e69de29b..00000000

From 1f608c1f6c7cb8991111f0379c6fcdcdd3acb5f0 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 20 Feb 2023 10:01:58 -0800
Subject: [PATCH 134/310] Support custom patch based mem tree delta

---
 server/src/engine/idx/mtchm/access.rs | 152 -----------
 server/src/engine/idx/mtchm/imp.rs    |  21 +-
 server/src/engine/idx/mtchm/mod.rs    |  99 ++++---
 server/src/engine/idx/mtchm/patch.rs  | 362 ++++++++++++++++++++++++++
 4 files changed, 431 insertions(+), 203 deletions(-)
 create mode 100644 server/src/engine/idx/mtchm/patch.rs

diff --git a/server/src/engine/idx/mtchm/access.rs b/server/src/engine/idx/mtchm/access.rs
index 4b5177dc..5f5a7647 100644
--- a/server/src/engine/idx/mtchm/access.rs
+++ b/server/src/engine/idx/mtchm/access.rs
@@ -26,16 +26,6 @@
 
 use super::meta::TreeElement;
 
-/// write mode flag
-type WriteFlag = u8;
-pub const WRITEMODE_DELETE: WriteFlag = 0xFF;
-/// fresh
-pub const WRITEMODE_FRESH: WriteFlag = 0b01;
-/// refresh
-pub const WRITEMODE_REFRESH: WriteFlag = 0b10;
-/// any
-pub const WRITEMODE_ANY: WriteFlag = 0b11;
-
 pub trait ReadMode<T>: 'static {
     type Ret<'a>;
     fn ex<'a>(v: &'a T) -> Self::Ret<'a>;
@@ -74,145 +64,3 @@ impl<T: TreeElement> ReadMode<T> for RModeClone {
         None
     }
 }
-
-pub trait WriteMode<T>: 'static {
-    const WMODE: WriteFlag;
-    type Ret<'a>;
-    fn ex<'a>(v: &'a T) -> Self::Ret<'a>;
-    fn nx<'a>() -> Self::Ret<'a>;
-}
-
-pub struct WModeFresh;
-impl<T> WriteMode<T> for WModeFresh {
-    const WMODE: WriteFlag = WRITEMODE_FRESH;
-    type Ret<'a> = bool;
-    #[inline(always)]
-    fn ex(_: &T) -> Self::Ret<'static> {
-        false
-    }
-    #[inline(always)]
-    fn nx<'a>() -> Self::Ret<'a> {
-        true
-    }
-}
-
-pub struct WModeUpdate;
-impl<T> WriteMode<T> for WModeUpdate {
-    const WMODE: WriteFlag = WRITEMODE_REFRESH;
-    type Ret<'a> = bool;
-    #[inline(always)]
-    fn ex(_: &T) -> Self::Ret<'static> {
-        true
-    }
-    #[inline(always)]
-    fn nx<'a>() -> Self::Ret<'a> {
-        false
-    }
-}
-
-pub struct WModeUpdateRetClone;
-impl<T: TreeElement> WriteMode<T> for WModeUpdateRetClone {
-    const WMODE: WriteFlag = WRITEMODE_REFRESH;
-    type Ret<'a> = Option<T::Value>;
-    #[inline(always)]
-    fn ex(v: &T) -> Self::Ret<'static> {
-        Some(v.val().clone())
-    }
-    #[inline(always)]
-    fn nx<'a>() -> Self::Ret<'a> {
-        None
-    }
-}
-
-pub struct WModeUpdateRetRef;
-impl<T: TreeElement> WriteMode<T> for WModeUpdateRetRef {
-    const WMODE: WriteFlag = WRITEMODE_REFRESH;
-    type Ret<'a> = Option<&'a T::Value>;
-    #[inline(always)]
-    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
-        Some(v.val())
-    }
-    #[inline(always)]
-    fn nx<'a>() -> Self::Ret<'a> {
-        None
-    }
-}
-
-pub struct WModeUpsert;
-impl<T> WriteMode<T> for WModeUpsert {
-    const WMODE: WriteFlag = WRITEMODE_ANY;
-    type Ret<'a> = ();
-    #[inline(always)]
-    fn ex(_: &T) -> Self::Ret<'static> {
-        ()
-    }
-    #[inline(always)]
-    fn nx<'a>() -> Self::Ret<'a> {
-        ()
-    }
-}
-
-pub struct WModeUpsertRef;
-impl<T: TreeElement> WriteMode<T> for WModeUpsertRef {
-    const WMODE: WriteFlag = WRITEMODE_ANY;
-    type Ret<'a> = Option<&'a T::Value>;
-    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
-        Some(v.val())
-    }
-    fn nx<'a>() -> Self::Ret<'a> {
-        None
-    }
-}
-
-pub struct WModeUpsertClone;
-impl<T: TreeElement> WriteMode<T> for WModeUpsertClone {
-    const WMODE: WriteFlag = WRITEMODE_ANY;
-    type Ret<'a> = Option<T::Value>;
-    fn ex<'a>(v: &'a T) -> Self::Ret<'static> {
-        Some(v.val().clone())
-    }
-    fn nx<'a>() -> Self::Ret<'a> {
-        None
-    }
-}
-pub struct WModeDelete;
-impl<T: TreeElement> WriteMode<T> for WModeDelete {
-    const WMODE: WriteFlag = WRITEMODE_DELETE;
-    type Ret<'a> = bool;
-    #[inline(always)]
-    fn ex<'a>(_: &'a T) -> Self::Ret<'a> {
-        true
-    }
-    #[inline(always)]
-    fn nx<'a>() -> Self::Ret<'a> {
-        false
-    }
-}
-
-pub struct WModeDeleteRef;
-impl<T: TreeElement> WriteMode<T> for WModeDeleteRef {
-    const WMODE: WriteFlag = WRITEMODE_DELETE;
-    type Ret<'a> = Option<&'a T::Value>;
-    #[inline(always)]
-    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
-        Some(v.val())
-    }
-    #[inline(always)]
-    fn nx<'a>() -> Self::Ret<'a> {
-        None
-    }
-}
-
-pub struct WModeDeleteClone;
-impl<T: TreeElement> WriteMode<T> for WModeDeleteClone {
-    const WMODE: WriteFlag = WRITEMODE_DELETE;
-    type Ret<'a> = Option<T::Value>;
-    #[inline(always)]
-    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
-        Some(v.val().clone())
-    }
-    #[inline(always)]
-    fn nx<'a>() -> Self::Ret<'a> {
-        None
-    }
-}
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index 01f0151f..df435219 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -72,7 +72,6 @@ where
 impl<K, V, C> MTIndex<K, V> for ChmArc<K, V, C>
 where
     C: Config,
-
     K: Key,
     V: Value,
 {
@@ -103,11 +102,11 @@ where
     }
 
     fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool {
-        self.insert(arc(key, val), g)
+        self.patch_insert(key, val, g)
     }
 
     fn mt_upsert(&self, key: K, val: V, g: &Guard) {
-        self.upsert(arc(key, val), g)
+        self.patch_upsert(key, val, g)
     }
 
     fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
@@ -137,7 +136,7 @@ where
     }
 
     fn mt_update(&self, key: K, val: V, g: &Guard) -> bool {
-        self.update(arc(key, val), g)
+        self.patch_update(key, val, g)
     }
 
     fn mt_update_return<'t, 'g, 'v>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
@@ -145,7 +144,7 @@ where
         't: 'v,
         'g: 't + 'v,
     {
-        self.update_return(arc(key, val), g)
+        self.patch_update_return(key, val, g)
     }
 
     fn mt_delete<Q>(&self, key: &Q, g: &Guard) -> bool
@@ -195,7 +194,6 @@ where
 impl<K, V, C> MTIndex<K, V> for ChmCopy<K, V, C>
 where
     C: Config,
-
     K: Key,
     V: Value,
 {
@@ -226,11 +224,11 @@ where
     }
 
     fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool {
-        self.insert((key, val), g)
+        self.patch_insert(key, val, g)
     }
 
     fn mt_upsert(&self, key: K, val: V, g: &Guard) {
-        self.upsert((key, val), g)
+        self.patch_upsert(key, val, g)
     }
 
     fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
@@ -260,7 +258,7 @@ where
     }
 
     fn mt_update(&self, key: K, val: V, g: &Guard) -> bool {
-        self.update((key, val), g)
+        self.patch_update(key, val, g)
     }
 
     fn mt_update_return<'t, 'g, 'v>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
@@ -268,7 +266,7 @@ where
         't: 'v,
         'g: 't + 'v,
     {
-        self.update_return((key, val), g)
+        self.patch_update_return(key, val, g)
     }
 
     fn mt_delete<Q>(&self, key: &Q, g: &Guard) -> bool
@@ -297,7 +295,8 @@ impl<T: TreeElement, C: Config> FromIterator<T> for Tree<T, C> {
             upin()
         };
         let t = Tree::new();
-        iter.into_iter().for_each(|te| assert!(t.insert(te, &g)));
+        iter.into_iter()
+            .for_each(|te| assert!(t.patch_insert(te.key().clone(), te.val().clone(), &g)));
         t
     }
 }
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index b10d0c7b..df33a7d6 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -28,6 +28,7 @@ mod access;
 pub(super) mod imp;
 mod iter;
 pub(super) mod meta;
+mod patch;
 #[cfg(test)]
 mod tests;
 
@@ -35,9 +36,9 @@ mod tests;
 use crate::engine::sync::atm::ORD_ACQ;
 use {
     self::{
-        access::{ReadMode, WriteMode},
         iter::{IterKV, IterKey, IterVal},
         meta::{CompressState, Config, DefConfig, LNode, NodeFlag, TreeElement},
+        patch::{TreeKeyComparable, TreeKeyComparableUpgradeable},
     },
     crate::engine::{
         idx::AsKey,
@@ -47,6 +48,7 @@ use {
     crossbeam_epoch::CompareExchangeError,
     std::{
         borrow::Borrow,
+        hash::Hash,
         hash::{BuildHasher, Hasher},
         marker::PhantomData,
         mem,
@@ -213,7 +215,7 @@ impl<T, C: Config> Tree<T, C> {
 impl<T, C: Config> Tree<T, C> {
     fn hash<Q>(&self, k: &Q) -> u64
     where
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Hash,
     {
         let mut state = self.h.build_hasher();
         k.hash(&mut state);
@@ -240,23 +242,38 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
             let _ = self.remove(k, g);
         });
     }
-    fn insert(&self, elem: T, g: &Guard) -> bool {
-        self._insert::<access::WModeFresh>(elem, g)
-    }
-    fn update(&self, elem: T, g: &Guard) -> bool {
-        self._insert::<access::WModeUpdate>(elem, g)
-    }
-    fn update_return<'g>(&'g self, elem: T, g: &'g Guard) -> Option<&'g T::Value> {
-        self._insert::<access::WModeUpdateRetRef>(elem, g)
+    fn patch_insert<Q: TreeKeyComparableUpgradeable<T>>(
+        &self,
+        key: Q,
+        v: T::Value,
+        g: &Guard,
+    ) -> bool {
+        self.patch(patch::Insert::new(key, v), g)
+    }
+    fn patch_upsert<Q: TreeKeyComparableUpgradeable<T>>(&self, key: Q, v: T::Value, g: &Guard) {
+        self.patch(patch::Upsert::new(key, v), g)
+    }
+    fn patch_upsert_return<'g, Q: TreeKeyComparableUpgradeable<T>>(
+        &'g self,
+        key: Q,
+        v: T::Value,
+        g: &'g Guard,
+    ) -> Option<&'g T::Value> {
+        self.patch(patch::UpsertReturn::new(key, v), g)
     }
-    fn upsert(&self, elem: T, g: &Guard) {
-        self._insert::<access::WModeUpsert>(elem, g)
+    fn patch_update<Q: TreeKeyComparable<T>>(&self, key: Q, v: T::Value, g: &Guard) -> bool {
+        self.patch(patch::UpdateReplace::new(key, v), g)
     }
-    fn upsert_return<'g>(&'g self, elem: T, g: &'g Guard) -> Option<&'g T::Value> {
-        self._insert::<access::WModeUpsertRef>(elem, g)
+    fn patch_update_return<'g, Q: TreeKeyComparable<T>>(
+        &'g self,
+        key: Q,
+        v: T::Value,
+        g: &'g Guard,
+    ) -> Option<&'g T::Value> {
+        self.patch(patch::UpdateReplaceRet::new(key, v), g)
     }
-    fn _insert<'g, W: WriteMode<T>>(&'g self, elem: T, g: &'g Guard) -> W::Ret<'g> {
-        let hash = self.hash(elem.key());
+    fn patch<'g, P: patch::Patch<T>>(&'g self, mut patch: P, g: &'g Guard) -> P::Ret<'g> {
+        let hash = self.hash(&patch.target());
         let mut level = C::LEVEL_ZERO;
         let mut current = &self.root;
         let mut parent = None;
@@ -282,17 +299,17 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                 }
                 _ if node.is_null() => {
                     // this is an empty slot
-                    if W::WMODE == access::WRITEMODE_REFRESH {
+                    if P::WMODE == patch::WRITEMODE_REFRESH {
                         // I call that a job well done
-                        return W::nx();
+                        return P::nx_ret();
                     }
-                    if (W::WMODE == access::WRITEMODE_ANY) | (W::WMODE == access::WRITEMODE_FRESH) {
-                        let new = Self::new_data(elem.clone());
+                    if (P::WMODE == patch::WRITEMODE_ANY) | (P::WMODE == patch::WRITEMODE_FRESH) {
+                        let new = Self::new_data(patch.nx_new());
                         match current.cx_rel(node, new, g) {
                             Ok(_) => {
                                 // we're done here
                                 self.incr_len();
-                                return W::nx();
+                                return P::nx_ret();
                             }
                             Err(CompareExchangeError { new, .. }) => unsafe {
                                 /*
@@ -312,15 +329,16 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                         Self::read_data(node)
                     };
                     debug_assert!(!data.is_empty(), "logic,empty node not compressed");
-                    if data[0].key() != elem.key() && level < C::MAX_TREE_HEIGHT_UB {
+                    if !patch.target().cmp_eq(&data[0]) && level < C::MAX_TREE_HEIGHT_UB {
                         /*
                             so this is a collision and since we haven't reached the max height, we should always
                             create a new branch so let's do that
                         */
                         self.m.hsplit();
                         debug_assert_eq!(data.len(), 1, "logic,lnode before height ub");
-                        if W::WMODE == access::WRITEMODE_REFRESH {
+                        if P::WMODE == patch::WRITEMODE_REFRESH {
                             // another job well done; an snode with the wrong key; so basically it's missing
+                            return P::nx_ret();
                         }
                         let next_chunk = (self.hash(data[0].key()) >> level) & C::MASK;
                         let mut new_branch = Node::null();
@@ -333,20 +351,20 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                             in this case we either have the same key or we found an lnode. resolve any conflicts and attempt
                             to update
                         */
-                        let p = data.iter().position(|e| e.key() == elem.key());
+                        let p = data.iter().position(|e| patch.target().cmp_eq(e));
                         match p {
-                            Some(v) if W::WMODE == access::WRITEMODE_FRESH => {
-                                return W::ex(&data[v])
+                            Some(v) if P::WMODE == patch::WRITEMODE_FRESH => {
+                                return P::ex_ret(&data[v])
                             }
                             Some(i)
-                                if W::WMODE == access::WRITEMODE_REFRESH
-                                    || W::WMODE == access::WRITEMODE_ANY =>
+                                if P::WMODE == patch::WRITEMODE_REFRESH
+                                    || P::WMODE == patch::WRITEMODE_ANY =>
                             {
                                 // update the entry and create a new node
                                 let mut new_ln = LNode::new();
                                 new_ln.extend(data[..i].iter().cloned());
                                 new_ln.extend(data[i + 1..].iter().cloned());
-                                new_ln.push(elem.clone());
+                                new_ln.push(patch.ex_apply(&data[i]));
                                 match current.cx_rel(node, Self::new_lnode(new_ln), g) {
                                     Ok(new) => {
                                         if cfg!(debug_assertions)
@@ -363,7 +381,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                                                 node.as_raw() as *const LNode<_>
                                             ))
                                         }
-                                        return W::ex(&data[i]);
+                                        return P::ex_ret(&data[i]);
                                     }
                                     Err(CompareExchangeError { new, .. }) => {
                                         // failed to swap it in
@@ -373,12 +391,12 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                                     }
                                 }
                             }
-                            None if W::WMODE == access::WRITEMODE_ANY
-                                || W::WMODE == access::WRITEMODE_FRESH =>
+                            None if P::WMODE == patch::WRITEMODE_ANY
+                                || P::WMODE == patch::WRITEMODE_FRESH =>
                             {
                                 // no funk here
                                 let mut new_node = data.clone();
-                                new_node.push(elem.clone());
+                                new_node.push(patch.nx_new());
                                 match current.cx_rel(node, Self::new_lnode(new_node), g) {
                                     Ok(new) => {
                                         if cfg!(debug_assertions)
@@ -394,7 +412,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                                             ));
                                         }
                                         self.incr_len();
-                                        return W::nx();
+                                        return P::nx_ret();
                                     }
                                     Err(CompareExchangeError { new, .. }) => {
                                         // failed to swap it
@@ -405,9 +423,9 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                                     }
                                 }
                             }
-                            None if W::WMODE == access::WRITEMODE_REFRESH => return W::nx(),
+                            None if P::WMODE == patch::WRITEMODE_REFRESH => return P::nx_ret(),
                             _ => {
-                                unreachable!("logic, WMODE mismatch: `{}`", W::WMODE);
+                                unreachable!("logic, WMODE mismatch: `{}`", P::WMODE);
                             }
                         }
                     }
@@ -423,6 +441,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
             }
         }
     }
+
     fn contains_key<'g, Q>(&'g self, k: &Q, g: &'g Guard) -> bool
     where
         T::Key: Borrow<Q>,
@@ -437,7 +456,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
     {
         self._lookup::<Q, access::RModeRef>(k, g)
     }
-    fn _lookup<'g, Q, R: ReadMode<T>>(&'g self, k: &Q, g: &'g Guard) -> R::Ret<'g>
+    fn _lookup<'g, Q, R: access::ReadMode<T>>(&'g self, k: &Q, g: &'g Guard) -> R::Ret<'g>
     where
         T::Key: Borrow<Q>,
         Q: AsKey + ?Sized,
@@ -477,16 +496,16 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
         T::Key: Borrow<Q>,
         Q: AsKey + ?Sized,
     {
-        self._remove::<Q, access::WModeDelete>(k, g)
+        self._remove::<Q, patch::Delete>(k, g)
     }
     fn remove_return<'g, Q>(&'g self, k: &Q, g: &'g Guard) -> Option<&'g T::Value>
     where
         T::Key: Borrow<Q>,
         Q: AsKey + ?Sized,
     {
-        self._remove::<Q, access::WModeDeleteRef>(k, g)
+        self._remove::<Q, patch::DeleteRet>(k, g)
     }
-    fn _remove<'g, Q, W: WriteMode<T>>(&'g self, k: &Q, g: &'g Guard) -> W::Ret<'g>
+    fn _remove<'g, Q, W: patch::PatchDelete<T>>(&'g self, k: &Q, g: &'g Guard) -> W::Ret<'g>
     where
         T::Key: Borrow<Q>,
         Q: AsKey + ?Sized,
diff --git a/server/src/engine/idx/mtchm/patch.rs b/server/src/engine/idx/mtchm/patch.rs
new file mode 100644
index 00000000..5c9db438
--- /dev/null
+++ b/server/src/engine/idx/mtchm/patch.rs
@@ -0,0 +1,362 @@
+/*
+ * Created on Sun Feb 19 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use std::borrow::Borrow;
+
+use {
+    super::meta::TreeElement,
+    core::{hash::Hash, marker::PhantomData},
+};
+
+/// write mode flag
+pub type WriteFlag = u8;
+pub const WRITEMODE_DELETE: WriteFlag = 0xFF;
+/// fresh
+pub const WRITEMODE_FRESH: WriteFlag = 0b01;
+/// refresh
+pub const WRITEMODE_REFRESH: WriteFlag = 0b10;
+/// any
+pub const WRITEMODE_ANY: WriteFlag = 0b11;
+
+pub trait TreeKeyComparable<T: TreeElement>: Hash {
+    fn cmp_eq(&self, them: &T) -> bool;
+}
+
+impl<E: TreeElement, T> TreeKeyComparable<E> for T
+where
+    E::Key: core::borrow::Borrow<T>,
+    T: Hash + PartialEq,
+{
+    fn cmp_eq(&self, them: &E) -> bool {
+        self == them.key().borrow()
+    }
+}
+
+pub trait TreeKeyComparableUpgradeable<T: TreeElement>: TreeKeyComparable<T> {
+    fn upgrade_key(&self) -> T::Key;
+}
+
+impl<E: TreeElement> TreeKeyComparableUpgradeable<E> for E::Key {
+    fn upgrade_key(&self) -> E::Key {
+        self.clone()
+    }
+}
+
+/// A [`Patch`] is intended to atomically update the state of the tree, which means that all your deltas should be atomic
+pub trait Patch<E: TreeElement> {
+    const WMODE: WriteFlag;
+    type Ret<'a>;
+    type Target<'a>: TreeKeyComparable<E>
+    where
+        Self: 'a;
+    fn target<'a>(&'a self) -> &Self::Target<'a>;
+    fn nx_new(&mut self) -> E;
+    fn nx_ret<'a>() -> Self::Ret<'a>;
+    fn ex_apply(&mut self, current: &E) -> E;
+    fn ex_ret<'a>(current: &'a E) -> Self::Ret<'a>;
+}
+
+/// insert
+pub struct Insert<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> {
+    target: U,
+    new_data: E::Value,
+    _m: PhantomData<E>,
+}
+
+impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Insert<E, U> {
+    pub fn new(target: U, new_data: E::Value) -> Self {
+        Self {
+            target,
+            new_data,
+            _m: PhantomData,
+        }
+    }
+}
+
+impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Patch<E> for Insert<E, U> {
+    const WMODE: WriteFlag = WRITEMODE_FRESH;
+    type Ret<'a> = bool;
+    type Target<'a> = U
+    where
+        Self: 'a;
+
+    fn target<'a>(&'a self) -> &Self::Target<'a> {
+        &self.target
+    }
+    fn nx_new(&mut self) -> E {
+        E::new(self.target.upgrade_key(), self.new_data.clone())
+    }
+    fn nx_ret<'a>() -> Self::Ret<'a> {
+        true
+    }
+    fn ex_apply(&mut self, _: &E) -> E {
+        unreachable!()
+    }
+    fn ex_ret<'a>(_: &'a E) -> Self::Ret<'a> {
+        false
+    }
+}
+
+/// upsert
+pub struct Upsert<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> {
+    target: U,
+    new_data: E::Value,
+    _m: PhantomData<E>,
+}
+
+impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Upsert<E, U> {
+    pub fn new(target: U, new_data: E::Value) -> Self {
+        Self {
+            target,
+            new_data,
+            _m: PhantomData,
+        }
+    }
+}
+
+impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Patch<E> for Upsert<E, U> {
+    const WMODE: WriteFlag = WRITEMODE_ANY;
+    type Ret<'a> = ();
+    type Target<'a> = U
+    where
+        Self: 'a;
+
+    fn target<'a>(&'a self) -> &Self::Target<'a> {
+        &self.target
+    }
+    fn nx_new(&mut self) -> E {
+        E::new(self.target.upgrade_key(), self.new_data.clone())
+    }
+    fn nx_ret<'a>() -> Self::Ret<'a> {
+        ()
+    }
+    fn ex_apply(&mut self, _: &E) -> E {
+        self.nx_new()
+    }
+    fn ex_ret<'a>(_: &'a E) -> Self::Ret<'a> {
+        ()
+    }
+}
+
+/// upsert return
+pub struct UpsertReturn<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> {
+    target: U,
+    new_data: E::Value,
+    _m: PhantomData<E>,
+}
+
+impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> UpsertReturn<E, U> {
+    pub fn new(target: U, new_data: E::Value) -> Self {
+        Self {
+            target,
+            new_data,
+            _m: PhantomData,
+        }
+    }
+}
+
+impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Patch<E> for UpsertReturn<E, U> {
+    const WMODE: WriteFlag = WRITEMODE_ANY;
+    type Ret<'a> = Option<&'a E::Value>;
+    type Target<'a> = U
+    where
+        Self: 'a;
+
+    fn target<'a>(&'a self) -> &Self::Target<'a> {
+        &self.target
+    }
+    fn nx_new(&mut self) -> E {
+        E::new(self.target.upgrade_key(), self.new_data.clone())
+    }
+    fn nx_ret<'a>() -> Self::Ret<'a> {
+        None
+    }
+    fn ex_apply(&mut self, _: &E) -> E {
+        self.nx_new()
+    }
+    fn ex_ret<'a>(e: &'a E) -> Self::Ret<'a> {
+        Some(e.val())
+    }
+}
+
+/// update
+pub struct UpdateReplace<E: TreeElement, U: TreeKeyComparable<E>> {
+    target: U,
+    new_data: E::Value,
+    _m: PhantomData<E>,
+}
+
+impl<E: TreeElement, U: TreeKeyComparable<E>> UpdateReplace<E, U> {
+    pub fn new(target: U, new_data: E::Value) -> Self {
+        Self {
+            target,
+            new_data,
+            _m: PhantomData,
+        }
+    }
+}
+
+impl<E: TreeElement, U: TreeKeyComparable<E>> Patch<E> for UpdateReplace<E, U> {
+    const WMODE: WriteFlag = WRITEMODE_REFRESH;
+
+    type Ret<'a> = bool;
+
+    type Target<'a> = U
+    where
+        Self: 'a;
+
+    fn target<'a>(&'a self) -> &Self::Target<'a> {
+        &self.target
+    }
+
+    fn nx_new(&mut self) -> E {
+        unreachable!()
+    }
+
+    fn nx_ret<'a>() -> Self::Ret<'a> {
+        false
+    }
+
+    fn ex_apply(&mut self, c: &E) -> E {
+        E::new(c.key().clone(), self.new_data.clone())
+    }
+
+    fn ex_ret<'a>(_: &'a E) -> Self::Ret<'a> {
+        true
+    }
+}
+
+/// update_return
+pub struct UpdateReplaceRet<E: TreeElement, U: TreeKeyComparable<E>> {
+    target: U,
+    new_data: E::Value,
+    _m: PhantomData<E>,
+}
+
+impl<E: TreeElement, U: TreeKeyComparable<E>> UpdateReplaceRet<E, U> {
+    pub fn new(target: U, new_data: E::Value) -> Self {
+        Self {
+            target,
+            new_data,
+            _m: PhantomData,
+        }
+    }
+}
+
+impl<E: TreeElement, U: TreeKeyComparable<E>> Patch<E> for UpdateReplaceRet<E, U> {
+    const WMODE: WriteFlag = WRITEMODE_REFRESH;
+
+    type Ret<'a> = Option<&'a E::Value>;
+
+    type Target<'a> = U
+    where
+        Self: 'a;
+
+    fn target<'a>(&'a self) -> &Self::Target<'a> {
+        &self.target
+    }
+
+    fn nx_new(&mut self) -> E {
+        unreachable!()
+    }
+
+    fn nx_ret<'a>() -> Self::Ret<'a> {
+        None
+    }
+
+    fn ex_apply(&mut self, c: &E) -> E {
+        E::new(c.key().clone(), self.new_data.clone())
+    }
+
+    fn ex_ret<'a>(c: &'a E) -> Self::Ret<'a> {
+        Some(c.val())
+    }
+}
+
+pub struct InsertDirect<T: TreeElement> {
+    data: T,
+}
+
+impl<T: TreeElement> InsertDirect<T> {
+    pub fn new(key: T::Key, val: T::Value) -> Self {
+        Self {
+            data: T::new(key, val),
+        }
+    }
+}
+
+impl<T: TreeElement> Patch<T> for InsertDirect<T> {
+    const WMODE: WriteFlag = WRITEMODE_FRESH;
+    type Ret<'a> = bool;
+    type Target<'a> = T::Key;
+    fn target<'a>(&'a self) -> &Self::Target<'a> {
+        self.data.key()
+    }
+    fn nx_ret<'a>() -> Self::Ret<'a> {
+        true
+    }
+    fn nx_new(&mut self) -> T {
+        self.data.clone()
+    }
+    fn ex_apply(&mut self, _: &T) -> T {
+        unreachable!()
+    }
+    fn ex_ret<'a>(_: &'a T) -> Self::Ret<'a> {
+        false
+    }
+}
+
+pub trait PatchDelete<T>: 'static {
+    type Ret<'a>;
+    fn ex<'a>(v: &'a T) -> Self::Ret<'a>;
+    fn nx<'a>() -> Self::Ret<'a>;
+}
+
+pub struct Delete;
+impl<T: TreeElement> PatchDelete<T> for Delete {
+    type Ret<'a> = bool;
+    #[inline(always)]
+    fn ex<'a>(_: &'a T) -> Self::Ret<'a> {
+        true
+    }
+    #[inline(always)]
+    fn nx<'a>() -> Self::Ret<'a> {
+        false
+    }
+}
+
+pub struct DeleteRet;
+impl<T: TreeElement> PatchDelete<T> for DeleteRet {
+    type Ret<'a> = Option<&'a T::Value>;
+    #[inline(always)]
+    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
+        Some(v.val())
+    }
+    #[inline(always)]
+    fn nx<'a>() -> Self::Ret<'a> {
+        None
+    }
+}

From 70810445cb10c18952ace0f76d59a82d2a128e8e Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 21 Feb 2023 08:42:36 -0800
Subject: [PATCH 135/310] Make cmp more obvious

---
 server/src/engine/idx/meta.rs        |  35 ++++++++-
 server/src/engine/idx/mtchm/mod.rs   |  24 ++++---
 server/src/engine/idx/mtchm/patch.rs | 103 +++++++++------------------
 3 files changed, 81 insertions(+), 81 deletions(-)

diff --git a/server/src/engine/idx/meta.rs b/server/src/engine/idx/meta.rs
index c486e87b..24c4b317 100644
--- a/server/src/engine/idx/meta.rs
+++ b/server/src/engine/idx/meta.rs
@@ -24,7 +24,40 @@
  *
 */
 
-use std::hash::BuildHasher;
+use core::{
+    borrow::Borrow,
+    hash::{BuildHasher, Hash},
+};
 
 pub trait AsHasher: BuildHasher + Default {}
 impl<T> AsHasher for T where T: BuildHasher + Default {}
+
+/// The [`Comparable`] trait is like [`PartialEq`], but is different due to its expectations, and escapes its scandalous relations with [`Eq`] and the consequential
+/// implications across the [`std`].
+///
+/// ☢️ WARNING ☢️: In some cases implementations of the [`Comparable`] set of traits COMPLETELY VIOLATES [`Eq`]'s invariants. BE VERY CAREFUL WHEN USING IN EXPRESSIONS
+/*
+    FIXME(@ohsayan): The gradual idea is to completely move to Comparable, but that means we'll have to go ahead as much as replacing the impls for some items in the
+    standard library. We don't have the time to do that right now, but I hope we can do it soon
+*/
+pub trait Comparable<K: ?Sized>: Hash {
+    fn cmp_eq(&self, key: &K) -> bool;
+}
+
+pub trait ComparableUpgradeable<K>: Comparable<K> {
+    fn upgrade(&self) -> K;
+}
+
+impl<K: Borrow<T>, T: Eq + Hash + ?Sized> Comparable<K> for T {
+    fn cmp_eq(&self, key: &K) -> bool {
+        self == key.borrow()
+    }
+}
+
+impl<K: Hash, T: ToOwned<Owned = K> + Hash + Comparable<K> + ?Sized> ComparableUpgradeable<K>
+    for T
+{
+    fn upgrade(&self) -> K {
+        self.to_owned()
+    }
+}
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index df33a7d6..beae4e08 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -38,10 +38,12 @@ use {
     self::{
         iter::{IterKV, IterKey, IterVal},
         meta::{CompressState, Config, DefConfig, LNode, NodeFlag, TreeElement},
-        patch::{TreeKeyComparable, TreeKeyComparableUpgradeable},
     },
     crate::engine::{
-        idx::AsKey,
+        idx::{
+            meta::{Comparable, ComparableUpgradeable},
+            AsKey,
+        },
         mem::UArray,
         sync::atm::{self, cpin, upin, Atomic, Guard, Owned, Shared, ORD_ACR, ORD_RLX},
     },
@@ -242,7 +244,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
             let _ = self.remove(k, g);
         });
     }
-    fn patch_insert<Q: TreeKeyComparableUpgradeable<T>>(
+    fn patch_insert<Q: ComparableUpgradeable<T::Key>>(
         &self,
         key: Q,
         v: T::Value,
@@ -250,10 +252,10 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
     ) -> bool {
         self.patch(patch::Insert::new(key, v), g)
     }
-    fn patch_upsert<Q: TreeKeyComparableUpgradeable<T>>(&self, key: Q, v: T::Value, g: &Guard) {
+    fn patch_upsert<Q: ComparableUpgradeable<T::Key>>(&self, key: Q, v: T::Value, g: &Guard) {
         self.patch(patch::Upsert::new(key, v), g)
     }
-    fn patch_upsert_return<'g, Q: TreeKeyComparableUpgradeable<T>>(
+    fn patch_upsert_return<'g, Q: ComparableUpgradeable<T::Key>>(
         &'g self,
         key: Q,
         v: T::Value,
@@ -261,10 +263,10 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
     ) -> Option<&'g T::Value> {
         self.patch(patch::UpsertReturn::new(key, v), g)
     }
-    fn patch_update<Q: TreeKeyComparable<T>>(&self, key: Q, v: T::Value, g: &Guard) -> bool {
+    fn patch_update<Q: Comparable<T::Key>>(&self, key: Q, v: T::Value, g: &Guard) -> bool {
         self.patch(patch::UpdateReplace::new(key, v), g)
     }
-    fn patch_update_return<'g, Q: TreeKeyComparable<T>>(
+    fn patch_update_return<'g, Q: Comparable<T::Key>>(
         &'g self,
         key: Q,
         v: T::Value,
@@ -273,7 +275,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
         self.patch(patch::UpdateReplaceRet::new(key, v), g)
     }
     fn patch<'g, P: patch::Patch<T>>(&'g self, mut patch: P, g: &'g Guard) -> P::Ret<'g> {
-        let hash = self.hash(&patch.target());
+        let hash = self.hash(patch.target());
         let mut level = C::LEVEL_ZERO;
         let mut current = &self.root;
         let mut parent = None;
@@ -329,7 +331,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                         Self::read_data(node)
                     };
                     debug_assert!(!data.is_empty(), "logic,empty node not compressed");
-                    if !patch.target().cmp_eq(&data[0]) && level < C::MAX_TREE_HEIGHT_UB {
+                    if !patch.target().cmp_eq(data[0].key()) && level < C::MAX_TREE_HEIGHT_UB {
                         /*
                             so this is a collision and since we haven't reached the max height, we should always
                             create a new branch so let's do that
@@ -351,7 +353,9 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                             in this case we either have the same key or we found an lnode. resolve any conflicts and attempt
                             to update
                         */
-                        let p = data.iter().position(|e| patch.target().cmp_eq(e));
+                        let p = data
+                            .iter()
+                            .position(|e| patch.target().cmp_eq(e.key()));
                         match p {
                             Some(v) if P::WMODE == patch::WRITEMODE_FRESH => {
                                 return P::ex_ret(&data[v])
diff --git a/server/src/engine/idx/mtchm/patch.rs b/server/src/engine/idx/mtchm/patch.rs
index 5c9db438..2cf645d7 100644
--- a/server/src/engine/idx/mtchm/patch.rs
+++ b/server/src/engine/idx/mtchm/patch.rs
@@ -24,10 +24,9 @@
  *
 */
 
-use std::borrow::Borrow;
-
 use {
     super::meta::TreeElement,
+    crate::engine::idx::meta::{Comparable, ComparableUpgradeable},
     core::{hash::Hash, marker::PhantomData},
 };
 
@@ -41,38 +40,12 @@ pub const WRITEMODE_REFRESH: WriteFlag = 0b10;
 /// any
 pub const WRITEMODE_ANY: WriteFlag = 0b11;
 
-pub trait TreeKeyComparable<T: TreeElement>: Hash {
-    fn cmp_eq(&self, them: &T) -> bool;
-}
-
-impl<E: TreeElement, T> TreeKeyComparable<E> for T
-where
-    E::Key: core::borrow::Borrow<T>,
-    T: Hash + PartialEq,
-{
-    fn cmp_eq(&self, them: &E) -> bool {
-        self == them.key().borrow()
-    }
-}
-
-pub trait TreeKeyComparableUpgradeable<T: TreeElement>: TreeKeyComparable<T> {
-    fn upgrade_key(&self) -> T::Key;
-}
-
-impl<E: TreeElement> TreeKeyComparableUpgradeable<E> for E::Key {
-    fn upgrade_key(&self) -> E::Key {
-        self.clone()
-    }
-}
-
 /// A [`Patch`] is intended to atomically update the state of the tree, which means that all your deltas should be atomic
 pub trait Patch<E: TreeElement> {
     const WMODE: WriteFlag;
     type Ret<'a>;
-    type Target<'a>: TreeKeyComparable<E>
-    where
-        Self: 'a;
-    fn target<'a>(&'a self) -> &Self::Target<'a>;
+    type Target: Hash + Comparable<E::Key>;
+    fn target<'a>(&'a self) -> &Self::Target;
     fn nx_new(&mut self) -> E;
     fn nx_ret<'a>() -> Self::Ret<'a>;
     fn ex_apply(&mut self, current: &E) -> E;
@@ -80,13 +53,13 @@ pub trait Patch<E: TreeElement> {
 }
 
 /// insert
-pub struct Insert<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> {
+pub struct Insert<E: TreeElement, U: ComparableUpgradeable<E::Key>> {
     target: U,
     new_data: E::Value,
     _m: PhantomData<E>,
 }
 
-impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Insert<E, U> {
+impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> Insert<E, U> {
     pub fn new(target: U, new_data: E::Value) -> Self {
         Self {
             target,
@@ -96,18 +69,16 @@ impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Insert<E, U> {
     }
 }
 
-impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Patch<E> for Insert<E, U> {
+impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> Patch<E> for Insert<E, U> {
     const WMODE: WriteFlag = WRITEMODE_FRESH;
     type Ret<'a> = bool;
-    type Target<'a> = U
-    where
-        Self: 'a;
+    type Target = U;
 
-    fn target<'a>(&'a self) -> &Self::Target<'a> {
+    fn target<'a>(&'a self) -> &Self::Target {
         &self.target
     }
     fn nx_new(&mut self) -> E {
-        E::new(self.target.upgrade_key(), self.new_data.clone())
+        E::new(self.target.upgrade(), self.new_data.clone())
     }
     fn nx_ret<'a>() -> Self::Ret<'a> {
         true
@@ -121,13 +92,13 @@ impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Patch<E> for Insert<E,
 }
 
 /// upsert
-pub struct Upsert<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> {
+pub struct Upsert<E: TreeElement, U: ComparableUpgradeable<E::Key>> {
     target: U,
     new_data: E::Value,
     _m: PhantomData<E>,
 }
 
-impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Upsert<E, U> {
+impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> Upsert<E, U> {
     pub fn new(target: U, new_data: E::Value) -> Self {
         Self {
             target,
@@ -137,18 +108,16 @@ impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Upsert<E, U> {
     }
 }
 
-impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Patch<E> for Upsert<E, U> {
+impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> Patch<E> for Upsert<E, U> {
     const WMODE: WriteFlag = WRITEMODE_ANY;
     type Ret<'a> = ();
-    type Target<'a> = U
-    where
-        Self: 'a;
+    type Target = U;
 
-    fn target<'a>(&'a self) -> &Self::Target<'a> {
+    fn target<'a>(&'a self) -> &Self::Target {
         &self.target
     }
     fn nx_new(&mut self) -> E {
-        E::new(self.target.upgrade_key(), self.new_data.clone())
+        E::new(self.target.upgrade(), self.new_data.clone())
     }
     fn nx_ret<'a>() -> Self::Ret<'a> {
         ()
@@ -162,13 +131,13 @@ impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Patch<E> for Upsert<E,
 }
 
 /// upsert return
-pub struct UpsertReturn<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> {
+pub struct UpsertReturn<E: TreeElement, U: ComparableUpgradeable<E::Key>> {
     target: U,
     new_data: E::Value,
     _m: PhantomData<E>,
 }
 
-impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> UpsertReturn<E, U> {
+impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> UpsertReturn<E, U> {
     pub fn new(target: U, new_data: E::Value) -> Self {
         Self {
             target,
@@ -178,18 +147,16 @@ impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> UpsertReturn<E, U> {
     }
 }
 
-impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Patch<E> for UpsertReturn<E, U> {
+impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> Patch<E> for UpsertReturn<E, U> {
     const WMODE: WriteFlag = WRITEMODE_ANY;
     type Ret<'a> = Option<&'a E::Value>;
-    type Target<'a> = U
-    where
-        Self: 'a;
+    type Target = U;
 
-    fn target<'a>(&'a self) -> &Self::Target<'a> {
+    fn target<'a>(&'a self) -> &Self::Target {
         &self.target
     }
     fn nx_new(&mut self) -> E {
-        E::new(self.target.upgrade_key(), self.new_data.clone())
+        E::new(self.target.upgrade(), self.new_data.clone())
     }
     fn nx_ret<'a>() -> Self::Ret<'a> {
         None
@@ -203,13 +170,13 @@ impl<E: TreeElement, U: TreeKeyComparableUpgradeable<E>> Patch<E> for UpsertRetu
 }
 
 /// update
-pub struct UpdateReplace<E: TreeElement, U: TreeKeyComparable<E>> {
+pub struct UpdateReplace<E: TreeElement, U: Comparable<E::Key>> {
     target: U,
     new_data: E::Value,
     _m: PhantomData<E>,
 }
 
-impl<E: TreeElement, U: TreeKeyComparable<E>> UpdateReplace<E, U> {
+impl<E: TreeElement, U: Comparable<E::Key>> UpdateReplace<E, U> {
     pub fn new(target: U, new_data: E::Value) -> Self {
         Self {
             target,
@@ -219,16 +186,14 @@ impl<E: TreeElement, U: TreeKeyComparable<E>> UpdateReplace<E, U> {
     }
 }
 
-impl<E: TreeElement, U: TreeKeyComparable<E>> Patch<E> for UpdateReplace<E, U> {
+impl<E: TreeElement, U: Comparable<E::Key>> Patch<E> for UpdateReplace<E, U> {
     const WMODE: WriteFlag = WRITEMODE_REFRESH;
 
     type Ret<'a> = bool;
 
-    type Target<'a> = U
-    where
-        Self: 'a;
+    type Target = U;
 
-    fn target<'a>(&'a self) -> &Self::Target<'a> {
+    fn target<'a>(&'a self) -> &Self::Target {
         &self.target
     }
 
@@ -250,13 +215,13 @@ impl<E: TreeElement, U: TreeKeyComparable<E>> Patch<E> for UpdateReplace<E, U> {
 }
 
 /// update_return
-pub struct UpdateReplaceRet<E: TreeElement, U: TreeKeyComparable<E>> {
+pub struct UpdateReplaceRet<E: TreeElement, U: Comparable<E::Key>> {
     target: U,
     new_data: E::Value,
     _m: PhantomData<E>,
 }
 
-impl<E: TreeElement, U: TreeKeyComparable<E>> UpdateReplaceRet<E, U> {
+impl<E: TreeElement, U: Comparable<E::Key>> UpdateReplaceRet<E, U> {
     pub fn new(target: U, new_data: E::Value) -> Self {
         Self {
             target,
@@ -266,16 +231,14 @@ impl<E: TreeElement, U: TreeKeyComparable<E>> UpdateReplaceRet<E, U> {
     }
 }
 
-impl<E: TreeElement, U: TreeKeyComparable<E>> Patch<E> for UpdateReplaceRet<E, U> {
+impl<E: TreeElement, U: Comparable<E::Key>> Patch<E> for UpdateReplaceRet<E, U> {
     const WMODE: WriteFlag = WRITEMODE_REFRESH;
 
     type Ret<'a> = Option<&'a E::Value>;
 
-    type Target<'a> = U
-    where
-        Self: 'a;
+    type Target = U;
 
-    fn target<'a>(&'a self) -> &Self::Target<'a> {
+    fn target<'a>(&'a self) -> &Self::Target {
         &self.target
     }
 
@@ -311,8 +274,8 @@ impl<T: TreeElement> InsertDirect<T> {
 impl<T: TreeElement> Patch<T> for InsertDirect<T> {
     const WMODE: WriteFlag = WRITEMODE_FRESH;
     type Ret<'a> = bool;
-    type Target<'a> = T::Key;
-    fn target<'a>(&'a self) -> &Self::Target<'a> {
+    type Target = T::Key;
+    fn target<'a>(&'a self) -> &Self::Target {
         self.data.key()
     }
     fn nx_ret<'a>() -> Self::Ret<'a> {

From 36b1e4fec21db4010b3c463fd80f95123bf6ac49 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 21 Feb 2023 09:42:19 -0800
Subject: [PATCH 136/310] Get rid of borrow in MT idx specs and related objects

---
 server/src/engine/idx/mod.rs          |  20 ++-
 server/src/engine/idx/mtchm/access.rs |  63 +++++++---
 server/src/engine/idx/mtchm/imp.rs    | 175 +++++++++++++++++++++-----
 server/src/engine/idx/mtchm/mod.rs    |  78 +++++-------
 server/src/engine/idx/mtchm/patch.rs  |  64 +++++++---
 5 files changed, 278 insertions(+), 122 deletions(-)

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 9b371761..b7dbbf53 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -34,6 +34,7 @@ mod stord;
 mod tests;
 
 use {
+    self::meta::{Comparable, ComparableUpgradeable},
     crate::engine::sync::atm::Guard,
     core::{borrow::Borrow, hash::Hash},
 };
@@ -152,9 +153,9 @@ pub trait MTIndex<K, V>: IndexBaseSpec<K, V> {
     // write
     /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
     /// violated
-    fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool
+    fn mt_insert<U>(&self, key: U, val: V, g: &Guard) -> bool
     where
-        K: AsKeyClone,
+        U: ComparableUpgradeable<K>,
         V: AsValue;
     /// Updates or inserts the given value
     fn mt_upsert(&self, key: K, val: V, g: &Guard)
@@ -164,20 +165,17 @@ pub trait MTIndex<K, V>: IndexBaseSpec<K, V> {
     // read
     fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
     where
-        K: Borrow<Q> + AsKeyClone,
-        Q: ?Sized + AsKey;
+        Q: ?Sized + Comparable<K>;
     /// Returns a reference to the value corresponding to the key, if it exists
     fn mt_get<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
     where
-        K: AsKeyClone + Borrow<Q>,
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Comparable<K>,
         't: 'v,
         'g: 't + 'v;
     /// Returns a clone of the value corresponding to the key, if it exists
     fn mt_get_cloned<Q>(&self, key: &Q, g: &Guard) -> Option<V>
     where
-        K: AsKeyClone + Borrow<Q>,
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Comparable<K>,
         V: AsValueClone;
     // update
     /// Returns true if the entry is updated
@@ -196,13 +194,11 @@ pub trait MTIndex<K, V>: IndexBaseSpec<K, V> {
     /// Returns true if the entry was deleted
     fn mt_delete<Q>(&self, key: &Q, g: &Guard) -> bool
     where
-        K: AsKeyClone + Borrow<Q>,
-        Q: ?Sized + AsKey;
+        Q: ?Sized + Comparable<K>;
     /// Removes the entry and returns it, if it exists
     fn mt_delete_return<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
     where
-        K: AsKeyClone + Borrow<Q>,
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Comparable<K>,
         't: 'v,
         'g: 't + 'v;
 }
diff --git a/server/src/engine/idx/mtchm/access.rs b/server/src/engine/idx/mtchm/access.rs
index 5f5a7647..946175c6 100644
--- a/server/src/engine/idx/mtchm/access.rs
+++ b/server/src/engine/idx/mtchm/access.rs
@@ -24,17 +24,39 @@
  *
 */
 
-use super::meta::TreeElement;
+use {
+    super::meta::TreeElement,
+    crate::engine::idx::meta::Comparable,
+    core::{hash::Hash, marker::PhantomData},
+};
 
-pub trait ReadMode<T>: 'static {
+pub trait ReadMode<T: TreeElement> {
     type Ret<'a>;
+    type Target: Comparable<T::Key> + ?Sized + Hash;
+    fn target(&self) -> &Self::Target;
     fn ex<'a>(v: &'a T) -> Self::Ret<'a>;
     fn nx<'a>() -> Self::Ret<'a>;
 }
 
-pub struct RModeExists;
-impl<T> ReadMode<T> for RModeExists {
+pub struct RModeExists<'a, T, U: ?Sized> {
+    target: &'a U,
+    _d: PhantomData<T>,
+}
+
+impl<'a, T, U: ?Sized> RModeExists<'a, T, U> {
+    pub fn new(target: &'a U) -> Self {
+        Self {
+            target,
+            _d: PhantomData,
+        }
+    }
+}
+impl<'re, T: TreeElement, U: Comparable<T::Key> + ?Sized> ReadMode<T> for RModeExists<'re, T, U> {
     type Ret<'a> = bool;
+    type Target = U;
+    fn target(&self) -> &Self::Target {
+        &self.target
+    }
     fn ex<'a>(_: &'a T) -> Self::Ret<'a> {
         true
     }
@@ -43,22 +65,27 @@ impl<T> ReadMode<T> for RModeExists {
     }
 }
 
-pub struct RModeRef;
-impl<T: TreeElement> ReadMode<T> for RModeRef {
-    type Ret<'a> = Option<&'a T::Value>;
-    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
-        Some(v.val())
-    }
-    fn nx<'a>() -> Self::Ret<'a> {
-        None
-    }
+pub struct RModeRef<'a, T, U: ?Sized> {
+    target: &'a U,
+    _d: PhantomData<T>,
 }
 
-pub struct RModeClone;
-impl<T: TreeElement> ReadMode<T> for RModeClone {
-    type Ret<'a> = Option<T::Value>;
-    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
-        Some(v.val().clone())
+impl<'a, T, U: ?Sized> RModeRef<'a, T, U> {
+    pub fn new(target: &'a U) -> Self {
+        Self {
+            target,
+            _d: PhantomData,
+        }
+    }
+}
+impl<'re, T: TreeElement, U: Comparable<T::Key> + ?Sized> ReadMode<T> for RModeRef<'re, T, U> {
+    type Ret<'a> = Option<&'a T::Value>;
+    type Target = U;
+    fn target(&self) -> &Self::Target {
+        &self.target
+    }
+    fn ex<'a>(c: &'a T) -> Self::Ret<'a> {
+        Some(c.val())
     }
     fn nx<'a>() -> Self::Ret<'a> {
         None
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index df435219..ae2fe39c 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -30,13 +30,17 @@ use {
     super::{
         iter::{IterKV, IterKey, IterVal},
         meta::{Config, Key, TreeElement, Value},
+        patch::{PatchWrite, WriteFlag, WRITEMODE_ANY, WRITEMODE_FRESH, WRITEMODE_REFRESH},
         Tree,
     },
     crate::engine::{
-        idx::{AsKey, IndexBaseSpec, MTIndex},
+        idx::{
+            meta::{Comparable, ComparableUpgradeable},
+            IndexBaseSpec, MTIndex,
+        },
         sync::atm::{upin, Guard},
     },
-    std::{borrow::Borrow, sync::Arc},
+    std::sync::Arc,
 };
 
 #[inline(always)]
@@ -46,6 +50,125 @@ fn arc<K, V>(k: K, v: V) -> Arc<(K, V)> {
 
 pub type ChmArc<K, V, C> = Tree<Arc<(K, V)>, C>;
 
+pub struct ArcInsert<K, V>(Arc<(K, V)>);
+
+impl<K: Key, V: Value> PatchWrite<Arc<(K, V)>> for ArcInsert<K, V> {
+    const WMODE: WriteFlag = WRITEMODE_FRESH;
+
+    type Ret<'a> = bool;
+
+    type Target = K;
+
+    fn target<'a>(&'a self) -> &Self::Target {
+        self.0.key()
+    }
+
+    fn nx_new(&mut self) -> Arc<(K, V)> {
+        self.0.clone()
+    }
+
+    fn nx_ret<'a>() -> Self::Ret<'a> {
+        true
+    }
+
+    fn ex_apply(&mut self, _: &Arc<(K, V)>) -> Arc<(K, V)> {
+        unreachable!()
+    }
+
+    fn ex_ret<'a>(_: &'a Arc<(K, V)>) -> Self::Ret<'a> {
+        false
+    }
+}
+
+pub struct ArcUpsert<K, V>(Arc<(K, V)>);
+
+impl<K: Key, V: Value> PatchWrite<Arc<(K, V)>> for ArcUpsert<K, V> {
+    const WMODE: WriteFlag = WRITEMODE_ANY;
+
+    type Ret<'a> = ();
+
+    type Target = K;
+
+    fn target<'a>(&'a self) -> &Self::Target {
+        self.0.key()
+    }
+
+    fn nx_new(&mut self) -> Arc<(K, V)> {
+        self.0.clone()
+    }
+
+    fn nx_ret<'a>() -> Self::Ret<'a> {
+        ()
+    }
+
+    fn ex_apply(&mut self, _: &Arc<(K, V)>) -> Arc<(K, V)> {
+        self.0.clone()
+    }
+
+    fn ex_ret<'a>(_: &'a Arc<(K, V)>) -> Self::Ret<'a> {
+        ()
+    }
+}
+
+pub struct ArcUpdate<K, V>(Arc<(K, V)>);
+
+impl<K: Key, V: Value> PatchWrite<Arc<(K, V)>> for ArcUpdate<K, V> {
+    const WMODE: WriteFlag = WRITEMODE_REFRESH;
+
+    type Ret<'a> = bool;
+
+    type Target = K;
+
+    fn target<'a>(&'a self) -> &Self::Target {
+        self.0.key()
+    }
+
+    fn nx_new(&mut self) -> Arc<(K, V)> {
+        unreachable!()
+    }
+
+    fn nx_ret<'a>() -> Self::Ret<'a> {
+        false
+    }
+
+    fn ex_apply(&mut self, _: &Arc<(K, V)>) -> Arc<(K, V)> {
+        self.0.clone()
+    }
+
+    fn ex_ret<'a>(_: &'a Arc<(K, V)>) -> Self::Ret<'a> {
+        true
+    }
+}
+pub struct ArcUpdateRet<K, V>(Arc<(K, V)>);
+
+impl<K: Key, V: Value> PatchWrite<Arc<(K, V)>> for ArcUpdateRet<K, V> {
+    const WMODE: WriteFlag = WRITEMODE_REFRESH;
+
+    type Ret<'a> = Option<&'a V>;
+
+    type Target = K;
+
+    fn target<'a>(&'a self) -> &Self::Target {
+        self.0.key()
+    }
+
+    fn nx_new(&mut self) -> Arc<(K, V)> {
+        unreachable!()
+    }
+
+    fn nx_ret<'a>() -> Self::Ret<'a> {
+        None
+    }
+
+    fn ex_apply(&mut self, _: &Arc<(K, V)>) -> Arc<(K, V)> {
+        self.0.clone()
+    }
+
+    fn ex_ret<'a>(c: &'a Arc<(K, V)>) -> Self::Ret<'a> {
+        Some(c.val())
+    }
+}
+
 impl<K, V, C> IndexBaseSpec<K, V> for ChmArc<K, V, C>
 where
     C: Config,
@@ -101,26 +224,27 @@ where
         self.nontransactional_clear(g)
     }
 
-    fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool {
-        self.patch_insert(key, val, g)
+    fn mt_insert<U>(&self, key: U, val: V, g: &Guard) -> bool
+    where
+        U: ComparableUpgradeable<K>,
+    {
+        self.patch(ArcInsert(arc(key.upgrade(), val)), g)
     }
 
     fn mt_upsert(&self, key: K, val: V, g: &Guard) {
-        self.patch_upsert(key, val, g)
+        self.patch(ArcUpsert(arc(key.upgrade(), val)), g)
     }
 
     fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
     where
-        K: Borrow<Q>,
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Comparable<K>,
     {
         self.contains_key(key, g)
     }
 
     fn mt_get<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
     where
-        K: Borrow<Q>,
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Comparable<K>,
         't: 'v,
         'g: 't + 'v,
     {
@@ -129,14 +253,13 @@ where
 
     fn mt_get_cloned<Q>(&self, key: &Q, g: &Guard) -> Option<V>
     where
-        K: Borrow<Q>,
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Comparable<K>,
     {
         self.get(key, g).cloned()
     }
 
     fn mt_update(&self, key: K, val: V, g: &Guard) -> bool {
-        self.patch_update(key, val, g)
+        self.patch(ArcUpdate(arc(key, val)), g)
     }
 
     fn mt_update_return<'t, 'g, 'v>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
@@ -144,21 +267,19 @@ where
         't: 'v,
         'g: 't + 'v,
     {
-        self.patch_update_return(key, val, g)
+        self.patch(ArcUpdateRet(arc(key, val)), g)
     }
 
     fn mt_delete<Q>(&self, key: &Q, g: &Guard) -> bool
     where
-        K: Borrow<Q>,
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Comparable<K>,
     {
         self.remove(key, g)
     }
 
     fn mt_delete_return<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
     where
-        K: Borrow<Q>,
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Comparable<K>,
         't: 'v,
         'g: 't + 'v,
     {
@@ -223,7 +344,10 @@ where
         self.nontransactional_clear(g)
     }
 
-    fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool {
+    fn mt_insert<U>(&self, key: U, val: V, g: &Guard) -> bool
+    where
+        U: ComparableUpgradeable<K>,
+    {
         self.patch_insert(key, val, g)
     }
 
@@ -233,16 +357,14 @@ where
 
     fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
     where
-        K: Borrow<Q>,
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Comparable<K>,
     {
         self.contains_key(key, g)
     }
 
     fn mt_get<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
     where
-        K: Borrow<Q>,
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Comparable<K>,
         't: 'v,
         'g: 't + 'v,
     {
@@ -251,8 +373,7 @@ where
 
     fn mt_get_cloned<Q>(&self, key: &Q, g: &Guard) -> Option<V>
     where
-        K: Borrow<Q>,
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Comparable<K>,
     {
         self.get(key, g).cloned()
     }
@@ -271,16 +392,14 @@ where
 
     fn mt_delete<Q>(&self, key: &Q, g: &Guard) -> bool
     where
-        K: Borrow<Q>,
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Comparable<K>,
     {
         self.remove(key, g)
     }
 
     fn mt_delete_return<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
     where
-        K: Borrow<Q>,
-        Q: ?Sized + AsKey,
+        Q: ?Sized + Comparable<K>,
         't: 'v,
         'g: 't + 'v,
     {
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index beae4e08..0dc9a372 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -40,16 +40,12 @@ use {
         meta::{CompressState, Config, DefConfig, LNode, NodeFlag, TreeElement},
     },
     crate::engine::{
-        idx::{
-            meta::{Comparable, ComparableUpgradeable},
-            AsKey,
-        },
+        idx::meta::{Comparable, ComparableUpgradeable},
         mem::UArray,
         sync::atm::{self, cpin, upin, Atomic, Guard, Owned, Shared, ORD_ACR, ORD_RLX},
     },
     crossbeam_epoch::CompareExchangeError,
     std::{
-        borrow::Borrow,
         hash::Hash,
         hash::{BuildHasher, Hasher},
         marker::PhantomData,
@@ -274,7 +270,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
     ) -> Option<&'g T::Value> {
         self.patch(patch::UpdateReplaceRet::new(key, v), g)
     }
-    fn patch<'g, P: patch::Patch<T>>(&'g self, mut patch: P, g: &'g Guard) -> P::Ret<'g> {
+    fn patch<'g, P: patch::PatchWrite<T>>(&'g self, mut patch: P, g: &'g Guard) -> P::Ret<'g> {
         let hash = self.hash(patch.target());
         let mut level = C::LEVEL_ZERO;
         let mut current = &self.root;
@@ -353,9 +349,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                             in this case we either have the same key or we found an lnode. resolve any conflicts and attempt
                             to update
                         */
-                        let p = data
-                            .iter()
-                            .position(|e| patch.target().cmp_eq(e.key()));
+                        let p = data.iter().position(|e| patch.target().cmp_eq(e.key()));
                         match p {
                             Some(v) if P::WMODE == patch::WRITEMODE_FRESH => {
                                 return P::ex_ret(&data[v])
@@ -446,26 +440,18 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
         }
     }
 
-    fn contains_key<'g, Q>(&'g self, k: &Q, g: &'g Guard) -> bool
-    where
-        T::Key: Borrow<Q>,
-        Q: AsKey + ?Sized,
-    {
-        self._lookup::<Q, access::RModeExists>(k, g)
+    fn contains_key<'g, Q: ?Sized + Comparable<T::Key>>(&'g self, k: &Q, g: &'g Guard) -> bool {
+        self._lookup(access::RModeExists::new(k), g)
     }
-    fn get<'g, Q>(&'g self, k: &Q, g: &'g Guard) -> Option<&'g T::Value>
-    where
-        T::Key: Borrow<Q>,
-        Q: AsKey + ?Sized,
-    {
-        self._lookup::<Q, access::RModeRef>(k, g)
+    fn get<'g, Q: ?Sized + Comparable<T::Key>>(
+        &'g self,
+        k: &Q,
+        g: &'g Guard,
+    ) -> Option<&'g T::Value> {
+        self._lookup(access::RModeRef::new(k), g)
     }
-    fn _lookup<'g, Q, R: access::ReadMode<T>>(&'g self, k: &Q, g: &'g Guard) -> R::Ret<'g>
-    where
-        T::Key: Borrow<Q>,
-        Q: AsKey + ?Sized,
-    {
-        let mut hash = self.hash(k);
+    fn _lookup<'g, R: access::ReadMode<T>>(&'g self, r: R, g: &'g Guard) -> R::Ret<'g> {
+        let mut hash = self.hash(r.target());
         let mut current = &self.root;
         loop {
             let node = current.ld_acq(g);
@@ -479,7 +465,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                     return unsafe {
                         // UNSAFE(@ohsayan): checked flag + nullck
                         Self::read_data(node).iter().find_map(|e| {
-                            e.key().borrow().eq(k).then_some({
+                            r.target().cmp_eq(e.key()).then_some({
                                 ret = R::ex(e);
                                 Some(())
                             })
@@ -495,26 +481,18 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
             }
         }
     }
-    fn remove<'g, Q>(&'g self, k: &Q, g: &'g Guard) -> bool
-    where
-        T::Key: Borrow<Q>,
-        Q: AsKey + ?Sized,
-    {
-        self._remove::<Q, patch::Delete>(k, g)
+    fn remove<'g, Q: Comparable<T::Key> + ?Sized>(&'g self, k: &Q, g: &'g Guard) -> bool {
+        self._remove(patch::Delete::new(k), g)
     }
-    fn remove_return<'g, Q>(&'g self, k: &Q, g: &'g Guard) -> Option<&'g T::Value>
-    where
-        T::Key: Borrow<Q>,
-        Q: AsKey + ?Sized,
-    {
-        self._remove::<Q, patch::DeleteRet>(k, g)
+    fn remove_return<'g, Q: Comparable<T::Key> + ?Sized>(
+        &'g self,
+        k: &Q,
+        g: &'g Guard,
+    ) -> Option<&'g T::Value> {
+        self._remove(patch::DeleteRet::new(k), g)
     }
-    fn _remove<'g, Q, W: patch::PatchDelete<T>>(&'g self, k: &Q, g: &'g Guard) -> W::Ret<'g>
-    where
-        T::Key: Borrow<Q>,
-        Q: AsKey + ?Sized,
-    {
-        let hash = self.hash(k);
+    fn _remove<'g, P: patch::PatchDelete<T>>(&'g self, patch: P, g: &'g Guard) -> P::Ret<'g> {
+        let hash = self.hash(patch.target());
         let mut current = &self.root;
         let mut level = C::LEVEL_ZERO;
         let mut levels = UArray::<{ <DefConfig as Config>::BRANCH_MX }, _>::new();
@@ -523,7 +501,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
             match ldfl(&node) {
                 _ if node.is_null() => {
                     // lol
-                    return W::nx();
+                    return P::nx();
                 }
                 flag if hf(flag, NodeFlag::PENDING_DELETE) => {
                     let (p, c) = levels.pop().unwrap();
@@ -544,7 +522,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                         // UNSAFE(@ohsayan): flagck
                         Self::read_data(node)
                     };
-                    let mut ret = W::nx();
+                    let mut ret = P::nx();
                     let mut rem = false;
                     // this node shouldn't be empty
                     debug_assert!(!data.is_empty(), "logic,empty node not collected");
@@ -552,8 +530,8 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
                     let r: LNode<T> = data
                         .iter()
                         .filter_map(|this_elem| {
-                            if this_elem.key().borrow() == k {
-                                ret = W::ex(this_elem);
+                            if patch.target().cmp_eq(this_elem.key()) {
+                                ret = P::ex(this_elem);
                                 rem = true;
                                 None
                             } else {
diff --git a/server/src/engine/idx/mtchm/patch.rs b/server/src/engine/idx/mtchm/patch.rs
index 2cf645d7..10fda6d0 100644
--- a/server/src/engine/idx/mtchm/patch.rs
+++ b/server/src/engine/idx/mtchm/patch.rs
@@ -41,7 +41,7 @@ pub const WRITEMODE_REFRESH: WriteFlag = 0b10;
 pub const WRITEMODE_ANY: WriteFlag = 0b11;
 
 /// A [`Patch`] is intended to atomically update the state of the tree, which means that all your deltas should be atomic
-pub trait Patch<E: TreeElement> {
+pub trait PatchWrite<E: TreeElement> {
     const WMODE: WriteFlag;
     type Ret<'a>;
     type Target: Hash + Comparable<E::Key>;
@@ -69,7 +69,7 @@ impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> Insert<E, U> {
     }
 }
 
-impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> Patch<E> for Insert<E, U> {
+impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> PatchWrite<E> for Insert<E, U> {
     const WMODE: WriteFlag = WRITEMODE_FRESH;
     type Ret<'a> = bool;
     type Target = U;
@@ -108,7 +108,7 @@ impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> Upsert<E, U> {
     }
 }
 
-impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> Patch<E> for Upsert<E, U> {
+impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> PatchWrite<E> for Upsert<E, U> {
     const WMODE: WriteFlag = WRITEMODE_ANY;
     type Ret<'a> = ();
     type Target = U;
@@ -147,7 +147,7 @@ impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> UpsertReturn<E, U> {
     }
 }
 
-impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> Patch<E> for UpsertReturn<E, U> {
+impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> PatchWrite<E> for UpsertReturn<E, U> {
     const WMODE: WriteFlag = WRITEMODE_ANY;
     type Ret<'a> = Option<&'a E::Value>;
     type Target = U;
@@ -186,7 +186,7 @@ impl<E: TreeElement, U: Comparable<E::Key>> UpdateReplace<E, U> {
     }
 }
 
-impl<E: TreeElement, U: Comparable<E::Key>> Patch<E> for UpdateReplace<E, U> {
+impl<E: TreeElement, U: Comparable<E::Key>> PatchWrite<E> for UpdateReplace<E, U> {
     const WMODE: WriteFlag = WRITEMODE_REFRESH;
 
     type Ret<'a> = bool;
@@ -231,7 +231,7 @@ impl<E: TreeElement, U: Comparable<E::Key>> UpdateReplaceRet<E, U> {
     }
 }
 
-impl<E: TreeElement, U: Comparable<E::Key>> Patch<E> for UpdateReplaceRet<E, U> {
+impl<E: TreeElement, U: Comparable<E::Key>> PatchWrite<E> for UpdateReplaceRet<E, U> {
     const WMODE: WriteFlag = WRITEMODE_REFRESH;
 
     type Ret<'a> = Option<&'a E::Value>;
@@ -271,7 +271,7 @@ impl<T: TreeElement> InsertDirect<T> {
     }
 }
 
-impl<T: TreeElement> Patch<T> for InsertDirect<T> {
+impl<T: TreeElement> PatchWrite<T> for InsertDirect<T> {
     const WMODE: WriteFlag = WRITEMODE_FRESH;
     type Ret<'a> = bool;
     type Target = T::Key;
@@ -292,15 +292,34 @@ impl<T: TreeElement> Patch<T> for InsertDirect<T> {
     }
 }
 
-pub trait PatchDelete<T>: 'static {
+pub trait PatchDelete<T: TreeElement> {
     type Ret<'a>;
+    type Target: Comparable<T::Key> + ?Sized + Hash;
+    fn target(&self) -> &Self::Target;
     fn ex<'a>(v: &'a T) -> Self::Ret<'a>;
     fn nx<'a>() -> Self::Ret<'a>;
 }
 
-pub struct Delete;
-impl<T: TreeElement> PatchDelete<T> for Delete {
+pub struct Delete<'a, T: TreeElement, U: ?Sized> {
+    target: &'a U,
+    _m: PhantomData<T>,
+}
+
+impl<'a, T: TreeElement, U: ?Sized> Delete<'a, T, U> {
+    pub fn new(target: &'a U) -> Self {
+        Self {
+            target,
+            _m: PhantomData,
+        }
+    }
+}
+
+impl<'d, T: TreeElement, U: Comparable<T::Key> + ?Sized> PatchDelete<T> for Delete<'d, T, U> {
     type Ret<'a> = bool;
+    type Target = U;
+    fn target(&self) -> &Self::Target {
+        &self.target
+    }
     #[inline(always)]
     fn ex<'a>(_: &'a T) -> Self::Ret<'a> {
         true
@@ -311,12 +330,29 @@ impl<T: TreeElement> PatchDelete<T> for Delete {
     }
 }
 
-pub struct DeleteRet;
-impl<T: TreeElement> PatchDelete<T> for DeleteRet {
+pub struct DeleteRet<'a, T: TreeElement, U: ?Sized> {
+    target: &'a U,
+    _m: PhantomData<T>,
+}
+
+impl<'a, T: TreeElement, U: ?Sized> DeleteRet<'a, T, U> {
+    pub fn new(target: &'a U) -> Self {
+        Self {
+            target,
+            _m: PhantomData,
+        }
+    }
+}
+
+impl<'dr, T: TreeElement, U: Comparable<T::Key> + ?Sized> PatchDelete<T> for DeleteRet<'dr, T, U> {
     type Ret<'a> = Option<&'a T::Value>;
+    type Target = U;
+    fn target(&self) -> &Self::Target {
+        &self.target
+    }
     #[inline(always)]
-    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
-        Some(v.val())
+    fn ex<'a>(c: &'a T) -> Self::Ret<'a> {
+        Some(c.val())
     }
     #[inline(always)]
     fn nx<'a>() -> Self::Ret<'a> {

From 5e8501320d6ed304bf7043a14ec936e5d9bdbc31 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 23 Feb 2023 05:24:46 -0800
Subject: [PATCH 137/310] Implement vanilla patches and get rid of `Clone`
 bounds

---
 server/src/engine/idx/mod.rs         |  14 +-
 server/src/engine/idx/mtchm/imp.rs   | 181 +++----------------
 server/src/engine/idx/mtchm/iter.rs  |  14 +-
 server/src/engine/idx/mtchm/meta.rs  |  15 +-
 server/src/engine/idx/mtchm/mod.rs   |  52 ++----
 server/src/engine/idx/mtchm/patch.rs | 251 ++++++++-------------------
 server/src/engine/idx/mtchm/tests.rs |   2 +-
 7 files changed, 125 insertions(+), 404 deletions(-)

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index b7dbbf53..6b30e5d4 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -34,7 +34,7 @@ mod stord;
 mod tests;
 
 use {
-    self::meta::{Comparable, ComparableUpgradeable},
+    self::meta::Comparable,
     crate::engine::sync::atm::Guard,
     core::{borrow::Borrow, hash::Hash},
 };
@@ -48,12 +48,12 @@ pub type IndexST<K, V, S = std::collections::hash_map::RandomState> =
     std::collections::hash_map::HashMap<K, V, S>;
 
 /// Any type implementing this trait can be used as a key inside memory engine structures
-pub trait AsKey: Hash + Eq {
+pub trait AsKey: Hash + Eq + 'static {
     /// Read the key
     fn read_key(&self) -> &Self;
 }
 
-impl<T: Hash + Eq + ?Sized> AsKey for T {
+impl<T: Hash + Eq + ?Sized + 'static> AsKey for T {
     fn read_key(&self) -> &Self {
         self
     }
@@ -72,10 +72,10 @@ impl<T: AsKey + Clone + ?Sized> AsKeyClone for T {
     }
 }
 
-pub trait AsValue {
+pub trait AsValue: 'static {
     fn read_value(&self) -> &Self;
 }
-impl<T: ?Sized> AsValue for T {
+impl<T: ?Sized + 'static> AsValue for T {
     fn read_value(&self) -> &Self {
         self
     }
@@ -153,14 +153,12 @@ pub trait MTIndex<K, V>: IndexBaseSpec<K, V> {
     // write
     /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
     /// violated
-    fn mt_insert<U>(&self, key: U, val: V, g: &Guard) -> bool
+    fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool
     where
-        U: ComparableUpgradeable<K>,
         V: AsValue;
     /// Updates or inserts the given value
     fn mt_upsert(&self, key: K, val: V, g: &Guard)
     where
-        K: AsKeyClone,
         V: AsValue;
     // read
     fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index ae2fe39c..7c88f69a 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -29,15 +29,12 @@ use super::CHTRuntimeLog;
 use {
     super::{
         iter::{IterKV, IterKey, IterVal},
-        meta::{Config, Key, TreeElement, Value},
-        patch::{PatchWrite, WriteFlag, WRITEMODE_ANY, WRITEMODE_FRESH, WRITEMODE_REFRESH},
-        Tree,
+        meta::{Config, TreeElement},
+        patch::{VanillaInsert, VanillaUpdate, VanillaUpdateRet, VanillaUpsert},
+        RawTree,
     },
     crate::engine::{
-        idx::{
-            meta::{Comparable, ComparableUpgradeable},
-            IndexBaseSpec, MTIndex,
-        },
+        idx::{meta::Comparable, AsKey, AsKeyClone, AsValue, AsValueClone, IndexBaseSpec, MTIndex},
         sync::atm::{upin, Guard},
     },
     std::sync::Arc,
@@ -48,126 +45,7 @@ fn arc<K, V>(k: K, v: V) -> Arc<(K, V)> {
     Arc::new((k, v))
 }
 
-pub type ChmArc<K, V, C> = Tree<Arc<(K, V)>, C>;
-
-pub struct ArcInsert<K, V>(Arc<(K, V)>);
-
-impl<K: Key, V: Value> PatchWrite<Arc<(K, V)>> for ArcInsert<K, V> {
-    const WMODE: WriteFlag = WRITEMODE_FRESH;
-
-    type Ret<'a> = bool;
-
-    type Target = K;
-
-    fn target<'a>(&'a self) -> &Self::Target {
-        self.0.key()
-    }
-
-    fn nx_new(&mut self) -> Arc<(K, V)> {
-        self.0.clone()
-    }
-
-    fn nx_ret<'a>() -> Self::Ret<'a> {
-        true
-    }
-
-    fn ex_apply(&mut self, _: &Arc<(K, V)>) -> Arc<(K, V)> {
-        unreachable!()
-    }
-
-    fn ex_ret<'a>(_: &'a Arc<(K, V)>) -> Self::Ret<'a> {
-        false
-    }
-}
-
-pub struct ArcUpsert<K, V>(Arc<(K, V)>);
-
-impl<K: Key, V: Value> PatchWrite<Arc<(K, V)>> for ArcUpsert<K, V> {
-    const WMODE: WriteFlag = WRITEMODE_ANY;
-
-    type Ret<'a> = ();
-
-    type Target = K;
-
-    fn target<'a>(&'a self) -> &Self::Target {
-        self.0.key()
-    }
-
-    fn nx_new(&mut self) -> Arc<(K, V)> {
-        self.0.clone()
-    }
-
-    fn nx_ret<'a>() -> Self::Ret<'a> {
-        ()
-    }
-
-    fn ex_apply(&mut self, _: &Arc<(K, V)>) -> Arc<(K, V)> {
-        self.0.clone()
-    }
-
-    fn ex_ret<'a>(_: &'a Arc<(K, V)>) -> Self::Ret<'a> {
-        ()
-    }
-}
-
-pub struct ArcUpdate<K, V>(Arc<(K, V)>);
-
-impl<K: Key, V: Value> PatchWrite<Arc<(K, V)>> for ArcUpdate<K, V> {
-    const WMODE: WriteFlag = WRITEMODE_REFRESH;
-
-    type Ret<'a> = bool;
-
-    type Target = K;
-
-    fn target<'a>(&'a self) -> &Self::Target {
-        self.0.key()
-    }
-
-    fn nx_new(&mut self) -> Arc<(K, V)> {
-        unreachable!()
-    }
-
-    fn nx_ret<'a>() -> Self::Ret<'a> {
-        false
-    }
-
-    fn ex_apply(&mut self, _: &Arc<(K, V)>) -> Arc<(K, V)> {
-        self.0.clone()
-    }
-
-    fn ex_ret<'a>(_: &'a Arc<(K, V)>) -> Self::Ret<'a> {
-        true
-    }
-}
-pub struct ArcUpdateRet<K, V>(Arc<(K, V)>);
-
-impl<K: Key, V: Value> PatchWrite<Arc<(K, V)>> for ArcUpdateRet<K, V> {
-    const WMODE: WriteFlag = WRITEMODE_REFRESH;
-
-    type Ret<'a> = Option<&'a V>;
-
-    type Target = K;
-
-    fn target<'a>(&'a self) -> &Self::Target {
-        self.0.key()
-    }
-
-    fn nx_new(&mut self) -> Arc<(K, V)> {
-        unreachable!()
-    }
-
-    fn nx_ret<'a>() -> Self::Ret<'a> {
-        None
-    }
-
-    fn ex_apply(&mut self, _: &Arc<(K, V)>) -> Arc<(K, V)> {
-        self.0.clone()
-    }
-
-    fn ex_ret<'a>(c: &'a Arc<(K, V)>) -> Self::Ret<'a> {
-        Some(c.val())
-    }
-}
+pub type ChmArc<K, V, C> = RawTree<Arc<(K, V)>, C>;
 
 impl<K, V, C> IndexBaseSpec<K, V> for ChmArc<K, V, C>
 where
@@ -195,10 +73,10 @@ where
 impl<K, V, C> MTIndex<K, V> for ChmArc<K, V, C>
 where
     C: Config,
-    K: Key,
-    V: Value,
+    K: AsKey,
+    V: AsValue,
 {
-    type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, (K, V), C>
+    type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, Arc<(K, V)>, C>
     where
         'g: 't + 'v,
         't: 'v,
@@ -206,14 +84,14 @@ where
         V: 'v,
         Self: 't;
 
-    type IterKey<'t, 'g, 'v> = IterKey<'t, 'g, 'v, (K, V), C>
+    type IterKey<'t, 'g, 'v> = IterKey<'t, 'g, 'v, Arc<(K, V)>, C>
     where
         'g: 't + 'v,
         't: 'v,
         K: 'v,
         Self: 't;
 
-    type IterVal<'t, 'g, 'v> = IterVal<'t, 'g, 'v, (K, V), C>
+    type IterVal<'t, 'g, 'v> = IterVal<'t, 'g, 'v, Arc<(K, V)>, C>
     where
         'g: 't + 'v,
         't: 'v,
@@ -224,15 +102,12 @@ where
         self.nontransactional_clear(g)
     }
 
-    fn mt_insert<U>(&self, key: U, val: V, g: &Guard) -> bool
-    where
-        U: ComparableUpgradeable<K>,
-    {
-        self.patch(ArcInsert(arc(key.upgrade(), val)), g)
+    fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool {
+        self.patch(VanillaInsert(arc(key, val)), g)
     }
 
     fn mt_upsert(&self, key: K, val: V, g: &Guard) {
-        self.patch(ArcUpsert(arc(key.upgrade(), val)), g)
+        self.patch(VanillaUpsert(arc(key, val)), g)
     }
 
     fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
@@ -254,12 +129,13 @@ where
     fn mt_get_cloned<Q>(&self, key: &Q, g: &Guard) -> Option<V>
     where
         Q: ?Sized + Comparable<K>,
+        V: AsValueClone,
     {
         self.get(key, g).cloned()
     }
 
     fn mt_update(&self, key: K, val: V, g: &Guard) -> bool {
-        self.patch(ArcUpdate(arc(key, val)), g)
+        self.patch(VanillaUpdate(arc(key, val)), g)
     }
 
     fn mt_update_return<'t, 'g, 'v>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
@@ -267,7 +143,7 @@ where
         't: 'v,
         'g: 't + 'v,
     {
-        self.patch(ArcUpdateRet(arc(key, val)), g)
+        self.patch(VanillaUpdateRet(arc(key, val)), g)
     }
 
     fn mt_delete<Q>(&self, key: &Q, g: &Guard) -> bool
@@ -287,7 +163,7 @@ where
     }
 }
 
-pub type ChmCopy<K, V, C> = Tree<(K, V), C>;
+pub type ChmCopy<K, V, C> = RawTree<(K, V), C>;
 
 impl<K, V, C> IndexBaseSpec<K, V> for ChmCopy<K, V, C>
 where
@@ -315,8 +191,8 @@ where
 impl<K, V, C> MTIndex<K, V> for ChmCopy<K, V, C>
 where
     C: Config,
-    K: Key,
-    V: Value,
+    K: AsKeyClone,
+    V: AsValueClone,
 {
     type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, (K, V), C>
     where
@@ -344,15 +220,12 @@ where
         self.nontransactional_clear(g)
     }
 
-    fn mt_insert<U>(&self, key: U, val: V, g: &Guard) -> bool
-    where
-        U: ComparableUpgradeable<K>,
-    {
-        self.patch_insert(key, val, g)
+    fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool {
+        self.patch(VanillaInsert((key, val)), g)
     }
 
     fn mt_upsert(&self, key: K, val: V, g: &Guard) {
-        self.patch_upsert(key, val, g)
+        self.patch(VanillaUpsert((key, val)), g)
     }
 
     fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
@@ -379,7 +252,7 @@ where
     }
 
     fn mt_update(&self, key: K, val: V, g: &Guard) -> bool {
-        self.patch_update(key, val, g)
+        self.patch(VanillaUpdate((key, val)), g)
     }
 
     fn mt_update_return<'t, 'g, 'v>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
@@ -387,7 +260,7 @@ where
         't: 'v,
         'g: 't + 'v,
     {
-        self.patch_update_return(key, val, g)
+        self.patch(VanillaUpdateRet((key, val)), g)
     }
 
     fn mt_delete<Q>(&self, key: &Q, g: &Guard) -> bool
@@ -407,15 +280,15 @@ where
     }
 }
 
-impl<T: TreeElement, C: Config> FromIterator<T> for Tree<T, C> {
+impl<T: TreeElement, C: Config> FromIterator<T> for RawTree<T, C> {
     fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
         let g = unsafe {
             // UNSAFE(@ohsayan): it's me, hi, I'm the problem, it's me. yeah, Taylor knows it too. it's just us
             upin()
         };
-        let t = Tree::new();
+        let t = RawTree::new();
         iter.into_iter()
-            .for_each(|te| assert!(t.patch_insert(te.key().clone(), te.val().clone(), &g)));
+            .for_each(|te| assert!(t.patch(VanillaInsert(te), &g)));
         t
     }
 }
diff --git a/server/src/engine/idx/mtchm/iter.rs b/server/src/engine/idx/mtchm/iter.rs
index 432b42e2..8081a009 100644
--- a/server/src/engine/idx/mtchm/iter.rs
+++ b/server/src/engine/idx/mtchm/iter.rs
@@ -27,7 +27,7 @@
 use {
     super::{
         meta::{Config, DefConfig, NodeFlag, TreeElement},
-        Node, Tree,
+        Node, RawTree,
     },
     crate::engine::{
         mem::UArray,
@@ -53,7 +53,7 @@ where
     C: Config,
     T: TreeElement,
 {
-    pub fn new(t: &'t Tree<T, C>, g: &'g Guard) -> Self {
+    pub fn new(t: &'t RawTree<T, C>, g: &'g Guard) -> Self {
         Self {
             i: RawIter::new(t, g),
         }
@@ -91,7 +91,7 @@ where
     C: Config,
     T: TreeElement,
 {
-    pub fn new(t: &'t Tree<T, C>, g: &'g Guard) -> Self {
+    pub fn new(t: &'t RawTree<T, C>, g: &'g Guard) -> Self {
         Self {
             i: RawIter::new(t, g),
         }
@@ -129,7 +129,7 @@ where
     C: Config,
     T: TreeElement,
 {
-    pub fn new(t: &'t Tree<T, C>, g: &'g Guard) -> Self {
+    pub fn new(t: &'t RawTree<T, C>, g: &'g Guard) -> Self {
         Self {
             i: RawIter::new(t, g),
         }
@@ -195,7 +195,7 @@ where
 {
     g: &'g Guard,
     stack: UArray<{ <DefConfig as Config>::BRANCH_MX + 1 }, DFSCNodeCtx<'g, C>>,
-    _m: PhantomData<(&'v T, C, &'t Tree<T, C>, I)>,
+    _m: PhantomData<(&'v T, C, &'t RawTree<T, C>, I)>,
 }
 
 impl<'t, 'g, 'v, T, C, I> RawIter<'t, 'g, 'v, T, C, I>
@@ -205,7 +205,7 @@ where
     I: IterConfig<T>,
     C: Config,
 {
-    pub(super) fn new(tree: &'t Tree<T, C>, g: &'g Guard) -> Self {
+    pub(super) fn new(tree: &'t RawTree<T, C>, g: &'g Guard) -> Self {
         let mut stack = UArray::new();
         let sptr = tree.root.ld_acq(g);
         stack.push(DFSCNodeCtx { sptr, idx: 0 });
@@ -229,7 +229,7 @@ where
                 flag if super::hf(flag, NodeFlag::DATA) => {
                     let data = unsafe {
                         // UNSAFE(@ohsayan): flagck
-                        Tree::<T, C>::read_data(current.sptr)
+                        RawTree::<T, C>::read_data(current.sptr)
                     };
                     if current.idx < data.len() {
                         let ref ret = data[current.idx];
diff --git a/server/src/engine/idx/mtchm/meta.rs b/server/src/engine/idx/mtchm/meta.rs
index 14d46568..45edb746 100644
--- a/server/src/engine/idx/mtchm/meta.rs
+++ b/server/src/engine/idx/mtchm/meta.rs
@@ -26,7 +26,7 @@
 
 use {
     crate::engine::{
-        idx::{meta::AsHasher, AsKeyClone},
+        idx::{meta::AsHasher, AsKey, AsKeyClone, AsValue, AsValueClone},
         mem::VInline,
     },
     std::{collections::hash_map::RandomState, sync::Arc},
@@ -69,20 +69,15 @@ impl<T: AsHasher> PreConfig for Config2B<T> {
     type HState = T;
 }
 
-pub trait Key: AsKeyClone + 'static {}
-impl<T> Key for T where T: AsKeyClone + 'static {}
-pub trait Value: Clone + 'static {}
-impl<T> Value for T where T: Clone + 'static {}
-
 pub trait TreeElement: Clone + 'static {
-    type Key: Key;
-    type Value: Value;
+    type Key: AsKey;
+    type Value: AsValue;
     fn key(&self) -> &Self::Key;
     fn val(&self) -> &Self::Value;
     fn new(k: Self::Key, v: Self::Value) -> Self;
 }
 
-impl<K: Key, V: Value> TreeElement for (K, V) {
+impl<K: AsKeyClone, V: AsValueClone> TreeElement for (K, V) {
     type Key = K;
     type Value = V;
     #[inline(always)]
@@ -98,7 +93,7 @@ impl<K: Key, V: Value> TreeElement for (K, V) {
     }
 }
 
-impl<K: Key, V: Value> TreeElement for Arc<(K, V)> {
+impl<K: AsKey, V: AsValue> TreeElement for Arc<(K, V)> {
     type Key = K;
     type Value = V;
     #[inline(always)]
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index 0dc9a372..ae99bfab 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -40,7 +40,7 @@ use {
         meta::{CompressState, Config, DefConfig, LNode, NodeFlag, TreeElement},
     },
     crate::engine::{
-        idx::meta::{Comparable, ComparableUpgradeable},
+        idx::meta::Comparable,
         mem::UArray,
         sync::atm::{self, cpin, upin, Atomic, Guard, Owned, Shared, ORD_ACR, ORD_RLX},
     },
@@ -158,18 +158,18 @@ trait CTFlagAlign {
     const FLCK: () = Self::FLCK_A;
 }
 
-impl<T, C: Config> CTFlagAlign for Tree<T, C> {
+impl<T, C: Config> CTFlagAlign for RawTree<T, C> {
     const FL_A: bool = atm::ensure_flag_align::<LNode<T>, { NodeFlag::bits() }>();
     const FL_B: bool = atm::ensure_flag_align::<Node<C>, { NodeFlag::bits() }>();
 }
 
-impl<T, C: Config> Default for Tree<T, C> {
+impl<T, C: Config> Default for RawTree<T, C> {
     fn default() -> Self {
         Self::_new(C::HState::default())
     }
 }
 
-pub struct Tree<T, C: Config = DefConfig> {
+pub struct RawTree<T, C: Config = DefConfig> {
     root: Atomic<Node<C>>,
     h: C::HState,
     l: AtomicUsize,
@@ -177,7 +177,7 @@ pub struct Tree<T, C: Config = DefConfig> {
     _m: PhantomData<T>,
 }
 
-impl<T, C: Config> Tree<T, C> {
+impl<T, C: Config> RawTree<T, C> {
     #[inline(always)]
     const fn _new(h: C::HState) -> Self {
         let _ = Self::FLCK;
@@ -203,14 +203,14 @@ impl<T, C: Config> Tree<T, C> {
     }
 }
 
-impl<T, C: Config> Tree<T, C> {
+impl<T, C: Config> RawTree<T, C> {
     #[inline(always)]
     fn new() -> Self {
         Self::_new(C::HState::default())
     }
 }
 
-impl<T, C: Config> Tree<T, C> {
+impl<T, C: Config> RawTree<T, C> {
     fn hash<Q>(&self, k: &Q) -> u64
     where
         Q: ?Sized + Hash,
@@ -222,7 +222,7 @@ impl<T, C: Config> Tree<T, C> {
 }
 
 // iter
-impl<T: TreeElement, C: Config> Tree<T, C> {
+impl<T: TreeElement, C: Config> RawTree<T, C> {
     fn iter_kv<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterKV<'t, 'g, 'v, T, C> {
         IterKV::new(self, g)
     }
@@ -234,42 +234,12 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
     }
 }
 
-impl<T: TreeElement, C: Config> Tree<T, C> {
+impl<T: TreeElement, C: Config> RawTree<T, C> {
     fn nontransactional_clear(&self, g: &Guard) {
         self.iter_key(g).for_each(|k| {
             let _ = self.remove(k, g);
         });
     }
-    fn patch_insert<Q: ComparableUpgradeable<T::Key>>(
-        &self,
-        key: Q,
-        v: T::Value,
-        g: &Guard,
-    ) -> bool {
-        self.patch(patch::Insert::new(key, v), g)
-    }
-    fn patch_upsert<Q: ComparableUpgradeable<T::Key>>(&self, key: Q, v: T::Value, g: &Guard) {
-        self.patch(patch::Upsert::new(key, v), g)
-    }
-    fn patch_upsert_return<'g, Q: ComparableUpgradeable<T::Key>>(
-        &'g self,
-        key: Q,
-        v: T::Value,
-        g: &'g Guard,
-    ) -> Option<&'g T::Value> {
-        self.patch(patch::UpsertReturn::new(key, v), g)
-    }
-    fn patch_update<Q: Comparable<T::Key>>(&self, key: Q, v: T::Value, g: &Guard) -> bool {
-        self.patch(patch::UpdateReplace::new(key, v), g)
-    }
-    fn patch_update_return<'g, Q: Comparable<T::Key>>(
-        &'g self,
-        key: Q,
-        v: T::Value,
-        g: &'g Guard,
-    ) -> Option<&'g T::Value> {
-        self.patch(patch::UpdateReplaceRet::new(key, v), g)
-    }
     fn patch<'g, P: patch::PatchWrite<T>>(&'g self, mut patch: P, g: &'g Guard) -> P::Ret<'g> {
         let hash = self.hash(patch.target());
         let mut level = C::LEVEL_ZERO;
@@ -606,7 +576,7 @@ impl<T: TreeElement, C: Config> Tree<T, C> {
 }
 
 // low-level methods
-impl<T, C: Config> Tree<T, C> {
+impl<T, C: Config> RawTree<T, C> {
     // hilarious enough but true, l doesn't affect safety but only creates an incorrect state
     fn decr_len(&self) {
         self.decr_len_by(1)
@@ -747,7 +717,7 @@ impl<T, C: Config> Tree<T, C> {
     }
 }
 
-impl<T, C: Config> Drop for Tree<T, C> {
+impl<T, C: Config> Drop for RawTree<T, C> {
     fn drop(&mut self) {
         unsafe {
             // UNSAFE(@ohsayan): sole live owner
diff --git a/server/src/engine/idx/mtchm/patch.rs b/server/src/engine/idx/mtchm/patch.rs
index 10fda6d0..ada96d5c 100644
--- a/server/src/engine/idx/mtchm/patch.rs
+++ b/server/src/engine/idx/mtchm/patch.rs
@@ -26,8 +26,8 @@
 
 use {
     super::meta::TreeElement,
-    crate::engine::idx::meta::{Comparable, ComparableUpgradeable},
-    core::{hash::Hash, marker::PhantomData},
+    crate::engine::idx::meta::Comparable,
+    std::{hash::Hash, marker::PhantomData},
 };
 
 /// write mode flag
@@ -41,256 +41,141 @@ pub const WRITEMODE_REFRESH: WriteFlag = 0b10;
 pub const WRITEMODE_ANY: WriteFlag = 0b11;
 
 /// A [`Patch`] is intended to atomically update the state of the tree, which means that all your deltas should be atomic
+///
+/// Make sure you override the [`unreachable!`] behavior!
 pub trait PatchWrite<E: TreeElement> {
     const WMODE: WriteFlag;
     type Ret<'a>;
     type Target: Hash + Comparable<E::Key>;
     fn target<'a>(&'a self) -> &Self::Target;
-    fn nx_new(&mut self) -> E;
+    fn nx_new(&mut self) -> E {
+        unreachable!()
+    }
     fn nx_ret<'a>() -> Self::Ret<'a>;
-    fn ex_apply(&mut self, current: &E) -> E;
+    fn ex_apply(&mut self, _: &E) -> E {
+        unreachable!()
+    }
     fn ex_ret<'a>(current: &'a E) -> Self::Ret<'a>;
 }
 
-/// insert
-pub struct Insert<E: TreeElement, U: ComparableUpgradeable<E::Key>> {
-    target: U,
-    new_data: E::Value,
-    _m: PhantomData<E>,
-}
-
-impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> Insert<E, U> {
-    pub fn new(target: U, new_data: E::Value) -> Self {
-        Self {
-            target,
-            new_data,
-            _m: PhantomData,
-        }
-    }
-}
+/*
+    vanilla
+*/
 
-impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> PatchWrite<E> for Insert<E, U> {
+pub struct VanillaInsert<T: TreeElement>(pub T);
+impl<T: TreeElement> PatchWrite<T> for VanillaInsert<T> {
     const WMODE: WriteFlag = WRITEMODE_FRESH;
     type Ret<'a> = bool;
-    type Target = U;
-
+    type Target = T::Key;
     fn target<'a>(&'a self) -> &Self::Target {
-        &self.target
+        self.0.key()
     }
-    fn nx_new(&mut self) -> E {
-        E::new(self.target.upgrade(), self.new_data.clone())
+    // nx
+    fn nx_new(&mut self) -> T {
+        self.0.clone()
     }
     fn nx_ret<'a>() -> Self::Ret<'a> {
         true
     }
-    fn ex_apply(&mut self, _: &E) -> E {
-        unreachable!()
-    }
-    fn ex_ret<'a>(_: &'a E) -> Self::Ret<'a> {
+    // ex
+    fn ex_ret<'a>(_: &'a T) -> Self::Ret<'a> {
         false
     }
 }
 
-/// upsert
-pub struct Upsert<E: TreeElement, U: ComparableUpgradeable<E::Key>> {
-    target: U,
-    new_data: E::Value,
-    _m: PhantomData<E>,
-}
-
-impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> Upsert<E, U> {
-    pub fn new(target: U, new_data: E::Value) -> Self {
-        Self {
-            target,
-            new_data,
-            _m: PhantomData,
-        }
-    }
-}
-
-impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> PatchWrite<E> for Upsert<E, U> {
+pub struct VanillaUpsert<T: TreeElement>(pub T);
+impl<T: TreeElement> PatchWrite<T> for VanillaUpsert<T> {
     const WMODE: WriteFlag = WRITEMODE_ANY;
     type Ret<'a> = ();
-    type Target = U;
-
+    type Target = T::Key;
     fn target<'a>(&'a self) -> &Self::Target {
-        &self.target
+        self.0.key()
     }
-    fn nx_new(&mut self) -> E {
-        E::new(self.target.upgrade(), self.new_data.clone())
+    // nx
+    fn nx_new(&mut self) -> T {
+        self.0.clone()
     }
     fn nx_ret<'a>() -> Self::Ret<'a> {
         ()
     }
-    fn ex_apply(&mut self, _: &E) -> E {
-        self.nx_new()
+    // ex
+    fn ex_apply(&mut self, _: &T) -> T {
+        self.0.clone()
     }
-    fn ex_ret<'a>(_: &'a E) -> Self::Ret<'a> {
+    fn ex_ret<'a>(_: &'a T) -> Self::Ret<'a> {
         ()
     }
 }
 
-/// upsert return
-pub struct UpsertReturn<E: TreeElement, U: ComparableUpgradeable<E::Key>> {
-    target: U,
-    new_data: E::Value,
-    _m: PhantomData<E>,
-}
-
-impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> UpsertReturn<E, U> {
-    pub fn new(target: U, new_data: E::Value) -> Self {
-        Self {
-            target,
-            new_data,
-            _m: PhantomData,
-        }
-    }
-}
-
-impl<E: TreeElement, U: ComparableUpgradeable<E::Key>> PatchWrite<E> for UpsertReturn<E, U> {
+pub struct VanillaUpsertRet<T: TreeElement>(pub T);
+impl<T: TreeElement> PatchWrite<T> for VanillaUpsertRet<T> {
     const WMODE: WriteFlag = WRITEMODE_ANY;
-    type Ret<'a> = Option<&'a E::Value>;
-    type Target = U;
-
+    type Ret<'a> = Option<&'a T::Value>;
+    type Target = T::Key;
     fn target<'a>(&'a self) -> &Self::Target {
-        &self.target
+        self.0.key()
     }
-    fn nx_new(&mut self) -> E {
-        E::new(self.target.upgrade(), self.new_data.clone())
+    // nx
+    fn nx_new(&mut self) -> T {
+        self.0.clone()
     }
     fn nx_ret<'a>() -> Self::Ret<'a> {
         None
     }
-    fn ex_apply(&mut self, _: &E) -> E {
-        self.nx_new()
-    }
-    fn ex_ret<'a>(e: &'a E) -> Self::Ret<'a> {
-        Some(e.val())
+    // ex
+    fn ex_apply(&mut self, _: &T) -> T {
+        self.0.clone()
     }
-}
-
-/// update
-pub struct UpdateReplace<E: TreeElement, U: Comparable<E::Key>> {
-    target: U,
-    new_data: E::Value,
-    _m: PhantomData<E>,
-}
-
-impl<E: TreeElement, U: Comparable<E::Key>> UpdateReplace<E, U> {
-    pub fn new(target: U, new_data: E::Value) -> Self {
-        Self {
-            target,
-            new_data,
-            _m: PhantomData,
-        }
+    fn ex_ret<'a>(c: &'a T) -> Self::Ret<'a> {
+        Some(c.val())
     }
 }
 
-impl<E: TreeElement, U: Comparable<E::Key>> PatchWrite<E> for UpdateReplace<E, U> {
+pub struct VanillaUpdate<T: TreeElement>(pub T);
+impl<T: TreeElement> PatchWrite<T> for VanillaUpdate<T> {
     const WMODE: WriteFlag = WRITEMODE_REFRESH;
-
     type Ret<'a> = bool;
-
-    type Target = U;
-
+    type Target = T::Key;
     fn target<'a>(&'a self) -> &Self::Target {
-        &self.target
+        self.0.key()
     }
-
-    fn nx_new(&mut self) -> E {
-        unreachable!()
-    }
-
+    // nx
     fn nx_ret<'a>() -> Self::Ret<'a> {
         false
     }
-
-    fn ex_apply(&mut self, c: &E) -> E {
-        E::new(c.key().clone(), self.new_data.clone())
+    // ex
+    fn ex_apply(&mut self, _: &T) -> T {
+        self.0.clone()
     }
-
-    fn ex_ret<'a>(_: &'a E) -> Self::Ret<'a> {
+    fn ex_ret<'a>(_: &'a T) -> Self::Ret<'a> {
         true
     }
 }
 
-/// update_return
-pub struct UpdateReplaceRet<E: TreeElement, U: Comparable<E::Key>> {
-    target: U,
-    new_data: E::Value,
-    _m: PhantomData<E>,
-}
-
-impl<E: TreeElement, U: Comparable<E::Key>> UpdateReplaceRet<E, U> {
-    pub fn new(target: U, new_data: E::Value) -> Self {
-        Self {
-            target,
-            new_data,
-            _m: PhantomData,
-        }
-    }
-}
-
-impl<E: TreeElement, U: Comparable<E::Key>> PatchWrite<E> for UpdateReplaceRet<E, U> {
+pub struct VanillaUpdateRet<T: TreeElement>(pub T);
+impl<T: TreeElement> PatchWrite<T> for VanillaUpdateRet<T> {
     const WMODE: WriteFlag = WRITEMODE_REFRESH;
-
-    type Ret<'a> = Option<&'a E::Value>;
-
-    type Target = U;
-
+    type Ret<'a> = Option<&'a T::Value>;
+    type Target = T::Key;
     fn target<'a>(&'a self) -> &Self::Target {
-        &self.target
+        self.0.key()
     }
-
-    fn nx_new(&mut self) -> E {
-        unreachable!()
-    }
-
+    // nx
     fn nx_ret<'a>() -> Self::Ret<'a> {
         None
     }
-
-    fn ex_apply(&mut self, c: &E) -> E {
-        E::new(c.key().clone(), self.new_data.clone())
+    // ex
+    fn ex_apply(&mut self, _: &T) -> T {
+        self.0.clone()
     }
-
-    fn ex_ret<'a>(c: &'a E) -> Self::Ret<'a> {
+    fn ex_ret<'a>(c: &'a T) -> Self::Ret<'a> {
         Some(c.val())
     }
 }
 
-pub struct InsertDirect<T: TreeElement> {
-    data: T,
-}
-
-impl<T: TreeElement> InsertDirect<T> {
-    pub fn new(key: T::Key, val: T::Value) -> Self {
-        Self {
-            data: T::new(key, val),
-        }
-    }
-}
-
-impl<T: TreeElement> PatchWrite<T> for InsertDirect<T> {
-    const WMODE: WriteFlag = WRITEMODE_FRESH;
-    type Ret<'a> = bool;
-    type Target = T::Key;
-    fn target<'a>(&'a self) -> &Self::Target {
-        self.data.key()
-    }
-    fn nx_ret<'a>() -> Self::Ret<'a> {
-        true
-    }
-    fn nx_new(&mut self) -> T {
-        self.data.clone()
-    }
-    fn ex_apply(&mut self, _: &T) -> T {
-        unreachable!()
-    }
-    fn ex_ret<'a>(_: &'a T) -> Self::Ret<'a> {
-        false
-    }
-}
+/*
+    delete
+*/
 
 pub trait PatchDelete<T: TreeElement> {
     type Ret<'a>;
diff --git a/server/src/engine/idx/mtchm/tests.rs b/server/src/engine/idx/mtchm/tests.rs
index 00e24292..f60fc89b 100644
--- a/server/src/engine/idx/mtchm/tests.rs
+++ b/server/src/engine/idx/mtchm/tests.rs
@@ -336,7 +336,7 @@ fn multispam_delete() {
 
 #[test]
 fn multispam_lol() {
-    let idx = Arc::new(super::Tree::<StringTup, super::meta::Config2B<LolState>>::new());
+    let idx = Arc::new(super::RawTree::<StringTup, super::meta::Config2B<LolState>>::new());
     let token = ControlToken::new();
     let mut data = Vec::new();
     prepare_data(&mut data, TUP_INCR);

From 28af6f20ad65af46326812b0a1275b04dcaa4bbe Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 25 Feb 2023 09:15:57 -0800
Subject: [PATCH 138/310] Add word definitions to `mem`

---
 server/src/engine/idx/mtchm/access.rs |   2 +
 server/src/engine/mem/astr.rs         | 168 ++++++++++++++++++++++
 server/src/engine/mem/mod.rs          | 198 +++++++++++---------------
 3 files changed, 252 insertions(+), 116 deletions(-)
 create mode 100644 server/src/engine/mem/astr.rs

diff --git a/server/src/engine/idx/mtchm/access.rs b/server/src/engine/idx/mtchm/access.rs
index 946175c6..df6b2833 100644
--- a/server/src/engine/idx/mtchm/access.rs
+++ b/server/src/engine/idx/mtchm/access.rs
@@ -51,6 +51,7 @@ impl<'a, T, U: ?Sized> RModeExists<'a, T, U> {
         }
     }
 }
+
 impl<'re, T: TreeElement, U: Comparable<T::Key> + ?Sized> ReadMode<T> for RModeExists<'re, T, U> {
     type Ret<'a> = bool;
     type Target = U;
@@ -78,6 +79,7 @@ impl<'a, T, U: ?Sized> RModeRef<'a, T, U> {
         }
     }
 }
+
 impl<'re, T: TreeElement, U: Comparable<T::Key> + ?Sized> ReadMode<T> for RModeRef<'re, T, U> {
     type Ret<'a> = Option<&'a T::Value>;
     type Target = U;
diff --git a/server/src/engine/mem/astr.rs b/server/src/engine/mem/astr.rs
new file mode 100644
index 00000000..5d92ace3
--- /dev/null
+++ b/server/src/engine/mem/astr.rs
@@ -0,0 +1,168 @@
+/*
+ * Created on Sat Feb 25 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::UArray,
+    crate::engine::ql::lex::Ident,
+    std::{
+        borrow::Borrow,
+        fmt, mem,
+        ops::{Deref, DerefMut},
+    },
+};
+
+#[derive(PartialEq, Eq, Hash, Clone)]
+#[repr(transparent)]
+pub struct AStr<const N: usize> {
+    base: UArray<N, u8>,
+}
+impl<const N: usize> AStr<N> {
+    #[inline(always)]
+    pub fn check(v: &str) -> bool {
+        v.len() <= N
+    }
+    #[inline(always)]
+    pub fn try_new(s: &str) -> Option<Self> {
+        if Self::check(s) {
+            Some(unsafe {
+                // UNSAFE(@ohsayan): verified len
+                Self::from_len_unchecked(s)
+            })
+        } else {
+            None
+        }
+    }
+    #[inline(always)]
+    pub fn new(s: &str) -> Self {
+        Self::try_new(s).expect("length overflow")
+    }
+    #[inline(always)]
+    pub unsafe fn from_len_unchecked_ident(i: Ident<'_>) -> Self {
+        Self::from_len_unchecked(i.as_str())
+    }
+    #[inline(always)]
+    pub unsafe fn from_len_unchecked(s: &str) -> Self {
+        Self {
+            base: UArray::from_slice(s.as_bytes()),
+        }
+    }
+    #[inline(always)]
+    pub unsafe fn from_len_unchecked_bytes(b: &[u8]) -> Self {
+        Self::from_len_unchecked(mem::transmute(b))
+    }
+    #[inline(always)]
+    pub fn _as_str(&self) -> &str {
+        unsafe { mem::transmute(self._as_bytes()) }
+    }
+    #[inline(always)]
+    pub fn _as_mut_str(&mut self) -> &mut str {
+        unsafe { mem::transmute(self._as_bytes_mut()) }
+    }
+    pub fn _as_bytes(&self) -> &[u8] {
+        self.base.as_slice()
+    }
+    pub fn _as_bytes_mut(&mut self) -> &mut [u8] {
+        self.base.as_slice_mut()
+    }
+}
+impl<const N: usize> fmt::Debug for AStr<N> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(self._as_str())
+    }
+}
+impl<const N: usize> Deref for AStr<N> {
+    type Target = str;
+    #[inline(always)]
+    fn deref(&self) -> &Self::Target {
+        self._as_str()
+    }
+}
+impl<const N: usize> DerefMut for AStr<N> {
+    #[inline(always)]
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self._as_mut_str()
+    }
+}
+impl<'a, const N: usize> From<Ident<'a>> for AStr<N> {
+    #[inline(always)]
+    fn from(value: Ident<'a>) -> Self {
+        Self::new(value.as_str())
+    }
+}
+impl<'a, const N: usize> From<&'a str> for AStr<N> {
+    #[inline(always)]
+    fn from(s: &str) -> Self {
+        Self::new(s)
+    }
+}
+impl<const N: usize> PartialEq<str> for AStr<N> {
+    #[inline(always)]
+    fn eq(&self, other: &str) -> bool {
+        self._as_bytes() == other.as_bytes()
+    }
+}
+impl<const N: usize> PartialEq<AStr<N>> for str {
+    #[inline(always)]
+    fn eq(&self, other: &AStr<N>) -> bool {
+        other._as_bytes() == self.as_bytes()
+    }
+}
+impl<const N: usize> PartialEq<[u8]> for AStr<N> {
+    #[inline(always)]
+    fn eq(&self, other: &[u8]) -> bool {
+        self._as_bytes() == other
+    }
+}
+impl<const N: usize> PartialEq<AStr<N>> for [u8] {
+    #[inline(always)]
+    fn eq(&self, other: &AStr<N>) -> bool {
+        self == other.as_bytes()
+    }
+}
+impl<const N: usize> AsRef<[u8]> for AStr<N> {
+    #[inline(always)]
+    fn as_ref(&self) -> &[u8] {
+        self._as_bytes()
+    }
+}
+impl<const N: usize> AsRef<str> for AStr<N> {
+    #[inline(always)]
+    fn as_ref(&self) -> &str {
+        self._as_str()
+    }
+}
+impl<const N: usize> Default for AStr<N> {
+    #[inline(always)]
+    fn default() -> Self {
+        Self::new("")
+    }
+}
+impl<const N: usize> Borrow<[u8]> for AStr<N> {
+    #[inline(always)]
+    fn borrow(&self) -> &[u8] {
+        self._as_bytes()
+    }
+}
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index 149d3a7d..70c4f67e 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -24,152 +24,118 @@
  *
 */
 
+mod astr;
 #[cfg(test)]
 mod tests;
 mod uarray;
 mod vinline;
 
-use {
-    crate::engine::ql::lex::Ident,
-    std::{
-        borrow::Borrow,
-        fmt, mem,
-        ops::{Deref, DerefMut},
-    },
-};
-
+pub use astr::AStr;
 pub use uarray::UArray;
 pub use vinline::VInline;
 
-#[derive(PartialEq, Eq, Hash, Clone)]
-#[repr(transparent)]
-pub struct AStr<const N: usize> {
-    base: UArray<N, u8>,
+/// Native double pointer width
+pub type NativeDword = [usize; 2];
+/// Native triple pointer width
+pub type NativeTword = [usize; 3];
+
+/// Native tripe pointer stack (must also be usable as a double pointer stack, see [`SystemDword`])
+pub trait SystemTword: SystemDword {
+    fn store_full(a: usize, b: usize, c: usize) -> Self;
+    fn load_full(&self) -> [usize; 3];
 }
-impl<const N: usize> AStr<N> {
-    #[inline(always)]
-    pub fn check(v: &str) -> bool {
-        v.len() <= N
-    }
+
+/// Native double pointer stack
+pub trait SystemDword {
+    fn store_qw(u: u64) -> Self;
+    fn store_fat(a: usize, b: usize) -> Self;
+    fn load_qw(&self) -> u64;
+    fn load_fat(&self) -> [usize; 2];
+}
+
+impl SystemDword for NativeDword {
     #[inline(always)]
-    pub fn try_new(s: &str) -> Option<Self> {
-        if Self::check(s) {
-            Some(unsafe {
-                // UNSAFE(@ohsayan): verified len
-                Self::from_len_unchecked(s)
-            })
-        } else {
-            None
+    fn store_qw(u: u64) -> Self {
+        let x;
+        #[cfg(target_pointer_width = "32")]
+        {
+            x = unsafe { core::mem::transmute(u) };
         }
-    }
-    #[inline(always)]
-    pub fn new(s: &str) -> Self {
-        Self::try_new(s).expect("length overflow")
-    }
-    #[inline(always)]
-    pub unsafe fn from_len_unchecked_ident(i: Ident<'_>) -> Self {
-        Self::from_len_unchecked(i.as_str())
-    }
-    #[inline(always)]
-    pub unsafe fn from_len_unchecked(s: &str) -> Self {
-        Self {
-            base: UArray::from_slice(s.as_bytes()),
+        #[cfg(target_pointer_width = "64")]
+        {
+            x = [u as usize, 0]
         }
+        x
     }
     #[inline(always)]
-    pub unsafe fn from_len_unchecked_bytes(b: &[u8]) -> Self {
-        Self::from_len_unchecked(mem::transmute(b))
-    }
-    #[inline(always)]
-    pub fn _as_str(&self) -> &str {
-        unsafe { mem::transmute(self._as_bytes()) }
-    }
-    #[inline(always)]
-    pub fn _as_mut_str(&mut self) -> &mut str {
-        unsafe { mem::transmute(self._as_bytes_mut()) }
-    }
-    pub fn _as_bytes(&self) -> &[u8] {
-        self.base.as_slice()
-    }
-    pub fn _as_bytes_mut(&mut self) -> &mut [u8] {
-        self.base.as_slice_mut()
-    }
-}
-impl<const N: usize> fmt::Debug for AStr<N> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str(self._as_str())
-    }
-}
-impl<const N: usize> Deref for AStr<N> {
-    type Target = str;
-    #[inline(always)]
-    fn deref(&self) -> &Self::Target {
-        self._as_str()
-    }
-}
-impl<const N: usize> DerefMut for AStr<N> {
-    #[inline(always)]
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self._as_mut_str()
-    }
-}
-impl<'a, const N: usize> From<Ident<'a>> for AStr<N> {
-    #[inline(always)]
-    fn from(value: Ident<'a>) -> Self {
-        Self::new(value.as_str())
-    }
-}
-impl<'a, const N: usize> From<&'a str> for AStr<N> {
-    #[inline(always)]
-    fn from(s: &str) -> Self {
-        Self::new(s)
+    fn store_fat(a: usize, b: usize) -> Self {
+        [a, b]
     }
-}
-impl<const N: usize> PartialEq<str> for AStr<N> {
     #[inline(always)]
-    fn eq(&self, other: &str) -> bool {
-        self._as_bytes() == other.as_bytes()
+    fn load_qw(&self) -> u64 {
+        let x;
+        #[cfg(target_pointer_width = "32")]
+        {
+            x = unsafe { core::mem::transmute_copy(self) }
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            x = self[0] as _;
+        }
+        x
     }
-}
-impl<const N: usize> PartialEq<AStr<N>> for str {
     #[inline(always)]
-    fn eq(&self, other: &AStr<N>) -> bool {
-        other._as_bytes() == self.as_bytes()
+    fn load_fat(&self) -> [usize; 2] {
+        *self
     }
 }
-impl<const N: usize> PartialEq<[u8]> for AStr<N> {
-    #[inline(always)]
-    fn eq(&self, other: &[u8]) -> bool {
-        self._as_bytes() == other
+
+impl SystemDword for NativeTword {
+    #[inline(always)]
+    fn store_qw(u: u64) -> Self {
+        let x;
+        #[cfg(target_pointer_width = "32")]
+        {
+            let [a, b] = unsafe { core::mem::transmute(u) };
+            x = [a, b, 0];
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            x = [u as _, 0, 0];
+        }
+        x
     }
-}
-impl<const N: usize> PartialEq<AStr<N>> for [u8] {
     #[inline(always)]
-    fn eq(&self, other: &AStr<N>) -> bool {
-        self == other.as_bytes()
+    fn store_fat(a: usize, b: usize) -> Self {
+        [a, b, 0]
     }
-}
-impl<const N: usize> AsRef<[u8]> for AStr<N> {
     #[inline(always)]
-    fn as_ref(&self) -> &[u8] {
-        self._as_bytes()
+    fn load_qw(&self) -> u64 {
+        let x;
+        #[cfg(target_pointer_width = "32")]
+        {
+            let ab = [self[0], self[1]];
+            x = unsafe { core::mem::transmute(ab) };
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            x = self[0] as _;
+        }
+        x
     }
-}
-impl<const N: usize> AsRef<str> for AStr<N> {
     #[inline(always)]
-    fn as_ref(&self) -> &str {
-        self._as_str()
+    fn load_fat(&self) -> [usize; 2] {
+        [self[0], self[1]]
     }
 }
-impl<const N: usize> Default for AStr<N> {
+
+impl SystemTword for NativeTword {
     #[inline(always)]
-    fn default() -> Self {
-        Self::new("")
+    fn store_full(a: usize, b: usize, c: usize) -> Self {
+        [a, b, c]
     }
-}
-impl<const N: usize> Borrow<[u8]> for AStr<N> {
     #[inline(always)]
-    fn borrow(&self) -> &[u8] {
-        self._as_bytes()
+    fn load_full(&self) -> [usize; 3] {
+        *self
     }
 }

From 0410a019cf40fe77154c70bea0287bffc10d9c25 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 26 Feb 2023 09:43:14 -0800
Subject: [PATCH 139/310] Define explicit literals

---
 server/src/engine/data/lit.rs  | 260 ++++++++++++++++++++++++++++
 server/src/engine/data/mod.rs  |   2 +
 server/src/engine/data/spec.rs | 307 +++++++++++++++++++++++++++++++++
 server/src/engine/mem/mod.rs   |  30 ++--
 4 files changed, 584 insertions(+), 15 deletions(-)
 create mode 100644 server/src/engine/data/lit.rs
 create mode 100644 server/src/engine/data/spec.rs

diff --git a/server/src/engine/data/lit.rs b/server/src/engine/data/lit.rs
new file mode 100644
index 00000000..9e998c50
--- /dev/null
+++ b/server/src/engine/data/lit.rs
@@ -0,0 +1,260 @@
+/*
+ * Created on Sun Feb 26 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::spec::{Dataflag, Dataspec1D, DataspecMeta1D, DataspecMethods1D, DataspecRaw1D},
+    crate::engine::mem::{NativeDword, SystemDword},
+    core::{
+        fmt,
+        marker::PhantomData,
+        mem::{self, ManuallyDrop},
+    },
+};
+
+/*
+    Lit
+*/
+
+pub struct Lit<'a> {
+    data: NativeDword,
+    tag: Dataflag,
+    _lt: PhantomData<&'a [u8]>,
+}
+
+impl<'a> DataspecMeta1D for Lit<'a> {
+    type Target = NativeDword;
+    type StringItem = Box<str>;
+    fn new(flag: Dataflag, data: Self::Target) -> Self {
+        Self {
+            data,
+            tag: flag,
+            _lt: PhantomData,
+        }
+    }
+    fn kind(&self) -> Dataflag {
+        self.tag
+    }
+    fn data(&self) -> Self::Target {
+        unsafe {
+            // UNSAFE(@ohsayan): This function doesn't create any clones, so we're good
+            mem::transmute_copy(self)
+        }
+    }
+}
+
+/*
+    UNSAFE(@ohsayan): Safety checks:
+    - Heap str: yes
+    - Heap bin: no
+    - Drop str: yes, dealloc
+    - Drop bin: not needed
+    - Clone str: yes, alloc
+    - Clone bin: not needed
+*/
+unsafe impl<'a> DataspecRaw1D for Lit<'a> {
+    const HEAP_STR: bool = true;
+    const HEAP_BIN: bool = false;
+    unsafe fn drop_str(s: &str) {
+        drop(Box::from_raw(s as *const _ as *mut str));
+    }
+    unsafe fn drop_bin(_: &[u8]) {}
+    unsafe fn clone_str(s: &str) -> Self::Target {
+        let new_string = ManuallyDrop::new(s.to_owned().into_boxed_str());
+        SystemDword::store_fat(new_string.as_ptr() as usize, new_string.len())
+    }
+    unsafe fn clone_bin(b: &[u8]) -> Self::Target {
+        SystemDword::store_fat(b.as_ptr() as usize, b.len())
+    }
+}
+
+/*
+    UNSAFE(@ohsayan): Safety checks:
+    - We LEAK memory because, duh
+    - We don't touch our own targets, ever (well, I'm a bad boy so I do touch it in fmt::Debug)
+*/
+unsafe impl<'a> Dataspec1D for Lit<'a> {
+    fn Str(s: Box<str>) -> Self {
+        let md = ManuallyDrop::new(s);
+        Self::new(
+            Dataflag::Str,
+            SystemDword::store_fat(md.as_ptr() as _, md.len()),
+        )
+    }
+}
+
+/*
+    UNSAFE(@ohsayan):
+    - No target touch
+*/
+unsafe impl<'a> DataspecMethods1D for Lit<'a> {}
+
+impl<'a, T: DataspecMethods1D> PartialEq<T> for Lit<'a> {
+    fn eq(&self, other: &T) -> bool {
+        <Self as DataspecMethods1D>::self_eq(self, other)
+    }
+}
+impl<'a> fmt::Debug for Lit<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut f = f.debug_struct("Lit");
+        f.field("tag", &self.tag);
+        self.self_fmt_debug_data("data", &mut f);
+        f.field("_lt", &self._lt);
+        f.finish()
+    }
+}
+
+impl<'a> Drop for Lit<'a> {
+    fn drop(&mut self) {
+        self.self_drop();
+    }
+}
+
+impl<'a> Clone for Lit<'a> {
+    fn clone(&self) -> Self {
+        self.self_clone()
+    }
+}
+
+/*
+    LitIR
+*/
+
+pub struct LitIR<'a> {
+    tag: Dataflag,
+    data: NativeDword,
+    _lt: PhantomData<&'a str>,
+}
+
+impl<'a> DataspecMeta1D for LitIR<'a> {
+    type Target = NativeDword;
+    type StringItem = &'a str;
+    fn new(flag: Dataflag, data: Self::Target) -> Self {
+        Self {
+            tag: flag,
+            data,
+            _lt: PhantomData,
+        }
+    }
+    fn kind(&self) -> Dataflag {
+        self.tag
+    }
+    fn data(&self) -> Self::Target {
+        unsafe {
+            // UNSAFE(@ohsayan): We can freely copy our stack because everything is already allocated
+            mem::transmute_copy(self)
+        }
+    }
+}
+
+/*
+    UNSAFE(@ohsayan): Safety:
+    - Heap str: no
+    - Heap bin: no
+    - Drop str: no
+    - Drop bin: no
+    - Clone str: stack
+    - Clone bin: stack
+*/
+unsafe impl<'a> DataspecRaw1D for LitIR<'a> {
+    const HEAP_STR: bool = false;
+    const HEAP_BIN: bool = false;
+    unsafe fn drop_str(_: &str) {}
+    unsafe fn drop_bin(_: &[u8]) {}
+    unsafe fn clone_str(s: &str) -> Self::Target {
+        SystemDword::store_fat(s.as_ptr() as usize, s.len())
+    }
+    unsafe fn clone_bin(b: &[u8]) -> Self::Target {
+        SystemDword::store_fat(b.as_ptr() as usize, b.len())
+    }
+}
+
+/*
+    UNSAFE(@ohsayan): Safety:
+    - No touches :)
+*/
+unsafe impl<'a> Dataspec1D for LitIR<'a> {
+    fn Str(s: Self::StringItem) -> Self {
+        Self::new(
+            Dataflag::Str,
+            SystemDword::store_fat(s.as_ptr() as usize, s.len()),
+        )
+    }
+}
+
+/*
+    UNSAFE(@ohsayan): Safety:
+    - No touches
+*/
+unsafe impl<'a> DataspecMethods1D for LitIR<'a> {}
+
+impl<'a, T: DataspecMethods1D> PartialEq<T> for LitIR<'a> {
+    fn eq(&self, other: &T) -> bool {
+        <Self as DataspecMethods1D>::self_eq(self, other)
+    }
+}
+impl<'a> fmt::Debug for LitIR<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut f = f.debug_struct("LitIR");
+        f.field("tag", &self.tag);
+        self.self_fmt_debug_data("data", &mut f);
+        f.field("_lt", &self._lt);
+        f.finish()
+    }
+}
+
+impl<'a> Drop for LitIR<'a> {
+    fn drop(&mut self) {
+        self.self_drop();
+    }
+}
+
+impl<'a> Clone for LitIR<'a> {
+    fn clone(&self) -> Self {
+        self.self_clone()
+    }
+}
+
+#[test]
+fn tlit() {
+    let str1 = Lit::Str("hello".into());
+    let str2 = str1.clone();
+    assert_eq!(str1, str2);
+    assert_eq!(str1.str(), "hello");
+    assert_eq!(str2.str(), "hello");
+    drop(str1);
+    assert_eq!(str2.str(), "hello");
+}
+
+#[test]
+fn tlitir() {
+    let str1 = LitIR::Str("hello");
+    let str2 = str1.clone();
+    assert_eq!(str1, str2);
+    assert_eq!(str1.str(), "hello");
+    assert_eq!(str2.str(), "hello");
+    drop(str1);
+    assert_eq!(str2.str(), "hello");
+}
diff --git a/server/src/engine/data/mod.rs b/server/src/engine/data/mod.rs
index 8fdce8f8..4a81710a 100644
--- a/server/src/engine/data/mod.rs
+++ b/server/src/engine/data/mod.rs
@@ -24,7 +24,9 @@
  *
 */
 
+pub mod lit;
 pub mod md_dict;
+pub mod spec;
 #[cfg(test)]
 mod tests;
 
diff --git a/server/src/engine/data/spec.rs b/server/src/engine/data/spec.rs
new file mode 100644
index 00000000..09804d21
--- /dev/null
+++ b/server/src/engine/data/spec.rs
@@ -0,0 +1,307 @@
+/*
+ * Created on Sun Feb 26 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/*
+    So, I woke up and chose violence. God bless me and the stack memory. What I've done here is a sin. Do not follow my footsteps here if you want to write maintainable code.
+    -- @ohsayan
+*/
+
+use {
+    crate::engine::mem::SystemDword,
+    core::{fmt, mem, slice},
+};
+
+#[inline(always)]
+fn when_then<T>(cond: bool, then: T) -> Option<T> {
+    cond.then_some(then)
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
+/// The "reduced" data flag
+pub enum Dataflag {
+    Bool,
+    UnsignedInt,
+    SignedInt,
+    Float,
+    Bin,
+    Str,
+    List,
+}
+
+/// Information about the type that implements the dataspec traits
+pub trait DataspecMeta1D: Sized {
+    // assoc
+    /// The target must be able to store (atleast) a native dword
+    type Target: SystemDword;
+    /// The string item. This helps us remain correct with the dtors
+    type StringItem;
+    // fn
+    /// Create a new instance. Usually allocates zero memory *directly*
+    fn new(flag: Dataflag, data: Self::Target) -> Self;
+    /// Returns the reduced dataflag
+    fn kind(&self) -> Dataflag;
+    /// Returns the data stack
+    fn data(&self) -> Self::Target;
+}
+
+/// Unsafe dtor/ctor impls for dataspec items. We have no clue about these things, the implementor must take care of them
+///
+/// ## Safety
+///
+/// - Your dtors MUST BE correct
+pub unsafe trait DataspecRaw1D: DataspecMeta1D {
+    /// Is the string heap allocated...anywhere down the line?
+    const HEAP_STR: bool;
+    /// Is the binary heap allocated...anywhere down the line?
+    const HEAP_BIN: bool;
+    /// Drop the string, if you need a dtor
+    unsafe fn drop_str(s: &str);
+    /// Drop the binary, if you need a dtor
+    unsafe fn drop_bin(b: &[u8]);
+    /// Clone the string object. Note, we literally HAVE NO IDEA about what you're doing here
+    unsafe fn clone_str(s: &str) -> Self::Target;
+    /// Clone the binary object. Again, NOT A DAMN CLUE about whay you're doing down there
+    unsafe fn clone_bin(b: &[u8]) -> Self::Target;
+}
+
+/// Functions that can be used to read/write to/from dataspec objects
+///
+/// ## Safety
+/// - You must touch your targets by yourself
+pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
+    // store
+    /// Store a new bool. This function is always safe to call
+    #[allow(non_snake_case)]
+    fn Bool(b: bool) -> Self {
+        Self::new(Dataflag::Bool, SystemDword::store_qw(b as _))
+    }
+    /// Store a new uint. This function is always safe to call
+    #[allow(non_snake_case)]
+    fn UnsignedInt(u: u64) -> Self {
+        Self::new(Dataflag::UnsignedInt, SystemDword::store_qw(u as _))
+    }
+    /// Store a new sint. This function is always safe to call
+    #[allow(non_snake_case)]
+    fn SignedInt(s: i64) -> Self {
+        Self::new(Dataflag::SignedInt, SystemDword::store_qw(s as _))
+    }
+    /// Store a new float. This function is always safe to call
+    #[allow(non_snake_case)]
+    fn Float(f: f64) -> Self {
+        Self::new(Dataflag::Float, SystemDword::store_qw(f.to_bits()))
+    }
+    /// Store a new binary. This function is always safe to call
+    #[allow(non_snake_case)]
+    fn Bin(b: &[u8]) -> Self {
+        Self::new(
+            Dataflag::Bin,
+            SystemDword::store_fat(b.as_ptr() as usize, b.len()),
+        )
+    }
+
+    /// Store a new string. Now, I won't talk about this one's safety because it depends on the implementor
+    #[allow(non_snake_case)]
+    fn Str(s: Self::StringItem) -> Self;
+
+    // load
+    // bool
+    /// Load a bool (this is unsafe for logical verity)
+    unsafe fn read_bool_uck(&self) -> bool {
+        self.data().load_qw() == 1
+    }
+    /// Load a bool
+    fn read_bool_try(&self) -> Option<bool> {
+        when_then(self.kind() == Dataflag::Bool, unsafe {
+            // UNSAFE(@ohsayan): we've verified the flag. but lol because this isn't actually unsafe
+            self.read_bool_uck()
+        })
+    }
+    /// Load a bool
+    /// ## Panics
+    /// If you're not a bool, you panic
+    fn bool(&self) -> bool {
+        self.read_bool_try().unwrap()
+    }
+    // uint
+    /// Load a uint (this is unsafe for logical verity)
+    unsafe fn read_uint_uck(&self) -> u64 {
+        self.data().load_qw()
+    }
+    /// Load a uint
+    fn read_uint_try(&self) -> Option<u64> {
+        when_then(self.kind() == Dataflag::UnsignedInt, unsafe {
+            // UNSAFE(@ohsayan): we've verified the flag. but lol because this isn't actually unsafe
+            self.read_uint_uck()
+        })
+    }
+    /// Load a uint
+    /// ## Panics
+    /// If you're not a uint, you panic
+    fn uint(&self) -> u64 {
+        self.read_uint_try().unwrap()
+    }
+    // sint
+    /// Load a sint (unsafe for logical verity)
+    unsafe fn read_sint_uck(&self) -> i64 {
+        self.data().load_qw() as _
+    }
+    /// Load a sint
+    fn read_sint_try(&self) -> Option<i64> {
+        when_then(self.kind() == Dataflag::SignedInt, unsafe {
+            // UNSAFE(@ohsayan): we've verified the flag. but lol because this isn't actually unsafe
+            self.read_sint_uck()
+        })
+    }
+    /// Load a sint and panic if we're not a sint
+    fn sint(&self) -> i64 {
+        self.read_sint_try().unwrap()
+    }
+    // float
+    /// Load a float (unsafe for logical verity)
+    unsafe fn read_float_uck(&self) -> f64 {
+        self.data().load_qw() as _
+    }
+    /// Load a float
+    fn read_float_try(&self) -> Option<f64> {
+        when_then(self.kind() == Dataflag::Float, unsafe {
+            self.read_float_uck()
+        })
+    }
+    /// Load a float and panic if we aren't one
+    fn float(&self) -> f64 {
+        self.read_float_try().unwrap()
+    }
+    // bin
+    /// Load a binary
+    ///
+    /// ## Safety
+    /// Are you a binary? Did you store it correctly? Are you a victim of segfaults?
+    unsafe fn read_bin_uck(&self) -> &[u8] {
+        let [p, l] = self.data().load_fat();
+        slice::from_raw_parts(p as *const u8, l)
+    }
+    /// Load a bin
+    fn read_bin_try(&self) -> Option<&[u8]> {
+        when_then(self.kind() == Dataflag::Bin, unsafe { self.read_bin_uck() })
+    }
+    /// Load a bin or panic if we aren't one
+    fn bin(&self) -> &[u8] {
+        self.read_bin_try().unwrap()
+    }
+    // str
+    /// Load a str
+    ///
+    /// ## Safety
+    /// Are you a str? Did you store it correctly? Are you a victim of segfaults?
+    unsafe fn read_str_uck(&self) -> &str {
+        mem::transmute(self.read_bin_uck())
+    }
+    /// Load a str
+    fn read_str_try(&self) -> Option<&str> {
+        when_then(self.kind() == Dataflag::Str, unsafe { self.read_str_uck() })
+    }
+    /// Load a str and panic if we aren't one
+    fn str(&self) -> &str {
+        self.read_str_try().unwrap()
+    }
+}
+
+/// Common impls
+///
+/// ## Safety
+/// - You are not touching your target
+pub unsafe trait DataspecMethods1D: Dataspec1D {
+    fn self_drop(&mut self) {
+        match self.kind() {
+            Dataflag::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
+                // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
+                <Self as DataspecRaw1D>::drop_str(Dataspec1D::read_str_uck(self))
+            },
+            Dataflag::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
+                // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
+                <Self as DataspecRaw1D>::drop_bin(Dataspec1D::read_bin_uck(self))
+            },
+            _ => {}
+        }
+    }
+    fn self_clone(&self) -> Self {
+        let data = match self.kind() {
+            Dataflag::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
+                // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
+                <Self as DataspecRaw1D>::clone_str(Dataspec1D::read_str_uck(self))
+            },
+            Dataflag::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
+                // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
+                <Self as DataspecRaw1D>::clone_bin(Dataspec1D::read_bin_uck(self))
+            },
+            _ => self.data(),
+        };
+        Self::new(self.kind(), data)
+    }
+    fn self_eq(&self, other: &impl DataspecMethods1D) -> bool {
+        unsafe {
+            // UNSAFE(@ohsayan): we are checking our flags
+            match (self.kind(), other.kind()) {
+                (Dataflag::Bool, Dataflag::Bool) => self.read_bool_uck() & other.read_bool_uck(),
+                (Dataflag::UnsignedInt, Dataflag::UnsignedInt) => {
+                    self.read_uint_uck() == other.read_uint_uck()
+                }
+                (Dataflag::SignedInt, Dataflag::SignedInt) => {
+                    self.read_sint_uck() == other.read_sint_uck()
+                }
+                (Dataflag::Float, Dataflag::Float) => {
+                    self.read_float_uck() == other.read_float_uck()
+                }
+                (Dataflag::Bin, Dataflag::Bin) => self.read_bin_uck() == other.read_bin_uck(),
+                (Dataflag::Str, Dataflag::Str) => self.read_str_uck() == other.read_str_uck(),
+                _ => false,
+            }
+        }
+    }
+    fn self_fmt_debug_data(&self, data_field: &str, f: &mut fmt::DebugStruct) {
+        macro_rules! fmtdebug {
+            ($($(#[$attr:meta])* $match:pat => $ret:expr),* $(,)?) => {
+                match self.kind() {$($(#[$attr])* $match => { let x = $ret; f.field(data_field, &x) },)*}
+            }
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): we are checking our flags
+            fmtdebug!(
+                Dataflag::Bool => self.read_bool_uck(),
+                Dataflag::UnsignedInt => self.read_uint_uck(),
+                Dataflag::SignedInt => self.read_sint_uck(),
+                Dataflag::Float => self.read_float_uck(),
+                Dataflag::Bin => self.read_bin_uck(),
+                Dataflag::Str => self.read_str_uck(),
+                #[allow(unused_variables)]
+                #[allow(unreachable_code)]
+                Dataflag::List => {
+                    panic!("found 2D in 1D data")
+                },
+            )
+        };
+    }
+}
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index 70c4f67e..4032cebc 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -34,10 +34,10 @@ pub use astr::AStr;
 pub use uarray::UArray;
 pub use vinline::VInline;
 
-/// Native double pointer width
-pub type NativeDword = [usize; 2];
-/// Native triple pointer width
-pub type NativeTword = [usize; 3];
+/// Native double pointer width (note, native != arch native, but host native)
+pub struct NativeDword([usize; 2]);
+/// Native triple pointer width (note, native != arch native, but host native)
+pub struct NativeTword([usize; 3]);
 
 /// Native tripe pointer stack (must also be usable as a double pointer stack, see [`SystemDword`])
 pub trait SystemTword: SystemDword {
@@ -65,11 +65,11 @@ impl SystemDword for NativeDword {
         {
             x = [u as usize, 0]
         }
-        x
+        Self(x)
     }
     #[inline(always)]
     fn store_fat(a: usize, b: usize) -> Self {
-        [a, b]
+        Self([a, b])
     }
     #[inline(always)]
     fn load_qw(&self) -> u64 {
@@ -80,13 +80,13 @@ impl SystemDword for NativeDword {
         }
         #[cfg(target_pointer_width = "64")]
         {
-            x = self[0] as _;
+            x = self.0[0] as _;
         }
         x
     }
     #[inline(always)]
     fn load_fat(&self) -> [usize; 2] {
-        *self
+        self.0
     }
 }
 
@@ -103,39 +103,39 @@ impl SystemDword for NativeTword {
         {
             x = [u as _, 0, 0];
         }
-        x
+        Self(x)
     }
     #[inline(always)]
     fn store_fat(a: usize, b: usize) -> Self {
-        [a, b, 0]
+        Self([a, b, 0])
     }
     #[inline(always)]
     fn load_qw(&self) -> u64 {
         let x;
         #[cfg(target_pointer_width = "32")]
         {
-            let ab = [self[0], self[1]];
+            let ab = [self.0[0], self.0[1]];
             x = unsafe { core::mem::transmute(ab) };
         }
         #[cfg(target_pointer_width = "64")]
         {
-            x = self[0] as _;
+            x = self.0[0] as _;
         }
         x
     }
     #[inline(always)]
     fn load_fat(&self) -> [usize; 2] {
-        [self[0], self[1]]
+        [self.0[0], self.0[1]]
     }
 }
 
 impl SystemTword for NativeTword {
     #[inline(always)]
     fn store_full(a: usize, b: usize, c: usize) -> Self {
-        [a, b, c]
+        Self([a, b, c])
     }
     #[inline(always)]
     fn load_full(&self) -> [usize; 3] {
-        *self
+        self.0
     }
 }

From 66dab00eb3f4d95a4b0ab88484765de9b8e7dccc Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 26 Feb 2023 10:03:05 -0800
Subject: [PATCH 140/310] Add lit impls

Also fixed 32-bit word store
---
 server/src/engine/data/lit.rs  | 45 ++++++++++++++++++++++++++++++++++
 server/src/engine/data/spec.rs | 18 ++++++++------
 server/src/engine/mem/mod.rs   |  2 +-
 3 files changed, 57 insertions(+), 8 deletions(-)

diff --git a/server/src/engine/data/lit.rs b/server/src/engine/data/lit.rs
index 9e998c50..13f6f7fa 100644
--- a/server/src/engine/data/lit.rs
+++ b/server/src/engine/data/lit.rs
@@ -44,6 +44,15 @@ pub struct Lit<'a> {
     _lt: PhantomData<&'a [u8]>,
 }
 
+impl<'a> Lit<'a> {
+    fn as_ir(&'a self) -> LitIR<'a> {
+        unsafe {
+            // UNSAFE(@ohsayan): 'tis the lifetime. 'tis the savior
+            mem::transmute_copy(self)
+        }
+    }
+}
+
 impl<'a> DataspecMeta1D for Lit<'a> {
     type Target = NativeDword;
     type StringItem = Box<str>;
@@ -138,6 +147,25 @@ impl<'a> Clone for Lit<'a> {
     }
 }
 
+impl<'a> ToString for Lit<'a> {
+    fn to_string(&self) -> String {
+        <Self as DataspecMethods1D>::to_string_debug(self)
+    }
+}
+
+enum_impls! {
+    Lit<'a> => {
+        bool as Bool,
+        u64 as UnsignedInt,
+        i64 as SignedInt,
+        f64 as Float,
+        &'a str as Str,
+        String as Str,
+        Box<str> as Str,
+        &'a [u8] as Bin,
+    }
+}
+
 /*
     LitIR
 */
@@ -204,6 +232,12 @@ unsafe impl<'a> Dataspec1D for LitIR<'a> {
     }
 }
 
+impl<'a> ToString for LitIR<'a> {
+    fn to_string(&self) -> String {
+        <Self as DataspecMethods1D>::to_string_debug(self)
+    }
+}
+
 /*
     UNSAFE(@ohsayan): Safety:
     - No touches
@@ -237,6 +271,17 @@ impl<'a> Clone for LitIR<'a> {
     }
 }
 
+enum_impls! {
+    LitIR<'a> => {
+        bool as Bool,
+        u64 as UnsignedInt,
+        i64 as SignedInt,
+        f64 as Float,
+        &'a str as Str,
+        &'a [u8] as Bin,
+    }
+}
+
 #[test]
 fn tlit() {
     let str1 = Lit::Str("hello".into());
diff --git a/server/src/engine/data/spec.rs b/server/src/engine/data/spec.rs
index 09804d21..e375eaf7 100644
--- a/server/src/engine/data/spec.rs
+++ b/server/src/engine/data/spec.rs
@@ -25,7 +25,7 @@
 */
 
 /*
-    So, I woke up and chose violence. God bless me and the stack memory. What I've done here is a sin. Do not follow my footsteps here if you want to write maintainable code.
+    So, I woke up and chose violence. God bless me and the stack memory. What I've done here is a sin. Do not follow my footsteps here if you want to write safe and maintainable code.
     -- @ohsayan
 */
 
@@ -48,7 +48,6 @@ pub enum Dataflag {
     Float,
     Bin,
     Str,
-    List,
 }
 
 /// Information about the type that implements the dataspec traits
@@ -296,12 +295,17 @@ pub unsafe trait DataspecMethods1D: Dataspec1D {
                 Dataflag::Float => self.read_float_uck(),
                 Dataflag::Bin => self.read_bin_uck(),
                 Dataflag::Str => self.read_str_uck(),
-                #[allow(unused_variables)]
-                #[allow(unreachable_code)]
-                Dataflag::List => {
-                    panic!("found 2D in 1D data")
-                },
             )
         };
     }
+    fn to_string_debug(&self) -> String {
+        match self.kind() {
+            Dataflag::Bool => self.bool().to_string(),
+            Dataflag::UnsignedInt => self.uint().to_string(),
+            Dataflag::SignedInt => self.sint().to_string(),
+            Dataflag::Float => self.float().to_string(),
+            Dataflag::Bin => format!("{:?}", self.bin()),
+            Dataflag::Str => format!("{:?}", self.str()),
+        }
+    }
 }
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index 4032cebc..e7b22589 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -96,7 +96,7 @@ impl SystemDword for NativeTword {
         let x;
         #[cfg(target_pointer_width = "32")]
         {
-            let [a, b] = unsafe { core::mem::transmute(u) };
+            let [a, b]: [usize; 2] = unsafe { core::mem::transmute(u) };
             x = [a, b, 0];
         }
         #[cfg(target_pointer_width = "64")]

From ca04ea2cd127abcc4d70a518a7fd17bbab11e111 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 27 Feb 2023 05:50:14 -0800
Subject: [PATCH 141/310] Use custom lit impl

---
 server/src/engine/data/lit.rs               |  13 +-
 server/src/engine/data/macros.rs            |  60 ++++++++
 server/src/engine/data/md_dict.rs           |   6 +-
 server/src/engine/data/mod.rs               |  23 ++--
 server/src/engine/data/spec.rs              |  27 ++--
 server/src/engine/ql/ast/mod.rs             |   8 +-
 server/src/engine/ql/benches.rs             |   7 +-
 server/src/engine/ql/dml/mod.rs             |   4 +-
 server/src/engine/ql/dml/upd.rs             |   3 +-
 server/src/engine/ql/lex/mod.rs             |  14 +-
 server/src/engine/ql/lex/raw.rs             |  66 +--------
 server/src/engine/ql/tests.rs               |   2 +-
 server/src/engine/ql/tests/dml_tests.rs     | 144 +++++++++++---------
 server/src/engine/ql/tests/lexer_tests.rs   |  39 ++++--
 server/src/engine/ql/tests/schema_tests.rs  |  11 +-
 server/src/engine/ql/tests/structure_syn.rs |   6 +-
 16 files changed, 239 insertions(+), 194 deletions(-)
 create mode 100644 server/src/engine/data/macros.rs

diff --git a/server/src/engine/data/lit.rs b/server/src/engine/data/lit.rs
index 13f6f7fa..6d967c87 100644
--- a/server/src/engine/data/lit.rs
+++ b/server/src/engine/data/lit.rs
@@ -45,7 +45,7 @@ pub struct Lit<'a> {
 }
 
 impl<'a> Lit<'a> {
-    fn as_ir(&'a self) -> LitIR<'a> {
+    pub fn as_ir(&'a self) -> LitIR<'a> {
         unsafe {
             // UNSAFE(@ohsayan): 'tis the lifetime. 'tis the savior
             mem::transmute_copy(self)
@@ -86,10 +86,11 @@ impl<'a> DataspecMeta1D for Lit<'a> {
 unsafe impl<'a> DataspecRaw1D for Lit<'a> {
     const HEAP_STR: bool = true;
     const HEAP_BIN: bool = false;
-    unsafe fn drop_str(s: &str) {
-        drop(Box::from_raw(s as *const _ as *mut str));
+    unsafe fn drop_str(&mut self) {
+        let [ptr, len] = self.data().load_fat();
+        drop(String::from_raw_parts(ptr as *mut u8, len, len));
     }
-    unsafe fn drop_bin(_: &[u8]) {}
+    unsafe fn drop_bin(&mut self) {}
     unsafe fn clone_str(s: &str) -> Self::Target {
         let new_string = ManuallyDrop::new(s.to_owned().into_boxed_str());
         SystemDword::store_fat(new_string.as_ptr() as usize, new_string.len())
@@ -209,8 +210,8 @@ impl<'a> DataspecMeta1D for LitIR<'a> {
 unsafe impl<'a> DataspecRaw1D for LitIR<'a> {
     const HEAP_STR: bool = false;
     const HEAP_BIN: bool = false;
-    unsafe fn drop_str(_: &str) {}
-    unsafe fn drop_bin(_: &[u8]) {}
+    unsafe fn drop_str(&mut self) {}
+    unsafe fn drop_bin(&mut self) {}
     unsafe fn clone_str(s: &str) -> Self::Target {
         SystemDword::store_fat(s.as_ptr() as usize, s.len())
     }
diff --git a/server/src/engine/data/macros.rs b/server/src/engine/data/macros.rs
new file mode 100644
index 00000000..70dc8cc9
--- /dev/null
+++ b/server/src/engine/data/macros.rs
@@ -0,0 +1,60 @@
+/*
+ * Created on Mon Feb 27 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/// This is a pretty complex macro that emulates the behavior of an enumeration by making use of flags and macro hacks. You might literally feel it's like a lang match, but nope,
+/// there's a lot of wizardry beneath. Well, it's important to know that it works and you shouldn't touch it UNLESS YOU ABSOLUTELY KNOW what you're doing
+macro_rules! match_data {
+    (match ref $dataitem:ident $tail:tt) => {match_data!(@branch [ #[deny(unreachable_patterns)] match crate::engine::data::spec::DataspecMeta1D::kind($dataitem)] $dataitem [] $tail)};
+    (match $dataitem:ident $tail:tt) => {match_data!(@branch [ #[deny(unreachable_patterns)] match crate::engine::data::spec::DataspecMeta1D::kind(&$dataitem)] $dataitem [] $tail)};
+    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] {}) => {match_data!(@defeat0 $decl [$($branch)*])};
+    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $name:ident::$variant:ident($capture:ident) => $ret:expr, $($tail:tt)*}) => {
+        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::spec::Dataflag::$variant => {let $capture = unsafe { /* UNSAFE(@ohsayan): flagck */ match_data!(@extract $name $dataitem $variant) }; $ret},] {$($tail)*})
+    };
+    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $name:ident::$variant:ident(_) => $ret:expr, $($tail:tt)*}) => {
+        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::spec::Dataflag::$variant => $ret,] {$($tail)*})
+    };
+    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $name:ident::$variant:ident($capture:ident) if $guard:expr => $ret:expr, $($tail:tt)*}) => {
+        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::spec::Dataflag::$variant if { let $capture = unsafe { /* UNSAFE(@ohsayan): flagck */ match_data!(@extract $name $dataitem $variant) }; $guard } => {
+            let $capture = unsafe { /* UNSAFE(@ohsayan): flagck */  match_data!(@extract $name $dataitem $variant) }; let _ = &$capture; $ret}, ] {$($tail)*}
+        )
+    };
+    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $name:ident::$variant:ident(_) if $guard:expr => $ret:expr, $($tail:tt)*}) => {
+        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::spec::Dataflag::$variant if $guard => $ret,] {$($tail)*})
+    };
+    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* _ => $ret:expr, $($tail:tt)*}) => {
+        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* _ => $ret,] {$($tail)*})
+    };
+    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $capture:ident => $ret:expr, $($tail:tt)* }) => {
+        match_data!(@branch $decl $dataitem [ $($branch)* $(#[$attr])* $capture => { $ret},] {$($tail:tt)*})
+    };
+    (@defeat0 [$($decl:tt)*] [$($branch:tt)*]) => {$($decl)* { $($branch)* }};
+    (@extract $name:ident $dataitem:ident Bool) => {<$name as crate::engine::data::spec::Dataspec1D>::read_bool_uck(&$dataitem)};
+    (@extract $name:ident $dataitem:ident UnsignedInt) => {<$name as crate::engine::data::spec::Dataspec1D>::read_uint_uck(&$dataitem)};
+    (@extract $name:ident $dataitem:ident SignedInt) => {<$name as crate::engine::data::spec::Dataspec1D>::read_sint_uck(&$dataitem)};
+    (@extract $name:ident $dataitem:ident Float) => {<$name as crate::engine::data::spec::Dataspec1D>::read_float_uck(&$dataitem)};
+    (@extract $name:ident $dataitem:ident Bin) => {<$name as crate::engine::data::spec::Dataspec1D>::read_bin_uck(&$dataitem)};
+    (@extract $name:ident $dataitem:ident Str) => {<$name as crate::engine::data::spec::Dataspec1D>::read_str_uck(&$dataitem)};
+}
diff --git a/server/src/engine/data/md_dict.rs b/server/src/engine/data/md_dict.rs
index 309d13fc..bf0ff407 100644
--- a/server/src/engine/data/md_dict.rs
+++ b/server/src/engine/data/md_dict.rs
@@ -26,9 +26,11 @@
 
 use {
     crate::engine::{
-        data::HSData,
+        data::{
+            lit::{Lit, LitIR},
+            HSData,
+        },
         idx::STIndex,
-        ql::lex::{Lit, LitIR},
     },
     std::collections::HashMap,
 };
diff --git a/server/src/engine/data/mod.rs b/server/src/engine/data/mod.rs
index 4a81710a..dd071c0c 100644
--- a/server/src/engine/data/mod.rs
+++ b/server/src/engine/data/mod.rs
@@ -24,6 +24,8 @@
  *
 */
 
+#[macro_use]
+mod macros;
 pub mod lit;
 pub mod md_dict;
 pub mod spec;
@@ -32,10 +34,8 @@ mod tests;
 
 pub use md_dict::{DictEntryGeneric, DictGeneric, MetaDict};
 use {
-    crate::engine::{
-        mem::AStr,
-        ql::lex::{Lit, LitIR},
-    },
+    self::lit::Lit,
+    crate::engine::{data::lit::LitIR, mem::AStr},
     std::mem::{self, Discriminant},
 };
 
@@ -88,24 +88,25 @@ enum_impls! {
 impl HSData {
     #[inline(always)]
     pub(super) fn clone_from_lit(lit: Lit) -> Self {
-        match lit {
-            Lit::Str(s) => HSData::String(s.clone()),
+        match_data!(match lit {
+            Lit::Str(s) => HSData::String(s.to_string().into_boxed_str()),
             Lit::Bool(b) => HSData::Boolean(b),
             Lit::UnsignedInt(u) => HSData::UnsignedInt(u),
             Lit::SignedInt(i) => HSData::SignedInt(i),
+            Lit::Float(f) => HSData::Float(f),
             Lit::Bin(l) => HSData::Binary(l.to_vec().into_boxed_slice()),
-        }
+        })
     }
     #[inline(always)]
     pub(super) fn clone_from_litir<'a>(lit: LitIR<'a>) -> Self {
-        match lit {
+        match_data!(match lit {
             LitIR::Str(s) => Self::String(s.to_owned().into_boxed_str()),
             LitIR::Bin(b) => Self::Binary(b.to_owned().into_boxed_slice()),
             LitIR::Float(f) => Self::Float(f),
-            LitIR::SInt(s) => Self::SignedInt(s),
-            LitIR::UInt(u) => Self::UnsignedInt(u),
+            LitIR::SignedInt(s) => Self::SignedInt(s),
+            LitIR::UnsignedInt(u) => Self::UnsignedInt(u),
             LitIR::Bool(b) => Self::Boolean(b),
-        }
+        })
     }
     fn kind(&self) -> Discriminant<Self> {
         mem::discriminant(&self)
diff --git a/server/src/engine/data/spec.rs b/server/src/engine/data/spec.rs
index e375eaf7..1c7c3f69 100644
--- a/server/src/engine/data/spec.rs
+++ b/server/src/engine/data/spec.rs
@@ -77,9 +77,9 @@ pub unsafe trait DataspecRaw1D: DataspecMeta1D {
     /// Is the binary heap allocated...anywhere down the line?
     const HEAP_BIN: bool;
     /// Drop the string, if you need a dtor
-    unsafe fn drop_str(s: &str);
+    unsafe fn drop_str(&mut self);
     /// Drop the binary, if you need a dtor
-    unsafe fn drop_bin(b: &[u8]);
+    unsafe fn drop_bin(&mut self);
     /// Clone the string object. Note, we literally HAVE NO IDEA about what you're doing here
     unsafe fn clone_str(s: &str) -> Self::Target;
     /// Clone the binary object. Again, NOT A DAMN CLUE about whay you're doing down there
@@ -237,11 +237,11 @@ pub unsafe trait DataspecMethods1D: Dataspec1D {
         match self.kind() {
             Dataflag::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
                 // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
-                <Self as DataspecRaw1D>::drop_str(Dataspec1D::read_str_uck(self))
+                <Self as DataspecRaw1D>::drop_str(self)
             },
             Dataflag::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
                 // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
-                <Self as DataspecRaw1D>::drop_bin(Dataspec1D::read_bin_uck(self))
+                <Self as DataspecRaw1D>::drop_bin(self)
             },
             _ => {}
         }
@@ -264,7 +264,7 @@ pub unsafe trait DataspecMethods1D: Dataspec1D {
         unsafe {
             // UNSAFE(@ohsayan): we are checking our flags
             match (self.kind(), other.kind()) {
-                (Dataflag::Bool, Dataflag::Bool) => self.read_bool_uck() & other.read_bool_uck(),
+                (Dataflag::Bool, Dataflag::Bool) => self.read_bool_uck() == other.read_bool_uck(),
                 (Dataflag::UnsignedInt, Dataflag::UnsignedInt) => {
                     self.read_uint_uck() == other.read_uint_uck()
                 }
@@ -298,14 +298,15 @@ pub unsafe trait DataspecMethods1D: Dataspec1D {
             )
         };
     }
+    #[rustfmt::skip]
     fn to_string_debug(&self) -> String {
-        match self.kind() {
-            Dataflag::Bool => self.bool().to_string(),
-            Dataflag::UnsignedInt => self.uint().to_string(),
-            Dataflag::SignedInt => self.sint().to_string(),
-            Dataflag::Float => self.float().to_string(),
-            Dataflag::Bin => format!("{:?}", self.bin()),
-            Dataflag::Str => format!("{:?}", self.str()),
-        }
+        match_data!(match ref self {
+            Self::Bool(b) => b.to_string(),
+            Self::UnsignedInt(u) => u.to_string(),
+            Self::SignedInt(s) => s.to_string(),
+            Self::Float(f) => f.to_string(),
+            Self::Bin(b) => format!("{:?}", b),
+            Self::Str(s) => format!("{:?}", s),
+        })
     }
 }
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 55025110..8dd04a97 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -32,11 +32,11 @@ use {
     self::traits::ASTNode,
     super::{
         ddl, dml,
-        lex::{Ident, LitIR, Token},
+        lex::{Ident, Token},
     },
     crate::{
         engine::{
-            data::HSData,
+            data::{lit::LitIR, HSData},
             error::{LangError, LangResult},
         },
         util::{compiler, MaybeInit},
@@ -332,14 +332,14 @@ impl<'a> QueryData<'a> for SubstitutedData<'a> {
     #[inline(always)]
     unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a> {
         debug_assert!(Token![?].eq(tok));
-        let ret = self.data[0];
+        let ret = self.data[0].clone();
         self.data = &self.data[1..];
         ret
     }
     #[inline(always)]
     unsafe fn read_data_type(&mut self, tok: &'a Token) -> HSData {
         debug_assert!(Token![?].eq(tok));
-        let ret = self.data[0];
+        let ret = self.data[0].clone();
         self.data = &self.data[1..];
         HSData::from(ret)
     }
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index ebbe1e24..c042d99d 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -45,12 +45,15 @@ use {
 mod lexer {
     use {
         super::*,
-        crate::engine::ql::lex::{Lit, Token},
+        crate::engine::{
+            data::{lit::Lit, spec::Dataspec1D},
+            ql::lex::Token,
+        },
     };
     #[bench]
     fn lex_number(b: &mut Bencher) {
         let src = b"1234567890";
-        let expected = vec![Token::Lit(1234567890.into())];
+        let expected = vec![Token::Lit(1234567890_u64.into())];
         b.iter(|| assert_eq!(lex_insecure(src).unwrap(), expected));
     }
     #[bench]
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index 09ba4218..e09fbe8a 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -37,9 +37,9 @@ pub mod upd;
 use {
     super::{
         ast::{QueryData, State},
-        lex::{Ident, LitIR, Token},
+        lex::{Ident, Token},
     },
-    crate::util::compiler,
+    crate::{engine::data::lit::LitIR, util::compiler},
     std::collections::HashMap,
 };
 
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index 8f92f873..50c160bc 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -30,10 +30,11 @@ use {
     super::{read_ident, u, WhereClause},
     crate::{
         engine::{
+            data::lit::LitIR,
             error::{LangError, LangResult},
             ql::{
                 ast::{Entity, QueryData, State},
-                lex::{Ident, LitIR},
+                lex::Ident,
             },
         },
         util::{compiler, MaybeInit},
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index 6cad4d92..fcfb99b9 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -29,13 +29,19 @@ mod raw;
 use {
     self::raw::RawLexer,
     crate::{
-        engine::error::{LexError, LexResult},
+        engine::{
+            data::{
+                lit::{Lit, LitIR},
+                spec::Dataspec1D,
+            },
+            error::{LexError, LexResult},
+        },
         util::compiler,
     },
     core::{cmp, fmt, ops::BitOr, slice, str},
 };
 
-pub use self::raw::{Ident, Keyword, Lit, LitIR, Symbol, Token};
+pub use self::raw::{Ident, Keyword, Symbol, Token};
 pub type Slice<'a> = &'a [u8];
 
 /*
@@ -469,14 +475,14 @@ impl<'b> SafeQueryData<'b> {
     pub(super) fn uint<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
         let mut b = true;
         let r = decode_num_ub(src, &mut b, cnt);
-        data.push(LitIR::UInt(r));
+        data.push(LitIR::UnsignedInt(r));
         b
     }
     #[inline(always)]
     pub(super) fn sint<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
         let mut b = true;
         let r = decode_num_ub(src, &mut b, cnt);
-        data.push(LitIR::SInt(r));
+        data.push(LitIR::SignedInt(r));
         b
     }
     #[inline(always)]
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index 76f3c137..f947dbbd 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -26,7 +26,10 @@
 
 use {
     super::Slice,
-    crate::engine::error::LexError,
+    crate::engine::{
+        data::{lit::Lit, spec::Dataspec1D},
+        error::LexError,
+    },
     core::{borrow::Borrow, fmt, ops::Deref, slice, str},
 };
 
@@ -151,56 +154,6 @@ enum_impls! {
     }
 }
 
-#[derive(Debug, PartialEq, Clone)]
-#[repr(u8)]
-/// A [`Lit`] as represented by an insecure token stream
-pub enum Lit<'a> {
-    Str(Box<str>),
-    Bool(bool),
-    UnsignedInt(u64),
-    SignedInt(i64),
-    Bin(Slice<'a>),
-}
-
-impl<'a> ToString for Lit<'a> {
-    fn to_string(&self) -> String {
-        match self {
-            Self::Str(s) => format!("{:?}", s),
-            Self::Bool(b) => b.to_string(),
-            Self::UnsignedInt(u) => u.to_string(),
-            Self::SignedInt(s) => s.to_string(),
-            Self::Bin(b) => format!("{:?}", b),
-        }
-    }
-}
-
-impl<'a> Lit<'a> {
-    pub fn as_ir(&'a self) -> LitIR<'a> {
-        match self {
-            Self::Str(s) => LitIR::Str(s.as_ref()),
-            Self::Bool(b) => LitIR::Bool(*b),
-            Self::UnsignedInt(u) => LitIR::UInt(*u),
-            Self::SignedInt(s) => LitIR::SInt(*s),
-            Self::Bin(b) => LitIR::Bin(b),
-        }
-    }
-}
-
-impl<'a> From<&'static str> for Lit<'a> {
-    fn from(s: &'static str) -> Self {
-        Self::Str(s.into())
-    }
-}
-
-enum_impls! {
-    Lit<'a> => {
-        Box<str> as Str,
-        String as Str,
-        bool as Bool,
-        u64 as UnsignedInt,
-    }
-}
-
 build_lut!(
     static KW_LUT in kwlut;
     #[derive(Debug, PartialEq, Eq, Clone, Copy)]
@@ -406,17 +359,6 @@ impl<'a> AsRef<Token<'a>> for Token<'a> {
     }
 }
 
-#[derive(PartialEq, Debug, Clone, Copy)]
-/// Intermediate literal repr
-pub enum LitIR<'a> {
-    Str(&'a str),
-    Bin(Slice<'a>),
-    UInt(u64),
-    SInt(i64),
-    Bool(bool),
-    Float(f64),
-}
-
 #[derive(Debug)]
 pub struct RawLexer<'a> {
     c: *const u8,
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 6a678099..3b81622a 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -85,7 +85,7 @@ impl NullableDictEntry for Null {
     }
 }
 
-impl<'a> NullableDictEntry for super::lex::Lit<'a> {
+impl<'a> NullableDictEntry for crate::engine::data::lit::Lit<'a> {
     fn data(self) -> Option<crate::engine::data::DictEntryGeneric> {
         Some(crate::engine::data::DictEntryGeneric::from(self.as_ir()))
     }
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index eb654de0..4c830c40 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -27,10 +27,12 @@
 use super::*;
 mod list_parse {
     use super::*;
-    use crate::engine::ql::{
-        ast::{parse_ast_node_full, traits::ASTNode, State, SubstitutedData},
-        dml::ins::List,
-        lex::LitIR,
+    use crate::engine::{
+        data::{lit::LitIR, spec::Dataspec1D},
+        ql::{
+            ast::{parse_ast_node_full, traits::ASTNode, State, SubstitutedData},
+            dml::ins::List,
+        },
     };
 
     #[test]
@@ -64,10 +66,10 @@ mod list_parse {
         )
         .unwrap();
         let data = [
-            LitIR::UInt(1),
-            LitIR::UInt(2),
-            LitIR::UInt(3),
-            LitIR::UInt(4),
+            LitIR::UnsignedInt(1),
+            LitIR::UnsignedInt(2),
+            LitIR::UnsignedInt(3),
+            LitIR::UnsignedInt(4),
         ];
         let mut state = State::new(&tok[1..], SubstitutedData::new(&data));
         assert_eq!(
@@ -115,12 +117,12 @@ mod list_parse {
         )
         .unwrap();
         let data = [
-            LitIR::UInt(1),
-            LitIR::UInt(2),
-            LitIR::UInt(3),
-            LitIR::UInt(4),
-            LitIR::UInt(5),
-            LitIR::UInt(6),
+            LitIR::UnsignedInt(1),
+            LitIR::UnsignedInt(2),
+            LitIR::UnsignedInt(3),
+            LitIR::UnsignedInt(4),
+            LitIR::UnsignedInt(5),
+            LitIR::UnsignedInt(6),
         ];
         let mut state = State::new(&tok[1..], SubstitutedData::new(&data));
         assert_eq!(
@@ -173,18 +175,18 @@ mod list_parse {
         )
         .unwrap();
         let data = [
-            LitIR::UInt(1),
-            LitIR::UInt(1),
-            LitIR::UInt(2),
-            LitIR::UInt(2),
-            LitIR::UInt(4),
-            LitIR::UInt(4),
-            LitIR::UInt(5),
-            LitIR::UInt(5),
-            LitIR::UInt(6),
-            LitIR::UInt(6),
-            LitIR::UInt(7),
-            LitIR::UInt(7),
+            LitIR::UnsignedInt(1),
+            LitIR::UnsignedInt(1),
+            LitIR::UnsignedInt(2),
+            LitIR::UnsignedInt(2),
+            LitIR::UnsignedInt(4),
+            LitIR::UnsignedInt(4),
+            LitIR::UnsignedInt(5),
+            LitIR::UnsignedInt(5),
+            LitIR::UnsignedInt(6),
+            LitIR::UnsignedInt(6),
+            LitIR::UnsignedInt(7),
+            LitIR::UnsignedInt(7),
         ];
         let mut state = State::new(&tok[1..], SubstitutedData::new(&data));
         assert_eq!(
@@ -596,10 +598,13 @@ mod stmt_insert {
 mod stmt_select {
     use {
         super::*,
-        crate::engine::ql::{
-            ast::{parse_ast_node_full, Entity},
-            dml::{sel::SelectStatement, RelationalExpr},
-            lex::{Ident, LitIR},
+        crate::engine::{
+            data::{lit::LitIR, spec::Dataspec1D},
+            ql::{
+                ast::{parse_ast_node_full, Entity},
+                dml::{sel::SelectStatement, RelationalExpr},
+                lex::Ident,
+            },
         },
     };
     #[test]
@@ -690,10 +695,13 @@ mod stmt_select {
 mod expression_tests {
     use {
         super::*,
-        crate::engine::ql::{
-            ast::parse_ast_node_full,
-            dml::upd::{AssignmentExpression, Operator},
-            lex::{Ident, LitIR},
+        crate::engine::{
+            data::{lit::LitIR, spec::Dataspec1D},
+            ql::{
+                ast::parse_ast_node_full,
+                dml::upd::{AssignmentExpression, Operator},
+                lex::Ident,
+            },
         },
     };
     #[test]
@@ -717,7 +725,7 @@ mod expression_tests {
             r,
             AssignmentExpression::new(
                 Ident::from("followers"),
-                LitIR::UInt(100),
+                LitIR::UnsignedInt(100),
                 Operator::AddAssign
             )
         );
@@ -730,7 +738,7 @@ mod expression_tests {
             r,
             AssignmentExpression::new(
                 Ident::from("following"),
-                LitIR::UInt(150),
+                LitIR::UnsignedInt(150),
                 Operator::SubAssign
             )
         );
@@ -743,7 +751,7 @@ mod expression_tests {
             r,
             AssignmentExpression::new(
                 Ident::from("product_qty"),
-                LitIR::UInt(2),
+                LitIR::UnsignedInt(2),
                 Operator::MulAssign
             )
         );
@@ -756,7 +764,7 @@ mod expression_tests {
             r,
             AssignmentExpression::new(
                 Ident::from("image_crop_factor"),
-                LitIR::UInt(2),
+                LitIR::UnsignedInt(2),
                 Operator::DivAssign
             )
         );
@@ -765,13 +773,16 @@ mod expression_tests {
 mod update_statement {
     use {
         super::*,
-        crate::engine::ql::{
-            ast::{parse_ast_node_full, Entity},
-            dml::{
-                upd::{AssignmentExpression, Operator, UpdateStatement},
-                RelationalExpr, WhereClause,
+        crate::engine::{
+            data::{lit::LitIR, spec::Dataspec1D},
+            ql::{
+                ast::{parse_ast_node_full, Entity},
+                dml::{
+                    upd::{AssignmentExpression, Operator, UpdateStatement},
+                    RelationalExpr, WhereClause,
+                },
+                lex::Ident,
             },
-            lex::{Ident, LitIR},
         },
     };
     #[test]
@@ -843,10 +854,13 @@ mod update_statement {
 mod delete_stmt {
     use {
         super::*,
-        crate::engine::ql::{
-            ast::{parse_ast_node_full, Entity},
-            dml::{del::DeleteStatement, RelationalExpr},
-            lex::{Ident, LitIR},
+        crate::engine::{
+            data::{lit::LitIR, spec::Dataspec1D},
+            ql::{
+                ast::{parse_ast_node_full, Entity},
+                dml::{del::DeleteStatement, RelationalExpr},
+                lex::Ident,
+            },
         },
     };
 
@@ -900,10 +914,9 @@ mod delete_stmt {
 mod relational_expr {
     use {
         super::*,
-        crate::engine::ql::{
-            ast::parse_ast_node_full,
-            dml::RelationalExpr,
-            lex::{Ident, LitIR},
+        crate::engine::{
+            data::{lit::LitIR, spec::Dataspec1D},
+            ql::{ast::parse_ast_node_full, dml::RelationalExpr, lex::Ident},
         },
     };
 
@@ -914,7 +927,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: LitIR::UInt(10),
+                rhs: LitIR::UnsignedInt(10),
                 lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_EQ
             }
@@ -927,7 +940,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: LitIR::UInt(10),
+                rhs: LitIR::UnsignedInt(10),
                 lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_NE
             }
@@ -940,7 +953,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: LitIR::UInt(10),
+                rhs: LitIR::UnsignedInt(10),
                 lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_GT
             }
@@ -953,7 +966,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: LitIR::UInt(10),
+                rhs: LitIR::UnsignedInt(10),
                 lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_GE
             }
@@ -966,7 +979,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: LitIR::UInt(10),
+                rhs: LitIR::UnsignedInt(10),
                 lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_LT
             }
@@ -980,7 +993,7 @@ mod relational_expr {
             r,
             RelationalExpr::new(
                 Ident::from("primary_key"),
-                LitIR::UInt(10),
+                LitIR::UnsignedInt(10),
                 RelationalExpr::OP_LE
             )
         );
@@ -989,10 +1002,13 @@ mod relational_expr {
 mod where_clause {
     use {
         super::*,
-        crate::engine::ql::{
-            ast::parse_ast_node_full,
-            dml::{RelationalExpr, WhereClause},
-            lex::{Ident, LitIR},
+        crate::engine::{
+            data::{lit::LitIR, spec::Dataspec1D},
+            ql::{
+                ast::parse_ast_node_full,
+                dml::{RelationalExpr, WhereClause},
+                lex::Ident,
+            },
         },
     };
     #[test]
@@ -1006,7 +1022,7 @@ mod where_clause {
         let expected = WhereClause::new(dict! {
             Ident::from("x") => RelationalExpr::new(
                 Ident::from("x"),
-                LitIR::UInt(100),
+                LitIR::UnsignedInt(100),
                 RelationalExpr::OP_EQ
             )
         });
@@ -1023,7 +1039,7 @@ mod where_clause {
         let expected = WhereClause::new(dict! {
             Ident::from("userid") => RelationalExpr::new(
                 Ident::from("userid"),
-                LitIR::UInt(100),
+                LitIR::UnsignedInt(100),
                 RelationalExpr::OP_EQ
             ),
             Ident::from("pass") => RelationalExpr::new(
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index ad4e5e21..274cfa56 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -26,10 +26,13 @@
 
 use {
     super::{
-        super::lex::{Ident, Lit, Token},
+        super::lex::{Ident, Token},
         lex_insecure,
     },
-    crate::engine::error::LexError,
+    crate::engine::{
+        data::{lit::Lit, spec::Dataspec1D},
+        error::LexError,
+    },
 };
 
 macro_rules! v(
@@ -130,8 +133,9 @@ fn lex_string_escape_bs() {
         vec![Token::Lit(Lit::Str("windows has c:\\".into()))]
     );
     let lol = v!(r#"'\\\\\\\\\\'"#);
+    let lexed = lex_insecure(&lol).unwrap();
     assert_eq!(
-        lex_insecure(&lol).unwrap(),
+        lexed,
         vec![Token::Lit(Lit::Str("\\".repeat(5).into_boxed_str()))],
         "lol"
     )
@@ -277,9 +281,11 @@ mod num_tests {
 }
 
 mod safequery_params {
+    use crate::engine::{
+        data::{lit::LitIR, spec::Dataspec1D},
+        ql::lex::SafeQueryData,
+    };
     use rand::seq::SliceRandom;
-
-    use crate::engine::ql::lex::{LitIR, SafeQueryData};
     #[test]
     fn param_uint() {
         let src = b"12345\n";
@@ -287,7 +293,7 @@ mod safequery_params {
         let mut i = 0;
         assert!(SafeQueryData::uint(src, &mut i, &mut d));
         assert_eq!(i, src.len());
-        assert_eq!(d, vec![LitIR::UInt(12345)]);
+        assert_eq!(d, vec![LitIR::UnsignedInt(12345)]);
     }
     #[test]
     fn param_sint() {
@@ -296,7 +302,7 @@ mod safequery_params {
         let mut i = 0;
         assert!(SafeQueryData::sint(src, &mut i, &mut d));
         assert_eq!(i, src.len());
-        assert_eq!(d, vec![LitIR::SInt(-12345)]);
+        assert_eq!(d, vec![LitIR::SignedInt(-12345)]);
     }
     #[test]
     fn param_bool_true() {
@@ -347,13 +353,13 @@ mod safequery_params {
     fn param_full_uint() {
         let src = b"\x0012345\n";
         let r = SafeQueryData::p_revloop(src, 1).unwrap();
-        assert_eq!(r.as_ref(), [LitIR::UInt(12345)]);
+        assert_eq!(r.as_ref(), [LitIR::UnsignedInt(12345)]);
     }
     #[test]
     fn param_full_sint() {
         let src = b"\x01-12345\n";
         let r = SafeQueryData::p_revloop(src, 1).unwrap();
-        assert_eq!(r.as_ref(), [LitIR::SInt(-12345)]);
+        assert_eq!(r.as_ref(), [LitIR::SignedInt(-12345)]);
     }
     #[test]
     fn param_full_bool() {
@@ -396,9 +402,9 @@ mod safequery_params {
             b"\x0430\none two three four five binary",
             b"\x0527\none two three four five str",
         ];
-        const RETMAP: [LitIR; 6] = [
-            LitIR::UInt(12345),
-            LitIR::SInt(-12345),
+        let retmap: [LitIR; 6] = [
+            LitIR::UnsignedInt(12345),
+            LitIR::SignedInt(-12345),
             LitIR::Bool(true),
             LitIR::Float(12345.67890),
             LitIR::Bin(b"one two three four five binary"),
@@ -409,7 +415,7 @@ mod safequery_params {
             local_data.shuffle(&mut rng);
             let ret: Vec<LitIR> = local_data
                 .iter()
-                .map(|v| RETMAP[v[0] as usize].clone())
+                .map(|v| retmap[v[0] as usize].clone())
                 .collect();
             let src: Vec<u8> = local_data.into_iter().flat_map(|v| v.to_owned()).collect();
             let r = SafeQueryData::p_revloop(&src, 6).unwrap();
@@ -419,7 +425,10 @@ mod safequery_params {
 }
 
 mod safequery_full_param {
-    use crate::engine::ql::lex::{Ident, LitIR, SafeQueryData, Token};
+    use crate::engine::{
+        data::{lit::LitIR, spec::Dataspec1D},
+        ql::lex::{Ident, SafeQueryData, Token},
+    };
     #[test]
     fn p_mini() {
         let query = b"select * from myapp where username = ?";
@@ -477,7 +486,7 @@ mod safequery_full_param {
             sq,
             SafeQueryData::new_test(
                 vec![
-                    LitIR::UInt(100),
+                    LitIR::UnsignedInt(100),
                     LitIR::Str("sayan"),
                     LitIR::Bin(b"pass1234")
                 ]
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 4a18fdb7..a701ca36 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -24,9 +24,9 @@
  *
 */
 
-use super::{
-    super::lex::{Ident, Lit},
-    lex_insecure, *,
+use {
+    super::{super::lex::Ident, lex_insecure, *},
+    crate::engine::data::{lit::Lit, spec::Dataspec1D},
 };
 mod inspect {
     use {
@@ -70,7 +70,10 @@ mod inspect {
 mod alter_space {
     use {
         super::*,
-        crate::engine::ql::{ast::parse_ast_node_full, ddl::alt::AlterSpace, lex::Lit},
+        crate::engine::{
+            data::{lit::Lit, spec::Dataspec1D},
+            ql::{ast::parse_ast_node_full, ddl::alt::AlterSpace},
+        },
     };
     #[test]
     fn alter_space_mini() {
diff --git a/server/src/engine/ql/tests/structure_syn.rs b/server/src/engine/ql/tests/structure_syn.rs
index 466ad38e..08ff240e 100644
--- a/server/src/engine/ql/tests/structure_syn.rs
+++ b/server/src/engine/ql/tests/structure_syn.rs
@@ -27,8 +27,8 @@
 use {
     super::*,
     crate::engine::{
-        data::DictGeneric,
-        ql::{ast::parse_ast_node_full, ddl::syn::DictBasic, lex::Lit},
+        data::{lit::Lit, spec::Dataspec1D, DictGeneric},
+        ql::{ast::parse_ast_node_full, ddl::syn::DictBasic},
     },
 };
 
@@ -232,7 +232,7 @@ mod dict {
 mod null_dict_tests {
     use super::*;
     mod dict {
-        use {super::*, crate::engine::ql::lex::Lit};
+        use super::*;
 
         #[test]
         fn null_mini() {

From f2dbec02299f259432e34c1182de85a5be78695e Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 27 Feb 2023 09:21:59 -0800
Subject: [PATCH 142/310] Use custom tags

---
 server/src/engine/data/lit.rs    |  23 +++---
 server/src/engine/data/macros.rs |  12 +--
 server/src/engine/data/mod.rs    |   3 +
 server/src/engine/data/spec.rs   |  86 ++++++++++----------
 server/src/engine/data/tag.rs    | 130 +++++++++++++++++++++++++++++++
 5 files changed, 195 insertions(+), 59 deletions(-)
 create mode 100644 server/src/engine/data/tag.rs

diff --git a/server/src/engine/data/lit.rs b/server/src/engine/data/lit.rs
index 6d967c87..011e6a7b 100644
--- a/server/src/engine/data/lit.rs
+++ b/server/src/engine/data/lit.rs
@@ -25,7 +25,10 @@
 */
 
 use {
-    super::spec::{Dataflag, Dataspec1D, DataspecMeta1D, DataspecMethods1D, DataspecRaw1D},
+    super::{
+        spec::{Dataspec1D, DataspecMeta1D, DataspecMethods1D, DataspecRaw1D},
+        tag::{DataTag, FullTag},
+    },
     crate::engine::mem::{NativeDword, SystemDword},
     core::{
         fmt,
@@ -40,7 +43,7 @@ use {
 
 pub struct Lit<'a> {
     data: NativeDword,
-    tag: Dataflag,
+    tag: FullTag,
     _lt: PhantomData<&'a [u8]>,
 }
 
@@ -54,16 +57,17 @@ impl<'a> Lit<'a> {
 }
 
 impl<'a> DataspecMeta1D for Lit<'a> {
+    type Tag = FullTag;
     type Target = NativeDword;
     type StringItem = Box<str>;
-    fn new(flag: Dataflag, data: Self::Target) -> Self {
+    fn new(flag: Self::Tag, data: Self::Target) -> Self {
         Self {
             data,
             tag: flag,
             _lt: PhantomData,
         }
     }
-    fn kind(&self) -> Dataflag {
+    fn kind(&self) -> Self::Tag {
         self.tag
     }
     fn data(&self) -> Self::Target {
@@ -109,7 +113,7 @@ unsafe impl<'a> Dataspec1D for Lit<'a> {
     fn Str(s: Box<str>) -> Self {
         let md = ManuallyDrop::new(s);
         Self::new(
-            Dataflag::Str,
+            FullTag::STR,
             SystemDword::store_fat(md.as_ptr() as _, md.len()),
         )
     }
@@ -172,7 +176,7 @@ enum_impls! {
 */
 
 pub struct LitIR<'a> {
-    tag: Dataflag,
+    tag: FullTag,
     data: NativeDword,
     _lt: PhantomData<&'a str>,
 }
@@ -180,14 +184,15 @@ pub struct LitIR<'a> {
 impl<'a> DataspecMeta1D for LitIR<'a> {
     type Target = NativeDword;
     type StringItem = &'a str;
-    fn new(flag: Dataflag, data: Self::Target) -> Self {
+    type Tag = FullTag;
+    fn new(flag: Self::Tag, data: Self::Target) -> Self {
         Self {
             tag: flag,
             data,
             _lt: PhantomData,
         }
     }
-    fn kind(&self) -> Dataflag {
+    fn kind(&self) -> Self::Tag {
         self.tag
     }
     fn data(&self) -> Self::Target {
@@ -227,7 +232,7 @@ unsafe impl<'a> DataspecRaw1D for LitIR<'a> {
 unsafe impl<'a> Dataspec1D for LitIR<'a> {
     fn Str(s: Self::StringItem) -> Self {
         Self::new(
-            Dataflag::Str,
+            FullTag::STR,
             SystemDword::store_fat(s.as_ptr() as usize, s.len()),
         )
     }
diff --git a/server/src/engine/data/macros.rs b/server/src/engine/data/macros.rs
index 70dc8cc9..99e552b6 100644
--- a/server/src/engine/data/macros.rs
+++ b/server/src/engine/data/macros.rs
@@ -27,22 +27,22 @@
 /// This is a pretty complex macro that emulates the behavior of an enumeration by making use of flags and macro hacks. You might literally feel it's like a lang match, but nope,
 /// there's a lot of wizardry beneath. Well, it's important to know that it works and you shouldn't touch it UNLESS YOU ABSOLUTELY KNOW what you're doing
 macro_rules! match_data {
-    (match ref $dataitem:ident $tail:tt) => {match_data!(@branch [ #[deny(unreachable_patterns)] match crate::engine::data::spec::DataspecMeta1D::kind($dataitem)] $dataitem [] $tail)};
-    (match $dataitem:ident $tail:tt) => {match_data!(@branch [ #[deny(unreachable_patterns)] match crate::engine::data::spec::DataspecMeta1D::kind(&$dataitem)] $dataitem [] $tail)};
+    (match ref $dataitem:ident $tail:tt) => {match_data!(@branch [ #[deny(unreachable_patterns)] match crate::engine::data::tag::DataTag::tag_class(&crate::engine::data::spec::DataspecMeta1D::kind($dataitem))] $dataitem [] $tail)};
+    (match $dataitem:ident $tail:tt) => {match_data!(@branch [ #[deny(unreachable_patterns)] match crate::engine::data::tag::DataTag::tag_class(&crate::engine::data::spec::DataspecMeta1D::kind(&$dataitem))] $dataitem [] $tail)};
     (@branch $decl:tt $dataitem:ident [$($branch:tt)*] {}) => {match_data!(@defeat0 $decl [$($branch)*])};
     (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $name:ident::$variant:ident($capture:ident) => $ret:expr, $($tail:tt)*}) => {
-        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::spec::Dataflag::$variant => {let $capture = unsafe { /* UNSAFE(@ohsayan): flagck */ match_data!(@extract $name $dataitem $variant) }; $ret},] {$($tail)*})
+        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::tag::TagClass::$variant => {let $capture = unsafe { /* UNSAFE(@ohsayan): flagck */ match_data!(@extract $name $dataitem $variant) }; $ret},] {$($tail)*})
     };
     (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $name:ident::$variant:ident(_) => $ret:expr, $($tail:tt)*}) => {
-        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::spec::Dataflag::$variant => $ret,] {$($tail)*})
+        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::tag::TagClass::$variant => $ret,] {$($tail)*})
     };
     (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $name:ident::$variant:ident($capture:ident) if $guard:expr => $ret:expr, $($tail:tt)*}) => {
-        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::spec::Dataflag::$variant if { let $capture = unsafe { /* UNSAFE(@ohsayan): flagck */ match_data!(@extract $name $dataitem $variant) }; $guard } => {
+        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::tag::TagClass::$variant if { let $capture = unsafe { /* UNSAFE(@ohsayan): flagck */ match_data!(@extract $name $dataitem $variant) }; $guard } => {
             let $capture = unsafe { /* UNSAFE(@ohsayan): flagck */  match_data!(@extract $name $dataitem $variant) }; let _ = &$capture; $ret}, ] {$($tail)*}
         )
     };
     (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $name:ident::$variant:ident(_) if $guard:expr => $ret:expr, $($tail:tt)*}) => {
-        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::spec::Dataflag::$variant if $guard => $ret,] {$($tail)*})
+        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::tag::TagClass::$variant if $guard => $ret,] {$($tail)*})
     };
     (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* _ => $ret:expr, $($tail:tt)*}) => {
         match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* _ => $ret,] {$($tail)*})
diff --git a/server/src/engine/data/mod.rs b/server/src/engine/data/mod.rs
index dd071c0c..8e81cd83 100644
--- a/server/src/engine/data/mod.rs
+++ b/server/src/engine/data/mod.rs
@@ -29,6 +29,7 @@ mod macros;
 pub mod lit;
 pub mod md_dict;
 pub mod spec;
+pub mod tag;
 #[cfg(test)]
 mod tests;
 
@@ -95,6 +96,7 @@ impl HSData {
             Lit::SignedInt(i) => HSData::SignedInt(i),
             Lit::Float(f) => HSData::Float(f),
             Lit::Bin(l) => HSData::Binary(l.to_vec().into_boxed_slice()),
+            TagClass::List(_) => unreachable!("found 2D data in 1D"),
         })
     }
     #[inline(always)]
@@ -106,6 +108,7 @@ impl HSData {
             LitIR::SignedInt(s) => Self::SignedInt(s),
             LitIR::UnsignedInt(u) => Self::UnsignedInt(u),
             LitIR::Bool(b) => Self::Boolean(b),
+            TagClass::List(_) => unreachable!("found 2D data in 1D"),
         })
     }
     fn kind(&self) -> Discriminant<Self> {
diff --git a/server/src/engine/data/spec.rs b/server/src/engine/data/spec.rs
index 1c7c3f69..9e2bbef2 100644
--- a/server/src/engine/data/spec.rs
+++ b/server/src/engine/data/spec.rs
@@ -30,6 +30,7 @@
 */
 
 use {
+    super::tag::{DataTag, TagClass},
     crate::engine::mem::SystemDword,
     core::{fmt, mem, slice},
 };
@@ -39,29 +40,19 @@ fn when_then<T>(cond: bool, then: T) -> Option<T> {
     cond.then_some(then)
 }
 
-#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
-/// The "reduced" data flag
-pub enum Dataflag {
-    Bool,
-    UnsignedInt,
-    SignedInt,
-    Float,
-    Bin,
-    Str,
-}
-
 /// Information about the type that implements the dataspec traits
 pub trait DataspecMeta1D: Sized {
     // assoc
+    type Tag: DataTag;
     /// The target must be able to store (atleast) a native dword
     type Target: SystemDword;
     /// The string item. This helps us remain correct with the dtors
     type StringItem;
     // fn
     /// Create a new instance. Usually allocates zero memory *directly*
-    fn new(flag: Dataflag, data: Self::Target) -> Self;
+    fn new(tag: Self::Tag, data: Self::Target) -> Self;
     /// Returns the reduced dataflag
-    fn kind(&self) -> Dataflag;
+    fn kind(&self) -> Self::Tag;
     /// Returns the data stack
     fn data(&self) -> Self::Target;
 }
@@ -95,28 +86,28 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     /// Store a new bool. This function is always safe to call
     #[allow(non_snake_case)]
     fn Bool(b: bool) -> Self {
-        Self::new(Dataflag::Bool, SystemDword::store_qw(b as _))
+        Self::new(Self::Tag::BOOL, SystemDword::store_qw(b as _))
     }
     /// Store a new uint. This function is always safe to call
     #[allow(non_snake_case)]
     fn UnsignedInt(u: u64) -> Self {
-        Self::new(Dataflag::UnsignedInt, SystemDword::store_qw(u as _))
+        Self::new(Self::Tag::UINT, SystemDword::store_qw(u as _))
     }
     /// Store a new sint. This function is always safe to call
     #[allow(non_snake_case)]
     fn SignedInt(s: i64) -> Self {
-        Self::new(Dataflag::SignedInt, SystemDword::store_qw(s as _))
+        Self::new(Self::Tag::SINT, SystemDword::store_qw(s as _))
     }
     /// Store a new float. This function is always safe to call
     #[allow(non_snake_case)]
     fn Float(f: f64) -> Self {
-        Self::new(Dataflag::Float, SystemDword::store_qw(f.to_bits()))
+        Self::new(Self::Tag::FLOAT, SystemDword::store_qw(f.to_bits()))
     }
     /// Store a new binary. This function is always safe to call
     #[allow(non_snake_case)]
     fn Bin(b: &[u8]) -> Self {
         Self::new(
-            Dataflag::Bin,
+            Self::Tag::BIN,
             SystemDword::store_fat(b.as_ptr() as usize, b.len()),
         )
     }
@@ -133,7 +124,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     }
     /// Load a bool
     fn read_bool_try(&self) -> Option<bool> {
-        when_then(self.kind() == Dataflag::Bool, unsafe {
+        when_then(self.kind().tag_class() == TagClass::Bool, unsafe {
             // UNSAFE(@ohsayan): we've verified the flag. but lol because this isn't actually unsafe
             self.read_bool_uck()
         })
@@ -151,7 +142,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     }
     /// Load a uint
     fn read_uint_try(&self) -> Option<u64> {
-        when_then(self.kind() == Dataflag::UnsignedInt, unsafe {
+        when_then(self.kind().tag_class() == TagClass::UnsignedInt, unsafe {
             // UNSAFE(@ohsayan): we've verified the flag. but lol because this isn't actually unsafe
             self.read_uint_uck()
         })
@@ -169,7 +160,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     }
     /// Load a sint
     fn read_sint_try(&self) -> Option<i64> {
-        when_then(self.kind() == Dataflag::SignedInt, unsafe {
+        when_then(self.kind().tag_class() == TagClass::SignedInt, unsafe {
             // UNSAFE(@ohsayan): we've verified the flag. but lol because this isn't actually unsafe
             self.read_sint_uck()
         })
@@ -185,7 +176,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     }
     /// Load a float
     fn read_float_try(&self) -> Option<f64> {
-        when_then(self.kind() == Dataflag::Float, unsafe {
+        when_then(self.kind().tag_class() == TagClass::Float, unsafe {
             self.read_float_uck()
         })
     }
@@ -204,7 +195,9 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     }
     /// Load a bin
     fn read_bin_try(&self) -> Option<&[u8]> {
-        when_then(self.kind() == Dataflag::Bin, unsafe { self.read_bin_uck() })
+        when_then(self.kind().tag_class() == TagClass::Bin, unsafe {
+            self.read_bin_uck()
+        })
     }
     /// Load a bin or panic if we aren't one
     fn bin(&self) -> &[u8] {
@@ -220,7 +213,9 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     }
     /// Load a str
     fn read_str_try(&self) -> Option<&str> {
-        when_then(self.kind() == Dataflag::Str, unsafe { self.read_str_uck() })
+        when_then(self.kind().tag_class() == TagClass::Str, unsafe {
+            self.read_str_uck()
+        })
     }
     /// Load a str and panic if we aren't one
     fn str(&self) -> &str {
@@ -234,12 +229,12 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
 /// - You are not touching your target
 pub unsafe trait DataspecMethods1D: Dataspec1D {
     fn self_drop(&mut self) {
-        match self.kind() {
-            Dataflag::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
+        match self.kind().tag_class() {
+            TagClass::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
                 // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
                 <Self as DataspecRaw1D>::drop_str(self)
             },
-            Dataflag::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
+            TagClass::Bin if <Self as DataspecRaw1D>::HEAP_BIN => unsafe {
                 // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
                 <Self as DataspecRaw1D>::drop_bin(self)
             },
@@ -247,12 +242,12 @@ pub unsafe trait DataspecMethods1D: Dataspec1D {
         }
     }
     fn self_clone(&self) -> Self {
-        let data = match self.kind() {
-            Dataflag::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
+        let data = match self.kind().tag_class() {
+            TagClass::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
                 // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
                 <Self as DataspecRaw1D>::clone_str(Dataspec1D::read_str_uck(self))
             },
-            Dataflag::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
+            TagClass::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
                 // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
                 <Self as DataspecRaw1D>::clone_bin(Dataspec1D::read_bin_uck(self))
             },
@@ -263,19 +258,19 @@ pub unsafe trait DataspecMethods1D: Dataspec1D {
     fn self_eq(&self, other: &impl DataspecMethods1D) -> bool {
         unsafe {
             // UNSAFE(@ohsayan): we are checking our flags
-            match (self.kind(), other.kind()) {
-                (Dataflag::Bool, Dataflag::Bool) => self.read_bool_uck() == other.read_bool_uck(),
-                (Dataflag::UnsignedInt, Dataflag::UnsignedInt) => {
+            match (self.kind().tag_class(), other.kind().tag_class()) {
+                (TagClass::Bool, TagClass::Bool) => self.read_bool_uck() == other.read_bool_uck(),
+                (TagClass::UnsignedInt, TagClass::UnsignedInt) => {
                     self.read_uint_uck() == other.read_uint_uck()
                 }
-                (Dataflag::SignedInt, Dataflag::SignedInt) => {
+                (TagClass::SignedInt, TagClass::SignedInt) => {
                     self.read_sint_uck() == other.read_sint_uck()
                 }
-                (Dataflag::Float, Dataflag::Float) => {
+                (TagClass::Float, TagClass::Float) => {
                     self.read_float_uck() == other.read_float_uck()
                 }
-                (Dataflag::Bin, Dataflag::Bin) => self.read_bin_uck() == other.read_bin_uck(),
-                (Dataflag::Str, Dataflag::Str) => self.read_str_uck() == other.read_str_uck(),
+                (TagClass::Bin, TagClass::Bin) => self.read_bin_uck() == other.read_bin_uck(),
+                (TagClass::Str, TagClass::Str) => self.read_str_uck() == other.read_str_uck(),
                 _ => false,
             }
         }
@@ -283,18 +278,20 @@ pub unsafe trait DataspecMethods1D: Dataspec1D {
     fn self_fmt_debug_data(&self, data_field: &str, f: &mut fmt::DebugStruct) {
         macro_rules! fmtdebug {
             ($($(#[$attr:meta])* $match:pat => $ret:expr),* $(,)?) => {
-                match self.kind() {$($(#[$attr])* $match => { let x = $ret; f.field(data_field, &x) },)*}
+                match self.kind().tag_class() {$($(#[$attr])* $match => { let _x = $ret; f.field(data_field, &_x) },)*}
             }
         }
         unsafe {
             // UNSAFE(@ohsayan): we are checking our flags
             fmtdebug!(
-                Dataflag::Bool => self.read_bool_uck(),
-                Dataflag::UnsignedInt => self.read_uint_uck(),
-                Dataflag::SignedInt => self.read_sint_uck(),
-                Dataflag::Float => self.read_float_uck(),
-                Dataflag::Bin => self.read_bin_uck(),
-                Dataflag::Str => self.read_str_uck(),
+                TagClass::Bool => self.read_bool_uck(),
+                TagClass::UnsignedInt => self.read_uint_uck(),
+                TagClass::SignedInt => self.read_sint_uck(),
+                TagClass::Float => self.read_float_uck(),
+                TagClass::Bin => self.read_bin_uck(),
+                TagClass::Str => self.read_str_uck(),
+                #[allow(unreachable_code)]
+                TagClass::List => unreachable!("found 2D data in 1D"),
             )
         };
     }
@@ -307,6 +304,7 @@ pub unsafe trait DataspecMethods1D: Dataspec1D {
             Self::Float(f) => f.to_string(),
             Self::Bin(b) => format!("{:?}", b),
             Self::Str(s) => format!("{:?}", s),
+            Self::List(_) => unreachable!("found 2D data in 1D"),
         })
     }
 }
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
new file mode 100644
index 00000000..a68d21ab
--- /dev/null
+++ b/server/src/engine/data/tag.rs
@@ -0,0 +1,130 @@
+/*
+ * Created on Mon Feb 27 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[repr(u8)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord)]
+pub enum TagClass {
+    Bool = 0,
+    UnsignedInt = 1,
+    SignedInt = 2,
+    Float = 3,
+    Bin = 4,
+    Str = 5,
+    List = 6,
+}
+
+#[repr(u8)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord)]
+pub enum TagSelector {
+    Bool = 0,
+    UInt8 = 1,
+    UInt16 = 2,
+    UInt32 = 3,
+    UInt64 = 4,
+    SInt8 = 5,
+    SInt16 = 6,
+    SInt32 = 7,
+    SInt64 = 8,
+    Float32 = 9,
+    Float64 = 10,
+    Bin = 11,
+    Str = 12,
+    List = 13,
+}
+
+#[repr(u8)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord)]
+pub enum TagUnique {
+    UnsignedInt = 0,
+    SignedInt = 1,
+    Bin = 2,
+    Str = 3,
+    Illegal = 0xFF,
+}
+
+macro_rules! d {
+    ($($ty:ty),*) => {$(impl $ty { fn d(&self) -> u8 { unsafe { ::core::mem::transmute_copy(self) } } } )*}
+}
+
+d!(TagClass, TagSelector, TagUnique);
+
+pub trait DataTag {
+    const BOOL: Self;
+    const UINT: Self;
+    const SINT: Self;
+    const FLOAT: Self;
+    const BIN: Self;
+    const STR: Self;
+    const LIST: Self;
+    fn tag_class(&self) -> TagClass;
+    fn tag_selector(&self) -> TagSelector;
+    fn tag_unique(&self) -> TagUnique;
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
+pub struct FullTag {
+    class: TagClass,
+    selector: TagSelector,
+    unique: TagUnique,
+}
+
+impl FullTag {
+    const fn new(class: TagClass, selector: TagSelector, unique: TagUnique) -> Self {
+        Self {
+            class,
+            selector,
+            unique,
+        }
+    }
+}
+
+macro_rules! fulltag {
+    ($class:ident, $selector:ident, $unique:ident) => {
+        FullTag::new(TagClass::$class, TagSelector::$selector, TagUnique::$unique)
+    };
+    ($class:ident, $selector:ident) => {
+        FullTag::new(TagClass::$class, TagSelector::$selector, TagUnique::Illegal)
+    };
+}
+
+impl DataTag for FullTag {
+    const BOOL: Self = fulltag!(Bool, Bool);
+    const UINT: Self = fulltag!(UnsignedInt, UInt64, UnsignedInt);
+    const SINT: Self = fulltag!(SignedInt, SInt64, SignedInt);
+    const FLOAT: Self = fulltag!(Float, Float64);
+    const BIN: Self = fulltag!(Bin, Bin, Bin);
+    const STR: Self = fulltag!(Str, Str, Str);
+    const LIST: Self = fulltag!(List, List);
+    fn tag_class(&self) -> TagClass {
+        self.class
+    }
+    fn tag_selector(&self) -> TagSelector {
+        self.selector
+    }
+    fn tag_unique(&self) -> TagUnique {
+        self.unique
+    }
+}

From 9d9ffeed1a678686193faa7ea8927bd33736c072 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 28 Feb 2023 01:43:03 -0800
Subject: [PATCH 143/310] Skip expensive ql tests on miri [skip ci]

---
 server/src/engine/ql/tests/schema_tests.rs  | 1 +
 server/src/engine/ql/tests/structure_syn.rs | 1 +
 2 files changed, 2 insertions(+)

diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index a701ca36..943503e7 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -273,6 +273,7 @@ mod layer {
     }
 
     #[test]
+    #[cfg(not(miri))]
     fn fuzz_layer() {
         let tok = b"
             list {
diff --git a/server/src/engine/ql/tests/structure_syn.rs b/server/src/engine/ql/tests/structure_syn.rs
index 08ff240e..3a17af63 100644
--- a/server/src/engine/ql/tests/structure_syn.rs
+++ b/server/src/engine/ql/tests/structure_syn.rs
@@ -188,6 +188,7 @@ mod dict {
     }
 
     #[test]
+    #[cfg(not(miri))]
     fn fuzz_dict() {
         let tok = b"
             {

From 3b8a251a01035d8b02a4e57400757b916231017d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 28 Feb 2023 09:18:01 -0800
Subject: [PATCH 144/310] Define larger words

---
 server/src/engine/mem/mod.rs | 70 +++++++++++++++++++++++++++++++++---
 1 file changed, 65 insertions(+), 5 deletions(-)

diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index e7b22589..842e9601 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -38,6 +38,14 @@ pub use vinline::VInline;
 pub struct NativeDword([usize; 2]);
 /// Native triple pointer width (note, native != arch native, but host native)
 pub struct NativeTword([usize; 3]);
+/// Native quad pointer width (note, native != arch native, but host native)
+pub struct NativeQword([usize; 4]);
+
+/// Native quad pointer stack (must also be usable as a double and triple pointer stack. see [`SystemTword`] and [`SystemDword`])
+pub trait SystemQword: SystemTword {
+    fn store_full(a: usize, b: usize, c: usize, d: usize) -> Self;
+    fn load_full(&self) -> [usize; 4];
+}
 
 /// Native tripe pointer stack (must also be usable as a double pointer stack, see [`SystemDword`])
 pub trait SystemTword: SystemDword {
@@ -90,6 +98,17 @@ impl SystemDword for NativeDword {
     }
 }
 
+impl SystemTword for NativeTword {
+    #[inline(always)]
+    fn store_full(a: usize, b: usize, c: usize) -> Self {
+        Self([a, b, c])
+    }
+    #[inline(always)]
+    fn load_full(&self) -> [usize; 3] {
+        self.0
+    }
+}
+
 impl SystemDword for NativeTword {
     #[inline(always)]
     fn store_qw(u: u64) -> Self {
@@ -129,13 +148,54 @@ impl SystemDword for NativeTword {
     }
 }
 
-impl SystemTword for NativeTword {
-    #[inline(always)]
+impl SystemQword for NativeQword {
+    fn store_full(a: usize, b: usize, c: usize, d: usize) -> Self {
+        Self([a, b, c, d])
+    }
+    fn load_full(&self) -> [usize; 4] {
+        self.0
+    }
+}
+
+impl SystemTword for NativeQword {
     fn store_full(a: usize, b: usize, c: usize) -> Self {
-        Self([a, b, c])
+        Self([a, b, c, 0])
     }
-    #[inline(always)]
     fn load_full(&self) -> [usize; 3] {
-        self.0
+        [self.0[0], self.0[1], self.0[2]]
+    }
+}
+
+impl SystemDword for NativeQword {
+    fn store_qw(u: u64) -> Self {
+        let ret;
+        #[cfg(target_pointer_width = "32")]
+        {
+            let [a, b]: [usize; 2] = unsafe { core::mem::transmute(u) };
+            ret = <Self as SystemQword>::store_full(a, b, 0, 0);
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            ret = <Self as SystemQword>::store_full(u as _, 0, 0, 0);
+        }
+        ret
+    }
+    fn store_fat(a: usize, b: usize) -> Self {
+        <Self as SystemQword>::store_full(a, b, 0, 0)
+    }
+    fn load_qw(&self) -> u64 {
+        let ret;
+        #[cfg(target_pointer_width = "32")]
+        {
+            ret = unsafe { core::mem::transmute([self.0[0], self.0[1]]) };
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            ret = self.0[0] as _;
+        }
+        ret
+    }
+    fn load_fat(&self) -> [usize; 2] {
+        [self.0[0], self.0[1]]
     }
 }

From 5d9851a427d2134853c5c6e3c2ed44fc690a9bd9 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 1 Mar 2023 01:37:51 -0800
Subject: [PATCH 145/310] Add model cell definition and more word impls

---
 server/src/engine/core/model/cell.rs          | 298 +++++++++++++++++
 .../engine/core/{model.rs => model/mod.rs}    |   2 +
 server/src/engine/data/lit.rs                 |  22 +-
 server/src/engine/data/md_dict.rs             |   4 +-
 server/src/engine/data/mod.rs                 |   5 +-
 server/src/engine/data/spec.rs                |  25 +-
 server/src/engine/macros.rs                   |   2 +-
 server/src/engine/mem/mod.rs                  | 174 +---------
 server/src/engine/mem/word.rs                 | 302 ++++++++++++++++++
 server/src/engine/ql/lex/raw.rs               |   2 +-
 10 files changed, 640 insertions(+), 196 deletions(-)
 create mode 100644 server/src/engine/core/model/cell.rs
 rename server/src/engine/core/{model.rs => model/mod.rs} (99%)
 create mode 100644 server/src/engine/mem/word.rs

diff --git a/server/src/engine/core/model/cell.rs b/server/src/engine/core/model/cell.rs
new file mode 100644
index 00000000..1dc6f980
--- /dev/null
+++ b/server/src/engine/core/model/cell.rs
@@ -0,0 +1,298 @@
+/*
+ * Created on Tue Feb 28 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[cfg(test)]
+use core::mem;
+use {
+    crate::engine::{
+        self,
+        data::tag::TagClass,
+        mem::{NativeQword, SystemDword},
+    },
+    core::{fmt, mem::ManuallyDrop, slice, str},
+    parking_lot::RwLock,
+};
+
+pub struct Datacell {
+    tag: TagClass,
+    data: DataRaw,
+}
+
+impl Datacell {
+    // bool
+    pub fn new_bool(b: bool) -> Self {
+        unsafe { Self::new(TagClass::Bool, DataRaw::word(SystemDword::store_qw(b as _))) }
+    }
+    pub unsafe fn read_bool(&self) -> bool {
+        self.data.word.load_qw() == 1
+    }
+    pub fn try_bool(&self) -> Option<bool> {
+        self.checked_tag(TagClass::Bool, || unsafe { self.read_bool() })
+    }
+    pub fn bool(&self) -> bool {
+        self.try_bool().unwrap()
+    }
+    // uint
+    pub fn new_uint(u: u64) -> Self {
+        unsafe {
+            Self::new(
+                TagClass::UnsignedInt,
+                DataRaw::word(SystemDword::store_qw(u)),
+            )
+        }
+    }
+    pub unsafe fn read_uint(&self) -> u64 {
+        self.data.word.load_qw()
+    }
+    pub fn try_uint(&self) -> Option<u64> {
+        self.checked_tag(TagClass::UnsignedInt, || unsafe { self.read_uint() })
+    }
+    pub fn uint(&self) -> u64 {
+        self.try_uint().unwrap()
+    }
+    // sint
+    pub fn new_sint(u: i64) -> Self {
+        unsafe {
+            Self::new(
+                TagClass::SignedInt,
+                DataRaw::word(SystemDword::store_qw(u as _)),
+            )
+        }
+    }
+    pub unsafe fn read_sint(&self) -> i64 {
+        self.data.word.load_qw() as _
+    }
+    pub fn try_sint(&self) -> Option<i64> {
+        self.checked_tag(TagClass::SignedInt, || unsafe { self.read_sint() })
+    }
+    pub fn sint(&self) -> i64 {
+        self.try_sint().unwrap()
+    }
+    // float
+    pub fn new_float(f: f64) -> Self {
+        unsafe {
+            Self::new(
+                TagClass::Float,
+                DataRaw::word(SystemDword::store_qw(f.to_bits())),
+            )
+        }
+    }
+    pub unsafe fn read_float(&self) -> f64 {
+        f64::from_bits(self.data.word.load_qw())
+    }
+    pub fn try_float(&self) -> Option<f64> {
+        self.checked_tag(TagClass::Float, || unsafe { self.read_float() })
+    }
+    pub fn float(&self) -> f64 {
+        self.try_float().unwrap()
+    }
+    // bin
+    pub fn new_bin(s: Box<[u8]>) -> Self {
+        let md = ManuallyDrop::new(s);
+        unsafe {
+            Self::new(
+                TagClass::Bin,
+                DataRaw::word(SystemDword::store_fat(md.as_ptr() as usize, md.len())),
+            )
+        }
+    }
+    pub unsafe fn read_bin(&self) -> &[u8] {
+        let [p, l] = self.data.word.load_fat();
+        slice::from_raw_parts(p as *mut u8, l)
+    }
+    pub fn try_bin(&self) -> Option<&[u8]> {
+        self.checked_tag(TagClass::Bin, || unsafe { self.read_bin() })
+    }
+    pub fn bin(&self) -> &[u8] {
+        self.try_bin().unwrap()
+    }
+    // str
+    pub fn new_str(s: Box<str>) -> Self {
+        let md = ManuallyDrop::new(s.into_boxed_bytes());
+        unsafe {
+            Self::new(
+                TagClass::Str,
+                DataRaw::word(SystemDword::store_fat(md.as_ptr() as usize, md.len())),
+            )
+        }
+    }
+    pub unsafe fn read_str(&self) -> &str {
+        let [p, l] = self.data.word.load_fat();
+        str::from_utf8_unchecked(slice::from_raw_parts(p as *mut u8, l))
+    }
+    pub fn try_str(&self) -> Option<&str> {
+        self.checked_tag(TagClass::Str, || unsafe { self.read_str() })
+    }
+    pub fn str(&self) -> &str {
+        self.try_str().unwrap()
+    }
+    // list
+    pub fn new_list(l: Vec<Self>) -> Self {
+        unsafe { Self::new(TagClass::List, DataRaw::rwl(RwLock::new(l))) }
+    }
+    pub unsafe fn read_list(&self) -> &RwLock<Vec<Self>> {
+        &self.data.rwl
+    }
+    pub fn try_list(&self) -> Option<&RwLock<Vec<Self>>> {
+        self.checked_tag(TagClass::List, || unsafe { self.read_list() })
+    }
+    pub fn list(&self) -> &RwLock<Vec<Self>> {
+        self.try_list().unwrap()
+    }
+}
+
+direct_from! {
+    Datacell => {
+        bool as new_bool,
+        u64 as new_uint,
+        i64 as new_sint,
+        f64 as new_float,
+        f32 as new_float,
+        Vec<u8> as new_bin,
+        Box<[u8]> as new_bin,
+        &'static [u8] as new_bin,
+        String as new_str,
+        Box<str> as new_str,
+        &'static str as new_str,
+        Vec<Self> as new_list,
+        Box<[Self]> as new_list,
+    }
+}
+
+impl Datacell {
+    unsafe fn new(tag: TagClass, data: DataRaw) -> Self {
+        Self { tag, data }
+    }
+    fn checked_tag<T>(&self, tag: TagClass, f: impl FnOnce() -> T) -> Option<T> {
+        (self.tag == tag).then_some(f())
+    }
+}
+
+impl fmt::Debug for Datacell {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut f = f.debug_struct("Datacell");
+        f.field("tag", &self.tag);
+        macro_rules! fmtdbg {
+            ($($match:ident => $ret:expr),* $(,)?) => {
+                match self.tag {
+                    $(TagClass::$match => f.field("data", &$ret),)*
+                }
+            }
+        }
+        fmtdbg!(
+            Bool => self.bool(),
+            UnsignedInt => self.uint(),
+            SignedInt => self.sint(),
+            Float => self.float(),
+            Bin => self.bin(),
+            Str => self.str(),
+            List => self.list(),
+        );
+        f.finish()
+    }
+}
+
+impl PartialEq for Datacell {
+    fn eq(&self, other: &Datacell) -> bool {
+        match (self.tag, other.tag) {
+            (TagClass::Bool, TagClass::Bool) => self.bool() == other.bool(),
+            (TagClass::UnsignedInt, TagClass::UnsignedInt) => self.uint() == other.uint(),
+            (TagClass::SignedInt, TagClass::SignedInt) => self.sint() == other.sint(),
+            (TagClass::Float, TagClass::Float) => self.float() == other.float(),
+            (TagClass::Bin, TagClass::Bin) => self.bin() == other.bin(),
+            (TagClass::Str, TagClass::Str) => self.str() == other.str(),
+            (TagClass::List, TagClass::List) => {
+                let l1_l = self.list().read();
+                let l2_l = other.list().read();
+                let l1: &[Self] = l1_l.as_ref();
+                let l2: &[Self] = l2_l.as_ref();
+                l1 == l2
+            }
+            _ => false,
+        }
+    }
+}
+
+impl Eq for Datacell {}
+
+union! {
+    union DataRaw {
+        !word: NativeQword,
+        !rwl: RwLock<Vec<Datacell>>,
+    }
+}
+
+impl DataRaw {
+    fn word(word: NativeQword) -> Self {
+        Self {
+            word: ManuallyDrop::new(word),
+        }
+    }
+    fn rwl(rwl: RwLock<Vec<Datacell>>) -> Self {
+        Self {
+            rwl: ManuallyDrop::new(rwl),
+        }
+    }
+}
+
+impl Drop for Datacell {
+    fn drop(&mut self) {
+        match self.tag {
+            TagClass::Str | TagClass::Bin => unsafe {
+                let [p, l] = self.data.word.load_fat();
+                engine::mem::dealloc_array(p as *mut u8, l)
+            },
+            TagClass::List => unsafe { ManuallyDrop::drop(&mut self.data.rwl) },
+            _ => {}
+        }
+    }
+}
+
+#[cfg(test)]
+impl Clone for Datacell {
+    fn clone(&self) -> Self {
+        let data = match self.tag {
+            TagClass::Str | TagClass::Bin => unsafe {
+                let block = ManuallyDrop::new(self.read_bin().to_owned().into_boxed_slice());
+                DataRaw {
+                    word: ManuallyDrop::new(SystemDword::store((block.as_ptr(), block.len()))),
+                }
+            },
+            TagClass::List => unsafe {
+                let data = self.read_list().read().iter().cloned().collect();
+                DataRaw {
+                    rwl: ManuallyDrop::new(RwLock::new(data)),
+                }
+            },
+            _ => unsafe {
+                DataRaw {
+                    word: ManuallyDrop::new(mem::transmute_copy(&self.data.word)),
+                }
+            },
+        };
+        unsafe { Self::new(self.tag, data) }
+    }
+}
diff --git a/server/src/engine/core/model.rs b/server/src/engine/core/model/mod.rs
similarity index 99%
rename from server/src/engine/core/model.rs
rename to server/src/engine/core/model/mod.rs
index d1d31acf..c64cae53 100644
--- a/server/src/engine/core/model.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -24,6 +24,8 @@
  *
 */
 
+mod cell;
+
 // FIXME(@ohsayan): update this!
 
 #[derive(Debug)]
diff --git a/server/src/engine/data/lit.rs b/server/src/engine/data/lit.rs
index 011e6a7b..6de39129 100644
--- a/server/src/engine/data/lit.rs
+++ b/server/src/engine/data/lit.rs
@@ -97,10 +97,10 @@ unsafe impl<'a> DataspecRaw1D for Lit<'a> {
     unsafe fn drop_bin(&mut self) {}
     unsafe fn clone_str(s: &str) -> Self::Target {
         let new_string = ManuallyDrop::new(s.to_owned().into_boxed_str());
-        SystemDword::store_fat(new_string.as_ptr() as usize, new_string.len())
+        SystemDword::store((new_string.as_ptr(), new_string.len()))
     }
     unsafe fn clone_bin(b: &[u8]) -> Self::Target {
-        SystemDword::store_fat(b.as_ptr() as usize, b.len())
+        SystemDword::store((b.as_ptr(), b.len()))
     }
 }
 
@@ -112,10 +112,7 @@ unsafe impl<'a> DataspecRaw1D for Lit<'a> {
 unsafe impl<'a> Dataspec1D for Lit<'a> {
     fn Str(s: Box<str>) -> Self {
         let md = ManuallyDrop::new(s);
-        Self::new(
-            FullTag::STR,
-            SystemDword::store_fat(md.as_ptr() as _, md.len()),
-        )
+        Self::new(FullTag::STR, SystemDword::store((md.as_ptr(), md.len())))
     }
 }
 
@@ -158,7 +155,7 @@ impl<'a> ToString for Lit<'a> {
     }
 }
 
-enum_impls! {
+direct_from! {
     Lit<'a> => {
         bool as Bool,
         u64 as UnsignedInt,
@@ -218,10 +215,10 @@ unsafe impl<'a> DataspecRaw1D for LitIR<'a> {
     unsafe fn drop_str(&mut self) {}
     unsafe fn drop_bin(&mut self) {}
     unsafe fn clone_str(s: &str) -> Self::Target {
-        SystemDword::store_fat(s.as_ptr() as usize, s.len())
+        SystemDword::store((s.as_ptr(), s.len()))
     }
     unsafe fn clone_bin(b: &[u8]) -> Self::Target {
-        SystemDword::store_fat(b.as_ptr() as usize, b.len())
+        SystemDword::store((b.as_ptr(), b.len()))
     }
 }
 
@@ -231,10 +228,7 @@ unsafe impl<'a> DataspecRaw1D for LitIR<'a> {
 */
 unsafe impl<'a> Dataspec1D for LitIR<'a> {
     fn Str(s: Self::StringItem) -> Self {
-        Self::new(
-            FullTag::STR,
-            SystemDword::store_fat(s.as_ptr() as usize, s.len()),
-        )
+        Self::new(FullTag::STR, SystemDword::store((s.as_ptr(), s.len())))
     }
 }
 
@@ -277,7 +271,7 @@ impl<'a> Clone for LitIR<'a> {
     }
 }
 
-enum_impls! {
+direct_from! {
     LitIR<'a> => {
         bool as Bool,
         u64 as UnsignedInt,
diff --git a/server/src/engine/data/md_dict.rs b/server/src/engine/data/md_dict.rs
index bf0ff407..34552352 100644
--- a/server/src/engine/data/md_dict.rs
+++ b/server/src/engine/data/md_dict.rs
@@ -218,14 +218,14 @@ impl<'a> From<Lit<'a>> for DictEntryGeneric {
     }
 }
 
-enum_impls! {
+direct_from! {
     DictEntryGeneric => {
         HSData as Lit,
         DictGeneric as Map,
     }
 }
 
-enum_impls! {
+direct_from! {
     MetaDictEntry => {
         HSData as Data,
         MetaDict as Map,
diff --git a/server/src/engine/data/mod.rs b/server/src/engine/data/mod.rs
index 8e81cd83..2086e345 100644
--- a/server/src/engine/data/mod.rs
+++ b/server/src/engine/data/mod.rs
@@ -46,7 +46,8 @@ pub type ItemID = AStr<IDENT_MX>;
 /// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
 /// be of one type.
 // TODO(@ohsayan): Change the underlying structures, there are just rudimentary ones used during integration with the QL
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq)]
+#[cfg_attr(test, derive(Clone))]
 #[repr(u8)]
 pub enum HSData {
     /// An UTF-8 string
@@ -75,7 +76,7 @@ pub enum HSData {
     List(Vec<Self>) = DataKind::LIST.d(),
 }
 
-enum_impls! {
+direct_from! {
     HSData => {
         String as String,
         Vec<u8> as Binary,
diff --git a/server/src/engine/data/spec.rs b/server/src/engine/data/spec.rs
index 9e2bbef2..94202565 100644
--- a/server/src/engine/data/spec.rs
+++ b/server/src/engine/data/spec.rs
@@ -86,30 +86,27 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     /// Store a new bool. This function is always safe to call
     #[allow(non_snake_case)]
     fn Bool(b: bool) -> Self {
-        Self::new(Self::Tag::BOOL, SystemDword::store_qw(b as _))
+        Self::new(Self::Tag::BOOL, SystemDword::store(b))
     }
     /// Store a new uint. This function is always safe to call
     #[allow(non_snake_case)]
     fn UnsignedInt(u: u64) -> Self {
-        Self::new(Self::Tag::UINT, SystemDword::store_qw(u as _))
+        Self::new(Self::Tag::UINT, SystemDword::store(u))
     }
     /// Store a new sint. This function is always safe to call
     #[allow(non_snake_case)]
     fn SignedInt(s: i64) -> Self {
-        Self::new(Self::Tag::SINT, SystemDword::store_qw(s as _))
+        Self::new(Self::Tag::SINT, SystemDword::store(s))
     }
     /// Store a new float. This function is always safe to call
     #[allow(non_snake_case)]
     fn Float(f: f64) -> Self {
-        Self::new(Self::Tag::FLOAT, SystemDword::store_qw(f.to_bits()))
+        Self::new(Self::Tag::FLOAT, SystemDword::store(f.to_bits()))
     }
     /// Store a new binary. This function is always safe to call
     #[allow(non_snake_case)]
     fn Bin(b: &[u8]) -> Self {
-        Self::new(
-            Self::Tag::BIN,
-            SystemDword::store_fat(b.as_ptr() as usize, b.len()),
-        )
+        Self::new(Self::Tag::BIN, SystemDword::store((b.as_ptr(), b.len())))
     }
 
     /// Store a new string. Now, I won't talk about this one's safety because it depends on the implementor
@@ -120,7 +117,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // bool
     /// Load a bool (this is unsafe for logical verity)
     unsafe fn read_bool_uck(&self) -> bool {
-        self.data().load_qw() == 1
+        self.data().load::<bool>()
     }
     /// Load a bool
     fn read_bool_try(&self) -> Option<bool> {
@@ -138,7 +135,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // uint
     /// Load a uint (this is unsafe for logical verity)
     unsafe fn read_uint_uck(&self) -> u64 {
-        self.data().load_qw()
+        self.data().load::<u64>()
     }
     /// Load a uint
     fn read_uint_try(&self) -> Option<u64> {
@@ -156,7 +153,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // sint
     /// Load a sint (unsafe for logical verity)
     unsafe fn read_sint_uck(&self) -> i64 {
-        self.data().load_qw() as _
+        self.data().load::<i64>()
     }
     /// Load a sint
     fn read_sint_try(&self) -> Option<i64> {
@@ -172,7 +169,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // float
     /// Load a float (unsafe for logical verity)
     unsafe fn read_float_uck(&self) -> f64 {
-        self.data().load_qw() as _
+        self.data().load::<f64>()
     }
     /// Load a float
     fn read_float_try(&self) -> Option<f64> {
@@ -190,8 +187,8 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     /// ## Safety
     /// Are you a binary? Did you store it correctly? Are you a victim of segfaults?
     unsafe fn read_bin_uck(&self) -> &[u8] {
-        let [p, l] = self.data().load_fat();
-        slice::from_raw_parts(p as *const u8, l)
+        let (p, l) = self.data().load::<(*const u8, usize)>();
+        slice::from_raw_parts(p, l)
     }
     /// Load a bin
     fn read_bin_try(&self) -> Option<&[u8]> {
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index 412b54cb..4fd251ea 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -52,7 +52,7 @@ macro_rules! multi_assert_eq {
     };
 }
 
-macro_rules! enum_impls {
+macro_rules! direct_from {
     ($for:ident<$lt:lifetime> => {$($other:ty as $me:ident),*$(,)?}) => {
         $(impl<$lt> ::core::convert::From<$other> for $for<$lt> {fn from(v: $other) -> Self {Self::$me(v.into())}})*
     };
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index 842e9601..9190cdda 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -25,14 +25,23 @@
 */
 
 mod astr;
-#[cfg(test)]
-mod tests;
 mod uarray;
 mod vinline;
-
+mod word;
+// test
+#[cfg(test)]
+mod tests;
+// re-exports
 pub use astr::AStr;
 pub use uarray::UArray;
 pub use vinline::VInline;
+pub use word::{SystemDword, SystemQword, SystemTword, WordRW};
+// imports
+use std::alloc::{self, Layout};
+
+pub unsafe fn dealloc_array<T>(ptr: *mut T, l: usize) {
+    alloc::dealloc(ptr as *mut u8, Layout::array::<T>(l).unwrap_unchecked())
+}
 
 /// Native double pointer width (note, native != arch native, but host native)
 pub struct NativeDword([usize; 2]);
@@ -40,162 +49,3 @@ pub struct NativeDword([usize; 2]);
 pub struct NativeTword([usize; 3]);
 /// Native quad pointer width (note, native != arch native, but host native)
 pub struct NativeQword([usize; 4]);
-
-/// Native quad pointer stack (must also be usable as a double and triple pointer stack. see [`SystemTword`] and [`SystemDword`])
-pub trait SystemQword: SystemTword {
-    fn store_full(a: usize, b: usize, c: usize, d: usize) -> Self;
-    fn load_full(&self) -> [usize; 4];
-}
-
-/// Native tripe pointer stack (must also be usable as a double pointer stack, see [`SystemDword`])
-pub trait SystemTword: SystemDword {
-    fn store_full(a: usize, b: usize, c: usize) -> Self;
-    fn load_full(&self) -> [usize; 3];
-}
-
-/// Native double pointer stack
-pub trait SystemDword {
-    fn store_qw(u: u64) -> Self;
-    fn store_fat(a: usize, b: usize) -> Self;
-    fn load_qw(&self) -> u64;
-    fn load_fat(&self) -> [usize; 2];
-}
-
-impl SystemDword for NativeDword {
-    #[inline(always)]
-    fn store_qw(u: u64) -> Self {
-        let x;
-        #[cfg(target_pointer_width = "32")]
-        {
-            x = unsafe { core::mem::transmute(u) };
-        }
-        #[cfg(target_pointer_width = "64")]
-        {
-            x = [u as usize, 0]
-        }
-        Self(x)
-    }
-    #[inline(always)]
-    fn store_fat(a: usize, b: usize) -> Self {
-        Self([a, b])
-    }
-    #[inline(always)]
-    fn load_qw(&self) -> u64 {
-        let x;
-        #[cfg(target_pointer_width = "32")]
-        {
-            x = unsafe { core::mem::transmute_copy(self) }
-        }
-        #[cfg(target_pointer_width = "64")]
-        {
-            x = self.0[0] as _;
-        }
-        x
-    }
-    #[inline(always)]
-    fn load_fat(&self) -> [usize; 2] {
-        self.0
-    }
-}
-
-impl SystemTword for NativeTword {
-    #[inline(always)]
-    fn store_full(a: usize, b: usize, c: usize) -> Self {
-        Self([a, b, c])
-    }
-    #[inline(always)]
-    fn load_full(&self) -> [usize; 3] {
-        self.0
-    }
-}
-
-impl SystemDword for NativeTword {
-    #[inline(always)]
-    fn store_qw(u: u64) -> Self {
-        let x;
-        #[cfg(target_pointer_width = "32")]
-        {
-            let [a, b]: [usize; 2] = unsafe { core::mem::transmute(u) };
-            x = [a, b, 0];
-        }
-        #[cfg(target_pointer_width = "64")]
-        {
-            x = [u as _, 0, 0];
-        }
-        Self(x)
-    }
-    #[inline(always)]
-    fn store_fat(a: usize, b: usize) -> Self {
-        Self([a, b, 0])
-    }
-    #[inline(always)]
-    fn load_qw(&self) -> u64 {
-        let x;
-        #[cfg(target_pointer_width = "32")]
-        {
-            let ab = [self.0[0], self.0[1]];
-            x = unsafe { core::mem::transmute(ab) };
-        }
-        #[cfg(target_pointer_width = "64")]
-        {
-            x = self.0[0] as _;
-        }
-        x
-    }
-    #[inline(always)]
-    fn load_fat(&self) -> [usize; 2] {
-        [self.0[0], self.0[1]]
-    }
-}
-
-impl SystemQword for NativeQword {
-    fn store_full(a: usize, b: usize, c: usize, d: usize) -> Self {
-        Self([a, b, c, d])
-    }
-    fn load_full(&self) -> [usize; 4] {
-        self.0
-    }
-}
-
-impl SystemTword for NativeQword {
-    fn store_full(a: usize, b: usize, c: usize) -> Self {
-        Self([a, b, c, 0])
-    }
-    fn load_full(&self) -> [usize; 3] {
-        [self.0[0], self.0[1], self.0[2]]
-    }
-}
-
-impl SystemDword for NativeQword {
-    fn store_qw(u: u64) -> Self {
-        let ret;
-        #[cfg(target_pointer_width = "32")]
-        {
-            let [a, b]: [usize; 2] = unsafe { core::mem::transmute(u) };
-            ret = <Self as SystemQword>::store_full(a, b, 0, 0);
-        }
-        #[cfg(target_pointer_width = "64")]
-        {
-            ret = <Self as SystemQword>::store_full(u as _, 0, 0, 0);
-        }
-        ret
-    }
-    fn store_fat(a: usize, b: usize) -> Self {
-        <Self as SystemQword>::store_full(a, b, 0, 0)
-    }
-    fn load_qw(&self) -> u64 {
-        let ret;
-        #[cfg(target_pointer_width = "32")]
-        {
-            ret = unsafe { core::mem::transmute([self.0[0], self.0[1]]) };
-        }
-        #[cfg(target_pointer_width = "64")]
-        {
-            ret = self.0[0] as _;
-        }
-        ret
-    }
-    fn load_fat(&self) -> [usize; 2] {
-        [self.0[0], self.0[1]]
-    }
-}
diff --git a/server/src/engine/mem/word.rs b/server/src/engine/mem/word.rs
new file mode 100644
index 00000000..5a6f7f3b
--- /dev/null
+++ b/server/src/engine/mem/word.rs
@@ -0,0 +1,302 @@
+/*
+ * Created on Wed Mar 01 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::{NativeDword, NativeQword, NativeTword};
+
+/// Native quad pointer stack (must also be usable as a double and triple pointer stack. see [`SystemTword`] and [`SystemDword`])
+pub trait SystemQword: SystemTword {
+    fn store_full(a: usize, b: usize, c: usize, d: usize) -> Self;
+    fn load_full(&self) -> [usize; 4];
+    fn store<'a, T>(v: T) -> Self
+    where
+        T: WordRW<Self>,
+    {
+        WordRW::store(v)
+    }
+    fn load<'a, T>(&'a self) -> T::Target<'a>
+    where
+        T: WordRW<Self>,
+    {
+        <T>::load(self)
+    }
+}
+
+/// Native tripe pointer stack (must also be usable as a double pointer stack, see [`SystemDword`])
+pub trait SystemTword: SystemDword {
+    fn store_full(a: usize, b: usize, c: usize) -> Self;
+    fn load_full(&self) -> [usize; 3];
+    fn store<'a, T>(v: T) -> Self
+    where
+        T: WordRW<Self>,
+    {
+        WordRW::store(v)
+    }
+    fn load<'a, T>(&'a self) -> T::Target<'a>
+    where
+        T: WordRW<Self>,
+    {
+        <T>::load(self)
+    }
+}
+
+/// Native double pointer stack
+pub trait SystemDword: Sized {
+    fn store_qw(u: u64) -> Self;
+    fn store_fat(a: usize, b: usize) -> Self;
+    fn load_qw(&self) -> u64;
+    fn load_fat(&self) -> [usize; 2];
+    fn store<'a, T>(v: T) -> Self
+    where
+        T: WordRW<Self>,
+    {
+        WordRW::store(v)
+    }
+    fn load<'a, T>(&'a self) -> T::Target<'a>
+    where
+        T: WordRW<Self>,
+    {
+        <T>::load(self)
+    }
+}
+
+impl SystemDword for NativeDword {
+    #[inline(always)]
+    fn store_qw(u: u64) -> Self {
+        let x;
+        #[cfg(target_pointer_width = "32")]
+        {
+            x = unsafe { core::mem::transmute(u) };
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            x = [u as usize, 0]
+        }
+        Self(x)
+    }
+    #[inline(always)]
+    fn store_fat(a: usize, b: usize) -> Self {
+        Self([a, b])
+    }
+    #[inline(always)]
+    fn load_qw(&self) -> u64 {
+        let x;
+        #[cfg(target_pointer_width = "32")]
+        {
+            x = unsafe { core::mem::transmute_copy(self) }
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            x = self.0[0] as _;
+        }
+        x
+    }
+    #[inline(always)]
+    fn load_fat(&self) -> [usize; 2] {
+        self.0
+    }
+}
+
+impl SystemTword for NativeTword {
+    #[inline(always)]
+    fn store_full(a: usize, b: usize, c: usize) -> Self {
+        Self([a, b, c])
+    }
+    #[inline(always)]
+    fn load_full(&self) -> [usize; 3] {
+        self.0
+    }
+}
+
+impl SystemDword for NativeTword {
+    #[inline(always)]
+    fn store_qw(u: u64) -> Self {
+        let x;
+        #[cfg(target_pointer_width = "32")]
+        {
+            let [a, b]: [usize; 2] = unsafe { core::mem::transmute(u) };
+            x = [a, b, 0];
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            x = [u as _, 0, 0];
+        }
+        Self(x)
+    }
+    #[inline(always)]
+    fn store_fat(a: usize, b: usize) -> Self {
+        Self([a, b, 0])
+    }
+    #[inline(always)]
+    fn load_qw(&self) -> u64 {
+        let x;
+        #[cfg(target_pointer_width = "32")]
+        {
+            let ab = [self.0[0], self.0[1]];
+            x = unsafe { core::mem::transmute(ab) };
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            x = self.0[0] as _;
+        }
+        x
+    }
+    #[inline(always)]
+    fn load_fat(&self) -> [usize; 2] {
+        [self.0[0], self.0[1]]
+    }
+}
+
+impl SystemQword for NativeQword {
+    fn store_full(a: usize, b: usize, c: usize, d: usize) -> Self {
+        Self([a, b, c, d])
+    }
+    fn load_full(&self) -> [usize; 4] {
+        self.0
+    }
+}
+
+impl SystemTword for NativeQword {
+    fn store_full(a: usize, b: usize, c: usize) -> Self {
+        Self([a, b, c, 0])
+    }
+    fn load_full(&self) -> [usize; 3] {
+        [self.0[0], self.0[1], self.0[2]]
+    }
+}
+
+impl SystemDword for NativeQword {
+    fn store_qw(u: u64) -> Self {
+        let ret;
+        #[cfg(target_pointer_width = "32")]
+        {
+            let [a, b]: [usize; 2] = unsafe { core::mem::transmute(u) };
+            ret = <Self as SystemQword>::store_full(a, b, 0, 0);
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            ret = <Self as SystemQword>::store_full(u as _, 0, 0, 0);
+        }
+        ret
+    }
+    fn store_fat(a: usize, b: usize) -> Self {
+        <Self as SystemQword>::store_full(a, b, 0, 0)
+    }
+    fn load_qw(&self) -> u64 {
+        let ret;
+        #[cfg(target_pointer_width = "32")]
+        {
+            ret = unsafe { core::mem::transmute([self.0[0], self.0[1]]) };
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            ret = self.0[0] as _;
+        }
+        ret
+    }
+    fn load_fat(&self) -> [usize; 2] {
+        [self.0[0], self.0[1]]
+    }
+}
+
+pub trait WordRW<W> {
+    type Target<'a>
+    where
+        W: 'a;
+    fn store(self) -> W;
+    fn load<'a>(word: &'a W) -> Self::Target<'a>;
+}
+
+macro_rules! impl_wordrw {
+	($($ty:ty as $minword:ident => { type Target<'a> = $target:ty; |$selfname:ident| $store:expr; |$wordarg:ident| $load:expr;})*) => {
+		$(impl<W: $minword> WordRW<W> for $ty { type Target<'a> = $target where W: 'a; fn store($selfname: Self) -> W { $store } fn load<'a>($wordarg: &'a W) -> Self::Target<'a> { $load } })*
+	};
+	($($ty:ty as $minword:ident => { |$selfname:ident| $store:expr; |$wordarg:ident| $load:expr;})*) => { impl_wordrw!($($ty as $minword => { type Target<'a> = $ty; |$selfname| $store; |$wordarg| $load;})*); };
+}
+
+impl_wordrw! {
+    bool as SystemDword => {
+        |self| SystemDword::store_qw(self as _);
+        |word| SystemDword::load_qw(word) == 1;
+    }
+    u8 as SystemDword => {
+        |self| SystemDword::store_qw(self as _);
+        |word| SystemDword::load_qw(word) as u8;
+    }
+    u16 as SystemDword => {
+        |self| SystemDword::store_qw(self as _);
+        |word| SystemDword::load_qw(word) as u16;
+    }
+    u32 as SystemDword => {
+        |self| SystemDword::store_qw(self as _);
+        |word| SystemDword::load_qw(word) as u32;
+    }
+    u64 as SystemDword => {
+        |self| SystemDword::store_qw(self);
+        |word| SystemDword::load_qw(word);
+    }
+    i8 as SystemDword => {
+        |self| SystemDword::store_qw(self as _);
+        |word| SystemDword::load_qw(word) as i8;
+    }
+    i16 as SystemDword => {
+        |self| SystemDword::store_qw(self as _);
+        |word| SystemDword::load_qw(word) as i16;
+    }
+    i32 as SystemDword => {
+        |self| SystemDword::store_qw(self as _);
+        |word| SystemDword::load_qw(word) as i32;
+    }
+    i64 as SystemDword => {
+        |self| SystemDword::store_qw(self as _);
+        |word| SystemDword::load_qw(word) as i64;
+    }
+    f32 as SystemDword => {
+        |self| SystemDword::store_qw(self as _);
+        |word| f64::from_bits(SystemDword::load_qw(word)) as _;
+    }
+    f64 as SystemDword => {
+        |self| SystemDword::store_qw(self as _);
+        |word| f64::from_bits(SystemDword::load_qw(word));
+    }
+    [usize; 2] as SystemDword => {
+        |self| SystemDword::store_fat(self[0], self[1]);
+        |word| SystemDword::load_fat(word);
+    }
+    (*mut u8, usize) as SystemDword => {
+        |self| SystemDword::store_fat(self.0 as usize, self.1);
+        |word| {
+            let [a, b] = word.load_fat();
+            (a as *mut u8, b)
+        };
+    }
+    (*const u8, usize) as SystemDword => {
+        |self| SystemDword::store_fat(self.0 as usize, self.1);
+        |word| {
+            let [a, b] = word.load_fat();
+            (a as *const u8, b)
+        };
+    }
+}
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index f947dbbd..43b84b58 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -146,7 +146,7 @@ impl<'a> PartialEq<Symbol> for Token<'a> {
     }
 }
 
-enum_impls! {
+direct_from! {
     Token<'a> => {
         Keyword as Keyword,
         Symbol as Symbol,

From 8d789bd166bb00da7c7286b56adb69e751e7fba3 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 1 Mar 2023 09:24:11 -0800
Subject: [PATCH 146/310] Migrate to the newer datacell impl

---
 server/src/engine/core/mod.rs                 |   2 +
 server/src/engine/core/model/cell.rs          |  54 +++++++-
 server/src/engine/core/model/mod.rs           |   2 +-
 server/src/engine/core/tests/space/alter.rs   |  12 +-
 server/src/engine/core/tests/space/create.rs  |   4 +-
 server/src/engine/data/md_dict.rs             |  20 ++-
 server/src/engine/data/mod.rs                 | 126 +-----------------
 server/src/engine/data/tests/md_dict_tests.rs |  52 ++++----
 server/src/engine/ql/ast/mod.rs               |  15 ++-
 server/src/engine/ql/dml/ins.rs               |  59 ++++----
 server/src/engine/ql/tests.rs                 |  14 +-
 server/src/engine/ql/tests/dml_tests.rs       |  13 +-
 12 files changed, 154 insertions(+), 219 deletions(-)

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 599f1f9c..c8966d1b 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -35,6 +35,8 @@ use {
     std::sync::Arc,
 };
 
+pub use model::cell::Datacell;
+
 /// Use this for now since it substitutes for a file lock (and those syscalls are expensive),
 /// but something better is in the offing
 type RWLIdx<K, V> = RwLock<IndexST<K, V>>;
diff --git a/server/src/engine/core/model/cell.rs b/server/src/engine/core/model/cell.rs
index 1dc6f980..32d2d782 100644
--- a/server/src/engine/core/model/cell.rs
+++ b/server/src/engine/core/model/cell.rs
@@ -29,7 +29,11 @@ use core::mem;
 use {
     crate::engine::{
         self,
-        data::tag::TagClass,
+        data::{
+            lit::{Lit, LitIR},
+            spec::{Dataspec1D, DataspecMeta1D},
+            tag::{DataTag, TagClass},
+        },
         mem::{NativeQword, SystemDword},
     },
     core::{fmt, mem::ManuallyDrop, slice, str},
@@ -182,6 +186,51 @@ direct_from! {
     }
 }
 
+impl<'a> From<LitIR<'a>> for Datacell {
+    fn from(l: LitIR<'a>) -> Self {
+        match l.kind().tag_class() {
+            tag if tag < TagClass::Bin => unsafe {
+                let [a, b] = l.data().load_fat();
+                Datacell::new(
+                    l.kind().tag_class(),
+                    DataRaw::word(SystemDword::store_fat(a, b)),
+                )
+            },
+            tag @ (TagClass::Bin | TagClass::Str) => unsafe {
+                let mut bin = ManuallyDrop::new(l.read_bin_uck().to_owned().into_boxed_slice());
+                Datacell::new(
+                    tag,
+                    DataRaw::word(SystemDword::store((bin.as_mut_ptr(), bin.len()))),
+                )
+            },
+            _ => unreachable!(),
+        }
+    }
+}
+
+#[cfg(test)]
+impl From<i32> for Datacell {
+    fn from(i: i32) -> Self {
+        if i.is_negative() {
+            Self::new_sint(i as _)
+        } else {
+            Self::new_uint(i as _)
+        }
+    }
+}
+
+impl<'a> From<Lit<'a>> for Datacell {
+    fn from(l: Lit<'a>) -> Self {
+        Self::from(l.as_ir())
+    }
+}
+
+impl<const N: usize> From<[Datacell; N]> for Datacell {
+    fn from(l: [Datacell; N]) -> Self {
+        Self::new_list(l.into())
+    }
+}
+
 impl Datacell {
     unsafe fn new(tag: TagClass, data: DataRaw) -> Self {
         Self { tag, data }
@@ -189,6 +238,9 @@ impl Datacell {
     fn checked_tag<T>(&self, tag: TagClass, f: impl FnOnce() -> T) -> Option<T> {
         (self.tag == tag).then_some(f())
     }
+    pub fn kind(&self) -> TagClass {
+        self.tag
+    }
 }
 
 impl fmt::Debug for Datacell {
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index c64cae53..e90ba91a 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -24,7 +24,7 @@
  *
 */
 
-mod cell;
+pub mod cell;
 
 // FIXME(@ohsayan): update this!
 
diff --git a/server/src/engine/core/tests/space/alter.rs b/server/src/engine/core/tests/space/alter.rs
index 2509107a..8c18f6ee 100644
--- a/server/src/engine/core/tests/space/alter.rs
+++ b/server/src/engine/core/tests/space/alter.rs
@@ -26,10 +26,10 @@
 
 use crate::engine::{
     core::{
+        model::cell::Datacell,
         space::{Space, SpaceMeta},
         GlobalNS,
     },
-    data::HSData,
     error::DatabaseError,
 };
 
@@ -45,7 +45,7 @@ fn alter_add_prop_env_var() {
                 space.unwrap(),
                 &Space::new(
                     into_dict!(),
-                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => HSData::UnsignedInt(100)))
+                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint(100)))
                 )
             )
         },
@@ -61,7 +61,7 @@ fn alter_update_prop_env_var() {
         |space| {
             assert_eq!(
                 space.unwrap().meta.env.read().get("MY_NEW_PROP").unwrap(),
-                &(HSData::UnsignedInt(100).into())
+                &(Datacell::new_uint(100).into())
             )
         },
     );
@@ -73,7 +73,7 @@ fn alter_update_prop_env_var() {
                 space.unwrap(),
                 &Space::new(
                     into_dict!(),
-                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => HSData::UnsignedInt(200)))
+                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint(200)))
                 )
             )
         },
@@ -89,7 +89,7 @@ fn alter_remove_prop_env_var() {
         |space| {
             assert_eq!(
                 space.unwrap().meta.env.read().get("MY_NEW_PROP").unwrap(),
-                &(HSData::UnsignedInt(100).into())
+                &(Datacell::new_uint(100).into())
             )
         },
     );
@@ -124,7 +124,7 @@ fn alter_remove_all_env() {
         |space| {
             assert_eq!(
                 space.unwrap().meta.env.read().get("MY_NEW_PROP").unwrap(),
-                &(HSData::UnsignedInt(100).into())
+                &(Datacell::new_uint(100).into())
             )
         },
     );
diff --git a/server/src/engine/core/tests/space/create.rs b/server/src/engine/core/tests/space/create.rs
index d17b483a..0170add2 100644
--- a/server/src/engine/core/tests/space/create.rs
+++ b/server/src/engine/core/tests/space/create.rs
@@ -26,10 +26,10 @@
 
 use crate::engine::{
     core::{
+        model::cell::Datacell,
         space::{Space, SpaceMeta},
         GlobalNS,
     },
-    data::HSData,
     error::DatabaseError,
 };
 
@@ -57,7 +57,7 @@ fn exec_create_space_with_env() {
                 &Space::new(
                     into_dict! {},
                     SpaceMeta::with_env(into_dict! {
-                        "MAX_MODELS" => HSData::UnsignedInt(100)
+                        "MAX_MODELS" => Datacell::new_uint(100)
                     })
                 )
             );
diff --git a/server/src/engine/data/md_dict.rs b/server/src/engine/data/md_dict.rs
index 34552352..a6f98a9d 100644
--- a/server/src/engine/data/md_dict.rs
+++ b/server/src/engine/data/md_dict.rs
@@ -26,10 +26,8 @@
 
 use {
     crate::engine::{
-        data::{
-            lit::{Lit, LitIR},
-            HSData,
-        },
+        core::Datacell,
+        data::lit::{Lit, LitIR},
         idx::STIndex,
     },
     std::collections::HashMap,
@@ -47,7 +45,7 @@ pub type DictGeneric = HashMap<Box<str>, Option<DictEntryGeneric>>;
 #[derive(Debug, PartialEq)]
 /// A generic dict entry: either a literal or a recursive dictionary
 pub enum DictEntryGeneric {
-    Lit(HSData),
+    Lit(Datacell),
     Map(DictGeneric),
 }
 
@@ -55,7 +53,7 @@ pub enum DictEntryGeneric {
 #[cfg_attr(test, derive(Clone))]
 /// A metadata dictionary
 pub enum MetaDictEntry {
-    Data(HSData),
+    Data(Datacell),
     Map(MetaDict),
 }
 
@@ -67,7 +65,7 @@ pub enum MetaDictEntry {
 struct MetaDictPatch(HashMap<Box<str>, Option<MetaDictPatchEntry>>);
 #[derive(Debug, PartialEq)]
 enum MetaDictPatchEntry {
-    Data(HSData),
+    Data(Datacell),
     Map(MetaDictPatch),
 }
 
@@ -208,26 +206,26 @@ fn rmerge_metadata_prepare_patch(
 
 impl<'a> From<LitIR<'a>> for DictEntryGeneric {
     fn from(l: LitIR<'a>) -> Self {
-        Self::Lit(HSData::from(l))
+        Self::Lit(Datacell::from(l))
     }
 }
 
 impl<'a> From<Lit<'a>> for DictEntryGeneric {
     fn from(value: Lit<'a>) -> Self {
-        Self::Lit(HSData::from(value))
+        Self::Lit(Datacell::from(value))
     }
 }
 
 direct_from! {
     DictEntryGeneric => {
-        HSData as Lit,
+        Datacell as Lit,
         DictGeneric as Map,
     }
 }
 
 direct_from! {
     MetaDictEntry => {
-        HSData as Data,
+        Datacell as Data,
         MetaDict as Map,
     }
 }
diff --git a/server/src/engine/data/mod.rs b/server/src/engine/data/mod.rs
index 2086e345..e42ee2c5 100644
--- a/server/src/engine/data/mod.rs
+++ b/server/src/engine/data/mod.rs
@@ -33,132 +33,8 @@ pub mod tag;
 #[cfg(test)]
 mod tests;
 
+use crate::engine::mem::AStr;
 pub use md_dict::{DictEntryGeneric, DictGeneric, MetaDict};
-use {
-    self::lit::Lit,
-    crate::engine::{data::lit::LitIR, mem::AStr},
-    std::mem::{self, Discriminant},
-};
 
 const IDENT_MX: usize = 64;
 pub type ItemID = AStr<IDENT_MX>;
-
-/// A [`DataType`] represents the underlying data-type, although this enumeration when used in a collection will always
-/// be of one type.
-// TODO(@ohsayan): Change the underlying structures, there are just rudimentary ones used during integration with the QL
-#[derive(Debug, PartialEq)]
-#[cfg_attr(test, derive(Clone))]
-#[repr(u8)]
-pub enum HSData {
-    /// An UTF-8 string
-    String(Box<str>) = DataKind::STR_BX.d(),
-    /// Bytes
-    Binary(Box<[u8]>) = DataKind::BIN_BX.d(),
-    /// An unsigned integer
-    ///
-    /// **NOTE:** This is the default evaluated type for unsigned integers by the query processor. It is the
-    /// responsibility of the executor to ensure integrity checks depending on actual type width in the declared
-    /// schema (if any)
-    UnsignedInt(u64) = DataKind::UINT64.d(),
-    /// A signed integer
-    ///
-    /// **NOTE:** This is the default evaluated type for signed integers by the query processor. It is the
-    /// responsibility of the executor to ensure integrity checks depending on actual type width in the declared
-    /// schema (if any)
-    SignedInt(i64) = DataKind::SINT64.d(),
-    /// A boolean
-    Boolean(bool) = DataKind::BOOL.d(),
-    /// A float (64-bit)
-    Float(f64) = DataKind::FLOAT64.d(),
-    /// A single-type list. Note, you **need** to keep up the invariant that the [`DataType`] disc. remains the same for all
-    /// elements to ensure correctness in this specific context
-    /// FIXME(@ohsayan): Try enforcing this somehow
-    List(Vec<Self>) = DataKind::LIST.d(),
-}
-
-direct_from! {
-    HSData => {
-        String as String,
-        Vec<u8> as Binary,
-        u64 as UnsignedInt,
-        bool as Boolean,
-        Vec<Self> as List,
-        &'static str as String,
-    }
-}
-
-impl HSData {
-    #[inline(always)]
-    pub(super) fn clone_from_lit(lit: Lit) -> Self {
-        match_data!(match lit {
-            Lit::Str(s) => HSData::String(s.to_string().into_boxed_str()),
-            Lit::Bool(b) => HSData::Boolean(b),
-            Lit::UnsignedInt(u) => HSData::UnsignedInt(u),
-            Lit::SignedInt(i) => HSData::SignedInt(i),
-            Lit::Float(f) => HSData::Float(f),
-            Lit::Bin(l) => HSData::Binary(l.to_vec().into_boxed_slice()),
-            TagClass::List(_) => unreachable!("found 2D data in 1D"),
-        })
-    }
-    #[inline(always)]
-    pub(super) fn clone_from_litir<'a>(lit: LitIR<'a>) -> Self {
-        match_data!(match lit {
-            LitIR::Str(s) => Self::String(s.to_owned().into_boxed_str()),
-            LitIR::Bin(b) => Self::Binary(b.to_owned().into_boxed_slice()),
-            LitIR::Float(f) => Self::Float(f),
-            LitIR::SignedInt(s) => Self::SignedInt(s),
-            LitIR::UnsignedInt(u) => Self::UnsignedInt(u),
-            LitIR::Bool(b) => Self::Boolean(b),
-            TagClass::List(_) => unreachable!("found 2D data in 1D"),
-        })
-    }
-    fn kind(&self) -> Discriminant<Self> {
-        mem::discriminant(&self)
-    }
-}
-
-impl<'a> From<Lit<'a>> for HSData {
-    fn from(l: Lit<'a>) -> Self {
-        Self::clone_from_lit(l)
-    }
-}
-
-impl<'a> From<LitIR<'a>> for HSData {
-    fn from(l: LitIR<'a>) -> Self {
-        Self::clone_from_litir(l)
-    }
-}
-
-impl<const N: usize> From<[HSData; N]> for HSData {
-    fn from(f: [HSData; N]) -> Self {
-        Self::List(f.into())
-    }
-}
-
-flags! {
-    #[derive(PartialEq, Eq, Clone, Copy)]
-    pub struct DataKind: u8 {
-        // primitive: integer unsigned
-        UINT8 = 0,
-        UINT16 = 1,
-        UINT32 = 2,
-        UINT64 = 3,
-        // primitive: integer unsigned
-        SINT8 = 4,
-        SINT16 = 5,
-        SINT32 = 6,
-        SINT64 = 7,
-        // primitive: misc
-        BOOL = 8,
-        // primitive: floating point
-        FLOAT32 = 9,
-        FLOAT64 = 10,
-        // compound: flat
-        STR = 11,
-        STR_BX = DataKind::_BASE_HB | DataKind::STR.d(),
-        BIN = 12,
-        BIN_BX = DataKind::_BASE_HB | DataKind::BIN.d(),
-        // compound: recursive
-        LIST = 13,
-    }
-}
diff --git a/server/src/engine/data/tests/md_dict_tests.rs b/server/src/engine/data/tests/md_dict_tests.rs
index 9f2577f0..ffd07576 100644
--- a/server/src/engine/data/tests/md_dict_tests.rs
+++ b/server/src/engine/data/tests/md_dict_tests.rs
@@ -24,19 +24,19 @@
  *
 */
 
-use crate::engine::data::{
-    md_dict::{self, DictEntryGeneric, DictGeneric, MetaDict, MetaDictEntry},
-    HSData,
+use crate::engine::{
+    core::Datacell,
+    data::md_dict::{self, DictEntryGeneric, DictGeneric, MetaDict, MetaDictEntry},
 };
 
 #[test]
 fn t_simple_flatten() {
     let generic_dict: DictGeneric = into_dict! {
-        "a_valid_key" => Some(DictEntryGeneric::Lit(100.into())),
+        "a_valid_key" => Some(DictEntryGeneric::Lit(100u64.into())),
         "a_null_key" => None,
     };
     let expected: MetaDict = into_dict!(
-        "a_valid_key" => HSData::UnsignedInt(100)
+        "a_valid_key" => Datacell::new_uint(100)
     );
     let ret = md_dict::rflatten_metadata(generic_dict);
     assert_eq!(ret, expected);
@@ -45,18 +45,18 @@ fn t_simple_flatten() {
 #[test]
 fn t_simple_patch() {
     let mut current: MetaDict = into_dict! {
-        "a" => HSData::UnsignedInt(2),
-        "b" => HSData::UnsignedInt(3),
-        "z" => HSData::SignedInt(-100),
+        "a" => Datacell::new_uint(2),
+        "b" => Datacell::new_uint(3),
+        "z" => Datacell::new_sint(-100),
     };
     let new: DictGeneric = into_dict! {
-        "a" => Some(HSData::UnsignedInt(1).into()),
-        "b" => Some(HSData::UnsignedInt(2).into()),
+        "a" => Some(Datacell::new_uint(1).into()),
+        "b" => Some(Datacell::new_uint(2).into()),
         "z" => None,
     };
     let expected: MetaDict = into_dict! {
-        "a" => HSData::UnsignedInt(1),
-        "b" => HSData::UnsignedInt(2),
+        "a" => Datacell::new_uint(1),
+        "b" => Datacell::new_uint(2),
     };
     assert!(md_dict::rmerge_metadata(&mut current, new));
     assert_eq!(current, expected);
@@ -65,15 +65,15 @@ fn t_simple_patch() {
 #[test]
 fn t_bad_patch() {
     let mut current: MetaDict = into_dict! {
-        "a" => HSData::UnsignedInt(2),
-        "b" => HSData::UnsignedInt(3),
-        "z" => HSData::SignedInt(-100),
+        "a" => Datacell::new_uint(2),
+        "b" => Datacell::new_uint(3),
+        "z" => Datacell::new_sint(-100),
     };
     let backup = current.clone();
     let new: DictGeneric = into_dict! {
-        "a" => Some(HSData::UnsignedInt(1).into()),
-        "b" => Some(HSData::UnsignedInt(2).into()),
-        "z" => Some(HSData::String("omg".into()).into()),
+        "a" => Some(Datacell::new_uint(1).into()),
+        "b" => Some(Datacell::new_uint(2).into()),
+        "z" => Some(Datacell::new_str("omg".into()).into()),
     };
     assert!(!md_dict::rmerge_metadata(&mut current, new));
     assert_eq!(current, backup);
@@ -82,20 +82,20 @@ fn t_bad_patch() {
 #[test]
 fn patch_null_out_dict() {
     let mut current: MetaDict = into_dict! {
-        "a" => HSData::UnsignedInt(2),
-        "b" => HSData::UnsignedInt(3),
+        "a" => Datacell::new_uint(2),
+        "b" => Datacell::new_uint(3),
         "z" => MetaDictEntry::Map(into_dict!(
-            "c" => HSData::UnsignedInt(1),
-            "d" => HSData::UnsignedInt(2)
+            "c" => Datacell::new_uint(1),
+            "d" => Datacell::new_uint(2)
         )),
     };
     let expected: MetaDict = into_dict! {
-        "a" => HSData::UnsignedInt(2),
-        "b" => HSData::UnsignedInt(3),
+        "a" => Datacell::new_uint(2),
+        "b" => Datacell::new_uint(3),
     };
     let new: DictGeneric = into_dict! {
-        "a" => Some(HSData::UnsignedInt(2).into()),
-        "b" => Some(HSData::UnsignedInt(3).into()),
+        "a" => Some(Datacell::new_uint(2).into()),
+        "b" => Some(Datacell::new_uint(3).into()),
         "z" => None,
     };
     assert!(md_dict::rmerge_metadata(&mut current, new));
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 8dd04a97..cd631002 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -36,7 +36,8 @@ use {
     },
     crate::{
         engine::{
-            data::{lit::LitIR, HSData},
+            core::Datacell,
+            data::lit::LitIR,
             error::{LangError, LangResult},
         },
         util::{compiler, MaybeInit},
@@ -238,7 +239,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     ///
     /// Caller should have checked that the token matches a lit signature and that enough data is available
     /// in the data source. (ideally should run `can_read_lit_from` or `can_read_lit_rounded`)
-    pub unsafe fn read_lit_into_data_type_unchecked_from(&mut self, tok: &'a Token) -> HSData {
+    pub unsafe fn read_lit_into_data_type_unchecked_from(&mut self, tok: &'a Token) -> Datacell {
         self.d.read_data_type(tok)
     }
     #[inline(always)]
@@ -280,7 +281,7 @@ pub trait QueryData<'a> {
     ///
     /// ## Safety
     /// The current token must match the signature of a lit
-    unsafe fn read_data_type(&mut self, tok: &'a Token) -> HSData;
+    unsafe fn read_data_type(&mut self, tok: &'a Token) -> Datacell;
     /// Returns true if the data source has enough data
     fn nonzero(&self) -> bool;
 }
@@ -304,8 +305,8 @@ impl<'a> QueryData<'a> for InplaceData {
         extract!(tok, Token::Lit(l) => l.as_ir())
     }
     #[inline(always)]
-    unsafe fn read_data_type(&mut self, tok: &'a Token) -> HSData {
-        HSData::from(extract!(tok, Token::Lit(ref l) => l.to_owned()))
+    unsafe fn read_data_type(&mut self, tok: &'a Token) -> Datacell {
+        Datacell::from(extract!(tok, Token::Lit(ref l) => l.to_owned()))
     }
     #[inline(always)]
     fn nonzero(&self) -> bool {
@@ -337,11 +338,11 @@ impl<'a> QueryData<'a> for SubstitutedData<'a> {
         ret
     }
     #[inline(always)]
-    unsafe fn read_data_type(&mut self, tok: &'a Token) -> HSData {
+    unsafe fn read_data_type(&mut self, tok: &'a Token) -> Datacell {
         debug_assert!(Token![?].eq(tok));
         let ret = self.data[0].clone();
         self.data = &self.data[1..];
-        HSData::from(ret)
+        Datacell::from(ret)
     }
     #[inline(always)]
     fn nonzero(&self) -> bool {
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index c9242bb7..c267c30b 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -28,7 +28,7 @@ use {
     super::read_ident,
     crate::{
         engine::{
-            data::HSData,
+            core::Datacell,
             error::{LangError, LangResult},
             ql::{
                 ast::{Entity, QueryData, State},
@@ -37,10 +37,7 @@ use {
         },
         util::{compiler, MaybeInit},
     },
-    core::{
-        cmp,
-        mem::{discriminant, Discriminant},
-    },
+    core::cmp,
     std::{
         collections::HashMap,
         time::{Duration, SystemTime, UNIX_EPOCH},
@@ -56,7 +53,7 @@ pub const T_UUIDSTR: &str = "4593264b-0231-43e9-b0aa-50784f14e204";
 pub const T_UUIDBIN: &[u8] = T_UUIDSTR.as_bytes();
 pub const T_TIMESEC: u64 = 1673187839_u64;
 
-type ProducerFn = fn() -> HSData;
+type ProducerFn = fn() -> Datacell;
 
 // base
 #[inline(always)]
@@ -77,16 +74,16 @@ fn pfnbase_uuid() -> Uuid {
 }
 // impl
 #[inline(always)]
-fn pfn_timesec() -> HSData {
-    HSData::UnsignedInt(pfnbase_time().as_secs())
+fn pfn_timesec() -> Datacell {
+    Datacell::new_uint(pfnbase_time().as_secs())
 }
 #[inline(always)]
-fn pfn_uuidstr() -> HSData {
-    HSData::String(pfnbase_uuid().to_string().into_boxed_str())
+fn pfn_uuidstr() -> Datacell {
+    Datacell::new_str(pfnbase_uuid().to_string().into_boxed_str())
 }
 #[inline(always)]
-fn pfn_uuidbin() -> HSData {
-    HSData::Binary(pfnbase_uuid().as_bytes().to_vec().into_boxed_slice())
+fn pfn_uuidbin() -> Datacell {
+    Datacell::new_bin(pfnbase_uuid().as_bytes().to_vec().into_boxed_slice())
 }
 
 static PRODUCER_G: [u8; 4] = [0, 2, 3, 0];
@@ -141,8 +138,8 @@ unsafe fn ldfunc_unchecked(func: &[u8]) -> ProducerFn {
 /// - If tt length is less than 1
 pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
     state: &mut State<'a, Qd>,
-    list: &mut Vec<HSData>,
-) -> Option<Discriminant<HSData>> {
+    list: &mut Vec<Datacell>,
+) -> Option<TagClass> {
     let mut stop = state.cursor_eq(Token![close []]);
     state.cursor_ahead_if(stop);
     let mut overall_dscr = None;
@@ -169,7 +166,7 @@ pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
                 if prev_nlist_dscr.is_none() && nlist_dscr.is_some() {
                     prev_nlist_dscr = nlist_dscr;
                 }
-                HSData::List(nested_list)
+                Datacell::new_list(nested_list)
             }
             Token![@] if state.cursor_signature_match_fn_arity0_rounded() => match unsafe {
                 // UNSAFE(@ohsayan): Just verified at guard
@@ -187,8 +184,8 @@ pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
                 break;
             }
         };
-        state.poison_if_not(list.is_empty() || discriminant(&d) == discriminant(&list[0]));
-        overall_dscr = Some(discriminant(&d));
+        state.poison_if_not(list.is_empty() || d.kind() == list[0].kind());
+        overall_dscr = Some(d.kind());
         list.push(d);
         let nx_comma = state.cursor_rounded_eq(Token![,]);
         let nx_csqrb = state.cursor_rounded_eq(Token![close []]);
@@ -202,7 +199,7 @@ pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
 #[inline(always)]
 /// ## Safety
 /// - Cursor must match arity(0) function signature
-unsafe fn handle_func_sub<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> Option<HSData> {
+unsafe fn handle_func_sub<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> Option<Datacell> {
     let func = read_ident(state.fw_read());
     state.cursor_ahead_by(2); // skip tt:paren
     ldfunc(func).map(move |f| f())
@@ -212,7 +209,7 @@ unsafe fn handle_func_sub<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> O
 /// - If tt is empty
 pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
     state: &mut State<'a, Qd>,
-) -> Vec<Option<HSData>> {
+) -> Vec<Option<Datacell>> {
     let mut stop = state.cursor_eq(Token![() close]);
     state.cursor_ahead_if(stop);
     let mut data = Vec::new();
@@ -259,7 +256,7 @@ pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
 /// Panics if tt is empty
 pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
     state: &mut State<'a, Qd>,
-) -> HashMap<Ident<'a>, Option<HSData>> {
+) -> HashMap<Ident<'a>, Option<Datacell>> {
     let mut stop = state.cursor_eq(Token![close {}]);
     state.cursor_ahead_if(stop);
     let mut data = HashMap::with_capacity(2);
@@ -313,18 +310,18 @@ pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
 
 #[derive(Debug, PartialEq)]
 pub enum InsertData<'a> {
-    Ordered(Vec<Option<HSData>>),
-    Map(HashMap<Ident<'a>, Option<HSData>>),
+    Ordered(Vec<Option<Datacell>>),
+    Map(HashMap<Ident<'a>, Option<Datacell>>),
 }
 
-impl<'a> From<Vec<Option<HSData>>> for InsertData<'a> {
-    fn from(v: Vec<Option<HSData>>) -> Self {
+impl<'a> From<Vec<Option<Datacell>>> for InsertData<'a> {
+    fn from(v: Vec<Option<Datacell>>) -> Self {
         Self::Ordered(v)
     }
 }
 
-impl<'a> From<HashMap<Ident<'static>, Option<HSData>>> for InsertData<'a> {
-    fn from(m: HashMap<Ident<'static>, Option<HSData>>) -> Self {
+impl<'a> From<HashMap<Ident<'static>, Option<Datacell>>> for InsertData<'a> {
+    fn from(m: HashMap<Ident<'static>, Option<Datacell>>) -> Self {
         Self::Map(m)
     }
 }
@@ -392,6 +389,8 @@ impl<'a> InsertStatement<'a> {
 
 #[cfg(test)]
 pub use impls::test::{DataMap, DataTuple, List};
+
+use crate::engine::data::tag::TagClass;
 mod impls {
     use {
         super::InsertStatement,
@@ -409,7 +408,7 @@ mod impls {
     pub mod test {
         use {
             super::super::{
-                parse_data_map_syntax, parse_data_tuple_syntax, parse_list, HSData, HashMap,
+                parse_data_map_syntax, parse_data_tuple_syntax, parse_list, Datacell, HashMap,
             },
             crate::engine::{
                 error::LangResult,
@@ -417,7 +416,7 @@ mod impls {
             },
         };
         #[derive(sky_macros::Wrapper, Debug)]
-        pub struct List(Vec<HSData>);
+        pub struct List(Vec<Datacell>);
         impl<'a> ASTNode<'a> for List {
             // important: upstream must verify this
             const VERIFY: bool = true;
@@ -428,7 +427,7 @@ mod impls {
             }
         }
         #[derive(sky_macros::Wrapper, Debug)]
-        pub struct DataTuple(Vec<Option<HSData>>);
+        pub struct DataTuple(Vec<Option<Datacell>>);
         impl<'a> ASTNode<'a> for DataTuple {
             // important: upstream must verify this
             const VERIFY: bool = true;
@@ -438,7 +437,7 @@ mod impls {
             }
         }
         #[derive(sky_macros::Wrapper, Debug)]
-        pub struct DataMap(HashMap<Box<str>, Option<HSData>>);
+        pub struct DataMap(HashMap<Box<str>, Option<Datacell>>);
         impl<'a> ASTNode<'a> for DataMap {
             // important: upstream must verify this
             const VERIFY: bool = true;
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 3b81622a..ad779032 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -27,7 +27,7 @@
 use {
     super::lex::{InsecureLexer, SafeLexer, Symbol, Token},
     crate::{
-        engine::{data::HSData, error::LexResult},
+        engine::{core::Datacell, error::LexResult},
         util::test_utils,
     },
     rand::{self, Rng},
@@ -54,24 +54,24 @@ pub trait NullableData<T> {
     fn data(self) -> Option<T>;
 }
 
-impl<T> NullableData<HSData> for T
+impl<T> NullableData<Datacell> for T
 where
-    T: Into<HSData>,
+    T: Into<Datacell>,
 {
-    fn data(self) -> Option<HSData> {
+    fn data(self) -> Option<Datacell> {
         Some(self.into())
     }
 }
 
 struct Null;
 
-impl NullableData<HSData> for Null {
-    fn data(self) -> Option<HSData> {
+impl NullableData<Datacell> for Null {
+    fn data(self) -> Option<Datacell> {
         None
     }
 }
 
-fn nullable_datatype(v: impl NullableData<HSData>) -> Option<HSData> {
+fn nullable_datatype(v: impl NullableData<Datacell>) -> Option<Datacell> {
     v.data()
 }
 
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 4c830c40..6bd751eb 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -436,9 +436,16 @@ mod stmt_insert {
         let r = parse_ast_node_full::<InsertStatement>(&x[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(Ident::from("twitter"), Ident::from("users")),
-            into_array_nullable!["sayan", "Sayan", "sayan@example.com", true, 12345, 67890]
-                .to_vec()
-                .into(),
+            into_array_nullable![
+                "sayan",
+                "Sayan",
+                "sayan@example.com",
+                true,
+                12345,
+                67890
+            ]
+            .to_vec()
+            .into(),
         );
         assert_eq!(e, r);
     }

From b8dd8135f146955ed4887165fd7cdafe0fe29426 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 2 Mar 2023 06:45:44 -0800
Subject: [PATCH 147/310] Implement layer validation

---
 server/src/engine/core/model/cell.rs       |  52 +++-----
 server/src/engine/core/model/mod.rs        | 143 +++++++++++++++++++++
 server/src/engine/core/tests/mod.rs        |   2 +
 server/src/engine/core/tests/model/mod.rs  |  69 ++++++++++
 server/src/engine/data/spec.rs             |  10 +-
 server/src/engine/data/tag.rs              |   9 ++
 server/src/engine/error.rs                 |   2 +
 server/src/engine/mem/word.rs              |  24 +++-
 server/src/engine/ql/ddl/syn.rs            |  29 +++--
 server/src/engine/ql/tests/schema_tests.rs | 122 +++++++++---------
 10 files changed, 350 insertions(+), 112 deletions(-)
 create mode 100644 server/src/engine/core/tests/model/mod.rs

diff --git a/server/src/engine/core/model/cell.rs b/server/src/engine/core/model/cell.rs
index 32d2d782..f995f4c4 100644
--- a/server/src/engine/core/model/cell.rs
+++ b/server/src/engine/core/model/cell.rs
@@ -26,6 +26,8 @@
 
 #[cfg(test)]
 use core::mem;
+
+use crate::engine::mem::WordRW;
 use {
     crate::engine::{
         self,
@@ -48,10 +50,10 @@ pub struct Datacell {
 impl Datacell {
     // bool
     pub fn new_bool(b: bool) -> Self {
-        unsafe { Self::new(TagClass::Bool, DataRaw::word(SystemDword::store_qw(b as _))) }
+        unsafe { Self::new(TagClass::Bool, DataRaw::word(SystemDword::store(b))) }
     }
     pub unsafe fn read_bool(&self) -> bool {
-        self.data.word.load_qw() == 1
+        self.load_word()
     }
     pub fn try_bool(&self) -> Option<bool> {
         self.checked_tag(TagClass::Bool, || unsafe { self.read_bool() })
@@ -61,15 +63,10 @@ impl Datacell {
     }
     // uint
     pub fn new_uint(u: u64) -> Self {
-        unsafe {
-            Self::new(
-                TagClass::UnsignedInt,
-                DataRaw::word(SystemDword::store_qw(u)),
-            )
-        }
+        unsafe { Self::new(TagClass::UnsignedInt, DataRaw::word(SystemDword::store(u))) }
     }
     pub unsafe fn read_uint(&self) -> u64 {
-        self.data.word.load_qw()
+        self.load_word()
     }
     pub fn try_uint(&self) -> Option<u64> {
         self.checked_tag(TagClass::UnsignedInt, || unsafe { self.read_uint() })
@@ -79,15 +76,10 @@ impl Datacell {
     }
     // sint
     pub fn new_sint(u: i64) -> Self {
-        unsafe {
-            Self::new(
-                TagClass::SignedInt,
-                DataRaw::word(SystemDword::store_qw(u as _)),
-            )
-        }
+        unsafe { Self::new(TagClass::SignedInt, DataRaw::word(SystemDword::store(u))) }
     }
     pub unsafe fn read_sint(&self) -> i64 {
-        self.data.word.load_qw() as _
+        self.load_word()
     }
     pub fn try_sint(&self) -> Option<i64> {
         self.checked_tag(TagClass::SignedInt, || unsafe { self.read_sint() })
@@ -97,15 +89,10 @@ impl Datacell {
     }
     // float
     pub fn new_float(f: f64) -> Self {
-        unsafe {
-            Self::new(
-                TagClass::Float,
-                DataRaw::word(SystemDword::store_qw(f.to_bits())),
-            )
-        }
+        unsafe { Self::new(TagClass::Float, DataRaw::word(SystemDword::store(f))) }
     }
     pub unsafe fn read_float(&self) -> f64 {
-        f64::from_bits(self.data.word.load_qw())
+        self.load_word()
     }
     pub fn try_float(&self) -> Option<f64> {
         self.checked_tag(TagClass::Float, || unsafe { self.read_float() })
@@ -119,13 +106,13 @@ impl Datacell {
         unsafe {
             Self::new(
                 TagClass::Bin,
-                DataRaw::word(SystemDword::store_fat(md.as_ptr() as usize, md.len())),
+                DataRaw::word(SystemDword::store((md.as_ptr(), md.len()))),
             )
         }
     }
     pub unsafe fn read_bin(&self) -> &[u8] {
-        let [p, l] = self.data.word.load_fat();
-        slice::from_raw_parts(p as *mut u8, l)
+        let (p, l) = self.load_word();
+        slice::from_raw_parts::<u8>(p, l)
     }
     pub fn try_bin(&self) -> Option<&[u8]> {
         self.checked_tag(TagClass::Bin, || unsafe { self.read_bin() })
@@ -139,13 +126,13 @@ impl Datacell {
         unsafe {
             Self::new(
                 TagClass::Str,
-                DataRaw::word(SystemDword::store_fat(md.as_ptr() as usize, md.len())),
+                DataRaw::word(SystemDword::store((md.as_ptr(), md.len()))),
             )
         }
     }
     pub unsafe fn read_str(&self) -> &str {
-        let [p, l] = self.data.word.load_fat();
-        str::from_utf8_unchecked(slice::from_raw_parts(p as *mut u8, l))
+        let (p, l) = self.load_word();
+        str::from_utf8_unchecked(slice::from_raw_parts(p, l))
     }
     pub fn try_str(&self) -> Option<&str> {
         self.checked_tag(TagClass::Str, || unsafe { self.read_str() })
@@ -241,6 +228,9 @@ impl Datacell {
     pub fn kind(&self) -> TagClass {
         self.tag
     }
+    unsafe fn load_word<'a, T: WordRW<NativeQword, Target<'a> = T>>(&'a self) -> T {
+        self.data.word.ld()
+    }
 }
 
 impl fmt::Debug for Datacell {
@@ -314,8 +304,8 @@ impl Drop for Datacell {
     fn drop(&mut self) {
         match self.tag {
             TagClass::Str | TagClass::Bin => unsafe {
-                let [p, l] = self.data.word.load_fat();
-                engine::mem::dealloc_array(p as *mut u8, l)
+                let (p, l) = self.load_word();
+                engine::mem::dealloc_array::<u8>(p, l)
             },
             TagClass::List => unsafe { ManuallyDrop::drop(&mut self.data.rwl) },
             _ => {}
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index e90ba91a..90109ad9 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -26,6 +26,12 @@
 
 pub mod cell;
 
+use crate::engine::{
+    data::tag::{DataTag, FullTag, TagSelector},
+    error::{DatabaseError, DatabaseResult},
+    ql::ddl::syn::LayerSpec,
+};
+
 // FIXME(@ohsayan): update this!
 
 #[derive(Debug)]
@@ -37,3 +43,140 @@ impl PartialEq for ModelView {
         true
     }
 }
+
+/*
+    Layer
+*/
+
+static G: [u8; 15] = [0, 13, 12, 5, 6, 4, 3, 6, 1, 10, 4, 5, 7, 5, 5];
+static S1: [u8; 7] = [13, 9, 4, 14, 2, 4, 7];
+static S2: [u8; 7] = [12, 8, 2, 6, 4, 9, 9];
+
+static LUT: [(&str, FullTag); 14] = [
+    ("bool", FullTag::BOOL),
+    ("uint8", FullTag::new_uint(TagSelector::UInt8)),
+    ("uint16", FullTag::new_uint(TagSelector::UInt16)),
+    ("uint32", FullTag::new_uint(TagSelector::UInt32)),
+    ("uint64", FullTag::new_uint(TagSelector::UInt64)),
+    ("sint8", FullTag::new_sint(TagSelector::SInt8)),
+    ("sint16", FullTag::new_sint(TagSelector::SInt16)),
+    ("sint32", FullTag::new_sint(TagSelector::SInt32)),
+    ("sint64", FullTag::new_sint(TagSelector::SInt64)),
+    ("float32", FullTag::new_float(TagSelector::Float32)),
+    ("float64", FullTag::new_float(TagSelector::Float64)),
+    ("binary", FullTag::BIN),
+    ("string", FullTag::STR),
+    ("list", FullTag::LIST),
+];
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct LayerView(Box<[Layer]>);
+
+impl LayerView {
+    pub fn layers(&self) -> &[Layer] {
+        &self.0
+    }
+    pub fn parse_layers(spec: Vec<LayerSpec>) -> DatabaseResult<Self> {
+        let mut layers = spec.into_iter().rev();
+        let mut okay = true;
+        let mut fin = false;
+        let mut layerview = Vec::with_capacity(layers.len());
+        while (layers.len() != 0) & okay & !fin {
+            let LayerSpec { ty, props } = layers.next().unwrap();
+            okay &= props.is_empty(); // FIXME(@ohsayan): you know what to do here
+            match Layer::get_layer(&ty) {
+                Some(l) => {
+                    fin = l.tag.tag_selector() != TagSelector::List;
+                    layerview.push(l);
+                }
+                None => okay = false,
+            }
+        }
+        okay &= fin & (layers.len() == 0);
+        if okay {
+            Ok(Self(layerview.into_boxed_slice()))
+        } else {
+            Err(DatabaseError::DdlModelInvalidTypeDefinition)
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub struct Layer {
+    tag: FullTag,
+    config: [usize; 2],
+}
+
+impl Layer {
+    pub const fn bool() -> Self {
+        Self::empty(FullTag::BOOL)
+    }
+    pub const fn uint8() -> Self {
+        Self::empty(FullTag::new_uint(TagSelector::UInt8))
+    }
+    pub const fn uint16() -> Self {
+        Self::empty(FullTag::new_uint(TagSelector::UInt16))
+    }
+    pub const fn uint32() -> Self {
+        Self::empty(FullTag::new_uint(TagSelector::UInt32))
+    }
+    pub const fn uint64() -> Self {
+        Self::empty(FullTag::new_uint(TagSelector::UInt64))
+    }
+    pub const fn sint8() -> Self {
+        Self::empty(FullTag::new_sint(TagSelector::SInt8))
+    }
+    pub const fn sint16() -> Self {
+        Self::empty(FullTag::new_sint(TagSelector::SInt16))
+    }
+    pub const fn sint32() -> Self {
+        Self::empty(FullTag::new_sint(TagSelector::SInt32))
+    }
+    pub const fn sint64() -> Self {
+        Self::empty(FullTag::new_sint(TagSelector::SInt64))
+    }
+    pub const fn float32() -> Self {
+        Self::empty(FullTag::new_float(TagSelector::Float32))
+    }
+    pub const fn float64() -> Self {
+        Self::empty(FullTag::new_float(TagSelector::Float64))
+    }
+    pub const fn bin() -> Self {
+        Self::empty(FullTag::BIN)
+    }
+    pub const fn str() -> Self {
+        Self::empty(FullTag::STR)
+    }
+    pub const fn list() -> Self {
+        Self::empty(FullTag::LIST)
+    }
+}
+
+impl Layer {
+    const fn new(tag: FullTag, config: [usize; 2]) -> Self {
+        Self { tag, config }
+    }
+    const fn empty(tag: FullTag) -> Self {
+        Self::new(tag, [0; 2])
+    }
+    fn hf(key: &[u8], v: [u8; 7]) -> u16 {
+        let mut tot = 0;
+        let mut i = 0;
+        while i < key.len() {
+            tot += v[i % v.len()] as u16 * key[i] as u16;
+            i += 1;
+        }
+        tot % 15
+    }
+    fn pf(key: &[u8]) -> u16 {
+        (G[Self::hf(key, S1) as usize] as u16 + G[Self::hf(key, S2) as usize] as u16) % 15
+    }
+    fn get_layer(ident: &str) -> Option<Self> {
+        let idx = Self::pf(ident.as_bytes()) as usize;
+        if idx < LUT.len() && LUT[idx].0 == ident {
+            Some(Self::empty(LUT[idx].1))
+        } else {
+            None
+        }
+    }
+}
diff --git a/server/src/engine/core/tests/mod.rs b/server/src/engine/core/tests/mod.rs
index af49ff11..63b43696 100644
--- a/server/src/engine/core/tests/mod.rs
+++ b/server/src/engine/core/tests/mod.rs
@@ -27,3 +27,5 @@
 // ddl
 // space
 mod space;
+// model
+mod model;
diff --git a/server/src/engine/core/tests/model/mod.rs b/server/src/engine/core/tests/model/mod.rs
new file mode 100644
index 00000000..019ae3ca
--- /dev/null
+++ b/server/src/engine/core/tests/model/mod.rs
@@ -0,0 +1,69 @@
+/*
+ * Created on Thu Mar 02 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod layer_validation {
+    use crate::engine::{
+        core::model::{Layer, LayerView},
+        error::DatabaseError,
+        ql::{ast::parse_ast_node_multiple_full, tests::lex_insecure as lex},
+    };
+
+    #[test]
+    fn string() {
+        let tok = lex(b"string").unwrap();
+        let spec = parse_ast_node_multiple_full(&tok).unwrap();
+        let view = LayerView::parse_layers(spec).unwrap();
+        assert_eq!(view.layers(), [Layer::str()]);
+    }
+
+    #[test]
+    fn nested_list() {
+        let tok = lex(b"list { type: list { type: string } }").unwrap();
+        let spec = parse_ast_node_multiple_full(&tok).unwrap();
+        let view = LayerView::parse_layers(spec).unwrap();
+        assert_eq!(view.layers(), [Layer::list(), Layer::list(), Layer::str()]);
+    }
+
+    #[test]
+    fn invalid_list() {
+        let tok = lex(b"list").unwrap();
+        let spec = parse_ast_node_multiple_full(&tok).unwrap();
+        assert_eq!(
+            LayerView::parse_layers(spec).unwrap_err(),
+            DatabaseError::DdlModelInvalidTypeDefinition
+        );
+    }
+
+    #[test]
+    fn invalid_flat() {
+        let tok = lex(b"string { type: string }").unwrap();
+        let spec = parse_ast_node_multiple_full(&tok).unwrap();
+        assert_eq!(
+            LayerView::parse_layers(spec).unwrap_err(),
+            DatabaseError::DdlModelInvalidTypeDefinition
+        );
+    }
+}
diff --git a/server/src/engine/data/spec.rs b/server/src/engine/data/spec.rs
index 94202565..113e5cf3 100644
--- a/server/src/engine/data/spec.rs
+++ b/server/src/engine/data/spec.rs
@@ -117,7 +117,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // bool
     /// Load a bool (this is unsafe for logical verity)
     unsafe fn read_bool_uck(&self) -> bool {
-        self.data().load::<bool>()
+        self.data().ld()
     }
     /// Load a bool
     fn read_bool_try(&self) -> Option<bool> {
@@ -135,7 +135,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // uint
     /// Load a uint (this is unsafe for logical verity)
     unsafe fn read_uint_uck(&self) -> u64 {
-        self.data().load::<u64>()
+        self.data().ld()
     }
     /// Load a uint
     fn read_uint_try(&self) -> Option<u64> {
@@ -153,7 +153,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // sint
     /// Load a sint (unsafe for logical verity)
     unsafe fn read_sint_uck(&self) -> i64 {
-        self.data().load::<i64>()
+        self.data().ld()
     }
     /// Load a sint
     fn read_sint_try(&self) -> Option<i64> {
@@ -169,7 +169,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // float
     /// Load a float (unsafe for logical verity)
     unsafe fn read_float_uck(&self) -> f64 {
-        self.data().load::<f64>()
+        self.data().ld()
     }
     /// Load a float
     fn read_float_try(&self) -> Option<f64> {
@@ -187,7 +187,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     /// ## Safety
     /// Are you a binary? Did you store it correctly? Are you a victim of segfaults?
     unsafe fn read_bin_uck(&self) -> &[u8] {
-        let (p, l) = self.data().load::<(*const u8, usize)>();
+        let (p, l) = self.data().ld();
         slice::from_raw_parts(p, l)
     }
     /// Load a bin
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
index a68d21ab..a74973dd 100644
--- a/server/src/engine/data/tag.rs
+++ b/server/src/engine/data/tag.rs
@@ -99,6 +99,15 @@ impl FullTag {
             unique,
         }
     }
+    pub const fn new_uint(selector: TagSelector) -> Self {
+        Self::new(TagClass::UnsignedInt, selector, TagUnique::UnsignedInt)
+    }
+    pub const fn new_sint(selector: TagSelector) -> Self {
+        Self::new(TagClass::SignedInt, selector, TagUnique::SignedInt)
+    }
+    pub const fn new_float(selector: TagSelector) -> Self {
+        Self::new(TagClass::Float, selector, TagUnique::Illegal)
+    }
 }
 
 macro_rules! fulltag {
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index eee9f372..e763cf33 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -88,4 +88,6 @@ pub enum DatabaseError {
     DdlSpaceAlreadyExists,
     /// the space doesn't exist
     DdlSpaceNotFound,
+    /// bad definition for some typedef in a model
+    DdlModelInvalidTypeDefinition,
 }
diff --git a/server/src/engine/mem/word.rs b/server/src/engine/mem/word.rs
index 5a6f7f3b..090d1ecf 100644
--- a/server/src/engine/mem/word.rs
+++ b/server/src/engine/mem/word.rs
@@ -36,12 +36,18 @@ pub trait SystemQword: SystemTword {
     {
         WordRW::store(v)
     }
-    fn load<'a, T>(&'a self) -> T::Target<'a>
+    fn ld_special<'a, T>(&'a self) -> T::Target<'a>
     where
         T: WordRW<Self>,
     {
         <T>::load(self)
     }
+    fn ld<'a, T>(&'a self) -> T
+    where
+        T: WordRW<Self, Target<'a> = T>,
+    {
+        <T>::load(self)
+    }
 }
 
 /// Native tripe pointer stack (must also be usable as a double pointer stack, see [`SystemDword`])
@@ -54,12 +60,18 @@ pub trait SystemTword: SystemDword {
     {
         WordRW::store(v)
     }
-    fn load<'a, T>(&'a self) -> T::Target<'a>
+    fn ld_special<'a, T>(&'a self) -> T::Target<'a>
     where
         T: WordRW<Self>,
     {
         <T>::load(self)
     }
+    fn ld<'a, T>(&'a self) -> T
+    where
+        T: WordRW<Self, Target<'a> = T>,
+    {
+        <T>::load(self)
+    }
 }
 
 /// Native double pointer stack
@@ -74,12 +86,18 @@ pub trait SystemDword: Sized {
     {
         WordRW::store(v)
     }
-    fn load<'a, T>(&'a self) -> T::Target<'a>
+    fn ld_special<'a, T>(&'a self) -> T::Target<'a>
     where
         T: WordRW<Self>,
     {
         <T>::load(self)
     }
+    fn ld<'a, T>(&'a self) -> T
+    where
+        T: WordRW<Self, Target<'a> = T>,
+    {
+        <T>::load(self)
+    }
 }
 
 impl SystemDword for NativeDword {
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 27e443da..627f4659 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -217,12 +217,12 @@ pub(super) fn rfold_tymeta<'a, Qd: QueryData<'a>>(
 
 #[derive(Debug, PartialEq)]
 /// A layer contains a type and corresponding metadata
-pub struct Layer<'a> {
-    ty: Ident<'a>,
-    props: DictGeneric,
+pub struct LayerSpec<'a> {
+    pub(in crate::engine) ty: Ident<'a>,
+    pub(in crate::engine) props: DictGeneric,
 }
 
-impl<'a> Layer<'a> {
+impl<'a> LayerSpec<'a> {
     //// Create a new layer
     pub const fn new(ty: Ident<'a>, props: DictGeneric) -> Self {
         Self { ty, props }
@@ -243,7 +243,7 @@ states! {
 fn rfold_layers<'a, Qd: QueryData<'a>>(
     mut mstate: LayerFoldState,
     state: &mut State<'a, Qd>,
-    layers: &mut Vec<Layer<'a>>,
+    layers: &mut Vec<LayerSpec<'a>>,
 ) {
     let mut ty = MaybeInit::uninit();
     let mut props = Default::default();
@@ -296,7 +296,7 @@ fn rfold_layers<'a, Qd: QueryData<'a>>(
     }
     if ((mstate == LayerFoldState::FINAL) | (mstate == LayerFoldState::FINAL_OR_OB)) & state.okay()
     {
-        layers.push(Layer {
+        layers.push(LayerSpec {
             ty: unsafe { ty.take() },
             props,
         });
@@ -311,7 +311,7 @@ pub struct Field<'a> {
     /// the field name
     field_name: Ident<'a>,
     /// layers
-    layers: Vec<Layer<'a>>,
+    layers: Vec<LayerSpec<'a>>,
     /// is null
     null: bool,
     /// is primary
@@ -319,7 +319,12 @@ pub struct Field<'a> {
 }
 
 impl<'a> Field<'a> {
-    pub fn new(field_name: Ident<'a>, layers: Vec<Layer<'a>>, null: bool, primary: bool) -> Self {
+    pub fn new(
+        field_name: Ident<'a>,
+        layers: Vec<LayerSpec<'a>>,
+        null: bool,
+        primary: bool,
+    ) -> Self {
         Self {
             field_name,
             layers,
@@ -364,12 +369,12 @@ impl<'a> Field<'a> {
 /// An [`ExpandedField`] is a full field definition with advanced metadata
 pub struct ExpandedField<'a> {
     field_name: Ident<'a>,
-    layers: Vec<Layer<'a>>,
+    layers: Vec<LayerSpec<'a>>,
     props: DictGeneric,
 }
 
 impl<'a> ExpandedField<'a> {
-    pub fn new(field_name: Ident<'a>, layers: Vec<Layer<'a>>, props: DictGeneric) -> Self {
+    pub fn new(field_name: Ident<'a>, layers: Vec<LayerSpec<'a>>, props: DictGeneric) -> Self {
         Self {
             field_name,
             layers,
@@ -481,7 +486,7 @@ mod impls {
     use {
         super::{
             rfold_dict, rfold_layers, rfold_tymeta, DictFoldState, DictGeneric, ExpandedField,
-            Field, Layer, LayerFoldState,
+            Field, LayerFoldState, LayerSpec,
         },
         crate::engine::{
             error::LangResult,
@@ -498,7 +503,7 @@ mod impls {
             Self::parse_multiple(state).map(Vec::from)
         }
     }
-    impl<'a> ASTNode<'a> for Layer<'a> {
+    impl<'a> ASTNode<'a> for LayerSpec<'a> {
         // important: upstream must verify this
         const VERIFY: bool = true;
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 943503e7..079571c9 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -190,23 +190,23 @@ mod tymeta {
 }
 mod layer {
     use super::*;
-    use crate::engine::ql::{ast::parse_ast_node_multiple_full, ddl::syn::Layer};
+    use crate::engine::ql::{ast::parse_ast_node_multiple_full, ddl::syn::LayerSpec};
     #[test]
     fn layer_mini() {
         let tok = lex_insecure(b"string").unwrap();
-        let layers = parse_ast_node_multiple_full::<Layer>(&tok).unwrap();
+        let layers = parse_ast_node_multiple_full::<LayerSpec>(&tok).unwrap();
         assert_eq!(
             layers,
-            vec![Layer::new(Ident::from("string"), null_dict! {})]
+            vec![LayerSpec::new(Ident::from("string"), null_dict! {})]
         );
     }
     #[test]
     fn layer() {
         let tok = lex_insecure(b"string { maxlen: 100 }").unwrap();
-        let layers = parse_ast_node_multiple_full::<Layer>(&tok).unwrap();
+        let layers = parse_ast_node_multiple_full::<LayerSpec>(&tok).unwrap();
         assert_eq!(
             layers,
-            vec![Layer::new(
+            vec![LayerSpec::new(
                 Ident::from("string"),
                 null_dict! {
                     "maxlen" => Lit::UnsignedInt(100)
@@ -217,24 +217,24 @@ mod layer {
     #[test]
     fn layer_plus() {
         let tok = lex_insecure(b"list { type: string }").unwrap();
-        let layers = parse_ast_node_multiple_full::<Layer>(&tok).unwrap();
+        let layers = parse_ast_node_multiple_full::<LayerSpec>(&tok).unwrap();
         assert_eq!(
             layers,
             vec![
-                Layer::new(Ident::from("string"), null_dict! {}),
-                Layer::new(Ident::from("list"), null_dict! {})
+                LayerSpec::new(Ident::from("string"), null_dict! {}),
+                LayerSpec::new(Ident::from("list"), null_dict! {})
             ]
         );
     }
     #[test]
     fn layer_pro() {
         let tok = lex_insecure(b"list { unique: true, type: string, maxlen: 10 }").unwrap();
-        let layers = parse_ast_node_multiple_full::<Layer>(&tok).unwrap();
+        let layers = parse_ast_node_multiple_full::<LayerSpec>(&tok).unwrap();
         assert_eq!(
             layers,
             vec![
-                Layer::new(Ident::from("string"), null_dict! {}),
-                Layer::new(
+                LayerSpec::new(Ident::from("string"), null_dict! {}),
+                LayerSpec::new(
                     Ident::from("list"),
                     null_dict! {
                         "unique" => Lit::Bool(true),
@@ -250,18 +250,18 @@ mod layer {
             b"list { unique: true, type: string { ascii_only: true, maxlen: 255 }, maxlen: 10 }",
         )
         .unwrap();
-        let layers = parse_ast_node_multiple_full::<Layer>(&tok).unwrap();
+        let layers = parse_ast_node_multiple_full::<LayerSpec>(&tok).unwrap();
         assert_eq!(
             layers,
             vec![
-                Layer::new(
+                LayerSpec::new(
                     Ident::from("string"),
                     null_dict! {
                         "ascii_only" => Lit::Bool(true),
                         "maxlen" => Lit::UnsignedInt(255)
                     }
                 ),
-                Layer::new(
+                LayerSpec::new(
                     Ident::from("list"),
                     null_dict! {
                         "unique" => Lit::Bool(true),
@@ -285,17 +285,17 @@ mod layer {
             }
         ";
         let expected = vec![
-            Layer::new(Ident::from("string"), null_dict!()),
-            Layer::new(
+            LayerSpec::new(Ident::from("string"), null_dict!()),
+            LayerSpec::new(
                 Ident::from("list"),
                 null_dict! {
                     "maxlen" => Lit::UnsignedInt(100),
                 },
             ),
-            Layer::new(Ident::from("list"), null_dict!("unique" => Lit::Bool(true))),
+            LayerSpec::new(Ident::from("list"), null_dict!("unique" => Lit::Bool(true))),
         ];
         fuzz_tokens(tok.as_slice(), |should_pass, new_tok| {
-            let layers = parse_ast_node_multiple_full::<Layer>(&new_tok);
+            let layers = parse_ast_node_multiple_full::<LayerSpec>(&new_tok);
             let ok = layers.is_ok();
             if should_pass {
                 assert_eq!(layers.unwrap(), expected);
@@ -309,7 +309,7 @@ mod fields {
         super::*,
         crate::engine::ql::{
             ast::parse_ast_node_full,
-            ddl::syn::{Field, Layer},
+            ddl::syn::{Field, LayerSpec},
             lex::Ident,
         },
     };
@@ -321,7 +321,7 @@ mod fields {
             f,
             Field::new(
                 Ident::from("username"),
-                [Layer::new(Ident::from("string"), null_dict! {})].into(),
+                [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                 false,
                 false
             )
@@ -335,7 +335,7 @@ mod fields {
             f,
             Field::new(
                 Ident::from("username"),
-                [Layer::new(Ident::from("string"), null_dict! {})].into(),
+                [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                 false,
                 true
             )
@@ -357,7 +357,7 @@ mod fields {
             f,
             Field::new(
                 Ident::from("username"),
-                [Layer::new(
+                [LayerSpec::new(
                     Ident::from("string"),
                     null_dict! {
                         "maxlen" => Lit::UnsignedInt(10),
@@ -390,14 +390,14 @@ mod fields {
             Field::new(
                 Ident::from("notes"),
                 [
-                    Layer::new(
+                    LayerSpec::new(
                         Ident::from("string"),
                         null_dict! {
                             "maxlen" => Lit::UnsignedInt(255),
                             "ascii_only" => Lit::Bool(true),
                         }
                     ),
-                    Layer::new(
+                    LayerSpec::new(
                         Ident::from("list"),
                         null_dict! {
                             "unique" => Lit::Bool(true)
@@ -417,7 +417,7 @@ mod schemas {
         ast::parse_ast_node_full,
         ddl::{
             crt::CreateModel,
-            syn::{Field, Layer},
+            syn::{Field, LayerSpec},
         },
     };
     #[test]
@@ -443,13 +443,13 @@ mod schemas {
                 vec![
                     Field::new(
                         Ident::from("username"),
-                        vec![Layer::new(Ident::from("string"), null_dict! {})],
+                        vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
                         false,
                         true,
                     ),
                     Field::new(
                         Ident::from("password"),
-                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
+                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         false,
                         false,
                     )
@@ -482,19 +482,19 @@ mod schemas {
                 vec![
                     Field::new(
                         Ident::from("username"),
-                        vec![Layer::new(Ident::from("string"), null_dict! {})],
+                        vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
                         false,
                         true,
                     ),
                     Field::new(
                         Ident::from("password"),
-                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
+                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         false,
                         false,
                     ),
                     Field::new(
                         Ident::from("profile_pic"),
-                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
+                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         true,
                         false,
                     )
@@ -532,27 +532,27 @@ mod schemas {
                 vec![
                     Field::new(
                         Ident::from("username"),
-                        vec![Layer::new(Ident::from("string"), null_dict! {})],
+                        vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
                         false,
                         true
                     ),
                     Field::new(
                         Ident::from("password"),
-                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
+                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         false,
                         false
                     ),
                     Field::new(
                         Ident::from("profile_pic"),
-                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
+                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         true,
                         false
                     ),
                     Field::new(
                         Ident::from("notes"),
                         vec![
-                            Layer::new(Ident::from("string"), null_dict! {}),
-                            Layer::new(
+                            LayerSpec::new(Ident::from("string"), null_dict! {}),
+                            LayerSpec::new(
                                 Ident::from("list"),
                                 null_dict! {
                                     "unique" => Lit::Bool(true)
@@ -601,27 +601,27 @@ mod schemas {
                 vec![
                     Field::new(
                         Ident::from("username"),
-                        vec![Layer::new(Ident::from("string"), null_dict! {})],
+                        vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
                         false,
                         true
                     ),
                     Field::new(
                         Ident::from("password"),
-                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
+                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         false,
                         false
                     ),
                     Field::new(
                         Ident::from("profile_pic"),
-                        vec![Layer::new(Ident::from("binary"), null_dict! {})],
+                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         true,
                         false
                     ),
                     Field::new(
                         Ident::from("notes"),
                         vec![
-                            Layer::new(Ident::from("string"), null_dict! {}),
-                            Layer::new(
+                            LayerSpec::new(Ident::from("string"), null_dict! {}),
+                            LayerSpec::new(
                                 Ident::from("list"),
                                 null_dict! {
                                     "unique" => Lit::Bool(true)
@@ -646,7 +646,7 @@ mod dict_field_syntax {
     use super::*;
     use crate::engine::ql::{
         ast::parse_ast_node_full,
-        ddl::syn::{ExpandedField, Layer},
+        ddl::syn::{ExpandedField, LayerSpec},
     };
     #[test]
     fn field_syn_mini() {
@@ -656,7 +656,7 @@ mod dict_field_syntax {
             ef,
             ExpandedField::new(
                 Ident::from("username"),
-                vec![Layer::new(Ident::from("string"), null_dict! {})],
+                vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
                 null_dict! {}
             )
         )
@@ -677,7 +677,7 @@ mod dict_field_syntax {
             ef,
             ExpandedField::new(
                 Ident::from("username"),
-                vec![Layer::new(Ident::from("string"), null_dict! {})],
+                vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
                 null_dict! {
                     "nullable" => Lit::Bool(false),
                 },
@@ -704,7 +704,7 @@ mod dict_field_syntax {
             ef,
             ExpandedField::new(
                 Ident::from("username"),
-                vec![Layer::new(
+                vec![LayerSpec::new(
                     Ident::from("string"),
                     null_dict! {
                         "minlen" => Lit::UnsignedInt(6),
@@ -741,13 +741,13 @@ mod dict_field_syntax {
             ExpandedField::new(
                 Ident::from("notes"),
                 vec![
-                    Layer::new(
+                    LayerSpec::new(
                         Ident::from("string"),
                         null_dict! {
                             "ascii_only" => Lit::Bool(true),
                         }
                     ),
-                    Layer::new(
+                    LayerSpec::new(
                         Ident::from("list"),
                         null_dict! {
                             "unique" => Lit::Bool(true),
@@ -819,7 +819,7 @@ mod alter_model_add {
         ast::parse_ast_node_full,
         ddl::{
             alt::{AlterKind, AlterModel},
-            syn::{ExpandedField, Layer},
+            syn::{ExpandedField, LayerSpec},
         },
     };
     #[test]
@@ -837,7 +837,7 @@ mod alter_model_add {
                 AlterKind::Add(
                     [ExpandedField::new(
                         Ident::from("myfield"),
-                        [Layer::new(Ident::from("string"), null_dict! {})].into(),
+                        [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {},
                     )]
                     .into()
@@ -861,7 +861,7 @@ mod alter_model_add {
                 AlterKind::Add(
                     [ExpandedField::new(
                         Ident::from("myfield"),
-                        [Layer::new(Ident::from("string"), null_dict! {})].into(),
+                        [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {
                             "nullable" => Lit::Bool(true)
                         },
@@ -887,7 +887,7 @@ mod alter_model_add {
                 AlterKind::Add(
                     [ExpandedField::new(
                         Ident::from("myfield"),
-                        [Layer::new(Ident::from("string"), null_dict! {})].into(),
+                        [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {
                             "nullable" => Lit::Bool(true)
                         },
@@ -928,7 +928,7 @@ mod alter_model_add {
                     [
                         ExpandedField::new(
                             Ident::from("myfield"),
-                            [Layer::new(Ident::from("string"), null_dict! {})].into(),
+                            [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                             null_dict! {
                                 "nullable" => Lit::Bool(true)
                             },
@@ -936,13 +936,13 @@ mod alter_model_add {
                         ExpandedField::new(
                             Ident::from("another"),
                             [
-                                Layer::new(
+                                LayerSpec::new(
                                     Ident::from("string"),
                                     null_dict! {
                                         "maxlen" => Lit::UnsignedInt(255)
                                     }
                                 ),
-                                Layer::new(
+                                LayerSpec::new(
                                     Ident::from("list"),
                                     null_dict! {
                                        "unique" => Lit::Bool(true)
@@ -967,7 +967,7 @@ mod alter_model_update {
         ast::parse_ast_node_full,
         ddl::{
             alt::{AlterKind, AlterModel},
-            syn::{ExpandedField, Layer},
+            syn::{ExpandedField, LayerSpec},
         },
     };
 
@@ -987,7 +987,7 @@ mod alter_model_update {
                 AlterKind::Update(
                     [ExpandedField::new(
                         Ident::from("myfield"),
-                        [Layer::new(Ident::from("string"), null_dict! {})].into(),
+                        [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {},
                     )]
                     .into()
@@ -1011,7 +1011,7 @@ mod alter_model_update {
                 AlterKind::Update(
                     [ExpandedField::new(
                         Ident::from("myfield"),
-                        [Layer::new(Ident::from("string"), null_dict! {})].into(),
+                        [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {},
                     )]
                     .into()
@@ -1040,7 +1040,7 @@ mod alter_model_update {
                 AlterKind::Update(
                     [ExpandedField::new(
                         Ident::from("myfield"),
-                        [Layer::new(Ident::from("string"), null_dict! {})].into(),
+                        [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {
                             "nullable" => Lit::Bool(true)
                         },
@@ -1075,14 +1075,14 @@ mod alter_model_update {
                     [
                         ExpandedField::new(
                             Ident::from("myfield"),
-                            [Layer::new(Ident::from("string"), null_dict! {})].into(),
+                            [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                             null_dict! {
                                 "nullable" => Lit::Bool(true)
                             },
                         ),
                         ExpandedField::new(
                             Ident::from("myfield2"),
-                            [Layer::new(Ident::from("string"), null_dict! {})].into(),
+                            [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                             null_dict! {},
                         )
                     ]
@@ -1118,14 +1118,14 @@ mod alter_model_update {
                     [
                         ExpandedField::new(
                             Ident::from("myfield"),
-                            [Layer::new(Ident::from("string"), null_dict! {})].into(),
+                            [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                             null_dict! {
                                 "nullable" => Lit::Bool(true)
                             },
                         ),
                         ExpandedField::new(
                             Ident::from("myfield2"),
-                            [Layer::new(
+                            [LayerSpec::new(
                                 Ident::from("string"),
                                 null_dict! {"maxlen" => Lit::UnsignedInt(255)}
                             )]

From 1c9dfbaebe2fd385ac12c3c51c1cc1fb7f3e3189 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 2 Mar 2023 10:08:39 -0800
Subject: [PATCH 148/310] Implement layered data validation

Also fixed stacked borrows
---
 server/src/engine/core/model/cell.rs        |  12 +-
 server/src/engine/core/model/mod.rs         | 114 ++++++++++-
 server/src/engine/core/tests/model/layer.rs | 200 ++++++++++++++++++++
 server/src/engine/core/tests/model/mod.rs   |  44 +----
 server/src/engine/data/tag.rs               |   4 +-
 server/src/engine/mem/word.rs               |   6 +-
 server/src/storage/v1/tests.rs              |   2 +-
 server/src/util/macros.rs                   |  28 ++-
 8 files changed, 347 insertions(+), 63 deletions(-)
 create mode 100644 server/src/engine/core/tests/model/layer.rs

diff --git a/server/src/engine/core/model/cell.rs b/server/src/engine/core/model/cell.rs
index f995f4c4..b3272278 100644
--- a/server/src/engine/core/model/cell.rs
+++ b/server/src/engine/core/model/cell.rs
@@ -26,8 +26,6 @@
 
 #[cfg(test)]
 use core::mem;
-
-use crate::engine::mem::WordRW;
 use {
     crate::engine::{
         self,
@@ -36,7 +34,7 @@ use {
             spec::{Dataspec1D, DataspecMeta1D},
             tag::{DataTag, TagClass},
         },
-        mem::{NativeQword, SystemDword},
+        mem::{NativeQword, SystemDword, WordRW},
     },
     core::{fmt, mem::ManuallyDrop, slice, str},
     parking_lot::RwLock,
@@ -102,11 +100,11 @@ impl Datacell {
     }
     // bin
     pub fn new_bin(s: Box<[u8]>) -> Self {
-        let md = ManuallyDrop::new(s);
+        let mut md = ManuallyDrop::new(s);
         unsafe {
             Self::new(
                 TagClass::Bin,
-                DataRaw::word(SystemDword::store((md.as_ptr(), md.len()))),
+                DataRaw::word(SystemDword::store((md.as_mut_ptr(), md.len()))),
             )
         }
     }
@@ -122,11 +120,11 @@ impl Datacell {
     }
     // str
     pub fn new_str(s: Box<str>) -> Self {
-        let md = ManuallyDrop::new(s.into_boxed_bytes());
+        let mut md = ManuallyDrop::new(s.into_boxed_bytes());
         unsafe {
             Self::new(
                 TagClass::Str,
-                DataRaw::word(SystemDword::store((md.as_ptr(), md.len()))),
+                DataRaw::word(SystemDword::store((md.as_mut_ptr(), md.len()))),
             )
         }
     }
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 90109ad9..b523c684 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -27,10 +27,14 @@
 pub mod cell;
 
 use crate::engine::{
-    data::tag::{DataTag, FullTag, TagSelector},
+    core::model::cell::Datacell,
+    data::tag::{DataTag, FullTag, TagClass, TagSelector},
     error::{DatabaseError, DatabaseResult},
+    mem::VInline,
     ql::ddl::syn::LayerSpec,
 };
+#[cfg(test)]
+use std::cell::RefCell;
 
 // FIXME(@ohsayan): update this!
 
@@ -70,7 +74,7 @@ static LUT: [(&str, FullTag); 14] = [
 ];
 
 #[derive(Debug, PartialEq, Clone)]
-pub struct LayerView(Box<[Layer]>);
+pub struct LayerView(VInline<1, Layer>);
 
 impl LayerView {
     pub fn layers(&self) -> &[Layer] {
@@ -80,7 +84,7 @@ impl LayerView {
         let mut layers = spec.into_iter().rev();
         let mut okay = true;
         let mut fin = false;
-        let mut layerview = Vec::with_capacity(layers.len());
+        let mut layerview = VInline::new();
         while (layers.len() != 0) & okay & !fin {
             let LayerSpec { ty, props } = layers.next().unwrap();
             okay &= props.is_empty(); // FIXME(@ohsayan): you know what to do here
@@ -94,11 +98,41 @@ impl LayerView {
         }
         okay &= fin & (layers.len() == 0);
         if okay {
-            Ok(Self(layerview.into_boxed_slice()))
+            Ok(Self(layerview))
         } else {
             Err(DatabaseError::DdlModelInvalidTypeDefinition)
         }
     }
+    pub fn validate_data_fpath(&self, data: &Datacell) -> bool {
+        if (self.layers().len() == 0) & (self.layers()[0].tag.tag_class() == data.kind()) {
+            // if someone sends a PR with an added check, I'll come home and throw a brick on your head
+            let l = self.layers()[0];
+            unsafe { LVERIFY[l.tag.tag_selector().word()](l, data) }
+        } else {
+            Self::rverify_layers(self.layers(), data)
+        }
+    }
+    // TODO(@ohsayan): improve algo with dfs
+    fn rverify_layers(layers: &[Layer], data: &Datacell) -> bool {
+        let layer = layers[0];
+        let layers = &layers[1..];
+        match (layer.tag.tag_class(), data.kind()) {
+            (layer_tag, data_tag) if (layer_tag == data_tag) & (layer_tag < TagClass::List) => {
+                // time to go home
+                (unsafe { LVERIFY[layer.tag.tag_class().word()](layer, data) } & layers.is_empty())
+            }
+            (TagClass::List, TagClass::List) => unsafe {
+                let mut okay = !layers.is_empty() & LVERIFY[TagClass::List.word()](layer, data);
+                let list = data.read_list().read();
+                let mut it = list.iter();
+                while (it.len() != 0) & okay {
+                    okay &= Self::rverify_layers(layers, it.next().unwrap());
+                }
+                okay
+            },
+            _ => false,
+        }
+    }
 }
 
 #[derive(Debug, PartialEq, Clone, Copy)]
@@ -180,3 +214,75 @@ impl Layer {
         }
     }
 }
+
+static LVERIFY: [unsafe fn(Layer, &Datacell) -> bool; 7] = [
+    lverify_bool,
+    lverify_uint,
+    lverify_sint,
+    lverify_float,
+    lverify_bin,
+    lverify_str,
+    lverify_list,
+];
+
+#[cfg(test)]
+thread_local! {
+    pub static LAYER_TRACE: RefCell<Vec<Box<str>>> = RefCell::new(Vec::new());
+}
+
+#[inline(always)]
+fn layertrace(_layer: impl ToString) {
+    #[cfg(test)]
+    {
+        LAYER_TRACE.with(|v| v.borrow_mut().push(_layer.to_string().into()));
+    }
+}
+
+#[cfg(test)]
+/// Obtain a layer trace and clear older traces
+pub(super) fn layer_traces() -> Box<[Box<str>]> {
+    LAYER_TRACE.with(|x| {
+        let ret = x.borrow().iter().cloned().collect();
+        x.borrow_mut().clear();
+        ret
+    })
+}
+
+unsafe fn lverify_bool(_: Layer, _: &Datacell) -> bool {
+    layertrace("bool");
+    true
+}
+unsafe fn lverify_uint(l: Layer, d: &Datacell) -> bool {
+    layertrace("uint");
+    const MX: [u64; 4] = [u8::MAX as _, u16::MAX as _, u32::MAX as _, u64::MAX];
+    d.read_uint() <= MX[l.tag.tag_selector().word() - 1]
+}
+unsafe fn lverify_sint(l: Layer, d: &Datacell) -> bool {
+    layertrace("sint");
+    const MN_MX: [(i64, i64); 4] = [
+        (i8::MIN as _, i8::MAX as _),
+        (i16::MIN as _, i16::MAX as _),
+        (i32::MIN as _, i32::MAX as _),
+        (i64::MIN, i64::MAX),
+    ];
+    let (mn, mx) = MN_MX[l.tag.tag_selector().word() - 5];
+    (d.read_sint() >= mn) & (d.read_sint() <= mx)
+}
+unsafe fn lverify_float(l: Layer, d: &Datacell) -> bool {
+    layertrace("float");
+    const MN_MX: [(f64, f64); 2] = [(f32::MIN as _, f32::MAX as _), (f64::MIN, f64::MAX)];
+    let (mn, mx) = MN_MX[l.tag.tag_selector().word() - 9];
+    (d.read_float() >= mn) & (d.read_float() <= mx)
+}
+unsafe fn lverify_bin(_: Layer, _: &Datacell) -> bool {
+    layertrace("binary");
+    true
+}
+unsafe fn lverify_str(_: Layer, _: &Datacell) -> bool {
+    layertrace("string");
+    true
+}
+unsafe fn lverify_list(_: Layer, _: &Datacell) -> bool {
+    layertrace("list");
+    true
+}
diff --git a/server/src/engine/core/tests/model/layer.rs b/server/src/engine/core/tests/model/layer.rs
new file mode 100644
index 00000000..b31ee03c
--- /dev/null
+++ b/server/src/engine/core/tests/model/layer.rs
@@ -0,0 +1,200 @@
+/*
+ * Created on Thu Mar 02 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    core::model::LayerView,
+    error::DatabaseResult,
+    ql::{ast::parse_ast_node_multiple_full, tests::lex_insecure},
+};
+
+fn layerview(layer_def: &str) -> DatabaseResult<LayerView> {
+    let tok = lex_insecure(layer_def.as_bytes()).unwrap();
+    let spec = parse_ast_node_multiple_full(&tok).unwrap();
+    LayerView::parse_layers(spec)
+}
+
+mod layer_spec_validation {
+    use {
+        super::layerview,
+        crate::engine::{core::model::Layer, error::DatabaseError},
+    };
+
+    #[test]
+    fn string() {
+        assert_eq!(layerview("string").unwrap().layers(), [Layer::str()]);
+    }
+
+    #[test]
+    fn nested_list() {
+        assert_eq!(
+            layerview("list { type: list { type: string } }")
+                .unwrap()
+                .layers(),
+            [Layer::list(), Layer::list(), Layer::str()]
+        );
+    }
+
+    #[test]
+    fn invalid_list() {
+        assert_eq!(
+            layerview("list").unwrap_err(),
+            DatabaseError::DdlModelInvalidTypeDefinition
+        );
+    }
+
+    #[test]
+    fn invalid_flat() {
+        assert_eq!(
+            layerview("string { type: string }").unwrap_err(),
+            DatabaseError::DdlModelInvalidTypeDefinition
+        );
+    }
+}
+
+mod layer_data_validation {
+    use {
+        super::layerview,
+        crate::engine::core::model::{self, cell::Datacell},
+    };
+    #[test]
+    fn bool() {
+        let dc = Datacell::new_bool(true);
+        let layer = layerview("bool").unwrap();
+        assert!(layer.validate_data_fpath(&dc));
+        assert_vecstreq_exact!(model::layer_traces(), ["bool"]);
+    }
+    #[test]
+    fn uint() {
+        let targets = [
+            ("uint8", u8::MAX as u64),
+            ("uint16", u16::MAX as _),
+            ("uint32", u32::MAX as _),
+            ("uint64", u64::MAX),
+        ];
+        targets
+            .into_iter()
+            .enumerate()
+            .for_each(|(i, (layer, max))| {
+                let this_layer = layerview(layer).unwrap();
+                let dc = Datacell::new_uint(max);
+                assert!(this_layer.validate_data_fpath(&dc), "{:?}", this_layer);
+                assert_vecstreq_exact!(model::layer_traces(), ["uint"]);
+                for (lower, _) in targets[..i].iter() {
+                    let layer = layerview(lower).unwrap();
+                    assert!(!layer.validate_data_fpath(&dc), "{:?}", layer);
+                    assert_vecstreq_exact!(model::layer_traces(), ["uint"]);
+                }
+                for (higher, _) in targets[i + 1..].iter() {
+                    let layer = layerview(higher).unwrap();
+                    assert!(layer.validate_data_fpath(&dc), "{:?}", layer);
+                    assert_vecstreq_exact!(model::layer_traces(), ["uint"]);
+                }
+            });
+    }
+    #[test]
+    fn sint() {
+        let targets = [
+            ("sint8", (i8::MIN as i64, i8::MAX as i64)),
+            ("sint16", (i16::MIN as _, i16::MAX as _)),
+            ("sint32", (i32::MIN as _, i32::MAX as _)),
+            ("sint64", (i64::MIN, i64::MAX)),
+        ];
+        targets
+            .into_iter()
+            .enumerate()
+            .for_each(|(i, (layer, (min, max)))| {
+                let this_layer = layerview(layer).unwrap();
+                let dc_min = Datacell::new_sint(min);
+                let dc_max = Datacell::new_sint(max);
+                assert!(this_layer.validate_data_fpath(&dc_min), "{:?}", this_layer);
+                assert!(this_layer.validate_data_fpath(&dc_max), "{:?}", this_layer);
+                assert_vecstreq_exact!(model::layer_traces(), ["sint", "sint"]);
+                for (lower, _) in targets[..i].iter() {
+                    let layer = layerview(lower).unwrap();
+                    assert!(!layer.validate_data_fpath(&dc_min), "{:?}", layer);
+                    assert!(!layer.validate_data_fpath(&dc_max), "{:?}", layer);
+                    assert_vecstreq_exact!(model::layer_traces(), ["sint", "sint"]);
+                }
+                for (higher, _) in targets[i + 1..].iter() {
+                    let layer = layerview(higher).unwrap();
+                    assert!(layer.validate_data_fpath(&dc_min), "{:?}", layer);
+                    assert!(layer.validate_data_fpath(&dc_max), "{:?}", layer);
+                    assert_vecstreq_exact!(model::layer_traces(), ["sint", "sint"]);
+                }
+            });
+    }
+    #[test]
+    fn float() {
+        // l
+        let f32_l = layerview("float32").unwrap();
+        let f64_l = layerview("float64").unwrap();
+        // dc
+        let f32_dc_min = Datacell::new_float(f32::MIN as _);
+        let f32_dc_max = Datacell::new_float(f32::MAX as _);
+        let f64_dc_min = Datacell::new_float(f64::MIN as _);
+        let f64_dc_max = Datacell::new_float(f64::MAX as _);
+        // check (32)
+        assert!(f32_l.validate_data_fpath(&f32_dc_min));
+        assert!(f32_l.validate_data_fpath(&f32_dc_max));
+        assert_vecstreq_exact!(model::layer_traces(), ["float", "float"]);
+        assert!(f64_l.validate_data_fpath(&f32_dc_min));
+        assert!(f64_l.validate_data_fpath(&f32_dc_max));
+        assert_vecstreq_exact!(model::layer_traces(), ["float", "float"]);
+        // check (64)
+        assert!(!f32_l.validate_data_fpath(&f64_dc_min));
+        assert!(!f32_l.validate_data_fpath(&f64_dc_max));
+        assert_vecstreq_exact!(model::layer_traces(), ["float", "float"]);
+        assert!(f64_l.validate_data_fpath(&f64_dc_min));
+        assert!(f64_l.validate_data_fpath(&f64_dc_max));
+        assert_vecstreq_exact!(model::layer_traces(), ["float", "float"]);
+    }
+    #[test]
+    fn bin() {
+        let layer = layerview("binary").unwrap();
+        assert!(layer.validate_data_fpath(&Datacell::from("hello".as_bytes())));
+        assert_vecstreq_exact!(model::layer_traces(), ["binary"]);
+    }
+    #[test]
+    fn str() {
+        let layer = layerview("string").unwrap();
+        assert!(layer.validate_data_fpath(&Datacell::from("hello")));
+        assert_vecstreq_exact!(model::layer_traces(), ["string"]);
+    }
+    #[test]
+    fn list_simple() {
+        let layer = layerview("list { type: string }").unwrap();
+        let dc = Datacell::new_list(vec![
+            Datacell::from("I"),
+            Datacell::from("love"),
+            Datacell::from("cats"),
+        ]);
+        assert!(layer.validate_data_fpath(&dc));
+        assert_vecstreq_exact!(
+            model::layer_traces(),
+            ["list", "string", "string", "string"]
+        );
+    }
+}
diff --git a/server/src/engine/core/tests/model/mod.rs b/server/src/engine/core/tests/model/mod.rs
index 019ae3ca..e07ae0a6 100644
--- a/server/src/engine/core/tests/model/mod.rs
+++ b/server/src/engine/core/tests/model/mod.rs
@@ -24,46 +24,4 @@
  *
 */
 
-mod layer_validation {
-    use crate::engine::{
-        core::model::{Layer, LayerView},
-        error::DatabaseError,
-        ql::{ast::parse_ast_node_multiple_full, tests::lex_insecure as lex},
-    };
-
-    #[test]
-    fn string() {
-        let tok = lex(b"string").unwrap();
-        let spec = parse_ast_node_multiple_full(&tok).unwrap();
-        let view = LayerView::parse_layers(spec).unwrap();
-        assert_eq!(view.layers(), [Layer::str()]);
-    }
-
-    #[test]
-    fn nested_list() {
-        let tok = lex(b"list { type: list { type: string } }").unwrap();
-        let spec = parse_ast_node_multiple_full(&tok).unwrap();
-        let view = LayerView::parse_layers(spec).unwrap();
-        assert_eq!(view.layers(), [Layer::list(), Layer::list(), Layer::str()]);
-    }
-
-    #[test]
-    fn invalid_list() {
-        let tok = lex(b"list").unwrap();
-        let spec = parse_ast_node_multiple_full(&tok).unwrap();
-        assert_eq!(
-            LayerView::parse_layers(spec).unwrap_err(),
-            DatabaseError::DdlModelInvalidTypeDefinition
-        );
-    }
-
-    #[test]
-    fn invalid_flat() {
-        let tok = lex(b"string { type: string }").unwrap();
-        let spec = parse_ast_node_multiple_full(&tok).unwrap();
-        assert_eq!(
-            LayerView::parse_layers(spec).unwrap_err(),
-            DatabaseError::DdlModelInvalidTypeDefinition
-        );
-    }
-}
+mod layer;
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
index a74973dd..d3823410 100644
--- a/server/src/engine/data/tag.rs
+++ b/server/src/engine/data/tag.rs
@@ -66,7 +66,7 @@ pub enum TagUnique {
 }
 
 macro_rules! d {
-    ($($ty:ty),*) => {$(impl $ty { fn d(&self) -> u8 { unsafe { ::core::mem::transmute_copy(self) } } } )*}
+    ($($ty:ty),*) => {$(impl $ty { pub fn d(&self) -> u8 {unsafe{::core::mem::transmute_copy(self)}} pub fn word(&self) -> usize {Self::d(self) as usize} } )*}
 }
 
 d!(TagClass, TagSelector, TagUnique);
@@ -115,7 +115,7 @@ macro_rules! fulltag {
         FullTag::new(TagClass::$class, TagSelector::$selector, TagUnique::$unique)
     };
     ($class:ident, $selector:ident) => {
-        FullTag::new(TagClass::$class, TagSelector::$selector, TagUnique::Illegal)
+        fulltag!($class, $selector, Illegal)
     };
 }
 
diff --git a/server/src/engine/mem/word.rs b/server/src/engine/mem/word.rs
index 090d1ecf..ddd7ffaa 100644
--- a/server/src/engine/mem/word.rs
+++ b/server/src/engine/mem/word.rs
@@ -292,11 +292,11 @@ impl_wordrw! {
         |word| SystemDword::load_qw(word) as i64;
     }
     f32 as SystemDword => {
-        |self| SystemDword::store_qw(self as _);
-        |word| f64::from_bits(SystemDword::load_qw(word)) as _;
+        |self| SystemDword::store_qw(self.to_bits() as u64);
+        |word| f32::from_bits(SystemDword::load_qw(word) as u32);
     }
     f64 as SystemDword => {
-        |self| SystemDword::store_qw(self as _);
+        |self| SystemDword::store_qw(self.to_bits());
         |word| f64::from_bits(SystemDword::load_qw(word));
     }
     [usize; 2] as SystemDword => {
diff --git a/server/src/storage/v1/tests.rs b/server/src/storage/v1/tests.rs
index f4e854d5..ca306d12 100644
--- a/server/src/storage/v1/tests.rs
+++ b/server/src/storage/v1/tests.rs
@@ -209,7 +209,7 @@ mod preload_tests {
             .into_iter()
             .map(|each| unsafe { each.as_str().to_owned() })
             .collect();
-        assert_veceq!(de, vec!["default".to_owned(), "system".to_owned()]);
+        assert_veceq_transposed!(de, vec!["default".to_owned(), "system".to_owned()]);
     }
 }
 
diff --git a/server/src/util/macros.rs b/server/src/util/macros.rs
index 0079e3da..138b0691 100644
--- a/server/src/util/macros.rs
+++ b/server/src/util/macros.rs
@@ -78,16 +78,38 @@ macro_rules! cfg_test {
 
 #[macro_export]
 /// Compare two vectors irrespective of their elements' position
-macro_rules! veceq {
+macro_rules! veceq_transposed {
     ($v1:expr, $v2:expr) => {
         $v1.len() == $v2.len() && $v1.iter().all(|v| $v2.contains(v))
     };
 }
 
 #[macro_export]
-macro_rules! assert_veceq {
+macro_rules! assert_veceq_transposed {
     ($v1:expr, $v2:expr) => {
-        assert!(veceq!($v1, $v2))
+        assert!(veceq_transposed!($v1, $v2))
+    };
+}
+
+#[cfg(test)]
+macro_rules! vecstreq_exact {
+    ($v1:expr, $v2:expr) => {
+        $v1.iter()
+            .zip($v2.iter())
+            .all(|(a, b)| a.as_bytes() == b.as_bytes())
+    };
+}
+
+#[cfg(test)]
+macro_rules! assert_vecstreq_exact {
+    ($v1:expr, $v2:expr) => {
+        if !vecstreq_exact!($v1, $v2) {
+            ::core::panic!(
+                "failed to assert vector data equality. lhs: {:?}, rhs: {:?}",
+                $v1,
+                $v2
+            );
+        }
     };
 }
 

From b9453041e49e99042d009cc9009cdfbb5ba82b25 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 2 Mar 2023 21:10:13 -0800
Subject: [PATCH 149/310] Trace fpath in layers

---
 server/src/engine/core/model/mod.rs         |  5 ++--
 server/src/engine/core/tests/model/layer.rs | 32 ++++++++++++---------
 2 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index b523c684..92455755 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -104,10 +104,11 @@ impl LayerView {
         }
     }
     pub fn validate_data_fpath(&self, data: &Datacell) -> bool {
-        if (self.layers().len() == 0) & (self.layers()[0].tag.tag_class() == data.kind()) {
+        if (self.layers().len() == 1) & (self.layers()[0].tag.tag_class() == data.kind()) {
             // if someone sends a PR with an added check, I'll come home and throw a brick on your head
+            layertrace("fpath");
             let l = self.layers()[0];
-            unsafe { LVERIFY[l.tag.tag_selector().word()](l, data) }
+            unsafe { LVERIFY[l.tag.tag_class().word()](l, data) }
         } else {
             Self::rverify_layers(self.layers(), data)
         }
diff --git a/server/src/engine/core/tests/model/layer.rs b/server/src/engine/core/tests/model/layer.rs
index b31ee03c..00e41734 100644
--- a/server/src/engine/core/tests/model/layer.rs
+++ b/server/src/engine/core/tests/model/layer.rs
@@ -84,7 +84,7 @@ mod layer_data_validation {
         let dc = Datacell::new_bool(true);
         let layer = layerview("bool").unwrap();
         assert!(layer.validate_data_fpath(&dc));
-        assert_vecstreq_exact!(model::layer_traces(), ["bool"]);
+        assert_vecstreq_exact!(model::layer_traces(), ["fpath", "bool"]);
     }
     #[test]
     fn uint() {
@@ -101,16 +101,16 @@ mod layer_data_validation {
                 let this_layer = layerview(layer).unwrap();
                 let dc = Datacell::new_uint(max);
                 assert!(this_layer.validate_data_fpath(&dc), "{:?}", this_layer);
-                assert_vecstreq_exact!(model::layer_traces(), ["uint"]);
+                assert_vecstreq_exact!(model::layer_traces(), ["fpath", "uint"]);
                 for (lower, _) in targets[..i].iter() {
                     let layer = layerview(lower).unwrap();
                     assert!(!layer.validate_data_fpath(&dc), "{:?}", layer);
-                    assert_vecstreq_exact!(model::layer_traces(), ["uint"]);
+                    assert_vecstreq_exact!(model::layer_traces(), ["fpath", "uint"]);
                 }
                 for (higher, _) in targets[i + 1..].iter() {
                     let layer = layerview(higher).unwrap();
                     assert!(layer.validate_data_fpath(&dc), "{:?}", layer);
-                    assert_vecstreq_exact!(model::layer_traces(), ["uint"]);
+                    assert_vecstreq_exact!(model::layer_traces(), ["fpath", "uint"]);
                 }
             });
     }
@@ -131,18 +131,24 @@ mod layer_data_validation {
                 let dc_max = Datacell::new_sint(max);
                 assert!(this_layer.validate_data_fpath(&dc_min), "{:?}", this_layer);
                 assert!(this_layer.validate_data_fpath(&dc_max), "{:?}", this_layer);
-                assert_vecstreq_exact!(model::layer_traces(), ["sint", "sint"]);
+                assert_vecstreq_exact!(model::layer_traces(), ["fpath", "sint", "fpath", "sint"]);
                 for (lower, _) in targets[..i].iter() {
                     let layer = layerview(lower).unwrap();
                     assert!(!layer.validate_data_fpath(&dc_min), "{:?}", layer);
                     assert!(!layer.validate_data_fpath(&dc_max), "{:?}", layer);
-                    assert_vecstreq_exact!(model::layer_traces(), ["sint", "sint"]);
+                    assert_vecstreq_exact!(
+                        model::layer_traces(),
+                        ["fpath", "sint", "fpath", "sint"]
+                    );
                 }
                 for (higher, _) in targets[i + 1..].iter() {
                     let layer = layerview(higher).unwrap();
                     assert!(layer.validate_data_fpath(&dc_min), "{:?}", layer);
                     assert!(layer.validate_data_fpath(&dc_max), "{:?}", layer);
-                    assert_vecstreq_exact!(model::layer_traces(), ["sint", "sint"]);
+                    assert_vecstreq_exact!(
+                        model::layer_traces(),
+                        ["fpath", "sint", "fpath", "sint"]
+                    );
                 }
             });
     }
@@ -159,29 +165,29 @@ mod layer_data_validation {
         // check (32)
         assert!(f32_l.validate_data_fpath(&f32_dc_min));
         assert!(f32_l.validate_data_fpath(&f32_dc_max));
-        assert_vecstreq_exact!(model::layer_traces(), ["float", "float"]);
+        assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]);
         assert!(f64_l.validate_data_fpath(&f32_dc_min));
         assert!(f64_l.validate_data_fpath(&f32_dc_max));
-        assert_vecstreq_exact!(model::layer_traces(), ["float", "float"]);
+        assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]);
         // check (64)
         assert!(!f32_l.validate_data_fpath(&f64_dc_min));
         assert!(!f32_l.validate_data_fpath(&f64_dc_max));
-        assert_vecstreq_exact!(model::layer_traces(), ["float", "float"]);
+        assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]);
         assert!(f64_l.validate_data_fpath(&f64_dc_min));
         assert!(f64_l.validate_data_fpath(&f64_dc_max));
-        assert_vecstreq_exact!(model::layer_traces(), ["float", "float"]);
+        assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]);
     }
     #[test]
     fn bin() {
         let layer = layerview("binary").unwrap();
         assert!(layer.validate_data_fpath(&Datacell::from("hello".as_bytes())));
-        assert_vecstreq_exact!(model::layer_traces(), ["binary"]);
+        assert_vecstreq_exact!(model::layer_traces(), ["fpath", "binary"]);
     }
     #[test]
     fn str() {
         let layer = layerview("string").unwrap();
         assert!(layer.validate_data_fpath(&Datacell::from("hello")));
-        assert_vecstreq_exact!(model::layer_traces(), ["string"]);
+        assert_vecstreq_exact!(model::layer_traces(), ["fpath", "string"]);
     }
     #[test]
     fn list_simple() {

From 188f9ecbb44dd4d43450cb582f2f93d58c4185c5 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 4 Mar 2023 05:14:45 -0800
Subject: [PATCH 150/310] Support null in cell

---
 server/src/engine/core/model/cell.rs | 47 +++++++++++++++++++++++-----
 server/src/engine/ql/dml/ins.rs      | 40 +++++++++++------------
 server/src/engine/ql/tests.rs        | 12 +++----
 3 files changed, 64 insertions(+), 35 deletions(-)

diff --git a/server/src/engine/core/model/cell.rs b/server/src/engine/core/model/cell.rs
index b3272278..86685ecb 100644
--- a/server/src/engine/core/model/cell.rs
+++ b/server/src/engine/core/model/cell.rs
@@ -41,6 +41,7 @@ use {
 };
 
 pub struct Datacell {
+    init: bool,
     tag: TagClass,
     data: DataRaw,
 }
@@ -217,18 +218,43 @@ impl<const N: usize> From<[Datacell; N]> for Datacell {
 }
 
 impl Datacell {
-    unsafe fn new(tag: TagClass, data: DataRaw) -> Self {
-        Self { tag, data }
-    }
-    fn checked_tag<T>(&self, tag: TagClass, f: impl FnOnce() -> T) -> Option<T> {
-        (self.tag == tag).then_some(f())
-    }
     pub fn kind(&self) -> TagClass {
         self.tag
     }
+    pub fn null() -> Self {
+        unsafe {
+            Self::_new(
+                TagClass::Bool,
+                DataRaw::word(NativeQword::store_qw(0)),
+                false,
+            )
+        }
+    }
+    pub fn is_null(&self) -> bool {
+        !self.init
+    }
+    pub fn is_init(&self) -> bool {
+        self.init
+    }
+    pub fn as_option(&self) -> Option<&Datacell> {
+        if self.init {
+            Some(self)
+        } else {
+            None
+        }
+    }
     unsafe fn load_word<'a, T: WordRW<NativeQword, Target<'a> = T>>(&'a self) -> T {
         self.data.word.ld()
     }
+    unsafe fn _new(tag: TagClass, data: DataRaw, init: bool) -> Self {
+        Self { init, tag, data }
+    }
+    unsafe fn new(tag: TagClass, data: DataRaw) -> Self {
+        Self::_new(tag, data, true)
+    }
+    fn checked_tag<T>(&self, tag: TagClass, f: impl FnOnce() -> T) -> Option<T> {
+        ((self.tag == tag) & (self.is_init())).then_some(f())
+    }
 }
 
 impl fmt::Debug for Datacell {
@@ -238,7 +264,9 @@ impl fmt::Debug for Datacell {
         macro_rules! fmtdbg {
             ($($match:ident => $ret:expr),* $(,)?) => {
                 match self.tag {
-                    $(TagClass::$match => f.field("data", &$ret),)*
+                    $(TagClass::$match if self.is_init() => { f.field("data", &Some($ret));},)*
+                    TagClass::Bool if self.is_null() => {f.field("data", &Option::<u8>::None);},
+                    _ => unreachable!("incorrect state"),
                 }
             }
         }
@@ -257,6 +285,9 @@ impl fmt::Debug for Datacell {
 
 impl PartialEq for Datacell {
     fn eq(&self, other: &Datacell) -> bool {
+        if self.is_null() {
+            return other.is_null();
+        }
         match (self.tag, other.tag) {
             (TagClass::Bool, TagClass::Bool) => self.bool() == other.bool(),
             (TagClass::UnsignedInt, TagClass::UnsignedInt) => self.uint() == other.uint(),
@@ -333,6 +364,6 @@ impl Clone for Datacell {
                 }
             },
         };
-        unsafe { Self::new(self.tag, data) }
+        unsafe { Self::_new(self.tag, data, self.init) }
     }
 }
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index c267c30b..669ecfaa 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -209,7 +209,7 @@ unsafe fn handle_func_sub<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> O
 /// - If tt is empty
 pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
     state: &mut State<'a, Qd>,
-) -> Vec<Option<Datacell>> {
+) -> Vec<Datacell> {
     let mut stop = state.cursor_eq(Token![() close]);
     state.cursor_ahead_if(stop);
     let mut data = Vec::new();
@@ -217,21 +217,19 @@ pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
         match state.fw_read() {
             tok if state.can_read_lit_from(tok) => unsafe {
                 // UNSAFE(@ohsayan): if guard guarantees correctness
-                data.push(Some(state.read_lit_into_data_type_unchecked_from(tok)));
+                data.push(state.read_lit_into_data_type_unchecked_from(tok));
             },
             Token![open []] if state.not_exhausted() => {
                 let mut l = Vec::new();
                 let _ = parse_list(state, &mut l);
-                data.push(Some(l.into()));
-            }
-            Token![null] => {
-                data.push(None);
+                data.push(l.into());
             }
+            Token![null] => data.push(Datacell::null()),
             Token![@] if state.cursor_signature_match_fn_arity0_rounded() => match unsafe {
                 // UNSAFE(@ohsayan): Just verified at guard
                 handle_func_sub(state)
             } {
-                Some(value) => data.push(Some(value)),
+                Some(value) => data.push(value),
                 None => {
                     state.poison();
                     break;
@@ -256,7 +254,7 @@ pub(super) fn parse_data_tuple_syntax<'a, Qd: QueryData<'a>>(
 /// Panics if tt is empty
 pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
     state: &mut State<'a, Qd>,
-) -> HashMap<Ident<'a>, Option<Datacell>> {
+) -> HashMap<Ident<'a>, Datacell> {
     let mut stop = state.cursor_eq(Token![close {}]);
     state.cursor_ahead_if(stop);
     let mut data = HashMap::with_capacity(2);
@@ -267,26 +265,26 @@ pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
         state.poison_if_not(Token![:].eq(colon));
         match (field, expr) {
             (Token::Ident(id), tok) if state.can_read_lit_from(tok) => {
-                let ldata = Some(unsafe {
+                let ldata = unsafe {
                     // UNSAFE(@ohsayan): The if guard guarantees correctness
                     state.read_lit_into_data_type_unchecked_from(tok)
-                });
+                };
                 state.poison_if_not(data.insert(*id, ldata).is_none());
             }
             (Token::Ident(id), Token![null]) => {
-                state.poison_if_not(data.insert(*id, None).is_none());
+                state.poison_if_not(data.insert(*id, Datacell::null()).is_none());
             }
             (Token::Ident(id), Token![open []]) if state.not_exhausted() => {
                 let mut l = Vec::new();
                 let _ = parse_list(state, &mut l);
-                state.poison_if_not(data.insert(*id, Some(l.into())).is_none());
+                state.poison_if_not(data.insert(*id, l.into()).is_none());
             }
             (Token::Ident(id), Token![@]) if state.cursor_signature_match_fn_arity0_rounded() => {
                 match unsafe {
                     // UNSAFE(@ohsayan): Just verified at guard
                     handle_func_sub(state)
                 } {
-                    Some(value) => state.poison_if_not(data.insert(*id, Some(value)).is_none()),
+                    Some(value) => state.poison_if_not(data.insert(*id, value).is_none()),
                     None => {
                         state.poison();
                         break;
@@ -310,18 +308,18 @@ pub(super) fn parse_data_map_syntax<'a, Qd: QueryData<'a>>(
 
 #[derive(Debug, PartialEq)]
 pub enum InsertData<'a> {
-    Ordered(Vec<Option<Datacell>>),
-    Map(HashMap<Ident<'a>, Option<Datacell>>),
+    Ordered(Vec<Datacell>),
+    Map(HashMap<Ident<'a>, Datacell>),
 }
 
-impl<'a> From<Vec<Option<Datacell>>> for InsertData<'a> {
-    fn from(v: Vec<Option<Datacell>>) -> Self {
+impl<'a> From<Vec<Datacell>> for InsertData<'a> {
+    fn from(v: Vec<Datacell>) -> Self {
         Self::Ordered(v)
     }
 }
 
-impl<'a> From<HashMap<Ident<'static>, Option<Datacell>>> for InsertData<'a> {
-    fn from(m: HashMap<Ident<'static>, Option<Datacell>>) -> Self {
+impl<'a> From<HashMap<Ident<'static>, Datacell>> for InsertData<'a> {
+    fn from(m: HashMap<Ident<'static>, Datacell>) -> Self {
         Self::Map(m)
     }
 }
@@ -427,7 +425,7 @@ mod impls {
             }
         }
         #[derive(sky_macros::Wrapper, Debug)]
-        pub struct DataTuple(Vec<Option<Datacell>>);
+        pub struct DataTuple(Vec<Datacell>);
         impl<'a> ASTNode<'a> for DataTuple {
             // important: upstream must verify this
             const VERIFY: bool = true;
@@ -437,7 +435,7 @@ mod impls {
             }
         }
         #[derive(sky_macros::Wrapper, Debug)]
-        pub struct DataMap(HashMap<Box<str>, Option<Datacell>>);
+        pub struct DataMap(HashMap<Box<str>, Datacell>);
         impl<'a> ASTNode<'a> for DataMap {
             // important: upstream must verify this
             const VERIFY: bool = true;
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index ad779032..e4421bda 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -51,27 +51,27 @@ pub fn lex_secure(src: &[u8]) -> LexResult<Vec<Token>> {
 }
 
 pub trait NullableData<T> {
-    fn data(self) -> Option<T>;
+    fn data(self) -> T;
 }
 
 impl<T> NullableData<Datacell> for T
 where
     T: Into<Datacell>,
 {
-    fn data(self) -> Option<Datacell> {
-        Some(self.into())
+    fn data(self) -> Datacell {
+        self.into()
     }
 }
 
 struct Null;
 
 impl NullableData<Datacell> for Null {
-    fn data(self) -> Option<Datacell> {
-        None
+    fn data(self) -> Datacell {
+        Datacell::null()
     }
 }
 
-fn nullable_datatype(v: impl NullableData<Datacell>) -> Option<Datacell> {
+fn nullable_datatype(v: impl NullableData<Datacell>) -> Datacell {
     v.data()
 }
 

From a88319b8b9199f1213cbcd06d7e173359b6e4778 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 4 Mar 2023 06:57:30 -0800
Subject: [PATCH 151/310] Enable fpath on nullable data

---
 server/src/engine/core/model/mod.rs         | 37 ++++++++++++++++-----
 server/src/engine/core/tests/model/layer.rs | 21 ++++++++++--
 2 files changed, 46 insertions(+), 12 deletions(-)

diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 92455755..3ccec215 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -74,13 +74,16 @@ static LUT: [(&str, FullTag); 14] = [
 ];
 
 #[derive(Debug, PartialEq, Clone)]
-pub struct LayerView(VInline<1, Layer>);
+pub struct LayerView {
+    layers: VInline<1, Layer>,
+    nullable: bool,
+}
 
 impl LayerView {
     pub fn layers(&self) -> &[Layer] {
-        &self.0
+        &self.layers
     }
-    pub fn parse_layers(spec: Vec<LayerSpec>) -> DatabaseResult<Self> {
+    pub fn parse_layers(spec: Vec<LayerSpec>, nullable: bool) -> DatabaseResult<Self> {
         let mut layers = spec.into_iter().rev();
         let mut okay = true;
         let mut fin = false;
@@ -98,17 +101,29 @@ impl LayerView {
         }
         okay &= fin & (layers.len() == 0);
         if okay {
-            Ok(Self(layerview))
+            Ok(Self {
+                layers: layerview,
+                nullable,
+            })
         } else {
             Err(DatabaseError::DdlModelInvalidTypeDefinition)
         }
     }
+    #[inline(always)]
+    fn single_pass_for(&self, dc: &Datacell) -> bool {
+        ((self.layers().len() == 1) & (self.layers()[0].tag.tag_class() == dc.kind()))
+            | (self.nullable & dc.is_null())
+    }
+    #[inline(always)]
+    fn compute_index(&self, dc: &Datacell) -> usize {
+        // escape check if it makes sense to
+        !(self.nullable & dc.is_null()) as usize * self.layers()[0].tag.tag_class().word()
+    }
     pub fn validate_data_fpath(&self, data: &Datacell) -> bool {
-        if (self.layers().len() == 1) & (self.layers()[0].tag.tag_class() == data.kind()) {
-            // if someone sends a PR with an added check, I'll come home and throw a brick on your head
+        // if someone sends a PR with an added check, I'll personally come to your house and throw a brick on your head
+        if self.single_pass_for(data) {
             layertrace("fpath");
-            let l = self.layers()[0];
-            unsafe { LVERIFY[l.tag.tag_class().word()](l, data) }
+            unsafe { LVERIFY[self.compute_index(data)](self.layers()[0], data) }
         } else {
             Self::rverify_layers(self.layers(), data)
         }
@@ -188,6 +203,10 @@ impl Layer {
 }
 
 impl Layer {
+    #[inline(always)]
+    fn compute_index(&self, dc: &Datacell) -> usize {
+        self.tag.tag_class().word() * (dc.is_null() as usize)
+    }
     const fn new(tag: FullTag, config: [usize; 2]) -> Self {
         Self { tag, config }
     }
@@ -228,7 +247,7 @@ static LVERIFY: [unsafe fn(Layer, &Datacell) -> bool; 7] = [
 
 #[cfg(test)]
 thread_local! {
-    pub static LAYER_TRACE: RefCell<Vec<Box<str>>> = RefCell::new(Vec::new());
+    static LAYER_TRACE: RefCell<Vec<Box<str>>> = RefCell::new(Vec::new());
 }
 
 #[inline(always)]
diff --git a/server/src/engine/core/tests/model/layer.rs b/server/src/engine/core/tests/model/layer.rs
index 00e41734..1c5608b1 100644
--- a/server/src/engine/core/tests/model/layer.rs
+++ b/server/src/engine/core/tests/model/layer.rs
@@ -30,10 +30,13 @@ use crate::engine::{
     ql::{ast::parse_ast_node_multiple_full, tests::lex_insecure},
 };
 
-fn layerview(layer_def: &str) -> DatabaseResult<LayerView> {
+fn layerview_nullable(layer_def: &str, nullable: bool) -> DatabaseResult<LayerView> {
     let tok = lex_insecure(layer_def.as_bytes()).unwrap();
     let spec = parse_ast_node_multiple_full(&tok).unwrap();
-    LayerView::parse_layers(spec)
+    LayerView::parse_layers(spec, nullable)
+}
+fn layerview(layer_def: &str) -> DatabaseResult<LayerView> {
+    layerview_nullable(layer_def, false)
 }
 
 mod layer_spec_validation {
@@ -76,7 +79,7 @@ mod layer_spec_validation {
 
 mod layer_data_validation {
     use {
-        super::layerview,
+        super::{layerview, layerview_nullable},
         crate::engine::core::model::{self, cell::Datacell},
     };
     #[test]
@@ -203,4 +206,16 @@ mod layer_data_validation {
             ["list", "string", "string", "string"]
         );
     }
+    #[test]
+    fn nullval_fpath() {
+        let layer = layerview_nullable("string", true).unwrap();
+        assert!(layer.validate_data_fpath(&Datacell::null()));
+        assert_vecstreq_exact!(model::layer_traces(), ["fpath", "bool"]);
+    }
+    #[test]
+    fn nullval_nested_but_fpath() {
+        let layer = layerview_nullable("list { type: string }", true).unwrap();
+        assert!(layer.validate_data_fpath(&Datacell::null()));
+        assert_vecstreq_exact!(model::layer_traces(), ["fpath", "bool"]);
+    }
 }

From feac086edc37ac22a14641c4caf4ee15de2e6bf2 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 4 Mar 2023 07:58:30 -0800
Subject: [PATCH 152/310] Implement exec for create model

---
 server/src/engine/core/model/mod.rs         |  66 +++++++++--
 server/src/engine/core/tests/model/crt.rs   | 115 ++++++++++++++++++++
 server/src/engine/core/tests/model/layer.rs |   8 +-
 server/src/engine/core/tests/model/mod.rs   |   1 +
 server/src/engine/data/tag.rs               |   8 +-
 server/src/engine/error.rs                  |   2 +
 server/src/engine/idx/mod.rs                |   6 +-
 server/src/engine/idx/stord/config.rs       |   4 +
 server/src/engine/idx/stord/mod.rs          |  28 ++++-
 server/src/engine/mem/vinline.rs            |   7 ++
 server/src/engine/mod.rs                    |  12 --
 server/src/engine/ql/ddl/crt.rs             |  12 +-
 server/src/engine/ql/ddl/syn.rs             |  18 +--
 server/src/engine/ql/tests/schema_tests.rs  |  46 ++++----
 14 files changed, 265 insertions(+), 68 deletions(-)
 create mode 100644 server/src/engine/core/tests/model/crt.rs

diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 3ccec215..fed63f0e 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -30,21 +30,61 @@ use crate::engine::{
     core::model::cell::Datacell,
     data::tag::{DataTag, FullTag, TagClass, TagSelector},
     error::{DatabaseError, DatabaseResult},
+    idx::{IndexSTSeqCns, STIndex, STIndexSeq},
     mem::VInline,
-    ql::ddl::syn::LayerSpec,
+    ql::ddl::{
+        crt::CreateModel,
+        syn::{FieldSpec, LayerSpec},
+    },
 };
 #[cfg(test)]
 use std::cell::RefCell;
 
 // FIXME(@ohsayan): update this!
 
-#[derive(Debug)]
-pub struct ModelView {}
+#[derive(Debug, PartialEq)]
+pub struct ModelView {
+    pub(super) p_key: Box<str>,
+    pub(super) p_tag: FullTag,
+    pub(super) fields: IndexSTSeqCns<Box<str>, Field>,
+}
 
-#[cfg(test)]
-impl PartialEq for ModelView {
-    fn eq(&self, _: &Self) -> bool {
-        true
+impl ModelView {
+    pub fn create(CreateModel { fields, props, .. }: CreateModel) -> DatabaseResult<Self> {
+        let mut okay = props.is_empty() & !fields.is_empty();
+        // validate fields
+        let mut field_spec = fields.into_iter();
+        let mut fields = IndexSTSeqCns::with_capacity(field_spec.len());
+        let mut last_pk = None;
+        let mut pk_cnt = 0;
+        while (field_spec.len() != 0) & okay {
+            let FieldSpec {
+                field_name,
+                layers,
+                null,
+                primary,
+            } = field_spec.next().unwrap();
+            if primary {
+                pk_cnt += 1usize;
+                last_pk = Some(field_name.as_str());
+                okay &= !null;
+            }
+            let layer = Field::parse_layers(layers, null)?;
+            okay &= fields.st_insert(field_name.as_str().to_string().into_boxed_str(), layer);
+        }
+        okay &= pk_cnt <= 1;
+        if okay {
+            let last_pk = last_pk.unwrap_or(fields.stseq_ord_key().next().unwrap());
+            let tag = fields.st_get(last_pk).unwrap().layers()[0].tag;
+            if tag.tag_unique().is_unique() {
+                return Ok(Self {
+                    p_key: last_pk.into(),
+                    p_tag: tag,
+                    fields,
+                });
+            }
+        }
+        Err(DatabaseError::DdlModelBadDefinition)
     }
 }
 
@@ -74,12 +114,16 @@ static LUT: [(&str, FullTag); 14] = [
 ];
 
 #[derive(Debug, PartialEq, Clone)]
-pub struct LayerView {
+pub struct Field {
     layers: VInline<1, Layer>,
     nullable: bool,
 }
 
-impl LayerView {
+impl Field {
+    #[cfg(test)]
+    pub fn new_test(layers: VInline<1, Layer>, nullable: bool) -> Self {
+        Self { layers, nullable }
+    }
     pub fn layers(&self) -> &[Layer] {
         &self.layers
     }
@@ -203,6 +247,10 @@ impl Layer {
 }
 
 impl Layer {
+    #[cfg(test)]
+    pub fn new_test(tag: FullTag, config: [usize; 2]) -> Self {
+        Self::new(tag, config)
+    }
     #[inline(always)]
     fn compute_index(&self, dc: &Datacell) -> usize {
         self.tag.tag_class().word() * (dc.is_null() as usize)
diff --git a/server/src/engine/core/tests/model/crt.rs b/server/src/engine/core/tests/model/crt.rs
new file mode 100644
index 00000000..f55a3051
--- /dev/null
+++ b/server/src/engine/core/tests/model/crt.rs
@@ -0,0 +1,115 @@
+/*
+ * Created on Sat Mar 04 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    core::model::{Field, Layer, ModelView},
+    data::tag::{DataTag, FullTag},
+    error::{DatabaseError, DatabaseResult},
+    idx::STIndexSeq,
+    ql::{ast::parse_ast_node_full, tests::lex_insecure},
+};
+
+fn create(s: &str) -> DatabaseResult<ModelView> {
+    let tok = lex_insecure(s.as_bytes()).unwrap();
+    let create_model = parse_ast_node_full(&tok[2..]).unwrap();
+    ModelView::create(create_model)
+}
+
+#[test]
+fn simple() {
+    let ModelView {
+        p_key,
+        p_tag,
+        fields,
+    } = create("create model mymodel(username: string, password: binary)").unwrap();
+    assert_eq!(p_key.as_ref(), "username");
+    assert_eq!(p_tag, FullTag::STR);
+    assert_eq!(
+        fields.stseq_ord_value().cloned().collect::<Vec<Field>>(),
+        [
+            Field::new_test([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
+            Field::new_test([Layer::new_test(FullTag::BIN, [0; 2])].into(), false)
+        ]
+    );
+}
+
+#[test]
+fn idiotic_order() {
+    let ModelView {
+        p_key,
+        p_tag,
+        fields,
+    } = create("create model mymodel(password: binary, primary username: string)").unwrap();
+    assert_eq!(p_key.as_ref(), "username");
+    assert_eq!(p_tag, FullTag::STR);
+    assert_eq!(
+        fields.stseq_ord_value().cloned().collect::<Vec<Field>>(),
+        [
+            Field::new_test([Layer::new_test(FullTag::BIN, [0; 2])].into(), false),
+            Field::new_test([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
+        ]
+    );
+}
+
+#[test]
+fn duplicate_primary_key() {
+    assert_eq!(
+        create("create model mymodel(primary username: string, primary contract_location: binary)")
+            .unwrap_err(),
+        DatabaseError::DdlModelBadDefinition
+    );
+}
+
+#[test]
+fn duplicate_fields() {
+    assert_eq!(
+        create("create model mymodel(primary username: string, username: binary)").unwrap_err(),
+        DatabaseError::DdlModelBadDefinition
+    );
+}
+
+#[test]
+fn illegal_props() {
+    assert_eq!(
+        create("create model mymodel(primary username: string, password: binary) with { lol_prop: false }").unwrap_err(),
+        DatabaseError::DdlModelBadDefinition
+    );
+}
+
+#[test]
+fn illegal_pk() {
+    assert_eq!(
+        create(
+            "create model mymodel(primary username_bytes: list { type: uint8 }, password: binary)"
+        )
+        .unwrap_err(),
+        DatabaseError::DdlModelBadDefinition
+    );
+    assert_eq!(
+        create("create model mymodel(primary username: float32, password: binary)").unwrap_err(),
+        DatabaseError::DdlModelBadDefinition
+    );
+}
diff --git a/server/src/engine/core/tests/model/layer.rs b/server/src/engine/core/tests/model/layer.rs
index 1c5608b1..626940fa 100644
--- a/server/src/engine/core/tests/model/layer.rs
+++ b/server/src/engine/core/tests/model/layer.rs
@@ -25,17 +25,17 @@
 */
 
 use crate::engine::{
-    core::model::LayerView,
+    core::model::Field,
     error::DatabaseResult,
     ql::{ast::parse_ast_node_multiple_full, tests::lex_insecure},
 };
 
-fn layerview_nullable(layer_def: &str, nullable: bool) -> DatabaseResult<LayerView> {
+fn layerview_nullable(layer_def: &str, nullable: bool) -> DatabaseResult<Field> {
     let tok = lex_insecure(layer_def.as_bytes()).unwrap();
     let spec = parse_ast_node_multiple_full(&tok).unwrap();
-    LayerView::parse_layers(spec, nullable)
+    Field::parse_layers(spec, nullable)
 }
-fn layerview(layer_def: &str) -> DatabaseResult<LayerView> {
+fn layerview(layer_def: &str) -> DatabaseResult<Field> {
     layerview_nullable(layer_def, false)
 }
 
diff --git a/server/src/engine/core/tests/model/mod.rs b/server/src/engine/core/tests/model/mod.rs
index e07ae0a6..8b8690cb 100644
--- a/server/src/engine/core/tests/model/mod.rs
+++ b/server/src/engine/core/tests/model/mod.rs
@@ -24,4 +24,5 @@
  *
 */
 
+mod crt;
 mod layer;
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
index d3823410..f64ea3a5 100644
--- a/server/src/engine/data/tag.rs
+++ b/server/src/engine/data/tag.rs
@@ -65,8 +65,14 @@ pub enum TagUnique {
     Illegal = 0xFF,
 }
 
+impl TagUnique {
+    pub const fn is_unique(&self) -> bool {
+        self.d() != 0xFF
+    }
+}
+
 macro_rules! d {
-    ($($ty:ty),*) => {$(impl $ty { pub fn d(&self) -> u8 {unsafe{::core::mem::transmute_copy(self)}} pub fn word(&self) -> usize {Self::d(self) as usize} } )*}
+    ($($ty:ty),*) => {$(impl $ty { pub const fn d(&self) -> u8 {unsafe{::core::mem::transmute(*self)}} pub const fn word(&self) -> usize {Self::d(self) as usize} } )*}
 }
 
 d!(TagClass, TagSelector, TagUnique);
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index e763cf33..577f27ac 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -90,4 +90,6 @@ pub enum DatabaseError {
     DdlSpaceNotFound,
     /// bad definition for some typedef in a model
     DdlModelInvalidTypeDefinition,
+    /// bad model definition; most likely an illegal primary key
+    DdlModelBadDefinition,
 }
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 6b30e5d4..5912dcc2 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -100,7 +100,7 @@ pub struct DummyMetrics;
 
 /// The base spec for any index. Iterators have meaningless order, and that is intentional and oftentimes
 /// consequential. For more specialized impls, use the [`STIndex`], [`MTIndex`] or [`STIndexSeq`] traits
-pub trait IndexBaseSpec<K, V>: Sized {
+pub trait IndexBaseSpec<K: ?Sized, V: ?Sized>: Sized {
     /// Index supports prealloc?
     const PREALLOC: bool;
     #[cfg(debug_assertions)]
@@ -202,7 +202,7 @@ pub trait MTIndex<K, V>: IndexBaseSpec<K, V> {
 }
 
 /// An unordered STIndex
-pub trait STIndex<K, V>: IndexBaseSpec<K, V> {
+pub trait STIndex<K: ?Sized, V>: IndexBaseSpec<K, V> {
     /// An iterator over the keys and values
     type IterKV<'a>: Iterator<Item = (&'a K, &'a V)>
     where
@@ -284,7 +284,7 @@ pub trait STIndex<K, V>: IndexBaseSpec<K, V> {
     fn st_iter_value<'a>(&'a self) -> Self::IterValue<'a>;
 }
 
-pub trait STIndexSeq<K, V>: STIndex<K, V> {
+pub trait STIndexSeq<K: ?Sized, V>: STIndex<K, V> {
     /// An ordered iterator over the keys and values
     type IterOrdKV<'a>: Iterator<Item = (&'a K, &'a V)> + DoubleEndedIterator<Item = (&'a K, &'a V)>
     where
diff --git a/server/src/engine/idx/stord/config.rs b/server/src/engine/idx/stord/config.rs
index 241e487a..211a2041 100644
--- a/server/src/engine/idx/stord/config.rs
+++ b/server/src/engine/idx/stord/config.rs
@@ -30,6 +30,7 @@ use {
     std::{collections::hash_map::RandomState, marker::PhantomData, ptr},
 };
 
+#[derive(Debug)]
 pub struct LiberalStrategy<K, V> {
     f: *mut IndexSTSeqDllNode<K, V>,
 }
@@ -76,6 +77,7 @@ impl<K, V> AllocStrategy<K, V> for LiberalStrategy<K, V> {
     }
 }
 
+#[derive(Debug)]
 pub struct ConservativeStrategy<K, V> {
     _d: PhantomData<IndexSTSeqDllNodePtr<K, V>>,
 }
@@ -113,6 +115,7 @@ pub trait Config<K, V> {
     type AllocStrategy: AllocStrategy<K, V>;
 }
 
+#[derive(Debug)]
 pub struct ConservativeConfig<K, V>(PhantomData<super::IndexSTSeqDll<K, V, Self>>);
 
 impl<K, V> Config<K, V> for ConservativeConfig<K, V> {
@@ -120,6 +123,7 @@ impl<K, V> Config<K, V> for ConservativeConfig<K, V> {
     type AllocStrategy = ConservativeStrategy<K, V>;
 }
 
+#[derive(Debug)]
 pub struct LiberalConfig<K, V>(PhantomData<super::IndexSTSeqDll<K, V, Self>>);
 
 impl<K, V> Config<K, V> for LiberalConfig<K, V> {
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index 237a817f..5ee485d1 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -40,7 +40,7 @@ use {
         alloc::{alloc as std_alloc, dealloc as std_dealloc, Layout},
         borrow::Borrow,
         collections::HashMap as StdMap,
-        fmt::Debug,
+        fmt::{self, Debug},
         hash::{Hash, Hasher},
         mem,
         ptr::{self, NonNull},
@@ -275,6 +275,15 @@ impl<K, V, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
     }
 }
 
+impl<K, V, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
+    pub fn new() -> Self {
+        Self::with_hasher(C::Hasher::default())
+    }
+    pub fn with_capacity(cap: usize) -> Self {
+        Self::with_capacity_and_hasher(cap, C::Hasher::default())
+    }
+}
+
 impl<K, V, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
     #[inline(always)]
     fn metrics_update_f_decr(&mut self) {
@@ -709,3 +718,20 @@ impl<K: AsKeyClone, V: AsValueClone, C: Config<K, V>> Clone for IndexSTSeqDll<K,
 
 unsafe impl<K: Send, V: Send, C: Config<K, V> + Send> Send for IndexSTSeqDll<K, V, C> {}
 unsafe impl<K: Sync, V: Sync, C: Config<K, V> + Sync> Sync for IndexSTSeqDll<K, V, C> {}
+
+impl<K: fmt::Debug, V: fmt::Debug, C: Config<K, V> + fmt::Debug> fmt::Debug
+    for IndexSTSeqDll<K, V, C>
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_map().entries(self._iter_ord_kv()).finish()
+    }
+}
+
+impl<K: AsKey, V: AsValue + PartialEq, C: Config<K, V>> PartialEq for IndexSTSeqDll<K, V, C> {
+    fn eq(&self, other: &Self) -> bool {
+        self.len() == other.len()
+            && self
+                ._iter_unord_kv()
+                .all(|(k, v)| other._get(k).unwrap().eq(v))
+    }
+}
diff --git a/server/src/engine/mem/vinline.rs b/server/src/engine/mem/vinline.rs
index 22db1bcd..f0702406 100644
--- a/server/src/engine/mem/vinline.rs
+++ b/server/src/engine/mem/vinline.rs
@@ -291,6 +291,13 @@ impl<T, const N: usize> Extend<T> for VInline<N, T> {
     }
 }
 
+#[cfg(test)]
+impl<T, const N: usize> From<[T; N]> for VInline<N, T> {
+    fn from(a: [T; N]) -> Self {
+        a.into_iter().collect()
+    }
+}
+
 impl<T: Clone, const N: usize> Clone for VInline<N, T> {
     fn clone(&self) -> Self {
         self.iter().cloned().collect()
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 96b375fb..cb44d272 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -35,15 +35,3 @@ mod idx;
 mod mem;
 mod ql;
 mod sync;
-
-/*
-
-    A word on tests:
-
-    "Nature is not equal. That's the whole problem." - Freeman Dyson
-
-    Well, that applies to us for atleast all test cases since most of them are based on a quiescent
-    state than a chaotic state as in runtime. We do emulate such cases, but remember most assertions
-    that you'll make on most structures are just illusionary, and are only atomically correct at point
-    in time.
-*/
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index 5128eb50..ab376451 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    super::syn::{self, DictFoldState, Field},
+    super::syn::{self, DictFoldState, FieldSpec},
     crate::{
         engine::{
             data::DictGeneric,
@@ -82,11 +82,11 @@ impl<'a> CreateSpace<'a> {
 /// A model definition
 pub struct CreateModel<'a> {
     /// the model name
-    model_name: Ident<'a>,
+    pub(in crate::engine) model_name: Ident<'a>,
     /// the fields
-    fields: Vec<Field<'a>>,
+    pub(in crate::engine) fields: Vec<FieldSpec<'a>>,
     /// properties
-    props: DictGeneric,
+    pub(in crate::engine) props: DictGeneric,
 }
 
 /*
@@ -97,7 +97,7 @@ pub struct CreateModel<'a> {
 */
 
 impl<'a> CreateModel<'a> {
-    pub fn new(model_name: Ident<'a>, fields: Vec<Field<'a>>, props: DictGeneric) -> Self {
+    pub fn new(model_name: Ident<'a>, fields: Vec<FieldSpec<'a>>, props: DictGeneric) -> Self {
         Self {
             model_name,
             fields,
@@ -118,7 +118,7 @@ impl<'a> CreateModel<'a> {
         let mut stop = false;
         let mut fields = Vec::with_capacity(2);
         while state.loop_tt() && !stop {
-            fields.push(Field::parse(state)?);
+            fields.push(FieldSpec::parse(state)?);
             let nx_close = state.cursor_rounded_eq(Token![() close]);
             let nx_comma = state.cursor_rounded_eq(Token![,]);
             state.poison_if_not(nx_close | nx_comma);
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 627f4659..77a59655 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -307,18 +307,18 @@ fn rfold_layers<'a, Qd: QueryData<'a>>(
 
 #[derive(Debug, PartialEq)]
 /// A field definition
-pub struct Field<'a> {
+pub struct FieldSpec<'a> {
     /// the field name
-    field_name: Ident<'a>,
+    pub(in crate::engine) field_name: Ident<'a>,
     /// layers
-    layers: Vec<LayerSpec<'a>>,
+    pub(in crate::engine) layers: Vec<LayerSpec<'a>>,
     /// is null
-    null: bool,
+    pub(in crate::engine) null: bool,
     /// is primary
-    primary: bool,
+    pub(in crate::engine) primary: bool,
 }
 
-impl<'a> Field<'a> {
+impl<'a> FieldSpec<'a> {
     pub fn new(
         field_name: Ident<'a>,
         layers: Vec<LayerSpec<'a>>,
@@ -353,7 +353,7 @@ impl<'a> Field<'a> {
         let mut layers = Vec::new();
         rfold_layers(LayerFoldState::BEGIN_IDENT, state, &mut layers);
         if state.okay() {
-            Ok(Field {
+            Ok(FieldSpec {
                 field_name: field_name.clone(),
                 layers,
                 null: is_null,
@@ -486,7 +486,7 @@ mod impls {
     use {
         super::{
             rfold_dict, rfold_layers, rfold_tymeta, DictFoldState, DictGeneric, ExpandedField,
-            Field, LayerFoldState, LayerSpec,
+            FieldSpec, LayerFoldState, LayerSpec,
         },
         crate::engine::{
             error::LangResult,
@@ -553,7 +553,7 @@ mod impls {
             Ok(Self(dict))
         }
     }
-    impl<'a> ASTNode<'a> for Field<'a> {
+    impl<'a> ASTNode<'a> for FieldSpec<'a> {
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             Self::parse(state)
         }
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 079571c9..c9b793b9 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -309,17 +309,17 @@ mod fields {
         super::*,
         crate::engine::ql::{
             ast::parse_ast_node_full,
-            ddl::syn::{Field, LayerSpec},
+            ddl::syn::{FieldSpec, LayerSpec},
             lex::Ident,
         },
     };
     #[test]
     fn field_mini() {
         let tok = lex_insecure(b"username: string").unwrap();
-        let f = parse_ast_node_full::<Field>(&tok).unwrap();
+        let f = parse_ast_node_full::<FieldSpec>(&tok).unwrap();
         assert_eq!(
             f,
-            Field::new(
+            FieldSpec::new(
                 Ident::from("username"),
                 [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                 false,
@@ -330,10 +330,10 @@ mod fields {
     #[test]
     fn field() {
         let tok = lex_insecure(b"primary username: string").unwrap();
-        let f = parse_ast_node_full::<Field>(&tok).unwrap();
+        let f = parse_ast_node_full::<FieldSpec>(&tok).unwrap();
         assert_eq!(
             f,
-            Field::new(
+            FieldSpec::new(
                 Ident::from("username"),
                 [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                 false,
@@ -352,10 +352,10 @@ mod fields {
             ",
         )
         .unwrap();
-        let f = parse_ast_node_full::<Field>(&tok).unwrap();
+        let f = parse_ast_node_full::<FieldSpec>(&tok).unwrap();
         assert_eq!(
             f,
-            Field::new(
+            FieldSpec::new(
                 Ident::from("username"),
                 [LayerSpec::new(
                     Ident::from("string"),
@@ -384,10 +384,10 @@ mod fields {
             ",
         )
         .unwrap();
-        let f = parse_ast_node_full::<Field>(&tok).unwrap();
+        let f = parse_ast_node_full::<FieldSpec>(&tok).unwrap();
         assert_eq!(
             f,
-            Field::new(
+            FieldSpec::new(
                 Ident::from("notes"),
                 [
                     LayerSpec::new(
@@ -417,7 +417,7 @@ mod schemas {
         ast::parse_ast_node_full,
         ddl::{
             crt::CreateModel,
-            syn::{Field, LayerSpec},
+            syn::{FieldSpec, LayerSpec},
         },
     };
     #[test]
@@ -441,13 +441,13 @@ mod schemas {
             CreateModel::new(
                 Ident::from("mymodel"),
                 vec![
-                    Field::new(
+                    FieldSpec::new(
                         Ident::from("username"),
                         vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
                         false,
                         true,
                     ),
-                    Field::new(
+                    FieldSpec::new(
                         Ident::from("password"),
                         vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         false,
@@ -480,19 +480,19 @@ mod schemas {
             CreateModel::new(
                 Ident::from("mymodel"),
                 vec![
-                    Field::new(
+                    FieldSpec::new(
                         Ident::from("username"),
                         vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
                         false,
                         true,
                     ),
-                    Field::new(
+                    FieldSpec::new(
                         Ident::from("password"),
                         vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         false,
                         false,
                     ),
-                    Field::new(
+                    FieldSpec::new(
                         Ident::from("profile_pic"),
                         vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         true,
@@ -530,25 +530,25 @@ mod schemas {
             CreateModel::new(
                 Ident::from("mymodel"),
                 vec![
-                    Field::new(
+                    FieldSpec::new(
                         Ident::from("username"),
                         vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
                         false,
                         true
                     ),
-                    Field::new(
+                    FieldSpec::new(
                         Ident::from("password"),
                         vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         false,
                         false
                     ),
-                    Field::new(
+                    FieldSpec::new(
                         Ident::from("profile_pic"),
                         vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         true,
                         false
                     ),
-                    Field::new(
+                    FieldSpec::new(
                         Ident::from("notes"),
                         vec![
                             LayerSpec::new(Ident::from("string"), null_dict! {}),
@@ -599,25 +599,25 @@ mod schemas {
             CreateModel::new(
                 Ident::from("mymodel"),
                 vec![
-                    Field::new(
+                    FieldSpec::new(
                         Ident::from("username"),
                         vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
                         false,
                         true
                     ),
-                    Field::new(
+                    FieldSpec::new(
                         Ident::from("password"),
                         vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         false,
                         false
                     ),
-                    Field::new(
+                    FieldSpec::new(
                         Ident::from("profile_pic"),
                         vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
                         true,
                         false
                     ),
-                    Field::new(
+                    FieldSpec::new(
                         Ident::from("notes"),
                         vec![
                             LayerSpec::new(Ident::from("string"), null_dict! {}),

From 64c8b55b93467323a41068e663a91686cf1ccc42 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 5 Mar 2023 04:56:12 -0800
Subject: [PATCH 153/310] Add exec for drop space

---
 server/src/engine/core/mod.rs             |  5 ++---
 server/src/engine/core/model/mod.rs       |  2 +-
 server/src/engine/core/space.rs           | 27 ++++++++++++++++++-----
 server/src/engine/core/tests/model/crt.rs |  2 +-
 server/src/engine/core/tests/space/mod.rs |  5 +++--
 server/src/engine/error.rs                |  2 ++
 server/src/engine/idx/mod.rs              |  4 ++++
 server/src/engine/idx/stdhm.rs            | 18 +++++++++++++++
 server/src/engine/idx/stord/mod.rs        | 15 +++++++++++++
 server/src/engine/ql/ddl/drop.rs          |  4 ++--
 10 files changed, 69 insertions(+), 15 deletions(-)

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index c8966d1b..68a4cb8c 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -32,7 +32,6 @@ mod tests;
 use {
     crate::engine::{core::space::Space, data::ItemID, idx::IndexST},
     parking_lot::RwLock,
-    std::sync::Arc,
 };
 
 pub use model::cell::Datacell;
@@ -44,11 +43,11 @@ type RWLIdx<K, V> = RwLock<IndexST<K, V>>;
 // FIXME(@ohsayan): Make sure we update what all structures we're making use of here
 
 pub struct GlobalNS {
-    spaces: RWLIdx<ItemID, Arc<Space>>,
+    spaces: RWLIdx<ItemID, Space>,
 }
 
 impl GlobalNS {
-    pub(self) fn _spaces(&self) -> &RWLIdx<ItemID, Arc<Space>> {
+    pub(self) fn _spaces(&self) -> &RWLIdx<ItemID, Space> {
         &self.spaces
     }
     pub fn empty() -> Self {
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index fed63f0e..e14e2edc 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -50,7 +50,7 @@ pub struct ModelView {
 }
 
 impl ModelView {
-    pub fn create(CreateModel { fields, props, .. }: CreateModel) -> DatabaseResult<Self> {
+    pub fn create_process(CreateModel { fields, props, .. }: CreateModel) -> DatabaseResult<Self> {
         let mut okay = props.is_empty() & !fields.is_empty();
         // validate fields
         let mut field_spec = fields.into_iter();
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index c538fba4..dbc72126 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -30,16 +30,15 @@ use {
         data::{md_dict, DictEntryGeneric, MetaDict},
         error::{DatabaseError, DatabaseResult},
         idx::{IndexST, STIndex},
-        ql::ddl::{alt::AlterSpace, crt::CreateSpace},
+        ql::ddl::{alt::AlterSpace, crt::CreateSpace, drop::DropSpace},
     },
     parking_lot::RwLock,
-    std::sync::Arc,
 };
 
 #[derive(Debug)]
 /// A space with the model namespace
 pub struct Space {
-    mns: RWLIdx<ItemID, Arc<ModelView>>,
+    mns: RWLIdx<ItemID, ModelView>,
     pub(super) meta: SpaceMeta,
 }
 
@@ -79,7 +78,7 @@ impl Space {
         Space::new(Default::default(), SpaceMeta::with_env(into_dict! {}))
     }
     #[inline(always)]
-    pub fn new(mns: IndexST<ItemID, Arc<ModelView>>, meta: SpaceMeta) -> Self {
+    pub fn new(mns: IndexST<ItemID, ModelView>, meta: SpaceMeta) -> Self {
         Self {
             mns: RWLIdx::new(mns),
             meta,
@@ -117,7 +116,7 @@ impl Space {
     pub fn exec_create(gns: &super::GlobalNS, space: CreateSpace) -> DatabaseResult<()> {
         let ProcedureCreate { space_name, space } = Self::validate_create(space)?;
         let mut wl = gns._spaces().write();
-        if wl.st_insert(space_name, Arc::new(space)) {
+        if wl.st_insert(space_name, space) {
             Ok(())
         } else {
             Err(DatabaseError::DdlSpaceAlreadyExists)
@@ -131,7 +130,7 @@ impl Space {
             mut updated_props,
         }: AlterSpace,
     ) -> DatabaseResult<()> {
-        match gns._spaces().read().st_get_cloned(space_name.as_bytes()) {
+        match gns._spaces().read().st_get(space_name.as_bytes()) {
             Some(space) => {
                 let mut space_env = space.meta.env.write();
                 match updated_props.remove(SpaceMeta::KEY_ENV) {
@@ -149,6 +148,22 @@ impl Space {
             None => Err(DatabaseError::DdlSpaceNotFound),
         }
     }
+    pub fn exec_drop(
+        gns: &super::GlobalNS,
+        DropSpace { space, force: _ }: DropSpace,
+    ) -> DatabaseResult<()> {
+        // TODO(@ohsayan): force remove option
+        // TODO(@ohsayan): should a drop space block the entire global table?
+        match gns
+            ._spaces()
+            .write()
+            .st_delete_if(space.as_bytes(), |space| space.mns.read().len() == 0)
+        {
+            Some(true) => Ok(()),
+            Some(false) => Err(DatabaseError::DdlSpaceRemoveNonEmpty),
+            None => Err(DatabaseError::DdlSpaceNotFound),
+        }
+    }
 }
 
 #[cfg(test)]
diff --git a/server/src/engine/core/tests/model/crt.rs b/server/src/engine/core/tests/model/crt.rs
index f55a3051..619dc2ff 100644
--- a/server/src/engine/core/tests/model/crt.rs
+++ b/server/src/engine/core/tests/model/crt.rs
@@ -35,7 +35,7 @@ use crate::engine::{
 fn create(s: &str) -> DatabaseResult<ModelView> {
     let tok = lex_insecure(s.as_bytes()).unwrap();
     let create_model = parse_ast_node_full(&tok[2..]).unwrap();
-    ModelView::create(create_model)
+    ModelView::create_process(create_model)
 }
 
 #[test]
diff --git a/server/src/engine/core/tests/space/mod.rs b/server/src/engine/core/tests/space/mod.rs
index ede699f1..8c210b3b 100644
--- a/server/src/engine/core/tests/space/mod.rs
+++ b/server/src/engine/core/tests/space/mod.rs
@@ -46,8 +46,9 @@ fn exec_verify(
     let tok = lex(query.as_bytes()).unwrap();
     let ast_node = compile_test(&tok).unwrap();
     let (res, space_name) = exec(gns, ast_node);
-    let space_arc = gns._spaces().read().st_get_cloned(space_name.as_bytes());
-    let r = res.map(|_| space_arc.as_deref().unwrap());
+    let rl = gns._spaces().read();
+    let space_ref = rl.st_get(space_name.as_bytes());
+    let r = res.map(|_| space_ref.unwrap());
     verify(r);
 }
 
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 577f27ac..f6ce63e2 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -88,6 +88,8 @@ pub enum DatabaseError {
     DdlSpaceAlreadyExists,
     /// the space doesn't exist
     DdlSpaceNotFound,
+    /// the space that we attempted to remove is non-empty
+    DdlSpaceRemoveNonEmpty,
     /// bad definition for some typedef in a model
     DdlModelInvalidTypeDefinition,
     /// bad model definition; most likely an illegal primary key
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 5912dcc2..82922012 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -272,6 +272,10 @@ pub trait STIndex<K: ?Sized, V>: IndexBaseSpec<K, V> {
         Q: ?Sized + AsKey;
     /// Removes the entry and returns it, if it exists
     fn st_delete_return<Q>(&mut self, key: &Q) -> Option<V>
+    where
+        K: AsKey + Borrow<Q>,
+        Q: ?Sized + AsKey;
+    fn st_delete_if<Q>(&mut self, key: &Q, iff: impl Fn(&V) -> bool) -> Option<bool>
     where
         K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKey;
diff --git a/server/src/engine/idx/stdhm.rs b/server/src/engine/idx/stdhm.rs
index a78fcee3..e93cbf41 100644
--- a/server/src/engine/idx/stdhm.rs
+++ b/server/src/engine/idx/stdhm.rs
@@ -173,6 +173,24 @@ where
         self.remove(key)
     }
 
+    fn st_delete_if<Q>(&mut self, key: &Q, iff: impl Fn(&V) -> bool) -> Option<bool>
+    where
+        K: AsKey + Borrow<Q>,
+        Q: ?Sized + AsKey,
+    {
+        match self.get(key) {
+            Some(v) => {
+                if iff(v) {
+                    self.remove(key);
+                    Some(true)
+                } else {
+                    Some(false)
+                }
+            }
+            None => None,
+        }
+    }
+
     fn st_iter_kv<'a>(&'a self) -> Self::IterKV<'a> {
         self.iter()
     }
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index 5ee485d1..69485a59 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -656,6 +656,21 @@ where
         self._remove(key)
     }
 
+    fn st_delete_if<Q>(&mut self, key: &Q, iff: impl Fn(&V) -> bool) -> Option<bool>
+    where
+        K: AsKey + Borrow<Q>,
+        Q: ?Sized + AsKey,
+    {
+        match self._get(key) {
+            Some(v) if iff(v) => {
+                self._remove(key);
+                Some(true)
+            }
+            Some(_) => Some(false),
+            None => None,
+        }
+    }
+
     fn st_iter_kv<'a>(&'a self) -> Self::IterKV<'a> {
         self._iter_unord_kv()
     }
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index 9f18efd3..6d33bd07 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -35,8 +35,8 @@ use crate::engine::{
 #[derive(Debug, PartialEq)]
 /// A generic representation of `drop` query
 pub struct DropSpace<'a> {
-    pub(super) space: Ident<'a>,
-    pub(super) force: bool,
+    pub(in crate::engine) space: Ident<'a>,
+    pub(in crate::engine) force: bool,
 }
 
 impl<'a> DropSpace<'a> {

From 7069ab7bfbb0aabdc4c3d74c3caaf07b0a04b17f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 5 Mar 2023 08:04:03 -0800
Subject: [PATCH 154/310] Add exec for create model

---
 server/src/engine/core/mod.rs             |  19 +-
 server/src/engine/core/model/mod.rs       |  34 ++-
 server/src/engine/core/space.rs           |  23 +-
 server/src/engine/core/tests/model/crt.rs | 245 +++++++++++++++-------
 server/src/engine/core/tests/space/mod.rs |   2 +-
 server/src/engine/error.rs                |   2 +
 6 files changed, 238 insertions(+), 87 deletions(-)

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 68a4cb8c..67c535f0 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -28,12 +28,12 @@ mod model;
 mod space;
 #[cfg(test)]
 mod tests;
-
+// imports
 use {
     crate::engine::{core::space::Space, data::ItemID, idx::IndexST},
     parking_lot::RwLock,
 };
-
+// re-exports
 pub use model::cell::Datacell;
 
 /// Use this for now since it substitutes for a file lock (and those syscalls are expensive),
@@ -43,16 +43,23 @@ type RWLIdx<K, V> = RwLock<IndexST<K, V>>;
 // FIXME(@ohsayan): Make sure we update what all structures we're making use of here
 
 pub struct GlobalNS {
-    spaces: RWLIdx<ItemID, Space>,
+    index_space: RWLIdx<ItemID, Space>,
 }
 
 impl GlobalNS {
-    pub(self) fn _spaces(&self) -> &RWLIdx<ItemID, Space> {
-        &self.spaces
+    pub(self) fn spaces(&self) -> &RWLIdx<ItemID, Space> {
+        &self.index_space
     }
     pub fn empty() -> Self {
         Self {
-            spaces: RWLIdx::default(),
+            index_space: RWLIdx::default(),
         }
     }
+    #[cfg(test)]
+    pub(self) fn test_new_empty_space(&self, space_id: &str) -> bool {
+        use super::idx::STIndex;
+        self.index_space
+            .write()
+            .st_insert(space_id.into(), Space::empty())
+    }
 }
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index e14e2edc..aed0e2c7 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -28,7 +28,10 @@ pub mod cell;
 
 use crate::engine::{
     core::model::cell::Datacell,
-    data::tag::{DataTag, FullTag, TagClass, TagSelector},
+    data::{
+        tag::{DataTag, FullTag, TagClass, TagSelector},
+        ItemID,
+    },
     error::{DatabaseError, DatabaseResult},
     idx::{IndexSTSeqCns, STIndex, STIndexSeq},
     mem::VInline,
@@ -50,8 +53,20 @@ pub struct ModelView {
 }
 
 impl ModelView {
-    pub fn create_process(CreateModel { fields, props, .. }: CreateModel) -> DatabaseResult<Self> {
-        let mut okay = props.is_empty() & !fields.is_empty();
+    pub fn fields(&self) -> &IndexSTSeqCns<Box<str>, Field> {
+        &self.fields
+    }
+}
+
+impl ModelView {
+    pub fn process_create(
+        CreateModel {
+            model_name,
+            fields,
+            props,
+        }: CreateModel,
+    ) -> DatabaseResult<Self> {
+        let mut okay = props.is_empty() & !fields.is_empty() & ItemID::check(&model_name);
         // validate fields
         let mut field_spec = fields.into_iter();
         let mut fields = IndexSTSeqCns::with_capacity(field_spec.len());
@@ -86,6 +101,19 @@ impl ModelView {
         }
         Err(DatabaseError::DdlModelBadDefinition)
     }
+    pub fn exec_create(
+        gns: &super::GlobalNS,
+        space: &[u8],
+        stmt: CreateModel,
+    ) -> DatabaseResult<()> {
+        let name = stmt.model_name;
+        let model = Self::process_create(stmt)?;
+        let space_rl = gns.spaces().read();
+        let Some(space) = space_rl.get(space) else {
+            return Err(DatabaseError::DdlSpaceNotFound)
+        };
+        space.create_model(ItemID::new(&name), model)
+    }
 }
 
 /*
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index dbc72126..ae8faa6a 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -73,6 +73,19 @@ impl ProcedureCreate {
     }
 }
 
+impl Space {
+    pub fn create_model(&self, name: ItemID, model: ModelView) -> DatabaseResult<()> {
+        if self.mns.write().st_insert(name, model) {
+            Ok(())
+        } else {
+            Err(DatabaseError::DdlModelAlreadyExists)
+        }
+    }
+    pub(super) fn models(&self) -> &RWLIdx<ItemID, ModelView> {
+        &self.mns
+    }
+}
+
 impl Space {
     pub fn empty() -> Self {
         Space::new(Default::default(), SpaceMeta::with_env(into_dict! {}))
@@ -86,7 +99,7 @@ impl Space {
     }
     #[inline]
     /// Validate a `create` stmt
-    fn validate_create(
+    fn process_create(
         CreateSpace {
             space_name,
             mut props,
@@ -114,8 +127,8 @@ impl Space {
     }
     /// Execute a `create` stmt
     pub fn exec_create(gns: &super::GlobalNS, space: CreateSpace) -> DatabaseResult<()> {
-        let ProcedureCreate { space_name, space } = Self::validate_create(space)?;
-        let mut wl = gns._spaces().write();
+        let ProcedureCreate { space_name, space } = Self::process_create(space)?;
+        let mut wl = gns.spaces().write();
         if wl.st_insert(space_name, space) {
             Ok(())
         } else {
@@ -130,7 +143,7 @@ impl Space {
             mut updated_props,
         }: AlterSpace,
     ) -> DatabaseResult<()> {
-        match gns._spaces().read().st_get(space_name.as_bytes()) {
+        match gns.spaces().read().st_get(space_name.as_bytes()) {
             Some(space) => {
                 let mut space_env = space.meta.env.write();
                 match updated_props.remove(SpaceMeta::KEY_ENV) {
@@ -155,7 +168,7 @@ impl Space {
         // TODO(@ohsayan): force remove option
         // TODO(@ohsayan): should a drop space block the entire global table?
         match gns
-            ._spaces()
+            .spaces()
             .write()
             .st_delete_if(space.as_bytes(), |space| space.mns.read().len() == 0)
         {
diff --git a/server/src/engine/core/tests/model/crt.rs b/server/src/engine/core/tests/model/crt.rs
index 619dc2ff..af0d296b 100644
--- a/server/src/engine/core/tests/model/crt.rs
+++ b/server/src/engine/core/tests/model/crt.rs
@@ -24,92 +24,193 @@
  *
 */
 
-use crate::engine::{
-    core::model::{Field, Layer, ModelView},
-    data::tag::{DataTag, FullTag},
-    error::{DatabaseError, DatabaseResult},
-    idx::STIndexSeq,
-    ql::{ast::parse_ast_node_full, tests::lex_insecure},
-};
-
-fn create(s: &str) -> DatabaseResult<ModelView> {
-    let tok = lex_insecure(s.as_bytes()).unwrap();
-    let create_model = parse_ast_node_full(&tok[2..]).unwrap();
-    ModelView::create_process(create_model)
-}
+mod validation {
+    use crate::engine::{
+        core::model::{Field, Layer, ModelView},
+        data::tag::{DataTag, FullTag},
+        error::{DatabaseError, DatabaseResult},
+        idx::STIndexSeq,
+        ql::{ast::parse_ast_node_full, tests::lex_insecure},
+    };
 
-#[test]
-fn simple() {
-    let ModelView {
-        p_key,
-        p_tag,
-        fields,
-    } = create("create model mymodel(username: string, password: binary)").unwrap();
-    assert_eq!(p_key.as_ref(), "username");
-    assert_eq!(p_tag, FullTag::STR);
-    assert_eq!(
-        fields.stseq_ord_value().cloned().collect::<Vec<Field>>(),
-        [
-            Field::new_test([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
-            Field::new_test([Layer::new_test(FullTag::BIN, [0; 2])].into(), false)
-        ]
-    );
-}
+    fn create(s: &str) -> DatabaseResult<ModelView> {
+        let tok = lex_insecure(s.as_bytes()).unwrap();
+        let create_model = parse_ast_node_full(&tok[2..]).unwrap();
+        ModelView::process_create(create_model)
+    }
+    #[test]
+    fn simple() {
+        let ModelView {
+            p_key,
+            p_tag,
+            fields,
+        } = create("create model mymodel(username: string, password: binary)").unwrap();
+        assert_eq!(p_key.as_ref(), "username");
+        assert_eq!(p_tag, FullTag::STR);
+        assert_eq!(
+            fields.stseq_ord_value().cloned().collect::<Vec<Field>>(),
+            [
+                Field::new_test([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
+                Field::new_test([Layer::new_test(FullTag::BIN, [0; 2])].into(), false)
+            ]
+        );
+    }
 
-#[test]
-fn idiotic_order() {
-    let ModelView {
-        p_key,
-        p_tag,
-        fields,
-    } = create("create model mymodel(password: binary, primary username: string)").unwrap();
-    assert_eq!(p_key.as_ref(), "username");
-    assert_eq!(p_tag, FullTag::STR);
-    assert_eq!(
-        fields.stseq_ord_value().cloned().collect::<Vec<Field>>(),
-        [
-            Field::new_test([Layer::new_test(FullTag::BIN, [0; 2])].into(), false),
-            Field::new_test([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
-        ]
-    );
-}
+    #[test]
+    fn idiotic_order() {
+        let ModelView {
+            p_key,
+            p_tag,
+            fields,
+        } = create("create model mymodel(password: binary, primary username: string)").unwrap();
+        assert_eq!(p_key.as_ref(), "username");
+        assert_eq!(p_tag, FullTag::STR);
+        assert_eq!(
+            fields.stseq_ord_value().cloned().collect::<Vec<Field>>(),
+            [
+                Field::new_test([Layer::new_test(FullTag::BIN, [0; 2])].into(), false),
+                Field::new_test([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
+            ]
+        );
+    }
 
-#[test]
-fn duplicate_primary_key() {
-    assert_eq!(
-        create("create model mymodel(primary username: string, primary contract_location: binary)")
+    #[test]
+    fn duplicate_primary_key() {
+        assert_eq!(
+            create(
+                "create model mymodel(primary username: string, primary contract_location: binary)"
+            )
             .unwrap_err(),
-        DatabaseError::DdlModelBadDefinition
-    );
-}
+            DatabaseError::DdlModelBadDefinition
+        );
+    }
 
-#[test]
-fn duplicate_fields() {
-    assert_eq!(
-        create("create model mymodel(primary username: string, username: binary)").unwrap_err(),
-        DatabaseError::DdlModelBadDefinition
-    );
-}
+    #[test]
+    fn duplicate_fields() {
+        assert_eq!(
+            create("create model mymodel(primary username: string, username: binary)").unwrap_err(),
+            DatabaseError::DdlModelBadDefinition
+        );
+    }
 
-#[test]
-fn illegal_props() {
-    assert_eq!(
+    #[test]
+    fn illegal_props() {
+        assert_eq!(
         create("create model mymodel(primary username: string, password: binary) with { lol_prop: false }").unwrap_err(),
         DatabaseError::DdlModelBadDefinition
     );
-}
+    }
 
-#[test]
-fn illegal_pk() {
-    assert_eq!(
+    #[test]
+    fn illegal_pk() {
+        assert_eq!(
         create(
             "create model mymodel(primary username_bytes: list { type: uint8 }, password: binary)"
         )
         .unwrap_err(),
         DatabaseError::DdlModelBadDefinition
     );
-    assert_eq!(
-        create("create model mymodel(primary username: float32, password: binary)").unwrap_err(),
-        DatabaseError::DdlModelBadDefinition
-    );
+        assert_eq!(
+            create("create model mymodel(primary username: float32, password: binary)")
+                .unwrap_err(),
+            DatabaseError::DdlModelBadDefinition
+        );
+    }
+}
+
+/*
+    Exec
+*/
+
+mod exec {
+    use crate::engine::{
+        core::{
+            model::{Field, Layer, ModelView},
+            space::Space,
+            GlobalNS,
+        },
+        data::tag::{DataTag, FullTag},
+        error::DatabaseResult,
+        idx::{STIndex, STIndexSeq},
+        ql::{ast::parse_ast_node_full, tests::lex_insecure},
+    };
+
+    const SPACE: &str = "myspace";
+
+    pub fn exec_create(
+        gns: &GlobalNS,
+        create_stmt: &str,
+        space_id: &str,
+        create_new_space: bool,
+    ) -> DatabaseResult<()> {
+        if create_new_space {
+            assert!(gns.test_new_empty_space(space_id));
+        }
+        let tok = lex_insecure(create_stmt.as_bytes()).unwrap();
+        let create_model = parse_ast_node_full(&tok[2..]).unwrap();
+        ModelView::exec_create(gns, space_id.as_bytes(), create_model)
+    }
+
+    pub fn exec_create_new_space(
+        gns: &GlobalNS,
+        create_stmt: &str,
+        space_id: &str,
+    ) -> DatabaseResult<()> {
+        exec_create(gns, create_stmt, space_id, true)
+    }
+
+    pub fn exec_create_no_create(
+        gns: &GlobalNS,
+        create_stmt: &str,
+        space_id: &str,
+    ) -> DatabaseResult<()> {
+        exec_create(gns, create_stmt, space_id, false)
+    }
+
+    fn with_space(gns: &GlobalNS, space_name: &str, f: impl Fn(&Space)) {
+        let rl = gns.spaces().read();
+        let space = rl.st_get(space_name.as_bytes()).unwrap();
+        f(space);
+    }
+
+    fn with_model(gns: &GlobalNS, space_id: &str, model_name: &str, f: impl Fn(&ModelView)) {
+        with_space(gns, space_id, |space| {
+            let space_rl = space.models().read();
+            let model = space_rl.st_get(model_name.as_bytes()).unwrap();
+            f(model)
+        })
+    }
+
+    #[test]
+    fn simple() {
+        let gns = GlobalNS::empty();
+        exec_create_new_space(
+            &gns,
+            "create model mymodel(username: string, password: binary)",
+            SPACE,
+        )
+        .unwrap();
+        with_model(&gns, SPACE, "mymodel", |model| {
+            let models: Vec<(String, Field)> = model
+                .fields()
+                .stseq_ord_kv()
+                .map(|(k, v)| (k.to_string(), v.clone()))
+                .collect();
+            assert_eq!(model.p_key.as_ref(), "username");
+            assert_eq!(model.p_tag, FullTag::STR);
+            assert_eq!(
+                models,
+                [
+                    (
+                        "username".to_string(),
+                        Field::new_test([Layer::str()].into(), false)
+                    ),
+                    (
+                        "password".to_string(),
+                        Field::new_test([Layer::bin()].into(), false)
+                    )
+                ]
+            );
+        });
+    }
 }
diff --git a/server/src/engine/core/tests/space/mod.rs b/server/src/engine/core/tests/space/mod.rs
index 8c210b3b..87c0257e 100644
--- a/server/src/engine/core/tests/space/mod.rs
+++ b/server/src/engine/core/tests/space/mod.rs
@@ -46,7 +46,7 @@ fn exec_verify(
     let tok = lex(query.as_bytes()).unwrap();
     let ast_node = compile_test(&tok).unwrap();
     let (res, space_name) = exec(gns, ast_node);
-    let rl = gns._spaces().read();
+    let rl = gns.spaces().read();
     let space_ref = rl.st_get(space_name.as_bytes());
     let r = res.map(|_| space_ref.unwrap());
     verify(r);
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index f6ce63e2..a6a9bd06 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -94,4 +94,6 @@ pub enum DatabaseError {
     DdlModelInvalidTypeDefinition,
     /// bad model definition; most likely an illegal primary key
     DdlModelBadDefinition,
+    /// the model already exists
+    DdlModelAlreadyExists,
 }

From 1c95c2e70f8891a14df5f019cb1392aadec2c2a8 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 6 Mar 2023 06:45:50 -0800
Subject: [PATCH 155/310] Implement alter model plan

---
 server/src/engine/core/model/alt.rs       | 259 ++++++++++++++++++++++
 server/src/engine/core/model/mod.rs       |   7 +-
 server/src/engine/core/tests/model/alt.rs | 177 +++++++++++++++
 server/src/engine/core/tests/model/crt.rs |  83 ++-----
 server/src/engine/core/tests/model/mod.rs |  58 +++++
 server/src/engine/error.rs                |  10 +
 server/src/engine/idx/stdhm.rs            |   7 +
 server/src/engine/idx/stord/mod.rs        |   7 +
 server/src/engine/mem/mod.rs              |  55 +++++
 server/src/engine/ql/ddl/alt.rs           |   4 +-
 server/src/engine/ql/ddl/syn.rs           |   6 +-
 server/src/util/mod.rs                    |   7 +
 12 files changed, 607 insertions(+), 73 deletions(-)
 create mode 100644 server/src/engine/core/model/alt.rs
 create mode 100644 server/src/engine/core/tests/model/alt.rs

diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
new file mode 100644
index 00000000..e077bd4f
--- /dev/null
+++ b/server/src/engine/core/model/alt.rs
@@ -0,0 +1,259 @@
+/*
+ * Created on Sun Mar 05 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{Field, Layer, ModelView},
+    crate::{
+        engine::{
+            data::{
+                tag::{DataTag, TagClass},
+                DictEntryGeneric,
+            },
+            error::{DatabaseError, DatabaseResult},
+            idx::{IndexST, IndexSTSeqCns, STIndex},
+            ql::{
+                ddl::{
+                    alt::{AlterKind, AlterModel},
+                    syn::{ExpandedField, LayerSpec},
+                },
+                lex::Ident,
+            },
+        },
+        util,
+    },
+    std::collections::{HashMap, HashSet},
+};
+
+#[derive(Debug, PartialEq)]
+pub(in crate::engine::core) struct AlterPlan<'a> {
+    pub(in crate::engine::core) model: Ident<'a>,
+    pub(in crate::engine::core) no_lock: bool,
+    pub(in crate::engine::core) action: AlterAction<'a>,
+}
+
+#[derive(Debug, PartialEq)]
+pub(in crate::engine::core) enum AlterAction<'a> {
+    Ignore,
+    Add(IndexSTSeqCns<Box<str>, Field>),
+    Update(IndexST<Box<str>, Field>),
+    Remove(Box<[Ident<'a>]>),
+}
+
+macro_rules! can_ignore {
+    (AlterAction::$variant:ident($expr:expr)) => {
+        if crate::engine::mem::StatelessLen::stateless_empty(&$expr) {
+            AlterAction::Ignore
+        } else {
+            AlterAction::$variant($expr)
+        }
+    };
+}
+
+impl ModelView {
+    fn no_field(&self, new: &str) -> bool {
+        !self.fields().st_contains(new)
+    }
+    fn is_pk(&self, new: &str) -> bool {
+        self.p_key.as_bytes() == new.as_bytes()
+    }
+    fn not_pk(&self, new: &str) -> bool {
+        !self.is_pk(new)
+    }
+    fn guard_pk(&self, new: &str) -> DatabaseResult<()> {
+        if self.is_pk(new) {
+            Err(DatabaseError::DdlModelAlterProtectedField)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+fn check_nullable(props: &mut HashMap<Box<str>, Option<DictEntryGeneric>>) -> DatabaseResult<bool> {
+    match props.remove("nullable") {
+        Some(Some(DictEntryGeneric::Lit(b))) if b.kind() == TagClass::Bool => Ok(b.bool()),
+        Some(_) => Err(DatabaseError::DdlModelAlterBadProperty),
+        None => Ok(false),
+    }
+}
+
+impl<'a> AlterPlan<'a> {
+    pub fn fdeltas(
+        mv: &ModelView,
+        AlterModel { model, kind }: AlterModel<'a>,
+    ) -> DatabaseResult<AlterPlan<'a>> {
+        let mut no_lock = true;
+        let mut okay = true;
+        let action = match kind {
+            AlterKind::Remove(r) => {
+                let mut x = HashSet::new();
+                if !r.iter().all(|id| x.insert(id.as_str())) {
+                    return Err(DatabaseError::DdlModelAlterBad);
+                }
+                let mut not_found = false;
+                if r.iter().all(|id| {
+                    let not_pk = mv.not_pk(id);
+                    let exists = mv.fields().st_contains(id.as_str());
+                    not_found = !exists;
+                    not_pk & exists
+                }) {
+                    can_ignore!(AlterAction::Remove(r))
+                } else {
+                    if not_found {
+                        return Err(DatabaseError::DdlModelAlterFieldNotFound);
+                    } else {
+                        return Err(DatabaseError::DdlModelAlterProtectedField);
+                    }
+                }
+            }
+            AlterKind::Add(new_fields) => {
+                let mut fields = util::bx_to_vec(new_fields).into_iter();
+                let mut add = IndexSTSeqCns::with_capacity(fields.len());
+                while (fields.len() != 0) & okay {
+                    let ExpandedField {
+                        field_name,
+                        layers,
+                        mut props,
+                    } = fields.next().unwrap();
+                    okay &= mv.not_pk(&field_name);
+                    okay &= mv.no_field(&field_name);
+                    let is_nullable = check_nullable(&mut props)?;
+                    let layers = Field::parse_layers(layers, is_nullable)?;
+                    okay &= add.st_insert(field_name.to_string().into_boxed_str(), layers);
+                }
+                can_ignore!(AlterAction::Add(add))
+            }
+            AlterKind::Update(updated_fields) => {
+                let updated_fields = util::bx_to_vec::<ExpandedField<'a>>(updated_fields);
+                let mut updated_fields = updated_fields.into_iter();
+                let mut any_delta = 0;
+                let mut new_fields = IndexST::new();
+                while (updated_fields.len() != 0) & okay {
+                    let ExpandedField {
+                        field_name,
+                        layers,
+                        mut props,
+                    } = updated_fields.next().unwrap();
+                    // enforce pk
+                    mv.guard_pk(&field_name)?;
+                    // get the current field
+                    let Some(current_field) = mv.fields().st_get(field_name.as_str()) else {
+                        return Err(DatabaseError::DdlModelAlterFieldNotFound);
+                    };
+                    // check props
+                    let is_nullable = check_nullable(&mut props)?;
+                    okay &= props.is_empty();
+                    // check layers
+                    let (anydelta, new_field) =
+                        Self::ldeltas(current_field, layers, is_nullable, &mut no_lock, &mut okay)?;
+                    any_delta += anydelta as usize;
+                    okay &=
+                        new_fields.st_insert(field_name.to_string().into_boxed_str(), new_field);
+                }
+                if any_delta == 0 {
+                    AlterAction::Ignore
+                } else {
+                    AlterAction::Update(new_fields)
+                }
+            }
+        };
+        if okay {
+            Ok(Self {
+                model,
+                action,
+                no_lock,
+            })
+        } else {
+            Err(DatabaseError::DdlModelAlterBad)
+        }
+    }
+    fn ldeltas(
+        current: &Field,
+        layers: Vec<LayerSpec<'a>>,
+        nullable: bool,
+        super_nlck: &mut bool,
+        super_okay: &mut bool,
+    ) -> DatabaseResult<(bool, Field)> {
+        #[inline(always)]
+        fn classeq(current: &Layer, new: &Layer, class: TagClass) -> bool {
+            // KIDDOS, LEARN SOME RELATIONS BEFORE WRITING CODE
+            (current.tag.tag_class() == new.tag.tag_class()) & (current.tag.tag_class() == class)
+        }
+        #[inline(always)]
+        fn interop(current: &Layer, new: &Layer) -> bool {
+            classeq(current, new, TagClass::UnsignedInt)
+                | classeq(current, new, TagClass::SignedInt)
+                | classeq(current, new, TagClass::Float)
+        }
+        if layers.len() > current.layers().len() {
+            // simply a dumb tomato; ELIMINATE THESE DUMB TOMATOES
+            return Err(DatabaseError::DdlModelAlterBad);
+        }
+        let mut no_lock = !(current.is_nullable() & !nullable);
+        let mut deltasize = (current.is_nullable() ^ nullable) as usize;
+        let mut okay = true;
+        let mut new_field = current.clone();
+        new_field.nullable = nullable;
+        let mut zipped_layers = layers
+            .into_iter()
+            .rev()
+            .zip(current.layers())
+            .zip(new_field.layers.iter_mut());
+        // check all layers
+        while (zipped_layers.len() != 0) & okay {
+            let ((LayerSpec { ty, props }, current_layer), new_layer) =
+                zipped_layers.next().unwrap();
+            // actually parse the new layer
+            okay &= props.is_empty();
+            let Some(new_parsed_layer) = Layer::get_layer(&ty) else {
+                return Err(DatabaseError::DdlModelAlterBadTypedef)
+            };
+            match (
+                current_layer.tag.tag_selector(),
+                new_parsed_layer.tag.tag_selector(),
+            ) {
+                (current_tag, new_tag) if current_tag == new_tag => {
+                    // no delta
+                }
+                (current_selector, new_selector) if interop(current_layer, &new_parsed_layer) => {
+                    no_lock &= new_selector >= current_selector;
+                    deltasize += (new_selector != current_selector) as usize;
+                }
+                _ => {
+                    // can't cast this directly
+                    return Err(DatabaseError::DdlModelAlterBadTypedef);
+                }
+            }
+            *new_layer = new_parsed_layer;
+        }
+        *super_nlck &= no_lock;
+        *super_okay &= okay;
+        if okay {
+            Ok((deltasize != 0, new_field))
+        } else {
+            Err(DatabaseError::DdlModelAlterBad)
+        }
+    }
+}
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index aed0e2c7..b28756fc 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -24,6 +24,7 @@
  *
 */
 
+pub(super) mod alt;
 pub mod cell;
 
 use crate::engine::{
@@ -148,10 +149,12 @@ pub struct Field {
 }
 
 impl Field {
-    #[cfg(test)]
-    pub fn new_test(layers: VInline<1, Layer>, nullable: bool) -> Self {
+    pub fn new(layers: VInline<1, Layer>, nullable: bool) -> Self {
         Self { layers, nullable }
     }
+    pub fn is_nullable(&self) -> bool {
+        self.nullable
+    }
     pub fn layers(&self) -> &[Layer] {
         &self.layers
     }
diff --git a/server/src/engine/core/tests/model/alt.rs b/server/src/engine/core/tests/model/alt.rs
new file mode 100644
index 00000000..d2fbf581
--- /dev/null
+++ b/server/src/engine/core/tests/model/alt.rs
@@ -0,0 +1,177 @@
+/*
+ * Created on Mon Mar 06 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod plan {
+    use crate::engine::{
+        core::{
+            model::{
+                alt::{AlterAction, AlterPlan},
+                Field, Layer,
+            },
+            tests::model::create,
+        },
+        error::{DatabaseError, DatabaseResult},
+        ql::{ast::parse_ast_node_full, tests::lex_insecure},
+    };
+    fn with_plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) -> DatabaseResult<()> {
+        let model = create(model)?;
+        let tok = lex_insecure(plan.as_bytes()).unwrap();
+        let alter = parse_ast_node_full(&tok[2..]).unwrap();
+        let mv = AlterPlan::fdeltas(&model, alter)?;
+        Ok(f(mv))
+    }
+    fn plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) {
+        with_plan(model, plan, f).unwrap()
+    }
+    /*
+        Simple
+    */
+    #[test]
+    fn simple_add() {
+        plan(
+            "create model mymodel(username: string, password: binary)",
+            "alter model mymodel add myfield { type: string, nullable: true }",
+            |plan| {
+                assert_eq!(plan.model.as_str(), "mymodel");
+                assert!(plan.no_lock);
+                assert_eq!(
+                    plan.action,
+                    AlterAction::Add(
+                        into_dict! { "myfield" => Field::new([Layer::str()].into(), true) }
+                    )
+                )
+            },
+        )
+    }
+    #[test]
+    fn simple_remove() {
+        plan(
+            "create model mymodel(username: string, password: binary, useless_field: uint8)",
+            "alter model mymodel remove useless_field",
+            |plan| {
+                assert_eq!(plan.model.as_str(), "mymodel");
+                assert!(plan.no_lock);
+                assert_eq!(
+                    plan.action,
+                    AlterAction::Remove(["useless_field".into()].into())
+                )
+            },
+        );
+    }
+    #[test]
+    fn simple_update() {
+        // FREEDOM! DAMN THE PASSWORD!
+        plan(
+            "create model mymodel(username: string, password: binary)",
+            "alter model mymodel update password { nullable: true }",
+            |plan| {
+                assert_eq!(plan.model.as_str(), "mymodel");
+                assert!(plan.no_lock);
+                assert_eq!(
+                    plan.action,
+                    AlterAction::Update(into_dict! {
+                        "password" => Field::new([Layer::bin()].into(), true)
+                    })
+                );
+            },
+        );
+    }
+    /*
+        Illegal
+    */
+    #[test]
+    fn illegal_remove_nx() {
+        assert_eq!(
+            with_plan(
+                "create model mymodel(username: string, password: binary)",
+                "alter model mymodel remove password_e2e",
+                |_| {}
+            )
+            .unwrap_err(),
+            DatabaseError::DdlModelAlterFieldNotFound
+        );
+    }
+    #[test]
+    fn illegal_remove_pk() {
+        assert_eq!(
+            with_plan(
+                "create model mymodel(username: string, password: binary)",
+                "alter model mymodel remove username",
+                |_| {}
+            )
+            .unwrap_err(),
+            DatabaseError::DdlModelAlterProtectedField
+        );
+    }
+    #[test]
+    fn illegal_add_pk() {
+        assert_eq!(
+            with_plan(
+                "create model mymodel(username: string, password: binary)",
+                "alter model mymodel add username { type: string }",
+                |_| {}
+            )
+            .unwrap_err(),
+            DatabaseError::DdlModelAlterBad
+        );
+    }
+    #[test]
+    fn illegal_add_ex() {
+        assert_eq!(
+            with_plan(
+                "create model mymodel(username: string, password: binary)",
+                "alter model mymodel add password { type: string }",
+                |_| {}
+            )
+            .unwrap_err(),
+            DatabaseError::DdlModelAlterBad
+        );
+    }
+    #[test]
+    fn illegal_update_pk() {
+        assert_eq!(
+            with_plan(
+                "create model mymodel(username: string, password: binary)",
+                "alter model mymodel update username { type: string }",
+                |_| {}
+            )
+            .unwrap_err(),
+            DatabaseError::DdlModelAlterProtectedField
+        );
+    }
+    #[test]
+    fn illegal_update_nx() {
+        assert_eq!(
+            with_plan(
+                "create model mymodel(username: string, password: binary)",
+                "alter model mymodel update username_secret { type: string }",
+                |_| {}
+            )
+            .unwrap_err(),
+            DatabaseError::DdlModelAlterFieldNotFound
+        );
+    }
+}
diff --git a/server/src/engine/core/tests/model/crt.rs b/server/src/engine/core/tests/model/crt.rs
index af0d296b..d3c2f408 100644
--- a/server/src/engine/core/tests/model/crt.rs
+++ b/server/src/engine/core/tests/model/crt.rs
@@ -25,19 +25,16 @@
 */
 
 mod validation {
-    use crate::engine::{
-        core::model::{Field, Layer, ModelView},
-        data::tag::{DataTag, FullTag},
-        error::{DatabaseError, DatabaseResult},
-        idx::STIndexSeq,
-        ql::{ast::parse_ast_node_full, tests::lex_insecure},
+    use {
+        super::super::create,
+        crate::engine::{
+            core::model::{Field, Layer, ModelView},
+            data::tag::{DataTag, FullTag},
+            error::DatabaseError,
+            idx::STIndexSeq,
+        },
     };
 
-    fn create(s: &str) -> DatabaseResult<ModelView> {
-        let tok = lex_insecure(s.as_bytes()).unwrap();
-        let create_model = parse_ast_node_full(&tok[2..]).unwrap();
-        ModelView::process_create(create_model)
-    }
     #[test]
     fn simple() {
         let ModelView {
@@ -50,8 +47,8 @@ mod validation {
         assert_eq!(
             fields.stseq_ord_value().cloned().collect::<Vec<Field>>(),
             [
-                Field::new_test([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
-                Field::new_test([Layer::new_test(FullTag::BIN, [0; 2])].into(), false)
+                Field::new([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
+                Field::new([Layer::new_test(FullTag::BIN, [0; 2])].into(), false)
             ]
         );
     }
@@ -68,8 +65,8 @@ mod validation {
         assert_eq!(
             fields.stseq_ord_value().cloned().collect::<Vec<Field>>(),
             [
-                Field::new_test([Layer::new_test(FullTag::BIN, [0; 2])].into(), false),
-                Field::new_test([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
+                Field::new([Layer::new_test(FullTag::BIN, [0; 2])].into(), false),
+                Field::new([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
             ]
         );
     }
@@ -125,62 +122,16 @@ mod validation {
 mod exec {
     use crate::engine::{
         core::{
-            model::{Field, Layer, ModelView},
-            space::Space,
+            model::{Field, Layer},
+            tests::model::{exec_create_new_space, with_model},
             GlobalNS,
         },
         data::tag::{DataTag, FullTag},
-        error::DatabaseResult,
-        idx::{STIndex, STIndexSeq},
-        ql::{ast::parse_ast_node_full, tests::lex_insecure},
+        idx::STIndexSeq,
     };
 
     const SPACE: &str = "myspace";
 
-    pub fn exec_create(
-        gns: &GlobalNS,
-        create_stmt: &str,
-        space_id: &str,
-        create_new_space: bool,
-    ) -> DatabaseResult<()> {
-        if create_new_space {
-            assert!(gns.test_new_empty_space(space_id));
-        }
-        let tok = lex_insecure(create_stmt.as_bytes()).unwrap();
-        let create_model = parse_ast_node_full(&tok[2..]).unwrap();
-        ModelView::exec_create(gns, space_id.as_bytes(), create_model)
-    }
-
-    pub fn exec_create_new_space(
-        gns: &GlobalNS,
-        create_stmt: &str,
-        space_id: &str,
-    ) -> DatabaseResult<()> {
-        exec_create(gns, create_stmt, space_id, true)
-    }
-
-    pub fn exec_create_no_create(
-        gns: &GlobalNS,
-        create_stmt: &str,
-        space_id: &str,
-    ) -> DatabaseResult<()> {
-        exec_create(gns, create_stmt, space_id, false)
-    }
-
-    fn with_space(gns: &GlobalNS, space_name: &str, f: impl Fn(&Space)) {
-        let rl = gns.spaces().read();
-        let space = rl.st_get(space_name.as_bytes()).unwrap();
-        f(space);
-    }
-
-    fn with_model(gns: &GlobalNS, space_id: &str, model_name: &str, f: impl Fn(&ModelView)) {
-        with_space(gns, space_id, |space| {
-            let space_rl = space.models().read();
-            let model = space_rl.st_get(model_name.as_bytes()).unwrap();
-            f(model)
-        })
-    }
-
     #[test]
     fn simple() {
         let gns = GlobalNS::empty();
@@ -203,11 +154,11 @@ mod exec {
                 [
                     (
                         "username".to_string(),
-                        Field::new_test([Layer::str()].into(), false)
+                        Field::new([Layer::str()].into(), false)
                     ),
                     (
                         "password".to_string(),
-                        Field::new_test([Layer::bin()].into(), false)
+                        Field::new([Layer::bin()].into(), false)
                     )
                 ]
             );
diff --git a/server/src/engine/core/tests/model/mod.rs b/server/src/engine/core/tests/model/mod.rs
index 8b8690cb..2875b60e 100644
--- a/server/src/engine/core/tests/model/mod.rs
+++ b/server/src/engine/core/tests/model/mod.rs
@@ -24,5 +24,63 @@
  *
 */
 
+mod alt;
 mod crt;
 mod layer;
+
+use crate::engine::{
+    core::{model::ModelView, space::Space, GlobalNS},
+    error::DatabaseResult,
+    idx::STIndex,
+    ql::{ast::parse_ast_node_full, tests::lex_insecure},
+};
+
+fn create(s: &str) -> DatabaseResult<ModelView> {
+    let tok = lex_insecure(s.as_bytes()).unwrap();
+    let create_model = parse_ast_node_full(&tok[2..]).unwrap();
+    ModelView::process_create(create_model)
+}
+
+pub fn exec_create(
+    gns: &GlobalNS,
+    create_stmt: &str,
+    space_id: &str,
+    create_new_space: bool,
+) -> DatabaseResult<()> {
+    if create_new_space {
+        assert!(gns.test_new_empty_space(space_id));
+    }
+    let tok = lex_insecure(create_stmt.as_bytes()).unwrap();
+    let create_model = parse_ast_node_full(&tok[2..]).unwrap();
+    ModelView::exec_create(gns, space_id.as_bytes(), create_model)
+}
+
+pub fn exec_create_new_space(
+    gns: &GlobalNS,
+    create_stmt: &str,
+    space_id: &str,
+) -> DatabaseResult<()> {
+    exec_create(gns, create_stmt, space_id, true)
+}
+
+pub fn exec_create_no_create(
+    gns: &GlobalNS,
+    create_stmt: &str,
+    space_id: &str,
+) -> DatabaseResult<()> {
+    exec_create(gns, create_stmt, space_id, false)
+}
+
+fn with_space(gns: &GlobalNS, space_name: &str, f: impl Fn(&Space)) {
+    let rl = gns.spaces().read();
+    let space = rl.st_get(space_name.as_bytes()).unwrap();
+    f(space);
+}
+
+fn with_model(gns: &GlobalNS, space_id: &str, model_name: &str, f: impl Fn(&ModelView)) {
+    with_space(gns, space_id, |space| {
+        let space_rl = space.models().read();
+        let model = space_rl.st_get(model_name.as_bytes()).unwrap();
+        f(model)
+    })
+}
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index a6a9bd06..5c8d174b 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -96,4 +96,14 @@ pub enum DatabaseError {
     DdlModelBadDefinition,
     /// the model already exists
     DdlModelAlreadyExists,
+    /// an alter attempted to remove a protected field (usually the primary key)
+    DdlModelAlterProtectedField,
+    /// an alter model attempted to modify an invalid property/a property with an illegal value
+    DdlModelAlterBadProperty,
+    /// the alter model statement is "wrong"
+    DdlModelAlterBad,
+    /// an alter attempted to update an nx field
+    DdlModelAlterFieldNotFound,
+    /// bad type definition to alter
+    DdlModelAlterBadTypedef,
 }
diff --git a/server/src/engine/idx/stdhm.rs b/server/src/engine/idx/stdhm.rs
index e93cbf41..570d9c28 100644
--- a/server/src/engine/idx/stdhm.rs
+++ b/server/src/engine/idx/stdhm.rs
@@ -28,6 +28,7 @@
 use super::DummyMetrics;
 use {
     super::{AsKey, AsValue, AsValueClone, IndexBaseSpec, STIndex},
+    crate::engine::mem::StatelessLen,
     std::{
         borrow::Borrow,
         collections::{
@@ -203,3 +204,9 @@ where
         self.values()
     }
 }
+
+impl<K, V, S> StatelessLen for StdMap<K, V, S> {
+    fn stateless_len(&self) -> usize {
+        self.len()
+    }
+}
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index 69485a59..4cd8a6bb 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -36,6 +36,7 @@ use {
         },
     },
     super::{AsKey, AsKeyClone, AsValue, AsValueClone, IndexBaseSpec, STIndex, STIndexSeq},
+    crate::engine::mem::StatelessLen,
     std::{
         alloc::{alloc as std_alloc, dealloc as std_dealloc, Layout},
         borrow::Borrow,
@@ -750,3 +751,9 @@ impl<K: AsKey, V: AsValue + PartialEq, C: Config<K, V>> PartialEq for IndexSTSeq
                 .all(|(k, v)| other._get(k).unwrap().eq(v))
     }
 }
+
+impl<K, V, C: Config<K, V>> StatelessLen for IndexSTSeqDll<K, V, C> {
+    fn stateless_len(&self) -> usize {
+        self.len()
+    }
+}
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index 9190cdda..fc9a0bad 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -49,3 +49,58 @@ pub struct NativeDword([usize; 2]);
 pub struct NativeTword([usize; 3]);
 /// Native quad pointer width (note, native != arch native, but host native)
 pub struct NativeQword([usize; 4]);
+
+pub trait StatelessLen {
+    fn stateless_len(&self) -> usize;
+    fn stateless_empty(&self) -> bool {
+        self.stateless_len() == 0
+    }
+}
+
+impl<T> StatelessLen for Vec<T> {
+    fn stateless_len(&self) -> usize {
+        self.len()
+    }
+}
+
+impl<T> StatelessLen for Box<[T]> {
+    fn stateless_len(&self) -> usize {
+        self.len()
+    }
+}
+
+impl StatelessLen for String {
+    fn stateless_len(&self) -> usize {
+        self.len()
+    }
+}
+
+impl StatelessLen for str {
+    fn stateless_len(&self) -> usize {
+        self.len()
+    }
+}
+
+impl<T> StatelessLen for [T] {
+    fn stateless_len(&self) -> usize {
+        self.len()
+    }
+}
+
+impl<const N: usize, T> StatelessLen for VInline<N, T> {
+    fn stateless_len(&self) -> usize {
+        self.len()
+    }
+}
+
+impl<const N: usize> StatelessLen for AStr<N> {
+    fn stateless_len(&self) -> usize {
+        self.len()
+    }
+}
+
+impl<const N: usize, T> StatelessLen for UArray<N, T> {
+    fn stateless_len(&self) -> usize {
+        self.len()
+    }
+}
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 9890c93d..86e978a3 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -85,8 +85,8 @@ impl<'a> AlterSpace<'a> {
 
 #[derive(Debug, PartialEq)]
 pub struct AlterModel<'a> {
-    model: Ident<'a>,
-    kind: AlterKind<'a>,
+    pub(in crate::engine) model: Ident<'a>,
+    pub(in crate::engine) kind: AlterKind<'a>,
 }
 
 impl<'a> AlterModel<'a> {
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 77a59655..50f0ebe8 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -368,9 +368,9 @@ impl<'a> FieldSpec<'a> {
 #[derive(Debug, PartialEq)]
 /// An [`ExpandedField`] is a full field definition with advanced metadata
 pub struct ExpandedField<'a> {
-    field_name: Ident<'a>,
-    layers: Vec<LayerSpec<'a>>,
-    props: DictGeneric,
+    pub(in crate::engine) field_name: Ident<'a>,
+    pub(in crate::engine) layers: Vec<LayerSpec<'a>>,
+    pub(in crate::engine) props: DictGeneric,
 }
 
 impl<'a> ExpandedField<'a> {
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index 7895f971..18d7c041 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -51,6 +51,13 @@ pub const IS_ON_CI: bool = option_env!("CI").is_some();
 
 const EXITCODE_ONE: i32 = 0x01;
 
+pub fn bx_to_vec<T>(mut bx: Box<[T]>) -> Vec<T> {
+    let ptr = bx.as_mut_ptr();
+    let len = bx.len();
+    mem::forget(bx);
+    unsafe { Vec::from_raw_parts(ptr, len, len) }
+}
+
 /// # Unsafe unwrapping
 ///
 /// This trait provides a method `unsafe_unwrap` that is potentially unsafe and has

From 73df6f9af43783b9b862474c7fb6a2f9c40e76fe Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 8 Mar 2023 06:10:57 -0800
Subject: [PATCH 156/310] Define intents for sync

---
 server/src/engine/core/model/alt.rs       |  30 +---
 server/src/engine/core/model/mod.rs       | 175 +++++++++++++++++++---
 server/src/engine/core/tests/model/alt.rs |   3 +-
 server/src/engine/core/tests/model/crt.rs |  38 ++---
 server/src/engine/error.rs                |  10 +-
 server/src/util/mod.rs                    |  11 +-
 6 files changed, 200 insertions(+), 67 deletions(-)

diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index e077bd4f..7b4ebab6 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    super::{Field, Layer, ModelView},
+    super::{Field, IWModel, Layer, ModelView},
     crate::{
         engine::{
             data::{
@@ -72,23 +72,9 @@ macro_rules! can_ignore {
     };
 }
 
-impl ModelView {
-    fn no_field(&self, new: &str) -> bool {
-        !self.fields().st_contains(new)
-    }
-    fn is_pk(&self, new: &str) -> bool {
-        self.p_key.as_bytes() == new.as_bytes()
-    }
-    fn not_pk(&self, new: &str) -> bool {
-        !self.is_pk(new)
-    }
-    fn guard_pk(&self, new: &str) -> DatabaseResult<()> {
-        if self.is_pk(new) {
-            Err(DatabaseError::DdlModelAlterProtectedField)
-        } else {
-            Ok(())
-        }
-    }
+#[inline(always)]
+fn no_field(mr: &IWModel, new: &str) -> bool {
+    !mr.fields().st_contains(new)
 }
 
 fn check_nullable(props: &mut HashMap<Box<str>, Option<DictEntryGeneric>>) -> DatabaseResult<bool> {
@@ -102,6 +88,7 @@ fn check_nullable(props: &mut HashMap<Box<str>, Option<DictEntryGeneric>>) -> Da
 impl<'a> AlterPlan<'a> {
     pub fn fdeltas(
         mv: &ModelView,
+        wm: &IWModel,
         AlterModel { model, kind }: AlterModel<'a>,
     ) -> DatabaseResult<AlterPlan<'a>> {
         let mut no_lock = true;
@@ -115,7 +102,7 @@ impl<'a> AlterPlan<'a> {
                 let mut not_found = false;
                 if r.iter().all(|id| {
                     let not_pk = mv.not_pk(id);
-                    let exists = mv.fields().st_contains(id.as_str());
+                    let exists = !no_field(wm, id.as_str());
                     not_found = !exists;
                     not_pk & exists
                 }) {
@@ -137,8 +124,7 @@ impl<'a> AlterPlan<'a> {
                         layers,
                         mut props,
                     } = fields.next().unwrap();
-                    okay &= mv.not_pk(&field_name);
-                    okay &= mv.no_field(&field_name);
+                    okay &= no_field(wm, &field_name) & mv.not_pk(&field_name);
                     let is_nullable = check_nullable(&mut props)?;
                     let layers = Field::parse_layers(layers, is_nullable)?;
                     okay &= add.st_insert(field_name.to_string().into_boxed_str(), layers);
@@ -159,7 +145,7 @@ impl<'a> AlterPlan<'a> {
                     // enforce pk
                     mv.guard_pk(&field_name)?;
                     // get the current field
-                    let Some(current_field) = mv.fields().st_get(field_name.as_str()) else {
+                    let Some(current_field) = wm.fields().st_get(field_name.as_str()) else {
                         return Err(DatabaseError::DdlModelAlterFieldNotFound);
                     };
                     // check props
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index b28756fc..3bbcb91b 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -27,35 +27,76 @@
 pub(super) mod alt;
 pub mod cell;
 
-use crate::engine::{
-    core::model::cell::Datacell,
-    data::{
-        tag::{DataTag, FullTag, TagClass, TagSelector},
-        ItemID,
-    },
-    error::{DatabaseError, DatabaseResult},
-    idx::{IndexSTSeqCns, STIndex, STIndexSeq},
-    mem::VInline,
-    ql::ddl::{
-        crt::CreateModel,
-        syn::{FieldSpec, LayerSpec},
-    },
-};
 #[cfg(test)]
 use std::cell::RefCell;
+use {
+    crate::engine::{
+        core::model::cell::Datacell,
+        data::{
+            tag::{DataTag, FullTag, TagClass, TagSelector},
+            ItemID,
+        },
+        error::{DatabaseError, DatabaseResult},
+        idx::{IndexSTSeqCns, STIndex, STIndexSeq},
+        mem::VInline,
+        ql::ddl::{
+            crt::CreateModel,
+            syn::{FieldSpec, LayerSpec},
+        },
+    },
+    core::cell::UnsafeCell,
+    parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard},
+};
+
+type Fields = IndexSTSeqCns<Box<str>, Field>;
 
 // FIXME(@ohsayan): update this!
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub struct ModelView {
     pub(super) p_key: Box<str>,
     pub(super) p_tag: FullTag,
-    pub(super) fields: IndexSTSeqCns<Box<str>, Field>,
+    fields: UnsafeCell<Fields>,
+    sync_matrix: ISyncMatrix,
+}
+
+#[cfg(test)]
+impl PartialEq for ModelView {
+    fn eq(&self, m: &Self) -> bool {
+        let mdl1 = self.intent_read_model();
+        let mdl2 = m.intent_read_model();
+        self.p_key == m.p_key && self.p_tag == m.p_tag && mdl1.fields() == mdl2.fields()
+    }
 }
 
 impl ModelView {
-    pub fn fields(&self) -> &IndexSTSeqCns<Box<str>, Field> {
-        &self.fields
+    pub fn sync_matrix(&self) -> &ISyncMatrix {
+        &self.sync_matrix
+    }
+    unsafe fn _read_fields<'a>(&'a self) -> &'a Fields {
+        &*self.fields.get().cast_const()
+    }
+    unsafe fn _read_fields_mut<'a>(&'a self) -> &'a mut Fields {
+        &mut *self.fields.get()
+    }
+    pub fn intent_read_model<'a>(&'a self) -> IRModel<'a> {
+        IRModel::new(self)
+    }
+    pub fn intent_write_model<'a>(&'a self) -> IWModel<'a> {
+        IWModel::new(self)
+    }
+    fn is_pk(&self, new: &str) -> bool {
+        self.p_key.as_bytes() == new.as_bytes()
+    }
+    fn not_pk(&self, new: &str) -> bool {
+        !self.is_pk(new)
+    }
+    fn guard_pk(&self, new: &str) -> DatabaseResult<()> {
+        if self.is_pk(new) {
+            Err(DatabaseError::DdlModelAlterProtectedField)
+        } else {
+            Ok(())
+        }
     }
 }
 
@@ -96,7 +137,8 @@ impl ModelView {
                 return Ok(Self {
                     p_key: last_pk.into(),
                     p_tag: tag,
-                    fields,
+                    fields: UnsafeCell::new(fields),
+                    sync_matrix: ISyncMatrix::new(),
                 });
             }
         }
@@ -385,3 +427,98 @@ unsafe fn lverify_list(_: Layer, _: &Datacell) -> bool {
     layertrace("list");
     true
 }
+
+// FIXME(@ohsayan): This an inefficient repr of the matrix; replace it with my other design
+#[derive(Debug)]
+pub struct ISyncMatrix {
+    // virtual privileges
+    v_priv_model_alter: RwLock<()>,
+    v_priv_data_new_or_revise: RwLock<()>,
+}
+
+#[cfg(test)]
+impl PartialEq for ISyncMatrix {
+    fn eq(&self, _: &Self) -> bool {
+        true
+    }
+}
+
+#[derive(Debug)]
+pub struct IRModelSMData<'a> {
+    rmodel: RwLockReadGuard<'a, ()>,
+    mdata: RwLockReadGuard<'a, ()>,
+    fields: &'a Fields,
+}
+
+impl<'a> IRModelSMData<'a> {
+    pub fn new(m: &'a ModelView) -> Self {
+        let rmodel = m.sync_matrix().v_priv_model_alter.read();
+        let mdata = m.sync_matrix().v_priv_data_new_or_revise.read();
+        Self {
+            rmodel,
+            mdata,
+            fields: unsafe {
+                // UNSAFE(@ohsayan): we already have acquired this resource
+                m._read_fields()
+            },
+        }
+    }
+    pub fn fields(&'a self) -> &'a Fields {
+        self.fields
+    }
+}
+
+#[derive(Debug)]
+pub struct IRModel<'a> {
+    rmodel: RwLockReadGuard<'a, ()>,
+    fields: &'a Fields,
+}
+
+impl<'a> IRModel<'a> {
+    pub fn new(m: &'a ModelView) -> Self {
+        Self {
+            rmodel: m.sync_matrix().v_priv_model_alter.read(),
+            fields: unsafe {
+                // UNSAFE(@ohsayan): we already have acquired this resource
+                m._read_fields()
+            },
+        }
+    }
+    pub fn fields(&'a self) -> &'a Fields {
+        self.fields
+    }
+}
+
+#[derive(Debug)]
+pub struct IWModel<'a> {
+    wmodel: RwLockWriteGuard<'a, ()>,
+    fields: &'a mut Fields,
+}
+
+impl<'a> IWModel<'a> {
+    pub fn new(m: &'a ModelView) -> Self {
+        Self {
+            wmodel: m.sync_matrix().v_priv_model_alter.write(),
+            fields: unsafe {
+                // UNSAFE(@ohsayan): we have exclusive access to this resource
+                m._read_fields_mut()
+            },
+        }
+    }
+    pub fn fields(&'a self) -> &'a Fields {
+        self.fields
+    }
+    // ALIASING
+    pub fn fields_mut(&'a mut self) -> &'a mut Fields {
+        self.fields
+    }
+}
+
+impl ISyncMatrix {
+    pub const fn new() -> Self {
+        Self {
+            v_priv_model_alter: RwLock::new(()),
+            v_priv_data_new_or_revise: RwLock::new(()),
+        }
+    }
+}
diff --git a/server/src/engine/core/tests/model/alt.rs b/server/src/engine/core/tests/model/alt.rs
index d2fbf581..62c2a808 100644
--- a/server/src/engine/core/tests/model/alt.rs
+++ b/server/src/engine/core/tests/model/alt.rs
@@ -40,7 +40,8 @@ mod plan {
         let model = create(model)?;
         let tok = lex_insecure(plan.as_bytes()).unwrap();
         let alter = parse_ast_node_full(&tok[2..]).unwrap();
-        let mv = AlterPlan::fdeltas(&model, alter)?;
+        let model_write = model.intent_write_model();
+        let mv = AlterPlan::fdeltas(&model, &model_write, alter)?;
         Ok(f(mv))
     }
     fn plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) {
diff --git a/server/src/engine/core/tests/model/crt.rs b/server/src/engine/core/tests/model/crt.rs
index d3c2f408..d8112845 100644
--- a/server/src/engine/core/tests/model/crt.rs
+++ b/server/src/engine/core/tests/model/crt.rs
@@ -28,7 +28,7 @@ mod validation {
     use {
         super::super::create,
         crate::engine::{
-            core::model::{Field, Layer, ModelView},
+            core::model::{Field, Layer},
             data::tag::{DataTag, FullTag},
             error::DatabaseError,
             idx::STIndexSeq,
@@ -37,15 +37,16 @@ mod validation {
 
     #[test]
     fn simple() {
-        let ModelView {
-            p_key,
-            p_tag,
-            fields,
-        } = create("create model mymodel(username: string, password: binary)").unwrap();
-        assert_eq!(p_key.as_ref(), "username");
-        assert_eq!(p_tag, FullTag::STR);
+        let model = create("create model mymodel(username: string, password: binary)").unwrap();
+        assert_eq!(model.p_key.as_ref(), "username");
+        assert_eq!(model.p_tag, FullTag::STR);
         assert_eq!(
-            fields.stseq_ord_value().cloned().collect::<Vec<Field>>(),
+            model
+                .intent_read_model()
+                .fields()
+                .stseq_ord_value()
+                .cloned()
+                .collect::<Vec<Field>>(),
             [
                 Field::new([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
                 Field::new([Layer::new_test(FullTag::BIN, [0; 2])].into(), false)
@@ -55,15 +56,17 @@ mod validation {
 
     #[test]
     fn idiotic_order() {
-        let ModelView {
-            p_key,
-            p_tag,
-            fields,
-        } = create("create model mymodel(password: binary, primary username: string)").unwrap();
-        assert_eq!(p_key.as_ref(), "username");
-        assert_eq!(p_tag, FullTag::STR);
+        let model =
+            create("create model mymodel(password: binary, primary username: string)").unwrap();
+        assert_eq!(model.p_key.as_ref(), "username");
+        assert_eq!(model.p_tag, FullTag::STR);
         assert_eq!(
-            fields.stseq_ord_value().cloned().collect::<Vec<Field>>(),
+            model
+                .intent_read_model()
+                .fields()
+                .stseq_ord_value()
+                .cloned()
+                .collect::<Vec<Field>>(),
             [
                 Field::new([Layer::new_test(FullTag::BIN, [0; 2])].into(), false),
                 Field::new([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
@@ -143,6 +146,7 @@ mod exec {
         .unwrap();
         with_model(&gns, SPACE, "mymodel", |model| {
             let models: Vec<(String, Field)> = model
+                .intent_read_model()
                 .fields()
                 .stseq_ord_kv()
                 .map(|(k, v)| (k.to_string(), v.clone()))
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 5c8d174b..0a6e0f45 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -28,7 +28,8 @@ pub type LangResult<T> = Result<T, LangError>;
 pub type LexResult<T> = Result<T, LexError>;
 pub type DatabaseResult<T> = Result<T, DatabaseError>;
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+#[repr(u8)]
 /// Lex phase errors
 pub enum LexError {
     // insecure lex
@@ -47,7 +48,9 @@ pub enum LexError {
     /// Unrecognized byte in stream   
     UnexpectedByte,
 }
-#[derive(Debug, PartialEq)]
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+#[repr(u8)]
 /// AST errors
 pub enum LangError {
     // generic
@@ -76,7 +79,8 @@ pub enum LangError {
     StmtUnknownDrop,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+#[repr(u8)]
 /// Executor errors
 pub enum DatabaseError {
     // sys
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index 18d7c041..b76b5c45 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -24,7 +24,7 @@
  *
 */
 
-use std::mem;
+use std::mem::{self, ManuallyDrop};
 
 #[macro_use]
 mod macros;
@@ -51,10 +51,11 @@ pub const IS_ON_CI: bool = option_env!("CI").is_some();
 
 const EXITCODE_ONE: i32 = 0x01;
 
-pub fn bx_to_vec<T>(mut bx: Box<[T]>) -> Vec<T> {
-    let ptr = bx.as_mut_ptr();
-    let len = bx.len();
-    mem::forget(bx);
+pub fn bx_to_vec<T>(bx: Box<[T]>) -> Vec<T> {
+    let mut md = ManuallyDrop::new(bx);
+    // damn you, miri
+    let ptr = md.as_mut_ptr() as usize as *mut T;
+    let len = md.len();
     unsafe { Vec::from_raw_parts(ptr, len, len) }
 }
 

From 9bc3d85b1c56d1e7a21f9edec65efdd19c22100b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 9 Mar 2023 06:45:53 -0800
Subject: [PATCH 157/310] Revise lit definitions and layout

---
 server/src/engine/core/model/cell.rs      |  4 +--
 server/src/engine/core/model/mod.rs       | 10 ++++--
 server/src/engine/core/tests/model/crt.rs | 12 ++++----
 server/src/engine/data/lit.rs             | 10 +++---
 server/src/engine/mem/mod.rs              | 12 ++++++++
 server/src/engine/mem/word.rs             | 37 +++++++++++------------
 6 files changed, 51 insertions(+), 34 deletions(-)

diff --git a/server/src/engine/core/model/cell.rs b/server/src/engine/core/model/cell.rs
index 86685ecb..bd0bb135 100644
--- a/server/src/engine/core/model/cell.rs
+++ b/server/src/engine/core/model/cell.rs
@@ -176,10 +176,10 @@ impl<'a> From<LitIR<'a>> for Datacell {
     fn from(l: LitIR<'a>) -> Self {
         match l.kind().tag_class() {
             tag if tag < TagClass::Bin => unsafe {
-                let [a, b] = l.data().load_fat();
+                // DO NOT RELY ON the payload's bit pattern; it's padded
                 Datacell::new(
                     l.kind().tag_class(),
-                    DataRaw::word(SystemDword::store_fat(a, b)),
+                    DataRaw::word(SystemDword::store_qw(l.data().load_qw())),
                 )
             },
             tag @ (TagClass::Bin | TagClass::Str) => unsafe {
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 3bbcb91b..5d32ccfa 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -54,8 +54,8 @@ type Fields = IndexSTSeqCns<Box<str>, Field>;
 
 #[derive(Debug)]
 pub struct ModelView {
-    pub(super) p_key: Box<str>,
-    pub(super) p_tag: FullTag,
+    p_key: Box<str>,
+    p_tag: FullTag,
     fields: UnsafeCell<Fields>,
     sync_matrix: ISyncMatrix,
 }
@@ -70,6 +70,12 @@ impl PartialEq for ModelView {
 }
 
 impl ModelView {
+    pub fn p_key(&self) -> &str {
+        &self.p_key
+    }
+    pub fn p_tag(&self) -> FullTag {
+        self.p_tag
+    }
     pub fn sync_matrix(&self) -> &ISyncMatrix {
         &self.sync_matrix
     }
diff --git a/server/src/engine/core/tests/model/crt.rs b/server/src/engine/core/tests/model/crt.rs
index d8112845..921113de 100644
--- a/server/src/engine/core/tests/model/crt.rs
+++ b/server/src/engine/core/tests/model/crt.rs
@@ -38,8 +38,8 @@ mod validation {
     #[test]
     fn simple() {
         let model = create("create model mymodel(username: string, password: binary)").unwrap();
-        assert_eq!(model.p_key.as_ref(), "username");
-        assert_eq!(model.p_tag, FullTag::STR);
+        assert_eq!(model.p_key(), "username");
+        assert_eq!(model.p_tag(), FullTag::STR);
         assert_eq!(
             model
                 .intent_read_model()
@@ -58,8 +58,8 @@ mod validation {
     fn idiotic_order() {
         let model =
             create("create model mymodel(password: binary, primary username: string)").unwrap();
-        assert_eq!(model.p_key.as_ref(), "username");
-        assert_eq!(model.p_tag, FullTag::STR);
+        assert_eq!(model.p_key(), "username");
+        assert_eq!(model.p_tag(), FullTag::STR);
         assert_eq!(
             model
                 .intent_read_model()
@@ -151,8 +151,8 @@ mod exec {
                 .stseq_ord_kv()
                 .map(|(k, v)| (k.to_string(), v.clone()))
                 .collect();
-            assert_eq!(model.p_key.as_ref(), "username");
-            assert_eq!(model.p_tag, FullTag::STR);
+            assert_eq!(model.p_key(), "username");
+            assert_eq!(model.p_tag(), FullTag::STR);
             assert_eq!(
                 models,
                 [
diff --git a/server/src/engine/data/lit.rs b/server/src/engine/data/lit.rs
index 6de39129..2c3d768b 100644
--- a/server/src/engine/data/lit.rs
+++ b/server/src/engine/data/lit.rs
@@ -29,7 +29,7 @@ use {
         spec::{Dataspec1D, DataspecMeta1D, DataspecMethods1D, DataspecRaw1D},
         tag::{DataTag, FullTag},
     },
-    crate::engine::mem::{NativeDword, SystemDword},
+    crate::engine::mem::{SpecialPaddedWord, SystemDword},
     core::{
         fmt,
         marker::PhantomData,
@@ -42,7 +42,7 @@ use {
 */
 
 pub struct Lit<'a> {
-    data: NativeDword,
+    data: SpecialPaddedWord,
     tag: FullTag,
     _lt: PhantomData<&'a [u8]>,
 }
@@ -58,7 +58,7 @@ impl<'a> Lit<'a> {
 
 impl<'a> DataspecMeta1D for Lit<'a> {
     type Tag = FullTag;
-    type Target = NativeDword;
+    type Target = SpecialPaddedWord;
     type StringItem = Box<str>;
     fn new(flag: Self::Tag, data: Self::Target) -> Self {
         Self {
@@ -174,12 +174,12 @@ direct_from! {
 
 pub struct LitIR<'a> {
     tag: FullTag,
-    data: NativeDword,
+    data: SpecialPaddedWord,
     _lt: PhantomData<&'a str>,
 }
 
 impl<'a> DataspecMeta1D for LitIR<'a> {
-    type Target = NativeDword;
+    type Target = SpecialPaddedWord;
     type StringItem = &'a str;
     type Tag = FullTag;
     fn new(flag: Self::Tag, data: Self::Target) -> Self {
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index fc9a0bad..a213bd5b 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -49,6 +49,18 @@ pub struct NativeDword([usize; 2]);
 pub struct NativeTword([usize; 3]);
 /// Native quad pointer width (note, native != arch native, but host native)
 pub struct NativeQword([usize; 4]);
+/// A special word with a special bit pattern padded (with a quad)
+///
+/// **WARNING**: DO NOT EXPECT this to have the same bit pattern as that of native word sizes. It's called "special" FOR A REASON
+pub struct SpecialPaddedWord {
+    a: u64,
+    b: usize,
+}
+impl SpecialPaddedWord {
+    const fn new(a: u64, b: usize) -> Self {
+        Self { a, b }
+    }
+}
 
 pub trait StatelessLen {
     fn stateless_len(&self) -> usize;
diff --git a/server/src/engine/mem/word.rs b/server/src/engine/mem/word.rs
index ddd7ffaa..76747dc0 100644
--- a/server/src/engine/mem/word.rs
+++ b/server/src/engine/mem/word.rs
@@ -24,7 +24,9 @@
  *
 */
 
-use super::{NativeDword, NativeQword, NativeTword};
+use super::{NativeDword, NativeQword, NativeTword, SpecialPaddedWord};
+
+static ZERO_BLOCK: [u8; 0] = [];
 
 /// Native quad pointer stack (must also be usable as a double and triple pointer stack. see [`SystemTword`] and [`SystemDword`])
 pub trait SystemQword: SystemTword {
@@ -36,12 +38,6 @@ pub trait SystemQword: SystemTword {
     {
         WordRW::store(v)
     }
-    fn ld_special<'a, T>(&'a self) -> T::Target<'a>
-    where
-        T: WordRW<Self>,
-    {
-        <T>::load(self)
-    }
     fn ld<'a, T>(&'a self) -> T
     where
         T: WordRW<Self, Target<'a> = T>,
@@ -60,12 +56,6 @@ pub trait SystemTword: SystemDword {
     {
         WordRW::store(v)
     }
-    fn ld_special<'a, T>(&'a self) -> T::Target<'a>
-    where
-        T: WordRW<Self>,
-    {
-        <T>::load(self)
-    }
     fn ld<'a, T>(&'a self) -> T
     where
         T: WordRW<Self, Target<'a> = T>,
@@ -86,12 +76,6 @@ pub trait SystemDword: Sized {
     {
         WordRW::store(v)
     }
-    fn ld_special<'a, T>(&'a self) -> T::Target<'a>
-    where
-        T: WordRW<Self>,
-    {
-        <T>::load(self)
-    }
     fn ld<'a, T>(&'a self) -> T
     where
         T: WordRW<Self, Target<'a> = T>,
@@ -100,6 +84,21 @@ pub trait SystemDword: Sized {
     }
 }
 
+impl SystemDword for SpecialPaddedWord {
+    fn store_qw(u: u64) -> Self {
+        Self::new(u, ZERO_BLOCK.as_ptr() as usize)
+    }
+    fn store_fat(a: usize, b: usize) -> Self {
+        Self::new(a as u64, b)
+    }
+    fn load_qw(&self) -> u64 {
+        self.a
+    }
+    fn load_fat(&self) -> [usize; 2] {
+        [self.a as usize, self.b]
+    }
+}
+
 impl SystemDword for NativeDword {
     #[inline(always)]
     fn store_qw(u: u64) -> Self {

From 8f77c1d73a5d4684ba7f7accd3391103e96fea09 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 25 Mar 2023 01:14:43 -0700
Subject: [PATCH 158/310] Add basic alter exec impl

---
 server/src/engine/core/model/alt.rs | 50 +++++++++++++++++++++++++++++
 server/src/engine/core/model/mod.rs |  7 ++--
 server/src/engine/core/space.rs     |  5 ++-
 server/src/engine/data/tag.rs       |  2 +-
 server/src/engine/error.rs          |  5 +++
 5 files changed, 65 insertions(+), 4 deletions(-)

diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index 7b4ebab6..cbae154b 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -28,6 +28,7 @@ use {
     super::{Field, IWModel, Layer, ModelView},
     crate::{
         engine::{
+            core::GlobalNS,
             data::{
                 tag::{DataTag, TagClass},
                 DictEntryGeneric,
@@ -224,6 +225,8 @@ impl<'a> AlterPlan<'a> {
                     // no delta
                 }
                 (current_selector, new_selector) if interop(current_layer, &new_parsed_layer) => {
+                    // now, we're not sure if we can run this
+                    // FIXME(@ohsayan): look, should we be explicit about this?
                     no_lock &= new_selector >= current_selector;
                     deltasize += (new_selector != current_selector) as usize;
                 }
@@ -243,3 +246,50 @@ impl<'a> AlterPlan<'a> {
         }
     }
 }
+
+impl ModelView {
+    pub fn exec_alter(gns: &GlobalNS, space: &[u8], alter: AlterModel) -> DatabaseResult<()> {
+        let gns = gns.spaces().read();
+        let Some(space) = gns.st_get(space) else {
+            return Err(DatabaseError::DdlSpaceNotFound)
+        };
+        let space = space.models().read();
+        let Some(model) = space.st_get(alter.model.as_bytes()) else {
+            return Err(DatabaseError::DdlModelNotFound);
+        };
+        // make intent
+        let iwm = model.intent_write_model();
+        // prepare plan
+        let plan = AlterPlan::fdeltas(model, &iwm, alter)?;
+        // we have a legal plan; acquire exclusive if we need it
+        if !plan.no_lock {
+            // TODO(@ohsayan): allow this later on, once we define the syntax
+            return Err(DatabaseError::NeedLock);
+        }
+        // fine, we're good
+        let mut iwm = iwm;
+        match plan.action {
+            AlterAction::Ignore => drop(iwm),
+            AlterAction::Add(new_fields) => {
+                // TODO(@ohsayan): this impacts lockdown duration; fix it
+                new_fields
+                    .st_iter_kv()
+                    .map(|(x, y)| (x.clone(), y.clone()))
+                    .for_each(|(field_id, field)| {
+                        iwm.fields_mut().st_insert(field_id, field);
+                    });
+            }
+            AlterAction::Remove(remove) => {
+                remove.into_iter().for_each(|field_id| {
+                    iwm.fields_mut().st_delete(field_id.as_str());
+                });
+            }
+            AlterAction::Update(u) => {
+                u.into_iter().for_each(|(field_id, field)| {
+                    iwm.fields_mut().st_update(&field_id, field);
+                });
+            }
+        }
+        Ok(())
+    }
+}
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 5d32ccfa..e531f753 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -150,6 +150,9 @@ impl ModelView {
         }
         Err(DatabaseError::DdlModelBadDefinition)
     }
+}
+
+impl ModelView {
     pub fn exec_create(
         gns: &super::GlobalNS,
         space: &[u8],
@@ -161,7 +164,7 @@ impl ModelView {
         let Some(space) = space_rl.get(space) else {
             return Err(DatabaseError::DdlSpaceNotFound)
         };
-        space.create_model(ItemID::new(&name), model)
+        space._create_model(ItemID::new(&name), model)
     }
 }
 
@@ -515,7 +518,7 @@ impl<'a> IWModel<'a> {
         self.fields
     }
     // ALIASING
-    pub fn fields_mut(&'a mut self) -> &'a mut Fields {
+    pub fn fields_mut(&mut self) -> &mut Fields {
         self.fields
     }
 }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index ae8faa6a..2507b951 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -74,7 +74,7 @@ impl ProcedureCreate {
 }
 
 impl Space {
-    pub fn create_model(&self, name: ItemID, model: ModelView) -> DatabaseResult<()> {
+    pub fn _create_model(&self, name: ItemID, model: ModelView) -> DatabaseResult<()> {
         if self.mns.write().st_insert(name, model) {
             Ok(())
         } else {
@@ -125,6 +125,9 @@ impl Space {
             ),
         })
     }
+}
+
+impl Space {
     /// Execute a `create` stmt
     pub fn exec_create(gns: &super::GlobalNS, space: CreateSpace) -> DatabaseResult<()> {
         let ProcedureCreate { space_name, space } = Self::process_create(space)?;
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
index f64ea3a5..3712d925 100644
--- a/server/src/engine/data/tag.rs
+++ b/server/src/engine/data/tag.rs
@@ -67,7 +67,7 @@ pub enum TagUnique {
 
 impl TagUnique {
     pub const fn is_unique(&self) -> bool {
-        self.d() != 0xFF
+        self.d() != Self::Illegal.d()
     }
 }
 
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 0a6e0f45..901590d4 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -85,6 +85,9 @@ pub enum LangError {
 pub enum DatabaseError {
     // sys
     SysBadItemID,
+    // query generic
+    /// this needs an explicit lock
+    NeedLock,
     // ddl: create space
     /// unknown property or bad type for property
     DdlSpaceBadProperty,
@@ -110,4 +113,6 @@ pub enum DatabaseError {
     DdlModelAlterFieldNotFound,
     /// bad type definition to alter
     DdlModelAlterBadTypedef,
+    /// didn't find the model
+    DdlModelNotFound,
 }

From c5d7f5f6f6b2dc71cfee4e8b709f6bcce0238db0 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 25 Mar 2023 07:12:05 -0700
Subject: [PATCH 159/310] Test illegal type casts

---
 server/src/engine/core/model/mod.rs       |   8 ++
 server/src/engine/core/tests/model/alt.rs | 140 ++++++++++++++++++++--
 server/src/util/test_utils.rs             |  31 +++++
 3 files changed, 171 insertions(+), 8 deletions(-)

diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index e531f753..277cbbee 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -193,6 +193,14 @@ static LUT: [(&str, FullTag); 14] = [
     ("list", FullTag::LIST),
 ];
 
+pub static TY_BOOL: &str = LUT[0].0;
+pub static TY_UINT: [&str; 4] = [LUT[1].0, LUT[2].0, LUT[3].0, LUT[4].0];
+pub static TY_SINT: [&str; 4] = [LUT[5].0, LUT[6].0, LUT[7].0, LUT[8].0];
+pub static TY_FLOAT: [&str; 2] = [LUT[9].0, LUT[10].0];
+pub static TY_BINARY: &str = LUT[11].0;
+pub static TY_STRING: &str = LUT[12].0;
+pub static TY_LIST: &str = LUT[13].0;
+
 #[derive(Debug, PartialEq, Clone)]
 pub struct Field {
     layers: VInline<1, Layer>,
diff --git a/server/src/engine/core/tests/model/alt.rs b/server/src/engine/core/tests/model/alt.rs
index 62c2a808..885cabe1 100644
--- a/server/src/engine/core/tests/model/alt.rs
+++ b/server/src/engine/core/tests/model/alt.rs
@@ -25,16 +25,20 @@
 */
 
 mod plan {
-    use crate::engine::{
-        core::{
-            model::{
-                alt::{AlterAction, AlterPlan},
-                Field, Layer,
+    use crate::{
+        engine::{
+            core::{
+                model::{
+                    self,
+                    alt::{AlterAction, AlterPlan},
+                    Field, Layer,
+                },
+                tests::model::create,
             },
-            tests::model::create,
+            error::{DatabaseError, DatabaseResult},
+            ql::{ast::parse_ast_node_full, tests::lex_insecure},
         },
-        error::{DatabaseError, DatabaseResult},
-        ql::{ast::parse_ast_node_full, tests::lex_insecure},
+        vecfuse,
     };
     fn with_plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) -> DatabaseResult<()> {
         let model = create(model)?;
@@ -175,4 +179,124 @@ mod plan {
             DatabaseError::DdlModelAlterFieldNotFound
         );
     }
+    fn bad_type_cast(orig_ty: &str, new_ty: &str) {
+        let create = format!("create model mymodel(username: string, silly_field: {orig_ty})");
+        let alter = format!("alter model mymodel update silly_field {{ type: {new_ty} }}");
+        assert_eq!(
+            with_plan(&create, &alter, |_| {}).expect_err(&format!(
+                "found no error in transformation: {orig_ty} -> {new_ty}"
+            )),
+            DatabaseError::DdlModelAlterBadTypedef,
+            "failed to match error in transformation: {orig_ty} -> {new_ty}",
+        )
+    }
+    fn enumerated_bad_type_casts<O, N>(orig_ty: O, new_ty: N)
+    where
+        O: IntoIterator<Item = &'static str>,
+        N: IntoIterator<Item = &'static str> + Clone,
+    {
+        for orig in orig_ty {
+            let new_ty = new_ty.clone();
+            for new in new_ty {
+                bad_type_cast(orig, new);
+            }
+        }
+    }
+    #[test]
+    fn illegal_bool_direct_cast() {
+        enumerated_bad_type_casts(
+            ["bool"],
+            vecfuse![
+                model::TY_UINT,
+                model::TY_SINT,
+                model::TY_BINARY,
+                model::TY_STRING,
+                model::TY_LIST
+            ],
+        );
+    }
+    #[test]
+    fn illegal_uint_direct_cast() {
+        enumerated_bad_type_casts(
+            model::TY_UINT,
+            vecfuse![
+                model::TY_BOOL,
+                model::TY_SINT,
+                model::TY_FLOAT,
+                model::TY_BINARY,
+                model::TY_STRING,
+                model::TY_LIST
+            ],
+        );
+    }
+    #[test]
+    fn illegal_sint_direct_cast() {
+        enumerated_bad_type_casts(
+            model::TY_SINT,
+            vecfuse![
+                model::TY_BOOL,
+                model::TY_UINT,
+                model::TY_FLOAT,
+                model::TY_BINARY,
+                model::TY_STRING,
+                model::TY_LIST
+            ],
+        );
+    }
+    #[test]
+    fn illegal_float_direct_cast() {
+        enumerated_bad_type_casts(
+            model::TY_FLOAT,
+            vecfuse![
+                model::TY_BOOL,
+                model::TY_UINT,
+                model::TY_SINT,
+                model::TY_BINARY,
+                model::TY_STRING,
+                model::TY_LIST
+            ],
+        );
+    }
+    #[test]
+    fn illegal_binary_direct_cast() {
+        enumerated_bad_type_casts(
+            [model::TY_BINARY],
+            vecfuse![
+                model::TY_BOOL,
+                model::TY_UINT,
+                model::TY_SINT,
+                model::TY_FLOAT,
+                model::TY_STRING,
+                model::TY_LIST
+            ],
+        );
+    }
+    #[test]
+    fn illegal_string_direct_cast() {
+        enumerated_bad_type_casts(
+            [model::TY_STRING],
+            vecfuse![
+                model::TY_BOOL,
+                model::TY_UINT,
+                model::TY_SINT,
+                model::TY_FLOAT,
+                model::TY_BINARY,
+                model::TY_LIST
+            ],
+        );
+    }
+    #[test]
+    fn illegal_list_direct_cast() {
+        enumerated_bad_type_casts(
+            ["list { type: string }"],
+            vecfuse![
+                model::TY_BOOL,
+                model::TY_UINT,
+                model::TY_SINT,
+                model::TY_FLOAT,
+                model::TY_BINARY,
+                model::TY_STRING
+            ],
+        );
+    }
 }
diff --git a/server/src/util/test_utils.rs b/server/src/util/test_utils.rs
index 299bd9c8..192dfa34 100644
--- a/server/src/util/test_utils.rs
+++ b/server/src/util/test_utils.rs
@@ -55,3 +55,34 @@ pub fn random_string_checked(rng: &mut impl Rng, l: usize, ck: impl Fn(&str) ->
         }
     }
 }
+
+pub trait VecFuse<T> {
+    fn fuse_append(self, arr: &mut Vec<T>);
+}
+
+impl<T> VecFuse<T> for T {
+    fn fuse_append(self, arr: &mut Vec<T>) {
+        arr.push(self);
+    }
+}
+
+impl<T, const N: usize> VecFuse<T> for [T; N] {
+    fn fuse_append(self, arr: &mut Vec<T>) {
+        arr.extend(self)
+    }
+}
+
+impl<T> VecFuse<T> for Vec<T> {
+    fn fuse_append(self, arr: &mut Vec<T>) {
+        arr.extend(self)
+    }
+}
+
+#[macro_export]
+macro_rules! vecfuse {
+    ($($expr:expr),* $(,)?) => {{
+        let mut v = Vec::new();
+        $(<_ as crate::util::test_utils::VecFuse<_>>::fuse_append($expr, &mut v);)*
+        v
+    }};
+}

From 0e67872e69915af9c89e6a901be3c60b7c74fcd5 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 25 Mar 2023 20:50:48 -0700
Subject: [PATCH 160/310] Add tests for simple alter

---
 server/src/engine/core/tests/model/alt.rs | 121 ++++++++++++++++------
 1 file changed, 90 insertions(+), 31 deletions(-)

diff --git a/server/src/engine/core/tests/model/alt.rs b/server/src/engine/core/tests/model/alt.rs
index 885cabe1..8ec4df65 100644
--- a/server/src/engine/core/tests/model/alt.rs
+++ b/server/src/engine/core/tests/model/alt.rs
@@ -24,39 +24,61 @@
  *
 */
 
+use crate::engine::{
+    core::{
+        model::{alt::AlterPlan, ModelView},
+        tests::model::{create, exec_create},
+        GlobalNS,
+    },
+    error::DatabaseResult,
+    idx::STIndex,
+    ql::{ast::parse_ast_node_full, ddl::alt::AlterModel, tests::lex_insecure},
+};
+
+fn with_plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) -> DatabaseResult<()> {
+    let model = create(model)?;
+    let tok = lex_insecure(plan.as_bytes()).unwrap();
+    let alter = parse_ast_node_full(&tok[2..]).unwrap();
+    let model_write = model.intent_write_model();
+    let mv = AlterPlan::fdeltas(&model, &model_write, alter)?;
+    Ok(f(mv))
+}
+fn plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) {
+    with_plan(model, plan, f).unwrap()
+}
+fn exec_plan(
+    gns: &GlobalNS,
+    new_space: bool,
+    model: &str,
+    plan: &str,
+    f: impl Fn(&ModelView),
+) -> DatabaseResult<()> {
+    exec_create(gns, model, "myspace", new_space)?;
+    let tok = lex_insecure(plan.as_bytes()).unwrap();
+    let alter = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+    let model_name = alter.model;
+    ModelView::exec_alter(gns, "myspace".as_bytes(), alter)?;
+    let gns_read = gns.spaces().read();
+    let space = gns_read.st_get("myspace".as_bytes()).unwrap();
+    let model = space.models().read();
+    f(model.st_get(model_name.as_bytes()).unwrap());
+    Ok(())
+}
+
 mod plan {
     use crate::{
         engine::{
-            core::{
-                model::{
-                    self,
-                    alt::{AlterAction, AlterPlan},
-                    Field, Layer,
-                },
-                tests::model::create,
-            },
-            error::{DatabaseError, DatabaseResult},
-            ql::{ast::parse_ast_node_full, tests::lex_insecure},
+            core::model::{self, alt::AlterAction, Field, Layer},
+            error::DatabaseError,
         },
         vecfuse,
     };
-    fn with_plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) -> DatabaseResult<()> {
-        let model = create(model)?;
-        let tok = lex_insecure(plan.as_bytes()).unwrap();
-        let alter = parse_ast_node_full(&tok[2..]).unwrap();
-        let model_write = model.intent_write_model();
-        let mv = AlterPlan::fdeltas(&model, &model_write, alter)?;
-        Ok(f(mv))
-    }
-    fn plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) {
-        with_plan(model, plan, f).unwrap()
-    }
     /*
         Simple
     */
     #[test]
     fn simple_add() {
-        plan(
+        super::plan(
             "create model mymodel(username: string, password: binary)",
             "alter model mymodel add myfield { type: string, nullable: true }",
             |plan| {
@@ -73,7 +95,7 @@ mod plan {
     }
     #[test]
     fn simple_remove() {
-        plan(
+        super::plan(
             "create model mymodel(username: string, password: binary, useless_field: uint8)",
             "alter model mymodel remove useless_field",
             |plan| {
@@ -89,7 +111,7 @@ mod plan {
     #[test]
     fn simple_update() {
         // FREEDOM! DAMN THE PASSWORD!
-        plan(
+        super::plan(
             "create model mymodel(username: string, password: binary)",
             "alter model mymodel update password { nullable: true }",
             |plan| {
@@ -104,13 +126,31 @@ mod plan {
             },
         );
     }
+    #[test]
+    fn update_need_lock() {
+        // FIGHT THE NULL
+        super::plan(
+            "create model mymodel(username: string, null password: binary)",
+            "alter model mymodel update password { nullable: false }",
+            |plan| {
+                assert_eq!(plan.model.as_str(), "mymodel");
+                assert!(!plan.no_lock);
+                assert_eq!(
+                    plan.action,
+                    AlterAction::Update(into_dict! {
+                        "password" => Field::new([Layer::bin()].into(), false)
+                    })
+                );
+            },
+        );
+    }
     /*
         Illegal
     */
     #[test]
     fn illegal_remove_nx() {
         assert_eq!(
-            with_plan(
+            super::with_plan(
                 "create model mymodel(username: string, password: binary)",
                 "alter model mymodel remove password_e2e",
                 |_| {}
@@ -122,7 +162,7 @@ mod plan {
     #[test]
     fn illegal_remove_pk() {
         assert_eq!(
-            with_plan(
+            super::with_plan(
                 "create model mymodel(username: string, password: binary)",
                 "alter model mymodel remove username",
                 |_| {}
@@ -134,7 +174,7 @@ mod plan {
     #[test]
     fn illegal_add_pk() {
         assert_eq!(
-            with_plan(
+            super::with_plan(
                 "create model mymodel(username: string, password: binary)",
                 "alter model mymodel add username { type: string }",
                 |_| {}
@@ -146,7 +186,7 @@ mod plan {
     #[test]
     fn illegal_add_ex() {
         assert_eq!(
-            with_plan(
+            super::with_plan(
                 "create model mymodel(username: string, password: binary)",
                 "alter model mymodel add password { type: string }",
                 |_| {}
@@ -158,7 +198,7 @@ mod plan {
     #[test]
     fn illegal_update_pk() {
         assert_eq!(
-            with_plan(
+            super::with_plan(
                 "create model mymodel(username: string, password: binary)",
                 "alter model mymodel update username { type: string }",
                 |_| {}
@@ -170,7 +210,7 @@ mod plan {
     #[test]
     fn illegal_update_nx() {
         assert_eq!(
-            with_plan(
+            super::with_plan(
                 "create model mymodel(username: string, password: binary)",
                 "alter model mymodel update username_secret { type: string }",
                 |_| {}
@@ -183,7 +223,7 @@ mod plan {
         let create = format!("create model mymodel(username: string, silly_field: {orig_ty})");
         let alter = format!("alter model mymodel update silly_field {{ type: {new_ty} }}");
         assert_eq!(
-            with_plan(&create, &alter, |_| {}).expect_err(&format!(
+            super::with_plan(&create, &alter, |_| {}).expect_err(&format!(
                 "found no error in transformation: {orig_ty} -> {new_ty}"
             )),
             DatabaseError::DdlModelAlterBadTypedef,
@@ -300,3 +340,22 @@ mod plan {
         );
     }
 }
+
+mod exec {
+    use crate::engine::{core::GlobalNS, idx::STIndex};
+    #[test]
+    fn exec_simple_alter() {
+        let gns = GlobalNS::empty();
+        super::exec_plan(
+            &gns,
+            true,
+            "create model mymodel(username: string, password: binary)",
+            "alter model mymodel update password { nullable: true }",
+            |model| {
+                let schema = model.intent_read_model();
+                assert!(schema.fields().st_get("password").unwrap().is_nullable());
+            },
+        )
+        .unwrap();
+    }
+}

From f351be2819c8bd005b8ac668e008505b46e2c683 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 27 Mar 2023 10:21:45 -0700
Subject: [PATCH 161/310] Remove partial entity syntax

This was a bad mistake that we made with the actions API, and we won't
make the same mistake with BlueQL again. PES has been a terrible idea
all along, and it was a leaky abstraction.

Instead, we'll be using a more robust framework for addressing entities.
---
 server/src/engine/core/model/mod.rs  |  1 +
 server/src/engine/ql/ast/mod.rs      | 28 ----------------------------
 server/src/engine/ql/benches.rs      | 10 ----------
 server/src/engine/ql/ddl/drop.rs     |  4 ++--
 server/src/engine/ql/tests/entity.rs |  7 +------
 5 files changed, 4 insertions(+), 46 deletions(-)

diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 277cbbee..2c289e93 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -29,6 +29,7 @@ pub mod cell;
 
 #[cfg(test)]
 use std::cell::RefCell;
+
 use {
     crate::engine::{
         core::model::cell::Datacell,
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index cd631002..af3c1e7e 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -357,14 +357,6 @@ impl<'a> QueryData<'a> for SubstitutedData<'a> {
 #[derive(Debug, PartialEq)]
 /// An [`Entity`] represents the location for a specific structure, such as a model
 pub enum Entity<'a> {
-    /// A partial entity is used when switching to a model wrt the currently set space (commonly used
-    /// when running `use` queries)
-    ///
-    /// syntax:
-    /// ```sql
-    /// :model
-    /// ```
-    Partial(Ident<'a>),
     /// A single entity is used when switching to a model wrt the currently set space (commonly used
     /// when running DML queries)
     ///
@@ -408,21 +400,6 @@ impl<'a> Entity<'a> {
         Entity::Single(extract!(&sl[0], Token::Ident(sl) => sl.clone()))
     }
     #[inline(always)]
-    /// Parse a partial entity from the given slice
-    ///
-    /// ## Safety
-    ///
-    /// Caller guarantees that the token stream matches the exact stream of tokens
-    /// expected for a partial entity
-    pub(super) unsafe fn partial_entity_from_slice(sl: &'a [Token]) -> Self {
-        Entity::Partial(extract!(&sl[1], Token::Ident(sl) => sl.clone()))
-    }
-    #[inline(always)]
-    /// Returns true if the given token stream matches the signature of partial entity syntax
-    pub(super) fn tokens_with_partial(tok: &[Token]) -> bool {
-        tok.len() > 1 && tok[0] == Token![:] && tok[1].is_ident()
-    }
-    #[inline(always)]
     /// Returns true if the given token stream matches the signature of single entity syntax
     ///
     /// ⚠ WARNING: This will pass for full and single
@@ -438,7 +415,6 @@ impl<'a> Entity<'a> {
     /// Attempt to parse an entity using the given token stream. It also accepts a counter
     /// argument to forward the cursor
     pub fn parse_from_tokens(tok: &'a [Token], c: &mut usize) -> LangResult<Self> {
-        let is_partial = Self::tokens_with_partial(tok);
         let is_current = Self::tokens_with_single(tok);
         let is_full = Self::tokens_with_full(tok);
         let r = match () {
@@ -450,10 +426,6 @@ impl<'a> Entity<'a> {
                 *c += 1;
                 Self::single_entity_from_slice(tok)
             },
-            _ if is_partial => unsafe {
-                *c += 2;
-                Self::partial_entity_from_slice(tok)
-            },
             _ => return Err(LangError::ExpectedEntity),
         };
         Ok(r)
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index c042d99d..89b76565 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -111,16 +111,6 @@ mod ast {
             assert!(state.exhausted());
         })
     }
-    #[bench]
-    fn parse_entity_partial(b: &mut Bencher) {
-        let e = Entity::Partial(Ident::from("user"));
-        b.iter(|| {
-            let src = lex_insecure(b":user").unwrap();
-            let mut i = 0;
-            assert_eq!(Entity::parse_from_tokens(&src, &mut i).unwrap(), e);
-            assert_eq!(i, src.len());
-        });
-    }
 }
 
 mod ddl_queries {
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index 6d33bd07..a651fb57 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -68,8 +68,8 @@ impl<'a> DropSpace<'a> {
 
 #[derive(Debug, PartialEq)]
 pub struct DropModel<'a> {
-    pub(super) entity: Entity<'a>,
-    pub(super) force: bool,
+    pub(in crate::engine) entity: Entity<'a>,
+    pub(in crate::engine) force: bool,
 }
 
 impl<'a> DropModel<'a> {
diff --git a/server/src/engine/ql/tests/entity.rs b/server/src/engine/ql/tests/entity.rs
index 4efbcfa5..74019455 100644
--- a/server/src/engine/ql/tests/entity.rs
+++ b/server/src/engine/ql/tests/entity.rs
@@ -32,12 +32,7 @@ fn entity_current() {
     let r = Entity::parse_from_tokens(&t, &mut 0).unwrap();
     assert_eq!(r, Entity::Single(Ident::from("hello")))
 }
-#[test]
-fn entity_partial() {
-    let t = lex_insecure(b":hello").unwrap();
-    let r = Entity::parse_from_tokens(&t, &mut 0).unwrap();
-    assert_eq!(r, Entity::Partial(Ident::from("hello")))
-}
+
 #[test]
 fn entity_full() {
     let t = lex_insecure(b"hello.world").unwrap();

From f0f67a98fc34b1aac1f50c16f8bccd47cde36a1d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 30 Mar 2023 04:48:08 -0700
Subject: [PATCH 162/310] Fix unsoundness in `alter space` and remove redundant
 macro usage

---
 server/src/engine/ql/ast/mod.rs  | 11 ++++-------
 server/src/engine/ql/ddl/alt.rs  |  8 ++++++--
 server/src/engine/ql/ddl/crt.rs  | 18 ++++++++++++------
 server/src/engine/ql/ddl/drop.rs |  2 +-
 server/src/engine/ql/ddl/ins.rs  |  2 +-
 server/src/engine/ql/ddl/syn.rs  |  5 ++++-
 server/src/engine/ql/dml/ins.rs  |  3 +--
 server/src/engine/ql/dml/mod.rs  | 11 ++++-------
 server/src/engine/ql/dml/upd.rs  |  5 +++--
 server/src/engine/ql/lex/raw.rs  |  9 +++++++++
 10 files changed, 45 insertions(+), 29 deletions(-)

diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index af3c1e7e..dd22510b 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -302,11 +302,11 @@ impl<'a> QueryData<'a> for InplaceData {
     }
     #[inline(always)]
     unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a> {
-        extract!(tok, Token::Lit(l) => l.as_ir())
+        tok.uck_read_lit().as_ir()
     }
     #[inline(always)]
     unsafe fn read_data_type(&mut self, tok: &'a Token) -> Datacell {
-        Datacell::from(extract!(tok, Token::Lit(ref l) => l.to_owned()))
+        Datacell::from(<Self as QueryData>::read_lit(self, tok))
     }
     #[inline(always)]
     fn nonzero(&self) -> bool {
@@ -384,10 +384,7 @@ impl<'a> Entity<'a> {
     /// Caller guarantees that the token stream matches the exact stream of tokens
     /// expected for a full entity
     pub(super) unsafe fn full_entity_from_slice(sl: &'a [Token]) -> Self {
-        Entity::Full(
-            extract!(&sl[0], Token::Ident(sl) => sl.clone()),
-            extract!(&sl[2], Token::Ident(sl) => sl.clone()),
-        )
+        Entity::Full(sl[0].uck_read_ident(), sl[1].uck_read_ident())
     }
     #[inline(always)]
     /// Parse a single entity from the given slice
@@ -397,7 +394,7 @@ impl<'a> Entity<'a> {
     /// Caller guarantees that the token stream matches the exact stream of tokens
     /// expected for a single entity
     pub(super) unsafe fn single_entity_from_slice(sl: &'a [Token]) -> Self {
-        Entity::Single(extract!(&sl[0], Token::Ident(sl) => sl.clone()))
+        Entity::Single(sl[0].uck_read_ident())
     }
     #[inline(always)]
     /// Returns true if the given token stream matches the signature of single entity syntax
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 86e978a3..43c24d64 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -60,6 +60,7 @@ impl<'a> AlterSpace<'a> {
             return compiler::cold_rerr(LangError::UnexpectedEOS);
         }
         let space_name = state.fw_read();
+        state.poison_if_not(space_name.is_ident());
         state.poison_if_not(state.cursor_eq(Token![with]));
         state.cursor_ahead(); // ignore errors
         state.poison_if_not(state.cursor_eq(Token![open {}]));
@@ -69,7 +70,10 @@ impl<'a> AlterSpace<'a> {
             return Err(LangError::BadSyntax);
         }
 
-        let space_name = unsafe { extract!(space_name, Token::Ident(ref space) => space.clone()) };
+        let space_name = unsafe {
+            // UNSAFE(@ohsayan): We just verified that `space_name` is an ident
+            space_name.uck_read_ident()
+        };
         let mut d = DictGeneric::new();
         syn::rfold_dict(DictFoldState::CB_OR_IDENT, state, &mut d);
         if state.okay() {
@@ -113,7 +117,7 @@ impl<'a> AlterModel<'a> {
             return compiler::cold_rerr(LangError::BadSyntax);
             // FIXME(@ohsayan): bad because no specificity
         }
-        let model_name = unsafe { extract!(state.fw_read(), Token::Ident(ref l) => l.clone()) };
+        let model_name = unsafe { state.fw_read().uck_read_ident() };
         let kind = match state.fw_read() {
             Token![add] => AlterKind::alter_add(state),
             Token![remove] => AlterKind::alter_remove(state),
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index ab376451..3cb36c50 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -32,7 +32,7 @@ use {
             error::{LangError, LangResult},
             ql::{
                 ast::{QueryData, State},
-                lex::{Ident, Token},
+                lex::Ident,
             },
         },
         util::compiler,
@@ -69,7 +69,10 @@ impl<'a> CreateSpace<'a> {
         }
         if state.okay() {
             Ok(CreateSpace {
-                space_name: unsafe { extract!(space_name, Token::Ident(ref id) => id.clone()) },
+                space_name: unsafe {
+                    // UNSAFE(@ohsayan): we checked if `space_name` with `is_ident` above
+                    space_name.uck_read_ident()
+                },
                 props: d,
             })
         } else {
@@ -109,9 +112,9 @@ impl<'a> CreateModel<'a> {
         if compiler::unlikely(state.remaining() < 10) {
             return compiler::cold_rerr(LangError::UnexpectedEOS);
         }
-        // field name; ignore errors
-        let field_name = state.fw_read();
-        state.poison_if_not(field_name.is_ident());
+        // model name; ignore errors
+        let model_name = state.fw_read();
+        state.poison_if_not(model_name.is_ident());
         state.poison_if_not(state.cursor_eq(Token![() open]));
         state.cursor_ahead();
         // fields
@@ -136,7 +139,10 @@ impl<'a> CreateModel<'a> {
         // we're done
         if state.okay() {
             Ok(Self {
-                model_name: unsafe { extract!(field_name, Token::Ident(id) => *id) },
+                model_name: unsafe {
+                    // UNSAFE(@ohsayan): we verified if `model_name` returns `is_ident`
+                    model_name.uck_read_ident()
+                },
                 fields,
                 props,
             })
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index a651fb57..1e5c7c0d 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -56,7 +56,7 @@ impl<'a> DropSpace<'a> {
                 return Ok(DropSpace::new(
                     unsafe {
                         // UNSAFE(@ohsayan): Safe because the match predicate ensures that tok[1] is indeed an ident
-                        extract!(ident, Token::Ident(ref space) => *space)
+                        ident.uck_read_ident()
                     },
                     force,
                 ));
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
index 33e7cc19..3dc8da88 100644
--- a/server/src/engine/ql/ddl/ins.rs
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -55,7 +55,7 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
         Token![space] if state.cursor_has_ident_rounded() => {
             Ok(Statement::InspectSpace(unsafe {
                 // UNSAFE(@ohsayan): Safe because of the match predicate
-                extract!(state.fw_read(), Token::Ident(ref space) => *space)
+                state.fw_read().uck_read_ident()
             }))
         }
         Token::Ident(id) if id.eq_ignore_ascii_case("spaces") && state.exhausted() => {
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 50f0ebe8..a0d67b34 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -418,7 +418,10 @@ impl<'a> ExpandedField<'a> {
         }
         if state.okay() {
             Ok(Self {
-                field_name: unsafe { extract!(field_name, Token::Ident(id) => *id) },
+                field_name: unsafe {
+                    // UNSAFE(@ohsayan): We just verified if `field_name` returns `is_ident`
+                    field_name.uck_read_ident()
+                },
                 props,
                 layers,
             })
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index 669ecfaa..8a9690b8 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -25,7 +25,6 @@
 */
 
 use {
-    super::read_ident,
     crate::{
         engine::{
             core::Datacell,
@@ -200,7 +199,7 @@ pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
 /// ## Safety
 /// - Cursor must match arity(0) function signature
 unsafe fn handle_func_sub<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> Option<Datacell> {
-    let func = read_ident(state.fw_read());
+    let func = state.fw_read().uck_read_ident();
     state.cursor_ahead_by(2); // skip tt:paren
     ldfunc(func).map(move |f| f())
 }
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index e09fbe8a..ca3ccf42 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -37,17 +37,12 @@ pub mod upd;
 use {
     super::{
         ast::{QueryData, State},
-        lex::{Ident, Token},
+        lex::Ident,
     },
     crate::{engine::data::lit::LitIR, util::compiler},
     std::collections::HashMap,
 };
 
-#[inline(always)]
-unsafe fn read_ident<'a>(tok: &'a Token<'a>) -> Ident<'a> {
-    extract!(tok, Token::Ident(ref tok) => *tok)
-}
-
 #[inline(always)]
 fn u(b: bool) -> u8 {
     b as _
@@ -108,9 +103,11 @@ impl<'a> RelationalExpr<'a> {
         state.poison_if_not(state.can_read_lit_rounded());
         if compiler::likely(state.okay()) {
             unsafe {
+                // UNSAFE(@ohsayan): we verified this above
                 let lit = state.read_cursor_lit_unchecked();
                 state.cursor_ahead();
-                Some(Self::new(read_ident(ident), lit, operator))
+                // UNSAFE(@ohsayan): we checked if `ident` returns `is_ident` and updated state
+                Some(Self::new(ident.uck_read_ident(), lit, operator))
             }
         } else {
             None
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index 50c160bc..8459fb13 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -27,7 +27,7 @@
 #[cfg(test)]
 use super::WhereClauseCollection;
 use {
-    super::{read_ident, u, WhereClause},
+    super::{u, WhereClause},
     crate::{
         engine::{
             data::lit::LitIR,
@@ -123,7 +123,8 @@ impl<'a> AssignmentExpression<'a> {
                 let rhs = state.read_cursor_lit_unchecked();
                 state.cursor_ahead();
                 expressions.push(AssignmentExpression::new(
-                    read_ident(lhs),
+                    // UNSAFE(@ohsayan): we verified if `lhs` returns `is_ident`
+                    lhs.uck_read_ident(),
                     rhs,
                     OPERATOR[operator_code as usize],
                 ))
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index 43b84b58..a0dc8b8b 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -124,6 +124,15 @@ pub enum Token<'a> {
     Lit(Lit<'a>), // literal
 }
 
+impl<'a> Token<'a> {
+    pub unsafe fn uck_read_ident(&self) -> Ident<'a> {
+        extract!(self, Self::Ident(id) => *id)
+    }
+    pub unsafe fn uck_read_lit(&self) -> &Lit<'a> {
+        extract!(self, Self::Lit(l) => l)
+    }
+}
+
 impl<'a> ToString for Token<'a> {
     fn to_string(&self) -> String {
         match self {

From ccfb7b2e12184568947be28507a6391824c2d011 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 1 Apr 2023 08:52:06 -0700
Subject: [PATCH 163/310] Unsafe code review (partial)

Unsafe code review for March, 2023.

NEEDCHECK: Still need to verify unsafe code in index implementations
---
 server/src/engine/core/model/cell.rs | 90 +++++++++++++++++++++------
 server/src/engine/core/model/mod.rs  | 55 ++++++++++-------
 server/src/engine/data/spec.rs       |  2 +-
 server/src/engine/mem/astr.rs        | 10 ++-
 server/src/engine/mem/uarray.rs      |  2 +
 server/src/engine/mem/vinline.rs     | 44 ++++++-------
 server/src/engine/mem/word.rs        | 30 +++++++--
 server/src/engine/ql/ast/mod.rs      |  4 ++
 server/src/engine/ql/ddl/alt.rs      |  5 +-
 server/src/engine/ql/ddl/drop.rs     |  2 +-
 server/src/engine/ql/ddl/syn.rs      | 71 +++++++++++++--------
 server/src/engine/ql/dml/ins.rs      |  4 +-
 server/src/engine/ql/lex/mod.rs      | 92 +++++++++++++++++++++-------
 server/src/engine/ql/lex/raw.rs      | 30 +++++++--
 14 files changed, 317 insertions(+), 124 deletions(-)

diff --git a/server/src/engine/core/model/cell.rs b/server/src/engine/core/model/cell.rs
index bd0bb135..8b42cd69 100644
--- a/server/src/engine/core/model/cell.rs
+++ b/server/src/engine/core/model/cell.rs
@@ -49,52 +49,76 @@ pub struct Datacell {
 impl Datacell {
     // bool
     pub fn new_bool(b: bool) -> Self {
-        unsafe { Self::new(TagClass::Bool, DataRaw::word(SystemDword::store(b))) }
+        unsafe {
+            // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
+            Self::new(TagClass::Bool, DataRaw::word(SystemDword::store(b)))
+        }
     }
     pub unsafe fn read_bool(&self) -> bool {
         self.load_word()
     }
     pub fn try_bool(&self) -> Option<bool> {
-        self.checked_tag(TagClass::Bool, || unsafe { self.read_bool() })
+        self.checked_tag(TagClass::Bool, || unsafe {
+            // UNSAFE(@ohsayan): correct because we just verified the tag
+            self.read_bool()
+        })
     }
     pub fn bool(&self) -> bool {
         self.try_bool().unwrap()
     }
     // uint
     pub fn new_uint(u: u64) -> Self {
-        unsafe { Self::new(TagClass::UnsignedInt, DataRaw::word(SystemDword::store(u))) }
+        unsafe {
+            // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
+            Self::new(TagClass::UnsignedInt, DataRaw::word(SystemDword::store(u)))
+        }
     }
     pub unsafe fn read_uint(&self) -> u64 {
         self.load_word()
     }
     pub fn try_uint(&self) -> Option<u64> {
-        self.checked_tag(TagClass::UnsignedInt, || unsafe { self.read_uint() })
+        self.checked_tag(TagClass::UnsignedInt, || unsafe {
+            // UNSAFE(@ohsayan): correct because we just verified the tag
+            self.read_uint()
+        })
     }
     pub fn uint(&self) -> u64 {
         self.try_uint().unwrap()
     }
     // sint
     pub fn new_sint(u: i64) -> Self {
-        unsafe { Self::new(TagClass::SignedInt, DataRaw::word(SystemDword::store(u))) }
+        unsafe {
+            // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
+            Self::new(TagClass::SignedInt, DataRaw::word(SystemDword::store(u)))
+        }
     }
     pub unsafe fn read_sint(&self) -> i64 {
         self.load_word()
     }
     pub fn try_sint(&self) -> Option<i64> {
-        self.checked_tag(TagClass::SignedInt, || unsafe { self.read_sint() })
+        self.checked_tag(TagClass::SignedInt, || unsafe {
+            // UNSAFE(@ohsayan): Correct because we just verified the tag
+            self.read_sint()
+        })
     }
     pub fn sint(&self) -> i64 {
         self.try_sint().unwrap()
     }
     // float
     pub fn new_float(f: f64) -> Self {
-        unsafe { Self::new(TagClass::Float, DataRaw::word(SystemDword::store(f))) }
+        unsafe {
+            // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
+            Self::new(TagClass::Float, DataRaw::word(SystemDword::store(f)))
+        }
     }
     pub unsafe fn read_float(&self) -> f64 {
         self.load_word()
     }
     pub fn try_float(&self) -> Option<f64> {
-        self.checked_tag(TagClass::Float, || unsafe { self.read_float() })
+        self.checked_tag(TagClass::Float, || unsafe {
+            // UNSAFE(@ohsayan): Correcrt because we just verified the tag
+            self.read_float()
+        })
     }
     pub fn float(&self) -> f64 {
         self.try_float().unwrap()
@@ -103,6 +127,7 @@ impl Datacell {
     pub fn new_bin(s: Box<[u8]>) -> Self {
         let mut md = ManuallyDrop::new(s);
         unsafe {
+            // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
                 TagClass::Bin,
                 DataRaw::word(SystemDword::store((md.as_mut_ptr(), md.len()))),
@@ -114,7 +139,10 @@ impl Datacell {
         slice::from_raw_parts::<u8>(p, l)
     }
     pub fn try_bin(&self) -> Option<&[u8]> {
-        self.checked_tag(TagClass::Bin, || unsafe { self.read_bin() })
+        self.checked_tag(TagClass::Bin, || unsafe {
+            // UNSAFE(@ohsayan): Correct because we just verified the tag
+            self.read_bin()
+        })
     }
     pub fn bin(&self) -> &[u8] {
         self.try_bin().unwrap()
@@ -123,6 +151,7 @@ impl Datacell {
     pub fn new_str(s: Box<str>) -> Self {
         let mut md = ManuallyDrop::new(s.into_boxed_bytes());
         unsafe {
+            // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
                 TagClass::Str,
                 DataRaw::word(SystemDword::store((md.as_mut_ptr(), md.len()))),
@@ -134,20 +163,29 @@ impl Datacell {
         str::from_utf8_unchecked(slice::from_raw_parts(p, l))
     }
     pub fn try_str(&self) -> Option<&str> {
-        self.checked_tag(TagClass::Str, || unsafe { self.read_str() })
+        self.checked_tag(TagClass::Str, || unsafe {
+            // UNSAFE(@ohsayan): Correct because we just verified the tag
+            self.read_str()
+        })
     }
     pub fn str(&self) -> &str {
         self.try_str().unwrap()
     }
     // list
     pub fn new_list(l: Vec<Self>) -> Self {
-        unsafe { Self::new(TagClass::List, DataRaw::rwl(RwLock::new(l))) }
+        unsafe {
+            // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
+            Self::new(TagClass::List, DataRaw::rwl(RwLock::new(l)))
+        }
     }
     pub unsafe fn read_list(&self) -> &RwLock<Vec<Self>> {
         &self.data.rwl
     }
     pub fn try_list(&self) -> Option<&RwLock<Vec<Self>>> {
-        self.checked_tag(TagClass::List, || unsafe { self.read_list() })
+        self.checked_tag(TagClass::List, || unsafe {
+            // UNSAFE(@ohsayan): Correct because we just verified the tag
+            self.read_list()
+        })
     }
     pub fn list(&self) -> &RwLock<Vec<Self>> {
         self.try_list().unwrap()
@@ -176,20 +214,25 @@ impl<'a> From<LitIR<'a>> for Datacell {
     fn from(l: LitIR<'a>) -> Self {
         match l.kind().tag_class() {
             tag if tag < TagClass::Bin => unsafe {
-                // DO NOT RELY ON the payload's bit pattern; it's padded
+                // UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type doesn't need any advanced construction
                 Datacell::new(
                     l.kind().tag_class(),
+                    // DO NOT RELY ON the payload's bit pattern; it's padded
                     DataRaw::word(SystemDword::store_qw(l.data().load_qw())),
                 )
             },
             tag @ (TagClass::Bin | TagClass::Str) => unsafe {
+                // UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type requires a new heap for construction
                 let mut bin = ManuallyDrop::new(l.read_bin_uck().to_owned().into_boxed_slice());
                 Datacell::new(
                     tag,
                     DataRaw::word(SystemDword::store((bin.as_mut_ptr(), bin.len()))),
                 )
             },
-            _ => unreachable!(),
+            _ => unsafe {
+                // UNSAFE(@ohsayan): a Lit will never be higher than a string
+                impossible!()
+            },
         }
     }
 }
@@ -223,6 +266,7 @@ impl Datacell {
     }
     pub fn null() -> Self {
         unsafe {
+            // UNSAFE(@ohsayan): This is a hack. It's safe because we set init to false
             Self::_new(
                 TagClass::Bool,
                 DataRaw::word(NativeQword::store_qw(0)),
@@ -333,10 +377,14 @@ impl Drop for Datacell {
     fn drop(&mut self) {
         match self.tag {
             TagClass::Str | TagClass::Bin => unsafe {
+                // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class
                 let (p, l) = self.load_word();
                 engine::mem::dealloc_array::<u8>(p, l)
             },
-            TagClass::List => unsafe { ManuallyDrop::drop(&mut self.data.rwl) },
+            TagClass::List => unsafe {
+                // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class
+                ManuallyDrop::drop(&mut self.data.rwl)
+            },
             _ => {}
         }
     }
@@ -347,23 +395,29 @@ impl Clone for Datacell {
     fn clone(&self) -> Self {
         let data = match self.tag {
             TagClass::Str | TagClass::Bin => unsafe {
-                let block = ManuallyDrop::new(self.read_bin().to_owned().into_boxed_slice());
+                // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class
+                let mut block = ManuallyDrop::new(self.read_bin().to_owned().into_boxed_slice());
                 DataRaw {
-                    word: ManuallyDrop::new(SystemDword::store((block.as_ptr(), block.len()))),
+                    word: ManuallyDrop::new(SystemDword::store((block.as_mut_ptr(), block.len()))),
                 }
             },
             TagClass::List => unsafe {
+                // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class
                 let data = self.read_list().read().iter().cloned().collect();
                 DataRaw {
                     rwl: ManuallyDrop::new(RwLock::new(data)),
                 }
             },
             _ => unsafe {
+                // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class
                 DataRaw {
                     word: ManuallyDrop::new(mem::transmute_copy(&self.data.word)),
                 }
             },
         };
-        unsafe { Self::_new(self.tag, data, self.init) }
+        unsafe {
+            // UNSAFE(@ohsayan): same tag, we correctly init data and also return the same init state
+            Self::_new(self.tag, data, self.init)
+        }
     }
 }
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 2c289e93..c4dd1e42 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -245,20 +245,22 @@ impl Field {
         }
     }
     #[inline(always)]
-    fn single_pass_for(&self, dc: &Datacell) -> bool {
-        ((self.layers().len() == 1) & (self.layers()[0].tag.tag_class() == dc.kind()))
-            | (self.nullable & dc.is_null())
-    }
-    #[inline(always)]
     fn compute_index(&self, dc: &Datacell) -> usize {
-        // escape check if it makes sense to
-        !(self.nullable & dc.is_null()) as usize * self.layers()[0].tag.tag_class().word()
+        if ((!self.is_nullable()) & dc.is_null()) | (self.layers[0].tag.tag_class() != dc.kind()) {
+            // illegal states: (1) bad null (2) tags don't match
+            7
+        } else {
+            self.layers()[0].tag.tag_class().word()
+        }
     }
     pub fn validate_data_fpath(&self, data: &Datacell) -> bool {
         // if someone sends a PR with an added check, I'll personally come to your house and throw a brick on your head
-        if self.single_pass_for(data) {
+        if self.layers.len() == 1 {
             layertrace("fpath");
-            unsafe { LVERIFY[self.compute_index(data)](self.layers()[0], data) }
+            unsafe {
+                // UNSAFE(@ohsayan): checked for non-null, and used correct class
+                LVERIFY[self.compute_index(data)](self.layers()[0], data)
+            }
         } else {
             Self::rverify_layers(self.layers(), data)
         }
@@ -268,19 +270,29 @@ impl Field {
         let layer = layers[0];
         let layers = &layers[1..];
         match (layer.tag.tag_class(), data.kind()) {
-            (layer_tag, data_tag) if (layer_tag == data_tag) & (layer_tag < TagClass::List) => {
-                // time to go home
-                (unsafe { LVERIFY[layer.tag.tag_class().word()](layer, data) } & layers.is_empty())
-            }
-            (TagClass::List, TagClass::List) => unsafe {
-                let mut okay = !layers.is_empty() & LVERIFY[TagClass::List.word()](layer, data);
-                let list = data.read_list().read();
-                let mut it = list.iter();
-                while (it.len() != 0) & okay {
-                    okay &= Self::rverify_layers(layers, it.next().unwrap());
+            (TagClass::List, TagClass::List) if !layers.is_empty() => {
+                let mut okay = unsafe {
+                    // UNSAFE(@ohsayan): we've verified this
+                    LVERIFY[TagClass::List.word()](layer, data)
+                };
+                let list = unsafe {
+                    // UNSAFE(@ohsayan): we verified tags
+                    data.read_list()
+                };
+                let lread = list.read();
+                let mut i = 0;
+                while (i < lread.len()) & okay {
+                    okay &= Self::rverify_layers(layers, &lread[i]);
+                    i += 1;
                 }
                 okay
-            },
+            }
+            (tag_a, tag_b) if tag_a == tag_b => {
+                unsafe {
+                    // UNSAFE(@ohsayan): same tags; not-null for now so no extra handling required here
+                    LVERIFY[tag_a.word()](layer, data)
+                }
+            }
             _ => false,
         }
     }
@@ -374,7 +386,7 @@ impl Layer {
     }
 }
 
-static LVERIFY: [unsafe fn(Layer, &Datacell) -> bool; 7] = [
+static LVERIFY: [unsafe fn(Layer, &Datacell) -> bool; 8] = [
     lverify_bool,
     lverify_uint,
     lverify_sint,
@@ -382,6 +394,7 @@ static LVERIFY: [unsafe fn(Layer, &Datacell) -> bool; 7] = [
     lverify_bin,
     lverify_str,
     lverify_list,
+    |_, _| false,
 ];
 
 #[cfg(test)]
diff --git a/server/src/engine/data/spec.rs b/server/src/engine/data/spec.rs
index 113e5cf3..19f56909 100644
--- a/server/src/engine/data/spec.rs
+++ b/server/src/engine/data/spec.rs
@@ -244,7 +244,7 @@ pub unsafe trait DataspecMethods1D: Dataspec1D {
                 // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
                 <Self as DataspecRaw1D>::clone_str(Dataspec1D::read_str_uck(self))
             },
-            TagClass::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
+            TagClass::Bin if <Self as DataspecRaw1D>::HEAP_BIN => unsafe {
                 // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
                 <Self as DataspecRaw1D>::clone_bin(Dataspec1D::read_bin_uck(self))
             },
diff --git a/server/src/engine/mem/astr.rs b/server/src/engine/mem/astr.rs
index 5d92ace3..6b4052b7 100644
--- a/server/src/engine/mem/astr.rs
+++ b/server/src/engine/mem/astr.rs
@@ -75,11 +75,17 @@ impl<const N: usize> AStr<N> {
     }
     #[inline(always)]
     pub fn _as_str(&self) -> &str {
-        unsafe { mem::transmute(self._as_bytes()) }
+        unsafe {
+            // UNSAFE(@ohsayan): same layout
+            mem::transmute(self._as_bytes())
+        }
     }
     #[inline(always)]
     pub fn _as_mut_str(&mut self) -> &mut str {
-        unsafe { mem::transmute(self._as_bytes_mut()) }
+        unsafe {
+            // UNSAFE(@ohsayan): same layout
+            mem::transmute(self._as_bytes_mut())
+        }
     }
     pub fn _as_bytes(&self) -> &[u8] {
         self.base.as_slice()
diff --git a/server/src/engine/mem/uarray.rs b/server/src/engine/mem/uarray.rs
index 0a0ed0a9..8d64e86f 100644
--- a/server/src/engine/mem/uarray.rs
+++ b/server/src/engine/mem/uarray.rs
@@ -151,7 +151,9 @@ impl<const N: usize, T: Copy> UArray<N, T> {
         debug_assert!(s.len() <= N);
         let mut new = Self::new();
         unsafe {
+            // UNSAFE(@ohsayan): the src pointer *will* be correct and the dst is us, and we own our stack here
             ptr::copy_nonoverlapping(s.as_ptr(), new.a.as_mut_ptr() as *mut T, s.len());
+            // UNSAFE(@ohsayan): and here goes the call; same length as the origin buffer
             new.set_len(s.len());
         }
         new
diff --git a/server/src/engine/mem/vinline.rs b/server/src/engine/mem/vinline.rs
index f0702406..04552860 100644
--- a/server/src/engine/mem/vinline.rs
+++ b/server/src/engine/mem/vinline.rs
@@ -82,7 +82,7 @@ impl<const N: usize, T> VInline<N, T> {
     }
     #[inline(always)]
     pub fn remove(&mut self, idx: usize) -> T {
-        if idx >= self.len() {
+        if !(idx < self.len()) {
             panic!("index out of range");
         }
         unsafe {
@@ -190,11 +190,13 @@ impl<const N: usize, T> VInline<N, T> {
             return;
         }
         if self.l <= N {
+            // the current can be fit into the stack, and we aren't on the stack. so copy data from heap and move it to the stack
             unsafe {
                 // UNSAFE(@ohsayan): non-null heap
                 self.mv_to_stack();
             }
         } else {
+            // in this case, we can't move to stack but can optimize the heap size. so create a new heap, memcpy old heap and destroy old heap (NO dtor)
             let nb = Self::alloc_block(self.len());
             unsafe {
                 // UNSAFE(@ohsayan): nonov; non-null
@@ -217,28 +219,27 @@ impl<const N: usize, T> VInline<N, T> {
     }
     #[inline]
     fn grow(&mut self) {
-        if !(self.l == self.capacity()) {
-            return;
-        }
-        // allocate new block
-        let nc = self.ncap();
-        let nb = Self::alloc_block(nc);
-        if self.on_stack() {
-            // stack -> heap
-            unsafe {
-                // UNSAFE(@ohsayan): non-null; valid len
-                ptr::copy_nonoverlapping(self.d.s.as_ptr() as *const T, nb, self.l);
-            }
-        } else {
-            unsafe {
-                // UNSAFE(@ohsayan): non-null; valid len
-                ptr::copy_nonoverlapping(self.d.h.cast_const(), nb, self.l);
-                // UNSAFE(@ohsayan): non-null heap
-                self.dealloc_heap(self.d.h);
+        if self.l == self.capacity() {
+            // allocate new block because we've run out of capacity
+            let nc = self.ncap();
+            let nb = Self::alloc_block(nc);
+            if self.on_stack() {
+                // stack -> heap
+                unsafe {
+                    // UNSAFE(@ohsayan): non-null; valid len
+                    ptr::copy_nonoverlapping(self.d.s.as_ptr() as *const T, nb, self.l);
+                }
+            } else {
+                unsafe {
+                    // UNSAFE(@ohsayan): non-null; valid len
+                    ptr::copy_nonoverlapping(self.d.h.cast_const(), nb, self.l);
+                    // UNSAFE(@ohsayan): non-null heap
+                    self.dealloc_heap(self.d.h);
+                }
             }
+            self.d.h = nb;
+            self.c = nc;
         }
-        self.d.h = nb;
-        self.c = nc;
     }
     #[inline(always)]
     unsafe fn dealloc_heap(&mut self, heap: *mut T) {
@@ -338,6 +339,7 @@ impl<const N: usize, T> IntoIter<N, T> {
             return None;
         }
         unsafe {
+            // UNSAFE(@ohsayan): we get the back pointer and move back; always behind EOA so we're chill
             self.l -= 1;
             ptr::read(self.v._as_ptr().add(self.l).cast())
         }
diff --git a/server/src/engine/mem/word.rs b/server/src/engine/mem/word.rs
index 76747dc0..97ba2099 100644
--- a/server/src/engine/mem/word.rs
+++ b/server/src/engine/mem/word.rs
@@ -105,7 +105,10 @@ impl SystemDword for NativeDword {
         let x;
         #[cfg(target_pointer_width = "32")]
         {
-            x = unsafe { core::mem::transmute(u) };
+            x = unsafe {
+                // UNSAFE(@ohsayan): same layout and this is a stupidly simple cast and it's wild that the rust std doesn't have a simpler way to do it
+                core::mem::transmute(u)
+            };
         }
         #[cfg(target_pointer_width = "64")]
         {
@@ -122,7 +125,10 @@ impl SystemDword for NativeDword {
         let x;
         #[cfg(target_pointer_width = "32")]
         {
-            x = unsafe { core::mem::transmute_copy(self) }
+            x = unsafe {
+                // UNSAFE(@ohsayan): same layout and this is a stupidly simple cast and it's wild that the rust std doesn't have a simpler way to do it
+                core::mem::transmute_copy(self)
+            }
         }
         #[cfg(target_pointer_width = "64")]
         {
@@ -153,7 +159,10 @@ impl SystemDword for NativeTword {
         let x;
         #[cfg(target_pointer_width = "32")]
         {
-            let [a, b]: [usize; 2] = unsafe { core::mem::transmute(u) };
+            let [a, b]: [usize; 2] = unsafe {
+                // UNSAFE(@ohsayan): same layout and this is a stupidly simple cast and it's wild that the rust std doesn't have a simpler way to do it
+                core::mem::transmute(u)
+            };
             x = [a, b, 0];
         }
         #[cfg(target_pointer_width = "64")]
@@ -172,7 +181,10 @@ impl SystemDword for NativeTword {
         #[cfg(target_pointer_width = "32")]
         {
             let ab = [self.0[0], self.0[1]];
-            x = unsafe { core::mem::transmute(ab) };
+            x = unsafe {
+                // UNSAFE(@ohsayan): same layout and this is a stupidly simple cast and it's wild that the rust std doesn't have a simpler way to do it
+                core::mem::transmute(ab)
+            };
         }
         #[cfg(target_pointer_width = "64")]
         {
@@ -209,7 +221,10 @@ impl SystemDword for NativeQword {
         let ret;
         #[cfg(target_pointer_width = "32")]
         {
-            let [a, b]: [usize; 2] = unsafe { core::mem::transmute(u) };
+            let [a, b]: [usize; 2] = unsafe {
+                // UNSAFE(@ohsayan): same layout and this is a stupidly simple cast and it's wild that the rust std doesn't have a simpler way to do it
+                core::mem::transmute(u)
+            };
             ret = <Self as SystemQword>::store_full(a, b, 0, 0);
         }
         #[cfg(target_pointer_width = "64")]
@@ -225,7 +240,10 @@ impl SystemDword for NativeQword {
         let ret;
         #[cfg(target_pointer_width = "32")]
         {
-            ret = unsafe { core::mem::transmute([self.0[0], self.0[1]]) };
+            ret = unsafe {
+                // UNSAFE(@ohsayan): same layout and this is a stupidly simple cast and it's wild that the rust std doesn't have a simpler way to do it
+                core::mem::transmute([self.0[0], self.0[1]])
+            };
         }
         #[cfg(target_pointer_width = "64")]
         {
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index dd22510b..27feba1b 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -416,10 +416,12 @@ impl<'a> Entity<'a> {
         let is_full = Self::tokens_with_full(tok);
         let r = match () {
             _ if is_full => unsafe {
+                // UNSAFE(@ohsayan): just verified signature
                 *c += 3;
                 Self::full_entity_from_slice(tok)
             },
             _ if is_current => unsafe {
+                // UNSAFE(@ohsayan): just verified signature
                 *c += 1;
                 Self::single_entity_from_slice(tok)
             },
@@ -451,6 +453,7 @@ impl<'a> Entity<'a> {
         let is_full = state.cursor_signature_match_entity_full_rounded();
         let is_single = state.cursor_has_ident_rounded();
         unsafe {
+            // UNSAFE(@ohsayan): verified signatures
             if is_full {
                 state.cursor_ahead_by(3);
                 *d = MaybeInit::new(Entity::full_entity_from_slice(tok));
@@ -469,6 +472,7 @@ impl<'a> Entity<'a> {
         let is_full = tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident();
         let is_single = tok[0].is_ident();
         unsafe {
+            // UNSAFE(@ohsayan): verified signatures
             if is_full {
                 state.cursor_ahead_by(3);
                 *d = MaybeInit::new(Entity::full_entity_from_slice(tok));
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 43c24d64..1b5e59b8 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -117,7 +117,10 @@ impl<'a> AlterModel<'a> {
             return compiler::cold_rerr(LangError::BadSyntax);
             // FIXME(@ohsayan): bad because no specificity
         }
-        let model_name = unsafe { state.fw_read().uck_read_ident() };
+        let model_name = unsafe {
+            // UNSAFE(@ohsayan): did rounded check for ident in the above branch
+            state.fw_read().uck_read_ident()
+        };
         let kind = match state.fw_read() {
             Token![add] => AlterKind::alter_add(state),
             Token![remove] => AlterKind::alter_remove(state),
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index 1e5c7c0d..cbf2605d 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -55,7 +55,7 @@ impl<'a> DropSpace<'a> {
             if state.exhausted() {
                 return Ok(DropSpace::new(
                     unsafe {
-                        // UNSAFE(@ohsayan): Safe because the match predicate ensures that tok[1] is indeed an ident
+                        // UNSAFE(@ohsayan): Safe because the if predicate ensures that tok[0] (relative) is indeed an ident
                         ident.uck_read_ident()
                     },
                     force,
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index a0d67b34..75917ff6 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -151,18 +151,35 @@ where
             }
             (tok, DictFoldState::LIT_OR_OB) if state.can_read_lit_from(tok) => {
                 // found lit
-                unsafe {
-                    let v = Some(state.read_lit_unchecked_from(tok).into());
-                    state.poison_if_not(dict.insert(key.take().as_str().into(), v).is_none());
-                }
+                let v = Some(unsafe {
+                    // UNSAFE(@ohsayan): verified at guard
+                    state.read_lit_unchecked_from(tok).into()
+                });
+                state.poison_if_not(
+                    dict.insert(
+                        unsafe {
+                            // UNSAFE(@ohsayan): we switch to this state only when we are in the LIT_OR_OB state. this means that we've already read in a key
+                            key.take().as_str().into()
+                        },
+                        v,
+                    )
+                    .is_none(),
+                );
                 // after lit we're either done or expect something else
                 mstate = DictFoldState::COMMA_OR_CB;
             }
             (Token![null], DictFoldState::LIT_OR_OB) => {
                 // found a null
-                unsafe {
-                    state.poison_if_not(dict.insert(key.take().as_str().into(), None).is_none());
-                }
+                state.poison_if_not(
+                    dict.insert(
+                        unsafe {
+                            // UNSAFE(@ohsayan): we only switch to this when we've already read in a key
+                            key.take().as_str().into()
+                        },
+                        None,
+                    )
+                    .is_none(),
+                );
                 // after a null (essentially counts as a lit) we're either done or expect something else
                 mstate = DictFoldState::COMMA_OR_CB;
             }
@@ -170,12 +187,16 @@ where
                 // found a nested dict
                 let mut ndict = DictGeneric::new();
                 _rfold_dict::<Qd, NoBreakpoint>(DictFoldState::CB_OR_IDENT, state, &mut ndict);
-                unsafe {
-                    state.poison_if_not(
-                        dict.insert(key.take().as_str().into(), Some(ndict.into()))
-                            .is_none(),
-                    );
-                }
+                state.poison_if_not(
+                    dict.insert(
+                        unsafe {
+                            // UNSAFE(@ohsayan): correct again because whenever we hit an expression position, we've already read in a key (ident)
+                            key.take().as_str().into()
+                        },
+                        Some(ndict.into()),
+                    )
+                    .is_none(),
+                );
                 mstate = DictFoldState::COMMA_OR_CB;
             }
             (Token![,], DictFoldState::COMMA_OR_CB) => {
@@ -240,11 +261,8 @@ states! {
     }
 }
 
-fn rfold_layers<'a, Qd: QueryData<'a>>(
-    mut mstate: LayerFoldState,
-    state: &mut State<'a, Qd>,
-    layers: &mut Vec<LayerSpec<'a>>,
-) {
+fn rfold_layers<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>, layers: &mut Vec<LayerSpec<'a>>) {
+    let mut mstate = LayerFoldState::BEGIN_IDENT;
     let mut ty = MaybeInit::uninit();
     let mut props = Default::default();
     while state.loop_tt() {
@@ -260,7 +278,7 @@ fn rfold_layers<'a, Qd: QueryData<'a>>(
                     // but we first need a colon
                     state.poison_if_not(state.cursor_rounded_eq(Token![:]));
                     state.cursor_ahead_if(state.okay());
-                    rfold_layers(LayerFoldState::BEGIN_IDENT, state, layers);
+                    rfold_layers(state, layers);
                     // we are yet to parse the remaining props
                     mstate = LayerFoldState::FOLD_INCOMPLETE;
                 } else {
@@ -297,7 +315,10 @@ fn rfold_layers<'a, Qd: QueryData<'a>>(
     if ((mstate == LayerFoldState::FINAL) | (mstate == LayerFoldState::FINAL_OR_OB)) & state.okay()
     {
         layers.push(LayerSpec {
-            ty: unsafe { ty.take() },
+            ty: unsafe {
+                // UNSAFE(@ohsayan): our start state always looks for an ident
+                ty.take()
+            },
             props,
         });
     } else {
@@ -351,7 +372,7 @@ impl<'a> FieldSpec<'a> {
         };
         // layers
         let mut layers = Vec::new();
-        rfold_layers(LayerFoldState::BEGIN_IDENT, state, &mut layers);
+        rfold_layers(state, &mut layers);
         if state.okay() {
             Ok(FieldSpec {
                 field_name: field_name.clone(),
@@ -403,7 +424,7 @@ impl<'a> ExpandedField<'a> {
             }
             state.poison_if_not(state.cursor_eq(Token![:]));
             state.cursor_ahead();
-            rfold_layers(LayerFoldState::BEGIN_IDENT, state, &mut layers);
+            rfold_layers(state, &mut layers);
             match state.fw_read() {
                 Token![,] => {
                     rfold_dict(DictFoldState::CB_OR_IDENT, state, &mut props);
@@ -489,7 +510,7 @@ mod impls {
     use {
         super::{
             rfold_dict, rfold_layers, rfold_tymeta, DictFoldState, DictGeneric, ExpandedField,
-            FieldSpec, LayerFoldState, LayerSpec,
+            FieldSpec, LayerSpec,
         },
         crate::engine::{
             error::LangResult,
@@ -511,7 +532,7 @@ mod impls {
         const VERIFY: bool = true;
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
             let mut layers = Vec::new();
-            rfold_layers(LayerFoldState::BEGIN_IDENT, state, &mut layers);
+            rfold_layers(state, &mut layers);
             assert!(layers.len() == 1);
             Ok(layers.swap_remove(0))
         }
@@ -519,7 +540,7 @@ mod impls {
             state: &mut State<'a, Qd>,
         ) -> LangResult<Vec<Self>> {
             let mut l = Vec::new();
-            rfold_layers(LayerFoldState::BEGIN_IDENT, state, &mut l);
+            rfold_layers(state, &mut l);
             Ok(l)
         }
     }
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index 8a9690b8..fab38226 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -368,12 +368,12 @@ impl<'a> InsertStatement<'a> {
         }
         if state.okay() {
             let data = unsafe {
-                // UNSAFE(@ohsayan): state's flag guarantees correctness
+                // UNSAFE(@ohsayan): state's flag guarantees correctness (see wildcard branch)
                 data.unwrap_unchecked()
             };
             Ok(InsertStatement {
                 entity: unsafe {
-                    // UNSAFE(@ohsayan): state's flag ensures correctness
+                    // UNSAFE(@ohsayan): state's flag ensures correctness (see Entity::parse_entity)
                     entity.assume_init()
                 },
                 data,
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index fcfb99b9..e0346a51 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -77,7 +77,10 @@ impl<'a> InsecureLexer<'a> {
     fn _lex(&mut self) {
         let ref mut slf = self.base;
         while slf.not_exhausted() && slf.no_error() {
-            match unsafe { slf.deref_cursor() } {
+            match unsafe {
+                // UNSAFE(@ohsayan): Verified non-null from pre
+                slf.deref_cursor()
+            } {
                 byte if byte.is_ascii_alphabetic() => slf.scan_ident_or_keyword(),
                 #[cfg(test)]
                 byte if byte == b'\x01' => {
@@ -104,6 +107,7 @@ impl<'a> InsecureLexer<'a> {
     #[inline(always)]
     fn scan_signed_integer(slf: &mut RawLexer<'a>) {
         unsafe {
+            // UNSAFE(@ohsayan): We hit an integer hence this was called
             slf.incr_cursor();
         }
         if slf.peek_is(|b| b.is_ascii_digit()) {
@@ -116,8 +120,10 @@ impl<'a> InsecureLexer<'a> {
             while slf.peek_is_and_forward(|b| b.is_ascii_digit()) {}
             let wseof = slf.peek_is(|char| !char.is_ascii_alphabetic()) || slf.exhausted();
             match unsafe {
+                // UNSAFE(@ohsayan): a sequence of ASCII bytes in the integer range will always be correct unicode
                 str::from_utf8_unchecked(slice::from_raw_parts(
                     start,
+                    // UNSAFE(@ohsayan): valid cursor and start pointers
                     slf.cursor().offset_from(start) as usize,
                 ))
             }
@@ -137,47 +143,66 @@ impl<'a> InsecureLexer<'a> {
     #[inline(always)]
     fn scan_unsigned_integer(slf: &mut RawLexer<'a>) {
         let s = slf.cursor();
-        unsafe {
-            while slf.peek_is(|b| b.is_ascii_digit()) {
+
+        while slf.peek_is(|b| b.is_ascii_digit()) {
+            unsafe {
+                // UNSAFE(@ohsayan): since we're going ahead, this is correct (until EOA)
                 slf.incr_cursor();
             }
+        }
+        /*
+            1234; // valid
+            1234} // valid
+            1234{ // invalid
+            1234, // valid
+            1234a // invalid
+        */
+        let wseof = slf.peek_is(|char| !char.is_ascii_alphabetic()) || slf.exhausted();
+        match unsafe {
             /*
-                1234; // valid
-                1234} // valid
-                1234{ // invalid
-                1234, // valid
-                1234a // invalid
+                UNSAFE(@ohsayan):
+                (1) Valid cursor and start pointer (since we copy it from the cursor which is correct)
+                (2) All ASCII alphabetic bytes are captured, hence this will always be a correct unicode string
             */
-            let wseof = slf.peek_is(|char| !char.is_ascii_alphabetic()) || slf.exhausted();
-            match str::from_utf8_unchecked(slice::from_raw_parts(
+            str::from_utf8_unchecked(slice::from_raw_parts(
                 s,
                 slf.cursor().offset_from(s) as usize,
             ))
-            .parse()
-            {
-                Ok(num) if compiler::likely(wseof) => {
-                    slf.tokens.push(Token::Lit(Lit::UnsignedInt(num)))
-                }
-                _ => slf.set_error(LexError::InvalidUnsignedLiteral),
+        }
+        .parse()
+        {
+            Ok(num) if compiler::likely(wseof) => {
+                slf.tokens.push(Token::Lit(Lit::UnsignedInt(num)))
             }
+            _ => slf.set_error(LexError::InvalidUnsignedLiteral),
         }
     }
 
     #[inline(always)]
     fn scan_binary_literal(slf: &mut RawLexer<'a>) {
         unsafe {
+            // UNSAFE(@ohsayan): cursor increment since we hit the marker byte (CR)
             slf.incr_cursor();
         }
         let mut size = 0usize;
         let mut okay = true;
-        while slf.not_exhausted() && unsafe { slf.deref_cursor() != b'\n' } && okay {
+        while slf.not_exhausted()
+            && unsafe {
+                // UNSAFE(@ohsayan): verified non-exhaustion
+                slf.deref_cursor() != b'\n'
+            }
+            && okay
+        {
             /*
                 Don't ask me how stupid this is. Like, I was probably in some "mood" when I wrote this
                 and it works duh, but isn't the most elegant of things (could I have just used a parse?
                 nah, I'm just a hardcore numeric normie)
                 -- Sayan
             */
-            let byte = unsafe { slf.deref_cursor() };
+            let byte = unsafe {
+                // UNSAFE(@ohsayan): The pre invariant guarantees that this is correct
+                slf.deref_cursor()
+            };
             okay &= byte.is_ascii_digit();
             let (prod, of_flag) = size.overflowing_mul(10);
             okay &= !of_flag;
@@ -185,6 +210,7 @@ impl<'a> InsecureLexer<'a> {
             size = sum;
             okay &= !of_flag;
             unsafe {
+                // UNSAFE(@ohsayan): We just read something, so this is fine (until EOA)
                 slf.incr_cursor();
             }
         }
@@ -192,7 +218,9 @@ impl<'a> InsecureLexer<'a> {
         okay &= slf.remaining() >= size;
         if compiler::likely(okay) {
             unsafe {
+                // UNSAFE(@ohsayan): Correct cursor and length (from above we know that we have enough bytes)
                 slf.push_token(Lit::Bin(slice::from_raw_parts(slf.cursor(), size)));
+                // UNSAFE(@ohsayan): Correct length increment
                 slf.incr_cursor_by(size);
             }
         } else {
@@ -202,22 +230,31 @@ impl<'a> InsecureLexer<'a> {
     #[inline(always)]
     fn scan_quoted_string(slf: &mut RawLexer<'a>, quote_style: u8) {
         debug_assert!(
-            unsafe { slf.deref_cursor() } == quote_style,
+            unsafe {
+                // UNSAFE(@ohsayan): yessir, we just hit this byte. if called elsewhere, this function will crash and burn (or simply, segfault)
+                slf.deref_cursor()
+            } == quote_style,
             "illegal call to scan_quoted_string"
         );
-        unsafe { slf.incr_cursor() }
+        unsafe {
+            // UNSAFE(@ohsayan): Increment this cursor (this is correct since we just hit the quote)
+            slf.incr_cursor()
+        }
         let mut buf = Vec::new();
         unsafe {
             while slf.peek_neq(quote_style) {
+                // UNSAFE(@ohsayan): deref is good since peek passed
                 match slf.deref_cursor() {
                     b if b != b'\\' => {
                         buf.push(b);
                     }
                     _ => {
+                        // UNSAFE(@ohsayan): we read one byte, so this should work
                         slf.incr_cursor();
                         if slf.exhausted() {
                             break;
                         }
+                        // UNSAFE(@ohsayan): correct because of the above branch
                         let b = slf.deref_cursor();
                         let quote = b == quote_style;
                         let bs = b == b'\\';
@@ -228,6 +265,11 @@ impl<'a> InsecureLexer<'a> {
                         }
                     }
                 }
+                /*
+                    UNSAFE(@ohsayan): This is correct because:
+                    (a) If we are in arm 1: we move the cursor ahead from the `\` byte (the branch doesn't do it)
+                    (b) If we are in arm 2: we don't skip the second quote byte in the branch, hence this is correct
+                */
                 slf.incr_cursor();
             }
             let terminated = slf.peek_eq_and_forward(quote_style);
@@ -260,7 +302,10 @@ impl<'a> SafeLexer<'a> {
     fn _lex(self) -> LexResult<Vec<Token<'a>>> {
         let Self { base: mut l } = self;
         while l.not_exhausted() && l.no_error() {
-            let b = unsafe { l.deref_cursor() };
+            let b = unsafe {
+                // UNSAFE(@ohsayan): This is correct because of the pre invariant
+                l.deref_cursor()
+            };
             match b {
                 // ident or kw
                 b if b.is_ascii_alphabetic() => l.scan_ident_or_keyword(),
@@ -469,7 +514,10 @@ impl<'b> SafeQueryData<'b> {
         // incr cursor
         i += mx_extract;
         *cnt += i;
-        unsafe { slice::from_raw_parts(src.as_ptr(), mx_extract) }
+        unsafe {
+            // UNSAFE(@ohsayan): src is correct (guaranteed). even if the decoded length returns an error we still remain within bounds of the EOA
+            slice::from_raw_parts(src.as_ptr(), mx_extract)
+        }
     }
     #[inline(always)]
     pub(super) fn uint<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index a0dc8b8b..4787eb34 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -412,7 +412,10 @@ impl<'a> RawLexer<'a> {
     }
     #[inline(always)]
     pub(super) fn remaining(&self) -> usize {
-        unsafe { self.e.offset_from(self.c) as usize }
+        unsafe {
+            // UNSAFE(@ohsayan): valid ptrs
+            self.e.offset_from(self.c) as usize
+        }
     }
     #[inline(always)]
     pub(super) unsafe fn deref_cursor(&self) -> u8 {
@@ -437,12 +440,21 @@ impl<'a> RawLexer<'a> {
     }
     #[inline(always)]
     pub(super) fn peek_is(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
-        self.not_exhausted() && unsafe { f(self.deref_cursor()) }
+        self.not_exhausted()
+            && unsafe {
+                // UNSAFE(@ohsayan): verified cursor is nonnull
+                f(self.deref_cursor())
+            }
     }
     #[inline(always)]
     pub(super) fn peek_is_and_forward(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
-        let did_fw = self.not_exhausted() && unsafe { f(self.deref_cursor()) };
+        let did_fw = self.not_exhausted()
+            && unsafe {
+                // UNSAFE(@ohsayan): verified ptr
+                f(self.deref_cursor())
+            };
         unsafe {
+            // UNSAFE(@ohsayan): increment cursor
             self.incr_cursor_if(did_fw);
         }
         did_fw
@@ -450,18 +462,25 @@ impl<'a> RawLexer<'a> {
     #[inline(always)]
     fn peek_eq_and_forward_or_eof(&mut self, eq: u8) -> bool {
         unsafe {
+            // UNSAFE(@ohsayan): verified cursor
             let eq = self.not_exhausted() && self.deref_cursor() == eq;
+            // UNSAFE(@ohsayan): incr cursor if matched
             self.incr_cursor_if(eq);
             eq | self.exhausted()
         }
     }
     #[inline(always)]
     pub(super) fn peek_neq(&self, b: u8) -> bool {
-        self.not_exhausted() && unsafe { self.deref_cursor() != b }
+        self.not_exhausted()
+            && unsafe {
+                // UNSAFE(@ohsayan): verified cursor
+                self.deref_cursor() != b
+            }
     }
     #[inline(always)]
     pub(super) fn peek_eq_and_forward(&mut self, b: u8) -> bool {
         unsafe {
+            // UNSAFE(@ohsayan): verified cursor
             let r = self.not_exhausted() && self.deref_cursor() == b;
             self.incr_cursor_if(r);
             r
@@ -488,8 +507,10 @@ impl<'a> RawLexer<'a> {
         let s = self.cursor();
         unsafe {
             while self.peek_is(|b| b.is_ascii_alphanumeric() || b == b'_') {
+                // UNSAFE(@ohsayan): increment cursor, this is valid
                 self.incr_cursor();
             }
+            // UNSAFE(@ohsayan): valid slice and ptrs
             slice::from_raw_parts(s, self.cursor().offset_from(s) as usize)
         }
     }
@@ -514,6 +535,7 @@ impl<'a> RawLexer<'a> {
             None => return self.set_error(LexError::UnexpectedByte),
         }
         unsafe {
+            // UNSAFE(@ohsayan): we are sent a byte, so fw cursor
             self.incr_cursor();
         }
     }

From d626f9a302b76ee7d49693d612cc84215864a45c Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 1 Apr 2023 21:46:29 -0700
Subject: [PATCH 164/310] Fix layer validation call and entity parse

---
 server/src/engine/core/model/mod.rs | 9 ++++++---
 server/src/engine/ql/ast/mod.rs     | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index c4dd1e42..868356ed 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -246,16 +246,19 @@ impl Field {
     }
     #[inline(always)]
     fn compute_index(&self, dc: &Datacell) -> usize {
-        if ((!self.is_nullable()) & dc.is_null()) | (self.layers[0].tag.tag_class() != dc.kind()) {
+        if {
+            ((!self.is_nullable()) & dc.is_null())
+                | ((self.layers[0].tag.tag_class() != dc.kind()) & !dc.is_null())
+        } {
             // illegal states: (1) bad null (2) tags don't match
             7
         } else {
-            self.layers()[0].tag.tag_class().word()
+            dc.kind().word()
         }
     }
     pub fn validate_data_fpath(&self, data: &Datacell) -> bool {
         // if someone sends a PR with an added check, I'll personally come to your house and throw a brick on your head
-        if self.layers.len() == 1 {
+        if (self.layers.len() == 1) | data.is_null() {
             layertrace("fpath");
             unsafe {
                 // UNSAFE(@ohsayan): checked for non-null, and used correct class
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 27feba1b..8a23a2c0 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -384,7 +384,7 @@ impl<'a> Entity<'a> {
     /// Caller guarantees that the token stream matches the exact stream of tokens
     /// expected for a full entity
     pub(super) unsafe fn full_entity_from_slice(sl: &'a [Token]) -> Self {
-        Entity::Full(sl[0].uck_read_ident(), sl[1].uck_read_ident())
+        Entity::Full(sl[0].uck_read_ident(), sl[2].uck_read_ident())
     }
     #[inline(always)]
     /// Parse a single entity from the given slice

From 30d1be48629ecfdfe4a4a279bccd11a9d40cb4d0 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 2 Apr 2023 00:25:20 -0700
Subject: [PATCH 165/310] Add misc fixes

---
 server/src/engine/core/model/alt.rs   | 10 ++---
 server/src/engine/core/space.rs       |  4 +-
 server/src/engine/data/md_dict.rs     |  7 ++-
 server/src/engine/idx/mtchm/access.rs |  8 ++--
 server/src/engine/idx/mtchm/imp.rs    |  2 +-
 server/src/engine/idx/mtchm/mod.rs    |  2 +-
 server/src/engine/idx/mtchm/patch.rs  |  8 +---
 server/src/engine/idx/stord/iter.rs   |  2 +-
 server/src/engine/idx/stord/mod.rs    |  4 +-
 server/src/engine/idx/tests.rs        |  2 +-
 server/src/engine/macros.rs           |  2 +-
 server/src/engine/mem/tests.rs        | 64 +++++++++------------------
 server/src/engine/ql/ast/mod.rs       | 36 +++++++--------
 server/src/engine/ql/benches.rs       |  4 +-
 server/src/engine/ql/ddl/alt.rs       |  4 +-
 server/src/engine/ql/ddl/drop.rs      |  2 +-
 server/src/engine/ql/ddl/ins.rs       |  2 +-
 server/src/engine/ql/ddl/syn.rs       |  2 +-
 server/src/engine/ql/dml/del.rs       |  2 +-
 server/src/engine/ql/dml/ins.rs       | 15 +++----
 server/src/engine/ql/dml/sel.rs       |  2 +-
 server/src/engine/ql/dml/upd.rs       |  4 +-
 server/src/engine/ql/lex/mod.rs       | 22 +++++----
 server/src/engine/ql/lex/raw.rs       |  2 +-
 server/src/engine/ql/tests.rs         |  5 +--
 server/src/engine/ql/tests/entity.rs  |  4 +-
 server/src/engine/sync/cell.rs        |  2 +-
 server/src/engine/sync/smart.rs       |  2 +-
 server/src/util/compiler.rs           |  6 +++
 server/src/util/mod.rs                |  2 +-
 server/src/util/test_utils.rs         |  2 +-
 31 files changed, 105 insertions(+), 130 deletions(-)

diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index cbae154b..e86f1a4c 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -108,12 +108,10 @@ impl<'a> AlterPlan<'a> {
                     not_pk & exists
                 }) {
                     can_ignore!(AlterAction::Remove(r))
+                } else if not_found {
+                    return Err(DatabaseError::DdlModelAlterFieldNotFound);
                 } else {
-                    if not_found {
-                        return Err(DatabaseError::DdlModelAlterFieldNotFound);
-                    } else {
-                        return Err(DatabaseError::DdlModelAlterProtectedField);
-                    }
+                    return Err(DatabaseError::DdlModelAlterProtectedField);
                 }
             }
             AlterKind::Add(new_fields) => {
@@ -280,7 +278,7 @@ impl ModelView {
                     });
             }
             AlterAction::Remove(remove) => {
-                remove.into_iter().for_each(|field_id| {
+                remove.iter().for_each(|field_id| {
                     iwm.fields_mut().st_delete(field_id.as_str());
                 });
             }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 2507b951..10ba679a 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -187,7 +187,7 @@ impl PartialEq for SpaceMeta {
     fn eq(&self, other: &Self) -> bool {
         let x = self.env.read();
         let y = other.env.read();
-        &*x == &*y
+        *x == *y
     }
 }
 
@@ -196,6 +196,6 @@ impl PartialEq for Space {
     fn eq(&self, other: &Self) -> bool {
         let self_mns = self.mns.read();
         let other_mns = other.mns.read();
-        self.meta == other.meta && &*self_mns == &*other_mns
+        self.meta == other.meta && *self_mns == *other_mns
     }
 }
diff --git a/server/src/engine/data/md_dict.rs b/server/src/engine/data/md_dict.rs
index a6f98a9d..dcff3aff 100644
--- a/server/src/engine/data/md_dict.rs
+++ b/server/src/engine/data/md_dict.rs
@@ -78,8 +78,8 @@ pub fn rflatten_metadata(new: DictGeneric) -> MetaDict {
 
 fn _rflatten_metadata(new: DictGeneric, empty: &mut MetaDict) {
     for (key, val) in new {
-        match val {
-            Some(v) => match v {
+        if let Some(v) = val {
+            match v {
                 DictEntryGeneric::Lit(l) => {
                     empty.insert(key, MetaDictEntry::Data(l));
                 }
@@ -88,8 +88,7 @@ fn _rflatten_metadata(new: DictGeneric, empty: &mut MetaDict) {
                     _rflatten_metadata(m, &mut rnew);
                     empty.insert(key, MetaDictEntry::Map(rnew));
                 }
-            },
-            _ => {}
+            }
         }
     }
 }
diff --git a/server/src/engine/idx/mtchm/access.rs b/server/src/engine/idx/mtchm/access.rs
index df6b2833..13f20310 100644
--- a/server/src/engine/idx/mtchm/access.rs
+++ b/server/src/engine/idx/mtchm/access.rs
@@ -56,9 +56,9 @@ impl<'re, T: TreeElement, U: Comparable<T::Key> + ?Sized> ReadMode<T> for RModeE
     type Ret<'a> = bool;
     type Target = U;
     fn target(&self) -> &Self::Target {
-        &self.target
+        self.target
     }
-    fn ex<'a>(_: &'a T) -> Self::Ret<'a> {
+    fn ex(_: &T) -> Self::Ret<'_> {
         true
     }
     fn nx<'a>() -> Self::Ret<'a> {
@@ -84,9 +84,9 @@ impl<'re, T: TreeElement, U: Comparable<T::Key> + ?Sized> ReadMode<T> for RModeR
     type Ret<'a> = Option<&'a T::Value>;
     type Target = U;
     fn target(&self) -> &Self::Target {
-        &self.target
+        self.target
     }
-    fn ex<'a>(c: &'a T) -> Self::Ret<'a> {
+    fn ex(c: &T) -> Self::Ret<'_> {
         Some(c.val())
     }
     fn nx<'a>() -> Self::Ret<'a> {
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index 7c88f69a..6098bd5b 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -288,7 +288,7 @@ impl<T: TreeElement, C: Config> FromIterator<T> for RawTree<T, C> {
         };
         let t = RawTree::new();
         iter.into_iter()
-            .for_each(|te| assert!(t.patch(VanillaInsert(te), &g)));
+            .for_each(|te| assert!(t.patch(VanillaInsert(te), g)));
         t
     }
 }
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index ae99bfab..6bb3673f 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -655,7 +655,7 @@ impl<T, C: Config> RawTree<T, C> {
 
         let mut i = 0;
         while i < C::BRANCH_MX {
-            let ref child_ref = branch.branch[i];
+            let child_ref = &branch.branch[i];
             let this_child = child_ref.fetch_or(NodeFlag::PENDING_DELETE.d(), ORD_ACR, g);
             let this_child = this_child.with_tag(cf(ldfl(&this_child), NodeFlag::PENDING_DELETE));
             match ldfl(&this_child) {
diff --git a/server/src/engine/idx/mtchm/patch.rs b/server/src/engine/idx/mtchm/patch.rs
index ada96d5c..849ddc09 100644
--- a/server/src/engine/idx/mtchm/patch.rs
+++ b/server/src/engine/idx/mtchm/patch.rs
@@ -95,16 +95,12 @@ impl<T: TreeElement> PatchWrite<T> for VanillaUpsert<T> {
     fn nx_new(&mut self) -> T {
         self.0.clone()
     }
-    fn nx_ret<'a>() -> Self::Ret<'a> {
-        ()
-    }
+    fn nx_ret<'a>() -> Self::Ret<'a> {}
     // ex
     fn ex_apply(&mut self, _: &T) -> T {
         self.0.clone()
     }
-    fn ex_ret<'a>(_: &'a T) -> Self::Ret<'a> {
-        ()
-    }
+    fn ex_ret<'a>(_: &'a T) -> Self::Ret<'a> {}
 }
 
 pub struct VanillaUpsertRet<T: TreeElement>(pub T);
diff --git a/server/src/engine/idx/stord/iter.rs b/server/src/engine/idx/stord/iter.rs
index fd887c7e..ea0ea5e2 100644
--- a/server/src/engine/idx/stord/iter.rs
+++ b/server/src/engine/idx/stord/iter.rs
@@ -128,7 +128,7 @@ impl<'a, K, V> Iterator for IndexSTSeqDllIterUnordKey<'a, K, V> {
         self.k.next().map(|k| {
             unsafe {
                 // UNSAFE(@ohsayan): nullck
-                &*(*k).p
+                &*k.p
             }
         })
     }
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index 4cd8a6bb..01b93bc7 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -413,7 +413,7 @@ impl<K: AsKey, V: AsValue, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
                 // UNSAFE(@ohsayan): Impl guarantees that entry presence == nullck head
                 self.__update(*e, v)
             },
-            None => return None,
+            None => None,
         }
     }
     #[inline(always)]
@@ -547,7 +547,7 @@ impl<K, V, C: Config<K, V>> IndexBaseSpec<K, V> for IndexSTSeqDll<K, V, C> {
     }
 
     fn idx_init_with(s: Self) -> Self {
-        Self::from(s)
+        s
     }
 
     fn idx_init_cap(cap: usize) -> Self {
diff --git a/server/src/engine/idx/tests.rs b/server/src/engine/idx/tests.rs
index 658b10e3..6bbee40c 100644
--- a/server/src/engine/idx/tests.rs
+++ b/server/src/engine/idx/tests.rs
@@ -56,7 +56,7 @@ mod idx_st_seq_dll {
     }
 
     #[inline(always)]
-    fn s<'a>(s: &'a str) -> String {
+    fn s(s: &str) -> String {
         s.to_owned()
     }
     fn ranstr(rand: &mut impl Rng) -> String {
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index 4fd251ea..13a29c48 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -79,7 +79,7 @@ macro_rules! flags {
             const LEN: usize = { let mut i = 0; $(let _ = $expr; i += 1;)+{i} };
             const A: [$ty; $group::LEN] = [$($expr,)+];
             const SANITY: () = {
-                let ref a = Self::A; let l = a.len(); let mut i = 0;
+                let a = &Self::A; let l = a.len(); let mut i = 0;
                 while i < l { let mut j = i + 1; while j < l { if a[i] == a[j] { panic!("conflict"); } j += 1; } i += 1; }
             };
             const ALL: $ty = { let mut r: $ty = 0; $( r |= $expr;)+ r };
diff --git a/server/src/engine/mem/tests.rs b/server/src/engine/mem/tests.rs
index ddfef1ec..60221b86 100644
--- a/server/src/engine/mem/tests.rs
+++ b/server/src/engine/mem/tests.rs
@@ -45,7 +45,7 @@ mod vinline {
         F: Clone + FnMut(usize) -> T,
     {
         let map2 = map.clone();
-        let r: VInline<CAP, _> = (0..upto).into_iter().map(map).collect();
+        let r: VInline<CAP, _> = (0..upto).map(map).collect();
         assert_eq!(r.len(), upto);
         if upto <= CAP {
             assert_eq!(r.capacity(), CAP);
@@ -53,11 +53,7 @@ mod vinline {
         } else {
             assert!(r.on_heap());
         }
-        assert!((0..upto)
-            .into_iter()
-            .map(map2)
-            .zip(r.iter())
-            .all(|(x, y)| { x == *y }));
+        assert!((0..upto).map(map2).zip(r.iter()).all(|(x, y)| { x == *y }));
         r
     }
     fn mkvi(upto: usize) -> VInline<CAP, usize> {
@@ -103,7 +99,7 @@ mod vinline {
     fn optimize_capacity_none_on_heap() {
         let mut vi = mkvi(CAP + 1);
         assert_eq!(vi.capacity(), CAP * 2);
-        vi.extend((CAP + 1..CAP * 2).into_iter());
+        vi.extend(CAP + 1..CAP * 2);
         assert_eq!(vi.capacity(), CAP * 2);
         vi.optimize_capacity();
         assert_eq!(vi.capacity(), CAP * 2);
@@ -155,8 +151,7 @@ mod vinline {
         let v1 = mkvi_str(CAP);
         let v: Vec<String> = v1.into_iter().collect();
         (0..CAP)
-            .into_iter()
-            .zip(v.into_iter())
+            .zip(v)
             .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
     #[test]
@@ -164,8 +159,7 @@ mod vinline {
         let v1 = mkvi_str(CAP);
         let v: Vec<String> = v1.into_iter().take(CAP / 2).collect();
         (0..CAP / 2)
-            .into_iter()
-            .zip(v.into_iter())
+            .zip(v)
             .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
     #[test]
@@ -173,8 +167,7 @@ mod vinline {
         let v1 = mkvi_str(CAP + 2);
         let v: Vec<String> = v1.into_iter().collect();
         (0..CAP)
-            .into_iter()
-            .zip(v.into_iter())
+            .zip(v)
             .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
     #[test]
@@ -182,8 +175,7 @@ mod vinline {
         let v1 = mkvi_str(CAP + 2);
         let v: Vec<String> = v1.into_iter().take(CAP / 2).collect();
         (0..CAP / 2)
-            .into_iter()
-            .zip(v.into_iter())
+            .zip(v)
             .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
     #[test]
@@ -192,17 +184,15 @@ mod vinline {
         let v: Vec<String> = v1.into_iter().rev().collect();
         (0..CAP)
             .rev()
-            .into_iter()
-            .zip(v.into_iter())
+            .zip(v)
             .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
     #[test]
     fn into_iter_rev_stack_partial() {
         let v1 = mkvi_str(CAP);
         let v: Vec<String> = v1.into_iter().rev().take(CAP / 2).collect();
-        (CAP..CAP / 2)
+        (CAP / 2..CAP)
             .rev()
-            .into_iter()
             .zip(v.into_iter())
             .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
@@ -212,19 +202,16 @@ mod vinline {
         let v: Vec<String> = v1.into_iter().rev().collect();
         (0..CAP + 2)
             .rev()
-            .into_iter()
-            .zip(v.into_iter())
+            .zip(v)
             .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
     #[test]
     fn into_iter_rev_heap_partial() {
         let v1 = mkvi_str(CAP + 2);
         let v: Vec<String> = v1.into_iter().rev().take(CAP / 2).collect();
-        (CAP..CAP / 2)
-            .rev()
-            .into_iter()
-            .zip(v.into_iter())
-            .for_each(|(x, y)| assert_eq!(x.to_string(), y));
+        (0..CAP + 2).rev().zip(v).for_each(|(x, y)| {
+            assert_eq!(x.to_string(), y);
+        })
     }
 }
 
@@ -260,12 +247,12 @@ mod uarray {
     }
     #[test]
     fn slice() {
-        let a: UArray<CAP, _> = (1u8..=8).into_iter().collect();
+        let a: UArray<CAP, _> = (1u8..=8).collect();
         assert_eq!(a.as_slice(), [1, 2, 3, 4, 5, 6, 7, 8]);
     }
     #[test]
     fn slice_mut() {
-        let mut a: UArray<CAP, _> = (0u8..8).into_iter().collect();
+        let mut a: UArray<CAP, _> = (0u8..8).collect();
         a.iter_mut().for_each(|v| *v += 1);
         assert_eq!(a.as_slice(), [1, 2, 3, 4, 5, 6, 7, 8])
     }
@@ -277,60 +264,53 @@ mod uarray {
     }
     #[test]
     fn into_iter() {
-        let a: UArray<CAP, _> = (0u8..8).into_iter().collect();
+        let a: UArray<CAP, _> = (0u8..8).collect();
         let r: Vec<u8> = a.into_iter().collect();
         (0..8)
-            .into_iter()
             .zip(r.into_iter())
             .for_each(|(x, y)| assert_eq!(x, y));
     }
     #[test]
     fn into_iter_partial() {
-        let a: UArray<CAP, String> = (0u8..8)
-            .into_iter()
-            .map(|v| ToString::to_string(&v))
-            .collect();
+        let a: UArray<CAP, String> = (0u8..8).map(|v| ToString::to_string(&v)).collect();
         let r: Vec<String> = a.into_iter().take(4).collect();
         (0..4)
-            .into_iter()
             .zip(r.into_iter())
             .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
     #[test]
     fn clone() {
-        let a: UArray<CAP, u8> = (0u8..CAP as _).into_iter().collect();
+        let a: UArray<CAP, u8> = (0u8..CAP as _).collect();
         let b = a.clone();
         assert_eq!(a, b);
     }
     #[test]
     fn into_iter_rev() {
-        let a: UArray<CAP, String> = (0u8..8).into_iter().map(|v| v.to_string()).collect();
+        let a: UArray<CAP, String> = (0u8..8).map(|v| v.to_string()).collect();
         let r: Vec<String> = a.into_iter().rev().collect();
         (0..8)
             .rev()
-            .into_iter()
             .zip(r.into_iter())
             .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
     #[test]
     fn into_iter_rev_partial() {
-        let a: UArray<CAP, String> = (0u8..8).into_iter().map(|v| v.to_string()).collect();
+        let a: UArray<CAP, String> = (0u8..8).map(|v| v.to_string()).collect();
         let r: Vec<String> = a.into_iter().rev().take(4).collect();
         (4..8)
             .rev()
-            .into_iter()
             .zip(r.into_iter())
             .for_each(|(x, y)| assert_eq!(x.to_string(), y));
     }
     #[test]
     fn pop_array() {
-        let mut a: UArray<CAP, String> = (0u8..8).into_iter().map(|v| v.to_string()).collect();
+        let mut a: UArray<CAP, String> = (0u8..8).map(|v| v.to_string()).collect();
         assert_eq!(a.pop().unwrap(), "7");
         assert_eq!(a.len(), CAP - 1);
     }
     #[test]
     fn clear_array() {
-        let mut a: UArray<CAP, String> = (0u8..8).into_iter().map(|v| v.to_string()).collect();
+        let mut a: UArray<CAP, String> = (0u8..8).map(|v| v.to_string()).collect();
         a.clear();
         assert!(a.is_empty());
     }
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 8a23a2c0..503e1d1a 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -383,7 +383,7 @@ impl<'a> Entity<'a> {
     ///
     /// Caller guarantees that the token stream matches the exact stream of tokens
     /// expected for a full entity
-    pub(super) unsafe fn full_entity_from_slice(sl: &'a [Token]) -> Self {
+    pub(super) unsafe fn parse_uck_tokens_full(sl: &'a [Token]) -> Self {
         Entity::Full(sl[0].uck_read_ident(), sl[2].uck_read_ident())
     }
     #[inline(always)]
@@ -393,48 +393,48 @@ impl<'a> Entity<'a> {
     ///
     /// Caller guarantees that the token stream matches the exact stream of tokens
     /// expected for a single entity
-    pub(super) unsafe fn single_entity_from_slice(sl: &'a [Token]) -> Self {
+    pub(super) unsafe fn parse_uck_tokens_single(sl: &'a [Token]) -> Self {
         Entity::Single(sl[0].uck_read_ident())
     }
     #[inline(always)]
     /// Returns true if the given token stream matches the signature of single entity syntax
     ///
     /// ⚠ WARNING: This will pass for full and single
-    pub(super) fn tokens_with_single(tok: &[Token]) -> bool {
+    pub(super) fn signature_matches_single_len_checked(tok: &[Token]) -> bool {
         !tok.is_empty() && tok[0].is_ident()
     }
     #[inline(always)]
     /// Returns true if the given token stream matches the signature of full entity syntax
-    pub(super) fn tokens_with_full(tok: &[Token]) -> bool {
+    pub(super) fn signature_matches_full_len_checked(tok: &[Token]) -> bool {
         tok.len() > 2 && tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident()
     }
     #[inline(always)]
     /// Attempt to parse an entity using the given token stream. It also accepts a counter
     /// argument to forward the cursor
-    pub fn parse_from_tokens(tok: &'a [Token], c: &mut usize) -> LangResult<Self> {
-        let is_current = Self::tokens_with_single(tok);
-        let is_full = Self::tokens_with_full(tok);
+    pub fn parse_from_tokens_len_checked(tok: &'a [Token], c: &mut usize) -> LangResult<Self> {
+        let is_current = Self::signature_matches_single_len_checked(tok);
+        let is_full = Self::signature_matches_full_len_checked(tok);
         let r = match () {
             _ if is_full => unsafe {
                 // UNSAFE(@ohsayan): just verified signature
                 *c += 3;
-                Self::full_entity_from_slice(tok)
+                Self::parse_uck_tokens_full(tok)
             },
             _ if is_current => unsafe {
                 // UNSAFE(@ohsayan): just verified signature
                 *c += 1;
-                Self::single_entity_from_slice(tok)
+                Self::parse_uck_tokens_single(tok)
             },
             _ => return Err(LangError::ExpectedEntity),
         };
         Ok(r)
     }
     #[inline(always)]
-    pub fn attempt_process_entity_result<Qd: QueryData<'a>>(
+    pub fn parse_from_state_rounded_result<Qd: QueryData<'a>>(
         state: &mut State<'a, Qd>,
     ) -> LangResult<Self> {
         let mut e = MaybeInit::uninit();
-        Self::attempt_process_entity(state, &mut e);
+        Self::parse_from_state_rounded(state, &mut e);
         if compiler::likely(state.okay()) {
             unsafe {
                 // UNSAFE(@ohsayan): just checked if okay
@@ -445,7 +445,7 @@ impl<'a> Entity<'a> {
         }
     }
     #[inline(always)]
-    pub fn attempt_process_entity<Qd: QueryData<'a>>(
+    pub fn parse_from_state_rounded<Qd: QueryData<'a>>(
         state: &mut State<'a, Qd>,
         d: &mut MaybeInit<Entity<'a>>,
     ) {
@@ -456,15 +456,15 @@ impl<'a> Entity<'a> {
             // UNSAFE(@ohsayan): verified signatures
             if is_full {
                 state.cursor_ahead_by(3);
-                *d = MaybeInit::new(Entity::full_entity_from_slice(tok));
+                *d = MaybeInit::new(Entity::parse_uck_tokens_full(tok));
             } else if is_single {
                 state.cursor_ahead();
-                *d = MaybeInit::new(Entity::single_entity_from_slice(tok));
+                *d = MaybeInit::new(Entity::parse_uck_tokens_single(tok));
             }
         }
         state.poison_if_not(is_full | is_single);
     }
-    pub fn parse_entity<Qd: QueryData<'a>>(
+    pub fn parse_from_state_len_unchecked<Qd: QueryData<'a>>(
         state: &mut State<'a, Qd>,
         d: &mut MaybeInit<Entity<'a>>,
     ) {
@@ -475,10 +475,10 @@ impl<'a> Entity<'a> {
             // UNSAFE(@ohsayan): verified signatures
             if is_full {
                 state.cursor_ahead_by(3);
-                *d = MaybeInit::new(Entity::full_entity_from_slice(tok));
+                *d = MaybeInit::new(Entity::parse_uck_tokens_full(tok));
             } else if is_single {
                 state.cursor_ahead();
-                *d = MaybeInit::new(Entity::single_entity_from_slice(tok));
+                *d = MaybeInit::new(Entity::parse_uck_tokens_single(tok));
             }
         }
         state.poison_if_not(is_full | is_single);
@@ -540,7 +540,7 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> LangResult
     let mut state = State::new(tok, d);
     match state.fw_read() {
         // DDL
-        Token![use] => Entity::attempt_process_entity_result(&mut state).map(Statement::Use),
+        Token![use] => Entity::parse_from_state_rounded_result(&mut state).map(Statement::Use),
         Token![create] => match state.fw_read() {
             Token![model] => ASTNode::from_state(&mut state).map(Statement::CreateModel),
             Token![space] => ASTNode::from_state(&mut state).map(Statement::CreateSpace),
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index 89b76565..848c4d7d 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -95,7 +95,7 @@ mod ast {
         b.iter(|| {
             let src = lex_insecure(b"user").unwrap();
             let mut state = State::new(&src, InplaceData::new());
-            let re = Entity::attempt_process_entity_result(&mut state).unwrap();
+            let re = Entity::parse_from_state_rounded_result(&mut state).unwrap();
             assert_eq!(e, re);
             assert!(state.exhausted());
         })
@@ -106,7 +106,7 @@ mod ast {
         b.iter(|| {
             let src = lex_insecure(b"tweeter.user").unwrap();
             let mut state = State::new(&src, InplaceData::new());
-            let re = Entity::attempt_process_entity_result(&mut state).unwrap();
+            let re = Entity::parse_from_state_rounded_result(&mut state).unwrap();
             assert_eq!(e, re);
             assert!(state.exhausted());
         })
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 1b5e59b8..2b492ebd 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -155,14 +155,14 @@ impl<'a> AlterKind<'a> {
         }
 
         let r = match state.fw_read() {
-            Token::Ident(id) => Box::new([id.clone()]),
+            Token::Ident(id) => Box::new([*id]),
             Token![() open] => {
                 let mut stop = false;
                 let mut cols = Vec::with_capacity(DEFAULT_REMOVE_COL_CNT);
                 while state.loop_tt() && !stop {
                     match state.fw_read() {
                         Token::Ident(ref ident) => {
-                            cols.push(ident.clone());
+                            cols.push(*ident);
                             let nx_comma = state.cursor_rounded_eq(Token![,]);
                             let nx_close = state.cursor_rounded_eq(Token![() close]);
                             state.poison_if_not(nx_comma | nx_close);
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index cbf2605d..bba4ee95 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -78,7 +78,7 @@ impl<'a> DropModel<'a> {
         Self { entity, force }
     }
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-        let e = Entity::attempt_process_entity_result(state)?;
+        let e = Entity::parse_from_state_rounded_result(state)?;
         let force = state.cursor_rounded_eq(Token::Ident(Ident::from("force")));
         state.cursor_ahead_if(force);
         if state.exhausted() {
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
index 3dc8da88..4552d34e 100644
--- a/server/src/engine/ql/ddl/ins.rs
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -51,7 +51,7 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
     }
 
     match state.fw_read() {
-        Token![model] => Entity::attempt_process_entity_result(state).map(Statement::InspectModel),
+        Token![model] => Entity::parse_from_state_rounded_result(state).map(Statement::InspectModel),
         Token![space] if state.cursor_has_ident_rounded() => {
             Ok(Statement::InspectSpace(unsafe {
                 // UNSAFE(@ohsayan): Safe because of the match predicate
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 75917ff6..6c019be9 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -375,7 +375,7 @@ impl<'a> FieldSpec<'a> {
         rfold_layers(state, &mut layers);
         if state.okay() {
             Ok(FieldSpec {
-                field_name: field_name.clone(),
+                field_name: *field_name,
                 layers,
                 null: is_null,
                 primary: is_primary,
diff --git a/server/src/engine/ql/dml/del.rs b/server/src/engine/ql/dml/del.rs
index 48f73778..00e3dbec 100644
--- a/server/src/engine/ql/dml/del.rs
+++ b/server/src/engine/ql/dml/del.rs
@@ -75,7 +75,7 @@ impl<'a> DeleteStatement<'a> {
         state.poison_if_not(state.cursor_eq(Token![from]));
         state.cursor_ahead(); // ignore errors (if any)
         let mut entity = MaybeInit::uninit();
-        Entity::parse_entity(state, &mut entity);
+        Entity::parse_from_state_len_unchecked(state, &mut entity);
         // where + clauses
         state.poison_if_not(state.cursor_eq(Token![where]));
         state.cursor_ahead(); // ignore errors
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index fab38226..8aa30097 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -100,7 +100,7 @@ fn hashf(key: &[u8], m: &[u8]) -> u32 {
     let mut i = 0;
     let mut s = 0;
     while i < key.len() {
-        s += m[(i % MAGIC_L) as usize] as u32 * key[i] as u32;
+        s += m[i % MAGIC_L] as u32 * key[i] as u32;
         i += 1;
     }
     s % PRODUCER_G.len() as u32
@@ -115,7 +115,7 @@ fn ldfunc(func: Ident<'_>) -> Option<ProducerFn> {
     let func = func.as_bytes();
     let ph = hashp(func) as usize;
     let min = cmp::min(ph, PRODUCER_F.len() - 1);
-    let data = PRODUCER_F[min as usize];
+    let data = PRODUCER_F[min];
     if data.0 == func {
         Some(data.1)
     } else {
@@ -129,8 +129,8 @@ fn ldfunc_exists(func: Ident<'_>) -> bool {
 #[inline(always)]
 unsafe fn ldfunc_unchecked(func: &[u8]) -> ProducerFn {
     let ph = hashp(func) as usize;
-    debug_assert_eq!(PRODUCER_F[ph as usize].0, func);
-    PRODUCER_F[ph as usize].1
+    debug_assert_eq!(PRODUCER_F[ph].0, func);
+    PRODUCER_F[ph].1
 }
 
 /// ## Panics
@@ -146,11 +146,10 @@ pub(super) fn parse_list<'a, Qd: QueryData<'a>>(
     while state.not_exhausted() && state.okay() && !stop {
         let d = match state.fw_read() {
             tok if state.can_read_lit_from(tok) => {
-                let r = unsafe {
+                unsafe {
                     // UNSAFE(@ohsayan): the if guard guarantees correctness
                     state.read_lit_into_data_type_unchecked_from(tok)
-                };
-                r
+                }
             }
             Token![open []] => {
                 // a nested list
@@ -351,7 +350,7 @@ impl<'a> InsertStatement<'a> {
 
         // entity
         let mut entity = MaybeInit::uninit();
-        Entity::parse_entity(state, &mut entity);
+        Entity::parse_from_state_len_unchecked(state, &mut entity);
         let mut data = None;
         match state.fw_read() {
             Token![() open] if state.not_exhausted() => {
diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs
index 729d148a..416d3d4c 100644
--- a/server/src/engine/ql/dml/sel.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -113,7 +113,7 @@ impl<'a> SelectStatement<'a> {
         state.poison_if_not(state.cursor_eq(Token![from]));
         state.cursor_ahead(); // ignore errors
         let mut entity = MaybeInit::uninit();
-        Entity::attempt_process_entity(state, &mut entity);
+        Entity::parse_from_state_rounded(state, &mut entity);
         let mut clauses = <_ as Default>::default();
         if state.cursor_rounded_eq(Token![where]) {
             state.cursor_ahead();
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index 8459fb13..22428725 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -96,7 +96,7 @@ impl<'a> AssignmentExpression<'a> {
         }
         let lhs = state.fw_read();
         state.poison_if_not(lhs.is_ident());
-        let op_ass = u(state.cursor_eq(Token![=])) * 1;
+        let op_ass = u(state.cursor_eq(Token![=]));
         let op_add = u(state.cursor_eq(Token![+])) * 2;
         let op_sub = u(state.cursor_eq(Token![-])) * 3;
         let op_mul = u(state.cursor_eq(Token![*])) * 4;
@@ -175,7 +175,7 @@ impl<'a> UpdateStatement<'a> {
         }
         // parse entity
         let mut entity = MaybeInit::uninit();
-        Entity::parse_entity(state, &mut entity);
+        Entity::parse_from_state_len_unchecked(state, &mut entity);
         if !(state.has_remaining(6)) {
             unsafe {
                 // UNSAFE(@ohsayan): Obvious from above, max 3 fw
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index e0346a51..b376631f 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -75,7 +75,7 @@ impl<'a> InsecureLexer<'a> {
     }
     #[inline(always)]
     fn _lex(&mut self) {
-        let ref mut slf = self.base;
+        let slf = &mut self.base;
         while slf.not_exhausted() && slf.no_error() {
             match unsafe {
                 // UNSAFE(@ohsayan): Verified non-null from pre
@@ -133,7 +133,7 @@ impl<'a> InsecureLexer<'a> {
                     slf.push_token(Lit::SignedInt(num));
                 }
                 _ => {
-                    compiler::cold_val(slf.set_error(LexError::InvalidSignedNumericLit));
+                    compiler::cold_call(|| slf.set_error(LexError::InvalidSignedNumericLit));
                 }
             }
         } else {
@@ -399,15 +399,14 @@ where
     let mut number = N::zero();
     let mut nx_stop = false;
 
-    let is_signed;
-    if N::ALLOW_SIGNED {
+    let is_signed = if N::ALLOW_SIGNED {
         let loc_s = i < l && src[i] == b'-';
         i += loc_s as usize;
         okay &= (i + 2) <= l; // [-][digit][LF]
-        is_signed = loc_s;
+        loc_s
     } else {
-        is_signed = false;
-    }
+        false
+    };
 
     while i < l && okay && !nx_stop {
         // potential exit
@@ -416,16 +415,15 @@ where
         let mut local_ok = src[i].is_ascii_digit();
         let (p, p_of) = number.mul_of(10);
         local_ok &= !p_of;
-        let lfret;
-        if N::ALLOW_SIGNED && is_signed {
+        let lfret = if N::ALLOW_SIGNED && is_signed {
             let (d, d_of) = p.sub_of(src[i] & 0x0f);
             local_ok &= !d_of;
-            lfret = d;
+            d
         } else {
             let (s, s_of) = p.add_of(src[i] & 0x0f);
             local_ok &= !s_of;
-            lfret = s;
-        }
+            s
+        };
         // reassign or assign
         let reassign = number.b(nx_stop);
         let assign = lfret.b(!nx_stop);
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index 4787eb34..a40b511e 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -329,7 +329,7 @@ fn kwhf(k: &[u8], mg: &[u8]) -> u32 {
     let mut i = 0;
     let mut s = 0;
     while i < k.len() {
-        s += mg[(i % KWMG_S) as usize] as u32 * k[i] as u32;
+        s += mg[i % KWMG_S] as u32 * k[i] as u32;
         i += 1;
     }
     s % KWG.len() as u32
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index e4421bda..18bc3348 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -41,7 +41,7 @@ mod structure_syn;
 
 #[inline(always)]
 /// Uses the [`InsecureLexer`] to lex the given input
-pub fn lex_insecure<'a>(src: &'a [u8]) -> LexResult<Vec<Token<'a>>> {
+pub fn lex_insecure(src: &[u8]) -> LexResult<Vec<Token<'_>>> {
     InsecureLexer::lex(src)
 }
 #[inline(always)]
@@ -132,8 +132,7 @@ fn fuzz_tokens(src: &[u8], fuzzverify: impl Fn(bool, &[Token]) -> bool) {
                 "expected failure for `{}`, but it passed",
                 new_src
                     .iter()
-                    .map(|tok| format!("{} ", tok.to_string()).into_bytes())
-                    .flatten()
+                    .flat_map(|tok| format!("{} ", tok.to_string()).into_bytes())
                     .map(char::from)
                     .collect::<String>()
             )
diff --git a/server/src/engine/ql/tests/entity.rs b/server/src/engine/ql/tests/entity.rs
index 74019455..7999d36e 100644
--- a/server/src/engine/ql/tests/entity.rs
+++ b/server/src/engine/ql/tests/entity.rs
@@ -29,13 +29,13 @@ use crate::engine::ql::{ast::Entity, lex::Ident};
 #[test]
 fn entity_current() {
     let t = lex_insecure(b"hello").unwrap();
-    let r = Entity::parse_from_tokens(&t, &mut 0).unwrap();
+    let r = Entity::parse_from_tokens_len_checked(&t, &mut 0).unwrap();
     assert_eq!(r, Entity::Single(Ident::from("hello")))
 }
 
 #[test]
 fn entity_full() {
     let t = lex_insecure(b"hello.world").unwrap();
-    let r = Entity::parse_from_tokens(&t, &mut 0).unwrap();
+    let r = Entity::parse_from_tokens_len_checked(&t, &mut 0).unwrap();
     assert_eq!(r, Entity::Full(Ident::from("hello"), Ident::from("world")))
 }
diff --git a/server/src/engine/sync/cell.rs b/server/src/engine/sync/cell.rs
index d34e1d65..21d722fa 100644
--- a/server/src/engine/sync/cell.rs
+++ b/server/src/engine/sync/cell.rs
@@ -68,7 +68,7 @@ impl<T> Drop for TMCell<T> {
         unsafe {
             // UNSAFE(@ohsayan): Sole owner with mutable access
             let g = upin();
-            let shptr = self.a.ld_rlx(&g);
+            let shptr = self.a.ld_rlx(g);
             g.defer_destroy(shptr);
         }
     }
diff --git a/server/src/engine/sync/smart.rs b/server/src/engine/sync/smart.rs
index 7302cc58..d872e985 100644
--- a/server/src/engine/sync/smart.rs
+++ b/server/src/engine/sync/smart.rs
@@ -42,7 +42,7 @@ use {
 
 pub type BytesRC = SliceRC<u8>;
 
-#[derive(Debug, Clone, Hash)]
+#[derive(Debug, Clone)]
 pub struct StrRC {
     base: SliceRC<u8>,
 }
diff --git a/server/src/util/compiler.rs b/server/src/util/compiler.rs
index c865dfbf..a2b85dd0 100644
--- a/server/src/util/compiler.rs
+++ b/server/src/util/compiler.rs
@@ -47,6 +47,12 @@ pub const fn unlikely(b: bool) -> bool {
     b
 }
 
+#[cold]
+#[inline(never)]
+pub fn cold_call<U>(mut v: impl FnMut() -> U) -> U {
+    v()
+}
+
 #[cold]
 #[inline(never)]
 pub const fn cold_val<T>(v: T) -> T {
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index b76b5c45..42d8deec 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -319,7 +319,7 @@ impl<T: fmt::Debug> fmt::Debug for MaybeInit<T> {
         let dat_fmt = if self.is_init {
             unsafe { format!("{:?}", self.base.assume_init_ref()) }
         } else {
-            format!("MaybeUninit {{...}}")
+            "MaybeUninit {..}".to_string()
         };
         f.debug_struct("MaybeInit")
             .field("is_init", &self.is_init)
diff --git a/server/src/util/test_utils.rs b/server/src/util/test_utils.rs
index 192dfa34..48128db5 100644
--- a/server/src/util/test_utils.rs
+++ b/server/src/util/test_utils.rs
@@ -82,7 +82,7 @@ impl<T> VecFuse<T> for Vec<T> {
 macro_rules! vecfuse {
     ($($expr:expr),* $(,)?) => {{
         let mut v = Vec::new();
-        $(<_ as crate::util::test_utils::VecFuse<_>>::fuse_append($expr, &mut v);)*
+        $(<_ as $crate::util::test_utils::VecFuse<_>>::fuse_append($expr, &mut v);)*
         v
     }};
 }

From 45ef72e4003764aaf2c04fc319c8f2030cfd6f6d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 2 Apr 2023 06:59:56 -0700
Subject: [PATCH 166/310] Require full entity path in `alter model`

---
 server/src/engine/core/model/alt.rs        | 12 +++++---
 server/src/engine/core/tests/model/alt.rs  | 36 +++++++++++-----------
 server/src/engine/error.rs                 |  2 ++
 server/src/engine/ql/ast/mod.rs            | 16 +++++++++-
 server/src/engine/ql/ddl/alt.rs            | 11 +++----
 server/src/engine/ql/tests/schema_tests.rs | 30 +++++++++---------
 6 files changed, 62 insertions(+), 45 deletions(-)

diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index e86f1a4c..be1d0e55 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -36,6 +36,7 @@ use {
             error::{DatabaseError, DatabaseResult},
             idx::{IndexST, IndexSTSeqCns, STIndex},
             ql::{
+                ast::Entity,
                 ddl::{
                     alt::{AlterKind, AlterModel},
                     syn::{ExpandedField, LayerSpec},
@@ -50,7 +51,7 @@ use {
 
 #[derive(Debug, PartialEq)]
 pub(in crate::engine::core) struct AlterPlan<'a> {
-    pub(in crate::engine::core) model: Ident<'a>,
+    pub(in crate::engine::core) model: Entity<'a>,
     pub(in crate::engine::core) no_lock: bool,
     pub(in crate::engine::core) action: AlterAction<'a>,
 }
@@ -246,13 +247,16 @@ impl<'a> AlterPlan<'a> {
 }
 
 impl ModelView {
-    pub fn exec_alter(gns: &GlobalNS, space: &[u8], alter: AlterModel) -> DatabaseResult<()> {
+    pub fn exec_alter(gns: &GlobalNS, alter: AlterModel) -> DatabaseResult<()> {
+        let Some((space, model)) = alter.model.into_full() else {
+            return Err(DatabaseError::ExpectedEntity);
+        };
         let gns = gns.spaces().read();
-        let Some(space) = gns.st_get(space) else {
+        let Some(space) = gns.st_get(space.as_bytes()) else {
             return Err(DatabaseError::DdlSpaceNotFound)
         };
         let space = space.models().read();
-        let Some(model) = space.st_get(alter.model.as_bytes()) else {
+        let Some(model) = space.st_get(model.as_bytes()) else {
             return Err(DatabaseError::DdlModelNotFound);
         };
         // make intent
diff --git a/server/src/engine/core/tests/model/alt.rs b/server/src/engine/core/tests/model/alt.rs
index 8ec4df65..480fb875 100644
--- a/server/src/engine/core/tests/model/alt.rs
+++ b/server/src/engine/core/tests/model/alt.rs
@@ -56,8 +56,8 @@ fn exec_plan(
     exec_create(gns, model, "myspace", new_space)?;
     let tok = lex_insecure(plan.as_bytes()).unwrap();
     let alter = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
-    let model_name = alter.model;
-    ModelView::exec_alter(gns, "myspace".as_bytes(), alter)?;
+    let (_space, model_name) = alter.model.into_full().unwrap();
+    ModelView::exec_alter(gns, alter)?;
     let gns_read = gns.spaces().read();
     let space = gns_read.st_get("myspace".as_bytes()).unwrap();
     let model = space.models().read();
@@ -80,9 +80,9 @@ mod plan {
     fn simple_add() {
         super::plan(
             "create model mymodel(username: string, password: binary)",
-            "alter model mymodel add myfield { type: string, nullable: true }",
+            "alter model myspace.mymodel add myfield { type: string, nullable: true }",
             |plan| {
-                assert_eq!(plan.model.as_str(), "mymodel");
+                assert_eq!(plan.model.into_full().unwrap().1.as_str(), "mymodel");
                 assert!(plan.no_lock);
                 assert_eq!(
                     plan.action,
@@ -97,9 +97,9 @@ mod plan {
     fn simple_remove() {
         super::plan(
             "create model mymodel(username: string, password: binary, useless_field: uint8)",
-            "alter model mymodel remove useless_field",
+            "alter model myspace.mymodel remove useless_field",
             |plan| {
-                assert_eq!(plan.model.as_str(), "mymodel");
+                assert_eq!(plan.model.into_full().unwrap().1.as_str(), "mymodel");
                 assert!(plan.no_lock);
                 assert_eq!(
                     plan.action,
@@ -113,9 +113,9 @@ mod plan {
         // FREEDOM! DAMN THE PASSWORD!
         super::plan(
             "create model mymodel(username: string, password: binary)",
-            "alter model mymodel update password { nullable: true }",
+            "alter model myspace.mymodel update password { nullable: true }",
             |plan| {
-                assert_eq!(plan.model.as_str(), "mymodel");
+                assert_eq!(plan.model.into_full().unwrap().1.as_str(), "mymodel");
                 assert!(plan.no_lock);
                 assert_eq!(
                     plan.action,
@@ -131,9 +131,9 @@ mod plan {
         // FIGHT THE NULL
         super::plan(
             "create model mymodel(username: string, null password: binary)",
-            "alter model mymodel update password { nullable: false }",
+            "alter model myspace.mymodel update password { nullable: false }",
             |plan| {
-                assert_eq!(plan.model.as_str(), "mymodel");
+                assert_eq!(plan.model.into_full().unwrap().1.as_str(), "mymodel");
                 assert!(!plan.no_lock);
                 assert_eq!(
                     plan.action,
@@ -152,7 +152,7 @@ mod plan {
         assert_eq!(
             super::with_plan(
                 "create model mymodel(username: string, password: binary)",
-                "alter model mymodel remove password_e2e",
+                "alter model myspace.mymodel remove password_e2e",
                 |_| {}
             )
             .unwrap_err(),
@@ -164,7 +164,7 @@ mod plan {
         assert_eq!(
             super::with_plan(
                 "create model mymodel(username: string, password: binary)",
-                "alter model mymodel remove username",
+                "alter model myspace.mymodel remove username",
                 |_| {}
             )
             .unwrap_err(),
@@ -176,7 +176,7 @@ mod plan {
         assert_eq!(
             super::with_plan(
                 "create model mymodel(username: string, password: binary)",
-                "alter model mymodel add username { type: string }",
+                "alter model myspace.mymodel add username { type: string }",
                 |_| {}
             )
             .unwrap_err(),
@@ -188,7 +188,7 @@ mod plan {
         assert_eq!(
             super::with_plan(
                 "create model mymodel(username: string, password: binary)",
-                "alter model mymodel add password { type: string }",
+                "alter model myspace.mymodel add password { type: string }",
                 |_| {}
             )
             .unwrap_err(),
@@ -200,7 +200,7 @@ mod plan {
         assert_eq!(
             super::with_plan(
                 "create model mymodel(username: string, password: binary)",
-                "alter model mymodel update username { type: string }",
+                "alter model myspace.mymodel update username { type: string }",
                 |_| {}
             )
             .unwrap_err(),
@@ -212,7 +212,7 @@ mod plan {
         assert_eq!(
             super::with_plan(
                 "create model mymodel(username: string, password: binary)",
-                "alter model mymodel update username_secret { type: string }",
+                "alter model myspace.mymodel update username_secret { type: string }",
                 |_| {}
             )
             .unwrap_err(),
@@ -221,7 +221,7 @@ mod plan {
     }
     fn bad_type_cast(orig_ty: &str, new_ty: &str) {
         let create = format!("create model mymodel(username: string, silly_field: {orig_ty})");
-        let alter = format!("alter model mymodel update silly_field {{ type: {new_ty} }}");
+        let alter = format!("alter model myspace.mymodel update silly_field {{ type: {new_ty} }}");
         assert_eq!(
             super::with_plan(&create, &alter, |_| {}).expect_err(&format!(
                 "found no error in transformation: {orig_ty} -> {new_ty}"
@@ -350,7 +350,7 @@ mod exec {
             &gns,
             true,
             "create model mymodel(username: string, password: binary)",
-            "alter model mymodel update password { nullable: true }",
+            "alter model myspace.mymodel update password { nullable: true }",
             |model| {
                 let schema = model.intent_read_model();
                 assert!(schema.fields().st_get("password").unwrap().is_nullable());
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 901590d4..e7858e62 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -88,6 +88,8 @@ pub enum DatabaseError {
     // query generic
     /// this needs an explicit lock
     NeedLock,
+    /// expected a full entity, but found a single implicit entity
+    ExpectedEntity,
     // ddl: create space
     /// unknown property or bad type for property
     DdlSpaceBadProperty,
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 503e1d1a..53b3eaf8 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -354,7 +354,7 @@ impl<'a> QueryData<'a> for SubstitutedData<'a> {
     AST
 */
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone, Copy)]
 /// An [`Entity`] represents the location for a specific structure, such as a model
 pub enum Entity<'a> {
     /// A single entity is used when switching to a model wrt the currently set space (commonly used
@@ -376,6 +376,20 @@ pub enum Entity<'a> {
 }
 
 impl<'a> Entity<'a> {
+    pub fn into_full(self) -> Option<(Ident<'a>, Ident<'a>)> {
+        if let Self::Full(a, b) = self {
+            Some((a, b))
+        } else {
+            None
+        }
+    }
+    pub fn into_single(self) -> Option<Ident<'a>> {
+        if let Self::Single(a) = self {
+            Some(a)
+        } else {
+            None
+        }
+    }
     #[inline(always)]
     /// Parse a full entity from the given slice
     ///
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 2b492ebd..1d2d571c 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -31,7 +31,7 @@ use {
             data::DictGeneric,
             error::{LangError, LangResult},
             ql::{
-                ast::{QueryData, State},
+                ast::{Entity, QueryData, State},
                 lex::{Ident, Token},
             },
         },
@@ -89,13 +89,13 @@ impl<'a> AlterSpace<'a> {
 
 #[derive(Debug, PartialEq)]
 pub struct AlterModel<'a> {
-    pub(in crate::engine) model: Ident<'a>,
+    pub(in crate::engine) model: Entity<'a>,
     pub(in crate::engine) kind: AlterKind<'a>,
 }
 
 impl<'a> AlterModel<'a> {
     #[inline(always)]
-    pub fn new(model: Ident<'a>, kind: AlterKind<'a>) -> Self {
+    pub fn new(model: Entity<'a>, kind: AlterKind<'a>) -> Self {
         Self { model, kind }
     }
 }
@@ -117,10 +117,7 @@ impl<'a> AlterModel<'a> {
             return compiler::cold_rerr(LangError::BadSyntax);
             // FIXME(@ohsayan): bad because no specificity
         }
-        let model_name = unsafe {
-            // UNSAFE(@ohsayan): did rounded check for ident in the above branch
-            state.fw_read().uck_read_ident()
-        };
+        let model_name = Entity::parse_from_state_rounded_result(state)?;
         let kind = match state.fw_read() {
             Token![add] => AlterKind::alter_add(state),
             Token![remove] => AlterKind::alter_remove(state),
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index c9b793b9..ca541c41 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -765,7 +765,7 @@ mod dict_field_syntax {
 mod alter_model_remove {
     use super::*;
     use crate::engine::ql::{
-        ast::parse_ast_node_full,
+        ast::{parse_ast_node_full, Entity},
         ddl::alt::{AlterKind, AlterModel},
         lex::Ident,
     };
@@ -776,7 +776,7 @@ mod alter_model_remove {
         assert_eq!(
             remove,
             AlterModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 AlterKind::Remove(Box::from([Ident::from("myfield")]))
             )
         );
@@ -788,7 +788,7 @@ mod alter_model_remove {
         assert_eq!(
             remove,
             AlterModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 AlterKind::Remove(Box::from([Ident::from("myfield")]))
             )
         );
@@ -802,7 +802,7 @@ mod alter_model_remove {
         assert_eq!(
             remove,
             AlterModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 AlterKind::Remove(Box::from([
                     Ident::from("myfield1"),
                     Ident::from("myfield2"),
@@ -816,7 +816,7 @@ mod alter_model_remove {
 mod alter_model_add {
     use super::*;
     use crate::engine::ql::{
-        ast::parse_ast_node_full,
+        ast::{parse_ast_node_full, Entity},
         ddl::{
             alt::{AlterKind, AlterModel},
             syn::{ExpandedField, LayerSpec},
@@ -833,7 +833,7 @@ mod alter_model_add {
         assert_eq!(
             parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap(),
             AlterModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 AlterKind::Add(
                     [ExpandedField::new(
                         Ident::from("myfield"),
@@ -857,7 +857,7 @@ mod alter_model_add {
         assert_eq!(
             r,
             AlterModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 AlterKind::Add(
                     [ExpandedField::new(
                         Ident::from("myfield"),
@@ -883,7 +883,7 @@ mod alter_model_add {
         assert_eq!(
             r,
             AlterModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 AlterKind::Add(
                     [ExpandedField::new(
                         Ident::from("myfield"),
@@ -923,7 +923,7 @@ mod alter_model_add {
         assert_eq!(
             r,
             AlterModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 AlterKind::Add(
                     [
                         ExpandedField::new(
@@ -964,7 +964,7 @@ mod alter_model_add {
 mod alter_model_update {
     use super::*;
     use crate::engine::ql::{
-        ast::parse_ast_node_full,
+        ast::{parse_ast_node_full, Entity},
         ddl::{
             alt::{AlterKind, AlterModel},
             syn::{ExpandedField, LayerSpec},
@@ -983,7 +983,7 @@ mod alter_model_update {
         assert_eq!(
             r,
             AlterModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 AlterKind::Update(
                     [ExpandedField::new(
                         Ident::from("myfield"),
@@ -1007,7 +1007,7 @@ mod alter_model_update {
         assert_eq!(
             r,
             AlterModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 AlterKind::Update(
                     [ExpandedField::new(
                         Ident::from("myfield"),
@@ -1036,7 +1036,7 @@ mod alter_model_update {
         assert_eq!(
             r,
             AlterModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 AlterKind::Update(
                     [ExpandedField::new(
                         Ident::from("myfield"),
@@ -1070,7 +1070,7 @@ mod alter_model_update {
         assert_eq!(
             r,
             AlterModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 AlterKind::Update(
                     [
                         ExpandedField::new(
@@ -1113,7 +1113,7 @@ mod alter_model_update {
         assert_eq!(
             r,
             AlterModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 AlterKind::Update(
                     [
                         ExpandedField::new(

From fcc187901df95af1fc60a55e63e673d7c5f81cbd Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 2 Apr 2023 08:05:54 -0700
Subject: [PATCH 167/310] Require full entity in `create model`

---
 server/src/engine/core/model/mod.rs        | 16 ++++++-------
 server/src/engine/core/tests/model/alt.rs  | 27 +++++++++++-----------
 server/src/engine/core/tests/model/crt.rs  |  3 +--
 server/src/engine/core/tests/model/mod.rs  | 27 ++++++++--------------
 server/src/engine/ql/ast/mod.rs            |  6 +++++
 server/src/engine/ql/ddl/crt.rs            | 16 ++++++-------
 server/src/engine/ql/tests/schema_tests.rs | 10 ++++----
 7 files changed, 51 insertions(+), 54 deletions(-)

diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 868356ed..5aecd4f8 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -115,7 +115,8 @@ impl ModelView {
             props,
         }: CreateModel,
     ) -> DatabaseResult<Self> {
-        let mut okay = props.is_empty() & !fields.is_empty() & ItemID::check(&model_name);
+        let mut okay =
+            props.is_empty() & !fields.is_empty() & ItemID::check(model_name.any_single().as_str());
         // validate fields
         let mut field_spec = fields.into_iter();
         let mut fields = IndexSTSeqCns::with_capacity(field_spec.len());
@@ -154,18 +155,17 @@ impl ModelView {
 }
 
 impl ModelView {
-    pub fn exec_create(
-        gns: &super::GlobalNS,
-        space: &[u8],
-        stmt: CreateModel,
-    ) -> DatabaseResult<()> {
+    pub fn exec_create(gns: &super::GlobalNS, stmt: CreateModel) -> DatabaseResult<()> {
         let name = stmt.model_name;
         let model = Self::process_create(stmt)?;
         let space_rl = gns.spaces().read();
-        let Some(space) = space_rl.get(space) else {
+        let Some((space_name, model_name)) = name.into_full() else {
+            return Err(DatabaseError::ExpectedEntity);
+        };
+        let Some(space) = space_rl.get(space_name.as_bytes()) else {
             return Err(DatabaseError::DdlSpaceNotFound)
         };
-        space._create_model(ItemID::new(&name), model)
+        space._create_model(ItemID::new(model_name.as_str()), model)
     }
 }
 
diff --git a/server/src/engine/core/tests/model/alt.rs b/server/src/engine/core/tests/model/alt.rs
index 480fb875..482a1eb2 100644
--- a/server/src/engine/core/tests/model/alt.rs
+++ b/server/src/engine/core/tests/model/alt.rs
@@ -53,7 +53,7 @@ fn exec_plan(
     plan: &str,
     f: impl Fn(&ModelView),
 ) -> DatabaseResult<()> {
-    exec_create(gns, model, "myspace", new_space)?;
+    exec_create(gns, model, new_space)?;
     let tok = lex_insecure(plan.as_bytes()).unwrap();
     let alter = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
     let (_space, model_name) = alter.model.into_full().unwrap();
@@ -79,7 +79,7 @@ mod plan {
     #[test]
     fn simple_add() {
         super::plan(
-            "create model mymodel(username: string, password: binary)",
+            "create model myspace.mymodel(username: string, password: binary)",
             "alter model myspace.mymodel add myfield { type: string, nullable: true }",
             |plan| {
                 assert_eq!(plan.model.into_full().unwrap().1.as_str(), "mymodel");
@@ -96,7 +96,7 @@ mod plan {
     #[test]
     fn simple_remove() {
         super::plan(
-            "create model mymodel(username: string, password: binary, useless_field: uint8)",
+            "create model myspace.mymodel(username: string, password: binary, useless_field: uint8)",
             "alter model myspace.mymodel remove useless_field",
             |plan| {
                 assert_eq!(plan.model.into_full().unwrap().1.as_str(), "mymodel");
@@ -112,7 +112,7 @@ mod plan {
     fn simple_update() {
         // FREEDOM! DAMN THE PASSWORD!
         super::plan(
-            "create model mymodel(username: string, password: binary)",
+            "create model myspace.mymodel(username: string, password: binary)",
             "alter model myspace.mymodel update password { nullable: true }",
             |plan| {
                 assert_eq!(plan.model.into_full().unwrap().1.as_str(), "mymodel");
@@ -130,7 +130,7 @@ mod plan {
     fn update_need_lock() {
         // FIGHT THE NULL
         super::plan(
-            "create model mymodel(username: string, null password: binary)",
+            "create model myspace.mymodel(username: string, null password: binary)",
             "alter model myspace.mymodel update password { nullable: false }",
             |plan| {
                 assert_eq!(plan.model.into_full().unwrap().1.as_str(), "mymodel");
@@ -151,7 +151,7 @@ mod plan {
     fn illegal_remove_nx() {
         assert_eq!(
             super::with_plan(
-                "create model mymodel(username: string, password: binary)",
+                "create model myspace.mymodel(username: string, password: binary)",
                 "alter model myspace.mymodel remove password_e2e",
                 |_| {}
             )
@@ -163,7 +163,7 @@ mod plan {
     fn illegal_remove_pk() {
         assert_eq!(
             super::with_plan(
-                "create model mymodel(username: string, password: binary)",
+                "create model myspace.mymodel(username: string, password: binary)",
                 "alter model myspace.mymodel remove username",
                 |_| {}
             )
@@ -175,7 +175,7 @@ mod plan {
     fn illegal_add_pk() {
         assert_eq!(
             super::with_plan(
-                "create model mymodel(username: string, password: binary)",
+                "create model myspace.mymodel(username: string, password: binary)",
                 "alter model myspace.mymodel add username { type: string }",
                 |_| {}
             )
@@ -187,7 +187,7 @@ mod plan {
     fn illegal_add_ex() {
         assert_eq!(
             super::with_plan(
-                "create model mymodel(username: string, password: binary)",
+                "create model myspace.mymodel(username: string, password: binary)",
                 "alter model myspace.mymodel add password { type: string }",
                 |_| {}
             )
@@ -199,7 +199,7 @@ mod plan {
     fn illegal_update_pk() {
         assert_eq!(
             super::with_plan(
-                "create model mymodel(username: string, password: binary)",
+                "create model myspace.mymodel(username: string, password: binary)",
                 "alter model myspace.mymodel update username { type: string }",
                 |_| {}
             )
@@ -211,7 +211,7 @@ mod plan {
     fn illegal_update_nx() {
         assert_eq!(
             super::with_plan(
-                "create model mymodel(username: string, password: binary)",
+                "create model myspace.mymodel(username: string, password: binary)",
                 "alter model myspace.mymodel update username_secret { type: string }",
                 |_| {}
             )
@@ -220,7 +220,8 @@ mod plan {
         );
     }
     fn bad_type_cast(orig_ty: &str, new_ty: &str) {
-        let create = format!("create model mymodel(username: string, silly_field: {orig_ty})");
+        let create =
+            format!("create model myspace.mymodel(username: string, silly_field: {orig_ty})");
         let alter = format!("alter model myspace.mymodel update silly_field {{ type: {new_ty} }}");
         assert_eq!(
             super::with_plan(&create, &alter, |_| {}).expect_err(&format!(
@@ -349,7 +350,7 @@ mod exec {
         super::exec_plan(
             &gns,
             true,
-            "create model mymodel(username: string, password: binary)",
+            "create model myspace.mymodel(username: string, password: binary)",
             "alter model myspace.mymodel update password { nullable: true }",
             |model| {
                 let schema = model.intent_read_model();
diff --git a/server/src/engine/core/tests/model/crt.rs b/server/src/engine/core/tests/model/crt.rs
index 921113de..a069a63e 100644
--- a/server/src/engine/core/tests/model/crt.rs
+++ b/server/src/engine/core/tests/model/crt.rs
@@ -140,8 +140,7 @@ mod exec {
         let gns = GlobalNS::empty();
         exec_create_new_space(
             &gns,
-            "create model mymodel(username: string, password: binary)",
-            SPACE,
+            "create model myspace.mymodel(username: string, password: binary)",
         )
         .unwrap();
         with_model(&gns, SPACE, "mymodel", |model| {
diff --git a/server/src/engine/core/tests/model/mod.rs b/server/src/engine/core/tests/model/mod.rs
index 2875b60e..0b9c313e 100644
--- a/server/src/engine/core/tests/model/mod.rs
+++ b/server/src/engine/core/tests/model/mod.rs
@@ -32,7 +32,7 @@ use crate::engine::{
     core::{model::ModelView, space::Space, GlobalNS},
     error::DatabaseResult,
     idx::STIndex,
-    ql::{ast::parse_ast_node_full, tests::lex_insecure},
+    ql::{ast::parse_ast_node_full, ddl::crt::CreateModel, tests::lex_insecure},
 };
 
 fn create(s: &str) -> DatabaseResult<ModelView> {
@@ -44,31 +44,22 @@ fn create(s: &str) -> DatabaseResult<ModelView> {
 pub fn exec_create(
     gns: &GlobalNS,
     create_stmt: &str,
-    space_id: &str,
     create_new_space: bool,
 ) -> DatabaseResult<()> {
+    let tok = lex_insecure(create_stmt.as_bytes()).unwrap();
+    let create_model = parse_ast_node_full::<CreateModel>(&tok[2..]).unwrap();
     if create_new_space {
-        assert!(gns.test_new_empty_space(space_id));
+        gns.test_new_empty_space(&create_model.model_name.into_full().unwrap().0);
     }
-    let tok = lex_insecure(create_stmt.as_bytes()).unwrap();
-    let create_model = parse_ast_node_full(&tok[2..]).unwrap();
-    ModelView::exec_create(gns, space_id.as_bytes(), create_model)
+    ModelView::exec_create(gns, create_model)
 }
 
-pub fn exec_create_new_space(
-    gns: &GlobalNS,
-    create_stmt: &str,
-    space_id: &str,
-) -> DatabaseResult<()> {
-    exec_create(gns, create_stmt, space_id, true)
+pub fn exec_create_new_space(gns: &GlobalNS, create_stmt: &str) -> DatabaseResult<()> {
+    exec_create(gns, create_stmt, true)
 }
 
-pub fn exec_create_no_create(
-    gns: &GlobalNS,
-    create_stmt: &str,
-    space_id: &str,
-) -> DatabaseResult<()> {
-    exec_create(gns, create_stmt, space_id, false)
+pub fn exec_create_no_create(gns: &GlobalNS, create_stmt: &str) -> DatabaseResult<()> {
+    exec_create(gns, create_stmt, false)
 }
 
 fn with_space(gns: &GlobalNS, space_name: &str, f: impl Fn(&Space)) {
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 53b3eaf8..3e24a2e1 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -376,6 +376,12 @@ pub enum Entity<'a> {
 }
 
 impl<'a> Entity<'a> {
+    pub fn any_single(self) -> Ident<'a> {
+        match self {
+            Self::Full(_, e) => e,
+            Self::Single(e) => e,
+        }
+    }
     pub fn into_full(self) -> Option<(Ident<'a>, Ident<'a>)> {
         if let Self::Full(a, b) = self {
             Some((a, b))
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index 3cb36c50..ede26fa1 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -31,11 +31,11 @@ use {
             data::DictGeneric,
             error::{LangError, LangResult},
             ql::{
-                ast::{QueryData, State},
+                ast::{Entity, QueryData, State},
                 lex::Ident,
             },
         },
-        util::compiler,
+        util::{compiler, MaybeInit},
     },
 };
 
@@ -85,7 +85,7 @@ impl<'a> CreateSpace<'a> {
 /// A model definition
 pub struct CreateModel<'a> {
     /// the model name
-    pub(in crate::engine) model_name: Ident<'a>,
+    pub(in crate::engine) model_name: Entity<'a>,
     /// the fields
     pub(in crate::engine) fields: Vec<FieldSpec<'a>>,
     /// properties
@@ -100,7 +100,7 @@ pub struct CreateModel<'a> {
 */
 
 impl<'a> CreateModel<'a> {
-    pub fn new(model_name: Ident<'a>, fields: Vec<FieldSpec<'a>>, props: DictGeneric) -> Self {
+    pub fn new(model_name: Entity<'a>, fields: Vec<FieldSpec<'a>>, props: DictGeneric) -> Self {
         Self {
             model_name,
             fields,
@@ -113,8 +113,8 @@ impl<'a> CreateModel<'a> {
             return compiler::cold_rerr(LangError::UnexpectedEOS);
         }
         // model name; ignore errors
-        let model_name = state.fw_read();
-        state.poison_if_not(model_name.is_ident());
+        let mut model_uninit = MaybeInit::uninit();
+        Entity::parse_from_state_len_unchecked(state, &mut model_uninit);
         state.poison_if_not(state.cursor_eq(Token![() open]));
         state.cursor_ahead();
         // fields
@@ -140,8 +140,8 @@ impl<'a> CreateModel<'a> {
         if state.okay() {
             Ok(Self {
                 model_name: unsafe {
-                    // UNSAFE(@ohsayan): we verified if `model_name` returns `is_ident`
-                    model_name.uck_read_ident()
+                    // UNSAFE(@ohsayan): we verified if `model_name` is initialized through the state
+                    model_uninit.assume_init()
                 },
                 fields,
                 props,
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index ca541c41..32f26f1a 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -414,7 +414,7 @@ mod fields {
 mod schemas {
     use super::*;
     use crate::engine::ql::{
-        ast::parse_ast_node_full,
+        ast::{parse_ast_node_full, Entity},
         ddl::{
             crt::CreateModel,
             syn::{FieldSpec, LayerSpec},
@@ -439,7 +439,7 @@ mod schemas {
         assert_eq!(
             model,
             CreateModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 vec![
                     FieldSpec::new(
                         Ident::from("username"),
@@ -478,7 +478,7 @@ mod schemas {
         assert_eq!(
             model,
             CreateModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 vec![
                     FieldSpec::new(
                         Ident::from("username"),
@@ -528,7 +528,7 @@ mod schemas {
         assert_eq!(
             model,
             CreateModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 vec![
                     FieldSpec::new(
                         Ident::from("username"),
@@ -597,7 +597,7 @@ mod schemas {
         assert_eq!(
             model,
             CreateModel::new(
-                Ident::from("mymodel"),
+                Entity::Single(Ident::from("mymodel")),
                 vec![
                     FieldSpec::new(
                         Ident::from("username"),

From 1586b05bbd72a3896113f3613717549452242fa9 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 2 Apr 2023 08:31:24 -0700
Subject: [PATCH 168/310] Add `drop model` exec without advanced params

---
 server/src/engine/core/model/mod.rs       | 20 ++++++++++++++++++++
 server/src/engine/core/tests/model/alt.rs | 19 +++++++++++++++++--
 server/src/engine/error.rs                |  2 ++
 3 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 5aecd4f8..d30daa6e 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -42,6 +42,7 @@ use {
         mem::VInline,
         ql::ddl::{
             crt::CreateModel,
+            drop::DropModel,
             syn::{FieldSpec, LayerSpec},
         },
     },
@@ -105,6 +106,10 @@ impl ModelView {
             Ok(())
         }
     }
+    pub fn is_empty_atomic(&self) -> bool {
+        // TODO(@ohsayan): change this!
+        true
+    }
 }
 
 impl ModelView {
@@ -167,6 +172,21 @@ impl ModelView {
         };
         space._create_model(ItemID::new(model_name.as_str()), model)
     }
+    pub fn exec_drop(gns: &super::GlobalNS, stmt: DropModel) -> DatabaseResult<()> {
+        let Some((space, model)) = stmt.entity.into_full() else {
+            return Err(DatabaseError::ExpectedEntity);
+        };
+        let spaces = gns.spaces().read();
+        let Some(space) =  spaces.st_get(space.as_bytes()) else {
+            return Err(DatabaseError::DdlSpaceNotFound);
+        };
+        let mut w_space = space.models().write();
+        match w_space.st_delete_if(model.as_bytes(), |mdl| !mdl.is_empty_atomic()) {
+            Some(true) => Ok(()),
+            Some(false) => Err(DatabaseError::DdlModelViewNotEmpty),
+            None => Err(DatabaseError::DdlModelNotFound),
+        }
+    }
 }
 
 /*
diff --git a/server/src/engine/core/tests/model/alt.rs b/server/src/engine/core/tests/model/alt.rs
index 482a1eb2..efac130b 100644
--- a/server/src/engine/core/tests/model/alt.rs
+++ b/server/src/engine/core/tests/model/alt.rs
@@ -343,9 +343,9 @@ mod plan {
 }
 
 mod exec {
-    use crate::engine::{core::GlobalNS, idx::STIndex};
+    use crate::engine::{core::GlobalNS, error::DatabaseError, idx::STIndex};
     #[test]
-    fn exec_simple_alter() {
+    fn simple_alter() {
         let gns = GlobalNS::empty();
         super::exec_plan(
             &gns,
@@ -359,4 +359,19 @@ mod exec {
         )
         .unwrap();
     }
+    #[test]
+    fn failing_alter_nullable_switch_need_lock() {
+        let gns = GlobalNS::empty();
+        assert_eq!(
+            super::exec_plan(
+                &gns,
+                true,
+                "create model myspace.mymodel(username: string, null gh_handle: string)",
+                "alter model myspace.mymodel update gh_handle { nullable: false }",
+                |_| {},
+            )
+            .unwrap_err(),
+            DatabaseError::NeedLock
+        );
+    }
 }
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index e7858e62..0c4ffb81 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -117,4 +117,6 @@ pub enum DatabaseError {
     DdlModelAlterBadTypedef,
     /// didn't find the model
     DdlModelNotFound,
+    /// attempted a remove, but the model view is nonempty
+    DdlModelViewNotEmpty,
 }

From f98c5d3aa4225a06f3db8619a80160f08b2a8e2a Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 3 Apr 2023 00:11:59 -0700
Subject: [PATCH 169/310] Reduce memory usage per GNS index item

---
 server/src/engine/core/mod.rs             |  6 +++---
 server/src/engine/core/model/alt.rs       |  4 ++--
 server/src/engine/core/model/mod.rs       | 18 ++++++----------
 server/src/engine/core/space.rs           | 26 +++++++++++++----------
 server/src/engine/core/tests/model/alt.rs |  4 ++--
 server/src/engine/core/tests/model/mod.rs |  4 ++--
 server/src/engine/core/tests/space/mod.rs |  2 +-
 server/src/engine/data/mod.rs             |  4 ----
 server/src/engine/data/tests/mod.rs       | 22 -------------------
 9 files changed, 32 insertions(+), 58 deletions(-)

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 67c535f0..673698bf 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -30,7 +30,7 @@ mod space;
 mod tests;
 // imports
 use {
-    crate::engine::{core::space::Space, data::ItemID, idx::IndexST},
+    crate::engine::{core::space::Space, idx::IndexST},
     parking_lot::RwLock,
 };
 // re-exports
@@ -43,11 +43,11 @@ type RWLIdx<K, V> = RwLock<IndexST<K, V>>;
 // FIXME(@ohsayan): Make sure we update what all structures we're making use of here
 
 pub struct GlobalNS {
-    index_space: RWLIdx<ItemID, Space>,
+    index_space: RWLIdx<Box<str>, Space>,
 }
 
 impl GlobalNS {
-    pub(self) fn spaces(&self) -> &RWLIdx<ItemID, Space> {
+    pub(self) fn spaces(&self) -> &RWLIdx<Box<str>, Space> {
         &self.index_space
     }
     pub fn empty() -> Self {
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index be1d0e55..7ac2060d 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -252,11 +252,11 @@ impl ModelView {
             return Err(DatabaseError::ExpectedEntity);
         };
         let gns = gns.spaces().read();
-        let Some(space) = gns.st_get(space.as_bytes()) else {
+        let Some(space) = gns.st_get(space.as_str()) else {
             return Err(DatabaseError::DdlSpaceNotFound)
         };
         let space = space.models().read();
-        let Some(model) = space.st_get(model.as_bytes()) else {
+        let Some(model) = space.st_get(model.as_str()) else {
             return Err(DatabaseError::DdlModelNotFound);
         };
         // make intent
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index d30daa6e..4edbd7a7 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -33,10 +33,7 @@ use std::cell::RefCell;
 use {
     crate::engine::{
         core::model::cell::Datacell,
-        data::{
-            tag::{DataTag, FullTag, TagClass, TagSelector},
-            ItemID,
-        },
+        data::tag::{DataTag, FullTag, TagClass, TagSelector},
         error::{DatabaseError, DatabaseResult},
         idx::{IndexSTSeqCns, STIndex, STIndexSeq},
         mem::VInline,
@@ -115,13 +112,12 @@ impl ModelView {
 impl ModelView {
     pub fn process_create(
         CreateModel {
-            model_name,
+            model_name: _,
             fields,
             props,
         }: CreateModel,
     ) -> DatabaseResult<Self> {
-        let mut okay =
-            props.is_empty() & !fields.is_empty() & ItemID::check(model_name.any_single().as_str());
+        let mut okay = props.is_empty() & !fields.is_empty();
         // validate fields
         let mut field_spec = fields.into_iter();
         let mut fields = IndexSTSeqCns::with_capacity(field_spec.len());
@@ -167,21 +163,21 @@ impl ModelView {
         let Some((space_name, model_name)) = name.into_full() else {
             return Err(DatabaseError::ExpectedEntity);
         };
-        let Some(space) = space_rl.get(space_name.as_bytes()) else {
+        let Some(space) = space_rl.get(space_name.as_str()) else {
             return Err(DatabaseError::DdlSpaceNotFound)
         };
-        space._create_model(ItemID::new(model_name.as_str()), model)
+        space._create_model(model_name.as_str(), model)
     }
     pub fn exec_drop(gns: &super::GlobalNS, stmt: DropModel) -> DatabaseResult<()> {
         let Some((space, model)) = stmt.entity.into_full() else {
             return Err(DatabaseError::ExpectedEntity);
         };
         let spaces = gns.spaces().read();
-        let Some(space) =  spaces.st_get(space.as_bytes()) else {
+        let Some(space) =  spaces.st_get(space.as_str()) else {
             return Err(DatabaseError::DdlSpaceNotFound);
         };
         let mut w_space = space.models().write();
-        match w_space.st_delete_if(model.as_bytes(), |mdl| !mdl.is_empty_atomic()) {
+        match w_space.st_delete_if(model.as_str(), |mdl| !mdl.is_empty_atomic()) {
             Some(true) => Ok(()),
             Some(false) => Err(DatabaseError::DdlModelViewNotEmpty),
             None => Err(DatabaseError::DdlModelNotFound),
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 10ba679a..b953cba6 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -26,7 +26,7 @@
 
 use {
     crate::engine::{
-        core::{model::ModelView, ItemID, RWLIdx},
+        core::{model::ModelView, RWLIdx},
         data::{md_dict, DictEntryGeneric, MetaDict},
         error::{DatabaseError, DatabaseResult},
         idx::{IndexST, STIndex},
@@ -38,7 +38,7 @@ use {
 #[derive(Debug)]
 /// A space with the model namespace
 pub struct Space {
-    mns: RWLIdx<ItemID, ModelView>,
+    mns: RWLIdx<Box<str>, ModelView>,
     pub(super) meta: SpaceMeta,
 }
 
@@ -61,27 +61,31 @@ impl SpaceMeta {
 #[cfg_attr(test, derive(PartialEq))]
 /// Procedure for `create space`
 struct ProcedureCreate {
-    space_name: ItemID,
+    space_name: Box<str>,
     space: Space,
 }
 
 impl ProcedureCreate {
     #[inline(always)]
     /// Define the procedure
-    fn new(space_name: ItemID, space: Space) -> Self {
+    fn new(space_name: Box<str>, space: Space) -> Self {
         Self { space_name, space }
     }
 }
 
 impl Space {
-    pub fn _create_model(&self, name: ItemID, model: ModelView) -> DatabaseResult<()> {
-        if self.mns.write().st_insert(name, model) {
+    pub fn _create_model(&self, name: &str, model: ModelView) -> DatabaseResult<()> {
+        if self
+            .mns
+            .write()
+            .st_insert(name.to_string().into_boxed_str(), model)
+        {
             Ok(())
         } else {
             Err(DatabaseError::DdlModelAlreadyExists)
         }
     }
-    pub(super) fn models(&self) -> &RWLIdx<ItemID, ModelView> {
+    pub(super) fn models(&self) -> &RWLIdx<Box<str>, ModelView> {
         &self.mns
     }
 }
@@ -91,7 +95,7 @@ impl Space {
         Space::new(Default::default(), SpaceMeta::with_env(into_dict! {}))
     }
     #[inline(always)]
-    pub fn new(mns: IndexST<ItemID, ModelView>, meta: SpaceMeta) -> Self {
+    pub fn new(mns: IndexST<Box<str>, ModelView>, meta: SpaceMeta) -> Self {
         Self {
             mns: RWLIdx::new(mns),
             meta,
@@ -105,7 +109,7 @@ impl Space {
             mut props,
         }: CreateSpace,
     ) -> DatabaseResult<ProcedureCreate> {
-        let space_name = ItemID::try_new(&space_name).ok_or(DatabaseError::SysBadItemID)?;
+        let space_name = space_name.to_string().into_boxed_str();
         // check env
         let env = match props.remove(SpaceMeta::KEY_ENV) {
             Some(Some(DictEntryGeneric::Map(m))) if props.is_empty() => m,
@@ -146,7 +150,7 @@ impl Space {
             mut updated_props,
         }: AlterSpace,
     ) -> DatabaseResult<()> {
-        match gns.spaces().read().st_get(space_name.as_bytes()) {
+        match gns.spaces().read().st_get(space_name.as_str()) {
             Some(space) => {
                 let mut space_env = space.meta.env.write();
                 match updated_props.remove(SpaceMeta::KEY_ENV) {
@@ -173,7 +177,7 @@ impl Space {
         match gns
             .spaces()
             .write()
-            .st_delete_if(space.as_bytes(), |space| space.mns.read().len() == 0)
+            .st_delete_if(space.as_str(), |space| space.mns.read().len() == 0)
         {
             Some(true) => Ok(()),
             Some(false) => Err(DatabaseError::DdlSpaceRemoveNonEmpty),
diff --git a/server/src/engine/core/tests/model/alt.rs b/server/src/engine/core/tests/model/alt.rs
index efac130b..8868a2c9 100644
--- a/server/src/engine/core/tests/model/alt.rs
+++ b/server/src/engine/core/tests/model/alt.rs
@@ -59,9 +59,9 @@ fn exec_plan(
     let (_space, model_name) = alter.model.into_full().unwrap();
     ModelView::exec_alter(gns, alter)?;
     let gns_read = gns.spaces().read();
-    let space = gns_read.st_get("myspace".as_bytes()).unwrap();
+    let space = gns_read.st_get("myspace").unwrap();
     let model = space.models().read();
-    f(model.st_get(model_name.as_bytes()).unwrap());
+    f(model.st_get(model_name.as_str()).unwrap());
     Ok(())
 }
 
diff --git a/server/src/engine/core/tests/model/mod.rs b/server/src/engine/core/tests/model/mod.rs
index 0b9c313e..85f31ec8 100644
--- a/server/src/engine/core/tests/model/mod.rs
+++ b/server/src/engine/core/tests/model/mod.rs
@@ -64,14 +64,14 @@ pub fn exec_create_no_create(gns: &GlobalNS, create_stmt: &str) -> DatabaseResul
 
 fn with_space(gns: &GlobalNS, space_name: &str, f: impl Fn(&Space)) {
     let rl = gns.spaces().read();
-    let space = rl.st_get(space_name.as_bytes()).unwrap();
+    let space = rl.st_get(space_name).unwrap();
     f(space);
 }
 
 fn with_model(gns: &GlobalNS, space_id: &str, model_name: &str, f: impl Fn(&ModelView)) {
     with_space(gns, space_id, |space| {
         let space_rl = space.models().read();
-        let model = space_rl.st_get(model_name.as_bytes()).unwrap();
+        let model = space_rl.st_get(model_name).unwrap();
         f(model)
     })
 }
diff --git a/server/src/engine/core/tests/space/mod.rs b/server/src/engine/core/tests/space/mod.rs
index 87c0257e..0ca2f61a 100644
--- a/server/src/engine/core/tests/space/mod.rs
+++ b/server/src/engine/core/tests/space/mod.rs
@@ -47,7 +47,7 @@ fn exec_verify(
     let ast_node = compile_test(&tok).unwrap();
     let (res, space_name) = exec(gns, ast_node);
     let rl = gns.spaces().read();
-    let space_ref = rl.st_get(space_name.as_bytes());
+    let space_ref = rl.st_get(&space_name);
     let r = res.map(|_| space_ref.unwrap());
     verify(r);
 }
diff --git a/server/src/engine/data/mod.rs b/server/src/engine/data/mod.rs
index e42ee2c5..f58ba49a 100644
--- a/server/src/engine/data/mod.rs
+++ b/server/src/engine/data/mod.rs
@@ -33,8 +33,4 @@ pub mod tag;
 #[cfg(test)]
 mod tests;
 
-use crate::engine::mem::AStr;
 pub use md_dict::{DictEntryGeneric, DictGeneric, MetaDict};
-
-const IDENT_MX: usize = 64;
-pub type ItemID = AStr<IDENT_MX>;
diff --git a/server/src/engine/data/tests/mod.rs b/server/src/engine/data/tests/mod.rs
index 430bef15..1eff0f69 100644
--- a/server/src/engine/data/tests/mod.rs
+++ b/server/src/engine/data/tests/mod.rs
@@ -25,25 +25,3 @@
 */
 
 mod md_dict_tests;
-
-use crate::engine::{data::ItemID, ql::lex::Ident};
-
-#[test]
-fn item_id_okay() {
-    let _ = ItemID::from(Ident::from("hello"));
-}
-
-#[test]
-fn test_item_id_exact() {
-    let _ = ItemID::from(Ident::from(
-        "Abe76d912c6e205aa05edf974cd21cd48061d86d12d92ac1028e5b90f3132f4e",
-    ));
-}
-
-#[test]
-#[should_panic(expected = "length overflow")]
-fn item_id_too_long() {
-    let _ = ItemID::from(Ident::from(
-        "Abe76d912c6e205aa05edf974cd21cd48061d86d12d92ac1028e5b90f3132f4e_",
-    ));
-}

From 87adc1046d3d6d4bf30a491facf571d6b58e7217 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 6 Apr 2023 11:02:40 -0700
Subject: [PATCH 170/310] Simplify entity handling

---
 server/src/engine/core/mod.rs       | 30 ++++++++++-
 server/src/engine/core/model/alt.rs | 77 +++++++++++++----------------
 server/src/engine/core/model/mod.rs | 37 ++++++--------
 server/src/engine/core/space.rs     | 36 ++++++++------
 server/src/engine/core/util.rs      | 51 +++++++++++++++++++
 server/src/engine/ql/ast/mod.rs     | 14 ++++++
 6 files changed, 163 insertions(+), 82 deletions(-)
 create mode 100644 server/src/engine/core/util.rs

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 673698bf..df97ce22 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -26,11 +26,18 @@
 
 mod model;
 mod space;
+mod util;
+// test
 #[cfg(test)]
 mod tests;
 // imports
 use {
-    crate::engine::{core::space::Space, idx::IndexST},
+    self::{model::ModelView, util::EntityLocator},
+    crate::engine::{
+        core::space::Space,
+        error::{DatabaseError, DatabaseResult},
+        idx::{IndexST, STIndex},
+    },
     parking_lot::RwLock,
 };
 // re-exports
@@ -57,9 +64,28 @@ impl GlobalNS {
     }
     #[cfg(test)]
     pub(self) fn test_new_empty_space(&self, space_id: &str) -> bool {
-        use super::idx::STIndex;
         self.index_space
             .write()
             .st_insert(space_id.into(), Space::empty())
     }
+    pub fn with_space<T>(
+        &self,
+        space: &str,
+        f: impl FnOnce(&Space) -> DatabaseResult<T>,
+    ) -> DatabaseResult<T> {
+        let sread = self.index_space.read();
+        let Some(space) = sread.st_get(space) else {
+            return Err(DatabaseError::DdlSpaceNotFound);
+        };
+        f(space)
+    }
+    pub fn with_model<'a, T, E, F>(&self, entity: E, f: F) -> DatabaseResult<T>
+    where
+        F: FnOnce(&ModelView) -> DatabaseResult<T>,
+        E: 'a + EntityLocator<'a>,
+    {
+        entity
+            .parse_entity()
+            .and_then(|(space, model)| self.with_space(space, |space| space.with_model(model, f)))
+    }
 }
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index 7ac2060d..16cce2e8 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -248,50 +248,41 @@ impl<'a> AlterPlan<'a> {
 
 impl ModelView {
     pub fn exec_alter(gns: &GlobalNS, alter: AlterModel) -> DatabaseResult<()> {
-        let Some((space, model)) = alter.model.into_full() else {
-            return Err(DatabaseError::ExpectedEntity);
-        };
-        let gns = gns.spaces().read();
-        let Some(space) = gns.st_get(space.as_str()) else {
-            return Err(DatabaseError::DdlSpaceNotFound)
-        };
-        let space = space.models().read();
-        let Some(model) = space.st_get(model.as_str()) else {
-            return Err(DatabaseError::DdlModelNotFound);
-        };
-        // make intent
-        let iwm = model.intent_write_model();
-        // prepare plan
-        let plan = AlterPlan::fdeltas(model, &iwm, alter)?;
-        // we have a legal plan; acquire exclusive if we need it
-        if !plan.no_lock {
-            // TODO(@ohsayan): allow this later on, once we define the syntax
-            return Err(DatabaseError::NeedLock);
-        }
-        // fine, we're good
-        let mut iwm = iwm;
-        match plan.action {
-            AlterAction::Ignore => drop(iwm),
-            AlterAction::Add(new_fields) => {
-                // TODO(@ohsayan): this impacts lockdown duration; fix it
-                new_fields
-                    .st_iter_kv()
-                    .map(|(x, y)| (x.clone(), y.clone()))
-                    .for_each(|(field_id, field)| {
-                        iwm.fields_mut().st_insert(field_id, field);
-                    });
-            }
-            AlterAction::Remove(remove) => {
-                remove.iter().for_each(|field_id| {
-                    iwm.fields_mut().st_delete(field_id.as_str());
-                });
+        gns.with_model(alter.model, |model| {
+            // make intent
+            let iwm = model.intent_write_model();
+            // prepare plan
+            let plan = AlterPlan::fdeltas(model, &iwm, alter)?;
+            // we have a legal plan; acquire exclusive if we need it
+            if !plan.no_lock {
+                // TODO(@ohsayan): allow this later on, once we define the syntax
+                return Err(DatabaseError::NeedLock);
             }
-            AlterAction::Update(u) => {
-                u.into_iter().for_each(|(field_id, field)| {
-                    iwm.fields_mut().st_update(&field_id, field);
-                });
+            // fine, we're good
+            let mut iwm = iwm;
+            match plan.action {
+                AlterAction::Ignore => drop(iwm),
+                AlterAction::Add(new_fields) => {
+                    // TODO(@ohsayan): this impacts lockdown duration; fix it
+                    new_fields
+                        .st_iter_kv()
+                        .map(|(x, y)| (x.clone(), y.clone()))
+                        .for_each(|(field_id, field)| {
+                            iwm.fields_mut().st_insert(field_id, field);
+                        });
+                }
+                AlterAction::Remove(remove) => {
+                    remove.iter().for_each(|field_id| {
+                        iwm.fields_mut().st_delete(field_id.as_str());
+                    });
+                }
+                AlterAction::Update(u) => {
+                    u.into_iter().for_each(|(field_id, field)| {
+                        iwm.fields_mut().st_update(&field_id, field);
+                    });
+                }
             }
-        }
-        Ok(())
+            Ok(())
+        })
     }
 }
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 4edbd7a7..99c4c023 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -30,6 +30,8 @@ pub mod cell;
 #[cfg(test)]
 use std::cell::RefCell;
 
+use super::util::EntityLocator;
+
 use {
     crate::engine::{
         core::model::cell::Datacell,
@@ -47,7 +49,7 @@ use {
     parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard},
 };
 
-type Fields = IndexSTSeqCns<Box<str>, Field>;
+pub(in crate::engine::core) type Fields = IndexSTSeqCns<Box<str>, Field>;
 
 // FIXME(@ohsayan): update this!
 
@@ -157,31 +159,20 @@ impl ModelView {
 
 impl ModelView {
     pub fn exec_create(gns: &super::GlobalNS, stmt: CreateModel) -> DatabaseResult<()> {
-        let name = stmt.model_name;
+        let (space_name, model_name) = stmt.model_name.parse_entity()?;
         let model = Self::process_create(stmt)?;
-        let space_rl = gns.spaces().read();
-        let Some((space_name, model_name)) = name.into_full() else {
-            return Err(DatabaseError::ExpectedEntity);
-        };
-        let Some(space) = space_rl.get(space_name.as_str()) else {
-            return Err(DatabaseError::DdlSpaceNotFound)
-        };
-        space._create_model(model_name.as_str(), model)
+        gns.with_space(space_name, |space| space._create_model(model_name, model))
     }
     pub fn exec_drop(gns: &super::GlobalNS, stmt: DropModel) -> DatabaseResult<()> {
-        let Some((space, model)) = stmt.entity.into_full() else {
-            return Err(DatabaseError::ExpectedEntity);
-        };
-        let spaces = gns.spaces().read();
-        let Some(space) =  spaces.st_get(space.as_str()) else {
-            return Err(DatabaseError::DdlSpaceNotFound);
-        };
-        let mut w_space = space.models().write();
-        match w_space.st_delete_if(model.as_str(), |mdl| !mdl.is_empty_atomic()) {
-            Some(true) => Ok(()),
-            Some(false) => Err(DatabaseError::DdlModelViewNotEmpty),
-            None => Err(DatabaseError::DdlModelNotFound),
-        }
+        let (space, model) = stmt.entity.parse_entity()?;
+        gns.with_space(space, |space| {
+            let mut w_space = space.models().write();
+            match w_space.st_delete_if(model, |mdl| !mdl.is_empty_atomic()) {
+                Some(true) => Ok(()),
+                Some(false) => Err(DatabaseError::DdlModelViewNotEmpty),
+                None => Err(DatabaseError::DdlModelNotFound),
+            }
+        })
     }
 }
 
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index b953cba6..abbc96bf 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -88,6 +88,17 @@ impl Space {
     pub(super) fn models(&self) -> &RWLIdx<Box<str>, ModelView> {
         &self.mns
     }
+    pub fn with_model<T>(
+        &self,
+        model: &str,
+        f: impl FnOnce(&ModelView) -> DatabaseResult<T>,
+    ) -> DatabaseResult<T> {
+        let mread = self.mns.read();
+        let Some(model) = mread.st_get(model) else {
+            return Err(DatabaseError::DdlModelNotFound);
+        };
+        f(model)
+    }
 }
 
 impl Space {
@@ -150,23 +161,20 @@ impl Space {
             mut updated_props,
         }: AlterSpace,
     ) -> DatabaseResult<()> {
-        match gns.spaces().read().st_get(space_name.as_str()) {
-            Some(space) => {
-                let mut space_env = space.meta.env.write();
-                match updated_props.remove(SpaceMeta::KEY_ENV) {
-                    Some(Some(DictEntryGeneric::Map(env))) if updated_props.is_empty() => {
-                        if !md_dict::rmerge_metadata(&mut space_env, env) {
-                            return Err(DatabaseError::DdlSpaceBadProperty);
-                        }
+        gns.with_space(&space_name, |space| {
+            let mut space_env = space.meta.env.write();
+            match updated_props.remove(SpaceMeta::KEY_ENV) {
+                Some(Some(DictEntryGeneric::Map(env))) if updated_props.is_empty() => {
+                    if !md_dict::rmerge_metadata(&mut space_env, env) {
+                        return Err(DatabaseError::DdlSpaceBadProperty);
                     }
-                    Some(None) if updated_props.is_empty() => space_env.clear(),
-                    None => {}
-                    _ => return Err(DatabaseError::DdlSpaceBadProperty),
                 }
-                Ok(())
+                Some(None) if updated_props.is_empty() => space_env.clear(),
+                None => {}
+                _ => return Err(DatabaseError::DdlSpaceBadProperty),
             }
-            None => Err(DatabaseError::DdlSpaceNotFound),
-        }
+            Ok(())
+        })
     }
     pub fn exec_drop(
         gns: &super::GlobalNS,
diff --git a/server/src/engine/core/util.rs b/server/src/engine/core/util.rs
new file mode 100644
index 00000000..6d83edfc
--- /dev/null
+++ b/server/src/engine/core/util.rs
@@ -0,0 +1,51 @@
+/*
+ * Created on Thu Apr 06 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    error::{DatabaseError, DatabaseResult},
+    ql::ast::Entity,
+};
+
+pub trait EntityLocator<'a> {
+    fn parse_entity(self) -> DatabaseResult<(&'a str, &'a str)>
+    where
+        Self: 'a;
+}
+
+impl<'a> EntityLocator<'a> for (&'a str, &'a str) {
+    fn parse_entity(self) -> DatabaseResult<(&'a str, &'a str)> {
+        Ok(self)
+    }
+}
+
+impl<'a> EntityLocator<'a> for Entity<'a> {
+    fn parse_entity(self) -> DatabaseResult<(&'a str, &'a str)>
+    where
+        Self: 'a,
+    {
+        self.into_full_str().ok_or(DatabaseError::ExpectedEntity)
+    }
+}
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 3e24a2e1..296f33a2 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -389,6 +389,13 @@ impl<'a> Entity<'a> {
             None
         }
     }
+    pub fn into_full_str(self) -> Option<(&'a str, &'a str)> {
+        if let Self::Full(a, b) = self {
+            Some((a.as_str(), b.as_str()))
+        } else {
+            None
+        }
+    }
     pub fn into_single(self) -> Option<Ident<'a>> {
         if let Self::Single(a) = self {
             Some(a)
@@ -396,6 +403,13 @@ impl<'a> Entity<'a> {
             None
         }
     }
+    pub fn into_single_str(self) -> Option<&'a str> {
+        if let Self::Single(a) = self {
+            Some(a.as_str())
+        } else {
+            None
+        }
+    }
     #[inline(always)]
     /// Parse a full entity from the given slice
     ///

From 554a4789178a68551fa265b92f98d1bbd48d84b1 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 10 Apr 2023 05:48:52 -0700
Subject: [PATCH 171/310] Add index key

---
 server/src/engine/core/index/key.rs  | 276 +++++++++++++++++++++++++++
 server/src/engine/core/index/mod.rs  |  27 +++
 server/src/engine/core/mod.rs        |   1 +
 server/src/engine/core/model/cell.rs |  73 +++----
 server/src/engine/data/lit.rs        |  38 +++-
 server/src/engine/data/spec.rs       |  29 +--
 server/src/engine/data/tag.rs        |  55 ++++++
 server/src/engine/idx/mod.rs         |   2 +-
 server/src/engine/mem/mod.rs         |   1 +
 server/src/engine/mem/word.rs        |  38 ++--
 server/src/util/test_utils.rs        |  22 ++-
 11 files changed, 482 insertions(+), 80 deletions(-)
 create mode 100644 server/src/engine/core/index/key.rs
 create mode 100644 server/src/engine/core/index/mod.rs

diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
new file mode 100644
index 00000000..212aac4b
--- /dev/null
+++ b/server/src/engine/core/index/key.rs
@@ -0,0 +1,276 @@
+/*
+ * Created on Sun Apr 09 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[cfg(test)]
+use crate::{engine::data::spec::Dataspec1D, util::test_utils};
+use {
+    crate::engine::{
+        core::model::cell::Datacell,
+        data::{
+            lit::LitIR,
+            spec::DataspecMeta1D,
+            tag::{DataTag, TagUnique},
+        },
+        idx::meta::Comparable,
+        mem::{self, NativeDword, SystemDword},
+    },
+    core::{
+        fmt,
+        hash::{Hash, Hasher},
+        mem::ManuallyDrop,
+        slice, str,
+    },
+};
+
+pub struct PrimaryIndexKey {
+    tag: TagUnique,
+    data: NativeDword,
+}
+
+impl PrimaryIndexKey {
+    pub unsafe fn read_uint(&self) -> u64 {
+        self.data.load_qw()
+    }
+    pub fn uint(&self) -> Option<u64> {
+        (self.tag == TagUnique::UnsignedInt).then_some(unsafe {
+            // UNSAFE(@ohsayan): verified tag
+            self.read_uint()
+        })
+    }
+    pub unsafe fn read_sint(&self) -> i64 {
+        self.data.load_qw() as _
+    }
+    pub fn sint(&self) -> Option<i64> {
+        (self.tag == TagUnique::SignedInt).then_some(unsafe {
+            // UNSAFE(@ohsayan): verified tag
+            self.read_sint()
+        })
+    }
+    pub unsafe fn read_bin(&self) -> &[u8] {
+        self.vdata()
+    }
+    pub fn bin(&self) -> Option<&[u8]> {
+        (self.tag == TagUnique::Bin).then(|| unsafe {
+            // UNSAFE(@ohsayan): verified tag
+            self.read_bin()
+        })
+    }
+    pub unsafe fn read_str(&self) -> &str {
+        str::from_utf8_unchecked(self.vdata())
+    }
+    pub fn str(&self) -> Option<&str> {
+        (self.tag == TagUnique::Str).then(|| unsafe {
+            // UNSAFE(@ohsayan): verified tag
+            self.read_str()
+        })
+    }
+}
+
+impl PrimaryIndexKey {
+    pub fn try_from_dc(dc: Datacell) -> Option<Self> {
+        Self::check(&dc).then(|| unsafe { Self::new_from_dc(dc) })
+    }
+    /// ## Safety
+    ///
+    /// Make sure that the [`Datacell`] is an eligible candidate key (ensuring uniqueness constraints + allocation correctness).
+    ///
+    /// If you violate this:
+    /// - You might leak memory
+    /// - You might segfault
+    /// - Even if you escape both, it will produce incorrect results which is something you DO NOT want in an index
+    pub unsafe fn new_from_dc(dc: Datacell) -> Self {
+        debug_assert!(Self::check(&dc));
+        let tag = dc.tag().tag_unique();
+        let dc = ManuallyDrop::new(dc);
+        let dword = unsafe {
+            // UNSAFE(@ohsayan): this doesn't do anything "bad" by itself. needs the construction to be broken for it to do something silly
+            dc.as_raw()
+        }
+        .load_double();
+        Self {
+            tag,
+            data: unsafe {
+                // UNSAFE(@ohsayan): Perfectly safe since we're tranforming it and THIS will not by itself crash anything
+                core::mem::transmute(dword)
+            },
+        }
+    }
+    pub fn check(dc: &Datacell) -> bool {
+        dc.tag().tag_unique().is_unique()
+    }
+    pub fn check_opt(dc: &Option<Datacell>) -> bool {
+        dc.as_ref().map(Self::check).unwrap_or(false)
+    }
+    /// ## Safety
+    /// If you mess up construction, everything will fall apart
+    pub unsafe fn new(tag: TagUnique, data: NativeDword) -> Self {
+        Self { tag, data }
+    }
+    fn __compute_vdata_offset(&self) -> [usize; 2] {
+        let [len, data] = self.data.load_double();
+        let actual_len = len * (self.tag >= TagUnique::Bin) as usize;
+        [data, actual_len]
+    }
+    fn vdata(&self) -> &[u8] {
+        let [data, actual_len] = self.__compute_vdata_offset();
+        unsafe {
+            // UNSAFE(@ohsayan): Safe, due to construction
+            slice::from_raw_parts(data as *const u8, actual_len)
+        }
+    }
+    fn vdata_mut(&mut self) -> &mut [u8] {
+        let [data, actual_len] = self.__compute_vdata_offset();
+        unsafe {
+            // UNSAFE(@ohsayan): safe due to construction
+            slice::from_raw_parts_mut(data as *mut u8, actual_len)
+        }
+    }
+}
+
+impl Drop for PrimaryIndexKey {
+    fn drop(&mut self) {
+        if let TagUnique::Bin | TagUnique::Str = self.tag {
+            unsafe {
+                // UNSAFE(@ohsayan): Aliasing, sole owner and correct initialization
+                let vdata = self.vdata_mut();
+                mem::dealloc_array(vdata.as_mut_ptr(), vdata.len());
+            }
+        }
+    }
+}
+
+impl PartialEq for PrimaryIndexKey {
+    fn eq(&self, other: &Self) -> bool {
+        let [data_1, data_2] = [self.data.load_double()[0], other.data.load_double()[0]];
+        ((self.tag == other.tag) & (data_1 == data_2)) && self.vdata() == other.vdata()
+    }
+}
+
+impl Eq for PrimaryIndexKey {}
+
+impl Hash for PrimaryIndexKey {
+    fn hash<H: Hasher>(&self, hasher: &mut H) {
+        self.tag.hash(hasher);
+        self.vdata().hash(hasher);
+    }
+}
+
+impl<'a> PartialEq<LitIR<'a>> for PrimaryIndexKey {
+    fn eq(&self, key: &LitIR<'a>) -> bool {
+        debug_assert!(key.kind().tag_unique().is_unique());
+        self.tag == key.kind().tag_unique() && self.vdata() == key.__vdata()
+    }
+}
+
+impl<'a> Comparable<LitIR<'a>> for PrimaryIndexKey {
+    fn cmp_eq(&self, key: &LitIR<'a>) -> bool {
+        self == key
+    }
+}
+
+impl fmt::Debug for PrimaryIndexKey {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut dbg_struct = f.debug_struct("PrimaryIndexKey");
+        dbg_struct.field("tag", &self.tag);
+        macro_rules! fmt {
+            ($($mtch:ident => $expr:expr),* $(,)?) => {
+                match self.tag {
+                    $(TagUnique::$mtch => dbg_struct.field("data", &($expr.unwrap())),)*
+                    TagUnique::Illegal => panic!("found illegal value. check ctor."),
+                }
+            };
+        }
+        fmt!(
+            UnsignedInt => self.uint(),
+            SignedInt => self.sint(),
+            Bin => self.bin(),
+            Str => self.str(),
+        );
+        dbg_struct.finish()
+    }
+}
+
+#[test]
+fn check_pk_wrong_type() {
+    let data = [
+        Datacell::from(false),
+        Datacell::from(100),
+        Datacell::from(-100),
+        Datacell::from(10.11),
+        Datacell::from("hello"),
+        Datacell::from("hello".as_bytes()),
+        Datacell::from([]),
+    ];
+    for datum in data {
+        let tag = datum.tag();
+        let candidate = PrimaryIndexKey::try_from_dc(datum);
+        if tag.tag_unique() == TagUnique::Illegal {
+            assert!(candidate.is_none(), "{:?}", &candidate);
+        } else {
+            assert!(candidate.is_some(), "{:?}", &candidate);
+        }
+    }
+}
+
+#[test]
+fn check_pk_eq_hash() {
+    let state = test_utils::randomstate();
+    let data = [
+        Datacell::from(100),
+        Datacell::from(-100),
+        Datacell::from("binary".as_bytes()),
+        Datacell::from("string"),
+    ];
+
+    for datum in data {
+        let pk1 = PrimaryIndexKey::try_from_dc(datum.clone()).unwrap();
+        let pk2 = PrimaryIndexKey::try_from_dc(datum).unwrap();
+        assert_eq!(pk1, pk2);
+        assert_eq!(
+            test_utils::hash_rs(&state, &pk1),
+            test_utils::hash_rs(&state, &pk2)
+        );
+    }
+}
+
+#[test]
+fn check_pk_lit_eq_hash() {
+    let state = test_utils::randomstate();
+    let data = [
+        LitIR::UnsignedInt(100),
+        LitIR::SignedInt(-100),
+        LitIR::Bin(b"binary bro"),
+        LitIR::Str("string bro"),
+    ];
+    for litir in data {
+        let pk = PrimaryIndexKey::try_from_dc(Datacell::from(litir.clone())).unwrap();
+        assert_eq!(pk, litir);
+        assert_eq!(
+            test_utils::hash_rs(&state, &litir),
+            test_utils::hash_rs(&state, &pk)
+        );
+    }
+}
diff --git a/server/src/engine/core/index/mod.rs b/server/src/engine/core/index/mod.rs
new file mode 100644
index 00000000..ad3db743
--- /dev/null
+++ b/server/src/engine/core/index/mod.rs
@@ -0,0 +1,27 @@
+/*
+ * Created on Sat Apr 08 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod key;
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index df97ce22..29d2f965 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -24,6 +24,7 @@
  *
 */
 
+mod index;
 mod model;
 mod space;
 mod util;
diff --git a/server/src/engine/core/model/cell.rs b/server/src/engine/core/model/cell.rs
index 8b42cd69..e4369db5 100644
--- a/server/src/engine/core/model/cell.rs
+++ b/server/src/engine/core/model/cell.rs
@@ -24,25 +24,23 @@
  *
 */
 
-#[cfg(test)]
-use core::mem;
 use {
     crate::engine::{
         self,
         data::{
             lit::{Lit, LitIR},
             spec::{Dataspec1D, DataspecMeta1D},
-            tag::{DataTag, TagClass},
+            tag::{CUTag, DataTag, TagClass},
         },
         mem::{NativeQword, SystemDword, WordRW},
     },
-    core::{fmt, mem::ManuallyDrop, slice, str},
+    core::{fmt, mem, mem::ManuallyDrop, slice, str},
     parking_lot::RwLock,
 };
 
 pub struct Datacell {
     init: bool,
-    tag: TagClass,
+    tag: CUTag,
     data: DataRaw,
 }
 
@@ -51,7 +49,7 @@ impl Datacell {
     pub fn new_bool(b: bool) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(TagClass::Bool, DataRaw::word(SystemDword::store(b)))
+            Self::new(CUTag::BOOL, DataRaw::word(SystemDword::store(b)))
         }
     }
     pub unsafe fn read_bool(&self) -> bool {
@@ -70,7 +68,7 @@ impl Datacell {
     pub fn new_uint(u: u64) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(TagClass::UnsignedInt, DataRaw::word(SystemDword::store(u)))
+            Self::new(CUTag::UINT, DataRaw::word(SystemDword::store(u)))
         }
     }
     pub unsafe fn read_uint(&self) -> u64 {
@@ -89,7 +87,7 @@ impl Datacell {
     pub fn new_sint(u: i64) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(TagClass::SignedInt, DataRaw::word(SystemDword::store(u)))
+            Self::new(CUTag::SINT, DataRaw::word(SystemDword::store(u)))
         }
     }
     pub unsafe fn read_sint(&self) -> i64 {
@@ -108,7 +106,7 @@ impl Datacell {
     pub fn new_float(f: f64) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(TagClass::Float, DataRaw::word(SystemDword::store(f)))
+            Self::new(CUTag::FLOAT, DataRaw::word(SystemDword::store(f)))
         }
     }
     pub unsafe fn read_float(&self) -> f64 {
@@ -129,13 +127,13 @@ impl Datacell {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
-                TagClass::Bin,
-                DataRaw::word(SystemDword::store((md.as_mut_ptr(), md.len()))),
+                CUTag::BIN,
+                DataRaw::word(SystemDword::store((md.len(), md.as_mut_ptr()))),
             )
         }
     }
     pub unsafe fn read_bin(&self) -> &[u8] {
-        let (p, l) = self.load_word();
+        let (l, p) = self.load_word();
         slice::from_raw_parts::<u8>(p, l)
     }
     pub fn try_bin(&self) -> Option<&[u8]> {
@@ -153,13 +151,13 @@ impl Datacell {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
-                TagClass::Str,
-                DataRaw::word(SystemDword::store((md.as_mut_ptr(), md.len()))),
+                CUTag::STR,
+                DataRaw::word(SystemDword::store((md.len(), md.as_mut_ptr()))),
             )
         }
     }
     pub unsafe fn read_str(&self) -> &str {
-        let (p, l) = self.load_word();
+        let (l, p) = self.load_word();
         str::from_utf8_unchecked(slice::from_raw_parts(p, l))
     }
     pub fn try_str(&self) -> Option<&str> {
@@ -175,7 +173,7 @@ impl Datacell {
     pub fn new_list(l: Vec<Self>) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(TagClass::List, DataRaw::rwl(RwLock::new(l)))
+            Self::new(CUTag::LIST, DataRaw::rwl(RwLock::new(l)))
         }
     }
     pub unsafe fn read_list(&self) -> &RwLock<Vec<Self>> {
@@ -215,18 +213,19 @@ impl<'a> From<LitIR<'a>> for Datacell {
         match l.kind().tag_class() {
             tag if tag < TagClass::Bin => unsafe {
                 // UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type doesn't need any advanced construction
+                let [a, b] = l.data().load_double();
                 Datacell::new(
-                    l.kind().tag_class(),
+                    CUTag::from(l.kind()),
                     // DO NOT RELY ON the payload's bit pattern; it's padded
-                    DataRaw::word(SystemDword::store_qw(l.data().load_qw())),
+                    DataRaw::word(SystemDword::store_fat(a, b)),
                 )
             },
-            tag @ (TagClass::Bin | TagClass::Str) => unsafe {
+            TagClass::Bin | TagClass::Str => unsafe {
                 // UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type requires a new heap for construction
                 let mut bin = ManuallyDrop::new(l.read_bin_uck().to_owned().into_boxed_slice());
                 Datacell::new(
-                    tag,
-                    DataRaw::word(SystemDword::store((bin.as_mut_ptr(), bin.len()))),
+                    CUTag::from(l.kind()),
+                    DataRaw::word(SystemDword::store((bin.len(), bin.as_mut_ptr()))),
                 )
             },
             _ => unsafe {
@@ -261,17 +260,16 @@ impl<const N: usize> From<[Datacell; N]> for Datacell {
 }
 
 impl Datacell {
-    pub fn kind(&self) -> TagClass {
+    pub fn tag(&self) -> CUTag {
         self.tag
     }
+    pub fn kind(&self) -> TagClass {
+        self.tag.tag_class()
+    }
     pub fn null() -> Self {
         unsafe {
             // UNSAFE(@ohsayan): This is a hack. It's safe because we set init to false
-            Self::_new(
-                TagClass::Bool,
-                DataRaw::word(NativeQword::store_qw(0)),
-                false,
-            )
+            Self::_new(CUTag::BOOL, DataRaw::word(NativeQword::store_qw(0)), false)
         }
     }
     pub fn is_null(&self) -> bool {
@@ -290,14 +288,17 @@ impl Datacell {
     unsafe fn load_word<'a, T: WordRW<NativeQword, Target<'a> = T>>(&'a self) -> T {
         self.data.word.ld()
     }
-    unsafe fn _new(tag: TagClass, data: DataRaw, init: bool) -> Self {
+    unsafe fn _new(tag: CUTag, data: DataRaw, init: bool) -> Self {
         Self { init, tag, data }
     }
-    unsafe fn new(tag: TagClass, data: DataRaw) -> Self {
+    unsafe fn new(tag: CUTag, data: DataRaw) -> Self {
         Self::_new(tag, data, true)
     }
     fn checked_tag<T>(&self, tag: TagClass, f: impl FnOnce() -> T) -> Option<T> {
-        ((self.tag == tag) & (self.is_init())).then_some(f())
+        ((self.kind() == tag) & (self.is_init())).then(f)
+    }
+    pub unsafe fn as_raw(&self) -> NativeQword {
+        mem::transmute_copy(&self.data.word)
     }
 }
 
@@ -307,7 +308,7 @@ impl fmt::Debug for Datacell {
         f.field("tag", &self.tag);
         macro_rules! fmtdbg {
             ($($match:ident => $ret:expr),* $(,)?) => {
-                match self.tag {
+                match self.kind() {
                     $(TagClass::$match if self.is_init() => { f.field("data", &Some($ret));},)*
                     TagClass::Bool if self.is_null() => {f.field("data", &Option::<u8>::None);},
                     _ => unreachable!("incorrect state"),
@@ -332,7 +333,7 @@ impl PartialEq for Datacell {
         if self.is_null() {
             return other.is_null();
         }
-        match (self.tag, other.tag) {
+        match (self.kind(), other.kind()) {
             (TagClass::Bool, TagClass::Bool) => self.bool() == other.bool(),
             (TagClass::UnsignedInt, TagClass::UnsignedInt) => self.uint() == other.uint(),
             (TagClass::SignedInt, TagClass::SignedInt) => self.sint() == other.sint(),
@@ -375,10 +376,10 @@ impl DataRaw {
 
 impl Drop for Datacell {
     fn drop(&mut self) {
-        match self.tag {
+        match self.kind() {
             TagClass::Str | TagClass::Bin => unsafe {
                 // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class
-                let (p, l) = self.load_word();
+                let (l, p) = self.load_word();
                 engine::mem::dealloc_array::<u8>(p, l)
             },
             TagClass::List => unsafe {
@@ -393,12 +394,12 @@ impl Drop for Datacell {
 #[cfg(test)]
 impl Clone for Datacell {
     fn clone(&self) -> Self {
-        let data = match self.tag {
+        let data = match self.kind() {
             TagClass::Str | TagClass::Bin => unsafe {
                 // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class
                 let mut block = ManuallyDrop::new(self.read_bin().to_owned().into_boxed_slice());
                 DataRaw {
-                    word: ManuallyDrop::new(SystemDword::store((block.as_mut_ptr(), block.len()))),
+                    word: ManuallyDrop::new(SystemDword::store((block.len(), block.as_mut_ptr()))),
                 }
             },
             TagClass::List => unsafe {
diff --git a/server/src/engine/data/lit.rs b/server/src/engine/data/lit.rs
index 2c3d768b..038a8531 100644
--- a/server/src/engine/data/lit.rs
+++ b/server/src/engine/data/lit.rs
@@ -27,13 +27,15 @@
 use {
     super::{
         spec::{Dataspec1D, DataspecMeta1D, DataspecMethods1D, DataspecRaw1D},
-        tag::{DataTag, FullTag},
+        tag::{DataTag, FullTag, TagUnique},
     },
     crate::engine::mem::{SpecialPaddedWord, SystemDword},
     core::{
         fmt,
+        hash::{Hash, Hasher},
         marker::PhantomData,
         mem::{self, ManuallyDrop},
+        slice,
     },
 };
 
@@ -91,16 +93,16 @@ unsafe impl<'a> DataspecRaw1D for Lit<'a> {
     const HEAP_STR: bool = true;
     const HEAP_BIN: bool = false;
     unsafe fn drop_str(&mut self) {
-        let [ptr, len] = self.data().load_fat();
+        let [len, ptr] = self.data().load_double();
         drop(String::from_raw_parts(ptr as *mut u8, len, len));
     }
     unsafe fn drop_bin(&mut self) {}
     unsafe fn clone_str(s: &str) -> Self::Target {
         let new_string = ManuallyDrop::new(s.to_owned().into_boxed_str());
-        SystemDword::store((new_string.as_ptr(), new_string.len()))
+        SystemDword::store((new_string.len(), new_string.as_ptr()))
     }
     unsafe fn clone_bin(b: &[u8]) -> Self::Target {
-        SystemDword::store((b.as_ptr(), b.len()))
+        SystemDword::store((b.len(), b.as_ptr()))
     }
 }
 
@@ -112,7 +114,7 @@ unsafe impl<'a> DataspecRaw1D for Lit<'a> {
 unsafe impl<'a> Dataspec1D for Lit<'a> {
     fn Str(s: Box<str>) -> Self {
         let md = ManuallyDrop::new(s);
-        Self::new(FullTag::STR, SystemDword::store((md.as_ptr(), md.len())))
+        Self::new(FullTag::STR, SystemDword::store((md.len(), md.as_ptr())))
     }
 }
 
@@ -172,12 +174,32 @@ direct_from! {
     LitIR
 */
 
+/// ☢️TRAIT WARNING☢️: The [`Hash`] implementation is strictly intended for usage with [`crate::engine::core`] components ONLY. This will FAIL and PRODUCE INCORRECT results
+/// when used elsewhere
 pub struct LitIR<'a> {
     tag: FullTag,
     data: SpecialPaddedWord,
     _lt: PhantomData<&'a str>,
 }
 
+impl<'a> LitIR<'a> {
+    pub fn __vdata(&self) -> &[u8] {
+        let [vlen, data] = self.data().load_double();
+        let len = vlen * (self.kind().tag_unique() >= TagUnique::Bin) as usize;
+        unsafe {
+            // UNSAFE(@ohsayan): either because of static or lt
+            slice::from_raw_parts(data as *const u8, len)
+        }
+    }
+}
+
+impl<'a> Hash for LitIR<'a> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.tag.tag_unique().hash(state);
+        self.__vdata().hash(state);
+    }
+}
+
 impl<'a> DataspecMeta1D for LitIR<'a> {
     type Target = SpecialPaddedWord;
     type StringItem = &'a str;
@@ -215,10 +237,10 @@ unsafe impl<'a> DataspecRaw1D for LitIR<'a> {
     unsafe fn drop_str(&mut self) {}
     unsafe fn drop_bin(&mut self) {}
     unsafe fn clone_str(s: &str) -> Self::Target {
-        SystemDword::store((s.as_ptr(), s.len()))
+        SystemDword::store((s.len(), s.as_ptr()))
     }
     unsafe fn clone_bin(b: &[u8]) -> Self::Target {
-        SystemDword::store((b.as_ptr(), b.len()))
+        SystemDword::store((b.len(), b.as_ptr()))
     }
 }
 
@@ -228,7 +250,7 @@ unsafe impl<'a> DataspecRaw1D for LitIR<'a> {
 */
 unsafe impl<'a> Dataspec1D for LitIR<'a> {
     fn Str(s: Self::StringItem) -> Self {
-        Self::new(FullTag::STR, SystemDword::store((s.as_ptr(), s.len())))
+        Self::new(FullTag::STR, SystemDword::store((s.len(), s.as_ptr())))
     }
 }
 
diff --git a/server/src/engine/data/spec.rs b/server/src/engine/data/spec.rs
index 19f56909..de241374 100644
--- a/server/src/engine/data/spec.rs
+++ b/server/src/engine/data/spec.rs
@@ -36,8 +36,8 @@ use {
 };
 
 #[inline(always)]
-fn when_then<T>(cond: bool, then: T) -> Option<T> {
-    cond.then_some(then)
+fn when_then<T, F: FnOnce() -> T>(cond: bool, then: F) -> Option<T> {
+    cond.then(then)
 }
 
 /// Information about the type that implements the dataspec traits
@@ -106,7 +106,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     /// Store a new binary. This function is always safe to call
     #[allow(non_snake_case)]
     fn Bin(b: &[u8]) -> Self {
-        Self::new(Self::Tag::BIN, SystemDword::store((b.as_ptr(), b.len())))
+        Self::new(Self::Tag::BIN, SystemDword::store((b.len(), b.as_ptr())))
     }
 
     /// Store a new string. Now, I won't talk about this one's safety because it depends on the implementor
@@ -121,7 +121,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     }
     /// Load a bool
     fn read_bool_try(&self) -> Option<bool> {
-        when_then(self.kind().tag_class() == TagClass::Bool, unsafe {
+        when_then(self.kind().tag_class() == TagClass::Bool, || unsafe {
             // UNSAFE(@ohsayan): we've verified the flag. but lol because this isn't actually unsafe
             self.read_bool_uck()
         })
@@ -139,10 +139,13 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     }
     /// Load a uint
     fn read_uint_try(&self) -> Option<u64> {
-        when_then(self.kind().tag_class() == TagClass::UnsignedInt, unsafe {
-            // UNSAFE(@ohsayan): we've verified the flag. but lol because this isn't actually unsafe
-            self.read_uint_uck()
-        })
+        when_then(
+            self.kind().tag_class() == TagClass::UnsignedInt,
+            || unsafe {
+                // UNSAFE(@ohsayan): we've verified the flag. but lol because this isn't actually unsafe
+                self.read_uint_uck()
+            },
+        )
     }
     /// Load a uint
     /// ## Panics
@@ -157,7 +160,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     }
     /// Load a sint
     fn read_sint_try(&self) -> Option<i64> {
-        when_then(self.kind().tag_class() == TagClass::SignedInt, unsafe {
+        when_then(self.kind().tag_class() == TagClass::SignedInt, || unsafe {
             // UNSAFE(@ohsayan): we've verified the flag. but lol because this isn't actually unsafe
             self.read_sint_uck()
         })
@@ -173,7 +176,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     }
     /// Load a float
     fn read_float_try(&self) -> Option<f64> {
-        when_then(self.kind().tag_class() == TagClass::Float, unsafe {
+        when_then(self.kind().tag_class() == TagClass::Float, || unsafe {
             self.read_float_uck()
         })
     }
@@ -187,12 +190,12 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     /// ## Safety
     /// Are you a binary? Did you store it correctly? Are you a victim of segfaults?
     unsafe fn read_bin_uck(&self) -> &[u8] {
-        let (p, l) = self.data().ld();
+        let (l, p) = self.data().ld();
         slice::from_raw_parts(p, l)
     }
     /// Load a bin
     fn read_bin_try(&self) -> Option<&[u8]> {
-        when_then(self.kind().tag_class() == TagClass::Bin, unsafe {
+        when_then(self.kind().tag_class() == TagClass::Bin, || unsafe {
             self.read_bin_uck()
         })
     }
@@ -210,7 +213,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     }
     /// Load a str
     fn read_str_try(&self) -> Option<&str> {
-        when_then(self.kind().tag_class() == TagClass::Str, unsafe {
+        when_then(self.kind().tag_class() == TagClass::Str, || unsafe {
             self.read_str_uck()
         })
     }
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
index 3712d925..16194a6c 100644
--- a/server/src/engine/data/tag.rs
+++ b/server/src/engine/data/tag.rs
@@ -143,3 +143,58 @@ impl DataTag for FullTag {
         self.unique
     }
 }
+
+#[derive(Debug, Clone, Copy)]
+pub struct CUTag {
+    class: TagClass,
+    unique: TagUnique,
+}
+
+impl PartialEq for CUTag {
+    fn eq(&self, other: &Self) -> bool {
+        self.class == other.class
+    }
+}
+
+macro_rules! cutag {
+    ($class:ident, $unique:ident) => {
+        CUTag::new(TagClass::$class, TagUnique::$unique)
+    };
+    ($class:ident) => {
+        CUTag::new(TagClass::$class, TagUnique::Illegal)
+    };
+}
+
+impl CUTag {
+    const fn new(class: TagClass, unique: TagUnique) -> Self {
+        Self { class, unique }
+    }
+}
+
+impl DataTag for CUTag {
+    const BOOL: Self = cutag!(Bool);
+    const UINT: Self = cutag!(UnsignedInt, UnsignedInt);
+    const SINT: Self = cutag!(SignedInt, SignedInt);
+    const FLOAT: Self = cutag!(Float);
+    const BIN: Self = cutag!(Bin, Bin);
+    const STR: Self = cutag!(Str, Str);
+    const LIST: Self = cutag!(List);
+
+    fn tag_class(&self) -> TagClass {
+        self.class
+    }
+
+    fn tag_selector(&self) -> TagSelector {
+        unimplemented!()
+    }
+
+    fn tag_unique(&self) -> TagUnique {
+        self.unique
+    }
+}
+
+impl From<FullTag> for CUTag {
+    fn from(f: FullTag) -> Self {
+        Self::new(f.tag_class(), f.tag_unique())
+    }
+}
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 82922012..0f7119fa 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -26,7 +26,7 @@
 
 #![deny(unreachable_patterns)]
 
-mod meta;
+pub mod meta;
 mod mtchm;
 mod stdhm;
 mod stord;
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index a213bd5b..fa83aeda 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -56,6 +56,7 @@ pub struct SpecialPaddedWord {
     a: u64,
     b: usize,
 }
+
 impl SpecialPaddedWord {
     const fn new(a: u64, b: usize) -> Self {
         Self { a, b }
diff --git a/server/src/engine/mem/word.rs b/server/src/engine/mem/word.rs
index 97ba2099..44979998 100644
--- a/server/src/engine/mem/word.rs
+++ b/server/src/engine/mem/word.rs
@@ -31,7 +31,7 @@ static ZERO_BLOCK: [u8; 0] = [];
 /// Native quad pointer stack (must also be usable as a double and triple pointer stack. see [`SystemTword`] and [`SystemDword`])
 pub trait SystemQword: SystemTword {
     fn store_full(a: usize, b: usize, c: usize, d: usize) -> Self;
-    fn load_full(&self) -> [usize; 4];
+    fn load_quad(&self) -> [usize; 4];
     fn store<'a, T>(v: T) -> Self
     where
         T: WordRW<Self>,
@@ -49,7 +49,7 @@ pub trait SystemQword: SystemTword {
 /// Native tripe pointer stack (must also be usable as a double pointer stack, see [`SystemDword`])
 pub trait SystemTword: SystemDword {
     fn store_full(a: usize, b: usize, c: usize) -> Self;
-    fn load_full(&self) -> [usize; 3];
+    fn load_triple(&self) -> [usize; 3];
     fn store<'a, T>(v: T) -> Self
     where
         T: WordRW<Self>,
@@ -69,7 +69,7 @@ pub trait SystemDword: Sized {
     fn store_qw(u: u64) -> Self;
     fn store_fat(a: usize, b: usize) -> Self;
     fn load_qw(&self) -> u64;
-    fn load_fat(&self) -> [usize; 2];
+    fn load_double(&self) -> [usize; 2];
     fn store<'a, T>(v: T) -> Self
     where
         T: WordRW<Self>,
@@ -94,7 +94,7 @@ impl SystemDword for SpecialPaddedWord {
     fn load_qw(&self) -> u64 {
         self.a
     }
-    fn load_fat(&self) -> [usize; 2] {
+    fn load_double(&self) -> [usize; 2] {
         [self.a as usize, self.b]
     }
 }
@@ -137,7 +137,7 @@ impl SystemDword for NativeDword {
         x
     }
     #[inline(always)]
-    fn load_fat(&self) -> [usize; 2] {
+    fn load_double(&self) -> [usize; 2] {
         self.0
     }
 }
@@ -148,7 +148,7 @@ impl SystemTword for NativeTword {
         Self([a, b, c])
     }
     #[inline(always)]
-    fn load_full(&self) -> [usize; 3] {
+    fn load_triple(&self) -> [usize; 3] {
         self.0
     }
 }
@@ -193,7 +193,7 @@ impl SystemDword for NativeTword {
         x
     }
     #[inline(always)]
-    fn load_fat(&self) -> [usize; 2] {
+    fn load_double(&self) -> [usize; 2] {
         [self.0[0], self.0[1]]
     }
 }
@@ -202,7 +202,7 @@ impl SystemQword for NativeQword {
     fn store_full(a: usize, b: usize, c: usize, d: usize) -> Self {
         Self([a, b, c, d])
     }
-    fn load_full(&self) -> [usize; 4] {
+    fn load_quad(&self) -> [usize; 4] {
         self.0
     }
 }
@@ -211,7 +211,7 @@ impl SystemTword for NativeQword {
     fn store_full(a: usize, b: usize, c: usize) -> Self {
         Self([a, b, c, 0])
     }
-    fn load_full(&self) -> [usize; 3] {
+    fn load_triple(&self) -> [usize; 3] {
         [self.0[0], self.0[1], self.0[2]]
     }
 }
@@ -251,7 +251,7 @@ impl SystemDword for NativeQword {
         }
         ret
     }
-    fn load_fat(&self) -> [usize; 2] {
+    fn load_double(&self) -> [usize; 2] {
         [self.0[0], self.0[1]]
     }
 }
@@ -318,20 +318,20 @@ impl_wordrw! {
     }
     [usize; 2] as SystemDword => {
         |self| SystemDword::store_fat(self[0], self[1]);
-        |word| SystemDword::load_fat(word);
+        |word| SystemDword::load_double(word);
     }
-    (*mut u8, usize) as SystemDword => {
-        |self| SystemDword::store_fat(self.0 as usize, self.1);
+    (usize, *mut u8) as SystemDword => {
+        |self| SystemDword::store_fat(self.0, self.1 as usize);
         |word| {
-            let [a, b] = word.load_fat();
-            (a as *mut u8, b)
+            let [a, b] = word.load_double();
+            (a, b as *mut u8)
         };
     }
-    (*const u8, usize) as SystemDword => {
-        |self| SystemDword::store_fat(self.0 as usize, self.1);
+    (usize, *const u8) as SystemDword => {
+        |self| SystemDword::store_fat(self.0, self.1 as usize);
         |word| {
-            let [a, b] = word.load_fat();
-            (a as *const u8, b)
+            let [a, b] = word.load_double();
+            (a, b as *const u8)
         };
     }
 }
diff --git a/server/src/util/test_utils.rs b/server/src/util/test_utils.rs
index 48128db5..e8581bc4 100644
--- a/server/src/util/test_utils.rs
+++ b/server/src/util/test_utils.rs
@@ -24,9 +24,15 @@
  *
 */
 
-use rand::{
-    distributions::{uniform::SampleUniform, Alphanumeric},
-    Rng,
+use {
+    rand::{
+        distributions::{uniform::SampleUniform, Alphanumeric},
+        Rng,
+    },
+    std::{
+        collections::hash_map::RandomState,
+        hash::{BuildHasher, Hash, Hasher},
+    },
 };
 
 // TODO(@ohsayan): Use my own PRNG algo here. Maybe my quadratic one?
@@ -86,3 +92,13 @@ macro_rules! vecfuse {
         v
     }};
 }
+
+pub fn randomstate() -> RandomState {
+    RandomState::default()
+}
+
+pub fn hash_rs<T: Hash + ?Sized>(rs: &RandomState, item: &T) -> u64 {
+    let mut hasher = rs.build_hasher();
+    item.hash(&mut hasher);
+    hasher.finish()
+}

From eca185d560ee36072434f87aa95f56f299f7303b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 16 Apr 2023 09:53:35 -0700
Subject: [PATCH 172/310] Make rc modular

---
 server/src/engine/sync/smart.rs | 121 ++++++++++++++++++++++----------
 1 file changed, 82 insertions(+), 39 deletions(-)

diff --git a/server/src/engine/sync/smart.rs b/server/src/engine/sync/smart.rs
index d872e985..1da74a18 100644
--- a/server/src/engine/sync/smart.rs
+++ b/server/src/engine/sync/smart.rs
@@ -93,7 +93,7 @@ impl Eq for StrRC {}
 pub struct SliceRC<T> {
     ptr: NonNull<T>,
     len: usize,
-    rc: NonNull<AtomicUsize>,
+    rc: EArc,
 }
 
 impl<T> SliceRC<T> {
@@ -103,8 +103,8 @@ impl<T> SliceRC<T> {
             ptr,
             len,
             rc: unsafe {
-                // UNSAFE(@ohsayan): box would either fail or return a collect alloc
-                NonNull::new_unchecked(Box::into_raw(Box::new(AtomicUsize::new(0))))
+                // UNSAFE(@ohsayan): we will eventually deallocate this
+                EArc::new()
             },
         }
     }
@@ -123,42 +123,24 @@ impl<T> SliceRC<T> {
             slice::from_raw_parts(self.ptr.as_ptr(), self.len)
         }
     }
-    #[inline(always)]
-    fn _rc(&self) -> &AtomicUsize {
-        unsafe {
-            // UNSAFE(@ohsayan): rc + ctor
-            self.rc.as_ref()
-        }
-    }
-    /// SAFETY: Synchronize last man alive
-    #[inline(never)]
-    unsafe fn drop_slow(&self) {
-        // dtor
-        if mem::needs_drop::<T>() {
-            // UNSAFE(@ohsayan): dtor through, the ctor guarantees correct alignment and len
-            ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.ptr.as_ptr(), self.len));
-        }
-        // dealloc
-        // UNSAFE(@ohsayan): we allocated it
-        let layout = Layout::array::<T>(self.len).unwrap_unchecked();
-        // UNSAFE(@ohsayan): layout structure guaranteed by ctor
-        dealloc(self.ptr.as_ptr() as *mut u8, layout);
-        // UNSAFE(@ohsayan): well cmon, look for yourself
-        drop(Box::from_raw(self.rc.as_ptr()));
-    }
 }
 
 impl<T> Drop for SliceRC<T> {
     fn drop(&mut self) {
-        if self._rc().fetch_sub(1, ORD_REL) != 1 {
-            // not the last man alive
-            return;
-        }
-        // emit a fence for sync with stores
-        atomic::fence(ORD_ACQ);
         unsafe {
-            // UNSAFE(@ohsayan): Confirmed, we're the last one alive
-            self.drop_slow();
+            // UNSAFE(@ohsayan): Calling this within the dtor itself
+            self.rc.rc_drop(|| {
+                // dtor
+                if mem::needs_drop::<T>() {
+                    // UNSAFE(@ohsayan): dtor through, the ctor guarantees correct alignment and len
+                    ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.ptr.as_ptr(), self.len));
+                }
+                // dealloc
+                // UNSAFE(@ohsayan): we allocated it
+                let layout = Layout::array::<T>(self.len).unwrap_unchecked();
+                // UNSAFE(@ohsayan): layout structure guaranteed by ctor
+                dealloc(self.ptr.as_ptr() as *mut u8, layout);
+            })
         }
     }
 }
@@ -166,12 +148,14 @@ impl<T> Drop for SliceRC<T> {
 impl<T> Clone for SliceRC<T> {
     #[inline(always)]
     fn clone(&self) -> Self {
-        let new_rc = self._rc().fetch_add(1, ORD_RLX);
-        if new_rc > (isize::MAX) as usize {
-            // some incredibly degenerate case; this won't ever happen but who knows if some fella decided to have atomic overflow fun?
-            process::abort();
+        let new_rc = unsafe {
+            // UNSAFE(@ohsayan): calling this within the clone routine
+            self.rc.rc_clone()
+        };
+        Self {
+            rc: new_rc,
+            ..*self
         }
-        Self { ..*self }
     }
 }
 
@@ -215,3 +199,62 @@ impl<T> Deref for SliceRC<T> {
 
 unsafe impl<T: Send> Send for SliceRC<T> {}
 unsafe impl<T: Sync> Sync for SliceRC<T> {}
+
+/// The core atomic reference counter implementation. All smart pointers use this inside
+pub struct EArc {
+    rc: NonNull<AtomicUsize>,
+}
+
+impl EArc {
+    /// Create a new [`EArc`] instance
+    ///
+    /// ## Safety
+    ///
+    /// While this is **not unsafe** in the eyes of the language specification for safety, it still does violate a very common
+    /// bug: memory leaks and we don't want that. So, it is upto the caller to clean this up
+    unsafe fn new() -> Self {
+        Self {
+            rc: NonNull::new_unchecked(Box::into_raw(Box::new(AtomicUsize::new(0)))),
+        }
+    }
+}
+
+impl EArc {
+    /// ## Safety
+    ///
+    /// Only call when you follow the appropriate ground rules for safety
+    unsafe fn _rc(&self) -> &AtomicUsize {
+        self.rc.as_ref()
+    }
+    /// ## Safety
+    ///
+    /// Only call in an actual [`Clone`] context
+    unsafe fn rc_clone(&self) -> Self {
+        let new_rc = self._rc().fetch_add(1, ORD_RLX);
+        if new_rc > (isize::MAX) as usize {
+            // some incredibly degenerate case; this won't ever happen but who knows if some fella decided to have atomic overflow fun?
+            process::abort();
+        }
+        Self { ..*self }
+    }
+    #[cold]
+    #[inline(never)]
+    unsafe fn rc_drop_slow(&mut self, mut dropfn: impl FnMut()) {
+        // deallocate object
+        dropfn();
+        // deallocate rc
+        drop(Box::from_raw(self.rc.as_ptr()));
+    }
+    /// ## Safety
+    ///
+    /// Only call in dtor context
+    unsafe fn rc_drop(&mut self, dropfn: impl FnMut()) {
+        if self._rc().fetch_sub(1, ORD_REL) != 1 {
+            // not the last man alive
+            return;
+        }
+        // emit a fence for sync with stores
+        atomic::fence(ORD_ACQ);
+        self.rc_drop_slow(dropfn);
+    }
+}

From b25899e04b735e73539de9e0b10c9fb14892ab25 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 16 Apr 2023 12:16:24 -0700
Subject: [PATCH 173/310] Fix native qw storage for `Datacell`

---
 server/src/engine/core/model/cell.rs |   9 ++-
 server/src/engine/data/spec.rs       |  10 +--
 server/src/engine/mem/word.rs        | 114 ++++++++++++++++++++-------
 3 files changed, 96 insertions(+), 37 deletions(-)

diff --git a/server/src/engine/core/model/cell.rs b/server/src/engine/core/model/cell.rs
index e4369db5..f46e1b83 100644
--- a/server/src/engine/core/model/cell.rs
+++ b/server/src/engine/core/model/cell.rs
@@ -32,7 +32,7 @@ use {
             spec::{Dataspec1D, DataspecMeta1D},
             tag::{CUTag, DataTag, TagClass},
         },
-        mem::{NativeQword, SystemDword, WordRW},
+        mem::{NativeQword, SystemDword, SystemTword, WordRW},
     },
     core::{fmt, mem, mem::ManuallyDrop, slice, str},
     parking_lot::RwLock,
@@ -213,11 +213,12 @@ impl<'a> From<LitIR<'a>> for Datacell {
         match l.kind().tag_class() {
             tag if tag < TagClass::Bin => unsafe {
                 // UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type doesn't need any advanced construction
-                let [a, b] = l.data().load_double();
+                let data = l.data().load_qw();
+                let ptr = l.data().load_double()[1];
                 Datacell::new(
                     CUTag::from(l.kind()),
                     // DO NOT RELY ON the payload's bit pattern; it's padded
-                    DataRaw::word(SystemDword::store_fat(a, b)),
+                    DataRaw::word(SystemTword::store_qw_nw(data, ptr as usize)),
                 )
             },
             TagClass::Bin | TagClass::Str => unsafe {
@@ -286,7 +287,7 @@ impl Datacell {
         }
     }
     unsafe fn load_word<'a, T: WordRW<NativeQword, Target<'a> = T>>(&'a self) -> T {
-        self.data.word.ld()
+        self.data.word.dword_ld()
     }
     unsafe fn _new(tag: CUTag, data: DataRaw, init: bool) -> Self {
         Self { init, tag, data }
diff --git a/server/src/engine/data/spec.rs b/server/src/engine/data/spec.rs
index de241374..40f71d39 100644
--- a/server/src/engine/data/spec.rs
+++ b/server/src/engine/data/spec.rs
@@ -117,7 +117,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // bool
     /// Load a bool (this is unsafe for logical verity)
     unsafe fn read_bool_uck(&self) -> bool {
-        self.data().ld()
+        self.data().dword_ld()
     }
     /// Load a bool
     fn read_bool_try(&self) -> Option<bool> {
@@ -135,7 +135,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // uint
     /// Load a uint (this is unsafe for logical verity)
     unsafe fn read_uint_uck(&self) -> u64 {
-        self.data().ld()
+        self.data().dword_ld()
     }
     /// Load a uint
     fn read_uint_try(&self) -> Option<u64> {
@@ -156,7 +156,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // sint
     /// Load a sint (unsafe for logical verity)
     unsafe fn read_sint_uck(&self) -> i64 {
-        self.data().ld()
+        self.data().dword_ld()
     }
     /// Load a sint
     fn read_sint_try(&self) -> Option<i64> {
@@ -172,7 +172,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // float
     /// Load a float (unsafe for logical verity)
     unsafe fn read_float_uck(&self) -> f64 {
-        self.data().ld()
+        self.data().dword_ld()
     }
     /// Load a float
     fn read_float_try(&self) -> Option<f64> {
@@ -190,7 +190,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     /// ## Safety
     /// Are you a binary? Did you store it correctly? Are you a victim of segfaults?
     unsafe fn read_bin_uck(&self) -> &[u8] {
-        let (l, p) = self.data().ld();
+        let (l, p) = self.data().dword_ld();
         slice::from_raw_parts(p, l)
     }
     /// Load a bin
diff --git a/server/src/engine/mem/word.rs b/server/src/engine/mem/word.rs
index 44979998..5cd7cb72 100644
--- a/server/src/engine/mem/word.rs
+++ b/server/src/engine/mem/word.rs
@@ -28,6 +28,21 @@ use super::{NativeDword, NativeQword, NativeTword, SpecialPaddedWord};
 
 static ZERO_BLOCK: [u8; 0] = [];
 
+#[cfg(target_pointer_width = "32")]
+fn quadsplit(q: u64) -> [usize; 2] {
+    unsafe {
+        // UNSAFE(@ohsayan): simple numeric ops
+        core::mem::transmute(q)
+    }
+}
+#[cfg(target_pointer_width = "32")]
+fn quadmerge(v: [usize; 2]) -> u64 {
+    unsafe {
+        // UNSAFE(@ohsayan): simple numeric ops
+        core::mem::transmute(v)
+    }
+}
+
 /// Native quad pointer stack (must also be usable as a double and triple pointer stack. see [`SystemTword`] and [`SystemDword`])
 pub trait SystemQword: SystemTword {
     fn store_full(a: usize, b: usize, c: usize, d: usize) -> Self;
@@ -38,7 +53,7 @@ pub trait SystemQword: SystemTword {
     {
         WordRW::store(v)
     }
-    fn ld<'a, T>(&'a self) -> T
+    fn qword_ld<'a, T>(&'a self) -> T
     where
         T: WordRW<Self, Target<'a> = T>,
     {
@@ -48,6 +63,10 @@ pub trait SystemQword: SystemTword {
 
 /// Native tripe pointer stack (must also be usable as a double pointer stack, see [`SystemDword`])
 pub trait SystemTword: SystemDword {
+    /// Store a quad and a native word
+    fn store_qw_nw(a: u64, b: usize) -> Self;
+    /// Load a quad and a native word
+    fn load_qw_nw(&self) -> (u64, usize);
     fn store_full(a: usize, b: usize, c: usize) -> Self;
     fn load_triple(&self) -> [usize; 3];
     fn store<'a, T>(v: T) -> Self
@@ -56,7 +75,7 @@ pub trait SystemTword: SystemDword {
     {
         WordRW::store(v)
     }
-    fn ld<'a, T>(&'a self) -> T
+    fn tword_ld<'a, T>(&'a self) -> T
     where
         T: WordRW<Self, Target<'a> = T>,
     {
@@ -76,7 +95,7 @@ pub trait SystemDword: Sized {
     {
         WordRW::store(v)
     }
-    fn ld<'a, T>(&'a self) -> T
+    fn dword_ld<'a, T>(&'a self) -> T
     where
         T: WordRW<Self, Target<'a> = T>,
     {
@@ -105,10 +124,7 @@ impl SystemDword for NativeDword {
         let x;
         #[cfg(target_pointer_width = "32")]
         {
-            x = unsafe {
-                // UNSAFE(@ohsayan): same layout and this is a stupidly simple cast and it's wild that the rust std doesn't have a simpler way to do it
-                core::mem::transmute(u)
-            };
+            x = quadsplit(u);
         }
         #[cfg(target_pointer_width = "64")]
         {
@@ -125,10 +141,7 @@ impl SystemDword for NativeDword {
         let x;
         #[cfg(target_pointer_width = "32")]
         {
-            x = unsafe {
-                // UNSAFE(@ohsayan): same layout and this is a stupidly simple cast and it's wild that the rust std doesn't have a simpler way to do it
-                core::mem::transmute_copy(self)
-            }
+            x = quadmerge(self.0);
         }
         #[cfg(target_pointer_width = "64")]
         {
@@ -151,6 +164,35 @@ impl SystemTword for NativeTword {
     fn load_triple(&self) -> [usize; 3] {
         self.0
     }
+    #[inline(always)]
+    fn store_qw_nw(a: u64, b: usize) -> Self {
+        let ret;
+        #[cfg(target_pointer_width = "32")]
+        {
+            let [qw_1, qw_2] = quadsplit(a);
+            ret = [qw_1, qw_2, b];
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            ret = [a as usize, b, 0];
+        }
+        Self(ret)
+    }
+    #[inline(always)]
+    fn load_qw_nw(&self) -> (u64, usize) {
+        let ret;
+        #[cfg(target_pointer_width = "32")]
+        {
+            let qw = quadmerge([self.0[0], self.0[1]]);
+            let nw = self.0[2];
+            ret = (qw, nw);
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            ret = (self.0[0] as u64, self.0[1]);
+        }
+        ret
+    }
 }
 
 impl SystemDword for NativeTword {
@@ -159,10 +201,7 @@ impl SystemDword for NativeTword {
         let x;
         #[cfg(target_pointer_width = "32")]
         {
-            let [a, b]: [usize; 2] = unsafe {
-                // UNSAFE(@ohsayan): same layout and this is a stupidly simple cast and it's wild that the rust std doesn't have a simpler way to do it
-                core::mem::transmute(u)
-            };
+            let [a, b]: [usize; 2] = quadsplit(u);
             x = [a, b, 0];
         }
         #[cfg(target_pointer_width = "64")]
@@ -180,11 +219,7 @@ impl SystemDword for NativeTword {
         let x;
         #[cfg(target_pointer_width = "32")]
         {
-            let ab = [self.0[0], self.0[1]];
-            x = unsafe {
-                // UNSAFE(@ohsayan): same layout and this is a stupidly simple cast and it's wild that the rust std doesn't have a simpler way to do it
-                core::mem::transmute(ab)
-            };
+            x = quadmerge([self.0[0], self.0[1]]);
         }
         #[cfg(target_pointer_width = "64")]
         {
@@ -214,6 +249,35 @@ impl SystemTword for NativeQword {
     fn load_triple(&self) -> [usize; 3] {
         [self.0[0], self.0[1], self.0[2]]
     }
+    /// Store a quadword and a native word
+    fn store_qw_nw(a: u64, b: usize) -> Self {
+        let ret;
+        #[cfg(target_pointer_width = "32")]
+        {
+            let [qw_1, qw_2] = quadsplit(a);
+            ret = [qw_1, qw_2, b, 0];
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            ret = [a as usize, b, 0, 0];
+        }
+        Self(ret)
+    }
+    #[inline(always)]
+    fn load_qw_nw(&self) -> (u64, usize) {
+        let ret;
+        #[cfg(target_pointer_width = "32")]
+        {
+            let qw = quadmerge([self.0[0], self.0[1]]);
+            let nw = self.0[2];
+            ret = (qw, nw);
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            ret = (self.0[0] as u64, self.0[1]);
+        }
+        ret
+    }
 }
 
 impl SystemDword for NativeQword {
@@ -221,10 +285,7 @@ impl SystemDword for NativeQword {
         let ret;
         #[cfg(target_pointer_width = "32")]
         {
-            let [a, b]: [usize; 2] = unsafe {
-                // UNSAFE(@ohsayan): same layout and this is a stupidly simple cast and it's wild that the rust std doesn't have a simpler way to do it
-                core::mem::transmute(u)
-            };
+            let [a, b] = quadsplit(u);
             ret = <Self as SystemQword>::store_full(a, b, 0, 0);
         }
         #[cfg(target_pointer_width = "64")]
@@ -240,10 +301,7 @@ impl SystemDword for NativeQword {
         let ret;
         #[cfg(target_pointer_width = "32")]
         {
-            ret = unsafe {
-                // UNSAFE(@ohsayan): same layout and this is a stupidly simple cast and it's wild that the rust std doesn't have a simpler way to do it
-                core::mem::transmute([self.0[0], self.0[1]])
-            };
+            ret = quadmerge([self.0[0], self.0[1]]);
         }
         #[cfg(target_pointer_width = "64")]
         {

From 329ef1a27edc608afc9f443cca515a19b09f6e1f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 22 Apr 2023 09:38:07 -0700
Subject: [PATCH 174/310] Add tests for large integers in data items

---
 server/src/engine/core/index/key.rs | 32 +++++++++++++++++++++--------
 server/src/engine/data/tests/mod.rs | 18 ++++++++++++++++
 2 files changed, 42 insertions(+), 8 deletions(-)

diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
index 212aac4b..dcc16f02 100644
--- a/server/src/engine/core/index/key.rs
+++ b/server/src/engine/core/index/key.rs
@@ -70,7 +70,7 @@ impl PrimaryIndexKey {
         })
     }
     pub unsafe fn read_bin(&self) -> &[u8] {
-        self.vdata()
+        self.virtual_block()
     }
     pub fn bin(&self) -> Option<&[u8]> {
         (self.tag == TagUnique::Bin).then(|| unsafe {
@@ -79,7 +79,7 @@ impl PrimaryIndexKey {
         })
     }
     pub unsafe fn read_str(&self) -> &str {
-        str::from_utf8_unchecked(self.vdata())
+        str::from_utf8_unchecked(self.virtual_block())
     }
     pub fn str(&self) -> Option<&str> {
         (self.tag == TagUnique::Str).then(|| unsafe {
@@ -134,14 +134,14 @@ impl PrimaryIndexKey {
         let actual_len = len * (self.tag >= TagUnique::Bin) as usize;
         [data, actual_len]
     }
-    fn vdata(&self) -> &[u8] {
+    fn virtual_block(&self) -> &[u8] {
         let [data, actual_len] = self.__compute_vdata_offset();
         unsafe {
             // UNSAFE(@ohsayan): Safe, due to construction
             slice::from_raw_parts(data as *const u8, actual_len)
         }
     }
-    fn vdata_mut(&mut self) -> &mut [u8] {
+    fn virtual_block_mut(&mut self) -> &mut [u8] {
         let [data, actual_len] = self.__compute_vdata_offset();
         unsafe {
             // UNSAFE(@ohsayan): safe due to construction
@@ -155,7 +155,7 @@ impl Drop for PrimaryIndexKey {
         if let TagUnique::Bin | TagUnique::Str = self.tag {
             unsafe {
                 // UNSAFE(@ohsayan): Aliasing, sole owner and correct initialization
-                let vdata = self.vdata_mut();
+                let vdata = self.virtual_block_mut();
                 mem::dealloc_array(vdata.as_mut_ptr(), vdata.len());
             }
         }
@@ -165,7 +165,8 @@ impl Drop for PrimaryIndexKey {
 impl PartialEq for PrimaryIndexKey {
     fn eq(&self, other: &Self) -> bool {
         let [data_1, data_2] = [self.data.load_double()[0], other.data.load_double()[0]];
-        ((self.tag == other.tag) & (data_1 == data_2)) && self.vdata() == other.vdata()
+        ((self.tag == other.tag) & (data_1 == data_2))
+            && self.virtual_block() == other.virtual_block()
     }
 }
 
@@ -174,14 +175,14 @@ impl Eq for PrimaryIndexKey {}
 impl Hash for PrimaryIndexKey {
     fn hash<H: Hasher>(&self, hasher: &mut H) {
         self.tag.hash(hasher);
-        self.vdata().hash(hasher);
+        self.virtual_block().hash(hasher);
     }
 }
 
 impl<'a> PartialEq<LitIR<'a>> for PrimaryIndexKey {
     fn eq(&self, key: &LitIR<'a>) -> bool {
         debug_assert!(key.kind().tag_unique().is_unique());
-        self.tag == key.kind().tag_unique() && self.vdata() == key.__vdata()
+        self.tag == key.kind().tag_unique() && self.virtual_block() == key.__vdata()
     }
 }
 
@@ -274,3 +275,18 @@ fn check_pk_lit_eq_hash() {
         );
     }
 }
+
+#[test]
+fn check_pk_extremes() {
+    let state = test_utils::randomstate();
+    let d1 = PrimaryIndexKey::try_from_dc(Datacell::new_uint(u64::MAX)).unwrap();
+    let d2 = PrimaryIndexKey::try_from_dc(Datacell::from(LitIR::UnsignedInt(u64::MAX))).unwrap();
+    assert_eq!(d1, d2);
+    assert_eq!(
+        test_utils::hash_rs(&state, &d1),
+        test_utils::hash_rs(&state, &d2)
+    );
+    assert_eq!(d1, LitIR::UnsignedInt(u64::MAX));
+    assert_eq!(d2, LitIR::UnsignedInt(u64::MAX));
+    assert_eq!(d1.uint().unwrap(), u64::MAX);
+}
diff --git a/server/src/engine/data/tests/mod.rs b/server/src/engine/data/tests/mod.rs
index 1eff0f69..4c931392 100644
--- a/server/src/engine/data/tests/mod.rs
+++ b/server/src/engine/data/tests/mod.rs
@@ -25,3 +25,21 @@
 */
 
 mod md_dict_tests;
+use super::{
+    lit::{Lit, LitIR},
+    spec::Dataspec1D,
+};
+
+#[test]
+fn t_largest_int_litir() {
+    let x = LitIR::UnsignedInt(u64::MAX);
+    let y = LitIR::UnsignedInt(u64::MAX);
+    assert_eq!(x, y);
+}
+
+#[test]
+fn t_largest_int_lit() {
+    let x = Lit::UnsignedInt(u64::MAX);
+    let y = Lit::UnsignedInt(u64::MAX);
+    assert_eq!(x, y);
+}

From 4eb2851cf998bec7bc161da73f33c4b7d323c967 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 24 Apr 2023 22:27:07 -0700
Subject: [PATCH 175/310] Improve word impls

---
 server/src/engine/core/index/key.rs           |  26 +-
 server/src/engine/core/model/cell.rs          |  37 +-
 server/src/engine/data/lit.rs                 |  22 +-
 server/src/engine/data/spec.rs                |  24 +-
 server/src/engine/mem/mod.rs                  |   4 +-
 .../src/engine/mem/{tests.rs => tests/mod.rs} |   1 +
 server/src/engine/mem/tests/word.rs           | 145 +++++
 server/src/engine/mem/word.rs                 | 511 +++++++++---------
 8 files changed, 453 insertions(+), 317 deletions(-)
 rename server/src/engine/mem/{tests.rs => tests/mod.rs} (99%)
 create mode 100644 server/src/engine/mem/tests/word.rs

diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
index dcc16f02..4090c380 100644
--- a/server/src/engine/core/index/key.rs
+++ b/server/src/engine/core/index/key.rs
@@ -35,7 +35,7 @@ use {
             tag::{DataTag, TagUnique},
         },
         idx::meta::Comparable,
-        mem::{self, NativeDword, SystemDword},
+        mem::{self, DwordQN, SpecialPaddedWord, WordIO},
     },
     core::{
         fmt,
@@ -47,12 +47,12 @@ use {
 
 pub struct PrimaryIndexKey {
     tag: TagUnique,
-    data: NativeDword,
+    data: SpecialPaddedWord,
 }
 
 impl PrimaryIndexKey {
     pub unsafe fn read_uint(&self) -> u64 {
-        self.data.load_qw()
+        self.data.load()
     }
     pub fn uint(&self) -> Option<u64> {
         (self.tag == TagUnique::UnsignedInt).then_some(unsafe {
@@ -61,7 +61,7 @@ impl PrimaryIndexKey {
         })
     }
     pub unsafe fn read_sint(&self) -> i64 {
-        self.data.load_qw() as _
+        self.data.load()
     }
     pub fn sint(&self) -> Option<i64> {
         (self.tag == TagUnique::SignedInt).then_some(unsafe {
@@ -105,16 +105,16 @@ impl PrimaryIndexKey {
         debug_assert!(Self::check(&dc));
         let tag = dc.tag().tag_unique();
         let dc = ManuallyDrop::new(dc);
-        let dword = unsafe {
+        let (a, b) = unsafe {
             // UNSAFE(@ohsayan): this doesn't do anything "bad" by itself. needs the construction to be broken for it to do something silly
             dc.as_raw()
         }
-        .load_double();
+        .dwordqn_load_qw_nw();
         Self {
             tag,
             data: unsafe {
-                // UNSAFE(@ohsayan): Perfectly safe since we're tranforming it and THIS will not by itself crash anything
-                core::mem::transmute(dword)
+                // UNSAFE(@ohsayan): loaded above, writing here
+                SpecialPaddedWord::new(a, b)
             },
         }
     }
@@ -126,12 +126,12 @@ impl PrimaryIndexKey {
     }
     /// ## Safety
     /// If you mess up construction, everything will fall apart
-    pub unsafe fn new(tag: TagUnique, data: NativeDword) -> Self {
+    pub unsafe fn new(tag: TagUnique, data: SpecialPaddedWord) -> Self {
         Self { tag, data }
     }
     fn __compute_vdata_offset(&self) -> [usize; 2] {
-        let [len, data] = self.data.load_double();
-        let actual_len = len * (self.tag >= TagUnique::Bin) as usize;
+        let (len, data) = self.data.dwordqn_load_qw_nw();
+        let actual_len = (len as usize) * (self.tag >= TagUnique::Bin) as usize;
         [data, actual_len]
     }
     fn virtual_block(&self) -> &[u8] {
@@ -164,7 +164,7 @@ impl Drop for PrimaryIndexKey {
 
 impl PartialEq for PrimaryIndexKey {
     fn eq(&self, other: &Self) -> bool {
-        let [data_1, data_2] = [self.data.load_double()[0], other.data.load_double()[0]];
+        let [data_1, data_2]: [u64; 2] = [self.data.load(), other.data.load()];
         ((self.tag == other.tag) & (data_1 == data_2))
             && self.virtual_block() == other.virtual_block()
     }
@@ -282,6 +282,8 @@ fn check_pk_extremes() {
     let d1 = PrimaryIndexKey::try_from_dc(Datacell::new_uint(u64::MAX)).unwrap();
     let d2 = PrimaryIndexKey::try_from_dc(Datacell::from(LitIR::UnsignedInt(u64::MAX))).unwrap();
     assert_eq!(d1, d2);
+    assert_eq!(d1.uint().unwrap(), u64::MAX);
+    assert_eq!(d2.uint().unwrap(), u64::MAX);
     assert_eq!(
         test_utils::hash_rs(&state, &d1),
         test_utils::hash_rs(&state, &d2)
diff --git a/server/src/engine/core/model/cell.rs b/server/src/engine/core/model/cell.rs
index f46e1b83..effd8cfc 100644
--- a/server/src/engine/core/model/cell.rs
+++ b/server/src/engine/core/model/cell.rs
@@ -32,7 +32,7 @@ use {
             spec::{Dataspec1D, DataspecMeta1D},
             tag::{CUTag, DataTag, TagClass},
         },
-        mem::{NativeQword, SystemDword, SystemTword, WordRW},
+        mem::{DwordNN, DwordQN, NativeQword, WordIO},
     },
     core::{fmt, mem, mem::ManuallyDrop, slice, str},
     parking_lot::RwLock,
@@ -49,7 +49,7 @@ impl Datacell {
     pub fn new_bool(b: bool) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(CUTag::BOOL, DataRaw::word(SystemDword::store(b)))
+            Self::new(CUTag::BOOL, DataRaw::word(WordIO::store(b)))
         }
     }
     pub unsafe fn read_bool(&self) -> bool {
@@ -68,7 +68,7 @@ impl Datacell {
     pub fn new_uint(u: u64) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(CUTag::UINT, DataRaw::word(SystemDword::store(u)))
+            Self::new(CUTag::UINT, DataRaw::word(WordIO::store(u)))
         }
     }
     pub unsafe fn read_uint(&self) -> u64 {
@@ -87,7 +87,7 @@ impl Datacell {
     pub fn new_sint(u: i64) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(CUTag::SINT, DataRaw::word(SystemDword::store(u)))
+            Self::new(CUTag::SINT, DataRaw::word(WordIO::store(u)))
         }
     }
     pub unsafe fn read_sint(&self) -> i64 {
@@ -106,7 +106,7 @@ impl Datacell {
     pub fn new_float(f: f64) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(CUTag::FLOAT, DataRaw::word(SystemDword::store(f)))
+            Self::new(CUTag::FLOAT, DataRaw::word(WordIO::store(f)))
         }
     }
     pub unsafe fn read_float(&self) -> f64 {
@@ -128,7 +128,7 @@ impl Datacell {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
                 CUTag::BIN,
-                DataRaw::word(SystemDword::store((md.len(), md.as_mut_ptr()))),
+                DataRaw::word(WordIO::store((md.len(), md.as_mut_ptr()))),
             )
         }
     }
@@ -152,7 +152,7 @@ impl Datacell {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
                 CUTag::STR,
-                DataRaw::word(SystemDword::store((md.len(), md.as_mut_ptr()))),
+                DataRaw::word(WordIO::store((md.len(), md.as_mut_ptr()))),
             )
         }
     }
@@ -213,12 +213,10 @@ impl<'a> From<LitIR<'a>> for Datacell {
         match l.kind().tag_class() {
             tag if tag < TagClass::Bin => unsafe {
                 // UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type doesn't need any advanced construction
-                let data = l.data().load_qw();
-                let ptr = l.data().load_double()[1];
                 Datacell::new(
                     CUTag::from(l.kind()),
                     // DO NOT RELY ON the payload's bit pattern; it's padded
-                    DataRaw::word(SystemTword::store_qw_nw(data, ptr as usize)),
+                    DataRaw::word(l.data().dword_promote()),
                 )
             },
             TagClass::Bin | TagClass::Str => unsafe {
@@ -226,7 +224,7 @@ impl<'a> From<LitIR<'a>> for Datacell {
                 let mut bin = ManuallyDrop::new(l.read_bin_uck().to_owned().into_boxed_slice());
                 Datacell::new(
                     CUTag::from(l.kind()),
-                    DataRaw::word(SystemDword::store((bin.len(), bin.as_mut_ptr()))),
+                    DataRaw::word(WordIO::store((bin.len(), bin.as_mut_ptr()))),
                 )
             },
             _ => unsafe {
@@ -270,7 +268,11 @@ impl Datacell {
     pub fn null() -> Self {
         unsafe {
             // UNSAFE(@ohsayan): This is a hack. It's safe because we set init to false
-            Self::_new(CUTag::BOOL, DataRaw::word(NativeQword::store_qw(0)), false)
+            Self::_new(
+                CUTag::BOOL,
+                DataRaw::word(NativeQword::dwordnn_store_qw(0)),
+                false,
+            )
         }
     }
     pub fn is_null(&self) -> bool {
@@ -286,8 +288,11 @@ impl Datacell {
             None
         }
     }
-    unsafe fn load_word<'a, T: WordRW<NativeQword, Target<'a> = T>>(&'a self) -> T {
-        self.data.word.dword_ld()
+    unsafe fn load_word<'a, T>(&'a self) -> T
+    where
+        NativeQword: WordIO<T>,
+    {
+        self.data.word.load()
     }
     unsafe fn _new(tag: CUTag, data: DataRaw, init: bool) -> Self {
         Self { init, tag, data }
@@ -400,7 +405,7 @@ impl Clone for Datacell {
                 // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class
                 let mut block = ManuallyDrop::new(self.read_bin().to_owned().into_boxed_slice());
                 DataRaw {
-                    word: ManuallyDrop::new(SystemDword::store((block.len(), block.as_mut_ptr()))),
+                    word: ManuallyDrop::new(WordIO::store((block.len(), block.as_mut_ptr()))),
                 }
             },
             TagClass::List => unsafe {
@@ -411,7 +416,7 @@ impl Clone for Datacell {
                 }
             },
             _ => unsafe {
-                // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class
+                // UNSAFE(@ohsayan): we have checked that the cell is a stack class
                 DataRaw {
                     word: ManuallyDrop::new(mem::transmute_copy(&self.data.word)),
                 }
diff --git a/server/src/engine/data/lit.rs b/server/src/engine/data/lit.rs
index 038a8531..0ef9e035 100644
--- a/server/src/engine/data/lit.rs
+++ b/server/src/engine/data/lit.rs
@@ -29,7 +29,7 @@ use {
         spec::{Dataspec1D, DataspecMeta1D, DataspecMethods1D, DataspecRaw1D},
         tag::{DataTag, FullTag, TagUnique},
     },
-    crate::engine::mem::{SpecialPaddedWord, SystemDword},
+    crate::engine::mem::{DwordQN, SpecialPaddedWord, WordIO},
     core::{
         fmt,
         hash::{Hash, Hasher},
@@ -93,16 +93,16 @@ unsafe impl<'a> DataspecRaw1D for Lit<'a> {
     const HEAP_STR: bool = true;
     const HEAP_BIN: bool = false;
     unsafe fn drop_str(&mut self) {
-        let [len, ptr] = self.data().load_double();
-        drop(String::from_raw_parts(ptr as *mut u8, len, len));
+        let (len, ptr) = self.data().load();
+        drop(String::from_raw_parts(ptr, len, len));
     }
     unsafe fn drop_bin(&mut self) {}
     unsafe fn clone_str(s: &str) -> Self::Target {
         let new_string = ManuallyDrop::new(s.to_owned().into_boxed_str());
-        SystemDword::store((new_string.len(), new_string.as_ptr()))
+        WordIO::store((new_string.len(), new_string.as_ptr()))
     }
     unsafe fn clone_bin(b: &[u8]) -> Self::Target {
-        SystemDword::store((b.len(), b.as_ptr()))
+        WordIO::store((b.len(), b.as_ptr()))
     }
 }
 
@@ -114,7 +114,7 @@ unsafe impl<'a> DataspecRaw1D for Lit<'a> {
 unsafe impl<'a> Dataspec1D for Lit<'a> {
     fn Str(s: Box<str>) -> Self {
         let md = ManuallyDrop::new(s);
-        Self::new(FullTag::STR, SystemDword::store((md.len(), md.as_ptr())))
+        Self::new(FullTag::STR, WordIO::store((md.len(), md.as_ptr())))
     }
 }
 
@@ -184,8 +184,8 @@ pub struct LitIR<'a> {
 
 impl<'a> LitIR<'a> {
     pub fn __vdata(&self) -> &[u8] {
-        let [vlen, data] = self.data().load_double();
-        let len = vlen * (self.kind().tag_unique() >= TagUnique::Bin) as usize;
+        let (vlen, data) = self.data().dwordqn_load_qw_nw();
+        let len = vlen as usize * (self.kind().tag_unique() >= TagUnique::Bin) as usize;
         unsafe {
             // UNSAFE(@ohsayan): either because of static or lt
             slice::from_raw_parts(data as *const u8, len)
@@ -237,10 +237,10 @@ unsafe impl<'a> DataspecRaw1D for LitIR<'a> {
     unsafe fn drop_str(&mut self) {}
     unsafe fn drop_bin(&mut self) {}
     unsafe fn clone_str(s: &str) -> Self::Target {
-        SystemDword::store((s.len(), s.as_ptr()))
+        WordIO::store((s.len(), s.as_ptr()))
     }
     unsafe fn clone_bin(b: &[u8]) -> Self::Target {
-        SystemDword::store((b.len(), b.as_ptr()))
+        WordIO::store((b.len(), b.as_ptr()))
     }
 }
 
@@ -250,7 +250,7 @@ unsafe impl<'a> DataspecRaw1D for LitIR<'a> {
 */
 unsafe impl<'a> Dataspec1D for LitIR<'a> {
     fn Str(s: Self::StringItem) -> Self {
-        Self::new(FullTag::STR, SystemDword::store((s.len(), s.as_ptr())))
+        Self::new(FullTag::STR, WordIO::store((s.len(), s.as_ptr())))
     }
 }
 
diff --git a/server/src/engine/data/spec.rs b/server/src/engine/data/spec.rs
index 40f71d39..9a9e71c2 100644
--- a/server/src/engine/data/spec.rs
+++ b/server/src/engine/data/spec.rs
@@ -31,7 +31,7 @@
 
 use {
     super::tag::{DataTag, TagClass},
-    crate::engine::mem::SystemDword,
+    crate::engine::mem::{DwordQN, WordIO},
     core::{fmt, mem, slice},
 };
 
@@ -45,7 +45,7 @@ pub trait DataspecMeta1D: Sized {
     // assoc
     type Tag: DataTag;
     /// The target must be able to store (atleast) a native dword
-    type Target: SystemDword;
+    type Target: DwordQN;
     /// The string item. This helps us remain correct with the dtors
     type StringItem;
     // fn
@@ -86,27 +86,27 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     /// Store a new bool. This function is always safe to call
     #[allow(non_snake_case)]
     fn Bool(b: bool) -> Self {
-        Self::new(Self::Tag::BOOL, SystemDword::store(b))
+        Self::new(Self::Tag::BOOL, WordIO::store(b))
     }
     /// Store a new uint. This function is always safe to call
     #[allow(non_snake_case)]
     fn UnsignedInt(u: u64) -> Self {
-        Self::new(Self::Tag::UINT, SystemDword::store(u))
+        Self::new(Self::Tag::UINT, WordIO::store(u))
     }
     /// Store a new sint. This function is always safe to call
     #[allow(non_snake_case)]
     fn SignedInt(s: i64) -> Self {
-        Self::new(Self::Tag::SINT, SystemDword::store(s))
+        Self::new(Self::Tag::SINT, WordIO::store(s))
     }
     /// Store a new float. This function is always safe to call
     #[allow(non_snake_case)]
     fn Float(f: f64) -> Self {
-        Self::new(Self::Tag::FLOAT, SystemDword::store(f.to_bits()))
+        Self::new(Self::Tag::FLOAT, WordIO::store(f.to_bits()))
     }
     /// Store a new binary. This function is always safe to call
     #[allow(non_snake_case)]
     fn Bin(b: &[u8]) -> Self {
-        Self::new(Self::Tag::BIN, SystemDword::store((b.len(), b.as_ptr())))
+        Self::new(Self::Tag::BIN, WordIO::store((b.len(), b.as_ptr())))
     }
 
     /// Store a new string. Now, I won't talk about this one's safety because it depends on the implementor
@@ -117,7 +117,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // bool
     /// Load a bool (this is unsafe for logical verity)
     unsafe fn read_bool_uck(&self) -> bool {
-        self.data().dword_ld()
+        self.data().load()
     }
     /// Load a bool
     fn read_bool_try(&self) -> Option<bool> {
@@ -135,7 +135,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // uint
     /// Load a uint (this is unsafe for logical verity)
     unsafe fn read_uint_uck(&self) -> u64 {
-        self.data().dword_ld()
+        self.data().load()
     }
     /// Load a uint
     fn read_uint_try(&self) -> Option<u64> {
@@ -156,7 +156,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // sint
     /// Load a sint (unsafe for logical verity)
     unsafe fn read_sint_uck(&self) -> i64 {
-        self.data().dword_ld()
+        self.data().load()
     }
     /// Load a sint
     fn read_sint_try(&self) -> Option<i64> {
@@ -172,7 +172,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     // float
     /// Load a float (unsafe for logical verity)
     unsafe fn read_float_uck(&self) -> f64 {
-        self.data().dword_ld()
+        self.data().load()
     }
     /// Load a float
     fn read_float_try(&self) -> Option<f64> {
@@ -190,7 +190,7 @@ pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
     /// ## Safety
     /// Are you a binary? Did you store it correctly? Are you a victim of segfaults?
     unsafe fn read_bin_uck(&self) -> &[u8] {
-        let (l, p) = self.data().dword_ld();
+        let (l, p) = self.data().load();
         slice::from_raw_parts(p, l)
     }
     /// Load a bin
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index fa83aeda..d99940e0 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -35,7 +35,7 @@ mod tests;
 pub use astr::AStr;
 pub use uarray::UArray;
 pub use vinline::VInline;
-pub use word::{SystemDword, SystemQword, SystemTword, WordRW};
+pub use word::{DwordNN, DwordQN, QwordNNNN, TwordNNN, WordIO};
 // imports
 use std::alloc::{self, Layout};
 
@@ -58,7 +58,7 @@ pub struct SpecialPaddedWord {
 }
 
 impl SpecialPaddedWord {
-    const fn new(a: u64, b: usize) -> Self {
+    pub const unsafe fn new(a: u64, b: usize) -> Self {
         Self { a, b }
     }
 }
diff --git a/server/src/engine/mem/tests.rs b/server/src/engine/mem/tests/mod.rs
similarity index 99%
rename from server/src/engine/mem/tests.rs
rename to server/src/engine/mem/tests/mod.rs
index 60221b86..efe152a8 100644
--- a/server/src/engine/mem/tests.rs
+++ b/server/src/engine/mem/tests/mod.rs
@@ -25,6 +25,7 @@
 */
 
 use super::*;
+mod word;
 
 mod vinline {
     use super::VInline;
diff --git a/server/src/engine/mem/tests/word.rs b/server/src/engine/mem/tests/word.rs
new file mode 100644
index 00000000..d6efed52
--- /dev/null
+++ b/server/src/engine/mem/tests/word.rs
@@ -0,0 +1,145 @@
+/*
+ * Created on Mon Apr 24 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::engine::mem::{
+        word::{DwordQN, QwordNNNN, TwordNNN, WordIO, ZERO_BLOCK},
+        NativeDword, NativeQword, NativeTword, SpecialPaddedWord,
+    },
+    core::{slice, str},
+};
+
+fn wordld<T: PartialEq, W: WordIO<T>>(w: &W, x: T) -> (T, T) {
+    (w.load(), x)
+}
+
+macro_rules! assert_wordeq {
+    ($a:expr, $b:expr) => {{
+        let (a, b) = wordld(&$a, $b);
+        assert_eq!(a, b);
+    }};
+}
+
+macro_rules! assert_wordeq_minmax {
+    ($word:ty => $($ty:ty),* $(,)?; with $extramin:ident, $extramax:ident) => {{
+        $(
+            let x = <$word>::store(<$ty>::MIN); assert_wordeq!(x, <$ty>::MIN);
+            $extramin(&x);
+            let x = <$word>::store(<$ty>::MAX); assert_wordeq!(x, <$ty>::MAX);
+            $extramax(&x);
+        )*
+    }};
+}
+
+fn check_primitives<W>(extramin: impl Fn(&W), extramax: impl Fn(&W))
+where
+    W: WordIO<bool>
+        + WordIO<u8>
+        + WordIO<u16>
+        + WordIO<u32>
+        + WordIO<u64>
+        + WordIO<i8>
+        + WordIO<i16>
+        + WordIO<i32>
+        + WordIO<i64>
+        + WordIO<f32>
+        + WordIO<f64>
+        + WordIO<(usize, usize)>
+        + WordIO<(usize, *mut u8)>
+        + WordIO<(usize, *const u8)>,
+{
+    assert_wordeq_minmax!(W => u8, u16, u32, u64, i8, i16, i32, i64, f32, f64; with extramin, extramax);
+    // bool
+    let x = W::store(false);
+    assert_wordeq!(x, false);
+    extramin(&x);
+    let x = W::store(true);
+    assert_wordeq!(x, true);
+    extramax(&x);
+    // str
+    let str = "hello, world";
+    let x = W::store((str.len(), str.as_ptr()));
+    unsafe {
+        let (len, ptr) = x.load();
+        assert_eq!(
+            str,
+            str::from_utf8_unchecked(slice::from_raw_parts(ptr, len))
+        );
+    }
+    // string (mut)
+    let mut string = String::from("hello, world");
+    let x = W::store((string.len(), string.as_mut_ptr()));
+    unsafe {
+        let (len, ptr) = x.load();
+        assert_eq!(
+            string,
+            str::from_utf8_unchecked(slice::from_raw_parts(ptr, len))
+        );
+    }
+}
+
+#[test]
+fn dwordnn_all() {
+    check_primitives::<NativeDword>(|_| {}, |_| {});
+}
+
+#[test]
+fn dwordqn_all() {
+    check_primitives::<SpecialPaddedWord>(
+        |minword| {
+            let (_a, b) = minword.dwordqn_load_qw_nw();
+            assert_eq!(b, ZERO_BLOCK.as_ptr() as usize);
+        },
+        |maxword| {
+            let (_a, b) = maxword.dwordqn_load_qw_nw();
+            assert_eq!(b, ZERO_BLOCK.as_ptr() as usize);
+        },
+    );
+}
+
+#[test]
+fn twordnnn_all() {
+    check_primitives::<NativeTword>(|_| {}, |_| {});
+}
+
+#[test]
+fn qwordnnn_all() {
+    check_primitives::<NativeQword>(|_| {}, |_| {});
+}
+
+#[test]
+fn dwordqn_promotions() {
+    let x = SpecialPaddedWord::store(u64::MAX);
+    let y: NativeTword = x.dword_promote();
+    let (uint, usize) = y.dwordqn_load_qw_nw();
+    assert_eq!(uint, u64::MAX);
+    assert_eq!(usize, ZERO_BLOCK.as_ptr() as usize);
+    let z: NativeQword = y.tword_promote();
+    let (uint, usize_1, usize_2) = z.qwordnnnn_load_qw_nw_nw();
+    assert_eq!(uint, u64::MAX);
+    assert_eq!(usize_1, ZERO_BLOCK.as_ptr() as usize);
+    assert_eq!(usize_2, 0);
+}
diff --git a/server/src/engine/mem/word.rs b/server/src/engine/mem/word.rs
index 5cd7cb72..035236ba 100644
--- a/server/src/engine/mem/word.rs
+++ b/server/src/engine/mem/word.rs
@@ -24,9 +24,12 @@
  *
 */
 
-use super::{NativeDword, NativeQword, NativeTword, SpecialPaddedWord};
+use {
+    super::{NativeDword, NativeQword, NativeTword, SpecialPaddedWord},
+    core::mem::size_of,
+};
 
-static ZERO_BLOCK: [u8; 0] = [];
+pub static ZERO_BLOCK: [u8; 0] = [];
 
 #[cfg(target_pointer_width = "32")]
 fn quadsplit(q: u64) -> [usize; 2] {
@@ -43,353 +46,333 @@ fn quadmerge(v: [usize; 2]) -> u64 {
     }
 }
 
-/// Native quad pointer stack (must also be usable as a double and triple pointer stack. see [`SystemTword`] and [`SystemDword`])
-pub trait SystemQword: SystemTword {
-    fn store_full(a: usize, b: usize, c: usize, d: usize) -> Self;
-    fn load_quad(&self) -> [usize; 4];
-    fn store<'a, T>(v: T) -> Self
-    where
-        T: WordRW<Self>,
-    {
-        WordRW::store(v)
-    }
-    fn qword_ld<'a, T>(&'a self) -> T
-    where
-        T: WordRW<Self, Target<'a> = T>,
-    {
-        <T>::load(self)
+pub trait WordIO<T> {
+    fn store(v: T) -> Self;
+    fn load(&self) -> T;
+}
+
+/*
+    dword
+    ---
+    kinds: NN (word * 2), QN (qword, word)
+    promotions: QN -> NNN
+*/
+
+pub trait DwordNN: Sized {
+    const DWORDNN_FROM_UPPER: bool = size_of::<Self>() > size_of::<[usize; 2]>();
+    fn dwordnn_store_native_full(a: usize, b: usize) -> Self;
+    fn dwordnn_store_qw(a: u64) -> Self {
+        debug_assert!(!Self::DWORDNN_FROM_UPPER, "NEED TO OVERRIDE STORE");
+        #[cfg(target_pointer_width = "32")]
+        {
+            let [a, b] = quadsplit(a);
+            Self::dwordnn_store_native_full(a, b)
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            Self::dwordnn_store_native_full(a as usize, 0)
+        }
+    }
+    fn dwordnn_load_native_full(&self) -> [usize; 2];
+    fn dwordnn_load_qw(&self) -> u64 {
+        debug_assert!(!Self::DWORDNN_FROM_UPPER, "NEED TO OVERRIDE LOAD");
+        #[cfg(target_pointer_width = "32")]
+        {
+            quadmerge(self.dwordnn_load_native_full())
+        }
+        #[cfg(target_pointer_width = "64")]
+        {
+            self.dwordnn_load_native_full()[0] as u64
+        }
     }
 }
 
-/// Native tripe pointer stack (must also be usable as a double pointer stack, see [`SystemDword`])
-pub trait SystemTword: SystemDword {
-    /// Store a quad and a native word
-    fn store_qw_nw(a: u64, b: usize) -> Self;
-    /// Load a quad and a native word
-    fn load_qw_nw(&self) -> (u64, usize);
-    fn store_full(a: usize, b: usize, c: usize) -> Self;
-    fn load_triple(&self) -> [usize; 3];
-    fn store<'a, T>(v: T) -> Self
-    where
-        T: WordRW<Self>,
-    {
-        WordRW::store(v)
-    }
-    fn tword_ld<'a, T>(&'a self) -> T
-    where
-        T: WordRW<Self, Target<'a> = T>,
-    {
-        <T>::load(self)
+pub trait DwordQN: Sized {
+    const DWORDQN_FROM_UPPER: bool = size_of::<Self>() > size_of::<(u64, usize)>();
+    fn dwordqn_store_qw_nw(a: u64, b: usize) -> Self;
+    fn dwordqn_load_qw_nw(&self) -> (u64, usize);
+    // overrides
+    fn overridable_dwordnn_store_qw(a: u64) -> Self {
+        Self::dwordqn_store_qw_nw(a, 0)
+    }
+    // promotions
+    fn dword_promote<W: DwordQN>(&self) -> W {
+        let (a, b) = self.dwordqn_load_qw_nw();
+        <W as DwordQN>::dwordqn_store_qw_nw(a, b)
     }
 }
 
-/// Native double pointer stack
-pub trait SystemDword: Sized {
-    fn store_qw(u: u64) -> Self;
-    fn store_fat(a: usize, b: usize) -> Self;
-    fn load_qw(&self) -> u64;
-    fn load_double(&self) -> [usize; 2];
-    fn store<'a, T>(v: T) -> Self
-    where
-        T: WordRW<Self>,
-    {
-        WordRW::store(v)
-    }
-    fn dword_ld<'a, T>(&'a self) -> T
-    where
-        T: WordRW<Self, Target<'a> = T>,
-    {
-        <T>::load(self)
+/*
+    dword: blanket impls
+*/
+
+impl<T: DwordQN> DwordNN for T {
+    fn dwordnn_store_native_full(a: usize, b: usize) -> Self {
+        Self::dwordqn_store_qw_nw(a as u64, b)
+    }
+    fn dwordnn_store_qw(a: u64) -> Self {
+        Self::overridable_dwordnn_store_qw(a)
+    }
+    fn dwordnn_load_native_full(&self) -> [usize; 2] {
+        let (a, b) = self.dwordqn_load_qw_nw();
+        debug_assert!(a <= usize::MAX as u64, "overflowed with: `{}`", a);
+        [a as usize, b]
+    }
+    fn dwordnn_load_qw(&self) -> u64 {
+        DwordQN::dwordqn_load_qw_nw(self).0
     }
 }
 
-impl SystemDword for SpecialPaddedWord {
-    fn store_qw(u: u64) -> Self {
-        Self::new(u, ZERO_BLOCK.as_ptr() as usize)
+/*
+    dword: impls
+*/
+
+impl DwordNN for NativeDword {
+    fn dwordnn_store_native_full(a: usize, b: usize) -> Self {
+        Self([a, b])
     }
-    fn store_fat(a: usize, b: usize) -> Self {
-        Self::new(a as u64, b)
+    fn dwordnn_load_native_full(&self) -> [usize; 2] {
+        self.0
     }
-    fn load_qw(&self) -> u64 {
-        self.a
+}
+
+impl DwordQN for SpecialPaddedWord {
+    fn dwordqn_store_qw_nw(a: u64, b: usize) -> Self {
+        unsafe {
+            // UNSAFE(@ohsayan): valid construction
+            Self::new(a, b)
+        }
+    }
+    fn dwordqn_load_qw_nw(&self) -> (u64, usize) {
+        (self.a, self.b)
+    }
+    // overrides
+    fn overridable_dwordnn_store_qw(a: u64) -> Self {
+        unsafe {
+            // UNSAFE(@ohsayan): valid construction
+            Self::new(a, ZERO_BLOCK.as_ptr() as usize)
+        }
     }
-    fn load_double(&self) -> [usize; 2] {
-        [self.a as usize, self.b]
+}
+
+/*
+    tword
+    ---
+    kinds: NNN (word * 3)
+    promotions: NNN -> NNNN
+*/
+
+pub trait TwordNNN: Sized {
+    const TWORDNNN_FROM_UPPER: bool = size_of::<Self>() > size_of::<[usize; 3]>();
+    fn twordnnn_store_native_full(a: usize, b: usize, c: usize) -> Self;
+    fn twordnnn_load_native_full(&self) -> [usize; 3];
+    // promotions
+    fn tword_promote<W: TwordNNN>(&self) -> W {
+        let [a, b, c] = self.twordnnn_load_native_full();
+        <W as TwordNNN>::twordnnn_store_native_full(a, b, c)
     }
 }
 
-impl SystemDword for NativeDword {
-    #[inline(always)]
-    fn store_qw(u: u64) -> Self {
-        let x;
+/*
+    tword: blanket impls
+*/
+
+impl<T: TwordNNN> DwordQN for T {
+    fn dwordqn_store_qw_nw(a: u64, b: usize) -> Self {
         #[cfg(target_pointer_width = "32")]
         {
-            x = quadsplit(u);
+            let [qw_1, qw_2] = quadsplit(a);
+            Self::twordnnn_store_native_full(qw_1, qw_2, b)
         }
         #[cfg(target_pointer_width = "64")]
         {
-            x = [u as usize, 0]
+            Self::twordnnn_store_native_full(a as usize, b, 0)
         }
-        Self(x)
-    }
-    #[inline(always)]
-    fn store_fat(a: usize, b: usize) -> Self {
-        Self([a, b])
     }
-    #[inline(always)]
-    fn load_qw(&self) -> u64 {
-        let x;
+    fn dwordqn_load_qw_nw(&self) -> (u64, usize) {
         #[cfg(target_pointer_width = "32")]
         {
-            x = quadmerge(self.0);
+            let [w1, w2, b] = self.twordnnn_load_native_full();
+            (quadmerge([w1, w2]), b)
         }
         #[cfg(target_pointer_width = "64")]
         {
-            x = self.0[0] as _;
+            let [a, b, _] = self.twordnnn_load_native_full();
+            (a as u64, b)
         }
-        x
-    }
-    #[inline(always)]
-    fn load_double(&self) -> [usize; 2] {
-        self.0
     }
 }
 
-impl SystemTword for NativeTword {
-    #[inline(always)]
-    fn store_full(a: usize, b: usize, c: usize) -> Self {
+/*
+    tword: impls
+*/
+
+impl TwordNNN for NativeTword {
+    fn twordnnn_store_native_full(a: usize, b: usize, c: usize) -> Self {
         Self([a, b, c])
     }
-    #[inline(always)]
-    fn load_triple(&self) -> [usize; 3] {
+    fn twordnnn_load_native_full(&self) -> [usize; 3] {
         self.0
     }
-    #[inline(always)]
-    fn store_qw_nw(a: u64, b: usize) -> Self {
-        let ret;
+}
+
+/*
+    qword
+    ---
+    kinds: NNNN (word * 4)
+    promotions: N/A
+*/
+
+pub trait QwordNNNN: Sized {
+    const QWORDNNNN_FROM_UPPER: bool = size_of::<Self>() > size_of::<[usize; 4]>();
+    fn qwordnnnn_store_native_full(a: usize, b: usize, c: usize, d: usize) -> Self;
+    fn qwordnnnn_store_qw_qw(a: u64, b: u64) -> Self {
         #[cfg(target_pointer_width = "32")]
         {
-            let [qw_1, qw_2] = quadsplit(a);
-            ret = [qw_1, qw_2, b];
+            let [qw1_a, qw1_b] = quadsplit(a);
+            let [qw2_a, qw2_b] = quadsplit(b);
+            Self::qwordnnnn_store_native_full(qw1_a, qw1_b, qw2_a, qw2_b)
         }
         #[cfg(target_pointer_width = "64")]
         {
-            ret = [a as usize, b, 0];
+            Self::qwordnnnn_store_native_full(a as usize, b as usize, 0, 0)
         }
-        Self(ret)
     }
-    #[inline(always)]
-    fn load_qw_nw(&self) -> (u64, usize) {
-        let ret;
+    fn qwordnnnn_store_qw_nw_nw(a: u64, b: usize, c: usize) -> Self {
         #[cfg(target_pointer_width = "32")]
         {
-            let qw = quadmerge([self.0[0], self.0[1]]);
-            let nw = self.0[2];
-            ret = (qw, nw);
+            let [qw_a, qw_b] = quadsplit(a);
+            Self::qwordnnnn_store_native_full(qw_a, qw_b, b, c)
         }
         #[cfg(target_pointer_width = "64")]
         {
-            ret = (self.0[0] as u64, self.0[1]);
+            Self::qwordnnnn_store_native_full(a as usize, b, c, 0)
         }
-        ret
     }
-}
-
-impl SystemDword for NativeTword {
-    #[inline(always)]
-    fn store_qw(u: u64) -> Self {
-        let x;
+    fn qwordnnnn_load_native_full(&self) -> [usize; 4];
+    fn qwordnnnn_load_qw_qw(&self) -> [u64; 2] {
+        let [a, b, c, d] = self.qwordnnnn_load_native_full();
         #[cfg(target_pointer_width = "32")]
         {
-            let [a, b]: [usize; 2] = quadsplit(u);
-            x = [a, b, 0];
+            [quadmerge([a, b]), quadmerge([c, d])]
         }
         #[cfg(target_pointer_width = "64")]
         {
-            x = [u as _, 0, 0];
+            let _ = (c, d);
+            [a as u64, b as u64]
         }
-        Self(x)
     }
-    #[inline(always)]
-    fn store_fat(a: usize, b: usize) -> Self {
-        Self([a, b, 0])
-    }
-    #[inline(always)]
-    fn load_qw(&self) -> u64 {
-        let x;
+    fn qwordnnnn_load_qw_nw_nw(&self) -> (u64, usize, usize) {
+        let [a, b, c, d] = self.qwordnnnn_load_native_full();
         #[cfg(target_pointer_width = "32")]
         {
-            x = quadmerge([self.0[0], self.0[1]]);
+            (quadmerge([a, b]), c, d)
         }
         #[cfg(target_pointer_width = "64")]
         {
-            x = self.0[0] as _;
+            let _ = d;
+            (a as u64, b, c)
         }
-        x
     }
-    #[inline(always)]
-    fn load_double(&self) -> [usize; 2] {
-        [self.0[0], self.0[1]]
+}
+
+/*
+    qword: blanket impls
+*/
+
+impl<T: QwordNNNN> TwordNNN for T {
+    fn twordnnn_store_native_full(a: usize, b: usize, c: usize) -> Self {
+        Self::qwordnnnn_store_native_full(a, b, c, 0)
+    }
+    fn twordnnn_load_native_full(&self) -> [usize; 3] {
+        let [a, b, c, _] = self.qwordnnnn_load_native_full();
+        [a, b, c]
     }
 }
 
-impl SystemQword for NativeQword {
-    fn store_full(a: usize, b: usize, c: usize, d: usize) -> Self {
+/*
+    qword: impls
+*/
+
+impl QwordNNNN for NativeQword {
+    fn qwordnnnn_store_native_full(a: usize, b: usize, c: usize, d: usize) -> Self {
         Self([a, b, c, d])
     }
-    fn load_quad(&self) -> [usize; 4] {
+    fn qwordnnnn_load_native_full(&self) -> [usize; 4] {
         self.0
     }
 }
 
-impl SystemTword for NativeQword {
-    fn store_full(a: usize, b: usize, c: usize) -> Self {
-        Self([a, b, c, 0])
+/*
+    impls: WordIO
+*/
+
+macro_rules! impl_numeric_io {
+    ($trait:ident => { $($ty:ty),* $(,)? }) => {
+        $(impl<T: $trait> WordIO<$ty> for T {
+            fn store(v: $ty) -> Self { Self::dwordnn_store_qw(v as _) }
+            fn load(&self) -> $ty { self.dwordnn_load_qw() as _ }
+        })*
     }
-    fn load_triple(&self) -> [usize; 3] {
-        [self.0[0], self.0[1], self.0[2]]
+}
+
+impl_numeric_io!(DwordNN => { u8, u16, u32, u64, i8, i16, i32, i64 });
+
+impl<T: DwordNN> WordIO<bool> for T {
+    fn store(v: bool) -> Self {
+        Self::dwordnn_store_qw(v as _)
     }
-    /// Store a quadword and a native word
-    fn store_qw_nw(a: u64, b: usize) -> Self {
-        let ret;
-        #[cfg(target_pointer_width = "32")]
-        {
-            let [qw_1, qw_2] = quadsplit(a);
-            ret = [qw_1, qw_2, b, 0];
-        }
-        #[cfg(target_pointer_width = "64")]
-        {
-            ret = [a as usize, b, 0, 0];
-        }
-        Self(ret)
-    }
-    #[inline(always)]
-    fn load_qw_nw(&self) -> (u64, usize) {
-        let ret;
-        #[cfg(target_pointer_width = "32")]
-        {
-            let qw = quadmerge([self.0[0], self.0[1]]);
-            let nw = self.0[2];
-            ret = (qw, nw);
-        }
-        #[cfg(target_pointer_width = "64")]
-        {
-            ret = (self.0[0] as u64, self.0[1]);
-        }
-        ret
+    fn load(&self) -> bool {
+        self.dwordnn_load_qw() == 1
     }
 }
 
-impl SystemDword for NativeQword {
-    fn store_qw(u: u64) -> Self {
-        let ret;
-        #[cfg(target_pointer_width = "32")]
-        {
-            let [a, b] = quadsplit(u);
-            ret = <Self as SystemQword>::store_full(a, b, 0, 0);
-        }
-        #[cfg(target_pointer_width = "64")]
-        {
-            ret = <Self as SystemQword>::store_full(u as _, 0, 0, 0);
-        }
-        ret
-    }
-    fn store_fat(a: usize, b: usize) -> Self {
-        <Self as SystemQword>::store_full(a, b, 0, 0)
+macro_rules! impl_float_io {
+    ($($float:ty),* $(,)?) => {
+        $(impl<T: DwordNN> WordIO<$float> for T {
+            fn store(v: $float) -> Self { Self::dwordnn_store_qw(v.to_bits() as u64) }
+            fn load(&self) -> $float { <$float>::from_bits(self.dwordnn_load_qw() as _) }
+        })*
     }
-    fn load_qw(&self) -> u64 {
-        let ret;
-        #[cfg(target_pointer_width = "32")]
-        {
-            ret = quadmerge([self.0[0], self.0[1]]);
-        }
-        #[cfg(target_pointer_width = "64")]
-        {
-            ret = self.0[0] as _;
-        }
-        ret
+}
+
+impl_float_io!(f32, f64);
+
+impl<T: DwordNN> WordIO<(usize, usize)> for T {
+    fn store((a, b): (usize, usize)) -> Self {
+        Self::dwordnn_store_native_full(a, b)
     }
-    fn load_double(&self) -> [usize; 2] {
-        [self.0[0], self.0[1]]
+    fn load(&self) -> (usize, usize) {
+        let [a, b] = self.dwordnn_load_native_full();
+        (a, b)
     }
 }
 
-pub trait WordRW<W> {
-    type Target<'a>
-    where
-        W: 'a;
-    fn store(self) -> W;
-    fn load<'a>(word: &'a W) -> Self::Target<'a>;
+impl<T: DwordNN> WordIO<[usize; 2]> for T {
+    fn store([a, b]: [usize; 2]) -> Self {
+        Self::dwordnn_store_native_full(a, b)
+    }
+    fn load(&self) -> [usize; 2] {
+        self.dwordnn_load_native_full()
+    }
 }
 
-macro_rules! impl_wordrw {
-	($($ty:ty as $minword:ident => { type Target<'a> = $target:ty; |$selfname:ident| $store:expr; |$wordarg:ident| $load:expr;})*) => {
-		$(impl<W: $minword> WordRW<W> for $ty { type Target<'a> = $target where W: 'a; fn store($selfname: Self) -> W { $store } fn load<'a>($wordarg: &'a W) -> Self::Target<'a> { $load } })*
-	};
-	($($ty:ty as $minword:ident => { |$selfname:ident| $store:expr; |$wordarg:ident| $load:expr;})*) => { impl_wordrw!($($ty as $minword => { type Target<'a> = $ty; |$selfname| $store; |$wordarg| $load;})*); };
+impl<T: DwordNN> WordIO<(usize, *mut u8)> for T {
+    fn store((a, b): (usize, *mut u8)) -> Self {
+        Self::dwordnn_store_native_full(a, b as usize)
+    }
+    fn load(&self) -> (usize, *mut u8) {
+        let [a, b] = self.dwordnn_load_native_full();
+        (a, b as *mut u8)
+    }
 }
 
-impl_wordrw! {
-    bool as SystemDword => {
-        |self| SystemDword::store_qw(self as _);
-        |word| SystemDword::load_qw(word) == 1;
-    }
-    u8 as SystemDword => {
-        |self| SystemDword::store_qw(self as _);
-        |word| SystemDword::load_qw(word) as u8;
-    }
-    u16 as SystemDword => {
-        |self| SystemDword::store_qw(self as _);
-        |word| SystemDword::load_qw(word) as u16;
-    }
-    u32 as SystemDword => {
-        |self| SystemDword::store_qw(self as _);
-        |word| SystemDword::load_qw(word) as u32;
-    }
-    u64 as SystemDword => {
-        |self| SystemDword::store_qw(self);
-        |word| SystemDword::load_qw(word);
-    }
-    i8 as SystemDword => {
-        |self| SystemDword::store_qw(self as _);
-        |word| SystemDword::load_qw(word) as i8;
-    }
-    i16 as SystemDword => {
-        |self| SystemDword::store_qw(self as _);
-        |word| SystemDword::load_qw(word) as i16;
-    }
-    i32 as SystemDword => {
-        |self| SystemDword::store_qw(self as _);
-        |word| SystemDword::load_qw(word) as i32;
-    }
-    i64 as SystemDword => {
-        |self| SystemDword::store_qw(self as _);
-        |word| SystemDword::load_qw(word) as i64;
-    }
-    f32 as SystemDword => {
-        |self| SystemDword::store_qw(self.to_bits() as u64);
-        |word| f32::from_bits(SystemDword::load_qw(word) as u32);
-    }
-    f64 as SystemDword => {
-        |self| SystemDword::store_qw(self.to_bits());
-        |word| f64::from_bits(SystemDword::load_qw(word));
-    }
-    [usize; 2] as SystemDword => {
-        |self| SystemDword::store_fat(self[0], self[1]);
-        |word| SystemDword::load_double(word);
-    }
-    (usize, *mut u8) as SystemDword => {
-        |self| SystemDword::store_fat(self.0, self.1 as usize);
-        |word| {
-            let [a, b] = word.load_double();
-            (a, b as *mut u8)
-        };
-    }
-    (usize, *const u8) as SystemDword => {
-        |self| SystemDword::store_fat(self.0, self.1 as usize);
-        |word| {
-            let [a, b] = word.load_double();
-            (a, b as *const u8)
-        };
+impl<T: DwordNN> WordIO<(usize, *const u8)> for T {
+    fn store((a, b): (usize, *const u8)) -> Self {
+        Self::dwordnn_store_native_full(a, b as usize)
+    }
+    fn load(&self) -> (usize, *const u8) {
+        let [a, b] = self.dwordnn_load_native_full();
+        (a, b as *const u8)
     }
 }

From e2112c8bc1c0fe976f492330ce4322f7f0af530c Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 25 Apr 2023 00:57:15 -0700
Subject: [PATCH 176/310] Fix error causing dangling block

---
 server/src/engine/core/index/key.rs  |  6 ++++++
 server/src/engine/core/model/cell.rs | 24 ++++++++++++++++++------
 server/src/engine/mem/mod.rs         |  2 +-
 server/src/util/mod.rs               | 10 ++--------
 4 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
index 4090c380..2b42fabc 100644
--- a/server/src/engine/core/index/key.rs
+++ b/server/src/engine/core/index/key.rs
@@ -110,6 +110,9 @@ impl PrimaryIndexKey {
             dc.as_raw()
         }
         .dwordqn_load_qw_nw();
+        if cfg!(debug_assertions) && tag < TagUnique::Bin {
+            assert_eq!(b, mem::ZERO_BLOCK.as_ptr() as usize);
+        }
         Self {
             tag,
             data: unsafe {
@@ -131,6 +134,9 @@ impl PrimaryIndexKey {
     }
     fn __compute_vdata_offset(&self) -> [usize; 2] {
         let (len, data) = self.data.dwordqn_load_qw_nw();
+        if cfg!(debug_assertions) && self.tag < TagUnique::Bin {
+            assert_eq!(data, mem::ZERO_BLOCK.as_ptr() as usize);
+        }
         let actual_len = (len as usize) * (self.tag >= TagUnique::Bin) as usize;
         [data, actual_len]
     }
diff --git a/server/src/engine/core/model/cell.rs b/server/src/engine/core/model/cell.rs
index effd8cfc..69c5d0e3 100644
--- a/server/src/engine/core/model/cell.rs
+++ b/server/src/engine/core/model/cell.rs
@@ -32,7 +32,7 @@ use {
             spec::{Dataspec1D, DataspecMeta1D},
             tag::{CUTag, DataTag, TagClass},
         },
-        mem::{DwordNN, DwordQN, NativeQword, WordIO},
+        mem::{DwordNN, DwordQN, NativeQword, SpecialPaddedWord, WordIO},
     },
     core::{fmt, mem, mem::ManuallyDrop, slice, str},
     parking_lot::RwLock,
@@ -49,7 +49,10 @@ impl Datacell {
     pub fn new_bool(b: bool) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(CUTag::BOOL, DataRaw::word(WordIO::store(b)))
+            Self::new(
+                CUTag::BOOL,
+                DataRaw::word(SpecialPaddedWord::store(b).dword_promote()),
+            )
         }
     }
     pub unsafe fn read_bool(&self) -> bool {
@@ -68,7 +71,10 @@ impl Datacell {
     pub fn new_uint(u: u64) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(CUTag::UINT, DataRaw::word(WordIO::store(u)))
+            Self::new(
+                CUTag::UINT,
+                DataRaw::word(SpecialPaddedWord::store(u).dword_promote()),
+            )
         }
     }
     pub unsafe fn read_uint(&self) -> u64 {
@@ -84,10 +90,13 @@ impl Datacell {
         self.try_uint().unwrap()
     }
     // sint
-    pub fn new_sint(u: i64) -> Self {
+    pub fn new_sint(i: i64) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(CUTag::SINT, DataRaw::word(WordIO::store(u)))
+            Self::new(
+                CUTag::SINT,
+                DataRaw::word(SpecialPaddedWord::store(i).dword_promote()),
+            )
         }
     }
     pub unsafe fn read_sint(&self) -> i64 {
@@ -106,7 +115,10 @@ impl Datacell {
     pub fn new_float(f: f64) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(CUTag::FLOAT, DataRaw::word(WordIO::store(f)))
+            Self::new(
+                CUTag::FLOAT,
+                DataRaw::word(SpecialPaddedWord::store(f).dword_promote()),
+            )
         }
     }
     pub unsafe fn read_float(&self) -> f64 {
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index d99940e0..4c318e9d 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -35,7 +35,7 @@ mod tests;
 pub use astr::AStr;
 pub use uarray::UArray;
 pub use vinline::VInline;
-pub use word::{DwordNN, DwordQN, QwordNNNN, TwordNNN, WordIO};
+pub use word::{DwordNN, DwordQN, QwordNNNN, TwordNNN, WordIO, ZERO_BLOCK};
 // imports
 use std::alloc::{self, Layout};
 
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index 42d8deec..042c74a8 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -24,8 +24,6 @@
  *
 */
 
-use std::mem::{self, ManuallyDrop};
-
 #[macro_use]
 mod macros;
 pub mod compiler;
@@ -41,7 +39,7 @@ use {
     core::{
         fmt::{self, Debug},
         marker::PhantomData,
-        mem::MaybeUninit,
+        mem::{self, MaybeUninit},
         ops::Deref,
     },
     std::process,
@@ -52,11 +50,7 @@ pub const IS_ON_CI: bool = option_env!("CI").is_some();
 const EXITCODE_ONE: i32 = 0x01;
 
 pub fn bx_to_vec<T>(bx: Box<[T]>) -> Vec<T> {
-    let mut md = ManuallyDrop::new(bx);
-    // damn you, miri
-    let ptr = md.as_mut_ptr() as usize as *mut T;
-    let len = md.len();
-    unsafe { Vec::from_raw_parts(ptr, len, len) }
+    Vec::from(bx)
 }
 
 /// # Unsafe unwrapping

From 9297095f45ce9b112596910a24177157d4fa4b99 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 26 Apr 2023 08:10:59 -0700
Subject: [PATCH 177/310] Add spl case tests

---
 server/src/engine/core/model/cell.rs | 10 +++++-----
 server/src/engine/mem/tests/word.rs  | 27 ++++++++++++++++++++++++++-
 server/src/engine/mem/word.rs        |  2 +-
 3 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/server/src/engine/core/model/cell.rs b/server/src/engine/core/model/cell.rs
index 69c5d0e3..ae5e2385 100644
--- a/server/src/engine/core/model/cell.rs
+++ b/server/src/engine/core/model/cell.rs
@@ -51,7 +51,7 @@ impl Datacell {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
                 CUTag::BOOL,
-                DataRaw::word(SpecialPaddedWord::store(b).dword_promote()),
+                DataRaw::word(SpecialPaddedWord::store(b).dwordqn_promote()),
             )
         }
     }
@@ -73,7 +73,7 @@ impl Datacell {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
                 CUTag::UINT,
-                DataRaw::word(SpecialPaddedWord::store(u).dword_promote()),
+                DataRaw::word(SpecialPaddedWord::store(u).dwordqn_promote()),
             )
         }
     }
@@ -95,7 +95,7 @@ impl Datacell {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
                 CUTag::SINT,
-                DataRaw::word(SpecialPaddedWord::store(i).dword_promote()),
+                DataRaw::word(SpecialPaddedWord::store(i).dwordqn_promote()),
             )
         }
     }
@@ -117,7 +117,7 @@ impl Datacell {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
                 CUTag::FLOAT,
-                DataRaw::word(SpecialPaddedWord::store(f).dword_promote()),
+                DataRaw::word(SpecialPaddedWord::store(f).dwordqn_promote()),
             )
         }
     }
@@ -228,7 +228,7 @@ impl<'a> From<LitIR<'a>> for Datacell {
                 Datacell::new(
                     CUTag::from(l.kind()),
                     // DO NOT RELY ON the payload's bit pattern; it's padded
-                    DataRaw::word(l.data().dword_promote()),
+                    DataRaw::word(l.data().dwordqn_promote()),
                 )
             },
             TagClass::Bin | TagClass::Str => unsafe {
diff --git a/server/src/engine/mem/tests/word.rs b/server/src/engine/mem/tests/word.rs
index d6efed52..50a41a41 100644
--- a/server/src/engine/mem/tests/word.rs
+++ b/server/src/engine/mem/tests/word.rs
@@ -133,7 +133,7 @@ fn qwordnnn_all() {
 #[test]
 fn dwordqn_promotions() {
     let x = SpecialPaddedWord::store(u64::MAX);
-    let y: NativeTword = x.dword_promote();
+    let y: NativeTword = x.dwordqn_promote();
     let (uint, usize) = y.dwordqn_load_qw_nw();
     assert_eq!(uint, u64::MAX);
     assert_eq!(usize, ZERO_BLOCK.as_ptr() as usize);
@@ -143,3 +143,28 @@ fn dwordqn_promotions() {
     assert_eq!(usize_1, ZERO_BLOCK.as_ptr() as usize);
     assert_eq!(usize_2, 0);
 }
+
+fn eval_special_case(x: SpecialPaddedWord, qw: u64, nw: usize) {
+    let y: NativeQword = x.dwordqn_promote();
+    assert_eq!(y.dwordqn_load_qw_nw(), (qw, nw));
+    let z: SpecialPaddedWord = unsafe {
+        let (a, b) = y.dwordqn_load_qw_nw();
+        SpecialPaddedWord::new(a, b)
+    };
+    assert_eq!(z.dwordqn_load_qw_nw(), (qw, nw));
+}
+
+#[test]
+fn dwordqn_special_case_ldpk() {
+    let hello = "hello, world";
+    eval_special_case(
+        SpecialPaddedWord::store((hello.len(), hello.as_ptr())),
+        hello.len() as u64,
+        hello.as_ptr() as usize,
+    );
+    eval_special_case(
+        SpecialPaddedWord::store(u64::MAX),
+        u64::MAX,
+        ZERO_BLOCK.as_ptr() as usize,
+    );
+}
diff --git a/server/src/engine/mem/word.rs b/server/src/engine/mem/word.rs
index 035236ba..8e77a87c 100644
--- a/server/src/engine/mem/word.rs
+++ b/server/src/engine/mem/word.rs
@@ -96,7 +96,7 @@ pub trait DwordQN: Sized {
         Self::dwordqn_store_qw_nw(a, 0)
     }
     // promotions
-    fn dword_promote<W: DwordQN>(&self) -> W {
+    fn dwordqn_promote<W: DwordQN>(&self) -> W {
         let (a, b) = self.dwordqn_load_qw_nw();
         <W as DwordQN>::dwordqn_store_qw_nw(a, b)
     }

From ac2ec6f71a18a2a79dc33369adc18157dc1a7897 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 27 Apr 2023 07:55:35 -0700
Subject: [PATCH 178/310] Move dc into data

---
 server/src/engine/core/index/key.rs            | 2 +-
 server/src/engine/core/mod.rs                  | 2 --
 server/src/engine/core/model/mod.rs            | 7 ++++---
 server/src/engine/core/tests/model/layer.rs    | 2 +-
 server/src/engine/core/tests/space/alter.rs    | 2 +-
 server/src/engine/core/tests/space/create.rs   | 2 +-
 server/src/engine/{core/model => data}/cell.rs | 0
 server/src/engine/data/md_dict.rs              | 6 ++++--
 server/src/engine/data/mod.rs                  | 1 +
 server/src/engine/data/tests/md_dict_tests.rs  | 6 +++---
 server/src/engine/ql/ast/mod.rs                | 3 +--
 server/src/engine/ql/dml/ins.rs                | 2 +-
 server/src/engine/ql/tests.rs                  | 2 +-
 13 files changed, 19 insertions(+), 18 deletions(-)
 rename server/src/engine/{core/model => data}/cell.rs (100%)

diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
index 2b42fabc..7037f405 100644
--- a/server/src/engine/core/index/key.rs
+++ b/server/src/engine/core/index/key.rs
@@ -28,8 +28,8 @@
 use crate::{engine::data::spec::Dataspec1D, util::test_utils};
 use {
     crate::engine::{
-        core::model::cell::Datacell,
         data::{
+            cell::Datacell,
             lit::LitIR,
             spec::DataspecMeta1D,
             tag::{DataTag, TagUnique},
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 29d2f965..57033691 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -41,8 +41,6 @@ use {
     },
     parking_lot::RwLock,
 };
-// re-exports
-pub use model::cell::Datacell;
 
 /// Use this for now since it substitutes for a file lock (and those syscalls are expensive),
 /// but something better is in the offing
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 99c4c023..83cb0074 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -25,7 +25,6 @@
 */
 
 pub(super) mod alt;
-pub mod cell;
 
 #[cfg(test)]
 use std::cell::RefCell;
@@ -34,8 +33,10 @@ use super::util::EntityLocator;
 
 use {
     crate::engine::{
-        core::model::cell::Datacell,
-        data::tag::{DataTag, FullTag, TagClass, TagSelector},
+        data::{
+            cell::Datacell,
+            tag::{DataTag, FullTag, TagClass, TagSelector},
+        },
         error::{DatabaseError, DatabaseResult},
         idx::{IndexSTSeqCns, STIndex, STIndexSeq},
         mem::VInline,
diff --git a/server/src/engine/core/tests/model/layer.rs b/server/src/engine/core/tests/model/layer.rs
index 626940fa..10ebf68f 100644
--- a/server/src/engine/core/tests/model/layer.rs
+++ b/server/src/engine/core/tests/model/layer.rs
@@ -80,7 +80,7 @@ mod layer_spec_validation {
 mod layer_data_validation {
     use {
         super::{layerview, layerview_nullable},
-        crate::engine::core::model::{self, cell::Datacell},
+        crate::engine::{core::model, data::cell::Datacell},
     };
     #[test]
     fn bool() {
diff --git a/server/src/engine/core/tests/space/alter.rs b/server/src/engine/core/tests/space/alter.rs
index 8c18f6ee..190d4fa8 100644
--- a/server/src/engine/core/tests/space/alter.rs
+++ b/server/src/engine/core/tests/space/alter.rs
@@ -26,10 +26,10 @@
 
 use crate::engine::{
     core::{
-        model::cell::Datacell,
         space::{Space, SpaceMeta},
         GlobalNS,
     },
+    data::cell::Datacell,
     error::DatabaseError,
 };
 
diff --git a/server/src/engine/core/tests/space/create.rs b/server/src/engine/core/tests/space/create.rs
index 0170add2..707a6305 100644
--- a/server/src/engine/core/tests/space/create.rs
+++ b/server/src/engine/core/tests/space/create.rs
@@ -26,10 +26,10 @@
 
 use crate::engine::{
     core::{
-        model::cell::Datacell,
         space::{Space, SpaceMeta},
         GlobalNS,
     },
+    data::cell::Datacell,
     error::DatabaseError,
 };
 
diff --git a/server/src/engine/core/model/cell.rs b/server/src/engine/data/cell.rs
similarity index 100%
rename from server/src/engine/core/model/cell.rs
rename to server/src/engine/data/cell.rs
diff --git a/server/src/engine/data/md_dict.rs b/server/src/engine/data/md_dict.rs
index dcff3aff..bb06af98 100644
--- a/server/src/engine/data/md_dict.rs
+++ b/server/src/engine/data/md_dict.rs
@@ -26,8 +26,10 @@
 
 use {
     crate::engine::{
-        core::Datacell,
-        data::lit::{Lit, LitIR},
+        data::{
+            cell::Datacell,
+            lit::{Lit, LitIR},
+        },
         idx::STIndex,
     },
     std::collections::HashMap,
diff --git a/server/src/engine/data/mod.rs b/server/src/engine/data/mod.rs
index f58ba49a..499f6092 100644
--- a/server/src/engine/data/mod.rs
+++ b/server/src/engine/data/mod.rs
@@ -26,6 +26,7 @@
 
 #[macro_use]
 mod macros;
+pub mod cell;
 pub mod lit;
 pub mod md_dict;
 pub mod spec;
diff --git a/server/src/engine/data/tests/md_dict_tests.rs b/server/src/engine/data/tests/md_dict_tests.rs
index ffd07576..467cc862 100644
--- a/server/src/engine/data/tests/md_dict_tests.rs
+++ b/server/src/engine/data/tests/md_dict_tests.rs
@@ -24,9 +24,9 @@
  *
 */
 
-use crate::engine::{
-    core::Datacell,
-    data::md_dict::{self, DictEntryGeneric, DictGeneric, MetaDict, MetaDictEntry},
+use crate::engine::data::{
+    cell::Datacell,
+    md_dict::{self, DictEntryGeneric, DictGeneric, MetaDict, MetaDictEntry},
 };
 
 #[test]
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 296f33a2..9d27d045 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -36,8 +36,7 @@ use {
     },
     crate::{
         engine::{
-            core::Datacell,
-            data::lit::LitIR,
+            data::{cell::Datacell, lit::LitIR},
             error::{LangError, LangResult},
         },
         util::{compiler, MaybeInit},
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index 8aa30097..0376e64f 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -27,7 +27,7 @@
 use {
     crate::{
         engine::{
-            core::Datacell,
+            data::cell::Datacell,
             error::{LangError, LangResult},
             ql::{
                 ast::{Entity, QueryData, State},
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 18bc3348..96310c80 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -27,7 +27,7 @@
 use {
     super::lex::{InsecureLexer, SafeLexer, Symbol, Token},
     crate::{
-        engine::{core::Datacell, error::LexResult},
+        engine::{data::cell::Datacell, error::LexResult},
         util::test_utils,
     },
     rand::{self, Rng},

From 05d93d210227f80d66dfcb282ab88c1b7d15f648 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 29 Apr 2023 10:54:05 -0700
Subject: [PATCH 179/310] Add basic row definition

---
 server/src/engine/core/index/key.rs           |   9 +
 server/src/engine/core/index/mod.rs           |   1 +
 server/src/engine/core/index/row.rs           | 105 ++++++++
 server/src/engine/idx/meta/hash.rs            | 241 ++++++++++++++++++
 .../src/engine/idx/{meta.rs => meta/mod.rs}   |   2 +
 server/src/engine/idx/mod.rs                  |   2 +-
 server/src/engine/idx/mtchm/mod.rs            |   4 +-
 server/src/engine/sync/smart.rs               |  86 +++++--
 8 files changed, 426 insertions(+), 24 deletions(-)
 create mode 100644 server/src/engine/core/index/row.rs
 create mode 100644 server/src/engine/idx/meta/hash.rs
 rename server/src/engine/idx/{meta.rs => meta/mod.rs} (99%)

diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
index 7037f405..c7bcd9d6 100644
--- a/server/src/engine/core/index/key.rs
+++ b/server/src/engine/core/index/key.rs
@@ -121,6 +121,15 @@ impl PrimaryIndexKey {
             },
         }
     }
+    pub unsafe fn raw_clone(&self) -> Self {
+        Self {
+            tag: self.tag,
+            data: {
+                let (qw, nw) = self.data.dwordqn_load_qw_nw();
+                SpecialPaddedWord::new(qw, nw)
+            },
+        }
+    }
     pub fn check(dc: &Datacell) -> bool {
         dc.tag().tag_unique().is_unique()
     }
diff --git a/server/src/engine/core/index/mod.rs b/server/src/engine/core/index/mod.rs
index ad3db743..0bb0ce98 100644
--- a/server/src/engine/core/index/mod.rs
+++ b/server/src/engine/core/index/mod.rs
@@ -25,3 +25,4 @@
 */
 
 mod key;
+mod row;
diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
new file mode 100644
index 00000000..3621572e
--- /dev/null
+++ b/server/src/engine/core/index/row.rs
@@ -0,0 +1,105 @@
+/*
+ * Created on Thu Apr 27 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::key::PrimaryIndexKey,
+    crate::engine::{
+        data::cell::Datacell,
+        idx::{meta::hash::HasherNativeFx, mtchm::meta::TreeElement, IndexST},
+        sync::smart::RawRC,
+    },
+    parking_lot::RwLock,
+    std::mem::ManuallyDrop,
+};
+
+type DcFieldIndex = IndexST<Box<str>, Datacell, HasherNativeFx>;
+
+pub struct Row {
+    pk: ManuallyDrop<PrimaryIndexKey>,
+    rc: RawRC<RwLock<DcFieldIndex>>,
+}
+
+impl TreeElement for Row {
+    type Key = PrimaryIndexKey;
+    type Value = RwLock<DcFieldIndex>;
+    fn key(&self) -> &Self::Key {
+        &self.pk
+    }
+    fn val(&self) -> &Self::Value {
+        self.rc.data()
+    }
+    fn new(k: Self::Key, v: Self::Value) -> Self {
+        Self::new(k, v)
+    }
+}
+
+impl Row {
+    pub fn new(pk: PrimaryIndexKey, data: RwLock<DcFieldIndex>) -> Self {
+        Self {
+            pk: ManuallyDrop::new(pk),
+            rc: unsafe {
+                // UNSAFE(@ohsayan): we free this up later
+                RawRC::new(data)
+            },
+        }
+    }
+    pub fn with_data_read<T>(&self, f: impl Fn(&DcFieldIndex) -> T) -> T {
+        let data = self.rc.data().read();
+        f(&data)
+    }
+    pub fn with_data_write<T>(&self, f: impl Fn(&mut DcFieldIndex) -> T) -> T {
+        let mut data = self.rc.data().write();
+        f(&mut data)
+    }
+}
+
+impl Clone for Row {
+    fn clone(&self) -> Self {
+        let rc = unsafe {
+            // UNSAFE(@ohsayan): we're calling this in the clone implementation
+            self.rc.rc_clone()
+        };
+        Self {
+            pk: unsafe {
+                // UNSAFE(@ohsayan): this is safe because of the refcount
+                ManuallyDrop::new(self.pk.raw_clone())
+            },
+            rc,
+        }
+    }
+}
+
+impl Drop for Row {
+    fn drop(&mut self) {
+        unsafe {
+            // UNSAFE(@ohsayan): we call in this the dtor itself
+            self.rc.rc_drop(|| {
+                // UNSAFE(@ohsayan): we rely on the correctness of the rc
+                ManuallyDrop::drop(&mut self.pk);
+            });
+        }
+    }
+}
diff --git a/server/src/engine/idx/meta/hash.rs b/server/src/engine/idx/meta/hash.rs
new file mode 100644
index 00000000..ed93873f
--- /dev/null
+++ b/server/src/engine/idx/meta/hash.rs
@@ -0,0 +1,241 @@
+/*
+ * Created on Sat Apr 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use std::hash::{BuildHasher, Hasher};
+
+pub type Hasher32Fx = HasherRawFx<u32>;
+pub type Hasher64Fx = HasherRawFx<u64>;
+pub type HasherNativeFx = HasherRawFx<usize>;
+
+const ROTATE: u32 = 5;
+const PRIME32: u32 = 0x9E3779B9; // golden
+const PRIME64: u64 = 0x517CC1B727220A95; // archimedes (obtained from rustc)
+
+pub trait WriteNumeric {
+    fn self_u32(self) -> u32;
+}
+
+macro_rules! impl_numeric_writes {
+    ($($ty:ty),*) => {
+        $(impl WriteNumeric for $ty { fn self_u32(self) -> u32 { self as u32 } })*
+    };
+}
+
+impl_numeric_writes!(u8, i8, u16, i16, u32, i32);
+
+pub trait HashWord: Sized {
+    const STATE: Self;
+    fn fin(&self) -> u64;
+    fn h_bytes(&mut self, bytes: &[u8]);
+    fn h_quad(&mut self, quad: u64);
+    fn h_word(&mut self, v: impl WriteNumeric);
+}
+
+impl HashWord for u32 {
+    const STATE: Self = 0;
+    fn fin(&self) -> u64 {
+        (*self) as _
+    }
+    fn h_bytes(&mut self, mut bytes: &[u8]) {
+        let mut state = *self;
+        while bytes.len() >= 4 {
+            // no need for ptr am; let opt with loop invariant
+            state = self::hash32(
+                state,
+                u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]),
+            );
+            bytes = &bytes[4..];
+        }
+
+        if bytes.len() >= 2 {
+            state = self::hash32(state, u16::from_ne_bytes([bytes[0], bytes[1]]) as u32);
+            bytes = &bytes[2..];
+        }
+
+        if !bytes.is_empty() {
+            state = self::hash32(state, bytes[0] as u32);
+        }
+
+        *self = state;
+    }
+    fn h_quad(&mut self, quad: u64) {
+        let mut state = *self;
+        let [x, y]: [u32; 2] = unsafe { core::mem::transmute(quad.to_ne_bytes()) };
+        state = self::hash32(state, x);
+        state = self::hash32(state, y);
+        *self = state;
+    }
+    fn h_word(&mut self, v: impl WriteNumeric) {
+        *self = self::hash32(*self, v.self_u32());
+    }
+}
+
+impl HashWord for u64 {
+    const STATE: Self = 0;
+    fn fin(&self) -> u64 {
+        (*self) as _
+    }
+    fn h_bytes(&mut self, mut bytes: &[u8]) {
+        let mut state = *self;
+        while bytes.len() >= 8 {
+            state = self::hash64(
+                state,
+                u64::from_ne_bytes([
+                    bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7],
+                ]),
+            );
+            bytes = &bytes[8..];
+        }
+
+        if bytes.len() >= 4 {
+            state = self::hash64(
+                state,
+                u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as u64,
+            );
+            bytes = &bytes[4..];
+        }
+
+        if bytes.len() >= 2 {
+            state = self::hash64(state, u16::from_ne_bytes([bytes[0], bytes[1]]) as u64);
+            bytes = &bytes[2..];
+        }
+
+        if !bytes.is_empty() {
+            state = self::hash64(state, bytes[0] as u64);
+        }
+
+        *self = state;
+    }
+    fn h_quad(&mut self, quad: u64) {
+        *self = self::hash64(*self, quad);
+    }
+    fn h_word(&mut self, v: impl WriteNumeric) {
+        *self = self::hash64(*self, v.self_u32() as _)
+    }
+}
+
+impl HashWord for usize {
+    const STATE: Self = 0;
+    fn fin(&self) -> u64 {
+        (*self) as _
+    }
+    fn h_bytes(&mut self, bytes: &[u8]) {
+        if cfg!(target_pointer_width = "32") {
+            let mut slf = *self as u32;
+            <u32 as HashWord>::h_bytes(&mut slf, bytes);
+            *self = slf as usize;
+        } else {
+            let mut slf = *self as u64;
+            <u64 as HashWord>::h_bytes(&mut slf, bytes);
+            *self = slf as usize;
+        }
+    }
+    fn h_quad(&mut self, quad: u64) {
+        if cfg!(target_pointer_width = "32") {
+            let mut slf = *self as u32;
+            <u32 as HashWord>::h_quad(&mut slf, quad);
+            *self = slf as usize;
+        } else {
+            let mut slf = *self as u64;
+            <u64 as HashWord>::h_quad(&mut slf, quad);
+            *self = slf as usize;
+        }
+    }
+    fn h_word(&mut self, v: impl WriteNumeric) {
+        if cfg!(target_pointer_width = "32") {
+            let mut slf = *self as u32;
+            <u32 as HashWord>::h_word(&mut slf, v);
+            *self = slf as usize;
+        } else {
+            let mut slf = *self as u64;
+            <u64 as HashWord>::h_word(&mut slf, v);
+            *self = slf as usize;
+        }
+    }
+}
+
+fn hash32(state: u32, word: u32) -> u32 {
+    (state.rotate_left(ROTATE) ^ word).wrapping_mul(PRIME32)
+}
+fn hash64(state: u64, word: u64) -> u64 {
+    (state.rotate_left(ROTATE) ^ word).wrapping_mul(PRIME64)
+}
+
+#[derive(Debug)]
+pub struct HasherRawFx<T>(T);
+
+impl<T: HashWord> HasherRawFx<T> {
+    pub const fn new() -> Self {
+        Self(T::STATE)
+    }
+}
+
+impl<T: HashWord> Hasher for HasherRawFx<T> {
+    fn finish(&self) -> u64 {
+        self.0.fin()
+    }
+    fn write(&mut self, bytes: &[u8]) {
+        T::h_bytes(&mut self.0, bytes)
+    }
+    fn write_u8(&mut self, i: u8) {
+        T::h_word(&mut self.0, i)
+    }
+    fn write_u16(&mut self, i: u16) {
+        T::h_word(&mut self.0, i)
+    }
+    fn write_u32(&mut self, i: u32) {
+        T::h_word(&mut self.0, i)
+    }
+    fn write_u64(&mut self, i: u64) {
+        T::h_quad(&mut self.0, i)
+    }
+    fn write_u128(&mut self, i: u128) {
+        let [a, b]: [u64; 2] = unsafe { core::mem::transmute(i) };
+        T::h_quad(&mut self.0, a);
+        T::h_quad(&mut self.0, b);
+    }
+    fn write_usize(&mut self, i: usize) {
+        if cfg!(target_pointer_width = "32") {
+            T::h_word(&mut self.0, i as u32);
+        } else {
+            T::h_quad(&mut self.0, i as u64);
+        }
+    }
+}
+
+impl<T: HashWord> BuildHasher for HasherRawFx<T> {
+    type Hasher = Self;
+
+    fn build_hasher(&self) -> Self::Hasher {
+        Self::new()
+    }
+}
+
+impl<T: HashWord> Default for HasherRawFx<T> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
diff --git a/server/src/engine/idx/meta.rs b/server/src/engine/idx/meta/mod.rs
similarity index 99%
rename from server/src/engine/idx/meta.rs
rename to server/src/engine/idx/meta/mod.rs
index 24c4b317..e3576450 100644
--- a/server/src/engine/idx/meta.rs
+++ b/server/src/engine/idx/meta/mod.rs
@@ -24,6 +24,8 @@
  *
 */
 
+pub mod hash;
+
 use core::{
     borrow::Borrow,
     hash::{BuildHasher, Hash},
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 0f7119fa..b427b655 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -27,7 +27,7 @@
 #![deny(unreachable_patterns)]
 
 pub mod meta;
-mod mtchm;
+pub mod mtchm;
 mod stdhm;
 mod stord;
 #[cfg(test)]
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index 6bb3673f..8674a394 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -25,9 +25,9 @@
 */
 
 mod access;
-pub(super) mod imp;
+pub mod imp;
 mod iter;
-pub(super) mod meta;
+pub mod meta;
 mod patch;
 #[cfg(test)]
 mod tests;
diff --git a/server/src/engine/sync/smart.rs b/server/src/engine/sync/smart.rs
index 1da74a18..47e75430 100644
--- a/server/src/engine/sync/smart.rs
+++ b/server/src/engine/sync/smart.rs
@@ -200,21 +200,18 @@ impl<T> Deref for SliceRC<T> {
 unsafe impl<T: Send> Send for SliceRC<T> {}
 unsafe impl<T: Sync> Sync for SliceRC<T> {}
 
-/// The core atomic reference counter implementation. All smart pointers use this inside
+/// A simple rc
 pub struct EArc {
-    rc: NonNull<AtomicUsize>,
+    base: RawRC<()>,
 }
 
 impl EArc {
-    /// Create a new [`EArc`] instance
-    ///
     /// ## Safety
     ///
-    /// While this is **not unsafe** in the eyes of the language specification for safety, it still does violate a very common
-    /// bug: memory leaks and we don't want that. So, it is upto the caller to clean this up
-    unsafe fn new() -> Self {
+    /// Clean up your own memory, sir
+    pub unsafe fn new() -> Self {
         Self {
-            rc: NonNull::new_unchecked(Box::into_raw(Box::new(AtomicUsize::new(0)))),
+            base: RawRC::new(()),
         }
     }
 }
@@ -222,14 +219,61 @@ impl EArc {
 impl EArc {
     /// ## Safety
     ///
-    /// Only call when you follow the appropriate ground rules for safety
-    unsafe fn _rc(&self) -> &AtomicUsize {
-        self.rc.as_ref()
+    /// Only call in an actual [`Clone`] context
+    pub unsafe fn rc_clone(&self) -> Self {
+        Self {
+            base: self.base.rc_clone(),
+        }
+    }
+    /// ## Safety
+    ///
+    /// Only call in dtor context
+    pub unsafe fn rc_drop(&mut self, dropfn: impl FnMut()) {
+        self.base.rc_drop(dropfn)
+    }
+}
+
+/// The core atomic reference counter implementation. All smart pointers use this inside
+pub struct RawRC<T> {
+    hptr: NonNull<RawRCData<T>>,
+}
+
+struct RawRCData<T> {
+    rc: AtomicUsize,
+    data: T,
+}
+
+impl<T> RawRC<T> {
+    /// Create a new [`RawRC`] instance
+    ///
+    /// ## Safety
+    ///
+    /// While this is **not unsafe** in the eyes of the language specification for safety, it still does violate a very common
+    /// bug: memory leaks and we don't want that. So, it is upto the caller to clean this up
+    pub unsafe fn new(data: T) -> Self {
+        Self {
+            hptr: NonNull::new_unchecked(Box::leak(Box::new(RawRCData {
+                rc: AtomicUsize::new(1),
+                data,
+            }))),
+        }
+    }
+    pub fn data(&self) -> &T {
+        unsafe {
+            // UNSAFE(@ohsayan): we believe in the power of barriers!
+            &(*self.hptr.as_ptr()).data
+        }
+    }
+    fn _rc(&self) -> &AtomicUsize {
+        unsafe {
+            // UNSAFE(@ohsayan): we believe in the power of barriers!
+            &(*self.hptr.as_ptr()).rc
+        }
     }
     /// ## Safety
     ///
     /// Only call in an actual [`Clone`] context
-    unsafe fn rc_clone(&self) -> Self {
+    pub unsafe fn rc_clone(&self) -> Self {
         let new_rc = self._rc().fetch_add(1, ORD_RLX);
         if new_rc > (isize::MAX) as usize {
             // some incredibly degenerate case; this won't ever happen but who knows if some fella decided to have atomic overflow fun?
@@ -237,18 +281,10 @@ impl EArc {
         }
         Self { ..*self }
     }
-    #[cold]
-    #[inline(never)]
-    unsafe fn rc_drop_slow(&mut self, mut dropfn: impl FnMut()) {
-        // deallocate object
-        dropfn();
-        // deallocate rc
-        drop(Box::from_raw(self.rc.as_ptr()));
-    }
     /// ## Safety
     ///
     /// Only call in dtor context
-    unsafe fn rc_drop(&mut self, dropfn: impl FnMut()) {
+    pub unsafe fn rc_drop(&mut self, dropfn: impl FnMut()) {
         if self._rc().fetch_sub(1, ORD_REL) != 1 {
             // not the last man alive
             return;
@@ -257,4 +293,12 @@ impl EArc {
         atomic::fence(ORD_ACQ);
         self.rc_drop_slow(dropfn);
     }
+    #[cold]
+    #[inline(never)]
+    unsafe fn rc_drop_slow(&mut self, mut dropfn: impl FnMut()) {
+        // deallocate object
+        dropfn();
+        // deallocate rc
+        drop(Box::from_raw(self.hptr.as_ptr()));
+    }
 }

From 60157e110bf3639412a9cc03d4de9a947f779381 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 1 May 2023 08:29:34 -0700
Subject: [PATCH 180/310] Simplify cht impls

---
 server/src/engine/error.rs              |   2 +-
 server/src/engine/idx/mod.rs            |   7 +-
 server/src/engine/idx/mtchm/imp.rs      | 216 +++++-------------------
 server/src/engine/idx/stdhm.rs          |   2 +-
 server/src/engine/idx/stord/mod.rs      |   2 +-
 server/src/engine/ql/ddl/ins.rs         |   4 +-
 server/src/engine/ql/tests/dml_tests.rs |  13 +-
 7 files changed, 58 insertions(+), 188 deletions(-)

diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 0c4ffb81..dee625bc 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -90,7 +90,7 @@ pub enum DatabaseError {
     NeedLock,
     /// expected a full entity, but found a single implicit entity
     ExpectedEntity,
-    // ddl: create space
+    // ddl
     /// unknown property or bad type for property
     DdlSpaceBadProperty,
     /// the space already exists
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index b427b655..93e91924 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -43,6 +43,7 @@ use {
 pub type IndexSTSeqCns<K, V> = stord::IndexSTSeqDll<K, V, stord::config::ConservativeConfig<K, V>>;
 pub type IndexSTSeqLib<K, V> = stord::IndexSTSeqDll<K, V, stord::config::LiberalConfig<K, V>>;
 pub type IndexMTRC<K, V> = mtchm::imp::ChmArc<K, V, mtchm::meta::DefConfig>;
+pub type IndexMTRaw<E> = mtchm::RawTree<E, mtchm::meta::DefConfig>;
 pub type IndexMTCp<K, V> = mtchm::imp::ChmCopy<K, V, mtchm::meta::DefConfig>;
 pub type IndexST<K, V, S = std::collections::hash_map::RandomState> =
     std::collections::hash_map::HashMap<K, V, S>;
@@ -100,7 +101,7 @@ pub struct DummyMetrics;
 
 /// The base spec for any index. Iterators have meaningless order, and that is intentional and oftentimes
 /// consequential. For more specialized impls, use the [`STIndex`], [`MTIndex`] or [`STIndexSeq`] traits
-pub trait IndexBaseSpec<K: ?Sized, V: ?Sized>: Sized {
+pub trait IndexBaseSpec: Sized {
     /// Index supports prealloc?
     const PREALLOC: bool;
     #[cfg(debug_assertions)]
@@ -126,7 +127,7 @@ pub trait IndexBaseSpec<K: ?Sized, V: ?Sized>: Sized {
 }
 
 /// An unordered MTIndex
-pub trait MTIndex<K, V>: IndexBaseSpec<K, V> {
+pub trait MTIndex<K, V>: IndexBaseSpec {
     type IterKV<'t, 'g, 'v>: Iterator<Item = (&'v K, &'v V)>
     where
         'g: 't + 'v,
@@ -202,7 +203,7 @@ pub trait MTIndex<K, V>: IndexBaseSpec<K, V> {
 }
 
 /// An unordered STIndex
-pub trait STIndex<K: ?Sized, V>: IndexBaseSpec<K, V> {
+pub trait STIndex<K: ?Sized, V>: IndexBaseSpec {
     /// An iterator over the keys and values
     type IterKV<'a>: Iterator<Item = (&'a K, &'a V)>
     where
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index 6098bd5b..67a652d9 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -34,261 +34,135 @@ use {
         RawTree,
     },
     crate::engine::{
-        idx::{meta::Comparable, AsKey, AsKeyClone, AsValue, AsValueClone, IndexBaseSpec, MTIndex},
-        sync::atm::{upin, Guard},
+        idx::{meta::Comparable, AsKeyClone, AsValue, AsValueClone, IndexBaseSpec, MTIndex},
+        sync::atm::Guard,
     },
     std::sync::Arc,
 };
 
-#[inline(always)]
-fn arc<K, V>(k: K, v: V) -> Arc<(K, V)> {
-    Arc::new((k, v))
-}
-
-pub type ChmArc<K, V, C> = RawTree<Arc<(K, V)>, C>;
+pub type Raw<E, C> = RawTree<E, C>;
+pub type ChmArc<K, V, C> = Raw<Arc<(K, V)>, C>;
+pub type ChmCopy<K, V, C> = Raw<(K, V), C>;
 
-impl<K, V, C> IndexBaseSpec<K, V> for ChmArc<K, V, C>
-where
-    C: Config,
-{
+impl<E, C: Config> IndexBaseSpec for Raw<E, C> {
     const PREALLOC: bool = false;
 
-    #[cfg(debug_assertions)]
     type Metrics = CHTRuntimeLog;
 
     fn idx_init() -> Self {
-        ChmArc::new()
+        Self::new()
     }
 
     fn idx_init_with(s: Self) -> Self {
         s
     }
 
-    #[cfg(debug_assertions)]
     fn idx_metrics(&self) -> &Self::Metrics {
         &self.m
     }
 }
 
-impl<K, V, C> MTIndex<K, V> for ChmArc<K, V, C>
-where
-    C: Config,
-    K: AsKey,
-    V: AsValue,
-{
-    type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, Arc<(K, V)>, C>
+impl<E: TreeElement, C: Config> MTIndex<E::Key, E::Value> for Raw<E, C> {
+    type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, E, C>
     where
         'g: 't + 'v,
         't: 'v,
-        K: 'v,
-        V: 'v,
+        E::Key: 'v,
+        E::Value: 'v,
         Self: 't;
 
-    type IterKey<'t, 'g, 'v> = IterKey<'t, 'g, 'v, Arc<(K, V)>, C>
+    type IterKey<'t, 'g, 'v> = IterKey<'t, 'g, 'v, E, C>
     where
         'g: 't + 'v,
         't: 'v,
-        K: 'v,
+        E::Key: 'v,
         Self: 't;
 
-    type IterVal<'t, 'g, 'v> = IterVal<'t, 'g, 'v, Arc<(K, V)>, C>
+    type IterVal<'t, 'g, 'v> = IterVal<'t, 'g, 'v, E, C>
     where
         'g: 't + 'v,
         't: 'v,
-        V: 'v,
+        E::Value: 'v,
         Self: 't;
 
     fn mt_clear(&self, g: &Guard) {
         self.nontransactional_clear(g)
     }
 
-    fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool {
-        self.patch(VanillaInsert(arc(key, val)), g)
-    }
-
-    fn mt_upsert(&self, key: K, val: V, g: &Guard) {
-        self.patch(VanillaUpsert(arc(key, val)), g)
-    }
-
-    fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
-    where
-        Q: ?Sized + Comparable<K>,
-    {
-        self.contains_key(key, g)
-    }
-
-    fn mt_get<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
-    where
-        Q: ?Sized + Comparable<K>,
-        't: 'v,
-        'g: 't + 'v,
-    {
-        self.get(key, g)
-    }
-
-    fn mt_get_cloned<Q>(&self, key: &Q, g: &Guard) -> Option<V>
-    where
-        Q: ?Sized + Comparable<K>,
-        V: AsValueClone,
-    {
-        self.get(key, g).cloned()
-    }
-
-    fn mt_update(&self, key: K, val: V, g: &Guard) -> bool {
-        self.patch(VanillaUpdate(arc(key, val)), g)
-    }
-
-    fn mt_update_return<'t, 'g, 'v>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
+    fn mt_insert(&self, key: E::Key, val: E::Value, g: &Guard) -> bool
     where
-        't: 'v,
-        'g: 't + 'v,
+        E::Value: AsValue,
     {
-        self.patch(VanillaUpdateRet(arc(key, val)), g)
+        self.patch(VanillaInsert(E::new(key, val)), g)
     }
 
-    fn mt_delete<Q>(&self, key: &Q, g: &Guard) -> bool
+    fn mt_upsert(&self, key: E::Key, val: E::Value, g: &Guard)
     where
-        Q: ?Sized + Comparable<K>,
+        E::Value: AsValue,
     {
-        self.remove(key, g)
-    }
-
-    fn mt_delete_return<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
-    where
-        Q: ?Sized + Comparable<K>,
-        't: 'v,
-        'g: 't + 'v,
-    {
-        self.remove_return(key, g)
-    }
-}
-
-pub type ChmCopy<K, V, C> = RawTree<(K, V), C>;
-
-impl<K, V, C> IndexBaseSpec<K, V> for ChmCopy<K, V, C>
-where
-    C: Config,
-{
-    const PREALLOC: bool = false;
-
-    #[cfg(debug_assertions)]
-    type Metrics = CHTRuntimeLog;
-
-    fn idx_init() -> Self {
-        ChmCopy::new()
-    }
-
-    fn idx_init_with(s: Self) -> Self {
-        s
-    }
-
-    #[cfg(debug_assertions)]
-    fn idx_metrics(&self) -> &Self::Metrics {
-        &self.m
-    }
-}
-
-impl<K, V, C> MTIndex<K, V> for ChmCopy<K, V, C>
-where
-    C: Config,
-    K: AsKeyClone,
-    V: AsValueClone,
-{
-    type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, (K, V), C>
-    where
-        'g: 't + 'v,
-        't: 'v,
-        K: 'v,
-        V: 'v,
-        Self: 't;
-
-    type IterKey<'t, 'g, 'v> = IterKey<'t, 'g, 'v, (K, V), C>
-    where
-        'g: 't + 'v,
-        't: 'v,
-        K: 'v,
-        Self: 't;
-
-    type IterVal<'t, 'g, 'v> = IterVal<'t, 'g, 'v, (K, V), C>
-    where
-        'g: 't + 'v,
-        't: 'v,
-        V: 'v,
-        Self: 't;
-
-    fn mt_clear(&self, g: &Guard) {
-        self.nontransactional_clear(g)
-    }
-
-    fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool {
-        self.patch(VanillaInsert((key, val)), g)
-    }
-
-    fn mt_upsert(&self, key: K, val: V, g: &Guard) {
-        self.patch(VanillaUpsert((key, val)), g)
+        self.patch(VanillaUpsert(E::new(key, val)), g)
     }
 
     fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
     where
-        Q: ?Sized + Comparable<K>,
+        Q: ?Sized + Comparable<E::Key>,
     {
         self.contains_key(key, g)
     }
 
-    fn mt_get<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
+    fn mt_get<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v E::Value>
     where
-        Q: ?Sized + Comparable<K>,
+        Q: ?Sized + Comparable<E::Key>,
         't: 'v,
         'g: 't + 'v,
     {
         self.get(key, g)
     }
 
-    fn mt_get_cloned<Q>(&self, key: &Q, g: &Guard) -> Option<V>
+    fn mt_get_cloned<Q>(&self, key: &Q, g: &Guard) -> Option<E::Value>
     where
-        Q: ?Sized + Comparable<K>,
+        Q: ?Sized + Comparable<E::Key>,
+        E::Value: AsValueClone,
     {
         self.get(key, g).cloned()
     }
 
-    fn mt_update(&self, key: K, val: V, g: &Guard) -> bool {
-        self.patch(VanillaUpdate((key, val)), g)
+    fn mt_update(&self, key: E::Key, val: E::Value, g: &Guard) -> bool
+    where
+        E::Key: AsKeyClone,
+        E::Value: AsValue,
+    {
+        self.patch(VanillaUpdate(E::new(key, val)), g)
     }
 
-    fn mt_update_return<'t, 'g, 'v>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
+    fn mt_update_return<'t, 'g, 'v>(
+        &'t self,
+        key: E::Key,
+        val: E::Value,
+        g: &'g Guard,
+    ) -> Option<&'v E::Value>
     where
+        E::Key: AsKeyClone,
+        E::Value: AsValue,
         't: 'v,
         'g: 't + 'v,
     {
-        self.patch(VanillaUpdateRet((key, val)), g)
+        self.patch(VanillaUpdateRet(E::new(key, val)), g)
     }
 
     fn mt_delete<Q>(&self, key: &Q, g: &Guard) -> bool
     where
-        Q: ?Sized + Comparable<K>,
+        Q: ?Sized + Comparable<E::Key>,
     {
         self.remove(key, g)
     }
 
-    fn mt_delete_return<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
+    fn mt_delete_return<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v E::Value>
     where
-        Q: ?Sized + Comparable<K>,
+        Q: ?Sized + Comparable<E::Key>,
         't: 'v,
         'g: 't + 'v,
     {
         self.remove_return(key, g)
     }
 }
-
-impl<T: TreeElement, C: Config> FromIterator<T> for RawTree<T, C> {
-    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
-        let g = unsafe {
-            // UNSAFE(@ohsayan): it's me, hi, I'm the problem, it's me. yeah, Taylor knows it too. it's just us
-            upin()
-        };
-        let t = RawTree::new();
-        iter.into_iter()
-            .for_each(|te| assert!(t.patch(VanillaInsert(te), g)));
-        t
-    }
-}
diff --git a/server/src/engine/idx/stdhm.rs b/server/src/engine/idx/stdhm.rs
index 570d9c28..43306513 100644
--- a/server/src/engine/idx/stdhm.rs
+++ b/server/src/engine/idx/stdhm.rs
@@ -42,7 +42,7 @@ use {
     },
 };
 
-impl<K, V, S> IndexBaseSpec<K, V> for StdMap<K, V, S>
+impl<K, V, S> IndexBaseSpec for StdMap<K, V, S>
 where
     S: BuildHasher + Default,
 {
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index 01b93bc7..969f12b4 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -536,7 +536,7 @@ impl<K: AsKey, V: AsValue, C: Config<K, V>> FromIterator<(K, V)> for IndexSTSeqD
     }
 }
 
-impl<K, V, C: Config<K, V>> IndexBaseSpec<K, V> for IndexSTSeqDll<K, V, C> {
+impl<K, V, C: Config<K, V>> IndexBaseSpec for IndexSTSeqDll<K, V, C> {
     const PREALLOC: bool = true;
 
     #[cfg(debug_assertions)]
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
index 4552d34e..757c36d8 100644
--- a/server/src/engine/ql/ddl/ins.rs
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -51,7 +51,9 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
     }
 
     match state.fw_read() {
-        Token![model] => Entity::parse_from_state_rounded_result(state).map(Statement::InspectModel),
+        Token![model] => {
+            Entity::parse_from_state_rounded_result(state).map(Statement::InspectModel)
+        }
         Token![space] if state.cursor_has_ident_rounded() => {
             Ok(Statement::InspectSpace(unsafe {
                 // UNSAFE(@ohsayan): Safe because of the match predicate
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 6bd751eb..4c830c40 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -436,16 +436,9 @@ mod stmt_insert {
         let r = parse_ast_node_full::<InsertStatement>(&x[1..]).unwrap();
         let e = InsertStatement::new(
             Entity::Full(Ident::from("twitter"), Ident::from("users")),
-            into_array_nullable![
-                "sayan",
-                "Sayan",
-                "sayan@example.com",
-                true,
-                12345,
-                67890
-            ]
-            .to_vec()
-            .into(),
+            into_array_nullable!["sayan", "Sayan", "sayan@example.com", true, 12345, 67890]
+                .to_vec()
+                .into(),
         );
         assert_eq!(e, r);
     }

From 5838941ce8c9e4241735e13213e3b72645758d56 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 2 May 2023 22:42:49 -0700
Subject: [PATCH 181/310] Impl basic executor for ins

---
 server/src/engine/core/dml/ins.rs           | 104 ++++++++++++++++++++
 server/src/engine/core/dml/mod.rs           |  29 ++++++
 server/src/engine/core/dml/result_set.rs    |  67 +++++++++++++
 server/src/engine/core/index/mod.rs         |  29 ++++++
 server/src/engine/core/index/row.rs         |   9 +-
 server/src/engine/core/mod.rs               |   1 +
 server/src/engine/core/model/mod.rs         |  15 ++-
 server/src/engine/core/tests/model/layer.rs |  19 ++++
 server/src/engine/error.rs                  |   5 +
 server/src/engine/idx/mod.rs                |   2 +-
 server/src/engine/idx/mtchm/mod.rs          |  12 +++
 server/src/engine/ql/dml/ins.rs             |  15 +++
 server/src/engine/sync/smart.rs             |  22 ++++-
 13 files changed, 318 insertions(+), 11 deletions(-)
 create mode 100644 server/src/engine/core/dml/ins.rs
 create mode 100644 server/src/engine/core/dml/mod.rs
 create mode 100644 server/src/engine/core/dml/result_set.rs

diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
new file mode 100644
index 00000000..a07e1b7e
--- /dev/null
+++ b/server/src/engine/core/dml/ins.rs
@@ -0,0 +1,104 @@
+/*
+ * Created on Mon May 01 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    core::{
+        index::{DcFieldIndex, PrimaryIndexKey},
+        model::{Fields, ModelView},
+        GlobalNS,
+    },
+    error::{DatabaseError, DatabaseResult},
+    idx::{IndexBaseSpec, STIndex},
+    ql::dml::ins::{InsertData, InsertStatement},
+    sync::atm::cpin,
+};
+
+pub fn insert(gns: &GlobalNS, insert: InsertStatement) -> DatabaseResult<()> {
+    gns.with_model(insert.entity(), |mdl| {
+        let irmwd = mdl.intent_write_new_data();
+        let (pk, data) = prepare_insert(mdl, irmwd.fields(), insert.data())?;
+        let g = cpin();
+        if mdl.primary_index().insert(pk, data, &g) {
+            Ok(())
+        } else {
+            Err(DatabaseError::DmlConstraintViolationDuplicate)
+        }
+    })
+}
+
+fn prepare_insert(
+    model: &ModelView,
+    fields: &Fields,
+    insert: InsertData,
+) -> DatabaseResult<(PrimaryIndexKey, DcFieldIndex)> {
+    let mut okay = fields.len() == insert.column_count();
+    let mut prepared_data = DcFieldIndex::idx_init_cap(fields.len());
+    match insert {
+        InsertData::Ordered(tuple) => {
+            let mut fields = fields.st_iter_kv();
+            let mut tuple = tuple.into_iter();
+            while (tuple.len() != 0) & okay {
+                let data;
+                let field;
+                unsafe {
+                    // UNSAFE(@ohsayan): safe because of invariant
+                    data = tuple.next().unwrap_unchecked();
+                    // UNSAFE(@ohsayan): safe because of flag
+                    field = fields.next().unwrap_unchecked();
+                }
+                let (field_id, field) = field;
+                okay &= field.validate_data_fpath(&data);
+                okay &= prepared_data.st_insert(field_id.clone(), data);
+            }
+        }
+        InsertData::Map(map) => {
+            let mut map = map.into_iter();
+            while (map.len() != 0) & okay {
+                let (field_id, field_data) = unsafe {
+                    // UNSAFE(@ohsayan): safe because of loop invariant
+                    map.next().unwrap_unchecked()
+                };
+                let Some(field) = fields.st_get_cloned(field_id.as_str()) else {
+                    okay = false;
+                    break;
+                };
+                okay &= field.validate_data_fpath(&field_data);
+                prepared_data.st_insert(field_id.boxed_str(), field_data);
+            }
+        }
+    }
+    let primary_key = prepared_data.remove(model.p_key());
+    okay &= primary_key.is_some();
+    if okay {
+        let primary_key = unsafe {
+            // UNSAFE(@ohsayan): okay check above
+            PrimaryIndexKey::new_from_dc(primary_key.unwrap_unchecked())
+        };
+        Ok((primary_key, prepared_data))
+    } else {
+        Err(DatabaseError::DmlDataValidationError)
+    }
+}
diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
new file mode 100644
index 00000000..e433558e
--- /dev/null
+++ b/server/src/engine/core/dml/mod.rs
@@ -0,0 +1,29 @@
+/*
+ * Created on Mon May 01 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod ins;
+// result
+mod result_set;
diff --git a/server/src/engine/core/dml/result_set.rs b/server/src/engine/core/dml/result_set.rs
new file mode 100644
index 00000000..03e2750b
--- /dev/null
+++ b/server/src/engine/core/dml/result_set.rs
@@ -0,0 +1,67 @@
+/*
+ * Created on Tue May 02 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/*
+    ☢ ISOLATION WARNING ☢
+    ----------------
+    I went with a rather suboptimal solution for v1. Once we have CC
+    we can do much better (at the cost of more complexity, ofcourse).
+
+    We'll roll that out in 0.8.1, I think.
+
+    FIXME(@ohsayan): Fix this
+*/
+
+use {
+    crate::engine::core::index::{DcFieldIndex, PrimaryIndexKey, Row},
+    parking_lot::RwLockReadGuard,
+};
+
+pub struct RowSnapshot<'a> {
+    key: &'a PrimaryIndexKey,
+    data: RwLockReadGuard<'a, DcFieldIndex>,
+}
+
+impl<'a> RowSnapshot<'a> {
+    /// The moment you take a snapshot, you essentially "freeze" the row and prevent any changes from happening.
+    ///
+    /// HOWEVER: This is very inefficient subject to isolation level scrutiny
+    #[inline(always)]
+    pub fn snapshot(row: &'a Row) -> RowSnapshot<'a> {
+        Self {
+            key: row.d_key(),
+            data: row.d_data().read(),
+        }
+    }
+    #[inline(always)]
+    pub fn row_key(&self) -> &'a PrimaryIndexKey {
+        self.key
+    }
+    #[inline(always)]
+    pub fn row_data(&self) -> &DcFieldIndex {
+        &self.data
+    }
+}
diff --git a/server/src/engine/core/index/mod.rs b/server/src/engine/core/index/mod.rs
index 0bb0ce98..d8af8b6e 100644
--- a/server/src/engine/core/index/mod.rs
+++ b/server/src/engine/core/index/mod.rs
@@ -26,3 +26,32 @@
 
 mod key;
 mod row;
+
+use {
+    crate::engine::{
+        idx::{IndexBaseSpec, IndexMTRaw, MTIndex},
+        sync::atm::Guard,
+    },
+    parking_lot::RwLock,
+};
+
+pub use {
+    key::PrimaryIndexKey,
+    row::{DcFieldIndex, Row},
+};
+
+#[derive(Debug)]
+pub struct PrimaryIndex {
+    data: IndexMTRaw<row::Row>,
+}
+
+impl PrimaryIndex {
+    pub fn new_empty() -> Self {
+        Self {
+            data: IndexMTRaw::idx_init(),
+        }
+    }
+    pub fn insert(&self, key: PrimaryIndexKey, data: row::DcFieldIndex, g: &Guard) -> bool {
+        self.data.mt_insert(key, RwLock::new(data), g)
+    }
+}
diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
index 3621572e..376659bd 100644
--- a/server/src/engine/core/index/row.rs
+++ b/server/src/engine/core/index/row.rs
@@ -35,8 +35,9 @@ use {
     std::mem::ManuallyDrop,
 };
 
-type DcFieldIndex = IndexST<Box<str>, Datacell, HasherNativeFx>;
+pub type DcFieldIndex = IndexST<Box<str>, Datacell, HasherNativeFx>;
 
+#[derive(Debug)]
 pub struct Row {
     pk: ManuallyDrop<PrimaryIndexKey>,
     rc: RawRC<RwLock<DcFieldIndex>>,
@@ -74,6 +75,12 @@ impl Row {
         let mut data = self.rc.data().write();
         f(&mut data)
     }
+    pub fn d_key(&self) -> &PrimaryIndexKey {
+        &self.pk
+    }
+    pub fn d_data(&self) -> &RwLock<DcFieldIndex> {
+        self.rc.data()
+    }
 }
 
 impl Clone for Row {
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 57033691..6bc1517b 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -24,6 +24,7 @@
  *
 */
 
+mod dml;
 mod index;
 mod model;
 mod space;
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 83cb0074..b61a0b19 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -29,9 +29,8 @@ pub(super) mod alt;
 #[cfg(test)]
 use std::cell::RefCell;
 
-use super::util::EntityLocator;
-
 use {
+    super::{index::PrimaryIndex, util::EntityLocator},
     crate::engine::{
         data::{
             cell::Datacell,
@@ -52,14 +51,13 @@ use {
 
 pub(in crate::engine::core) type Fields = IndexSTSeqCns<Box<str>, Field>;
 
-// FIXME(@ohsayan): update this!
-
 #[derive(Debug)]
 pub struct ModelView {
     p_key: Box<str>,
     p_tag: FullTag,
     fields: UnsafeCell<Fields>,
     sync_matrix: ISyncMatrix,
+    data: PrimaryIndex,
 }
 
 #[cfg(test)]
@@ -93,6 +91,9 @@ impl ModelView {
     pub fn intent_write_model<'a>(&'a self) -> IWModel<'a> {
         IWModel::new(self)
     }
+    pub fn intent_write_new_data<'a>(&'a self) -> IRModelSMData<'a> {
+        IRModelSMData::new(self)
+    }
     fn is_pk(&self, new: &str) -> bool {
         self.p_key.as_bytes() == new.as_bytes()
     }
@@ -110,6 +111,9 @@ impl ModelView {
         // TODO(@ohsayan): change this!
         true
     }
+    pub fn primary_index(&self) -> &PrimaryIndex {
+        &self.data
+    }
 }
 
 impl ModelView {
@@ -151,6 +155,7 @@ impl ModelView {
                     p_tag: tag,
                     fields: UnsafeCell::new(fields),
                     sync_matrix: ISyncMatrix::new(),
+                    data: PrimaryIndex::new_empty(),
                 });
             }
         }
@@ -474,7 +479,9 @@ unsafe fn lverify_list(_: Layer, _: &Datacell) -> bool {
 #[derive(Debug)]
 pub struct ISyncMatrix {
     // virtual privileges
+    /// read/write model
     v_priv_model_alter: RwLock<()>,
+    /// RW data/block all
     v_priv_data_new_or_revise: RwLock<()>,
 }
 
diff --git a/server/src/engine/core/tests/model/layer.rs b/server/src/engine/core/tests/model/layer.rs
index 10ebf68f..f4025af2 100644
--- a/server/src/engine/core/tests/model/layer.rs
+++ b/server/src/engine/core/tests/model/layer.rs
@@ -207,6 +207,25 @@ mod layer_data_validation {
         );
     }
     #[test]
+    fn list_nested_l1() {
+        let layer = layerview("list { type: list { type: string } }").unwrap();
+        let dc = Datacell::new_list(vec![
+            Datacell::new_list(vec![Datacell::from("hello_11"), Datacell::from("hello_12")]),
+            Datacell::new_list(vec![Datacell::from("hello_21"), Datacell::from("hello_22")]),
+            Datacell::new_list(vec![Datacell::from("hello_31"), Datacell::from("hello_32")]),
+        ]);
+        assert!(layer.validate_data_fpath(&dc));
+        assert_vecstreq_exact!(
+            model::layer_traces(),
+            [
+                "list", // low
+                "list", "string", "string", // cs: 1
+                "list", "string", "string", // cs: 2
+                "list", "string", "string", // cs: 3
+            ]
+        );
+    }
+    #[test]
     fn nullval_fpath() {
         let layer = layerview_nullable("string", true).unwrap();
         assert!(layer.validate_data_fpath(&Datacell::null()));
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index dee625bc..4720cc5f 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -119,4 +119,9 @@ pub enum DatabaseError {
     DdlModelNotFound,
     /// attempted a remove, but the model view is nonempty
     DdlModelViewNotEmpty,
+    // dml
+    /// Duplicate
+    DmlConstraintViolationDuplicate,
+    /// data validation error
+    DmlDataValidationError,
 }
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 93e91924..04410d14 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -43,7 +43,7 @@ use {
 pub type IndexSTSeqCns<K, V> = stord::IndexSTSeqDll<K, V, stord::config::ConservativeConfig<K, V>>;
 pub type IndexSTSeqLib<K, V> = stord::IndexSTSeqDll<K, V, stord::config::LiberalConfig<K, V>>;
 pub type IndexMTRC<K, V> = mtchm::imp::ChmArc<K, V, mtchm::meta::DefConfig>;
-pub type IndexMTRaw<E> = mtchm::RawTree<E, mtchm::meta::DefConfig>;
+pub type IndexMTRaw<E> = mtchm::imp::Raw<E, mtchm::meta::DefConfig>;
 pub type IndexMTCp<K, V> = mtchm::imp::ChmCopy<K, V, mtchm::meta::DefConfig>;
 pub type IndexST<K, V, S = std::collections::hash_map::RandomState> =
     std::collections::hash_map::HashMap<K, V, S>;
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index 8674a394..bac2f467 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -46,6 +46,7 @@ use {
     },
     crossbeam_epoch::CompareExchangeError,
     std::{
+        fmt,
         hash::Hash,
         hash::{BuildHasher, Hasher},
         marker::PhantomData,
@@ -726,3 +727,14 @@ impl<T, C: Config> Drop for RawTree<T, C> {
         gc(&cpin())
     }
 }
+
+impl<T: TreeElement, C: Config> fmt::Debug for RawTree<T, C>
+where
+    T::Key: fmt::Debug,
+    T::Value: fmt::Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let g = cpin();
+        f.debug_map().entries(self.iter_kv(&g)).finish()
+    }
+}
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index 0376e64f..608ccbed 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -310,6 +310,15 @@ pub enum InsertData<'a> {
     Map(HashMap<Ident<'a>, Datacell>),
 }
 
+impl<'a> InsertData<'a> {
+    pub fn column_count(&self) -> usize {
+        match self {
+            Self::Ordered(ord) => ord.len(),
+            Self::Map(m) => m.len(),
+        }
+    }
+}
+
 impl<'a> From<Vec<Datacell>> for InsertData<'a> {
     fn from(v: Vec<Datacell>) -> Self {
         Self::Ordered(v)
@@ -333,6 +342,12 @@ impl<'a> InsertStatement<'a> {
     pub fn new(entity: Entity<'a>, data: InsertData<'a>) -> Self {
         Self { entity, data }
     }
+    pub fn entity(&self) -> Entity<'a> {
+        self.entity
+    }
+    pub fn data(self) -> InsertData<'a> {
+        self.data
+    }
 }
 
 impl<'a> InsertStatement<'a> {
diff --git a/server/src/engine/sync/smart.rs b/server/src/engine/sync/smart.rs
index 47e75430..1d6a26a8 100644
--- a/server/src/engine/sync/smart.rs
+++ b/server/src/engine/sync/smart.rs
@@ -235,9 +235,10 @@ impl EArc {
 
 /// The core atomic reference counter implementation. All smart pointers use this inside
 pub struct RawRC<T> {
-    hptr: NonNull<RawRCData<T>>,
+    rc_data: NonNull<RawRCData<T>>,
 }
 
+#[derive(Debug)]
 struct RawRCData<T> {
     rc: AtomicUsize,
     data: T,
@@ -252,7 +253,7 @@ impl<T> RawRC<T> {
     /// bug: memory leaks and we don't want that. So, it is upto the caller to clean this up
     pub unsafe fn new(data: T) -> Self {
         Self {
-            hptr: NonNull::new_unchecked(Box::leak(Box::new(RawRCData {
+            rc_data: NonNull::new_unchecked(Box::leak(Box::new(RawRCData {
                 rc: AtomicUsize::new(1),
                 data,
             }))),
@@ -261,13 +262,13 @@ impl<T> RawRC<T> {
     pub fn data(&self) -> &T {
         unsafe {
             // UNSAFE(@ohsayan): we believe in the power of barriers!
-            &(*self.hptr.as_ptr()).data
+            &(*self.rc_data.as_ptr()).data
         }
     }
     fn _rc(&self) -> &AtomicUsize {
         unsafe {
             // UNSAFE(@ohsayan): we believe in the power of barriers!
-            &(*self.hptr.as_ptr()).rc
+            &(*self.rc_data.as_ptr()).rc
         }
     }
     /// ## Safety
@@ -299,6 +300,17 @@ impl<T> RawRC<T> {
         // deallocate object
         dropfn();
         // deallocate rc
-        drop(Box::from_raw(self.hptr.as_ptr()));
+        drop(Box::from_raw(self.rc_data.as_ptr()));
+    }
+}
+
+impl<T: fmt::Debug> fmt::Debug for RawRC<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("RawRC")
+            .field("rc_data", unsafe {
+                // UNSAFE(@ohsayan): rc guard
+                self.rc_data.as_ref()
+            })
+            .finish()
     }
 }

From 361acccc09be77c61d74352f823104cdbe3d2799 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 6 May 2023 10:58:37 -0700
Subject: [PATCH 182/310] Use delta for schema changes

Also fixed an issue where an `alter model` would fail to keep added
fields in order.
---
 server/src/engine/core/dml/ins.rs             |   5 +-
 server/src/engine/core/dml/mod.rs             |   2 -
 server/src/engine/core/index/mod.rs           |  24 +-
 .../engine/core/{dml => index}/result_set.rs  |  14 +-
 server/src/engine/core/index/row.rs           | 100 +++++--
 server/src/engine/core/model/alt.rs           |  12 +-
 server/src/engine/core/model/delta.rs         | 248 ++++++++++++++++++
 server/src/engine/core/model/mod.rs           | 108 +-------
 server/src/engine/core/tests/model/alt.rs     |  72 ++++-
 server/src/engine/core/tests/model/crt.rs     |  16 +-
 server/src/engine/idx/mod.rs                  |  10 +-
 server/src/engine/idx/mtchm/imp.rs            |  25 +-
 server/src/engine/idx/mtchm/meta.rs           |  18 +-
 server/src/engine/idx/mtchm/tests.rs          |   6 +-
 server/src/engine/sync/smart.rs               |  18 ++
 15 files changed, 512 insertions(+), 166 deletions(-)
 rename server/src/engine/core/{dml => index}/result_set.rs (88%)
 create mode 100644 server/src/engine/core/model/delta.rs

diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index a07e1b7e..6f701efb 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -41,7 +41,10 @@ pub fn insert(gns: &GlobalNS, insert: InsertStatement) -> DatabaseResult<()> {
         let irmwd = mdl.intent_write_new_data();
         let (pk, data) = prepare_insert(mdl, irmwd.fields(), insert.data())?;
         let g = cpin();
-        if mdl.primary_index().insert(pk, data, &g) {
+        if mdl
+            .primary_index()
+            .insert(pk, data, mdl.delta_state().current_version(), &g)
+        {
             Ok(())
         } else {
             Err(DatabaseError::DmlConstraintViolationDuplicate)
diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index e433558e..5739dfb9 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -25,5 +25,3 @@
 */
 
 mod ins;
-// result
-mod result_set;
diff --git a/server/src/engine/core/index/mod.rs b/server/src/engine/core/index/mod.rs
index d8af8b6e..5bd2c3e9 100644
--- a/server/src/engine/core/index/mod.rs
+++ b/server/src/engine/core/index/mod.rs
@@ -25,19 +25,18 @@
 */
 
 mod key;
+mod result_set;
 mod row;
 
-use {
-    crate::engine::{
-        idx::{IndexBaseSpec, IndexMTRaw, MTIndex},
-        sync::atm::Guard,
-    },
-    parking_lot::RwLock,
+use crate::engine::{
+    core::model::DeltaVersion,
+    idx::{IndexBaseSpec, IndexMTRaw, MTIndex},
+    sync::atm::Guard,
 };
 
 pub use {
     key::PrimaryIndexKey,
-    row::{DcFieldIndex, Row},
+    row::{DcFieldIndex, Row, RowData},
 };
 
 #[derive(Debug)]
@@ -51,7 +50,14 @@ impl PrimaryIndex {
             data: IndexMTRaw::idx_init(),
         }
     }
-    pub fn insert(&self, key: PrimaryIndexKey, data: row::DcFieldIndex, g: &Guard) -> bool {
-        self.data.mt_insert(key, RwLock::new(data), g)
+    pub fn insert(
+        &self,
+        key: PrimaryIndexKey,
+        data: row::DcFieldIndex,
+        delta_version: DeltaVersion,
+        g: &Guard,
+    ) -> bool {
+        self.data
+            .mt_insert(Row::new(key, data, delta_version, delta_version), g)
     }
 }
diff --git a/server/src/engine/core/dml/result_set.rs b/server/src/engine/core/index/result_set.rs
similarity index 88%
rename from server/src/engine/core/dml/result_set.rs
rename to server/src/engine/core/index/result_set.rs
index 03e2750b..d883b1f4 100644
--- a/server/src/engine/core/dml/result_set.rs
+++ b/server/src/engine/core/index/result_set.rs
@@ -36,13 +36,13 @@
 */
 
 use {
-    crate::engine::core::index::{DcFieldIndex, PrimaryIndexKey, Row},
+    crate::engine::core::index::{DcFieldIndex, PrimaryIndexKey, Row, RowData},
     parking_lot::RwLockReadGuard,
 };
 
 pub struct RowSnapshot<'a> {
     key: &'a PrimaryIndexKey,
-    data: RwLockReadGuard<'a, DcFieldIndex>,
+    data: RwLockReadGuard<'a, RowData>,
 }
 
 impl<'a> RowSnapshot<'a> {
@@ -51,10 +51,10 @@ impl<'a> RowSnapshot<'a> {
     /// HOWEVER: This is very inefficient subject to isolation level scrutiny
     #[inline(always)]
     pub fn snapshot(row: &'a Row) -> RowSnapshot<'a> {
-        Self {
-            key: row.d_key(),
-            data: row.d_data().read(),
-        }
+        Self::new_manual(row.d_key(), row.d_data().read())
+    }
+    pub fn new_manual(key: &'a PrimaryIndexKey, data: RwLockReadGuard<'a, RowData>) -> Self {
+        Self { key, data }
     }
     #[inline(always)]
     pub fn row_key(&self) -> &'a PrimaryIndexKey {
@@ -62,6 +62,6 @@ impl<'a> RowSnapshot<'a> {
     }
     #[inline(always)]
     pub fn row_data(&self) -> &DcFieldIndex {
-        &self.data
+        &self.data.fields()
     }
 }
diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
index 376659bd..bd8c82b4 100644
--- a/server/src/engine/core/index/row.rs
+++ b/server/src/engine/core/index/row.rs
@@ -25,13 +25,17 @@
 */
 
 use {
-    super::key::PrimaryIndexKey,
-    crate::engine::{
-        data::cell::Datacell,
-        idx::{meta::hash::HasherNativeFx, mtchm::meta::TreeElement, IndexST},
-        sync::smart::RawRC,
+    super::{key::PrimaryIndexKey, result_set::RowSnapshot},
+    crate::{
+        engine::{
+            core::model::{DeltaKind, DeltaState, DeltaVersion},
+            data::cell::Datacell,
+            idx::{meta::hash::HasherNativeFx, mtchm::meta::TreeElement, IndexST, STIndex},
+            sync::smart::RawRC,
+        },
+        util::compiler,
     },
-    parking_lot::RwLock,
+    parking_lot::{RwLock, RwLockUpgradableReadGuard, RwLockWriteGuard},
     std::mem::ManuallyDrop,
 };
 
@@ -39,50 +43,111 @@ pub type DcFieldIndex = IndexST<Box<str>, Datacell, HasherNativeFx>;
 
 #[derive(Debug)]
 pub struct Row {
+    txn_genesis: DeltaVersion,
     pk: ManuallyDrop<PrimaryIndexKey>,
-    rc: RawRC<RwLock<DcFieldIndex>>,
+    rc: RawRC<RwLock<RowData>>,
+}
+
+#[derive(Debug)]
+pub struct RowData {
+    fields: DcFieldIndex,
+    txn_revised: DeltaVersion,
+}
+
+impl RowData {
+    pub fn fields(&self) -> &DcFieldIndex {
+        &self.fields
+    }
 }
 
 impl TreeElement for Row {
+    type IKey = PrimaryIndexKey;
     type Key = PrimaryIndexKey;
-    type Value = RwLock<DcFieldIndex>;
+    type IValue = DcFieldIndex;
+    type Value = RwLock<RowData>;
+    type VEx1 = DeltaVersion;
+    type VEx2 = DeltaVersion;
     fn key(&self) -> &Self::Key {
-        &self.pk
+        self.d_key()
     }
     fn val(&self) -> &Self::Value {
-        self.rc.data()
+        self.d_data()
     }
-    fn new(k: Self::Key, v: Self::Value) -> Self {
-        Self::new(k, v)
+    fn new(
+        k: Self::Key,
+        v: Self::IValue,
+        txn_genesis: DeltaVersion,
+        txn_revised: DeltaVersion,
+    ) -> Self {
+        Self::new(k, v, txn_genesis, txn_revised)
     }
 }
 
 impl Row {
-    pub fn new(pk: PrimaryIndexKey, data: RwLock<DcFieldIndex>) -> Self {
+    pub fn new(
+        pk: PrimaryIndexKey,
+        data: DcFieldIndex,
+        txn_genesis: DeltaVersion,
+        txn_revised: DeltaVersion,
+    ) -> Self {
         Self {
+            txn_genesis,
             pk: ManuallyDrop::new(pk),
             rc: unsafe {
                 // UNSAFE(@ohsayan): we free this up later
-                RawRC::new(data)
+                RawRC::new(RwLock::new(RowData {
+                    fields: data,
+                    txn_revised,
+                }))
             },
         }
     }
     pub fn with_data_read<T>(&self, f: impl Fn(&DcFieldIndex) -> T) -> T {
         let data = self.rc.data().read();
-        f(&data)
+        f(&data.fields)
     }
     pub fn with_data_write<T>(&self, f: impl Fn(&mut DcFieldIndex) -> T) -> T {
         let mut data = self.rc.data().write();
-        f(&mut data)
+        f(&mut data.fields)
     }
     pub fn d_key(&self) -> &PrimaryIndexKey {
         &self.pk
     }
-    pub fn d_data(&self) -> &RwLock<DcFieldIndex> {
+    pub fn d_data(&self) -> &RwLock<RowData> {
         self.rc.data()
     }
 }
 
+impl Row {
+    pub fn resolve_deltas_and_freeze<'g>(&'g self, delta_state: &DeltaState) -> RowSnapshot<'g> {
+        let rwl_ug = self.d_data().upgradable_read();
+        let current_version = delta_state.current_version();
+        if compiler::likely(current_version <= rwl_ug.txn_revised) {
+            return RowSnapshot::new_manual(
+                self.d_key(),
+                RwLockUpgradableReadGuard::downgrade(rwl_ug),
+            );
+        }
+        // we have deltas to apply
+        let mut wl = RwLockUpgradableReadGuard::upgrade(rwl_ug);
+        let delta_read = delta_state.rguard();
+        let mut max_delta = wl.txn_revised;
+        for (delta_id, delta) in delta_read.resolve_iter_since(wl.txn_revised) {
+            match delta.kind() {
+                DeltaKind::FieldAdd(f) => {
+                    wl.fields.st_insert(f.clone(), Datacell::null());
+                }
+                DeltaKind::FieldRem(f) => {
+                    wl.fields.st_delete(f);
+                }
+            }
+            max_delta = *delta_id;
+        }
+        wl.txn_revised = max_delta;
+        return RowSnapshot::new_manual(self.d_key(), RwLockWriteGuard::downgrade(wl));
+    }
+}
+
 impl Clone for Row {
     fn clone(&self) -> Self {
         let rc = unsafe {
@@ -95,6 +160,7 @@ impl Clone for Row {
                 ManuallyDrop::new(self.pk.raw_clone())
             },
             rc,
+            ..*self
         }
     }
 }
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index 16cce2e8..5bd2d1c0 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -34,7 +34,7 @@ use {
                 DictEntryGeneric,
             },
             error::{DatabaseError, DatabaseResult},
-            idx::{IndexST, IndexSTSeqCns, STIndex},
+            idx::{IndexST, IndexSTSeqCns, STIndex, STIndexSeq},
             ql::{
                 ast::Entity,
                 ddl::{
@@ -263,16 +263,24 @@ impl ModelView {
             match plan.action {
                 AlterAction::Ignore => drop(iwm),
                 AlterAction::Add(new_fields) => {
+                    let mut guard = model.delta_state().wguard();
                     // TODO(@ohsayan): this impacts lockdown duration; fix it
                     new_fields
-                        .st_iter_kv()
+                        .stseq_ord_kv()
                         .map(|(x, y)| (x.clone(), y.clone()))
                         .for_each(|(field_id, field)| {
+                            model
+                                .delta_state()
+                                .append_unresolved_wl_field_add(&mut guard, &field_id);
                             iwm.fields_mut().st_insert(field_id, field);
                         });
                 }
                 AlterAction::Remove(remove) => {
+                    let mut guard = model.delta_state().wguard();
                     remove.iter().for_each(|field_id| {
+                        model
+                            .delta_state()
+                            .append_unresolved_wl_field_rem(&mut guard, field_id.as_str());
                         iwm.fields_mut().st_delete(field_id.as_str());
                     });
                 }
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
new file mode 100644
index 00000000..b82d0838
--- /dev/null
+++ b/server/src/engine/core/model/delta.rs
@@ -0,0 +1,248 @@
+/*
+ * Created on Sat May 06 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{Fields, ModelView},
+    parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard},
+    std::{
+        collections::btree_map::{BTreeMap, Range},
+        sync::atomic::{AtomicU64, Ordering},
+    },
+};
+
+/*
+    sync matrix
+*/
+
+// FIXME(@ohsayan): This an inefficient repr of the matrix; replace it with my other design
+#[derive(Debug)]
+pub struct ISyncMatrix {
+    // virtual privileges
+    /// read/write model
+    v_priv_model_alter: RwLock<()>,
+    /// RW data/block all
+    v_priv_data_new_or_revise: RwLock<()>,
+}
+
+#[cfg(test)]
+impl PartialEq for ISyncMatrix {
+    fn eq(&self, _: &Self) -> bool {
+        true
+    }
+}
+
+#[derive(Debug)]
+pub struct IRModelSMData<'a> {
+    rmodel: RwLockReadGuard<'a, ()>,
+    mdata: RwLockReadGuard<'a, ()>,
+    fields: &'a Fields,
+}
+
+impl<'a> IRModelSMData<'a> {
+    pub fn new(m: &'a ModelView) -> Self {
+        let rmodel = m.sync_matrix().v_priv_model_alter.read();
+        let mdata = m.sync_matrix().v_priv_data_new_or_revise.read();
+        Self {
+            rmodel,
+            mdata,
+            fields: unsafe {
+                // UNSAFE(@ohsayan): we already have acquired this resource
+                m._read_fields()
+            },
+        }
+    }
+    pub fn fields(&'a self) -> &'a Fields {
+        self.fields
+    }
+}
+
+#[derive(Debug)]
+pub struct IRModel<'a> {
+    rmodel: RwLockReadGuard<'a, ()>,
+    fields: &'a Fields,
+}
+
+impl<'a> IRModel<'a> {
+    pub fn new(m: &'a ModelView) -> Self {
+        Self {
+            rmodel: m.sync_matrix().v_priv_model_alter.read(),
+            fields: unsafe {
+                // UNSAFE(@ohsayan): we already have acquired this resource
+                m._read_fields()
+            },
+        }
+    }
+    pub fn fields(&'a self) -> &'a Fields {
+        self.fields
+    }
+}
+
+#[derive(Debug)]
+pub struct IWModel<'a> {
+    wmodel: RwLockWriteGuard<'a, ()>,
+    fields: &'a mut Fields,
+}
+
+impl<'a> IWModel<'a> {
+    pub fn new(m: &'a ModelView) -> Self {
+        Self {
+            wmodel: m.sync_matrix().v_priv_model_alter.write(),
+            fields: unsafe {
+                // UNSAFE(@ohsayan): we have exclusive access to this resource
+                m._read_fields_mut()
+            },
+        }
+    }
+    pub fn fields(&'a self) -> &'a Fields {
+        self.fields
+    }
+    // ALIASING
+    pub fn fields_mut(&mut self) -> &mut Fields {
+        self.fields
+    }
+}
+
+impl ISyncMatrix {
+    pub const fn new() -> Self {
+        Self {
+            v_priv_model_alter: RwLock::new(()),
+            v_priv_data_new_or_revise: RwLock::new(()),
+        }
+    }
+}
+
+/*
+    delta
+*/
+
+#[derive(Debug)]
+pub struct DeltaState {
+    current_version: AtomicU64,
+    deltas: RwLock<BTreeMap<DeltaVersion, DeltaPart>>,
+}
+
+#[derive(Debug)]
+pub struct DeltaPart {
+    kind: DeltaKind,
+}
+
+impl DeltaPart {
+    pub fn kind(&self) -> &DeltaKind {
+        &self.kind
+    }
+}
+
+#[derive(Debug)]
+pub enum DeltaKind {
+    FieldAdd(Box<str>),
+    FieldRem(Box<str>),
+}
+
+impl DeltaPart {
+    fn new(kind: DeltaKind) -> Self {
+        Self { kind }
+    }
+    fn field_add(field_name: &str) -> Self {
+        Self::new(DeltaKind::FieldAdd(field_name.to_owned().into_boxed_str()))
+    }
+    fn field_rem(field_name: &str) -> Self {
+        Self::new(DeltaKind::FieldRem(field_name.to_owned().into_boxed_str()))
+    }
+}
+
+pub struct DeltaIndexWGuard<'a>(RwLockWriteGuard<'a, BTreeMap<DeltaVersion, DeltaPart>>);
+pub struct DeltaIndexRGuard<'a>(RwLockReadGuard<'a, BTreeMap<DeltaVersion, DeltaPart>>);
+impl<'a> DeltaIndexRGuard<'a> {
+    pub fn resolve_iter_since(
+        &self,
+        current_version: DeltaVersion,
+    ) -> Range<DeltaVersion, DeltaPart> {
+        self.0.range(current_version.step()..)
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
+pub struct DeltaVersion(u64);
+impl DeltaVersion {
+    pub const fn genesis() -> Self {
+        Self(0)
+    }
+    #[cfg(test)]
+    pub fn test_new(v: u64) -> Self {
+        Self(v)
+    }
+    fn step(&self) -> Self {
+        Self(self.0 + 1)
+    }
+}
+
+impl DeltaState {
+    pub fn new_resolved() -> Self {
+        Self {
+            current_version: AtomicU64::new(0),
+            deltas: RwLock::new(BTreeMap::new()),
+        }
+    }
+    pub fn wguard<'a>(&'a self) -> DeltaIndexWGuard<'a> {
+        DeltaIndexWGuard(self.deltas.write())
+    }
+    pub fn rguard<'a>(&'a self) -> DeltaIndexRGuard<'a> {
+        DeltaIndexRGuard(self.deltas.read())
+    }
+    pub fn current_version(&self) -> DeltaVersion {
+        self.__delta_current()
+    }
+    pub fn append_unresolved_wl_field_add(&self, guard: &mut DeltaIndexWGuard, field_name: &str) {
+        self.__append_unresolved_delta(&mut guard.0, DeltaPart::field_add(field_name));
+    }
+    pub fn append_unresolved_wl_field_rem(&self, guard: &mut DeltaIndexWGuard, field_name: &str) {
+        self.__append_unresolved_delta(&mut guard.0, DeltaPart::field_rem(field_name));
+    }
+    pub fn append_unresolved_field_add(&self, field_name: &str) {
+        self.append_unresolved_wl_field_add(&mut self.wguard(), field_name);
+    }
+    pub fn append_unresolved_field_rem(&self, field_name: &str) {
+        self.append_unresolved_wl_field_rem(&mut self.wguard(), field_name);
+    }
+}
+
+impl DeltaState {
+    fn __delta_step(&self) -> DeltaVersion {
+        DeltaVersion(self.current_version.fetch_add(1, Ordering::AcqRel))
+    }
+    fn __delta_current(&self) -> DeltaVersion {
+        DeltaVersion(self.current_version.load(Ordering::Acquire))
+    }
+    fn __append_unresolved_delta(
+        &self,
+        w: &mut BTreeMap<DeltaVersion, DeltaPart>,
+        part: DeltaPart,
+    ) -> DeltaVersion {
+        let v = self.__delta_step();
+        w.insert(v, part);
+        v
+    }
+}
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index b61a0b19..1d2cffab 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -25,11 +25,13 @@
 */
 
 pub(super) mod alt;
+mod delta;
 
 #[cfg(test)]
 use std::cell::RefCell;
 
 use {
+    self::delta::{IRModel, IRModelSMData, ISyncMatrix, IWModel},
     super::{index::PrimaryIndex, util::EntityLocator},
     crate::engine::{
         data::{
@@ -45,10 +47,10 @@ use {
             syn::{FieldSpec, LayerSpec},
         },
     },
-    core::cell::UnsafeCell,
-    parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard},
+    std::cell::UnsafeCell,
 };
 
+pub(in crate::engine::core) use self::delta::{DeltaKind, DeltaState, DeltaVersion};
 pub(in crate::engine::core) type Fields = IndexSTSeqCns<Box<str>, Field>;
 
 #[derive(Debug)]
@@ -58,6 +60,7 @@ pub struct ModelView {
     fields: UnsafeCell<Fields>,
     sync_matrix: ISyncMatrix,
     data: PrimaryIndex,
+    delta: DeltaState,
 }
 
 #[cfg(test)]
@@ -114,6 +117,9 @@ impl ModelView {
     pub fn primary_index(&self) -> &PrimaryIndex {
         &self.data
     }
+    pub fn delta_state(&self) -> &DeltaState {
+        &self.delta
+    }
 }
 
 impl ModelView {
@@ -156,6 +162,7 @@ impl ModelView {
                     fields: UnsafeCell::new(fields),
                     sync_matrix: ISyncMatrix::new(),
                     data: PrimaryIndex::new_empty(),
+                    delta: DeltaState::new_resolved(),
                 });
             }
         }
@@ -474,100 +481,3 @@ unsafe fn lverify_list(_: Layer, _: &Datacell) -> bool {
     layertrace("list");
     true
 }
-
-// FIXME(@ohsayan): This an inefficient repr of the matrix; replace it with my other design
-#[derive(Debug)]
-pub struct ISyncMatrix {
-    // virtual privileges
-    /// read/write model
-    v_priv_model_alter: RwLock<()>,
-    /// RW data/block all
-    v_priv_data_new_or_revise: RwLock<()>,
-}
-
-#[cfg(test)]
-impl PartialEq for ISyncMatrix {
-    fn eq(&self, _: &Self) -> bool {
-        true
-    }
-}
-
-#[derive(Debug)]
-pub struct IRModelSMData<'a> {
-    rmodel: RwLockReadGuard<'a, ()>,
-    mdata: RwLockReadGuard<'a, ()>,
-    fields: &'a Fields,
-}
-
-impl<'a> IRModelSMData<'a> {
-    pub fn new(m: &'a ModelView) -> Self {
-        let rmodel = m.sync_matrix().v_priv_model_alter.read();
-        let mdata = m.sync_matrix().v_priv_data_new_or_revise.read();
-        Self {
-            rmodel,
-            mdata,
-            fields: unsafe {
-                // UNSAFE(@ohsayan): we already have acquired this resource
-                m._read_fields()
-            },
-        }
-    }
-    pub fn fields(&'a self) -> &'a Fields {
-        self.fields
-    }
-}
-
-#[derive(Debug)]
-pub struct IRModel<'a> {
-    rmodel: RwLockReadGuard<'a, ()>,
-    fields: &'a Fields,
-}
-
-impl<'a> IRModel<'a> {
-    pub fn new(m: &'a ModelView) -> Self {
-        Self {
-            rmodel: m.sync_matrix().v_priv_model_alter.read(),
-            fields: unsafe {
-                // UNSAFE(@ohsayan): we already have acquired this resource
-                m._read_fields()
-            },
-        }
-    }
-    pub fn fields(&'a self) -> &'a Fields {
-        self.fields
-    }
-}
-
-#[derive(Debug)]
-pub struct IWModel<'a> {
-    wmodel: RwLockWriteGuard<'a, ()>,
-    fields: &'a mut Fields,
-}
-
-impl<'a> IWModel<'a> {
-    pub fn new(m: &'a ModelView) -> Self {
-        Self {
-            wmodel: m.sync_matrix().v_priv_model_alter.write(),
-            fields: unsafe {
-                // UNSAFE(@ohsayan): we have exclusive access to this resource
-                m._read_fields_mut()
-            },
-        }
-    }
-    pub fn fields(&'a self) -> &'a Fields {
-        self.fields
-    }
-    // ALIASING
-    pub fn fields_mut(&mut self) -> &mut Fields {
-        self.fields
-    }
-}
-
-impl ISyncMatrix {
-    pub const fn new() -> Self {
-        Self {
-            v_priv_model_alter: RwLock::new(()),
-            v_priv_data_new_or_revise: RwLock::new(()),
-        }
-    }
-}
diff --git a/server/src/engine/core/tests/model/alt.rs b/server/src/engine/core/tests/model/alt.rs
index 8868a2c9..6377ce57 100644
--- a/server/src/engine/core/tests/model/alt.rs
+++ b/server/src/engine/core/tests/model/alt.rs
@@ -343,9 +343,73 @@ mod plan {
 }
 
 mod exec {
-    use crate::engine::{core::GlobalNS, error::DatabaseError, idx::STIndex};
+    use crate::engine::{
+        core::{
+            model::{DeltaVersion, Field, Layer},
+            GlobalNS,
+        },
+        error::DatabaseError,
+        idx::{STIndex, STIndexSeq},
+    };
     #[test]
-    fn simple_alter() {
+    fn simple_add() {
+        let gns = GlobalNS::empty();
+        super::exec_plan(
+            &gns,
+            true,
+            "create model myspace.mymodel(username: string, col1: uint64)",
+            "alter model myspace.mymodel add (col2 { type: uint32, nullable: true }, col3 { type: uint16, nullable: true })",
+            |model| {
+                let schema = model.intent_read_model();
+                assert_eq!(
+                    schema
+                        .fields()
+                        .stseq_ord_kv()
+                        .rev()
+                        .take(2)
+                        .map(|(id, f)| (id.clone(), f.clone()))
+                        .collect::<Vec<_>>(),
+                    [
+                        ("col3".into(), Field::new([Layer::uint16()].into(), true)),
+                        ("col2".into(), Field::new([Layer::uint32()].into(), true))
+                    ]
+                );
+                assert_eq!(
+                    model.delta_state().current_version(),
+                    DeltaVersion::test_new(2)
+                );
+            },
+        )
+        .unwrap();
+    }
+    #[test]
+    fn simple_remove() {
+        let gns = GlobalNS::empty();
+        super::exec_plan(
+            &gns,
+            true,
+            "create model myspace.mymodel(username: string, col1: uint64, col2: uint32, col3: uint16, col4: uint8)",
+            "alter model myspace.mymodel remove (col1, col2, col3, col4)",
+            |mdl| {
+                let schema = mdl.intent_read_model();
+                assert_eq!(
+                    schema
+                        .fields()
+                        .stseq_ord_kv()
+                        .rev()
+                        .map(|(a, b)| (a.clone(), b.clone()))
+                        .collect::<Vec<_>>(),
+                    [("username".into(), Field::new([Layer::str()].into(), false))]
+                );
+                assert_eq!(
+                    mdl.delta_state().current_version(),
+                    DeltaVersion::test_new(4)
+                );
+            }
+        ).unwrap();
+    }
+    #[test]
+    fn simple_update() {
         let gns = GlobalNS::empty();
         super::exec_plan(
             &gns,
@@ -355,6 +419,10 @@ mod exec {
             |model| {
                 let schema = model.intent_read_model();
                 assert!(schema.fields().st_get("password").unwrap().is_nullable());
+                assert_eq!(
+                    model.delta_state().current_version(),
+                    DeltaVersion::genesis()
+                );
             },
         )
         .unwrap();
diff --git a/server/src/engine/core/tests/model/crt.rs b/server/src/engine/core/tests/model/crt.rs
index a069a63e..a60be1a5 100644
--- a/server/src/engine/core/tests/model/crt.rs
+++ b/server/src/engine/core/tests/model/crt.rs
@@ -28,7 +28,7 @@ mod validation {
     use {
         super::super::create,
         crate::engine::{
-            core::model::{Field, Layer},
+            core::model::{DeltaVersion, Field, Layer},
             data::tag::{DataTag, FullTag},
             error::DatabaseError,
             idx::STIndexSeq,
@@ -52,6 +52,10 @@ mod validation {
                 Field::new([Layer::new_test(FullTag::BIN, [0; 2])].into(), false)
             ]
         );
+        assert_eq!(
+            model.delta_state().current_version(),
+            DeltaVersion::genesis()
+        );
     }
 
     #[test]
@@ -72,6 +76,10 @@ mod validation {
                 Field::new([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
             ]
         );
+        assert_eq!(
+            model.delta_state().current_version(),
+            DeltaVersion::genesis()
+        );
     }
 
     #[test]
@@ -125,7 +133,7 @@ mod validation {
 mod exec {
     use crate::engine::{
         core::{
-            model::{Field, Layer},
+            model::{DeltaVersion, Field, Layer},
             tests::model::{exec_create_new_space, with_model},
             GlobalNS,
         },
@@ -165,6 +173,10 @@ mod exec {
                     )
                 ]
             );
+            assert_eq!(
+                model.delta_state().current_version(),
+                DeltaVersion::genesis()
+            );
         });
     }
 }
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 04410d14..3851483c 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -127,7 +127,7 @@ pub trait IndexBaseSpec: Sized {
 }
 
 /// An unordered MTIndex
-pub trait MTIndex<K, V>: IndexBaseSpec {
+pub trait MTIndex<E, K, V>: IndexBaseSpec {
     type IterKV<'t, 'g, 'v>: Iterator<Item = (&'v K, &'v V)>
     where
         'g: 't + 'v,
@@ -154,11 +154,11 @@ pub trait MTIndex<K, V>: IndexBaseSpec {
     // write
     /// Returns true if the entry was inserted successfully; returns false if the uniqueness constraint is
     /// violated
-    fn mt_insert(&self, key: K, val: V, g: &Guard) -> bool
+    fn mt_insert(&self, e: E, g: &Guard) -> bool
     where
         V: AsValue;
     /// Updates or inserts the given value
-    fn mt_upsert(&self, key: K, val: V, g: &Guard)
+    fn mt_upsert(&self, e: E, g: &Guard)
     where
         V: AsValue;
     // read
@@ -178,12 +178,12 @@ pub trait MTIndex<K, V>: IndexBaseSpec {
         V: AsValueClone;
     // update
     /// Returns true if the entry is updated
-    fn mt_update(&self, key: K, val: V, g: &Guard) -> bool
+    fn mt_update(&self, e: E, g: &Guard) -> bool
     where
         K: AsKeyClone,
         V: AsValue;
     /// Updates the entry and returns the old value, if it exists
-    fn mt_update_return<'t, 'g, 'v>(&'t self, key: K, val: V, g: &'g Guard) -> Option<&'v V>
+    fn mt_update_return<'t, 'g, 'v>(&'t self, e: E, g: &'g Guard) -> Option<&'v V>
     where
         K: AsKeyClone,
         V: AsValue,
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index 67a652d9..c48e764e 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -47,6 +47,7 @@ pub type ChmCopy<K, V, C> = Raw<(K, V), C>;
 impl<E, C: Config> IndexBaseSpec for Raw<E, C> {
     const PREALLOC: bool = false;
 
+    #[cfg(debug_assertions)]
     type Metrics = CHTRuntimeLog;
 
     fn idx_init() -> Self {
@@ -57,12 +58,13 @@ impl<E, C: Config> IndexBaseSpec for Raw<E, C> {
         s
     }
 
+    #[cfg(debug_assertions)]
     fn idx_metrics(&self) -> &Self::Metrics {
         &self.m
     }
 }
 
-impl<E: TreeElement, C: Config> MTIndex<E::Key, E::Value> for Raw<E, C> {
+impl<E: TreeElement, C: Config> MTIndex<E, E::Key, E::Value> for Raw<E, C> {
     type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, E, C>
     where
         'g: 't + 'v,
@@ -89,18 +91,18 @@ impl<E: TreeElement, C: Config> MTIndex<E::Key, E::Value> for Raw<E, C> {
         self.nontransactional_clear(g)
     }
 
-    fn mt_insert(&self, key: E::Key, val: E::Value, g: &Guard) -> bool
+    fn mt_insert(&self, e: E, g: &Guard) -> bool
     where
         E::Value: AsValue,
     {
-        self.patch(VanillaInsert(E::new(key, val)), g)
+        self.patch(VanillaInsert(e), g)
     }
 
-    fn mt_upsert(&self, key: E::Key, val: E::Value, g: &Guard)
+    fn mt_upsert(&self, e: E, g: &Guard)
     where
         E::Value: AsValue,
     {
-        self.patch(VanillaUpsert(E::new(key, val)), g)
+        self.patch(VanillaUpsert(e), g)
     }
 
     fn mt_contains<Q>(&self, key: &Q, g: &Guard) -> bool
@@ -127,27 +129,22 @@ impl<E: TreeElement, C: Config> MTIndex<E::Key, E::Value> for Raw<E, C> {
         self.get(key, g).cloned()
     }
 
-    fn mt_update(&self, key: E::Key, val: E::Value, g: &Guard) -> bool
+    fn mt_update(&self, e: E, g: &Guard) -> bool
     where
         E::Key: AsKeyClone,
         E::Value: AsValue,
     {
-        self.patch(VanillaUpdate(E::new(key, val)), g)
+        self.patch(VanillaUpdate(e), g)
     }
 
-    fn mt_update_return<'t, 'g, 'v>(
-        &'t self,
-        key: E::Key,
-        val: E::Value,
-        g: &'g Guard,
-    ) -> Option<&'v E::Value>
+    fn mt_update_return<'t, 'g, 'v>(&'t self, e: E, g: &'g Guard) -> Option<&'v E::Value>
     where
         E::Key: AsKeyClone,
         E::Value: AsValue,
         't: 'v,
         'g: 't + 'v,
     {
-        self.patch(VanillaUpdateRet(E::new(key, val)), g)
+        self.patch(VanillaUpdateRet(e), g)
     }
 
     fn mt_delete<Q>(&self, key: &Q, g: &Guard) -> bool
diff --git a/server/src/engine/idx/mtchm/meta.rs b/server/src/engine/idx/mtchm/meta.rs
index 45edb746..874e18d2 100644
--- a/server/src/engine/idx/mtchm/meta.rs
+++ b/server/src/engine/idx/mtchm/meta.rs
@@ -71,15 +71,23 @@ impl<T: AsHasher> PreConfig for Config2B<T> {
 
 pub trait TreeElement: Clone + 'static {
     type Key: AsKey;
+    type IKey;
     type Value: AsValue;
+    type IValue;
+    type VEx1;
+    type VEx2;
     fn key(&self) -> &Self::Key;
     fn val(&self) -> &Self::Value;
-    fn new(k: Self::Key, v: Self::Value) -> Self;
+    fn new(k: Self::IKey, v: Self::IValue, vex1: Self::VEx1, vex2: Self::VEx2) -> Self;
 }
 
 impl<K: AsKeyClone, V: AsValueClone> TreeElement for (K, V) {
+    type IKey = K;
     type Key = K;
+    type IValue = V;
     type Value = V;
+    type VEx1 = ();
+    type VEx2 = ();
     #[inline(always)]
     fn key(&self) -> &K {
         &self.0
@@ -88,14 +96,18 @@ impl<K: AsKeyClone, V: AsValueClone> TreeElement for (K, V) {
     fn val(&self) -> &V {
         &self.1
     }
-    fn new(k: Self::Key, v: Self::Value) -> Self {
+    fn new(k: Self::Key, v: Self::Value, _: (), _: ()) -> Self {
         (k, v)
     }
 }
 
 impl<K: AsKey, V: AsValue> TreeElement for Arc<(K, V)> {
+    type IKey = K;
     type Key = K;
+    type IValue = V;
     type Value = V;
+    type VEx1 = ();
+    type VEx2 = ();
     #[inline(always)]
     fn key(&self) -> &K {
         &self.0
@@ -104,7 +116,7 @@ impl<K: AsKey, V: AsValue> TreeElement for Arc<(K, V)> {
     fn val(&self) -> &V {
         &self.1
     }
-    fn new(k: Self::Key, v: Self::Value) -> Self {
+    fn new(k: Self::Key, v: Self::Value, _: (), _: ()) -> Self {
         Arc::new((k, v))
     }
 }
diff --git a/server/src/engine/idx/mtchm/tests.rs b/server/src/engine/idx/mtchm/tests.rs
index f60fc89b..236b80d0 100644
--- a/server/src/engine/idx/mtchm/tests.rs
+++ b/server/src/engine/idx/mtchm/tests.rs
@@ -106,7 +106,7 @@ fn get_empty() {
 #[test]
 fn update_empty() {
     let idx = ChmU8::idx_init();
-    assert!(!idx.mt_update(10, 20, &cpin()));
+    assert!(!idx.mt_update((10, 20), &cpin()));
 }
 
 const SPAM_QCOUNT: usize = if crate::util::IS_ON_CI {
@@ -234,7 +234,7 @@ fn _action_put<C: Config>(
     let _token = token.acquire_permit();
     let g = cpin();
     data.into_iter().for_each(|(k, v)| {
-        assert!(idx.mt_insert(k, v, &g));
+        assert!(idx.mt_insert((k, v), &g));
     });
 }
 fn _verify_eq<C: Config>(
@@ -283,7 +283,7 @@ fn multispam_update() {
             let _permit = tok.acquire_permit();
             chunk.into_iter().for_each(|(k, v)| {
                 let ret = idx
-                    .mt_update_return(k.clone(), v, &g)
+                    .mt_update_return((k.clone(), v), &g)
                     .expect(&format!("couldn't find key: {}", k));
                 assert_eq!(
                     ret.as_str().parse::<usize>().unwrap(),
diff --git a/server/src/engine/sync/smart.rs b/server/src/engine/sync/smart.rs
index 1d6a26a8..a0770592 100644
--- a/server/src/engine/sync/smart.rs
+++ b/server/src/engine/sync/smart.rs
@@ -81,6 +81,24 @@ impl PartialEq<str> for StrRC {
     }
 }
 
+impl From<String> for StrRC {
+    fn from(s: String) -> Self {
+        Self::from_bx(s.into_boxed_str())
+    }
+}
+
+impl From<Box<str>> for StrRC {
+    fn from(bx: Box<str>) -> Self {
+        Self::from_bx(bx)
+    }
+}
+
+impl<'a> From<&'a str> for StrRC {
+    fn from(str: &'a str) -> Self {
+        Self::from_bx(str.to_string().into_boxed_str())
+    }
+}
+
 impl Deref for StrRC {
     type Target = str;
     fn deref(&self) -> &Self::Target {

From 2488053318ef3929b728855d854bba3e494de09a Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 9 May 2023 09:44:19 -0700
Subject: [PATCH 183/310] Implement delete and add impls required for other dml
 queries

---
 server/src/engine/core/dml/del.rs          |  46 ++++++++
 server/src/engine/core/dml/ins.rs          |   4 +-
 server/src/engine/core/dml/mod.rs          |  27 +++++
 server/src/engine/core/index/key.rs        |   6 +
 server/src/engine/core/index/mod.rs        |  12 +-
 server/src/engine/core/index/result_set.rs |  67 -----------
 server/src/engine/core/index/row.rs        |  18 +--
 server/src/engine/core/mod.rs              |   4 +-
 server/src/engine/core/model/alt.rs        |   6 +-
 server/src/engine/core/model/delta.rs      |   8 +-
 server/src/engine/core/model/mod.rs        |  16 +--
 server/src/engine/core/space.rs            |  12 +-
 server/src/engine/core/tests/mod.rs        |   4 +-
 server/src/engine/core/tests/model/alt.rs  |   6 +-
 server/src/engine/core/tests/model/mod.rs  |  10 +-
 server/src/engine/data/cell.rs             |  81 +++++++++++--
 server/src/engine/data/lit.rs              |   1 +
 server/src/engine/error.rs                 |   4 +
 server/src/engine/idx/mod.rs               |   7 ++
 server/src/engine/idx/mtchm/access.rs      |  30 +++++
 server/src/engine/idx/mtchm/imp.rs         |   9 ++
 server/src/engine/idx/mtchm/mod.rs         |   7 ++
 server/src/engine/idx/stord/mod.rs         |   1 +
 server/src/engine/idx/stord/shared.rs      | 131 +++++++++++++++++++++
 server/src/engine/macros.rs                |   6 +
 server/src/engine/ql/dml/del.rs            |  12 ++
 server/src/engine/ql/dml/mod.rs            |  11 +-
 server/src/engine/ql/dml/sel.rs            |  15 +++
 server/src/engine/sync/smart.rs            |   1 +
 29 files changed, 438 insertions(+), 124 deletions(-)
 create mode 100644 server/src/engine/core/dml/del.rs
 delete mode 100644 server/src/engine/core/index/result_set.rs
 create mode 100644 server/src/engine/idx/stord/shared.rs

diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs
new file mode 100644
index 00000000..f85b00fd
--- /dev/null
+++ b/server/src/engine/core/dml/del.rs
@@ -0,0 +1,46 @@
+/*
+ * Created on Sat May 06 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    core::GlobalNS,
+    error::{DatabaseError, DatabaseResult},
+    ql::dml::del::DeleteStatement,
+    sync,
+};
+
+pub fn delete(gns: &GlobalNS, mut delete: DeleteStatement) -> DatabaseResult<()> {
+    gns.with_model(delete.entity(), |model| {
+        let g = sync::atm::cpin();
+        if model
+            .primary_index()
+            .remove(model.resolve_where(delete.clauses_mut())?, &g)
+        {
+            Ok(())
+        } else {
+            Err(DatabaseError::DmlEntryNotFound)
+        }
+    })
+}
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index 6f701efb..3f18df10 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -27,7 +27,7 @@
 use crate::engine::{
     core::{
         index::{DcFieldIndex, PrimaryIndexKey},
-        model::{Fields, ModelView},
+        model::{Fields, ModelData},
         GlobalNS,
     },
     error::{DatabaseError, DatabaseResult},
@@ -53,7 +53,7 @@ pub fn insert(gns: &GlobalNS, insert: InsertStatement) -> DatabaseResult<()> {
 }
 
 fn prepare_insert(
-    model: &ModelView,
+    model: &ModelData,
     fields: &Fields,
     insert: InsertData,
 ) -> DatabaseResult<(PrimaryIndexKey, DcFieldIndex)> {
diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index 5739dfb9..272bf916 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -24,4 +24,31 @@
  *
 */
 
+mod del;
 mod ins;
+
+use crate::engine::{
+    core::model::ModelData,
+    data::{lit::LitIR, spec::DataspecMeta1D, tag::DataTag},
+    error::{DatabaseError, DatabaseResult},
+    ql::dml::WhereClause,
+};
+
+pub use {del::delete, ins::insert};
+
+impl ModelData {
+    pub(self) fn resolve_where<'a>(
+        &self,
+        where_clause: &mut WhereClause<'a>,
+    ) -> DatabaseResult<LitIR<'a>> {
+        match where_clause.clauses_mut().remove(self.p_key().as_bytes()) {
+            Some(clause)
+                if clause.filter_hint_none()
+                    & (clause.rhs().kind().tag_unique() == self.p_tag().tag_unique()) =>
+            {
+                Ok(clause.rhs())
+            }
+            _ => Err(DatabaseError::DmlWhereClauseUnindexedExpr),
+        }
+    }
+}
diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
index c7bcd9d6..ed2e1f61 100644
--- a/server/src/engine/core/index/key.rs
+++ b/server/src/engine/core/index/key.rs
@@ -207,6 +207,12 @@ impl<'a> Comparable<LitIR<'a>> for PrimaryIndexKey {
     }
 }
 
+impl<'a> Comparable<PrimaryIndexKey> for LitIR<'a> {
+    fn cmp_eq(&self, key: &PrimaryIndexKey) -> bool {
+        key == self
+    }
+}
+
 impl fmt::Debug for PrimaryIndexKey {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let mut dbg_struct = f.debug_struct("PrimaryIndexKey");
diff --git a/server/src/engine/core/index/mod.rs b/server/src/engine/core/index/mod.rs
index 5bd2c3e9..9cd33a81 100644
--- a/server/src/engine/core/index/mod.rs
+++ b/server/src/engine/core/index/mod.rs
@@ -25,11 +25,11 @@
 */
 
 mod key;
-mod result_set;
 mod row;
 
 use crate::engine::{
     core::model::DeltaVersion,
+    data::lit::LitIR,
     idx::{IndexBaseSpec, IndexMTRaw, MTIndex},
     sync::atm::Guard,
 };
@@ -60,4 +60,14 @@ impl PrimaryIndex {
         self.data
             .mt_insert(Row::new(key, data, delta_version, delta_version), g)
     }
+    pub fn remove<'a>(&self, key: LitIR<'a>, g: &Guard) -> bool {
+        self.data.mt_delete(&key, g)
+    }
+    pub fn select<'a, 'v, 't: 'v, 'g: 't>(
+        &'t self,
+        key: LitIR<'a>,
+        g: &'g Guard,
+    ) -> Option<&'v Row> {
+        self.data.mt_get_element(&key, g)
+    }
 }
diff --git a/server/src/engine/core/index/result_set.rs b/server/src/engine/core/index/result_set.rs
deleted file mode 100644
index d883b1f4..00000000
--- a/server/src/engine/core/index/result_set.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Created on Tue May 02 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
-    ☢ ISOLATION WARNING ☢
-    ----------------
-    I went with a rather suboptimal solution for v1. Once we have CC
-    we can do much better (at the cost of more complexity, ofcourse).
-
-    We'll roll that out in 0.8.1, I think.
-
-    FIXME(@ohsayan): Fix this
-*/
-
-use {
-    crate::engine::core::index::{DcFieldIndex, PrimaryIndexKey, Row, RowData},
-    parking_lot::RwLockReadGuard,
-};
-
-pub struct RowSnapshot<'a> {
-    key: &'a PrimaryIndexKey,
-    data: RwLockReadGuard<'a, RowData>,
-}
-
-impl<'a> RowSnapshot<'a> {
-    /// The moment you take a snapshot, you essentially "freeze" the row and prevent any changes from happening.
-    ///
-    /// HOWEVER: This is very inefficient subject to isolation level scrutiny
-    #[inline(always)]
-    pub fn snapshot(row: &'a Row) -> RowSnapshot<'a> {
-        Self::new_manual(row.d_key(), row.d_data().read())
-    }
-    pub fn new_manual(key: &'a PrimaryIndexKey, data: RwLockReadGuard<'a, RowData>) -> Self {
-        Self { key, data }
-    }
-    #[inline(always)]
-    pub fn row_key(&self) -> &'a PrimaryIndexKey {
-        self.key
-    }
-    #[inline(always)]
-    pub fn row_data(&self) -> &DcFieldIndex {
-        &self.data.fields()
-    }
-}
diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
index bd8c82b4..08cbbca0 100644
--- a/server/src/engine/core/index/row.rs
+++ b/server/src/engine/core/index/row.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    super::{key::PrimaryIndexKey, result_set::RowSnapshot},
+    super::key::PrimaryIndexKey,
     crate::{
         engine::{
             core::model::{DeltaKind, DeltaState, DeltaVersion},
@@ -35,7 +35,7 @@ use {
         },
         util::compiler,
     },
-    parking_lot::{RwLock, RwLockUpgradableReadGuard, RwLockWriteGuard},
+    parking_lot::{RwLock, RwLockReadGuard, RwLockUpgradableReadGuard, RwLockWriteGuard},
     std::mem::ManuallyDrop,
 };
 
@@ -48,7 +48,7 @@ pub struct Row {
     rc: RawRC<RwLock<RowData>>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub struct RowData {
     fields: DcFieldIndex,
     txn_revised: DeltaVersion,
@@ -119,14 +119,14 @@ impl Row {
 }
 
 impl Row {
-    pub fn resolve_deltas_and_freeze<'g>(&'g self, delta_state: &DeltaState) -> RowSnapshot<'g> {
+    pub fn resolve_deltas_and_freeze<'g>(
+        &'g self,
+        delta_state: &DeltaState,
+    ) -> RwLockReadGuard<'g, RowData> {
         let rwl_ug = self.d_data().upgradable_read();
         let current_version = delta_state.current_version();
         if compiler::likely(current_version <= rwl_ug.txn_revised) {
-            return RowSnapshot::new_manual(
-                self.d_key(),
-                RwLockUpgradableReadGuard::downgrade(rwl_ug),
-            );
+            return RwLockUpgradableReadGuard::downgrade(rwl_ug);
         }
         // we have deltas to apply
         let mut wl = RwLockUpgradableReadGuard::upgrade(rwl_ug);
@@ -144,7 +144,7 @@ impl Row {
             max_delta = *delta_id;
         }
         wl.txn_revised = max_delta;
-        return RowSnapshot::new_manual(self.d_key(), RwLockWriteGuard::downgrade(wl));
+        return RwLockWriteGuard::downgrade(wl);
     }
 }
 
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 6bc1517b..4806b3ab 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -34,7 +34,7 @@ mod util;
 mod tests;
 // imports
 use {
-    self::{model::ModelView, util::EntityLocator},
+    self::{model::ModelData, util::EntityLocator},
     crate::engine::{
         core::space::Space,
         error::{DatabaseError, DatabaseResult},
@@ -81,7 +81,7 @@ impl GlobalNS {
     }
     pub fn with_model<'a, T, E, F>(&self, entity: E, f: F) -> DatabaseResult<T>
     where
-        F: FnOnce(&ModelView) -> DatabaseResult<T>,
+        F: FnOnce(&ModelData) -> DatabaseResult<T>,
         E: 'a + EntityLocator<'a>,
     {
         entity
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index 5bd2d1c0..925defcb 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    super::{Field, IWModel, Layer, ModelView},
+    super::{Field, IWModel, Layer, ModelData},
     crate::{
         engine::{
             core::GlobalNS,
@@ -89,7 +89,7 @@ fn check_nullable(props: &mut HashMap<Box<str>, Option<DictEntryGeneric>>) -> Da
 
 impl<'a> AlterPlan<'a> {
     pub fn fdeltas(
-        mv: &ModelView,
+        mv: &ModelData,
         wm: &IWModel,
         AlterModel { model, kind }: AlterModel<'a>,
     ) -> DatabaseResult<AlterPlan<'a>> {
@@ -246,7 +246,7 @@ impl<'a> AlterPlan<'a> {
     }
 }
 
-impl ModelView {
+impl ModelData {
     pub fn exec_alter(gns: &GlobalNS, alter: AlterModel) -> DatabaseResult<()> {
         gns.with_model(alter.model, |model| {
             // make intent
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index b82d0838..b86257a5 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    super::{Fields, ModelView},
+    super::{Fields, ModelData},
     parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard},
     std::{
         collections::btree_map::{BTreeMap, Range},
@@ -62,7 +62,7 @@ pub struct IRModelSMData<'a> {
 }
 
 impl<'a> IRModelSMData<'a> {
-    pub fn new(m: &'a ModelView) -> Self {
+    pub fn new(m: &'a ModelData) -> Self {
         let rmodel = m.sync_matrix().v_priv_model_alter.read();
         let mdata = m.sync_matrix().v_priv_data_new_or_revise.read();
         Self {
@@ -86,7 +86,7 @@ pub struct IRModel<'a> {
 }
 
 impl<'a> IRModel<'a> {
-    pub fn new(m: &'a ModelView) -> Self {
+    pub fn new(m: &'a ModelData) -> Self {
         Self {
             rmodel: m.sync_matrix().v_priv_model_alter.read(),
             fields: unsafe {
@@ -107,7 +107,7 @@ pub struct IWModel<'a> {
 }
 
 impl<'a> IWModel<'a> {
-    pub fn new(m: &'a ModelView) -> Self {
+    pub fn new(m: &'a ModelData) -> Self {
         Self {
             wmodel: m.sync_matrix().v_priv_model_alter.write(),
             fields: unsafe {
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 1d2cffab..ef485c1b 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -39,7 +39,7 @@ use {
             tag::{DataTag, FullTag, TagClass, TagSelector},
         },
         error::{DatabaseError, DatabaseResult},
-        idx::{IndexSTSeqCns, STIndex, STIndexSeq},
+        idx::{IndexBaseSpec, IndexSTCOrdRC, STIndex, STIndexSeq},
         mem::VInline,
         ql::ddl::{
             crt::CreateModel,
@@ -51,10 +51,10 @@ use {
 };
 
 pub(in crate::engine::core) use self::delta::{DeltaKind, DeltaState, DeltaVersion};
-pub(in crate::engine::core) type Fields = IndexSTSeqCns<Box<str>, Field>;
+pub(in crate::engine::core) type Fields = IndexSTCOrdRC<Box<str>, Field>;
 
 #[derive(Debug)]
-pub struct ModelView {
+pub struct ModelData {
     p_key: Box<str>,
     p_tag: FullTag,
     fields: UnsafeCell<Fields>,
@@ -64,7 +64,7 @@ pub struct ModelView {
 }
 
 #[cfg(test)]
-impl PartialEq for ModelView {
+impl PartialEq for ModelData {
     fn eq(&self, m: &Self) -> bool {
         let mdl1 = self.intent_read_model();
         let mdl2 = m.intent_read_model();
@@ -72,7 +72,7 @@ impl PartialEq for ModelView {
     }
 }
 
-impl ModelView {
+impl ModelData {
     pub fn p_key(&self) -> &str {
         &self.p_key
     }
@@ -122,7 +122,7 @@ impl ModelView {
     }
 }
 
-impl ModelView {
+impl ModelData {
     pub fn process_create(
         CreateModel {
             model_name: _,
@@ -133,7 +133,7 @@ impl ModelView {
         let mut okay = props.is_empty() & !fields.is_empty();
         // validate fields
         let mut field_spec = fields.into_iter();
-        let mut fields = IndexSTSeqCns::with_capacity(field_spec.len());
+        let mut fields = Fields::idx_init_cap(field_spec.len());
         let mut last_pk = None;
         let mut pk_cnt = 0;
         while (field_spec.len() != 0) & okay {
@@ -170,7 +170,7 @@ impl ModelView {
     }
 }
 
-impl ModelView {
+impl ModelData {
     pub fn exec_create(gns: &super::GlobalNS, stmt: CreateModel) -> DatabaseResult<()> {
         let (space_name, model_name) = stmt.model_name.parse_entity()?;
         let model = Self::process_create(stmt)?;
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index abbc96bf..8d9e0e0e 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -26,7 +26,7 @@
 
 use {
     crate::engine::{
-        core::{model::ModelView, RWLIdx},
+        core::{model::ModelData, RWLIdx},
         data::{md_dict, DictEntryGeneric, MetaDict},
         error::{DatabaseError, DatabaseResult},
         idx::{IndexST, STIndex},
@@ -38,7 +38,7 @@ use {
 #[derive(Debug)]
 /// A space with the model namespace
 pub struct Space {
-    mns: RWLIdx<Box<str>, ModelView>,
+    mns: RWLIdx<Box<str>, ModelData>,
     pub(super) meta: SpaceMeta,
 }
 
@@ -74,7 +74,7 @@ impl ProcedureCreate {
 }
 
 impl Space {
-    pub fn _create_model(&self, name: &str, model: ModelView) -> DatabaseResult<()> {
+    pub fn _create_model(&self, name: &str, model: ModelData) -> DatabaseResult<()> {
         if self
             .mns
             .write()
@@ -85,13 +85,13 @@ impl Space {
             Err(DatabaseError::DdlModelAlreadyExists)
         }
     }
-    pub(super) fn models(&self) -> &RWLIdx<Box<str>, ModelView> {
+    pub(super) fn models(&self) -> &RWLIdx<Box<str>, ModelData> {
         &self.mns
     }
     pub fn with_model<T>(
         &self,
         model: &str,
-        f: impl FnOnce(&ModelView) -> DatabaseResult<T>,
+        f: impl FnOnce(&ModelData) -> DatabaseResult<T>,
     ) -> DatabaseResult<T> {
         let mread = self.mns.read();
         let Some(model) = mread.st_get(model) else {
@@ -106,7 +106,7 @@ impl Space {
         Space::new(Default::default(), SpaceMeta::with_env(into_dict! {}))
     }
     #[inline(always)]
-    pub fn new(mns: IndexST<Box<str>, ModelView>, meta: SpaceMeta) -> Self {
+    pub fn new(mns: IndexST<Box<str>, ModelData>, meta: SpaceMeta) -> Self {
         Self {
             mns: RWLIdx::new(mns),
             meta,
diff --git a/server/src/engine/core/tests/mod.rs b/server/src/engine/core/tests/mod.rs
index 63b43696..b40024aa 100644
--- a/server/src/engine/core/tests/mod.rs
+++ b/server/src/engine/core/tests/mod.rs
@@ -25,7 +25,5 @@
 */
 
 // ddl
-// space
-mod space;
-// model
 mod model;
+mod space;
diff --git a/server/src/engine/core/tests/model/alt.rs b/server/src/engine/core/tests/model/alt.rs
index 6377ce57..e39b13d5 100644
--- a/server/src/engine/core/tests/model/alt.rs
+++ b/server/src/engine/core/tests/model/alt.rs
@@ -26,7 +26,7 @@
 
 use crate::engine::{
     core::{
-        model::{alt::AlterPlan, ModelView},
+        model::{alt::AlterPlan, ModelData},
         tests::model::{create, exec_create},
         GlobalNS,
     },
@@ -51,13 +51,13 @@ fn exec_plan(
     new_space: bool,
     model: &str,
     plan: &str,
-    f: impl Fn(&ModelView),
+    f: impl Fn(&ModelData),
 ) -> DatabaseResult<()> {
     exec_create(gns, model, new_space)?;
     let tok = lex_insecure(plan.as_bytes()).unwrap();
     let alter = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
     let (_space, model_name) = alter.model.into_full().unwrap();
-    ModelView::exec_alter(gns, alter)?;
+    ModelData::exec_alter(gns, alter)?;
     let gns_read = gns.spaces().read();
     let space = gns_read.st_get("myspace").unwrap();
     let model = space.models().read();
diff --git a/server/src/engine/core/tests/model/mod.rs b/server/src/engine/core/tests/model/mod.rs
index 85f31ec8..dfebbba0 100644
--- a/server/src/engine/core/tests/model/mod.rs
+++ b/server/src/engine/core/tests/model/mod.rs
@@ -29,16 +29,16 @@ mod crt;
 mod layer;
 
 use crate::engine::{
-    core::{model::ModelView, space::Space, GlobalNS},
+    core::{model::ModelData, space::Space, GlobalNS},
     error::DatabaseResult,
     idx::STIndex,
     ql::{ast::parse_ast_node_full, ddl::crt::CreateModel, tests::lex_insecure},
 };
 
-fn create(s: &str) -> DatabaseResult<ModelView> {
+fn create(s: &str) -> DatabaseResult<ModelData> {
     let tok = lex_insecure(s.as_bytes()).unwrap();
     let create_model = parse_ast_node_full(&tok[2..]).unwrap();
-    ModelView::process_create(create_model)
+    ModelData::process_create(create_model)
 }
 
 pub fn exec_create(
@@ -51,7 +51,7 @@ pub fn exec_create(
     if create_new_space {
         gns.test_new_empty_space(&create_model.model_name.into_full().unwrap().0);
     }
-    ModelView::exec_create(gns, create_model)
+    ModelData::exec_create(gns, create_model)
 }
 
 pub fn exec_create_new_space(gns: &GlobalNS, create_stmt: &str) -> DatabaseResult<()> {
@@ -68,7 +68,7 @@ fn with_space(gns: &GlobalNS, space_name: &str, f: impl Fn(&Space)) {
     f(space);
 }
 
-fn with_model(gns: &GlobalNS, space_id: &str, model_name: &str, f: impl Fn(&ModelView)) {
+fn with_model(gns: &GlobalNS, space_id: &str, model_name: &str, f: impl Fn(&ModelData)) {
     with_space(gns, space_id, |space| {
         let space_rl = space.models().read();
         let model = space_rl.st_get(model_name).unwrap();
diff --git a/server/src/engine/data/cell.rs b/server/src/engine/data/cell.rs
index ae5e2385..fc9c8e59 100644
--- a/server/src/engine/data/cell.rs
+++ b/server/src/engine/data/cell.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use std::{marker::PhantomData, ops::Deref};
+
 use {
     crate::engine::{
         self,
@@ -236,7 +238,10 @@ impl<'a> From<LitIR<'a>> for Datacell {
                 let mut bin = ManuallyDrop::new(l.read_bin_uck().to_owned().into_boxed_slice());
                 Datacell::new(
                     CUTag::from(l.kind()),
-                    DataRaw::word(WordIO::store((bin.len(), bin.as_mut_ptr()))),
+                    DataRaw::word(DwordQN::dwordqn_store_qw_nw(
+                        bin.len() as u64,
+                        bin.as_mut_ptr() as usize,
+                    )),
                 )
             },
             _ => unsafe {
@@ -309,6 +314,13 @@ impl Datacell {
     unsafe fn _new(tag: CUTag, data: DataRaw, init: bool) -> Self {
         Self { init, tag, data }
     }
+    pub unsafe fn upgrade_from(a: u64, b: usize, tag: CUTag) -> Self {
+        Self {
+            init: true,
+            tag,
+            data: DataRaw::word(DwordQN::dwordqn_store_qw_nw(a, b)),
+        }
+    }
     unsafe fn new(tag: CUTag, data: DataRaw) -> Self {
         Self::_new(tag, data, true)
     }
@@ -416,22 +428,19 @@ impl Clone for Datacell {
             TagClass::Str | TagClass::Bin => unsafe {
                 // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class
                 let mut block = ManuallyDrop::new(self.read_bin().to_owned().into_boxed_slice());
-                DataRaw {
-                    word: ManuallyDrop::new(WordIO::store((block.len(), block.as_mut_ptr()))),
-                }
+                DataRaw::word(DwordQN::dwordqn_store_qw_nw(
+                    block.len() as u64,
+                    block.as_mut_ptr() as usize,
+                ))
             },
             TagClass::List => unsafe {
                 // UNSAFE(@ohsayan): we have checked that the cell is initialized (uninit will not satisfy this class), and we have checked its class
                 let data = self.read_list().read().iter().cloned().collect();
-                DataRaw {
-                    rwl: ManuallyDrop::new(RwLock::new(data)),
-                }
+                DataRaw::rwl(RwLock::new(data))
             },
             _ => unsafe {
                 // UNSAFE(@ohsayan): we have checked that the cell is a stack class
-                DataRaw {
-                    word: ManuallyDrop::new(mem::transmute_copy(&self.data.word)),
-                }
+                DataRaw::word(mem::transmute_copy(&self.data.word))
             },
         };
         unsafe {
@@ -440,3 +449,55 @@ impl Clone for Datacell {
         }
     }
 }
+
+#[derive(Debug)]
+pub struct VirtualDatacell<'a> {
+    dc: ManuallyDrop<Datacell>,
+    _lt: PhantomData<LitIR<'a>>,
+}
+
+impl<'a> VirtualDatacell<'a> {
+    pub fn new(litir: LitIR<'a>) -> Self {
+        Self {
+            dc: ManuallyDrop::new(unsafe {
+                // UNSAFE(@ohsayan): this is a "reference" to a "virtual" aka fake DC. this just works because of memory layouts
+                Datacell::new(
+                    CUTag::from(litir.kind()),
+                    DataRaw::word(litir.data().dwordqn_promote()),
+                )
+            }),
+            _lt: PhantomData,
+        }
+    }
+}
+
+impl<'a> From<LitIR<'a>> for VirtualDatacell<'a> {
+    fn from(l: LitIR<'a>) -> Self {
+        Self::new(l)
+    }
+}
+
+impl<'a> Deref for VirtualDatacell<'a> {
+    type Target = Datacell;
+    fn deref(&self) -> &Self::Target {
+        &self.dc
+    }
+}
+
+impl<'a> PartialEq<Datacell> for VirtualDatacell<'a> {
+    fn eq(&self, other: &Datacell) -> bool {
+        self.dc.deref() == other
+    }
+}
+
+impl<'a> Clone for VirtualDatacell<'a> {
+    fn clone(&self) -> Self {
+        unsafe { core::mem::transmute_copy(self) }
+    }
+}
+
+#[test]
+fn virtual_dc_damn() {
+    let dc = LitIR::Str("hello, world");
+    assert_eq!(VirtualDatacell::from(dc), Datacell::from("hello, world"));
+}
diff --git a/server/src/engine/data/lit.rs b/server/src/engine/data/lit.rs
index 0ef9e035..d40b02ff 100644
--- a/server/src/engine/data/lit.rs
+++ b/server/src/engine/data/lit.rs
@@ -271,6 +271,7 @@ impl<'a, T: DataspecMethods1D> PartialEq<T> for LitIR<'a> {
         <Self as DataspecMethods1D>::self_eq(self, other)
     }
 }
+
 impl<'a> fmt::Debug for LitIR<'a> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let mut f = f.debug_struct("LitIR");
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 4720cc5f..9ca4db2c 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -124,4 +124,8 @@ pub enum DatabaseError {
     DmlConstraintViolationDuplicate,
     /// data validation error
     DmlDataValidationError,
+    /// The expression in a where clause is not indexed (in the way the expression expects it to be)
+    DmlWhereClauseUnindexedExpr,
+    /// The entry was not found
+    DmlEntryNotFound,
 }
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 3851483c..7a07a1df 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -41,6 +41,8 @@ use {
 
 // re-exports
 pub type IndexSTSeqCns<K, V> = stord::IndexSTSeqDll<K, V, stord::config::ConservativeConfig<K, V>>;
+pub type IndexSTCOrdRC<K, V> =
+    stord::shared::OrderedIdxRC<K, V, stord::config::ConservativeConfig<K, V>>;
 pub type IndexSTSeqLib<K, V> = stord::IndexSTSeqDll<K, V, stord::config::LiberalConfig<K, V>>;
 pub type IndexMTRC<K, V> = mtchm::imp::ChmArc<K, V, mtchm::meta::DefConfig>;
 pub type IndexMTRaw<E> = mtchm::imp::Raw<E, mtchm::meta::DefConfig>;
@@ -167,6 +169,11 @@ pub trait MTIndex<E, K, V>: IndexBaseSpec {
         Q: ?Sized + Comparable<K>;
     /// Returns a reference to the value corresponding to the key, if it exists
     fn mt_get<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v V>
+    where
+        Q: ?Sized + Comparable<K>,
+        't: 'v,
+        'g: 't + 'v;
+    fn mt_get_element<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v E>
     where
         Q: ?Sized + Comparable<K>,
         't: 'v,
diff --git a/server/src/engine/idx/mtchm/access.rs b/server/src/engine/idx/mtchm/access.rs
index 13f20310..e762f64c 100644
--- a/server/src/engine/idx/mtchm/access.rs
+++ b/server/src/engine/idx/mtchm/access.rs
@@ -93,3 +93,33 @@ impl<'re, T: TreeElement, U: Comparable<T::Key> + ?Sized> ReadMode<T> for RModeR
         None
     }
 }
+
+pub struct RModeElementRef<'a, T, U: ?Sized> {
+    target: &'a U,
+    _d: PhantomData<T>,
+}
+
+impl<'a, T, U: ?Sized> RModeElementRef<'a, T, U> {
+    pub fn new(target: &'a U) -> Self {
+        Self {
+            target,
+            _d: PhantomData,
+        }
+    }
+}
+
+impl<'re, T: TreeElement, U: Comparable<T::Key> + ?Sized> ReadMode<T>
+    for RModeElementRef<'re, T, U>
+{
+    type Ret<'a> = Option<&'a T>;
+    type Target = U;
+    fn target(&self) -> &Self::Target {
+        self.target
+    }
+    fn ex(c: &T) -> Self::Ret<'_> {
+        Some(c)
+    }
+    fn nx<'a>() -> Self::Ret<'a> {
+        None
+    }
+}
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index c48e764e..ca71697d 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -121,6 +121,15 @@ impl<E: TreeElement, C: Config> MTIndex<E, E::Key, E::Value> for Raw<E, C> {
         self.get(key, g)
     }
 
+    fn mt_get_element<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v E>
+    where
+        Q: ?Sized + Comparable<E::Key>,
+        't: 'v,
+        'g: 't + 'v,
+    {
+        self.get_full(key, g)
+    }
+
     fn mt_get_cloned<Q>(&self, key: &Q, g: &Guard) -> Option<E::Value>
     where
         Q: ?Sized + Comparable<E::Key>,
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index bac2f467..b57b872c 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -421,6 +421,13 @@ impl<T: TreeElement, C: Config> RawTree<T, C> {
     ) -> Option<&'g T::Value> {
         self._lookup(access::RModeRef::new(k), g)
     }
+    fn get_full<'g, Q: ?Sized + Comparable<T::Key>>(
+        &'g self,
+        k: &Q,
+        g: &'g Guard,
+    ) -> Option<&'g T> {
+        self._lookup(access::RModeElementRef::new(k), g)
+    }
     fn _lookup<'g, R: access::ReadMode<T>>(&'g self, r: R, g: &'g Guard) -> R::Ret<'g> {
         let mut hash = self.hash(r.target());
         let mut current = &self.root;
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index 969f12b4..ebeb2dce 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -26,6 +26,7 @@
 
 pub(super) mod config;
 mod iter;
+pub(super) mod shared;
 
 use {
     self::{
diff --git a/server/src/engine/idx/stord/shared.rs b/server/src/engine/idx/stord/shared.rs
new file mode 100644
index 00000000..dfff4818
--- /dev/null
+++ b/server/src/engine/idx/stord/shared.rs
@@ -0,0 +1,131 @@
+/*
+ * Created on Mon May 08 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[cfg(debug_assertions)]
+use super::IndexSTSeqDllMetrics;
+
+use {
+    super::{config::Config, IndexSTSeqDll},
+    crate::engine::{
+        idx::{AsKey, AsValue, IndexBaseSpec},
+        sync::smart::EArc,
+    },
+    std::{
+        mem::ManuallyDrop,
+        ops::{Deref, DerefMut},
+    },
+};
+
+#[derive(Debug)]
+pub struct OrderedIdxRC<K, V, C: Config<K, V>> {
+    base: ManuallyDrop<IndexSTSeqDll<K, V, C>>,
+    rc: EArc,
+}
+
+impl<K, V, C: Config<K, V>> OrderedIdxRC<K, V, C> {
+    fn new() -> Self {
+        Self::new_with(IndexSTSeqDll::new())
+    }
+    fn new_with(idx: IndexSTSeqDll<K, V, C>) -> Self {
+        Self {
+            base: ManuallyDrop::new(idx),
+            rc: unsafe {
+                // UNSAFE(@ohsayan): we'll clean this up
+                EArc::new()
+            },
+        }
+    }
+}
+
+impl<K, V, C: Config<K, V>> Deref for OrderedIdxRC<K, V, C> {
+    type Target = IndexSTSeqDll<K, V, C>;
+    fn deref(&self) -> &Self::Target {
+        &self.base
+    }
+}
+
+impl<K, V, C: Config<K, V>> DerefMut for OrderedIdxRC<K, V, C> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.base
+    }
+}
+
+impl<K, V, C: Config<K, V>> Clone for OrderedIdxRC<K, V, C> {
+    fn clone(&self) -> Self {
+        let rc = unsafe {
+            // UNSAFE(@ohsayan): called at clone position
+            self.rc.rc_clone()
+        };
+        Self {
+            base: unsafe {
+                // UNSAFE(@ohsayan): just a raw clone. no big deal since this is an RC
+                core::mem::transmute_copy(&self.base)
+            },
+            rc,
+        }
+    }
+}
+
+impl<K, V, C: Config<K, V>> Drop for OrderedIdxRC<K, V, C> {
+    fn drop(&mut self) {
+        unsafe {
+            // UNSAFE(@ohsayan): this is the dtor
+            self.rc.rc_drop(|| ManuallyDrop::drop(&mut self.base))
+        }
+    }
+}
+
+impl<K, V, C: Config<K, V>> IndexBaseSpec for OrderedIdxRC<K, V, C> {
+    const PREALLOC: bool = true;
+    #[cfg(debug_assertions)]
+    type Metrics = IndexSTSeqDllMetrics;
+
+    fn idx_init_cap(cap: usize) -> Self {
+        Self::new_with(IndexSTSeqDll::with_capacity(cap))
+    }
+
+    fn idx_init() -> Self {
+        Self::new()
+    }
+
+    fn idx_init_with(s: Self) -> Self {
+        s
+    }
+
+    #[cfg(debug_assertions)]
+    fn idx_metrics(&self) -> &Self::Metrics {
+        self.base.idx_metrics()
+    }
+}
+
+impl<K: AsKey, V: AsValue + PartialEq, C: Config<K, V>> PartialEq for OrderedIdxRC<K, V, C> {
+    fn eq(&self, other: &Self) -> bool {
+        self.len() == other.len()
+            && self
+                ._iter_unord_kv()
+                .all(|(k, v)| other._get(k).unwrap().eq(v))
+    }
+}
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index 13a29c48..cb2552ae 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -145,3 +145,9 @@ macro_rules! into_dict {
         .collect()
     }};
 }
+
+#[cfg(test)]
+#[allow(unused_macros)]
+macro_rules! into_vec {
+    ($($val:expr),* $(,)?) => { vec![$(::core::convert::From::from($val),)*]};
+}
diff --git a/server/src/engine/ql/dml/del.rs b/server/src/engine/ql/dml/del.rs
index 00e3dbec..e68d21ca 100644
--- a/server/src/engine/ql/dml/del.rs
+++ b/server/src/engine/ql/dml/del.rs
@@ -50,6 +50,18 @@ pub struct DeleteStatement<'a> {
     pub(super) wc: WhereClause<'a>,
 }
 
+impl<'a> DeleteStatement<'a> {
+    pub const fn entity(&self) -> Entity<'a> {
+        self.entity
+    }
+    pub const fn clauses(&self) -> &WhereClause {
+        &self.wc
+    }
+    pub fn clauses_mut(&mut self) -> &mut WhereClause<'a> {
+        &mut self.wc
+    }
+}
+
 impl<'a> DeleteStatement<'a> {
     #[inline(always)]
     pub(super) fn new(entity: Entity<'a>, wc: WhereClause<'a>) -> Self {
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index ca3ccf42..f105ad85 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -74,9 +74,12 @@ impl<'a> RelationalExpr<'a> {
     pub(super) const OP_GE: u8 = 4;
     pub(super) const OP_LT: u8 = 5;
     pub(super) const OP_LE: u8 = 6;
-    fn filter_hint_none(&self) -> bool {
+    pub fn filter_hint_none(&self) -> bool {
         self.opc == Self::OP_EQ
     }
+    pub fn rhs(&self) -> LitIR<'a> {
+        self.rhs.clone()
+    }
     #[inline(always)]
     fn parse_operator<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> u8 {
         let tok = state.current();
@@ -127,6 +130,9 @@ impl<'a> WhereClause<'a> {
     pub(super) fn new(c: WhereClauseCollection<'a>) -> Self {
         Self { c }
     }
+    pub fn clauses_mut(&mut self) -> &mut WhereClauseCollection<'a> {
+        &mut self.c
+    }
     #[inline(always)]
     fn parse_where_and_append_to<Qd: QueryData<'a>>(
         state: &mut State<'a, Qd>,
@@ -152,6 +158,9 @@ impl<'a> WhereClause<'a> {
         state.poison_if(c.is_empty());
         Self { c }
     }
+    pub fn clauses(&self) -> &WhereClauseCollection {
+        &self.c
+    }
 }
 
 #[cfg(test)]
diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs
index 416d3d4c..c4f8b219 100644
--- a/server/src/engine/ql/dml/sel.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -78,6 +78,21 @@ impl<'a> SelectStatement<'a> {
             clause: WhereClause::new(clauses),
         }
     }
+    pub fn entity(&self) -> Entity<'a> {
+        self.entity
+    }
+    pub fn clauses(&self) -> &WhereClause<'a> {
+        &self.clause
+    }
+    pub fn clauses_mut(&mut self) -> &mut WhereClause<'a> {
+        &mut self.clause
+    }
+    pub fn is_wildcard(&self) -> bool {
+        self.wildcard
+    }
+    pub fn into_fields(self) -> Vec<Ident<'a>> {
+        self.fields
+    }
 }
 
 impl<'a> SelectStatement<'a> {
diff --git a/server/src/engine/sync/smart.rs b/server/src/engine/sync/smart.rs
index a0770592..66254cb8 100644
--- a/server/src/engine/sync/smart.rs
+++ b/server/src/engine/sync/smart.rs
@@ -219,6 +219,7 @@ unsafe impl<T: Send> Send for SliceRC<T> {}
 unsafe impl<T: Sync> Sync for SliceRC<T> {}
 
 /// A simple rc
+#[derive(Debug)]
 pub struct EArc {
     base: RawRC<()>,
 }

From b714647905e21fee53c3968556e694f04a0dbbc6 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 9 May 2023 23:50:41 -0700
Subject: [PATCH 184/310] Add tests for insert

---
 server/src/engine/core/dml/ins.rs             |  1 +
 server/src/engine/core/index/row.rs           |  9 +++
 .../core/tests/{model => ddl_model}/alt.rs    |  2 +-
 .../core/tests/{model => ddl_model}/crt.rs    |  2 +-
 .../core/tests/{model => ddl_model}/layer.rs  |  0
 .../core/tests/{model => ddl_model}/mod.rs    |  0
 .../core/tests/{space => ddl_space}/alter.rs  |  0
 .../core/tests/{space => ddl_space}/create.rs |  0
 .../core/tests/{space => ddl_space}/mod.rs    |  0
 server/src/engine/core/tests/dml/insert.rs    | 68 +++++++++++++++++++
 server/src/engine/core/tests/dml/mod.rs       | 65 ++++++++++++++++++
 server/src/engine/core/tests/mod.rs           |  6 +-
 server/src/engine/macros.rs                   |  5 +-
 13 files changed, 150 insertions(+), 8 deletions(-)
 rename server/src/engine/core/tests/{model => ddl_model}/alt.rs (99%)
 rename server/src/engine/core/tests/{model => ddl_model}/crt.rs (98%)
 rename server/src/engine/core/tests/{model => ddl_model}/layer.rs (100%)
 rename server/src/engine/core/tests/{model => ddl_model}/mod.rs (100%)
 rename server/src/engine/core/tests/{space => ddl_space}/alter.rs (100%)
 rename server/src/engine/core/tests/{space => ddl_space}/create.rs (100%)
 rename server/src/engine/core/tests/{space => ddl_space}/mod.rs (100%)
 create mode 100644 server/src/engine/core/tests/dml/insert.rs
 create mode 100644 server/src/engine/core/tests/dml/mod.rs

diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index 3f18df10..0c229d1e 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -52,6 +52,7 @@ pub fn insert(gns: &GlobalNS, insert: InsertStatement) -> DatabaseResult<()> {
     })
 }
 
+// TODO(@ohsayan): optimize null case
 fn prepare_insert(
     model: &ModelData,
     fields: &Fields,
diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
index 08cbbca0..46e5962e 100644
--- a/server/src/engine/core/index/row.rs
+++ b/server/src/engine/core/index/row.rs
@@ -116,6 +116,15 @@ impl Row {
     pub fn d_data(&self) -> &RwLock<RowData> {
         self.rc.data()
     }
+    #[cfg(test)]
+    pub fn cloned_data(&self) -> Vec<(Box<str>, Datacell)> {
+        self.d_data()
+            .read()
+            .fields()
+            .st_iter_kv()
+            .map(|(id, data)| (id.clone(), data.clone()))
+            .collect()
+    }
 }
 
 impl Row {
diff --git a/server/src/engine/core/tests/model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
similarity index 99%
rename from server/src/engine/core/tests/model/alt.rs
rename to server/src/engine/core/tests/ddl_model/alt.rs
index e39b13d5..f2ea295d 100644
--- a/server/src/engine/core/tests/model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -27,7 +27,7 @@
 use crate::engine::{
     core::{
         model::{alt::AlterPlan, ModelData},
-        tests::model::{create, exec_create},
+        tests::ddl_model::{create, exec_create},
         GlobalNS,
     },
     error::DatabaseResult,
diff --git a/server/src/engine/core/tests/model/crt.rs b/server/src/engine/core/tests/ddl_model/crt.rs
similarity index 98%
rename from server/src/engine/core/tests/model/crt.rs
rename to server/src/engine/core/tests/ddl_model/crt.rs
index a60be1a5..72e16501 100644
--- a/server/src/engine/core/tests/model/crt.rs
+++ b/server/src/engine/core/tests/ddl_model/crt.rs
@@ -134,7 +134,7 @@ mod exec {
     use crate::engine::{
         core::{
             model::{DeltaVersion, Field, Layer},
-            tests::model::{exec_create_new_space, with_model},
+            tests::ddl_model::{exec_create_new_space, with_model},
             GlobalNS,
         },
         data::tag::{DataTag, FullTag},
diff --git a/server/src/engine/core/tests/model/layer.rs b/server/src/engine/core/tests/ddl_model/layer.rs
similarity index 100%
rename from server/src/engine/core/tests/model/layer.rs
rename to server/src/engine/core/tests/ddl_model/layer.rs
diff --git a/server/src/engine/core/tests/model/mod.rs b/server/src/engine/core/tests/ddl_model/mod.rs
similarity index 100%
rename from server/src/engine/core/tests/model/mod.rs
rename to server/src/engine/core/tests/ddl_model/mod.rs
diff --git a/server/src/engine/core/tests/space/alter.rs b/server/src/engine/core/tests/ddl_space/alter.rs
similarity index 100%
rename from server/src/engine/core/tests/space/alter.rs
rename to server/src/engine/core/tests/ddl_space/alter.rs
diff --git a/server/src/engine/core/tests/space/create.rs b/server/src/engine/core/tests/ddl_space/create.rs
similarity index 100%
rename from server/src/engine/core/tests/space/create.rs
rename to server/src/engine/core/tests/ddl_space/create.rs
diff --git a/server/src/engine/core/tests/space/mod.rs b/server/src/engine/core/tests/ddl_space/mod.rs
similarity index 100%
rename from server/src/engine/core/tests/space/mod.rs
rename to server/src/engine/core/tests/ddl_space/mod.rs
diff --git a/server/src/engine/core/tests/dml/insert.rs b/server/src/engine/core/tests/dml/insert.rs
new file mode 100644
index 00000000..93399b89
--- /dev/null
+++ b/server/src/engine/core/tests/dml/insert.rs
@@ -0,0 +1,68 @@
+/*
+ * Created on Tue May 09 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{core::GlobalNS, data::cell::Datacell};
+
+#[derive(sky_macros::Wrapper)]
+struct Tuple(Vec<(Box<str>, Datacell)>);
+
+#[test]
+fn insert_simple() {
+    let gns = GlobalNS::empty();
+    super::exec_insert(
+        &gns,
+        "create model myspace.mymodel(username: string, password: string)",
+        "insert into myspace.mymodel('sayan', 'pass123')",
+        "sayan",
+        |row| {
+            assert_veceq_transposed!(row.cloned_data(), Tuple(pairvec!(("password", "pass123"))));
+        },
+    )
+    .unwrap();
+}
+
+#[test]
+fn insert_with_null() {
+    let gns = GlobalNS::empty();
+    super::exec_insert(
+        &gns,
+        "create model myspace.mymodel(username: string, null useless_password: string, null useless_email: string, null useless_random_column: uint64)",
+        "insert into myspace.mymodel('sayan', null, null, null)",
+        "sayan",
+        |row| {
+            assert_veceq_transposed!(
+                row.cloned_data(),
+                Tuple(
+                    pairvec!(
+                        ("useless_password", Datacell::null()),
+                        ("useless_email", Datacell::null()),
+                        ("useless_random_column", Datacell::null())
+                    )
+                )
+            )
+        }
+    ).unwrap();
+}
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
new file mode 100644
index 00000000..0798c7c9
--- /dev/null
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -0,0 +1,65 @@
+/*
+ * Created on Tue May 09 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod insert;
+
+use crate::engine::{
+    core::{dml, index::Row, model::ModelData, GlobalNS},
+    data::lit::LitIR,
+    error::DatabaseResult,
+    ql::{ast::parse_ast_node_full, dml::ins::InsertStatement, tests::lex_insecure},
+    sync,
+};
+
+pub(self) fn exec_insert<T>(
+    gns: &GlobalNS,
+    model: &str,
+    insert: &str,
+    key_name: &str,
+    f: impl Fn(Row) -> T,
+) -> DatabaseResult<T> {
+    if !gns.spaces().read().contains_key("myspace") {
+        gns.test_new_empty_space("myspace");
+    }
+    let lex_create_model = lex_insecure(model.as_bytes()).unwrap();
+    let stmt_create_model = parse_ast_node_full(&lex_create_model[2..]).unwrap();
+    ModelData::exec_create(gns, stmt_create_model)?;
+    let lex_insert = lex_insecure(insert.as_bytes()).unwrap();
+    let stmt_insert = parse_ast_node_full::<InsertStatement>(&lex_insert[1..]).unwrap();
+    let entity = stmt_insert.entity();
+    dml::insert(gns, stmt_insert)?;
+    let guard = sync::atm::cpin();
+    gns.with_model(entity, |mdl| {
+        let _irm = mdl.intent_read_model();
+        let row = mdl
+            .primary_index()
+            .select(LitIR::from(key_name), &guard)
+            .unwrap()
+            .clone();
+        drop(guard);
+        Ok(f(row))
+    })
+}
diff --git a/server/src/engine/core/tests/mod.rs b/server/src/engine/core/tests/mod.rs
index b40024aa..a97da00f 100644
--- a/server/src/engine/core/tests/mod.rs
+++ b/server/src/engine/core/tests/mod.rs
@@ -24,6 +24,6 @@
  *
 */
 
-// ddl
-mod model;
-mod space;
+mod ddl_model;
+mod ddl_space;
+mod dml;
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index cb2552ae..daa0ecb5 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -147,7 +147,6 @@ macro_rules! into_dict {
 }
 
 #[cfg(test)]
-#[allow(unused_macros)]
-macro_rules! into_vec {
-    ($($val:expr),* $(,)?) => { vec![$(::core::convert::From::from($val),)*]};
+macro_rules! pairvec {
+    ($($x:expr),*) => {{ let mut v = Vec::new(); $( let (a, b) = $x; v.push((a.into(), b.into())); )* v }};
 }

From abf1e2934417bb819fd90bf0f8c7e7de8cbb81da Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 10 May 2023 11:25:39 -0700
Subject: [PATCH 185/310] Add tests for insert and delete, and fix security bug

This commit makes an important security fix that caused non-matching
of field data which could have had terrible impacts.

Second, it adds tests for insert and delete.
---
 server/src/engine/core/dml/ins.rs          |  4 +-
 server/src/engine/core/tests/dml/delete.rs | 56 +++++++++++++++
 server/src/engine/core/tests/dml/insert.rs | 24 ++++++-
 server/src/engine/core/tests/dml/mod.rs    | 83 +++++++++++++++++++---
 server/src/util/macros.rs                  | 11 ++-
 5 files changed, 161 insertions(+), 17 deletions(-)
 create mode 100644 server/src/engine/core/tests/dml/delete.rs

diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index 0c229d1e..394800c3 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -31,7 +31,7 @@ use crate::engine::{
         GlobalNS,
     },
     error::{DatabaseError, DatabaseResult},
-    idx::{IndexBaseSpec, STIndex},
+    idx::{IndexBaseSpec, STIndex, STIndexSeq},
     ql::dml::ins::{InsertData, InsertStatement},
     sync::atm::cpin,
 };
@@ -62,7 +62,7 @@ fn prepare_insert(
     let mut prepared_data = DcFieldIndex::idx_init_cap(fields.len());
     match insert {
         InsertData::Ordered(tuple) => {
-            let mut fields = fields.st_iter_kv();
+            let mut fields = fields.stseq_ord_kv();
             let mut tuple = tuple.into_iter();
             while (tuple.len() != 0) & okay {
                 let data;
diff --git a/server/src/engine/core/tests/dml/delete.rs b/server/src/engine/core/tests/dml/delete.rs
new file mode 100644
index 00000000..1bb4fcc0
--- /dev/null
+++ b/server/src/engine/core/tests/dml/delete.rs
@@ -0,0 +1,56 @@
+/*
+ * Created on Wed May 10 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{core::GlobalNS, error::DatabaseError};
+
+#[test]
+fn simple_delete() {
+    let gns = GlobalNS::empty();
+    super::exec_delete(
+        &gns,
+        "create model myspace.mymodel(username: string, password: string)",
+        Some("insert into myspace.mymodel('sayan', 'pass123')"),
+        "delete from myspace.mymodel where username = 'sayan'",
+        "sayan",
+    )
+    .unwrap();
+}
+
+#[test]
+fn delete_nonexisting() {
+    let gns = GlobalNS::empty();
+    assert_eq!(
+        super::exec_delete(
+            &gns,
+            "create model myspace.mymodel(username: string, password: string)",
+            None,
+            "delete from myspace.mymodel where username = 'sayan'",
+            "sayan",
+        )
+        .unwrap_err(),
+        DatabaseError::DmlEntryNotFound
+    );
+}
diff --git a/server/src/engine/core/tests/dml/insert.rs b/server/src/engine/core/tests/dml/insert.rs
index 93399b89..10d77ff5 100644
--- a/server/src/engine/core/tests/dml/insert.rs
+++ b/server/src/engine/core/tests/dml/insert.rs
@@ -24,9 +24,9 @@
  *
 */
 
-use crate::engine::{core::GlobalNS, data::cell::Datacell};
+use crate::engine::{core::GlobalNS, data::cell::Datacell, error::DatabaseError};
 
-#[derive(sky_macros::Wrapper)]
+#[derive(sky_macros::Wrapper, Debug)]
 struct Tuple(Vec<(Box<str>, Datacell)>);
 
 #[test]
@@ -66,3 +66,23 @@ fn insert_with_null() {
         }
     ).unwrap();
 }
+
+#[test]
+fn insert_duplicate() {
+    let gns = GlobalNS::empty();
+    super::exec_insert(
+        &gns,
+        "create model myspace.mymodel(username: string, password: string)",
+        "insert into myspace.mymodel('sayan', 'pass123')",
+        "sayan",
+        |row| {
+            assert_veceq_transposed!(row.cloned_data(), Tuple(pairvec!(("password", "pass123"))));
+        },
+    )
+    .unwrap();
+    assert_eq!(
+        super::exec_insert_only(&gns, "insert into myspace.mymodel('sayan', 'pass123')")
+            .unwrap_err(),
+        DatabaseError::DmlConstraintViolationDuplicate
+    );
+}
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index 0798c7c9..d508ba50 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -24,33 +24,49 @@
  *
 */
 
+mod delete;
 mod insert;
 
 use crate::engine::{
     core::{dml, index::Row, model::ModelData, GlobalNS},
     data::lit::LitIR,
     error::DatabaseResult,
-    ql::{ast::parse_ast_node_full, dml::ins::InsertStatement, tests::lex_insecure},
+    ql::{
+        ast::{parse_ast_node_full, Entity},
+        dml::{del::DeleteStatement, ins::InsertStatement},
+        tests::lex_insecure,
+    },
     sync,
 };
 
-pub(self) fn exec_insert<T>(
-    gns: &GlobalNS,
-    model: &str,
-    insert: &str,
-    key_name: &str,
-    f: impl Fn(Row) -> T,
-) -> DatabaseResult<T> {
+fn _exec_only_create_space_model(gns: &GlobalNS, model: &str) -> DatabaseResult<()> {
     if !gns.spaces().read().contains_key("myspace") {
         gns.test_new_empty_space("myspace");
     }
     let lex_create_model = lex_insecure(model.as_bytes()).unwrap();
     let stmt_create_model = parse_ast_node_full(&lex_create_model[2..]).unwrap();
-    ModelData::exec_create(gns, stmt_create_model)?;
+    ModelData::exec_create(gns, stmt_create_model)
+}
+
+fn _exec_only_insert_only<T>(
+    gns: &GlobalNS,
+    insert: &str,
+    and_then: impl Fn(Entity) -> T,
+) -> DatabaseResult<T> {
     let lex_insert = lex_insecure(insert.as_bytes()).unwrap();
     let stmt_insert = parse_ast_node_full::<InsertStatement>(&lex_insert[1..]).unwrap();
     let entity = stmt_insert.entity();
     dml::insert(gns, stmt_insert)?;
+    let r = and_then(entity);
+    Ok(r)
+}
+
+fn _exec_only_read_key_and_then<T>(
+    gns: &GlobalNS,
+    entity: Entity,
+    key_name: &str,
+    and_then: impl Fn(Row) -> T,
+) -> DatabaseResult<T> {
     let guard = sync::atm::cpin();
     gns.with_model(entity, |mdl| {
         let _irm = mdl.intent_read_model();
@@ -60,6 +76,53 @@ pub(self) fn exec_insert<T>(
             .unwrap()
             .clone();
         drop(guard);
-        Ok(f(row))
+        Ok(and_then(row))
     })
 }
+
+fn _exec_delete_only(gns: &GlobalNS, delete: &str, key: &str) -> DatabaseResult<()> {
+    let lex_del = lex_insecure(delete.as_bytes()).unwrap();
+    let delete = parse_ast_node_full::<DeleteStatement>(&lex_del[1..]).unwrap();
+    let entity = delete.entity();
+    dml::delete(gns, delete)?;
+    assert_eq!(
+        gns.with_model(entity, |model| {
+            let _ = model.intent_read_model();
+            let g = sync::atm::cpin();
+            Ok(model.primary_index().select(key.into(), &g).is_none())
+        }),
+        Ok(true)
+    );
+    Ok(())
+}
+
+pub(self) fn exec_insert<T: Default>(
+    gns: &GlobalNS,
+    model: &str,
+    insert: &str,
+    key_name: &str,
+    f: impl Fn(Row) -> T,
+) -> DatabaseResult<T> {
+    _exec_only_create_space_model(gns, model)?;
+    _exec_only_insert_only(gns, insert, |entity| {
+        _exec_only_read_key_and_then(gns, entity, key_name, |row| f(row))
+    })?
+}
+
+pub(self) fn exec_insert_only(gns: &GlobalNS, insert: &str) -> DatabaseResult<()> {
+    _exec_only_insert_only(gns, insert, |_| {})
+}
+
+pub(self) fn exec_delete(
+    gns: &GlobalNS,
+    model: &str,
+    insert: Option<&str>,
+    delete: &str,
+    key: &str,
+) -> DatabaseResult<()> {
+    _exec_only_create_space_model(gns, model)?;
+    if let Some(insert) = insert {
+        _exec_only_insert_only(gns, insert, |_| {})?;
+    }
+    _exec_delete_only(gns, delete, key)
+}
diff --git a/server/src/util/macros.rs b/server/src/util/macros.rs
index 138b0691..d23fd723 100644
--- a/server/src/util/macros.rs
+++ b/server/src/util/macros.rs
@@ -86,9 +86,14 @@ macro_rules! veceq_transposed {
 
 #[macro_export]
 macro_rules! assert_veceq_transposed {
-    ($v1:expr, $v2:expr) => {
-        assert!(veceq_transposed!($v1, $v2))
-    };
+    ($v1:expr, $v2:expr) => {{
+        if !veceq_transposed!($v1, $v2) {
+            panic!(
+                "failed to assert transposed veceq. v1: `{:#?}`, v2: `{:#?}`",
+                $v1, $v2
+            )
+        }
+    }};
 }
 
 #[cfg(test)]

From dae2052773671cdd42419db5d42463ce807e1b8f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 11 May 2023 10:16:14 -0700
Subject: [PATCH 186/310] Implement and select and fix security bug in index

Discovered another security bug which could potentially be dangerous:
the low level impl of the index used a comparator where eager evaluation
of an expression led to an erroneous return.
---
 server/src/engine/core/dml/mod.rs             |   3 +-
 server/src/engine/core/dml/sel.rs             |  74 +++++++++++++
 server/src/engine/core/index/key.rs           |   4 +-
 server/src/engine/core/model/alt.rs           |   4 +-
 server/src/engine/core/tests/ddl_model/alt.rs |   4 +-
 server/src/engine/core/tests/dml/mod.rs       |  34 +++++-
 server/src/engine/core/tests/dml/select.rs    | 102 ++++++++++++++++++
 server/src/engine/error.rs                    |   2 +-
 server/src/engine/idx/mtchm/mod.rs            |  11 +-
 server/src/engine/macros.rs                   |   5 +
 server/src/util/test_utils.rs                 |   5 +
 11 files changed, 229 insertions(+), 19 deletions(-)
 create mode 100644 server/src/engine/core/dml/sel.rs
 create mode 100644 server/src/engine/core/tests/dml/select.rs

diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index 272bf916..1cc57ff6 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -26,6 +26,7 @@
 
 mod del;
 mod ins;
+mod sel;
 
 use crate::engine::{
     core::model::ModelData,
@@ -34,7 +35,7 @@ use crate::engine::{
     ql::dml::WhereClause,
 };
 
-pub use {del::delete, ins::insert};
+pub use {del::delete, ins::insert, sel::select_custom};
 
 impl ModelData {
     pub(self) fn resolve_where<'a>(
diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
new file mode 100644
index 00000000..869e9b6a
--- /dev/null
+++ b/server/src/engine/core/dml/sel.rs
@@ -0,0 +1,74 @@
+/*
+ * Created on Thu May 11 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    core::{index::DcFieldIndex, GlobalNS},
+    data::cell::{Datacell, VirtualDatacell},
+    error::{DatabaseError, DatabaseResult},
+    idx::{STIndex, STIndexSeq},
+    ql::dml::sel::SelectStatement,
+    sync,
+};
+
+pub fn select_custom<F>(
+    gns: &GlobalNS,
+    mut select: SelectStatement,
+    mut cellfn: F,
+) -> DatabaseResult<()>
+where
+    F: FnMut(&Datacell),
+{
+    gns.with_model(select.entity(), |mdl| {
+        let irm = mdl.intent_read_model();
+        let target_key = mdl.resolve_where(select.clauses_mut())?;
+        let pkdc = VirtualDatacell::new(target_key.clone());
+        let g = sync::atm::cpin();
+        let mut read_field = |key, fields: &DcFieldIndex| {
+            match fields.st_get(key) {
+                Some(dc) => cellfn(dc),
+                None if key == mdl.p_key() => cellfn(&pkdc),
+                None => return Err(DatabaseError::FieldNotFound),
+            }
+            Ok(())
+        };
+        match mdl.primary_index().select(target_key.clone(), &g) {
+            Some(row) => {
+                let r = row.resolve_deltas_and_freeze(mdl.delta_state());
+                if select.is_wildcard() {
+                    for key in irm.fields().stseq_ord_key() {
+                        read_field(key.as_ref(), r.fields())?;
+                    }
+                } else {
+                    for key in select.into_fields() {
+                        read_field(key.as_str(), r.fields())?;
+                    }
+                }
+            }
+            None => return Err(DatabaseError::DmlEntryNotFound),
+        }
+        Ok(())
+    })
+}
diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
index ed2e1f61..dea6c360 100644
--- a/server/src/engine/core/index/key.rs
+++ b/server/src/engine/core/index/key.rs
@@ -203,13 +203,13 @@ impl<'a> PartialEq<LitIR<'a>> for PrimaryIndexKey {
 
 impl<'a> Comparable<LitIR<'a>> for PrimaryIndexKey {
     fn cmp_eq(&self, key: &LitIR<'a>) -> bool {
-        self == key
+        <PrimaryIndexKey as PartialEq<LitIR>>::eq(self, key)
     }
 }
 
 impl<'a> Comparable<PrimaryIndexKey> for LitIR<'a> {
     fn cmp_eq(&self, key: &PrimaryIndexKey) -> bool {
-        key == self
+        <PrimaryIndexKey as PartialEq<LitIR>>::eq(key, self)
     }
 }
 
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index 925defcb..c7ac8d7f 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -110,7 +110,7 @@ impl<'a> AlterPlan<'a> {
                 }) {
                     can_ignore!(AlterAction::Remove(r))
                 } else if not_found {
-                    return Err(DatabaseError::DdlModelAlterFieldNotFound);
+                    return Err(DatabaseError::FieldNotFound);
                 } else {
                     return Err(DatabaseError::DdlModelAlterProtectedField);
                 }
@@ -146,7 +146,7 @@ impl<'a> AlterPlan<'a> {
                     mv.guard_pk(&field_name)?;
                     // get the current field
                     let Some(current_field) = wm.fields().st_get(field_name.as_str()) else {
-                        return Err(DatabaseError::DdlModelAlterFieldNotFound);
+                        return Err(DatabaseError::FieldNotFound);
                     };
                     // check props
                     let is_nullable = check_nullable(&mut props)?;
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index f2ea295d..adc1f6f7 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -156,7 +156,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            DatabaseError::DdlModelAlterFieldNotFound
+            DatabaseError::FieldNotFound
         );
     }
     #[test]
@@ -216,7 +216,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            DatabaseError::DdlModelAlterFieldNotFound
+            DatabaseError::FieldNotFound
         );
     }
     fn bad_type_cast(orig_ty: &str, new_ty: &str) {
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index d508ba50..76b5d113 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -26,10 +26,11 @@
 
 mod delete;
 mod insert;
+mod select;
 
 use crate::engine::{
     core::{dml, index::Row, model::ModelData, GlobalNS},
-    data::lit::LitIR,
+    data::{cell::Datacell, lit::LitIR},
     error::DatabaseResult,
     ql::{
         ast::{parse_ast_node_full, Entity},
@@ -48,7 +49,7 @@ fn _exec_only_create_space_model(gns: &GlobalNS, model: &str) -> DatabaseResult<
     ModelData::exec_create(gns, stmt_create_model)
 }
 
-fn _exec_only_insert_only<T>(
+fn _exec_only_insert<T>(
     gns: &GlobalNS,
     insert: &str,
     and_then: impl Fn(Entity) -> T,
@@ -96,6 +97,14 @@ fn _exec_delete_only(gns: &GlobalNS, delete: &str, key: &str) -> DatabaseResult<
     Ok(())
 }
 
+fn _exec_only_select(gns: &GlobalNS, select: &str) -> DatabaseResult<Vec<Datacell>> {
+    let lex_sel = lex_insecure(select.as_bytes()).unwrap();
+    let select = parse_ast_node_full(&lex_sel[1..]).unwrap();
+    let mut r = Vec::new();
+    dml::select_custom(gns, select, |cell| r.push(cell.clone()))?;
+    Ok(r)
+}
+
 pub(self) fn exec_insert<T: Default>(
     gns: &GlobalNS,
     model: &str,
@@ -104,13 +113,13 @@ pub(self) fn exec_insert<T: Default>(
     f: impl Fn(Row) -> T,
 ) -> DatabaseResult<T> {
     _exec_only_create_space_model(gns, model)?;
-    _exec_only_insert_only(gns, insert, |entity| {
+    _exec_only_insert(gns, insert, |entity| {
         _exec_only_read_key_and_then(gns, entity, key_name, |row| f(row))
     })?
 }
 
 pub(self) fn exec_insert_only(gns: &GlobalNS, insert: &str) -> DatabaseResult<()> {
-    _exec_only_insert_only(gns, insert, |_| {})
+    _exec_only_insert(gns, insert, |_| {})
 }
 
 pub(self) fn exec_delete(
@@ -122,7 +131,22 @@ pub(self) fn exec_delete(
 ) -> DatabaseResult<()> {
     _exec_only_create_space_model(gns, model)?;
     if let Some(insert) = insert {
-        _exec_only_insert_only(gns, insert, |_| {})?;
+        _exec_only_insert(gns, insert, |_| {})?;
     }
     _exec_delete_only(gns, delete, key)
 }
+
+pub(self) fn exec_select(
+    gns: &GlobalNS,
+    model: &str,
+    insert: &str,
+    select: &str,
+) -> DatabaseResult<Vec<Datacell>> {
+    _exec_only_create_space_model(gns, model)?;
+    _exec_only_insert(gns, insert, |_| {})?;
+    _exec_only_select(gns, select)
+}
+
+pub(self) fn exec_select_only(gns: &GlobalNS, select: &str) -> DatabaseResult<Vec<Datacell>> {
+    _exec_only_select(gns, select)
+}
diff --git a/server/src/engine/core/tests/dml/select.rs b/server/src/engine/core/tests/dml/select.rs
new file mode 100644
index 00000000..e7501d07
--- /dev/null
+++ b/server/src/engine/core/tests/dml/select.rs
@@ -0,0 +1,102 @@
+/*
+ * Created on Thu May 11 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{core::GlobalNS, data::cell::Datacell, error::DatabaseError};
+
+#[test]
+fn simple_select_wildcard() {
+    let gns = GlobalNS::empty();
+    assert_eq!(
+        super::exec_select(
+            &gns,
+            "create model myspace.mymodel(username: string, password: string)",
+            "insert into myspace.mymodel('sayan', 'pass123')",
+            "select * from myspace.mymodel where username = 'sayan'",
+        )
+        .unwrap(),
+        intovec!["sayan", "pass123"]
+    );
+}
+
+#[test]
+fn simple_select_specified_same_order() {
+    let gns = GlobalNS::empty();
+    assert_eq!(
+        super::exec_select(
+            &gns,
+            "create model myspace.mymodel(username: string, password: string)",
+            "insert into myspace.mymodel('sayan', 'pass123')",
+            "select username, password from myspace.mymodel where username = 'sayan'",
+        )
+        .unwrap(),
+        intovec!["sayan", "pass123"]
+    );
+}
+
+#[test]
+fn simple_select_specified_reversed_order() {
+    let gns = GlobalNS::empty();
+    assert_eq!(
+        super::exec_select(
+            &gns,
+            "create model myspace.mymodel(username: string, password: string)",
+            "insert into myspace.mymodel('sayan', 'pass123')",
+            "select password, username from myspace.mymodel where username = 'sayan'",
+        )
+        .unwrap(),
+        intovec!["pass123", "sayan"]
+    );
+}
+
+#[test]
+fn select_null() {
+    let gns = GlobalNS::empty();
+    assert_eq!(
+        super::exec_select(
+            &gns,
+            "create model myspace.mymodel(username: string, null password: string)",
+            "insert into myspace.mymodel('sayan', null)",
+            "select username, password from myspace.mymodel where username = 'sayan'",
+        )
+        .unwrap(),
+        intovec!["sayan", Datacell::null()]
+    );
+}
+
+#[test]
+fn select_nonexisting() {
+    let gns = GlobalNS::empty();
+    assert_eq!(
+        super::exec_select(
+            &gns,
+            "create model myspace.mymodel(username: string, null password: string)",
+            "insert into myspace.mymodel('sayan', null)",
+            "select username, password from myspace.mymodel where username = 'notsayan'",
+        )
+        .unwrap_err(),
+        DatabaseError::DmlEntryNotFound
+    );
+}
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 9ca4db2c..941d3136 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -112,7 +112,7 @@ pub enum DatabaseError {
     /// the alter model statement is "wrong"
     DdlModelAlterBad,
     /// an alter attempted to update an nx field
-    DdlModelAlterFieldNotFound,
+    FieldNotFound,
     /// bad type definition to alter
     DdlModelAlterBadTypedef,
     /// didn't find the model
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index b57b872c..0c49b6c3 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -428,8 +428,8 @@ impl<T: TreeElement, C: Config> RawTree<T, C> {
     ) -> Option<&'g T> {
         self._lookup(access::RModeElementRef::new(k), g)
     }
-    fn _lookup<'g, R: access::ReadMode<T>>(&'g self, r: R, g: &'g Guard) -> R::Ret<'g> {
-        let mut hash = self.hash(r.target());
+    fn _lookup<'g, R: access::ReadMode<T>>(&'g self, read_spec: R, g: &'g Guard) -> R::Ret<'g> {
+        let mut hash = self.hash(read_spec.target());
         let mut current = &self.root;
         loop {
             let node = current.ld_acq(g);
@@ -442,10 +442,9 @@ impl<T: TreeElement, C: Config> RawTree<T, C> {
                     let mut ret = R::nx();
                     return unsafe {
                         // UNSAFE(@ohsayan): checked flag + nullck
-                        Self::read_data(node).iter().find_map(|e| {
-                            r.target().cmp_eq(e.key()).then_some({
-                                ret = R::ex(e);
-                                Some(())
+                        Self::read_data(node).iter().find_map(|e_current| {
+                            read_spec.target().cmp_eq(e_current.key()).then(|| {
+                                ret = R::ex(e_current);
                             })
                         });
                         ret
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index daa0ecb5..af51b940 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -150,3 +150,8 @@ macro_rules! into_dict {
 macro_rules! pairvec {
     ($($x:expr),*) => {{ let mut v = Vec::new(); $( let (a, b) = $x; v.push((a.into(), b.into())); )* v }};
 }
+
+#[cfg(test)]
+macro_rules! intovec {
+    ($($x:expr),* $(,)?) => { vec![$(core::convert::From::from($x),)*] };
+}
diff --git a/server/src/util/test_utils.rs b/server/src/util/test_utils.rs
index e8581bc4..f35f1034 100644
--- a/server/src/util/test_utils.rs
+++ b/server/src/util/test_utils.rs
@@ -27,6 +27,7 @@
 use {
     rand::{
         distributions::{uniform::SampleUniform, Alphanumeric},
+        rngs::ThreadRng,
         Rng,
     },
     std::{
@@ -102,3 +103,7 @@ pub fn hash_rs<T: Hash + ?Sized>(rs: &RandomState, item: &T) -> u64 {
     item.hash(&mut hasher);
     hasher.finish()
 }
+
+pub fn randomizer() -> ThreadRng {
+    rand::thread_rng()
+}

From f2a0fda29d64a7075112e2ee6341ae41297ba116 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 11 May 2023 22:21:02 -0700
Subject: [PATCH 187/310] Remove the shared impl of seq index

I actually realized this is a terrible abstraction the day I implemented
it and have since been planning to remove it.

The issue is with aliasing and potential corruption in a multi-threaded
environment. Even though we have never used it in such a context (aka
triggering UB) but we can potentially make an *accidental* use of it;
don't take it lightly: it's like modifying data while someone else is
reading it. That's not bad, that's a nightmare.
---
 server/src/engine/core/model/mod.rs   |   4 +-
 server/src/engine/idx/mod.rs          |   2 -
 server/src/engine/idx/stord/mod.rs    |   1 -
 server/src/engine/idx/stord/shared.rs | 131 --------------------------
 4 files changed, 2 insertions(+), 136 deletions(-)
 delete mode 100644 server/src/engine/idx/stord/shared.rs

diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index ef485c1b..3213fc7e 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -39,7 +39,7 @@ use {
             tag::{DataTag, FullTag, TagClass, TagSelector},
         },
         error::{DatabaseError, DatabaseResult},
-        idx::{IndexBaseSpec, IndexSTCOrdRC, STIndex, STIndexSeq},
+        idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
         mem::VInline,
         ql::ddl::{
             crt::CreateModel,
@@ -51,7 +51,7 @@ use {
 };
 
 pub(in crate::engine::core) use self::delta::{DeltaKind, DeltaState, DeltaVersion};
-pub(in crate::engine::core) type Fields = IndexSTCOrdRC<Box<str>, Field>;
+pub(in crate::engine::core) type Fields = IndexSTSeqCns<Box<str>, Field>;
 
 #[derive(Debug)]
 pub struct ModelData {
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 7a07a1df..3731f4b3 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -41,8 +41,6 @@ use {
 
 // re-exports
 pub type IndexSTSeqCns<K, V> = stord::IndexSTSeqDll<K, V, stord::config::ConservativeConfig<K, V>>;
-pub type IndexSTCOrdRC<K, V> =
-    stord::shared::OrderedIdxRC<K, V, stord::config::ConservativeConfig<K, V>>;
 pub type IndexSTSeqLib<K, V> = stord::IndexSTSeqDll<K, V, stord::config::LiberalConfig<K, V>>;
 pub type IndexMTRC<K, V> = mtchm::imp::ChmArc<K, V, mtchm::meta::DefConfig>;
 pub type IndexMTRaw<E> = mtchm::imp::Raw<E, mtchm::meta::DefConfig>;
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index ebeb2dce..969f12b4 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -26,7 +26,6 @@
 
 pub(super) mod config;
 mod iter;
-pub(super) mod shared;
 
 use {
     self::{
diff --git a/server/src/engine/idx/stord/shared.rs b/server/src/engine/idx/stord/shared.rs
deleted file mode 100644
index dfff4818..00000000
--- a/server/src/engine/idx/stord/shared.rs
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Created on Mon May 08 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#[cfg(debug_assertions)]
-use super::IndexSTSeqDllMetrics;
-
-use {
-    super::{config::Config, IndexSTSeqDll},
-    crate::engine::{
-        idx::{AsKey, AsValue, IndexBaseSpec},
-        sync::smart::EArc,
-    },
-    std::{
-        mem::ManuallyDrop,
-        ops::{Deref, DerefMut},
-    },
-};
-
-#[derive(Debug)]
-pub struct OrderedIdxRC<K, V, C: Config<K, V>> {
-    base: ManuallyDrop<IndexSTSeqDll<K, V, C>>,
-    rc: EArc,
-}
-
-impl<K, V, C: Config<K, V>> OrderedIdxRC<K, V, C> {
-    fn new() -> Self {
-        Self::new_with(IndexSTSeqDll::new())
-    }
-    fn new_with(idx: IndexSTSeqDll<K, V, C>) -> Self {
-        Self {
-            base: ManuallyDrop::new(idx),
-            rc: unsafe {
-                // UNSAFE(@ohsayan): we'll clean this up
-                EArc::new()
-            },
-        }
-    }
-}
-
-impl<K, V, C: Config<K, V>> Deref for OrderedIdxRC<K, V, C> {
-    type Target = IndexSTSeqDll<K, V, C>;
-    fn deref(&self) -> &Self::Target {
-        &self.base
-    }
-}
-
-impl<K, V, C: Config<K, V>> DerefMut for OrderedIdxRC<K, V, C> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.base
-    }
-}
-
-impl<K, V, C: Config<K, V>> Clone for OrderedIdxRC<K, V, C> {
-    fn clone(&self) -> Self {
-        let rc = unsafe {
-            // UNSAFE(@ohsayan): called at clone position
-            self.rc.rc_clone()
-        };
-        Self {
-            base: unsafe {
-                // UNSAFE(@ohsayan): just a raw clone. no big deal since this is an RC
-                core::mem::transmute_copy(&self.base)
-            },
-            rc,
-        }
-    }
-}
-
-impl<K, V, C: Config<K, V>> Drop for OrderedIdxRC<K, V, C> {
-    fn drop(&mut self) {
-        unsafe {
-            // UNSAFE(@ohsayan): this is the dtor
-            self.rc.rc_drop(|| ManuallyDrop::drop(&mut self.base))
-        }
-    }
-}
-
-impl<K, V, C: Config<K, V>> IndexBaseSpec for OrderedIdxRC<K, V, C> {
-    const PREALLOC: bool = true;
-    #[cfg(debug_assertions)]
-    type Metrics = IndexSTSeqDllMetrics;
-
-    fn idx_init_cap(cap: usize) -> Self {
-        Self::new_with(IndexSTSeqDll::with_capacity(cap))
-    }
-
-    fn idx_init() -> Self {
-        Self::new()
-    }
-
-    fn idx_init_with(s: Self) -> Self {
-        s
-    }
-
-    #[cfg(debug_assertions)]
-    fn idx_metrics(&self) -> &Self::Metrics {
-        self.base.idx_metrics()
-    }
-}
-
-impl<K: AsKey, V: AsValue + PartialEq, C: Config<K, V>> PartialEq for OrderedIdxRC<K, V, C> {
-    fn eq(&self, other: &Self) -> bool {
-        self.len() == other.len()
-            && self
-                ._iter_unord_kv()
-                .all(|(k, v)| other._get(k).unwrap().eq(v))
-    }
-}

From 4580b76d55ee823d49410d78b0f4a941201ee2fa Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 15 May 2023 08:14:48 -0700
Subject: [PATCH 188/310] Implement v1 executor for update

---
 server/src/engine/core/dml/mod.rs          |  18 +-
 server/src/engine/core/dml/upd.rs          | 308 +++++++++++++++++++++
 server/src/engine/core/index/row.rs        |  35 +--
 server/src/engine/core/mod.rs              |   1 +
 server/src/engine/core/model/mod.rs        |   3 +
 server/src/engine/core/query_meta.rs       |  55 ++++
 server/src/engine/core/tests/dml/mod.rs    |  20 ++
 server/src/engine/core/tests/dml/update.rs | 105 +++++++
 server/src/engine/data/tag.rs              |   7 +
 server/src/engine/error.rs                 |   5 +
 server/src/engine/idx/mod.rs               |   4 +
 server/src/engine/idx/stdhm.rs             |   8 +
 server/src/engine/idx/stord/mod.rs         |   8 +
 server/src/engine/ql/dml/upd.rs            |  51 ++--
 server/src/engine/ql/tests/dml_tests.rs    |  26 +-
 15 files changed, 596 insertions(+), 58 deletions(-)
 create mode 100644 server/src/engine/core/dml/upd.rs
 create mode 100644 server/src/engine/core/query_meta.rs
 create mode 100644 server/src/engine/core/tests/dml/update.rs

diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index 1cc57ff6..325f881b 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -27,15 +27,19 @@
 mod del;
 mod ins;
 mod sel;
+mod upd;
 
-use crate::engine::{
-    core::model::ModelData,
-    data::{lit::LitIR, spec::DataspecMeta1D, tag::DataTag},
-    error::{DatabaseError, DatabaseResult},
-    ql::dml::WhereClause,
+use crate::{
+    engine::{
+        core::model::ModelData,
+        data::{lit::LitIR, spec::DataspecMeta1D, tag::DataTag},
+        error::{DatabaseError, DatabaseResult},
+        ql::dml::WhereClause,
+    },
+    util::compiler,
 };
 
-pub use {del::delete, ins::insert, sel::select_custom};
+pub use {del::delete, ins::insert, sel::select_custom, upd::update};
 
 impl ModelData {
     pub(self) fn resolve_where<'a>(
@@ -49,7 +53,7 @@ impl ModelData {
             {
                 Ok(clause.rhs())
             }
-            _ => Err(DatabaseError::DmlWhereClauseUnindexedExpr),
+            _ => compiler::cold_rerr(DatabaseError::DmlWhereClauseUnindexedExpr),
         }
     }
 }
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
new file mode 100644
index 00000000..9ae552c9
--- /dev/null
+++ b/server/src/engine/core/dml/upd.rs
@@ -0,0 +1,308 @@
+/*
+ * Created on Thu May 11 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::{
+        engine::{
+            core::{query_meta::AssignmentOperator, GlobalNS},
+            data::{
+                cell::Datacell,
+                lit::LitIR,
+                spec::{Dataspec1D, DataspecMeta1D},
+                tag::{DataTag, TagClass},
+            },
+            error::{DatabaseError, DatabaseResult},
+            idx::STIndex,
+            ql::dml::upd::{AssignmentExpression, UpdateStatement},
+            sync,
+        },
+        util::compiler,
+    },
+    std::mem,
+};
+
+#[inline(always)]
+unsafe fn dc_op_fail(_: &Datacell, _: LitIR) -> (bool, Datacell) {
+    (false, Datacell::null())
+}
+// bool
+unsafe fn dc_op_bool_ass(_: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    (true, Datacell::new_bool(rhs.read_bool_uck()))
+}
+// uint
+unsafe fn dc_op_uint_ass(_: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    (true, Datacell::new_uint(rhs.read_uint_uck()))
+}
+unsafe fn dc_op_uint_add(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let (sum, of) = dc.read_uint().overflowing_add(rhs.read_uint_uck());
+    (of, Datacell::new_uint(sum))
+}
+unsafe fn dc_op_uint_sub(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let (diff, of) = dc.read_uint().overflowing_sub(rhs.read_uint_uck());
+    (of, Datacell::new_uint(diff))
+}
+unsafe fn dc_op_uint_mul(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let (prod, of) = dc.read_uint().overflowing_mul(rhs.read_uint_uck());
+    (of, Datacell::new_uint(prod))
+}
+unsafe fn dc_op_uint_div(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let (quo, of) = dc.read_uint().overflowing_div(rhs.read_uint_uck());
+    (of, Datacell::new_uint(quo))
+}
+// sint
+unsafe fn dc_op_sint_ass(_: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    (true, Datacell::new_sint(rhs.read_sint_uck()))
+}
+unsafe fn dc_op_sint_add(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let (sum, of) = dc.read_sint().overflowing_add(rhs.read_sint_uck());
+    (of, Datacell::new_sint(sum))
+}
+unsafe fn dc_op_sint_sub(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let (diff, of) = dc.read_sint().overflowing_sub(rhs.read_sint_uck());
+    (of, Datacell::new_sint(diff))
+}
+unsafe fn dc_op_sint_mul(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let (prod, of) = dc.read_sint().overflowing_mul(rhs.read_sint_uck());
+    (of, Datacell::new_sint(prod))
+}
+unsafe fn dc_op_sint_div(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let (quo, of) = dc.read_sint().overflowing_div(rhs.read_sint_uck());
+    (of, Datacell::new_sint(quo))
+}
+/*
+    float
+    ---
+    FIXME(@ohsayan): floating point always upsets me now and then, this time its
+    the silent overflow boom and I think I should implement a strict mode (no MySQL,
+    not `STRICT_ALL_TABLES` unless we do actually end up going down that route. In
+    that case, oops)
+    --
+    TODO(@ohsayan): account for float32 overflow
+*/
+unsafe fn dc_op_float_ass(_: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    (true, Datacell::new_float(rhs.read_float_uck()))
+}
+unsafe fn dc_op_float_add(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let sum = dc.read_float() + rhs.read_float_uck();
+    (true, Datacell::new_float(sum))
+}
+unsafe fn dc_op_float_sub(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let diff = dc.read_float() - rhs.read_float_uck();
+    (true, Datacell::new_float(diff))
+}
+unsafe fn dc_op_float_mul(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let prod = dc.read_float() - rhs.read_float_uck();
+    (true, Datacell::new_float(prod))
+}
+unsafe fn dc_op_float_div(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let quo = dc.read_float() * rhs.read_float_uck();
+    (true, Datacell::new_float(quo))
+}
+// binary
+unsafe fn dc_op_bin_ass(_dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let new_bin = rhs.read_bin_uck();
+    let mut v = Vec::new();
+    if v.try_reserve_exact(new_bin.len()).is_err() {
+        return dc_op_fail(_dc, rhs);
+    }
+    v.extend_from_slice(new_bin);
+    (true, Datacell::new_bin(v.into_boxed_slice()))
+}
+unsafe fn dc_op_bin_add(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let push_into_bin = rhs.read_bin_uck();
+    let mut bin = Vec::new();
+    if compiler::unlikely(bin.try_reserve_exact(push_into_bin.len()).is_err()) {
+        return dc_op_fail(dc, rhs);
+    }
+    bin.extend_from_slice(dc.read_bin());
+    bin.extend_from_slice(push_into_bin);
+    (true, Datacell::new_bin(bin.into_boxed_slice()))
+}
+// string
+unsafe fn dc_op_str_ass(_dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let new_str = rhs.read_str_uck();
+    let mut v = String::new();
+    if v.try_reserve_exact(new_str.len()).is_err() {
+        return dc_op_fail(_dc, rhs);
+    }
+    v.push_str(new_str);
+    (true, Datacell::new_str(v.into_boxed_str()))
+}
+unsafe fn dc_op_str_add(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
+    let push_into_str = rhs.read_str_uck();
+    let mut str = String::new();
+    if compiler::unlikely(str.try_reserve_exact(push_into_str.len()).is_err()) {
+        return dc_op_fail(dc, rhs);
+    }
+    str.push_str(dc.read_str());
+    str.push_str(push_into_str);
+    (true, Datacell::new_str(str.into_boxed_str()))
+}
+
+static OPERATOR: [unsafe fn(&Datacell, LitIR) -> (bool, Datacell); {
+    TagClass::max() * (AssignmentOperator::max() + 1)
+}] = [
+    // bool
+    dc_op_bool_ass,
+    // -- pad: 4
+    dc_op_fail,
+    dc_op_fail,
+    dc_op_fail,
+    dc_op_fail,
+    // uint
+    dc_op_uint_ass,
+    dc_op_uint_add,
+    dc_op_uint_sub,
+    dc_op_uint_mul,
+    dc_op_uint_div,
+    // sint
+    dc_op_sint_ass,
+    dc_op_sint_add,
+    dc_op_sint_sub,
+    dc_op_sint_mul,
+    dc_op_sint_div,
+    // float
+    dc_op_float_ass,
+    dc_op_float_add,
+    dc_op_float_sub,
+    dc_op_float_mul,
+    dc_op_float_div,
+    // bin
+    dc_op_bin_ass,
+    dc_op_bin_add,
+    // -- pad: 3
+    dc_op_fail,
+    dc_op_fail,
+    dc_op_fail,
+    // str
+    dc_op_str_ass,
+    dc_op_str_add,
+    // -- pad: 3
+    dc_op_fail,
+    dc_op_fail,
+    dc_op_fail,
+];
+
+#[inline(always)]
+const fn opc(opr: TagClass, ope: AssignmentOperator) -> usize {
+    (AssignmentOperator::count() * opr.word()) + ope.word()
+}
+
+pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()> {
+    gns.with_model(update.entity(), |mdl| {
+        let mut ret = Ok(());
+        let key = mdl.resolve_where(update.clauses_mut())?;
+        let irm = mdl.intent_read_model();
+        let g = sync::atm::cpin();
+        let Some(row) = mdl.primary_index().select(key, &g) else {
+            return Err(DatabaseError::DmlEntryNotFound)
+        };
+        let mut row_data_wl = row.d_data().write();
+        let mut rollback_now = false;
+        let mut rollback_data = Vec::with_capacity(update.expressions().len());
+        let mut assn_expressions = update.into_expressions().into_iter();
+        /*
+            FIXME(@ohsayan): where's my usual magic? I'll do it once we have the SE stabilized
+        */
+        while (assn_expressions.len() != 0) & (!rollback_now) {
+            let AssignmentExpression {
+                lhs,
+                rhs,
+                operator_fn,
+            } = unsafe {
+                // UNSAFE(@ohsayan): pre-loop cond
+                assn_expressions.next().unwrap_unchecked()
+            };
+            let field_definition;
+            let field_data;
+            match (
+                irm.fields().st_get(lhs.as_str()),
+                row_data_wl.fields_mut().st_get_mut(lhs.as_str()),
+            ) {
+                (Some(fdef), Some(fdata)) => {
+                    field_definition = fdef;
+                    field_data = fdata;
+                }
+                _ => {
+                    rollback_now = true;
+                    ret = Err(DatabaseError::FieldNotFound);
+                    break;
+                }
+            }
+            match (
+                field_definition.layers()[0].tag().tag_class(),
+                rhs.kind().tag_class(),
+            ) {
+                (tag_a, tag_b)
+                    if (tag_a == tag_b) & (tag_a < TagClass::List) & field_data.is_init() =>
+                {
+                    let (okay, new) = unsafe { OPERATOR[opc(tag_a, operator_fn)](field_data, rhs) };
+                    rollback_now &= !okay;
+                    rollback_data.push((lhs.as_str(), mem::replace(field_data, new)));
+                }
+                (tag_a, tag_b)
+                    if (tag_a == tag_b)
+                        & field_data.is_null()
+                        & (operator_fn == AssignmentOperator::Assign) =>
+                {
+                    rollback_data.push((lhs.as_str(), mem::replace(field_data, rhs.into())));
+                }
+                (TagClass::List, tag_b) => {
+                    if field_definition.layers()[1].tag().tag_class() == tag_b {
+                        unsafe {
+                            // UNSAFE(@ohsayan): matched tags
+                            let mut list = field_data.read_list().write();
+                            if list.try_reserve(1).is_ok() {
+                                list.push(rhs.into());
+                            } else {
+                                rollback_now = true;
+                                ret = Err(DatabaseError::ServerError);
+                                break;
+                            }
+                        }
+                    } else {
+                        rollback_now = true;
+                        ret = Err(DatabaseError::DmlConstraintViolationFieldTypedef);
+                        break;
+                    }
+                }
+                _ => {
+                    ret = Err(DatabaseError::DmlConstraintViolationFieldTypedef);
+                    rollback_now = true;
+                    break;
+                }
+            }
+        }
+        if compiler::unlikely(rollback_now) {
+            rollback_data
+                .into_iter()
+                .for_each(|(field_id, restored_data)| {
+                    row_data_wl.fields_mut().st_update(field_id, restored_data);
+                });
+        }
+        ret
+    })
+}
diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
index 46e5962e..6cb7a907 100644
--- a/server/src/engine/core/index/row.rs
+++ b/server/src/engine/core/index/row.rs
@@ -43,9 +43,9 @@ pub type DcFieldIndex = IndexST<Box<str>, Datacell, HasherNativeFx>;
 
 #[derive(Debug)]
 pub struct Row {
-    txn_genesis: DeltaVersion,
-    pk: ManuallyDrop<PrimaryIndexKey>,
-    rc: RawRC<RwLock<RowData>>,
+    __txn_genesis: DeltaVersion,
+    __pk: ManuallyDrop<PrimaryIndexKey>,
+    __rc: RawRC<RwLock<RowData>>,
 }
 
 #[derive(Debug, PartialEq)]
@@ -58,6 +58,9 @@ impl RowData {
     pub fn fields(&self) -> &DcFieldIndex {
         &self.fields
     }
+    pub fn fields_mut(&mut self) -> &mut DcFieldIndex {
+        &mut self.fields
+    }
 }
 
 impl TreeElement for Row {
@@ -91,9 +94,9 @@ impl Row {
         txn_revised: DeltaVersion,
     ) -> Self {
         Self {
-            txn_genesis,
-            pk: ManuallyDrop::new(pk),
-            rc: unsafe {
+            __txn_genesis: txn_genesis,
+            __pk: ManuallyDrop::new(pk),
+            __rc: unsafe {
                 // UNSAFE(@ohsayan): we free this up later
                 RawRC::new(RwLock::new(RowData {
                     fields: data,
@@ -103,18 +106,18 @@ impl Row {
         }
     }
     pub fn with_data_read<T>(&self, f: impl Fn(&DcFieldIndex) -> T) -> T {
-        let data = self.rc.data().read();
+        let data = self.__rc.data().read();
         f(&data.fields)
     }
     pub fn with_data_write<T>(&self, f: impl Fn(&mut DcFieldIndex) -> T) -> T {
-        let mut data = self.rc.data().write();
+        let mut data = self.__rc.data().write();
         f(&mut data.fields)
     }
     pub fn d_key(&self) -> &PrimaryIndexKey {
-        &self.pk
+        &self.__pk
     }
     pub fn d_data(&self) -> &RwLock<RowData> {
-        self.rc.data()
+        self.__rc.data()
     }
     #[cfg(test)]
     pub fn cloned_data(&self) -> Vec<(Box<str>, Datacell)> {
@@ -161,14 +164,14 @@ impl Clone for Row {
     fn clone(&self) -> Self {
         let rc = unsafe {
             // UNSAFE(@ohsayan): we're calling this in the clone implementation
-            self.rc.rc_clone()
+            self.__rc.rc_clone()
         };
         Self {
-            pk: unsafe {
+            __pk: unsafe {
                 // UNSAFE(@ohsayan): this is safe because of the refcount
-                ManuallyDrop::new(self.pk.raw_clone())
+                ManuallyDrop::new(self.__pk.raw_clone())
             },
-            rc,
+            __rc: rc,
             ..*self
         }
     }
@@ -178,9 +181,9 @@ impl Drop for Row {
     fn drop(&mut self) {
         unsafe {
             // UNSAFE(@ohsayan): we call in this the dtor itself
-            self.rc.rc_drop(|| {
+            self.__rc.rc_drop(|| {
                 // UNSAFE(@ohsayan): we rely on the correctness of the rc
-                ManuallyDrop::drop(&mut self.pk);
+                ManuallyDrop::drop(&mut self.__pk);
             });
         }
     }
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 4806b3ab..d12b7f2f 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -27,6 +27,7 @@
 mod dml;
 mod index;
 mod model;
+pub(in crate::engine) mod query_meta;
 mod space;
 mod util;
 // test
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 3213fc7e..e5e14640 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -373,6 +373,9 @@ impl Layer {
 }
 
 impl Layer {
+    pub fn tag(&self) -> FullTag {
+        self.tag
+    }
     #[cfg(test)]
     pub fn new_test(tag: FullTag, config: [usize; 2]) -> Self {
         Self::new(tag, config)
diff --git a/server/src/engine/core/query_meta.rs b/server/src/engine/core/query_meta.rs
new file mode 100644
index 00000000..7c429cba
--- /dev/null
+++ b/server/src/engine/core/query_meta.rs
@@ -0,0 +1,55 @@
+/*
+ * Created on Fri May 12 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use std::mem;
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
+#[repr(u8)]
+pub enum AssignmentOperator {
+    Assign = 0,
+    AddAssign = 1,
+    SubAssign = 2,
+    MulAssign = 3,
+    DivAssign = 4,
+}
+
+impl AssignmentOperator {
+    pub const fn disc(&self) -> u8 {
+        unsafe {
+            // UNSAFE(@ohsayan): just go back to school already; dscr
+            mem::transmute(*self)
+        }
+    }
+    pub const fn max() -> usize {
+        Self::DivAssign.disc() as _
+    }
+    pub const fn word(&self) -> usize {
+        self.disc() as _
+    }
+    pub const fn count() -> usize {
+        5
+    }
+}
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index 76b5d113..4857c339 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -27,6 +27,7 @@
 mod delete;
 mod insert;
 mod select;
+mod update;
 
 use crate::engine::{
     core::{dml, index::Row, model::ModelData, GlobalNS},
@@ -105,6 +106,12 @@ fn _exec_only_select(gns: &GlobalNS, select: &str) -> DatabaseResult<Vec<Datacel
     Ok(r)
 }
 
+fn _exec_only_update(gns: &GlobalNS, update: &str) -> DatabaseResult<()> {
+    let lex_upd = lex_insecure(update.as_bytes()).unwrap();
+    let update = parse_ast_node_full(&lex_upd[1..]).unwrap();
+    dml::update(gns, update)
+}
+
 pub(self) fn exec_insert<T: Default>(
     gns: &GlobalNS,
     model: &str,
@@ -150,3 +157,16 @@ pub(self) fn exec_select(
 pub(self) fn exec_select_only(gns: &GlobalNS, select: &str) -> DatabaseResult<Vec<Datacell>> {
     _exec_only_select(gns, select)
 }
+
+pub(self) fn exec_update(
+    gns: &GlobalNS,
+    model: &str,
+    insert: &str,
+    update: &str,
+    select: &str,
+) -> DatabaseResult<Vec<Datacell>> {
+    _exec_only_create_space_model(gns, model)?;
+    _exec_only_insert(gns, insert, |_| {})?;
+    _exec_only_update(gns, update)?;
+    _exec_only_select(gns, select)
+}
diff --git a/server/src/engine/core/tests/dml/update.rs b/server/src/engine/core/tests/dml/update.rs
new file mode 100644
index 00000000..5dea2d81
--- /dev/null
+++ b/server/src/engine/core/tests/dml/update.rs
@@ -0,0 +1,105 @@
+/*
+ * Created on Sun May 14 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{core::GlobalNS, data::cell::Datacell, error::DatabaseError};
+
+#[test]
+fn simple() {
+    let gns = GlobalNS::empty();
+    assert_eq!(
+        super::exec_update(
+            &gns,
+            "create model myspace.mymodel(username: string, email: string, followers: uint64, following: uint64)",
+            "insert into myspace.mymodel('sayan', 'sayan@example.com', 0, 100)",
+            "update myspace.mymodel set followers += 200000, following -= 15, email = 'sn@example.com' where username = 'sayan'",
+            "select * from myspace.mymodel where username = 'sayan'"
+        ).unwrap(),
+        intovec!["sayan", "sn@example.com", 200_000_u64, 85_u64],
+    );
+}
+
+#[test]
+fn with_null() {
+    let gns = GlobalNS::empty();
+    assert_eq!(
+        super::exec_update(
+            &gns,
+            "create model myspace.mymodel(username: string, password: string, null email: string)",
+            "insert into myspace.mymodel('sayan', 'pass123', null)",
+            "update myspace.mymodel set email = 'sayan@example.com' where username = 'sayan'",
+            "select * from myspace.mymodel where username='sayan'"
+        )
+        .unwrap(),
+        intovec!["sayan", "pass123", "sayan@example.com"]
+    );
+}
+
+#[test]
+fn fail_unknown_fields() {
+    let gns = GlobalNS::empty();
+    assert_eq!(
+        super::exec_update(
+            &gns,
+            "create model myspace.mymodel(username: string, password: string, null email: string)",
+            "insert into myspace.mymodel('sayan', 'pass123', null)",
+            "update myspace.mymodel set email2 = 'sayan@example.com', password += '4' where username = 'sayan'",
+            "select * from myspace.mymodel where username='sayan'"
+        )
+        .unwrap_err(),
+        DatabaseError::FieldNotFound
+    );
+    // verify integrity
+    assert_eq!(
+        super::exec_select_only(&gns, "select * from myspace.mymodel where username='sayan'")
+            .unwrap(),
+        intovec!["sayan", "pass123", Datacell::null()]
+    );
+}
+
+#[test]
+fn fail_typedef_violation() {
+    let gns = GlobalNS::empty();
+    assert_eq!(
+        super::exec_update(
+            &gns,
+            "create model myspace.mymodel(username: string, password: string, rank: uint8)",
+            "insert into myspace.mymodel('sayan', 'pass123', 1)",
+            "update myspace.mymodel set password = 'pass1234', rank = 'one' where username = 'sayan'",
+            "select * from myspace.mymodel where username = 'sayan'"
+        )
+        .unwrap_err(),
+        DatabaseError::DmlConstraintViolationFieldTypedef
+    );
+    // verify integrity
+    assert_eq!(
+        super::exec_select_only(
+            &gns,
+            "select * from myspace.mymodel where username = 'sayan'"
+        )
+        .unwrap(),
+        intovec!["sayan", "pass123", 1u64]
+    );
+}
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
index 16194a6c..03d1c9cc 100644
--- a/server/src/engine/data/tag.rs
+++ b/server/src/engine/data/tag.rs
@@ -36,6 +36,13 @@ pub enum TagClass {
     List = 6,
 }
 
+impl TagClass {
+    /// ☢WARNING☢: Don't forget offset
+    pub const fn max() -> usize {
+        Self::List.d() as _
+    }
+}
+
 #[repr(u8)]
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord)]
 pub enum TagSelector {
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 941d3136..54894643 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -128,4 +128,9 @@ pub enum DatabaseError {
     DmlWhereClauseUnindexedExpr,
     /// The entry was not found
     DmlEntryNotFound,
+    /// illegal data
+    DmlIllegalData,
+    /// field definition violation
+    DmlConstraintViolationFieldTypedef,
+    ServerError,
 }
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 3731f4b3..211bcecf 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -257,6 +257,10 @@ pub trait STIndex<K: ?Sized, V>: IndexBaseSpec {
         K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKey,
         V: AsValueClone;
+    fn st_get_mut<Q>(&mut self, key: &Q) -> Option<&mut V>
+    where
+        K: AsKey + Borrow<Q>,
+        Q: ?Sized + AsKey;
     // update
     /// Returns true if the entry is updated
     fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
diff --git a/server/src/engine/idx/stdhm.rs b/server/src/engine/idx/stdhm.rs
index 43306513..4161d03b 100644
--- a/server/src/engine/idx/stdhm.rs
+++ b/server/src/engine/idx/stdhm.rs
@@ -138,6 +138,14 @@ where
         self.get(key).cloned()
     }
 
+    fn st_get_mut<Q>(&mut self, key: &Q) -> Option<&mut V>
+    where
+        K: AsKey + Borrow<Q>,
+        Q: ?Sized + AsKey,
+    {
+        self.get_mut(key)
+    }
+
     fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
     where
         K: Borrow<Q>,
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index 969f12b4..67ae4a93 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -625,6 +625,14 @@ where
         self._get(key).cloned()
     }
 
+    fn st_get_mut<Q>(&mut self, _: &Q) -> Option<&mut V>
+    where
+        K: AsKey + Borrow<Q>,
+        Q: ?Sized + AsKey,
+    {
+        todo!()
+    }
+
     fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
     where
         K: Borrow<Q>,
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index 22428725..6a395bc9 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -30,6 +30,7 @@ use {
     super::{u, WhereClause},
     crate::{
         engine::{
+            core::query_meta::AssignmentOperator,
             data::lit::LitIR,
             error::{LangError, LangResult},
             ql::{
@@ -45,37 +46,27 @@ use {
     Impls for update
 */
 
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
-/// TODO(@ohsayan): This only helps with the parser test for now. Replace this with actual operator expressions
-pub enum Operator {
-    Assign,
-    AddAssign,
-    SubAssign,
-    MulAssign,
-    DivAssign,
-}
-
-static OPERATOR: [Operator; 6] = [
-    Operator::Assign,
-    Operator::Assign,
-    Operator::AddAssign,
-    Operator::SubAssign,
-    Operator::MulAssign,
-    Operator::DivAssign,
+static OPERATOR: [AssignmentOperator; 6] = [
+    AssignmentOperator::Assign,
+    AssignmentOperator::Assign,
+    AssignmentOperator::AddAssign,
+    AssignmentOperator::SubAssign,
+    AssignmentOperator::MulAssign,
+    AssignmentOperator::DivAssign,
 ];
 
 #[derive(Debug, PartialEq)]
 pub struct AssignmentExpression<'a> {
     /// the LHS ident
-    pub(super) lhs: Ident<'a>,
+    pub lhs: Ident<'a>,
     /// the RHS lit
-    pub(super) rhs: LitIR<'a>,
+    pub rhs: LitIR<'a>,
     /// operator
-    pub(super) operator_fn: Operator,
+    pub operator_fn: AssignmentOperator,
 }
 
 impl<'a> AssignmentExpression<'a> {
-    pub fn new(lhs: Ident<'a>, rhs: LitIR<'a>, operator_fn: Operator) -> Self {
+    pub fn new(lhs: Ident<'a>, rhs: LitIR<'a>, operator_fn: AssignmentOperator) -> Self {
         Self {
             lhs,
             rhs,
@@ -140,6 +131,24 @@ pub struct UpdateStatement<'a> {
     pub(super) wc: WhereClause<'a>,
 }
 
+impl<'a> UpdateStatement<'a> {
+    pub fn entity(&self) -> Entity<'a> {
+        self.entity
+    }
+    pub fn expressions(&self) -> &[AssignmentExpression<'a>] {
+        &self.expressions
+    }
+    pub fn clauses(&self) -> &WhereClause<'a> {
+        &self.wc
+    }
+    pub fn clauses_mut(&mut self) -> &mut WhereClause<'a> {
+        &mut self.wc
+    }
+    pub fn into_expressions(self) -> Vec<AssignmentExpression<'a>> {
+        self.expressions
+    }
+}
+
 impl<'a> UpdateStatement<'a> {
     #[inline(always)]
     #[cfg(test)]
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 4c830c40..2b0cc879 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -696,12 +696,9 @@ mod expression_tests {
     use {
         super::*,
         crate::engine::{
+            core::query_meta::AssignmentOperator,
             data::{lit::LitIR, spec::Dataspec1D},
-            ql::{
-                ast::parse_ast_node_full,
-                dml::upd::{AssignmentExpression, Operator},
-                lex::Ident,
-            },
+            ql::{ast::parse_ast_node_full, dml::upd::AssignmentExpression, lex::Ident},
         },
     };
     #[test]
@@ -713,7 +710,7 @@ mod expression_tests {
             AssignmentExpression::new(
                 Ident::from("username"),
                 LitIR::Str("sayan"),
-                Operator::Assign
+                AssignmentOperator::Assign
             )
         );
     }
@@ -726,7 +723,7 @@ mod expression_tests {
             AssignmentExpression::new(
                 Ident::from("followers"),
                 LitIR::UnsignedInt(100),
-                Operator::AddAssign
+                AssignmentOperator::AddAssign
             )
         );
     }
@@ -739,7 +736,7 @@ mod expression_tests {
             AssignmentExpression::new(
                 Ident::from("following"),
                 LitIR::UnsignedInt(150),
-                Operator::SubAssign
+                AssignmentOperator::SubAssign
             )
         );
     }
@@ -752,7 +749,7 @@ mod expression_tests {
             AssignmentExpression::new(
                 Ident::from("product_qty"),
                 LitIR::UnsignedInt(2),
-                Operator::MulAssign
+                AssignmentOperator::MulAssign
             )
         );
     }
@@ -765,7 +762,7 @@ mod expression_tests {
             AssignmentExpression::new(
                 Ident::from("image_crop_factor"),
                 LitIR::UnsignedInt(2),
-                Operator::DivAssign
+                AssignmentOperator::DivAssign
             )
         );
     }
@@ -774,11 +771,12 @@ mod update_statement {
     use {
         super::*,
         crate::engine::{
+            core::query_meta::AssignmentOperator,
             data::{lit::LitIR, spec::Dataspec1D},
             ql::{
                 ast::{parse_ast_node_full, Entity},
                 dml::{
-                    upd::{AssignmentExpression, Operator, UpdateStatement},
+                    upd::{AssignmentExpression, UpdateStatement},
                     RelationalExpr, WhereClause,
                 },
                 lex::Ident,
@@ -799,7 +797,7 @@ mod update_statement {
             vec![AssignmentExpression::new(
                 Ident::from("notes"),
                 LitIR::Str("this is my new note"),
-                Operator::AddAssign,
+                AssignmentOperator::AddAssign,
             )],
             WhereClause::new(dict! {
                 Ident::from("username") => RelationalExpr::new(
@@ -832,12 +830,12 @@ mod update_statement {
                 AssignmentExpression::new(
                     Ident::from("notes"),
                     LitIR::Str("this is my new note"),
-                    Operator::AddAssign,
+                    AssignmentOperator::AddAssign,
                 ),
                 AssignmentExpression::new(
                     Ident::from("email"),
                     LitIR::Str("sayan@example.com"),
-                    Operator::Assign,
+                    AssignmentOperator::Assign,
                 ),
             ],
             WhereClause::new(dict! {

From dbc7128c4191ab0debc87d9e443769ec6b9c8068 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 15 May 2023 09:40:36 -0700
Subject: [PATCH 189/310] Add more tests for update

---
 server/src/engine/core/dml/mod.rs          |  2 +
 server/src/engine/core/dml/upd.rs          | 29 ++++++++++-
 server/src/engine/core/tests/dml/update.rs | 56 +++++++++++++++++++++-
 3 files changed, 85 insertions(+), 2 deletions(-)

diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index 325f881b..ab149447 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -39,6 +39,8 @@ use crate::{
     util::compiler,
 };
 
+#[cfg(test)]
+pub use upd::collect_trace_path as update_flow_trace;
 pub use {del::delete, ins::insert, sel::select_custom, upd::update};
 
 impl ModelData {
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index 9ae552c9..d15810a6 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -24,6 +24,8 @@
  *
 */
 
+#[cfg(test)]
+use std::cell::RefCell;
 use {
     crate::{
         engine::{
@@ -211,6 +213,24 @@ const fn opc(opr: TagClass, ope: AssignmentOperator) -> usize {
     (AssignmentOperator::count() * opr.word()) + ope.word()
 }
 
+#[cfg(test)]
+thread_local! {
+    pub(super) static ROUTE_TRACE: RefCell<Vec<&'static str>> = RefCell::new(Vec::new());
+}
+
+#[inline(always)]
+fn input_trace(v: &'static str) {
+    #[cfg(test)]
+    {
+        ROUTE_TRACE.with(|rcv| rcv.borrow_mut().push(v))
+    }
+    let _ = v;
+}
+#[cfg(test)]
+pub fn collect_trace_path() -> Vec<&'static str> {
+    ROUTE_TRACE.with(|v| v.borrow().iter().cloned().collect())
+}
+
 pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()> {
     gns.with_model(update.entity(), |mdl| {
         let mut ret = Ok(());
@@ -247,6 +267,7 @@ pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()>
                     field_data = fdata;
                 }
                 _ => {
+                    input_trace("fieldnotfound");
                     rollback_now = true;
                     ret = Err(DatabaseError::FieldNotFound);
                     break;
@@ -262,6 +283,7 @@ pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()>
                     let (okay, new) = unsafe { OPERATOR[opc(tag_a, operator_fn)](field_data, rhs) };
                     rollback_now &= !okay;
                     rollback_data.push((lhs.as_str(), mem::replace(field_data, new)));
+                    input_trace("sametag;nonnull");
                 }
                 (tag_a, tag_b)
                     if (tag_a == tag_b)
@@ -269,13 +291,15 @@ pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()>
                         & (operator_fn == AssignmentOperator::Assign) =>
                 {
                     rollback_data.push((lhs.as_str(), mem::replace(field_data, rhs.into())));
+                    input_trace("sametag;orignull");
                 }
-                (TagClass::List, tag_b) => {
+                (TagClass::List, tag_b) if operator_fn == AssignmentOperator::AddAssign => {
                     if field_definition.layers()[1].tag().tag_class() == tag_b {
                         unsafe {
                             // UNSAFE(@ohsayan): matched tags
                             let mut list = field_data.read_list().write();
                             if list.try_reserve(1).is_ok() {
+                                input_trace("list;sametag");
                                 list.push(rhs.into());
                             } else {
                                 rollback_now = true;
@@ -284,12 +308,14 @@ pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()>
                             }
                         }
                     } else {
+                        input_trace("list;badtag");
                         rollback_now = true;
                         ret = Err(DatabaseError::DmlConstraintViolationFieldTypedef);
                         break;
                     }
                 }
                 _ => {
+                    input_trace("unknown_reason;exitmainloop");
                     ret = Err(DatabaseError::DmlConstraintViolationFieldTypedef);
                     rollback_now = true;
                     break;
@@ -297,6 +323,7 @@ pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()>
             }
         }
         if compiler::unlikely(rollback_now) {
+            input_trace("rollback");
             rollback_data
                 .into_iter()
                 .for_each(|(field_id, restored_data)| {
diff --git a/server/src/engine/core/tests/dml/update.rs b/server/src/engine/core/tests/dml/update.rs
index 5dea2d81..5f74d26d 100644
--- a/server/src/engine/core/tests/dml/update.rs
+++ b/server/src/engine/core/tests/dml/update.rs
@@ -24,7 +24,11 @@
  *
 */
 
-use crate::engine::{core::GlobalNS, data::cell::Datacell, error::DatabaseError};
+use crate::engine::{
+    core::{dml, GlobalNS},
+    data::cell::Datacell,
+    error::DatabaseError,
+};
 
 #[test]
 fn simple() {
@@ -39,6 +43,10 @@ fn simple() {
         ).unwrap(),
         intovec!["sayan", "sn@example.com", 200_000_u64, 85_u64],
     );
+    assert_eq!(
+        dml::update_flow_trace(),
+        ["sametag;nonnull", "sametag;nonnull", "sametag;nonnull"]
+    );
 }
 
 #[test]
@@ -55,6 +63,47 @@ fn with_null() {
         .unwrap(),
         intovec!["sayan", "pass123", "sayan@example.com"]
     );
+    assert_eq!(dml::update_flow_trace(), ["sametag;orignull"]);
+}
+
+#[test]
+fn with_list() {
+    let gns = GlobalNS::empty();
+    assert_eq!(
+        super::exec_update(
+            &gns,
+            "create model myspace.mymodel(link: string, click_ids: list { type: string })",
+            "insert into myspace.mymodel('example.com', [])",
+            "update myspace.mymodel set click_ids += 'ios_client_uuid' where link = 'example.com'",
+            "select * from myspace.mymodel where link = 'example.com'"
+        )
+        .unwrap(),
+        intovec![
+            "example.com",
+            Datacell::new_list(intovec!["ios_client_uuid"])
+        ]
+    );
+    assert_eq!(dml::update_flow_trace(), ["list;sametag"]);
+}
+
+#[test]
+fn fail_operation_on_null() {
+    let gns = GlobalNS::empty();
+    assert_eq!(
+        super::exec_update(
+            &gns,
+            "create model myspace.mymodel(username: string, password: string, null email: string)",
+            "insert into myspace.mymodel('sayan', 'pass123', null)",
+            "update myspace.mymodel set email += '.com' where username = 'sayan'",
+            "select * from myspace.mymodel where username='sayan'"
+        )
+        .unwrap_err(),
+        DatabaseError::DmlConstraintViolationFieldTypedef
+    );
+    assert_eq!(
+        dml::update_flow_trace(),
+        ["unknown_reason;exitmainloop", "rollback"]
+    );
 }
 
 #[test]
@@ -71,6 +120,7 @@ fn fail_unknown_fields() {
         .unwrap_err(),
         DatabaseError::FieldNotFound
     );
+    assert_eq!(dml::update_flow_trace(), ["fieldnotfound", "rollback"]);
     // verify integrity
     assert_eq!(
         super::exec_select_only(&gns, "select * from myspace.mymodel where username='sayan'")
@@ -93,6 +143,10 @@ fn fail_typedef_violation() {
         .unwrap_err(),
         DatabaseError::DmlConstraintViolationFieldTypedef
     );
+    assert_eq!(
+        dml::update_flow_trace(),
+        ["sametag;nonnull", "unknown_reason;exitmainloop", "rollback"]
+    );
     // verify integrity
     assert_eq!(
         super::exec_select_only(

From b0651c9492ab2b8585a5442945416b17ef731d45 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 18 May 2023 02:20:25 -0700
Subject: [PATCH 190/310] Implement SDSS Header generators

---
 server/src/engine/macros.rs                   |   9 +
 server/src/engine/mod.rs                      |   1 +
 server/src/engine/storage/header.rs           | 312 ++++++++++++++++++
 server/src/engine/storage/macros.rs           |  65 ++++
 server/src/engine/storage/mod.rs              |  34 ++
 server/src/engine/storage/v1/header_impl.rs   | 278 ++++++++++++++++
 server/src/engine/storage/v1/mod.rs           |  27 ++
 server/src/engine/storage/versions/mod.rs     |  85 +++++
 .../engine/storage/versions/server_version.rs | 103 ++++++
 sky-macros/src/lib.rs                         |  50 ++-
 10 files changed, 963 insertions(+), 1 deletion(-)
 create mode 100644 server/src/engine/storage/header.rs
 create mode 100644 server/src/engine/storage/macros.rs
 create mode 100644 server/src/engine/storage/mod.rs
 create mode 100644 server/src/engine/storage/v1/header_impl.rs
 create mode 100644 server/src/engine/storage/v1/mod.rs
 create mode 100644 server/src/engine/storage/versions/mod.rs
 create mode 100644 server/src/engine/storage/versions/server_version.rs

diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index af51b940..13ba99ab 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -155,3 +155,12 @@ macro_rules! pairvec {
 macro_rules! intovec {
     ($($x:expr),* $(,)?) => { vec![$(core::convert::From::from($x),)*] };
 }
+
+macro_rules! sizeof {
+    ($ty:ty) => {
+        ::core::mem::size_of::<$ty>()
+    };
+    ($ty:ty, $by:literal) => {
+        ::core::mem::size_of::<$ty>() * $by
+    };
+}
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index cb44d272..0654b6c0 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -34,4 +34,5 @@ mod error;
 mod idx;
 mod mem;
 mod ql;
+mod storage;
 mod sync;
diff --git a/server/src/engine/storage/header.rs b/server/src/engine/storage/header.rs
new file mode 100644
index 00000000..0eebeb64
--- /dev/null
+++ b/server/src/engine/storage/header.rs
@@ -0,0 +1,312 @@
+/*
+ * Created on Mon May 15 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/*
+    SDSS Header
+    ---
+    SDSS headers have two sections:
+        - Static record: fixed-size record with fixed-layout
+        - Dynamic record: variable-size record with version-dependent layout (> 256B)
+    +--------------------------------------------------------------+
+    |                                                              |
+    |                        STATIC RECORD                         |
+    |                            128B                              |
+    +--------------------------------------------------------------+
+    +--------------------------------------------------------------+
+    |                                                              |
+    |                                                              |
+    |                       DYNAMIC RECORD                         |
+    |                          (256+?)B                            |
+    |                                                              |
+    +--------------------------------------------------------------+
+
+    We collectively define this as the SDSS Header. We'll attempt to statically compute
+    most of the sections, but for variable records we can't do the same. Also, our target
+    is to keep the SDSS Header at around 4K with page-padding.
+*/
+
+/*
+    Static record
+    ---
+    [MAGIC (8B), [HEADER_VERSION(4B), PTR_WIDTH(1B), ENDIAN(1B), ARCH(1B), OPERATING SYSTEM(1B)]]
+
+    ☢ HEADS UP: Static record is always little endian ☢
+*/
+
+use super::versions::HeaderVersion;
+
+const SR0_MAGIC: u64 = 0x4F48534159414E21;
+const SR2_PTR_WIDTH: u8 = HostPointerWidth::new().value_u8();
+const SR3_ENDIAN: u8 = HostEndian::new().value_u8();
+const SR4_ARCH: u8 = HostArch::new().value_u8();
+const SR5_OS: u8 = HostOS::new().value_u8();
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+pub enum HostArch {
+    X86 = 0,
+    X86_64 = 1,
+    ARM = 2,
+    ARM64 = 3,
+    MIPS = 4,
+    PowerPC = 5,
+}
+
+impl HostArch {
+    pub const fn new() -> Self {
+        if cfg!(target_arch = "x86") {
+            HostArch::X86
+        } else if cfg!(target_arch = "x86_64") {
+            HostArch::X86_64
+        } else if cfg!(target_arch = "arm") {
+            HostArch::ARM
+        } else if cfg!(target_arch = "aarch64") {
+            HostArch::ARM64
+        } else if cfg!(target_arch = "mips") {
+            HostArch::MIPS
+        } else if cfg!(target_arch = "powerpc") {
+            HostArch::PowerPC
+        } else {
+            panic!("Unsupported target architecture")
+        }
+    }
+    pub const fn new_with_val(v: u8) -> Self {
+        match v {
+            0 => HostArch::X86,
+            1 => HostArch::X86_64,
+            2 => HostArch::ARM,
+            3 => HostArch::ARM64,
+            4 => HostArch::MIPS,
+            5 => HostArch::PowerPC,
+            _ => panic!("unknown arch"),
+        }
+    }
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+pub enum HostOS {
+    // T1
+    Linux = 0,
+    Windows = 1,
+    MacOS = 2,
+    // T2
+    Android = 3,
+    AppleiOS = 4,
+    FreeBSD = 5,
+    OpenBSD = 6,
+    NetBSD = 7,
+    WASI = 8,
+    Emscripten = 9,
+    // T3
+    Solaris = 10,
+    Fuchsia = 11,
+    Redox = 12,
+    DragonFly = 13,
+}
+
+impl HostOS {
+    pub const fn new() -> Self {
+        if cfg!(target_os = "linux") {
+            HostOS::Linux
+        } else if cfg!(target_os = "windows") {
+            HostOS::Windows
+        } else if cfg!(target_os = "macos") {
+            HostOS::MacOS
+        } else if cfg!(target_os = "android") {
+            HostOS::Android
+        } else if cfg!(target_os = "ios") {
+            HostOS::AppleiOS
+        } else if cfg!(target_os = "freebsd") {
+            HostOS::FreeBSD
+        } else if cfg!(target_os = "openbsd") {
+            HostOS::OpenBSD
+        } else if cfg!(target_os = "netbsd") {
+            HostOS::NetBSD
+        } else if cfg!(target_os = "dragonfly") {
+            HostOS::DragonFly
+        } else if cfg!(target_os = "redox") {
+            HostOS::Redox
+        } else if cfg!(target_os = "fuchsia") {
+            HostOS::Fuchsia
+        } else if cfg!(target_os = "solaris") {
+            HostOS::Solaris
+        } else if cfg!(target_os = "emscripten") {
+            HostOS::Emscripten
+        } else if cfg!(target_os = "wasi") {
+            HostOS::WASI
+        } else {
+            panic!("unknown os")
+        }
+    }
+    pub const fn new_with_val(v: u8) -> Self {
+        match v {
+            0 => HostOS::Linux,
+            1 => HostOS::Windows,
+            2 => HostOS::MacOS,
+            3 => HostOS::Android,
+            4 => HostOS::AppleiOS,
+            5 => HostOS::FreeBSD,
+            6 => HostOS::OpenBSD,
+            7 => HostOS::NetBSD,
+            8 => HostOS::WASI,
+            9 => HostOS::Emscripten,
+            10 => HostOS::Solaris,
+            11 => HostOS::Fuchsia,
+            12 => HostOS::Redox,
+            13 => HostOS::DragonFly,
+            _ => panic!("unknown OS"),
+        }
+    }
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+pub enum HostEndian {
+    Big = 0,
+    Little = 1,
+}
+
+impl HostEndian {
+    pub const fn new() -> Self {
+        if cfg!(target_endian = "little") {
+            Self::Little
+        } else {
+            Self::Big
+        }
+    }
+    pub const fn new_with_val(v: u8) -> Self {
+        match v {
+            0 => HostEndian::Big,
+            1 => HostEndian::Little,
+            _ => panic!("Unknown endian"),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+#[repr(u8)]
+pub enum HostPointerWidth {
+    P32 = 0,
+    P64 = 1,
+}
+
+impl HostPointerWidth {
+    pub const fn new() -> Self {
+        match sizeof!(usize) {
+            4 => Self::P32,
+            8 => Self::P64,
+            _ => panic!("unknown pointer width"),
+        }
+    }
+    pub const fn new_with_val(v: u8) -> Self {
+        match v {
+            0 => HostPointerWidth::P32,
+            1 => HostPointerWidth::P64,
+            _ => panic!("Unknown pointer width"),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct StaticRecordUV {
+    data: [u8; 16],
+}
+
+impl StaticRecordUV {
+    pub const fn create(sr1_version: HeaderVersion) -> Self {
+        let mut data = [0u8; 16];
+        let magic_buf = SR0_MAGIC.to_le_bytes();
+        let version_buf = sr1_version.little_endian_u64();
+        let mut i = 0usize;
+        while i < sizeof!(u64) {
+            data[i] = magic_buf[i];
+            data[i + sizeof!(u64)] = version_buf[i];
+            i += 1;
+        }
+        data[sizeof!(u64, 2) - 4] = SR2_PTR_WIDTH;
+        data[sizeof!(u64, 2) - 3] = SR3_ENDIAN;
+        data[sizeof!(u64, 2) - 2] = SR4_ARCH;
+        data[sizeof!(u64, 2) - 1] = SR5_OS;
+        Self { data }
+    }
+    pub const fn read_p0_magic(&self) -> u64 {
+        self.read_qword(0)
+    }
+    pub const fn read_p1_header_version(&self) -> HeaderVersion {
+        HeaderVersion::__new(self.read_dword(sizeof!(u64)))
+    }
+    pub const fn read_p2_ptr_width(&self) -> HostPointerWidth {
+        HostPointerWidth::new_with_val(self.read_byte(12))
+    }
+    pub const fn read_p3_endian(&self) -> HostEndian {
+        HostEndian::new_with_val(self.read_byte(13))
+    }
+    pub const fn read_p4_arch(&self) -> HostArch {
+        HostArch::new_with_val(self.read_byte(14))
+    }
+    pub const fn read_p5_os(&self) -> HostOS {
+        HostOS::new_with_val(self.read_byte(15))
+    }
+}
+
+impl_stack_read_primitives!(unsafe impl for StaticRecordUV {});
+
+/*
+    File identity
+*/
+
+/// The file scope
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+pub enum FileScope {
+    TransactionLog = 0,
+    TransactionLogCompacted = 1,
+}
+
+impl FileScope {
+    pub const fn new(id: u32) -> Self {
+        match id {
+            0 => Self::TransactionLog,
+            1 => Self::TransactionLogCompacted,
+            _ => panic!("unknown filescope"),
+        }
+    }
+}
+
+#[test]
+fn test_static_record_encode_decode() {
+    let static_record = StaticRecordUV::create(super::versions::v1::V1_HEADER_VERSION);
+    assert_eq!(static_record.read_p0_magic(), SR0_MAGIC);
+    assert_eq!(
+        static_record.read_p1_header_version(),
+        super::versions::v1::V1_HEADER_VERSION
+    );
+    assert_eq!(static_record.read_p2_ptr_width(), HostPointerWidth::new());
+    assert_eq!(static_record.read_p3_endian(), HostEndian::new());
+    assert_eq!(static_record.read_p4_arch(), HostArch::new());
+    assert_eq!(static_record.read_p5_os(), HostOS::new());
+}
diff --git a/server/src/engine/storage/macros.rs b/server/src/engine/storage/macros.rs
new file mode 100644
index 00000000..38b0be82
--- /dev/null
+++ b/server/src/engine/storage/macros.rs
@@ -0,0 +1,65 @@
+/*
+ * Created on Thu May 18 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+// HACK(@ohsayan): until const traits are stable, this is the silly stuff we have to resort to
+macro_rules! impl_stack_read_primitives {
+    (unsafe impl for $ty:ty {}) => {
+        impl $ty {
+            const fn read_byte(&self, position: usize) -> u8 {
+                self.data[position]
+            }
+            const fn read_word(&self, position: usize) -> u16 {
+                unsafe {
+                    core::mem::transmute([self.read_byte(position), self.read_byte(position + 1)])
+                }
+            }
+            const fn read_dword(&self, position: usize) -> u32 {
+                unsafe {
+                    core::mem::transmute([
+                        self.read_word(position),
+                        self.read_word(position + sizeof!(u16)),
+                    ])
+                }
+            }
+            const fn read_qword(&self, position: usize) -> u64 {
+                unsafe {
+                    core::mem::transmute([
+                        self.read_dword(position),
+                        self.read_dword(position + sizeof!(u32)),
+                    ])
+                }
+            }
+            const fn read_xmmword(&self, position: usize) -> u128 {
+                unsafe {
+                    core::mem::transmute([
+                        self.read_qword(position),
+                        self.read_qword(position + sizeof!(u64)),
+                    ])
+                }
+            }
+        }
+    };
+}
diff --git a/server/src/engine/storage/mod.rs b/server/src/engine/storage/mod.rs
new file mode 100644
index 00000000..5a869195
--- /dev/null
+++ b/server/src/engine/storage/mod.rs
@@ -0,0 +1,34 @@
+/*
+ * Created on Mon May 15 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+//! Implementations of the Skytable Disk Storage Subsystem (SDSS)
+
+#[macro_use]
+mod macros;
+mod header;
+mod versions;
+// impls
+mod v1;
diff --git a/server/src/engine/storage/v1/header_impl.rs b/server/src/engine/storage/v1/header_impl.rs
new file mode 100644
index 00000000..7bd1f23a
--- /dev/null
+++ b/server/src/engine/storage/v1/header_impl.rs
@@ -0,0 +1,278 @@
+/*
+ * Created on Mon May 15 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/*
+ * SDSS Header layout:
+ *
+ * +--------------------------------------------------------------+
+ * |                                                              |
+ * |                        STATIC RECORD                         |
+ * |                            128B                              |
+ * +--------------------------------------------------------------+
+ * +--------------------------------------------------------------+
+ * |                                                              |
+ * |                                                              |
+ * |                       DYNAMIC RECORD                         |
+ * |                          (256+?)B                            |
+ * |        +--------------------------------------------+        |
+ * |        |                                            |        |
+ * |        |              METADATA RECORD               |        |
+ * |        |                   256B                     |        |
+ * |        +--------------------------------------------+        |
+ * |        +--------------------------------------------+        |
+ * |        |                                            |        |
+ * |        |            VARIABLE HOST RECORD            |        |
+ * |        |                    ?B                      |        |
+ * |        +--------------------------------------------+        |
+ * +--------------------------------------------------------------+
+ *
+*/
+
+use crate::engine::storage::{
+    header::{FileScope, StaticRecordUV},
+    versions::{self, DriverVersion, ServerVersion},
+};
+
+/// Static record
+pub struct StaticRecord {
+    base: StaticRecordUV,
+}
+
+impl StaticRecord {
+    pub const fn new() -> Self {
+        Self {
+            base: StaticRecordUV::create(versions::v1::V1_HEADER_VERSION),
+        }
+    }
+}
+
+/*
+    Dynamic record (1/2)
+    ---
+    Metadata record (8B x 3 + (4B x 2)):
+    +----------+----------+----------+---------+
+    |  Server  |  Driver  |   File   |File|Spec|
+    |  version |  Version |   Scope  |Spec|ID  |
+    +----------+----------+----------+---------+
+*/
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+#[repr(u8)]
+pub enum FileSpecifier {
+    GNSTxnLog = 0,
+}
+
+impl FileSpecifier {
+    pub const fn new(v: u32) -> Self {
+        match v {
+            0 => Self::GNSTxnLog,
+            _ => panic!("unknown filespecifier"),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct FileSpecifierVersion(u32);
+impl FileSpecifierVersion {
+    pub const fn __new(v: u32) -> Self {
+        Self(v)
+    }
+}
+
+pub struct MetadataRecord {
+    data: [u8; 32],
+}
+
+impl_stack_read_primitives!(unsafe impl for MetadataRecord {});
+
+impl MetadataRecord {
+    pub const fn new(
+        scope: FileScope,
+        specifier: FileSpecifier,
+        specifier_id: FileSpecifierVersion,
+    ) -> Self {
+        let mut ret = [0u8; 32];
+        let mut i = 0;
+        // read buf
+        let server_version = versions::v1::V1_SERVER_VERSION.native_endian();
+        let driver_version = versions::v1::V1_DRIVER_VERSION.native_endian();
+        let file_scope = scope.value_qword().to_ne_bytes();
+        // specifier + specifier ID
+        let file_specifier_and_id: u64 =
+            unsafe { core::mem::transmute([specifier.value_u8() as u32, specifier_id.0]) };
+        let file_specifier_and_id = file_specifier_and_id.to_ne_bytes();
+        while i < sizeof!(u64) {
+            ret[i] = server_version[i];
+            ret[i + sizeof!(u64, 1)] = driver_version[i];
+            ret[i + sizeof!(u64, 2)] = file_scope[i];
+            ret[i + sizeof!(u64, 3)] = file_specifier_and_id[i];
+            i += 1;
+        }
+        Self { data: ret }
+    }
+    pub const fn read_p0_server_version(&self) -> ServerVersion {
+        ServerVersion::__new(self.read_qword(0))
+    }
+    pub const fn read_p1_driver_version(&self) -> DriverVersion {
+        DriverVersion::__new(self.read_qword(sizeof!(u64)))
+    }
+    pub const fn read_p2_file_scope(&self) -> FileScope {
+        FileScope::new(self.read_qword(sizeof!(u128)) as u32)
+    }
+    pub const fn read_p3_file_spec(&self) -> FileSpecifier {
+        FileSpecifier::new(self.read_dword(sizeof!(u64, 3)))
+    }
+    pub const fn read_p4_file_spec_version(&self) -> FileSpecifierVersion {
+        FileSpecifierVersion(self.read_dword(sizeof!(u64, 3) + sizeof!(u32)))
+    }
+}
+
+/*
+    Dynamic Record (2/2)
+    ---
+    Variable record (?B; > 56B):
+    - 16B: Host epoch time in nanoseconds
+    - 16B: Host uptime in nanoseconds
+    - 08B:
+     - 04B: Host setting version ID
+     - 04B: Host run mode
+    - 08B: Host startup counter
+    - 08B: Host name length
+    - ??B: Host name
+*/
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+#[repr(u8)]
+pub enum HostRunMode {
+    Dev = 0,
+    Prod = 1,
+}
+
+impl HostRunMode {
+    pub const fn new_with_val(v: u8) -> Self {
+        match v {
+            0 => Self::Dev,
+            1 => Self::Prod,
+            _ => panic!("unknown hostrunmode"),
+        }
+    }
+}
+
+impl_stack_read_primitives!(unsafe impl for VariableHostRecord {});
+
+pub struct VariableHostRecord {
+    data: [u8; 56],
+    host_name: Box<[u8]>,
+}
+
+impl VariableHostRecord {
+    pub fn new(
+        p0_host_epoch_time: u128,
+        p1_host_uptime: u128,
+        p2a_host_setting_version_id: u32,
+        p2b_host_run_mode: HostRunMode,
+        p3_host_startup_counter: u64,
+        p5_host_name: Box<[u8]>,
+    ) -> Self {
+        let p4_host_name_length = p5_host_name.len();
+        let mut variable_record_fl = [0u8; 56];
+        variable_record_fl[0..16].copy_from_slice(&p0_host_epoch_time.to_ne_bytes());
+        variable_record_fl[16..32].copy_from_slice(&p1_host_uptime.to_ne_bytes());
+        variable_record_fl[32..36].copy_from_slice(&p2a_host_setting_version_id.to_ne_bytes());
+        variable_record_fl[36..40]
+            .copy_from_slice(&(p2b_host_run_mode.value_u8() as u32).to_ne_bytes());
+        variable_record_fl[40..48].copy_from_slice(&p3_host_startup_counter.to_ne_bytes());
+        variable_record_fl[48..56].copy_from_slice(&(p4_host_name_length as u64).to_ne_bytes());
+        Self {
+            data: variable_record_fl,
+            host_name: p5_host_name,
+        }
+    }
+    pub const fn read_p0_epoch_time(&self) -> u128 {
+        self.read_xmmword(0)
+    }
+    pub const fn read_p1_uptime(&self) -> u128 {
+        self.read_xmmword(sizeof!(u128))
+    }
+    pub const fn read_p2a_setting_version_id(&self) -> u32 {
+        self.read_dword(sizeof!(u128, 2))
+    }
+    pub const fn read_p2b_run_mode(&self) -> HostRunMode {
+        HostRunMode::new_with_val(self.read_dword(sizeof!(u128, 2) + sizeof!(u32)) as u8)
+    }
+    pub const fn read_p3_startup_counter(&self) -> u64 {
+        self.read_qword(sizeof!(u128, 2) + sizeof!(u32, 2))
+    }
+    pub const fn read_p4_host_name_length(&self) -> u64 {
+        self.read_qword(sizeof!(u128, 2) + sizeof!(u32, 2) + sizeof!(u64))
+    }
+    pub fn read_p5_host_name(&self) -> &[u8] {
+        &self.host_name
+    }
+}
+
+#[test]
+fn test_metadata_record_encode_decode() {
+    let md = MetadataRecord::new(
+        FileScope::TransactionLog,
+        FileSpecifier::GNSTxnLog,
+        FileSpecifierVersion(1),
+    );
+    assert_eq!(md.read_p0_server_version(), versions::v1::V1_SERVER_VERSION);
+    assert_eq!(md.read_p1_driver_version(), versions::v1::V1_DRIVER_VERSION);
+    assert_eq!(md.read_p2_file_scope(), FileScope::TransactionLog);
+    assert_eq!(md.read_p3_file_spec(), FileSpecifier::GNSTxnLog);
+    assert_eq!(md.read_p4_file_spec_version(), FileSpecifierVersion(1));
+}
+
+#[test]
+fn test_variable_host_record_encode_decode() {
+    const HOST_UPTIME: u128 = u128::MAX - 434324903;
+    const HOST_SETTING_VERSION_ID: u32 = 245;
+    const HOST_RUN_MODE: HostRunMode = HostRunMode::Prod;
+    const HOST_STARTUP_COUNTER: u64 = u32::MAX as _;
+    const HOST_NAME: &str = "skycloud";
+    use std::time::*;
+    let time = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_nanos();
+    let vhr = VariableHostRecord::new(
+        time,
+        HOST_UPTIME,
+        HOST_SETTING_VERSION_ID,
+        HOST_RUN_MODE,
+        HOST_STARTUP_COUNTER,
+        HOST_NAME.as_bytes().to_owned().into_boxed_slice(),
+    );
+    assert_eq!(vhr.read_p0_epoch_time(), time);
+    assert_eq!(vhr.read_p1_uptime(), HOST_UPTIME);
+    assert_eq!(vhr.read_p2a_setting_version_id(), HOST_SETTING_VERSION_ID);
+    assert_eq!(vhr.read_p2b_run_mode(), HOST_RUN_MODE);
+    assert_eq!(vhr.read_p3_startup_counter(), HOST_STARTUP_COUNTER);
+    assert_eq!(vhr.read_p4_host_name_length(), HOST_NAME.len() as u64);
+    assert_eq!(vhr.read_p5_host_name(), HOST_NAME.as_bytes());
+}
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
new file mode 100644
index 00000000..04ccb986
--- /dev/null
+++ b/server/src/engine/storage/v1/mod.rs
@@ -0,0 +1,27 @@
+/*
+ * Created on Mon May 15 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod header_impl;
diff --git a/server/src/engine/storage/versions/mod.rs b/server/src/engine/storage/versions/mod.rs
new file mode 100644
index 00000000..4d16c1e1
--- /dev/null
+++ b/server/src/engine/storage/versions/mod.rs
@@ -0,0 +1,85 @@
+/*
+ * Created on Mon May 15 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+//! SDSS Based Storage Engine versions
+
+pub mod server_version;
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
+/// The header version
+///
+/// The header version is part of the static record and *barely* changes (almost like once in a light year)
+pub struct HeaderVersion(u32);
+
+impl HeaderVersion {
+    pub const fn __new(v: u32) -> Self {
+        Self(v)
+    }
+    pub const fn little_endian_u64(&self) -> [u8; 8] {
+        (self.0 as u64).to_le_bytes()
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
+/// The server version (based on tag index)
+pub struct ServerVersion(u64);
+
+impl ServerVersion {
+    pub const fn __new(v: u64) -> Self {
+        Self(v)
+    }
+    pub const fn native_endian(&self) -> [u8; 8] {
+        self.0.to_ne_bytes()
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
+/// The driver version
+pub struct DriverVersion(u64);
+
+impl DriverVersion {
+    pub const fn __new(v: u64) -> Self {
+        Self(v)
+    }
+    pub const fn native_endian(&self) -> [u8; 8] {
+        self.0.to_ne_bytes()
+    }
+}
+
+pub mod v1 {
+    //! The first SDSS based storage engine implementation.
+    //! Target tag: 0.8.0
+
+    use super::{DriverVersion, HeaderVersion, ServerVersion};
+
+    /// The SDSS header version UID
+    pub const V1_HEADER_VERSION: HeaderVersion = HeaderVersion(0);
+    /// The server version UID
+    pub const V1_SERVER_VERSION: ServerVersion =
+        ServerVersion(super::server_version::fetch_id("v0.8.0") as _);
+    /// The driver version UID
+    pub const V1_DRIVER_VERSION: DriverVersion = DriverVersion(0);
+}
diff --git a/server/src/engine/storage/versions/server_version.rs b/server/src/engine/storage/versions/server_version.rs
new file mode 100644
index 00000000..257cde4e
--- /dev/null
+++ b/server/src/engine/storage/versions/server_version.rs
@@ -0,0 +1,103 @@
+/*
+ * Created on Wed May 17 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+const VERSION_TAGS: [&str; 52] = [
+    "v0.1.0",
+    "v0.2.0",
+    "v0.3.0",
+    "v0.3.1",
+    "v0.3.2",
+    "v0.4.0-alpha.1",
+    "v0.4.0-alpha.2",
+    "v0.4.0",
+    "v0.4.1-alpha.1",
+    "v0.4.1",
+    "v0.4.2-alpha.1",
+    "v0.4.2",
+    "v0.4.3-alpha.1",
+    "v0.4.3",
+    "v0.4.4",
+    "v0.4.5-alpha.1",
+    "v0.4.5-alpha.2",
+    "v0.4.5",
+    "v0.5.0-alpha.1",
+    "v0.5.0-alpha.2",
+    "v0.5.0",
+    "v0.5.1-alpha.1",
+    "v0.5.1",
+    "v0.5.2",
+    "v0.5.3",
+    "v0.6.0",
+    "v0.6.1",
+    "v0.6.2-testrelease.1",
+    "v0.6.2",
+    "v0.6.3-alpha.1",
+    "v0.6.3",
+    "v0.6.4-alpha.1",
+    "v0.6.4",
+    "v0.7.0-RC.1",
+    "v0.7.0-alpha.1",
+    "v0.7.0-alpha.2",
+    "v0.7.0-beta.1",
+    "v0.7.0",
+    "v0.7.1-alpha.1",
+    "v0.7.1",
+    "v0.7.2-alpha.1",
+    "v0.7.2",
+    "v0.7.3-alpha.1",
+    "v0.7.3-alpha.2",
+    "v0.7.3-alpha.3",
+    "v0.7.3",
+    "v0.7.4",
+    "v0.7.5",
+    "v0.7.6",
+    "v0.7.7",
+    "v0.8.0-alpha.1",
+    "v0.8.0",
+];
+const VERSION_TAGS_LEN: usize = VERSION_TAGS.len();
+pub const fn fetch_id(id: &str) -> usize {
+    // this is ct, so a O(n) doesn't matter
+    let mut i = 0;
+    while i < VERSION_TAGS_LEN {
+        let bytes = VERSION_TAGS[i].as_bytes();
+        let given = id.as_bytes();
+        let mut j = 0;
+        let mut eq = true;
+        while (j < bytes.len()) & (bytes.len() == given.len()) {
+            if bytes[i] != given[i] {
+                eq = false;
+                break;
+            }
+            j += 1;
+        }
+        if eq {
+            return i;
+        }
+        i += 1;
+    }
+    panic!("version not found")
+}
diff --git a/sky-macros/src/lib.rs b/sky-macros/src/lib.rs
index a51ace61..0b69f98d 100644
--- a/sky-macros/src/lib.rs
+++ b/sky-macros/src/lib.rs
@@ -47,7 +47,7 @@ use {
     proc_macro::TokenStream,
     proc_macro2::TokenStream as TokenStream2,
     quote::quote,
-    syn::{Data, DataStruct, DeriveInput, Fields, Lit},
+    syn::{parse_macro_input, Data, DataStruct, DeriveInput, Fields, Lit, Meta, NestedMeta},
 };
 
 mod dbtest_fn;
@@ -226,3 +226,51 @@ fn wrapper(item: DeriveInput) -> TokenStream2 {
         }
     }
 }
+
+#[proc_macro_derive(EnumMethods)]
+pub fn derive_value_methods(input: TokenStream) -> TokenStream {
+    let ast = parse_macro_input!(input as DeriveInput);
+    let enum_name = &ast.ident;
+    let mut repr_type = None;
+    // Get repr attribute
+    for attr in &ast.attrs {
+        if attr.path.is_ident("repr") {
+            if let Meta::List(list) = attr.parse_meta().unwrap() {
+                if let Some(NestedMeta::Meta(Meta::Path(path))) = list.nested.first() {
+                    repr_type = Some(path.get_ident().unwrap().to_string());
+                }
+            }
+        }
+    }
+    let repr_type = repr_type.expect("Must have repr(u8) or repr(u16) etc.");
+    // Ensure all variants have explicit discriminants
+    if let Data::Enum(data) = &ast.data {
+        for variant in &data.variants {
+            match &variant.fields {
+                Fields::Unit => {
+                    if variant.discriminant.as_ref().is_none() {
+                        panic!("All enum variants must have explicit discriminants");
+                    }
+                }
+                _ => panic!("All enum variants must be unit variants"),
+            }
+        }
+    } else {
+        panic!("This derive macro only works on enums");
+    }
+
+    let repr_type_ident = syn::Ident::new(&repr_type, proc_macro2::Span::call_site());
+    let repr_type_ident_func = syn::Ident::new(
+        &format!("value_{repr_type}"),
+        proc_macro2::Span::call_site(),
+    );
+
+    let gen = quote! {
+        impl #enum_name {
+            pub const fn #repr_type_ident_func(&self) -> #repr_type_ident { unsafe { core::mem::transmute(*self) } }
+            pub const fn value_word(&self) -> usize { self.#repr_type_ident_func() as usize }
+            pub const fn value_qword(&self) -> u64 { self.#repr_type_ident_func() as u64 }
+        }
+    };
+    gen.into()
+}

From f29807446f175e46655b3be5fdeb5484e2a3b3e2 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 21 May 2023 06:52:34 -0700
Subject: [PATCH 191/310] Add generation for full SDSS header

---
 server/src/engine/storage/v1/header_impl.rs | 100 +++++++++++++++++---
 server/src/engine/storage/versions/mod.rs   |   8 +-
 server/src/util/os.rs                       |  36 ++++++-
 3 files changed, 126 insertions(+), 18 deletions(-)

diff --git a/server/src/engine/storage/v1/header_impl.rs b/server/src/engine/storage/v1/header_impl.rs
index 7bd1f23a..992a53ea 100644
--- a/server/src/engine/storage/v1/header_impl.rs
+++ b/server/src/engine/storage/v1/header_impl.rs
@@ -36,7 +36,7 @@
  * |                                                              |
  * |                                                              |
  * |                       DYNAMIC RECORD                         |
- * |                          (256+?)B                            |
+ * |                         (256+56+?)B                          |
  * |        +--------------------------------------------+        |
  * |        |                                            |        |
  * |        |              METADATA RECORD               |        |
@@ -45,10 +45,11 @@
  * |        +--------------------------------------------+        |
  * |        |                                            |        |
  * |        |            VARIABLE HOST RECORD            |        |
- * |        |                    ?B                      |        |
+ * |        |                  >56B                      |        |
  * |        +--------------------------------------------+        |
  * +--------------------------------------------------------------+
  *
+ * Note: The entire part of the header is little endian encoded
 */
 
 use crate::engine::storage::{
@@ -117,13 +118,13 @@ impl MetadataRecord {
         let mut ret = [0u8; 32];
         let mut i = 0;
         // read buf
-        let server_version = versions::v1::V1_SERVER_VERSION.native_endian();
-        let driver_version = versions::v1::V1_DRIVER_VERSION.native_endian();
-        let file_scope = scope.value_qword().to_ne_bytes();
+        let server_version = versions::v1::V1_SERVER_VERSION.little_endian();
+        let driver_version = versions::v1::V1_DRIVER_VERSION.little_endian();
+        let file_scope = scope.value_qword().to_le_bytes();
         // specifier + specifier ID
         let file_specifier_and_id: u64 =
             unsafe { core::mem::transmute([specifier.value_u8() as u32, specifier_id.0]) };
-        let file_specifier_and_id = file_specifier_and_id.to_ne_bytes();
+        let file_specifier_and_id = file_specifier_and_id.to_le_bytes();
         while i < sizeof!(u64) {
             ret[i] = server_version[i];
             ret[i + sizeof!(u64, 1)] = driver_version[i];
@@ -181,10 +182,12 @@ impl HostRunMode {
     }
 }
 
+type VHRConstSection = [u8; 56];
+
 impl_stack_read_primitives!(unsafe impl for VariableHostRecord {});
 
 pub struct VariableHostRecord {
-    data: [u8; 56],
+    data: VHRConstSection,
     host_name: Box<[u8]>,
 }
 
@@ -199,18 +202,35 @@ impl VariableHostRecord {
     ) -> Self {
         let p4_host_name_length = p5_host_name.len();
         let mut variable_record_fl = [0u8; 56];
-        variable_record_fl[0..16].copy_from_slice(&p0_host_epoch_time.to_ne_bytes());
-        variable_record_fl[16..32].copy_from_slice(&p1_host_uptime.to_ne_bytes());
-        variable_record_fl[32..36].copy_from_slice(&p2a_host_setting_version_id.to_ne_bytes());
+        variable_record_fl[0..16].copy_from_slice(&p0_host_epoch_time.to_le_bytes());
+        variable_record_fl[16..32].copy_from_slice(&p1_host_uptime.to_le_bytes());
+        variable_record_fl[32..36].copy_from_slice(&p2a_host_setting_version_id.to_le_bytes());
         variable_record_fl[36..40]
-            .copy_from_slice(&(p2b_host_run_mode.value_u8() as u32).to_ne_bytes());
-        variable_record_fl[40..48].copy_from_slice(&p3_host_startup_counter.to_ne_bytes());
-        variable_record_fl[48..56].copy_from_slice(&(p4_host_name_length as u64).to_ne_bytes());
+            .copy_from_slice(&(p2b_host_run_mode.value_u8() as u32).to_le_bytes());
+        variable_record_fl[40..48].copy_from_slice(&p3_host_startup_counter.to_le_bytes());
+        variable_record_fl[48..56].copy_from_slice(&(p4_host_name_length as u64).to_le_bytes());
         Self {
             data: variable_record_fl,
             host_name: p5_host_name,
         }
     }
+    pub fn new_auto(
+        p2a_host_setting_version_id: u32,
+        p2b_host_run_mode: HostRunMode,
+        p3_host_startup_counter: u64,
+        p5_host_name: Box<[u8]>,
+    ) -> Self {
+        let p0_host_epoch_time = crate::util::os::get_epoch_time();
+        let p1_host_uptime = crate::util::os::get_uptime();
+        Self::new(
+            p0_host_epoch_time,
+            p1_host_uptime,
+            p2a_host_setting_version_id,
+            p2b_host_run_mode,
+            p3_host_startup_counter,
+            p5_host_name,
+        )
+    }
     pub const fn read_p0_epoch_time(&self) -> u128 {
         self.read_xmmword(0)
     }
@@ -234,6 +254,60 @@ impl VariableHostRecord {
     }
 }
 
+pub struct SDSSHeader {
+    sr: StaticRecord,
+    dr_0_mdr: MetadataRecord,
+    dr_1_vhr: VariableHostRecord,
+}
+
+impl SDSSHeader {
+    pub fn new(
+        sr: StaticRecord,
+        dr_0_mdr: MetadataRecord,
+        dr_1_vhr_const_section: VHRConstSection,
+        dr_1_vhr_host_name: Box<[u8]>,
+    ) -> Self {
+        Self {
+            sr,
+            dr_0_mdr,
+            dr_1_vhr: VariableHostRecord {
+                data: dr_1_vhr_const_section,
+                host_name: dr_1_vhr_host_name,
+            },
+        }
+    }
+    pub fn init(
+        mdr_file_scope: FileScope,
+        mdr_file_specifier: FileSpecifier,
+        mdr_file_specifier_id: FileSpecifierVersion,
+        vhr_host_setting_id: u32,
+        vhr_host_run_mode: HostRunMode,
+        vhr_host_startup_counter: u64,
+        vhr_host_name: Box<[u8]>,
+    ) -> Self {
+        Self {
+            sr: StaticRecord::new(),
+            dr_0_mdr: MetadataRecord::new(
+                mdr_file_scope,
+                mdr_file_specifier,
+                mdr_file_specifier_id,
+            ),
+            dr_1_vhr: VariableHostRecord::new_auto(
+                vhr_host_setting_id,
+                vhr_host_run_mode,
+                vhr_host_startup_counter,
+                vhr_host_name,
+            ),
+        }
+    }
+    pub fn calculate_header_size(&self) -> usize {
+        sizeof!(StaticRecord)
+            + sizeof!(MetadataRecord)
+            + sizeof!(VHRConstSection)
+            + self.dr_1_vhr.host_name.len()
+    }
+}
+
 #[test]
 fn test_metadata_record_encode_decode() {
     let md = MetadataRecord::new(
diff --git a/server/src/engine/storage/versions/mod.rs b/server/src/engine/storage/versions/mod.rs
index 4d16c1e1..5a7d3bb0 100644
--- a/server/src/engine/storage/versions/mod.rs
+++ b/server/src/engine/storage/versions/mod.rs
@@ -51,8 +51,8 @@ impl ServerVersion {
     pub const fn __new(v: u64) -> Self {
         Self(v)
     }
-    pub const fn native_endian(&self) -> [u8; 8] {
-        self.0.to_ne_bytes()
+    pub const fn little_endian(&self) -> [u8; 8] {
+        self.0.to_le_bytes()
     }
 }
 
@@ -64,8 +64,8 @@ impl DriverVersion {
     pub const fn __new(v: u64) -> Self {
         Self(v)
     }
-    pub const fn native_endian(&self) -> [u8; 8] {
-        self.0.to_ne_bytes()
+    pub const fn little_endian(&self) -> [u8; 8] {
+        self.0.to_le_bytes()
     }
 }
 
diff --git a/server/src/util/os.rs b/server/src/util/os.rs
index 028c5191..7e5ad261 100644
--- a/server/src/util/os.rs
+++ b/server/src/util/os.rs
@@ -31,7 +31,12 @@ pub use windows::*;
 
 use {
     crate::IoResult,
-    std::{ffi::OsStr, fs, path::Path},
+    std::{
+        ffi::OsStr,
+        fs,
+        path::Path,
+        time::{SystemTime, UNIX_EPOCH},
+    },
 };
 
 #[cfg(unix)]
@@ -297,3 +302,32 @@ fn dir_size_inner(dir: fs::ReadDir) -> IoResult<u64> {
 pub fn dirsize(path: impl AsRef<Path>) -> IoResult<u64> {
     dir_size_inner(fs::read_dir(path.as_ref())?)
 }
+
+/// Returns the current system uptime in milliseconds
+pub fn get_uptime() -> u128 {
+    uptime().unwrap() as u128 * if cfg!(unix) { 1000 } else { 1 }
+}
+
+/// Returns the current epoch time in nanoseconds
+pub fn get_epoch_time() -> u128 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_nanos()
+}
+
+#[cfg(unix)]
+fn uptime() -> std::io::Result<u64> {
+    let mut sysinfo: libc::sysinfo = unsafe { std::mem::zeroed() };
+    let res = unsafe { libc::sysinfo(&mut sysinfo) };
+    if res == 0 {
+        Ok(sysinfo.uptime as _)
+    } else {
+        Err(std::io::Error::last_os_error())
+    }
+}
+
+#[cfg(windows)]
+fn uptime() -> std::io::Result<u64> {
+    unsafe { Ok(winapi::um::sysinfoapi::GetTickCount64()) }
+}

From c8d3e6b7398e86d26a928994165af3a72b767ab9 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 21 May 2023 08:14:44 -0700
Subject: [PATCH 192/310] Fix uptime impl

---
 server/src/engine/storage/header.rs         |   3 +
 server/src/engine/storage/v1/header_impl.rs |  12 ++
 server/src/util/os.rs                       | 147 +++++++++++++-------
 3 files changed, 114 insertions(+), 48 deletions(-)

diff --git a/server/src/engine/storage/header.rs b/server/src/engine/storage/header.rs
index 0eebeb64..470d317d 100644
--- a/server/src/engine/storage/header.rs
+++ b/server/src/engine/storage/header.rs
@@ -253,6 +253,9 @@ impl StaticRecordUV {
         data[sizeof!(u64, 2) - 1] = SR5_OS;
         Self { data }
     }
+    pub const fn get_ref(&self) -> &[u8] {
+        &self.data
+    }
     pub const fn read_p0_magic(&self) -> u64 {
         self.read_qword(0)
     }
diff --git a/server/src/engine/storage/v1/header_impl.rs b/server/src/engine/storage/v1/header_impl.rs
index 992a53ea..ad43185c 100644
--- a/server/src/engine/storage/v1/header_impl.rs
+++ b/server/src/engine/storage/v1/header_impl.rs
@@ -300,6 +300,18 @@ impl SDSSHeader {
             ),
         }
     }
+    pub fn get0_sr(&self) -> &[u8] {
+        self.sr.base.get_ref()
+    }
+    pub fn get1_dr_0_mdr(&self) -> &[u8] {
+        &self.dr_0_mdr.data
+    }
+    pub fn get1_dr_1_vhr_0(&self) -> &[u8] {
+        &self.dr_1_vhr.data
+    }
+    pub fn get1_dr_1_vhr_1(&self) -> &[u8] {
+        self.dr_1_vhr.host_name.as_ref()
+    }
     pub fn calculate_header_size(&self) -> usize {
         sizeof!(StaticRecord)
             + sizeof!(MetadataRecord)
diff --git a/server/src/util/os.rs b/server/src/util/os.rs
index 7e5ad261..7c2ccc78 100644
--- a/server/src/util/os.rs
+++ b/server/src/util/os.rs
@@ -182,47 +182,6 @@ pub fn recursive_copy(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> IoResult<
     Ok(())
 }
 
-#[test]
-fn rcopy_okay() {
-    let dir_paths = [
-        "testdata/backups",
-        "testdata/ks/default",
-        "testdata/ks/system",
-        "testdata/rsnaps",
-        "testdata/snaps",
-    ];
-    let file_paths = [
-        "testdata/ks/default/default",
-        "testdata/ks/default/PARTMAP",
-        "testdata/ks/PRELOAD",
-        "testdata/ks/system/PARTMAP",
-    ];
-    let new_file_paths = [
-        "my-backups/ks/default/default",
-        "my-backups/ks/default/PARTMAP",
-        "my-backups/ks/PRELOAD",
-        "my-backups/ks/system/PARTMAP",
-    ];
-    let x = move || -> IoResult<()> {
-        for dir in dir_paths {
-            fs::create_dir_all(dir)?;
-        }
-        for file in file_paths {
-            fs::File::create(file)?;
-        }
-        Ok(())
-    };
-    x().unwrap();
-    // now copy all files inside testdata/* to my-backups/*
-    recursive_copy("testdata", "my-backups").unwrap();
-    new_file_paths
-        .iter()
-        .for_each(|path| assert!(Path::new(path).exists()));
-    // now remove the directories
-    fs::remove_dir_all("testdata").unwrap();
-    fs::remove_dir_all("my-backups").unwrap();
-}
-
 #[derive(Debug, PartialEq)]
 pub enum EntryKind {
     Directory(String),
@@ -305,7 +264,7 @@ pub fn dirsize(path: impl AsRef<Path>) -> IoResult<u64> {
 
 /// Returns the current system uptime in milliseconds
 pub fn get_uptime() -> u128 {
-    uptime().unwrap() as u128 * if cfg!(unix) { 1000 } else { 1 }
+    uptime().unwrap()
 }
 
 /// Returns the current epoch time in nanoseconds
@@ -316,18 +275,110 @@ pub fn get_epoch_time() -> u128 {
         .as_nanos()
 }
 
-#[cfg(unix)]
-fn uptime() -> std::io::Result<u64> {
+#[cfg(target_os = "linux")]
+fn uptime() -> std::io::Result<u128> {
     let mut sysinfo: libc::sysinfo = unsafe { std::mem::zeroed() };
     let res = unsafe { libc::sysinfo(&mut sysinfo) };
     if res == 0 {
-        Ok(sysinfo.uptime as _)
+        Ok(sysinfo.uptime as u128 * 1_000)
     } else {
         Err(std::io::Error::last_os_error())
     }
 }
 
-#[cfg(windows)]
-fn uptime() -> std::io::Result<u64> {
-    unsafe { Ok(winapi::um::sysinfoapi::GetTickCount64()) }
+#[cfg(any(
+    target_os = "macos",
+    target_os = "freebsd",
+    target_os = "openbsd",
+    target_os = "netbsd"
+))]
+fn uptime() -> std::io::Result<u128> {
+    use libc::{c_void, size_t, sysctl, timeval};
+    use std::ptr;
+
+    let mib = [libc::CTL_KERN, libc::KERN_BOOTTIME];
+    let mut boottime = timeval {
+        tv_sec: 0,
+        tv_usec: 0,
+    };
+    let mut size = std::mem::size_of::<libc::timeval>() as size_t;
+
+    let result = unsafe {
+        sysctl(
+            // this cast is fine. sysctl only needs to access the ptr to array base (read)
+            &mib as *const _ as *mut _,
+            2,
+            &mut boottime as *mut timeval as *mut c_void,
+            &mut size,
+            ptr::null_mut(),
+            0,
+        )
+    };
+
+    if result == 0 {
+        let current_time = unsafe { libc::time(ptr::null_mut()) };
+        let uptime_secs = current_time - boottime.tv_sec;
+        Ok((uptime_secs as u128) * 1_000)
+    } else {
+        Err(std::io::Error::last_os_error())
+    }
+}
+
+#[cfg(target_os = "windows")]
+fn uptime() -> std::io::Result<u128> {
+    Ok(unsafe { winapi::um::sysinfoapi::GetTickCount64() } as u128)
+}
+
+#[test]
+fn rcopy_okay() {
+    let dir_paths = [
+        "testdata/backups",
+        "testdata/ks/default",
+        "testdata/ks/system",
+        "testdata/rsnaps",
+        "testdata/snaps",
+    ];
+    let file_paths = [
+        "testdata/ks/default/default",
+        "testdata/ks/default/PARTMAP",
+        "testdata/ks/PRELOAD",
+        "testdata/ks/system/PARTMAP",
+    ];
+    let new_file_paths = [
+        "my-backups/ks/default/default",
+        "my-backups/ks/default/PARTMAP",
+        "my-backups/ks/PRELOAD",
+        "my-backups/ks/system/PARTMAP",
+    ];
+    let x = move || -> IoResult<()> {
+        for dir in dir_paths {
+            fs::create_dir_all(dir)?;
+        }
+        for file in file_paths {
+            fs::File::create(file)?;
+        }
+        Ok(())
+    };
+    x().unwrap();
+    // now copy all files inside testdata/* to my-backups/*
+    recursive_copy("testdata", "my-backups").unwrap();
+    new_file_paths
+        .iter()
+        .for_each(|path| assert!(Path::new(path).exists()));
+    // now remove the directories
+    fs::remove_dir_all("testdata").unwrap();
+    fs::remove_dir_all("my-backups").unwrap();
+}
+
+#[test]
+fn t_uptime() {
+    use std::{thread, time::Duration};
+    let uptime_1 = get_uptime();
+    thread::sleep(Duration::from_secs(1));
+    let uptime_2 = get_uptime();
+    // we're putting a 10s tolerance
+    assert!(
+        Duration::from_millis(uptime_2.try_into().unwrap())
+            <= (Duration::from_millis(uptime_1.try_into().unwrap()) + Duration::from_secs(10))
+    )
 }

From 96acecb2c36b58f35f282e13e35ce454c2b1ef46 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 21 May 2023 10:49:30 -0700
Subject: [PATCH 193/310] Impl basic SDSSWriter (raw)

---
 server/src/engine/storage/v1/mod.rs |   1 +
 server/src/engine/storage/v1/rw.rs  | 104 ++++++++++++++++++++++++++++
 server/src/util/mod.rs              |  32 +++++++++
 3 files changed, 137 insertions(+)
 create mode 100644 server/src/engine/storage/v1/rw.rs

diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index 04ccb986..ad07016f 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -25,3 +25,4 @@
 */
 
 mod header_impl;
+mod rw;
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
new file mode 100644
index 00000000..2474aff0
--- /dev/null
+++ b/server/src/engine/storage/v1/rw.rs
@@ -0,0 +1,104 @@
+/*
+ * Created on Fri May 19 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::header_impl::SDSSHeader,
+    crate::{
+        util::{ByteRepr, NumericRepr},
+        IoResult,
+    },
+    std::{
+        fs::{File, OpenOptions},
+        io::Write,
+    },
+};
+
+/*
+    Writer interface
+*/
+
+pub trait RawWriterInterface: Sized {
+    fn open_truncated(fname: &str) -> IoResult<Self>;
+    fn open_create(fname: &str) -> IoResult<Self>;
+    fn fwrite_all(&mut self, bytes: &[u8]) -> IoResult<()>;
+    fn fsync_all(&mut self) -> IoResult<()>;
+}
+
+impl RawWriterInterface for File {
+    fn open_truncated(fname: &str) -> IoResult<Self> {
+        OpenOptions::new()
+            .write(true)
+            .truncate(true)
+            .create(false)
+            .open(fname)
+    }
+
+    fn open_create(fname: &str) -> IoResult<Self> {
+        File::create(fname)
+    }
+
+    fn fwrite_all(&mut self, bytes: &[u8]) -> IoResult<()> {
+        Write::write_all(self, bytes)
+    }
+
+    fn fsync_all(&mut self) -> IoResult<()> {
+        // FIXME(@ohsayan): too slow? maybe fdatasync only?
+        File::sync_all(self)
+    }
+}
+
+/*
+    Writer
+*/
+
+pub struct SDSSWriter<W> {
+    writer: W,
+}
+
+impl<W: RawWriterInterface> SDSSWriter<W> {
+    pub fn open_create_with_header(file: &str, header: SDSSHeader) -> IoResult<Self> {
+        let mut w = W::open_create(file)?;
+        w.fwrite_all(header.get0_sr())?;
+        w.fwrite_all(header.get1_dr_0_mdr())?;
+        w.fwrite_all(header.get1_dr_1_vhr_0())?;
+        w.fwrite_all(header.get1_dr_1_vhr_1())?;
+        w.fsync_all()?;
+        Ok(Self { writer: w })
+    }
+    pub fn fsync_write<D: ?Sized + ByteRepr>(&mut self, data: &D) -> IoResult<()> {
+        self.writer.fwrite_all(data.repr())?;
+        self.writer.fsync_all()
+    }
+    pub fn newrite_numeric(&mut self, num: impl NumericRepr) -> IoResult<()> {
+        self.fsync_write(num.repr())
+    }
+    pub fn lewrite_numeric(&mut self, num: impl NumericRepr) -> IoResult<()> {
+        self.fsync_write(num.le().repr())
+    }
+    pub fn bewrite_numeric(&mut self, num: impl NumericRepr) -> IoResult<()> {
+        self.fsync_write(num.be().repr())
+    }
+}
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index 042c74a8..f29f920c 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -41,6 +41,7 @@ use {
         marker::PhantomData,
         mem::{self, MaybeUninit},
         ops::Deref,
+        slice,
     },
     std::process,
 };
@@ -330,3 +331,34 @@ impl<T: fmt::Debug> fmt::Debug for MaybeInit<T> {
             .finish()
     }
 }
+
+pub unsafe trait ByteRepr {
+    fn repr(&self) -> &[u8];
+}
+
+unsafe impl ByteRepr for [u8] {
+    fn repr(&self) -> &[u8] {
+        self
+    }
+}
+unsafe impl ByteRepr for str {
+    fn repr(&self) -> &[u8] {
+        self.as_bytes()
+    }
+}
+
+pub trait NumericRepr: ByteRepr {
+    fn be(&self) -> Self;
+    fn le(&self) -> Self;
+}
+
+macro_rules! byte_repr_impls {
+    ($($ty:ty),*) => {
+        $(
+            unsafe impl ByteRepr for $ty { fn repr(&self) -> &[u8] { unsafe { slice::from_raw_parts(self as *const $ty as *const u8, mem::size_of::<Self>()) } } }
+            impl NumericRepr for $ty { fn be(&self) -> $ty { <$ty>::to_be(*self) } fn le(&self) -> $ty { <$ty>::to_le(*self) } }
+        )*
+    };
+}
+
+byte_repr_impls!(u8, i8, u16, i16, u32, i32, u64, i64, u128, i128, usize, isize);

From 0e779467140a1cdb80d11246a471d2d0dbea784c Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 22 May 2023 10:30:25 -0700
Subject: [PATCH 194/310] Add decode impls for headers

---
 server/src/engine/storage/header.rs         | 164 ++++++++++++++------
 server/src/engine/storage/v1/header_impl.rs | 145 ++++++++++++++---
 server/src/engine/storage/v1/rw.rs          |  82 +++++++++-
 3 files changed, 309 insertions(+), 82 deletions(-)

diff --git a/server/src/engine/storage/header.rs b/server/src/engine/storage/header.rs
index 470d317d..7c3215db 100644
--- a/server/src/engine/storage/header.rs
+++ b/server/src/engine/storage/header.rs
@@ -58,14 +58,20 @@
 
 use super::versions::HeaderVersion;
 
+/// magic
 const SR0_MAGIC: u64 = 0x4F48534159414E21;
-const SR2_PTR_WIDTH: u8 = HostPointerWidth::new().value_u8();
-const SR3_ENDIAN: u8 = HostEndian::new().value_u8();
-const SR4_ARCH: u8 = HostArch::new().value_u8();
-const SR5_OS: u8 = HostOS::new().value_u8();
+/// host ptr width
+const SR2_PTR_WIDTH: HostPointerWidth = HostPointerWidth::new();
+/// host endian
+const SR3_ENDIAN: HostEndian = HostEndian::new();
+/// host arch
+const SR4_ARCH: HostArch = HostArch::new();
+/// host os
+const SR5_OS: HostOS = HostOS::new();
 
 #[repr(u8)]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+/// Host architecture enumeration for common platforms
 pub enum HostArch {
     X86 = 0,
     X86_64 = 1,
@@ -93,21 +99,28 @@ impl HostArch {
             panic!("Unsupported target architecture")
         }
     }
-    pub const fn new_with_val(v: u8) -> Self {
-        match v {
+    pub const fn try_new_with_val(v: u8) -> Option<Self> {
+        Some(match v {
             0 => HostArch::X86,
             1 => HostArch::X86_64,
             2 => HostArch::ARM,
             3 => HostArch::ARM64,
             4 => HostArch::MIPS,
             5 => HostArch::PowerPC,
-            _ => panic!("unknown arch"),
+            _ => return None,
+        })
+    }
+    pub const fn new_with_val(v: u8) -> Self {
+        match Self::try_new_with_val(v) {
+            Some(v) => v,
+            None => panic!("unknown arch"),
         }
     }
 }
 
 #[repr(u8)]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+/// Host OS enumeration for common operating systems
 pub enum HostOS {
     // T1
     Linux = 0,
@@ -162,8 +175,8 @@ impl HostOS {
             panic!("unknown os")
         }
     }
-    pub const fn new_with_val(v: u8) -> Self {
-        match v {
+    pub const fn try_new_with_val(v: u8) -> Option<Self> {
+        Some(match v {
             0 => HostOS::Linux,
             1 => HostOS::Windows,
             2 => HostOS::MacOS,
@@ -178,13 +191,20 @@ impl HostOS {
             11 => HostOS::Fuchsia,
             12 => HostOS::Redox,
             13 => HostOS::DragonFly,
-            _ => panic!("unknown OS"),
+            _ => return None,
+        })
+    }
+    pub const fn new_with_val(v: u8) -> Self {
+        match Self::try_new_with_val(v) {
+            Some(v) => v,
+            None => panic!("unknown OS"),
         }
     }
 }
 
 #[repr(u8)]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+/// Host endian enumeration
 pub enum HostEndian {
     Big = 0,
     Little = 1,
@@ -198,17 +218,24 @@ impl HostEndian {
             Self::Big
         }
     }
-    pub const fn new_with_val(v: u8) -> Self {
-        match v {
+    pub const fn try_new_with_val(v: u8) -> Option<Self> {
+        Some(match v {
             0 => HostEndian::Big,
             1 => HostEndian::Little,
-            _ => panic!("Unknown endian"),
+            _ => return None,
+        })
+    }
+    pub const fn new_with_val(v: u8) -> Self {
+        match Self::try_new_with_val(v) {
+            Some(v) => v,
+            None => panic!("Unknown endian"),
         }
     }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
 #[repr(u8)]
+/// Host pointer width enumeration
 pub enum HostPointerWidth {
     P32 = 0,
     P64 = 1,
@@ -222,22 +249,43 @@ impl HostPointerWidth {
             _ => panic!("unknown pointer width"),
         }
     }
-    pub const fn new_with_val(v: u8) -> Self {
-        match v {
+    pub const fn try_new_with_val(v: u8) -> Option<Self> {
+        Some(match v {
             0 => HostPointerWidth::P32,
             1 => HostPointerWidth::P64,
-            _ => panic!("Unknown pointer width"),
+            _ => return None,
+        })
+    }
+    pub const fn new_with_val(v: u8) -> Self {
+        match Self::try_new_with_val(v) {
+            Some(v) => v,
+            None => panic!("Unknown pointer width"),
         }
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
+/// The static record
 pub struct StaticRecordUV {
     data: [u8; 16],
 }
 
 impl StaticRecordUV {
-    pub const fn create(sr1_version: HeaderVersion) -> Self {
+    const OFFSET_P0: usize = 0;
+    const OFFSET_P1: usize = sizeof!(u64);
+    const OFFSET_P2: usize = Self::OFFSET_P1 + sizeof!(u32);
+    const OFFSET_P3: usize = Self::OFFSET_P2 + 1;
+    const OFFSET_P4: usize = Self::OFFSET_P3 + 1;
+    const OFFSET_P5: usize = Self::OFFSET_P4 + 1;
+    const _ENSURE: () = assert!(Self::OFFSET_P5 == (sizeof!(Self) - 1));
+    #[inline(always)]
+    pub const fn new(
+        sr1_version: HeaderVersion,
+        sr2_ptr_width: HostPointerWidth,
+        sr3_endian: HostEndian,
+        sr4_arch: HostArch,
+        sr5_os: HostOS,
+    ) -> Self {
         let mut data = [0u8; 16];
         let magic_buf = SR0_MAGIC.to_le_bytes();
         let version_buf = sr1_version.little_endian_u64();
@@ -247,61 +295,68 @@ impl StaticRecordUV {
             data[i + sizeof!(u64)] = version_buf[i];
             i += 1;
         }
-        data[sizeof!(u64, 2) - 4] = SR2_PTR_WIDTH;
-        data[sizeof!(u64, 2) - 3] = SR3_ENDIAN;
-        data[sizeof!(u64, 2) - 2] = SR4_ARCH;
-        data[sizeof!(u64, 2) - 1] = SR5_OS;
+        data[sizeof!(u64, 2) - 4] = sr2_ptr_width.value_u8();
+        data[sizeof!(u64, 2) - 3] = sr3_endian.value_u8();
+        data[sizeof!(u64, 2) - 2] = sr4_arch.value_u8();
+        data[sizeof!(u64, 2) - 1] = sr5_os.value_u8();
         Self { data }
     }
+    #[inline(always)]
+    pub const fn create(sr1_version: HeaderVersion) -> Self {
+        Self::new(sr1_version, SR2_PTR_WIDTH, SR3_ENDIAN, SR4_ARCH, SR5_OS)
+    }
+    /// Decode and validate a SR
+    ///
+    /// WARNING: NOT CONTEXTUAL! VALIDATE YOUR OWN STUFF!
+    pub fn decode(data: [u8; 16]) -> Option<Self> {
+        let _ = Self::_ENSURE;
+        let slf = Self { data };
+        // p0: magic; the magic HAS to be the same
+        if u64::from_le(slf.read_qword(Self::OFFSET_P0)) != SR0_MAGIC {
+            return None;
+        }
+        let sr2_ptr = HostPointerWidth::try_new_with_val(slf.read_byte(Self::OFFSET_P2))?; // p2: ptr width
+        let sr3_endian = HostEndian::try_new_with_val(slf.read_byte(Self::OFFSET_P3))?; // p3: endian
+        let sr4_arch = HostArch::try_new_with_val(slf.read_byte(Self::OFFSET_P4))?; // p4: arch
+        let sr5_os = HostOS::try_new_with_val(slf.read_byte(Self::OFFSET_P5))?; // p5: os
+        Some(Self::new(
+            HeaderVersion::__new(u32::from_le(slf.read_dword(Self::OFFSET_P1))),
+            sr2_ptr,
+            sr3_endian,
+            sr4_arch,
+            sr5_os,
+        ))
+    }
+}
+
+impl StaticRecordUV {
     pub const fn get_ref(&self) -> &[u8] {
         &self.data
     }
     pub const fn read_p0_magic(&self) -> u64 {
-        self.read_qword(0)
+        self.read_qword(Self::OFFSET_P0)
     }
     pub const fn read_p1_header_version(&self) -> HeaderVersion {
-        HeaderVersion::__new(self.read_dword(sizeof!(u64)))
+        HeaderVersion::__new(self.read_dword(Self::OFFSET_P1))
     }
     pub const fn read_p2_ptr_width(&self) -> HostPointerWidth {
-        HostPointerWidth::new_with_val(self.read_byte(12))
+        HostPointerWidth::new_with_val(self.read_byte(Self::OFFSET_P2))
     }
     pub const fn read_p3_endian(&self) -> HostEndian {
-        HostEndian::new_with_val(self.read_byte(13))
+        HostEndian::new_with_val(self.read_byte(Self::OFFSET_P3))
     }
     pub const fn read_p4_arch(&self) -> HostArch {
-        HostArch::new_with_val(self.read_byte(14))
+        HostArch::new_with_val(self.read_byte(Self::OFFSET_P4))
     }
     pub const fn read_p5_os(&self) -> HostOS {
-        HostOS::new_with_val(self.read_byte(15))
+        HostOS::new_with_val(self.read_byte(Self::OFFSET_P5))
     }
 }
 
 impl_stack_read_primitives!(unsafe impl for StaticRecordUV {});
 
-/*
-    File identity
-*/
-
-/// The file scope
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
-pub enum FileScope {
-    TransactionLog = 0,
-    TransactionLogCompacted = 1,
-}
-
-impl FileScope {
-    pub const fn new(id: u32) -> Self {
-        match id {
-            0 => Self::TransactionLog,
-            1 => Self::TransactionLogCompacted,
-            _ => panic!("unknown filescope"),
-        }
-    }
-}
-
 #[test]
-fn test_static_record_encode_decode() {
+fn test_static_record() {
     let static_record = StaticRecordUV::create(super::versions::v1::V1_HEADER_VERSION);
     assert_eq!(static_record.read_p0_magic(), SR0_MAGIC);
     assert_eq!(
@@ -313,3 +368,10 @@ fn test_static_record_encode_decode() {
     assert_eq!(static_record.read_p4_arch(), HostArch::new());
     assert_eq!(static_record.read_p5_os(), HostOS::new());
 }
+
+#[test]
+fn test_static_record_encode_decode() {
+    let static_record = StaticRecordUV::create(super::versions::v1::V1_HEADER_VERSION);
+    let static_record_decoded = StaticRecordUV::decode(static_record.data).unwrap();
+    assert_eq!(static_record, static_record_decoded);
+}
diff --git a/server/src/engine/storage/v1/header_impl.rs b/server/src/engine/storage/v1/header_impl.rs
index ad43185c..8b3519a5 100644
--- a/server/src/engine/storage/v1/header_impl.rs
+++ b/server/src/engine/storage/v1/header_impl.rs
@@ -53,7 +53,7 @@
 */
 
 use crate::engine::storage::{
-    header::{FileScope, StaticRecordUV},
+    header::StaticRecordUV,
     versions::{self, DriverVersion, ServerVersion},
 };
 
@@ -78,8 +78,33 @@ impl StaticRecord {
     |  Server  |  Driver  |   File   |File|Spec|
     |  version |  Version |   Scope  |Spec|ID  |
     +----------+----------+----------+---------+
+    0, 63
 */
 
+/// The file scope
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+pub enum FileScope {
+    TransactionLog = 0,
+    TransactionLogCompacted = 1,
+}
+
+impl FileScope {
+    pub const fn try_new(id: u64) -> Option<Self> {
+        Some(match id {
+            0 => Self::TransactionLog,
+            1 => Self::TransactionLogCompacted,
+            _ => return None,
+        })
+    }
+    pub const fn new(id: u64) -> Self {
+        match Self::try_new(id) {
+            Some(v) => v,
+            None => panic!("unknown filescope"),
+        }
+    }
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
 #[repr(u8)]
 pub enum FileSpecifier {
@@ -87,9 +112,15 @@ pub enum FileSpecifier {
 }
 
 impl FileSpecifier {
-    pub const fn new(v: u32) -> Self {
-        match v {
+    pub const fn try_new(v: u32) -> Option<Self> {
+        Some(match v {
             0 => Self::GNSTxnLog,
+            _ => return None,
+        })
+    }
+    pub const fn new(v: u32) -> Self {
+        match Self::try_new(v) {
+            Some(v) => v,
             _ => panic!("unknown filespecifier"),
         }
     }
@@ -110,16 +141,49 @@ pub struct MetadataRecord {
 impl_stack_read_primitives!(unsafe impl for MetadataRecord {});
 
 impl MetadataRecord {
-    pub const fn new(
+    /// Decodes a given metadata record, validating all data for correctness.
+    ///
+    /// WARNING: That means you need to do contextual validation! This function is not aware of any context
+    pub fn decode(data: [u8; 32]) -> Option<Self> {
+        let slf = Self { data };
+        let server_version =
+            ServerVersion::__new(u64::from_le(slf.read_qword(Self::MDR_OFFSET_P0)));
+        let driver_version =
+            DriverVersion::__new(u64::from_le(slf.read_qword(Self::MDR_OFFSET_P1)));
+        let file_scope = FileScope::try_new(u64::from_le(slf.read_qword(Self::MDR_OFFSET_P2)))?;
+        let file_spec = FileSpecifier::try_new(u32::from_le(slf.read_dword(Self::MDR_OFFSET_P3)))?;
+        let file_spec_id =
+            FileSpecifierVersion::__new(u32::from_le(slf.read_dword(Self::MDR_OFFSET_P4)));
+        Some(Self::new_full(
+            server_version,
+            driver_version,
+            file_scope,
+            file_spec,
+            file_spec_id,
+        ))
+    }
+}
+
+impl MetadataRecord {
+    const MDR_OFFSET_P0: usize = 0;
+    const MDR_OFFSET_P1: usize = sizeof!(u64);
+    const MDR_OFFSET_P2: usize = Self::MDR_OFFSET_P1 + sizeof!(u64);
+    const MDR_OFFSET_P3: usize = Self::MDR_OFFSET_P2 + sizeof!(u64);
+    const MDR_OFFSET_P4: usize = Self::MDR_OFFSET_P3 + sizeof!(u32);
+    const _ENSURE: () = assert!(Self::MDR_OFFSET_P4 == (sizeof!(Self) - sizeof!(u32)));
+    pub const fn new_full(
+        server_version: ServerVersion,
+        driver_version: DriverVersion,
         scope: FileScope,
         specifier: FileSpecifier,
         specifier_id: FileSpecifierVersion,
     ) -> Self {
+        let _ = Self::_ENSURE;
         let mut ret = [0u8; 32];
         let mut i = 0;
         // read buf
-        let server_version = versions::v1::V1_SERVER_VERSION.little_endian();
-        let driver_version = versions::v1::V1_DRIVER_VERSION.little_endian();
+        let server_version = server_version.little_endian();
+        let driver_version = driver_version.little_endian();
         let file_scope = scope.value_qword().to_le_bytes();
         // specifier + specifier ID
         let file_specifier_and_id: u64 =
@@ -134,20 +198,36 @@ impl MetadataRecord {
         }
         Self { data: ret }
     }
+    pub const fn new(
+        scope: FileScope,
+        specifier: FileSpecifier,
+        specifier_id: FileSpecifierVersion,
+    ) -> Self {
+        Self::new_full(
+            versions::v1::V1_SERVER_VERSION,
+            versions::v1::V1_DRIVER_VERSION,
+            scope,
+            specifier,
+            specifier_id,
+        )
+    }
+}
+
+impl MetadataRecord {
     pub const fn read_p0_server_version(&self) -> ServerVersion {
-        ServerVersion::__new(self.read_qword(0))
+        ServerVersion::__new(self.read_qword(Self::MDR_OFFSET_P0))
     }
     pub const fn read_p1_driver_version(&self) -> DriverVersion {
-        DriverVersion::__new(self.read_qword(sizeof!(u64)))
+        DriverVersion::__new(self.read_qword(Self::MDR_OFFSET_P1))
     }
     pub const fn read_p2_file_scope(&self) -> FileScope {
-        FileScope::new(self.read_qword(sizeof!(u128)) as u32)
+        FileScope::new(self.read_qword(Self::MDR_OFFSET_P2))
     }
     pub const fn read_p3_file_spec(&self) -> FileSpecifier {
-        FileSpecifier::new(self.read_dword(sizeof!(u64, 3)))
+        FileSpecifier::new(self.read_dword(Self::MDR_OFFSET_P3))
     }
     pub const fn read_p4_file_spec_version(&self) -> FileSpecifierVersion {
-        FileSpecifierVersion(self.read_dword(sizeof!(u64, 3) + sizeof!(u32)))
+        FileSpecifierVersion(self.read_dword(Self::MDR_OFFSET_P4))
     }
 }
 
@@ -173,11 +253,17 @@ pub enum HostRunMode {
 }
 
 impl HostRunMode {
-    pub const fn new_with_val(v: u8) -> Self {
-        match v {
+    pub const fn try_new_with_val(v: u8) -> Option<Self> {
+        Some(match v {
             0 => Self::Dev,
             1 => Self::Prod,
-            _ => panic!("unknown hostrunmode"),
+            _ => return None,
+        })
+    }
+    pub const fn new_with_val(v: u8) -> Self {
+        match Self::try_new_with_val(v) {
+            Some(v) => v,
+            None => panic!("unknown hostrunmode"),
         }
     }
 }
@@ -192,6 +278,13 @@ pub struct VariableHostRecord {
 }
 
 impl VariableHostRecord {
+    const VHR_OFFSET_P0: usize = 0;
+    const VHR_OFFSET_P1: usize = sizeof!(u128);
+    const VHR_OFFSET_P2A: usize = Self::VHR_OFFSET_P1 + sizeof!(u128);
+    const VHR_OFFSET_P2B: usize = Self::VHR_OFFSET_P2A + sizeof!(u32);
+    const VHR_OFFSET_P3: usize = Self::VHR_OFFSET_P2B + sizeof!(u32);
+    const VHR_OFFSET_P4: usize = Self::VHR_OFFSET_P3 + sizeof!(u64);
+    const _ENSURE: () = assert!(Self::VHR_OFFSET_P4 == sizeof!(VHRConstSection) - sizeof!(u64));
     pub fn new(
         p0_host_epoch_time: u128,
         p1_host_uptime: u128,
@@ -200,6 +293,7 @@ impl VariableHostRecord {
         p3_host_startup_counter: u64,
         p5_host_name: Box<[u8]>,
     ) -> Self {
+        let _ = Self::_ENSURE;
         let p4_host_name_length = p5_host_name.len();
         let mut variable_record_fl = [0u8; 56];
         variable_record_fl[0..16].copy_from_slice(&p0_host_epoch_time.to_le_bytes());
@@ -231,23 +325,26 @@ impl VariableHostRecord {
             p5_host_name,
         )
     }
+}
+
+impl VariableHostRecord {
     pub const fn read_p0_epoch_time(&self) -> u128 {
-        self.read_xmmword(0)
+        self.read_xmmword(Self::VHR_OFFSET_P0)
     }
     pub const fn read_p1_uptime(&self) -> u128 {
-        self.read_xmmword(sizeof!(u128))
+        self.read_xmmword(Self::VHR_OFFSET_P1)
     }
     pub const fn read_p2a_setting_version_id(&self) -> u32 {
-        self.read_dword(sizeof!(u128, 2))
+        self.read_dword(Self::VHR_OFFSET_P2A)
     }
     pub const fn read_p2b_run_mode(&self) -> HostRunMode {
-        HostRunMode::new_with_val(self.read_dword(sizeof!(u128, 2) + sizeof!(u32)) as u8)
+        HostRunMode::new_with_val(self.read_dword(Self::VHR_OFFSET_P2B) as u8)
     }
     pub const fn read_p3_startup_counter(&self) -> u64 {
-        self.read_qword(sizeof!(u128, 2) + sizeof!(u32, 2))
+        self.read_qword(Self::VHR_OFFSET_P3)
     }
     pub const fn read_p4_host_name_length(&self) -> u64 {
-        self.read_qword(sizeof!(u128, 2) + sizeof!(u32, 2) + sizeof!(u64))
+        self.read_qword(Self::VHR_OFFSET_P4)
     }
     pub fn read_p5_host_name(&self) -> &[u8] {
         &self.host_name
@@ -313,10 +410,10 @@ impl SDSSHeader {
         self.dr_1_vhr.host_name.as_ref()
     }
     pub fn calculate_header_size(&self) -> usize {
-        sizeof!(StaticRecord)
-            + sizeof!(MetadataRecord)
-            + sizeof!(VHRConstSection)
-            + self.dr_1_vhr.host_name.len()
+        Self::calculate_fixed_header_size() + self.dr_1_vhr.host_name.len()
+    }
+    pub const fn calculate_fixed_header_size() -> usize {
+        sizeof!(StaticRecord) + sizeof!(MetadataRecord) + sizeof!(VHRConstSection)
     }
 }
 
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 2474aff0..33fdffb0 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -31,24 +31,38 @@ use {
         IoResult,
     },
     std::{
-        fs::{File, OpenOptions},
-        io::Write,
+        fs::{self, File, OpenOptions},
+        io::{Error as IoError, ErrorKind, Read, Seek, SeekFrom, Write},
     },
 };
 
+#[derive(Debug)]
+pub enum SDSSError {
+    SRVersionMismatch,
+    IoError(IoError),
+}
+
+impl From<IoError> for SDSSError {
+    fn from(e: IoError) -> Self {
+        Self::IoError(e)
+    }
+}
+
+pub type SDSSResult<T> = Result<T, SDSSError>;
+
 /*
     Writer interface
 */
 
 pub trait RawWriterInterface: Sized {
-    fn open_truncated(fname: &str) -> IoResult<Self>;
-    fn open_create(fname: &str) -> IoResult<Self>;
+    fn fopen_truncated(fname: &str) -> IoResult<Self>;
+    fn fopen_create(fname: &str) -> IoResult<Self>;
     fn fwrite_all(&mut self, bytes: &[u8]) -> IoResult<()>;
     fn fsync_all(&mut self) -> IoResult<()>;
 }
 
 impl RawWriterInterface for File {
-    fn open_truncated(fname: &str) -> IoResult<Self> {
+    fn fopen_truncated(fname: &str) -> IoResult<Self> {
         OpenOptions::new()
             .write(true)
             .truncate(true)
@@ -56,7 +70,7 @@ impl RawWriterInterface for File {
             .open(fname)
     }
 
-    fn open_create(fname: &str) -> IoResult<Self> {
+    fn fopen_create(fname: &str) -> IoResult<Self> {
         File::create(fname)
     }
 
@@ -80,7 +94,7 @@ pub struct SDSSWriter<W> {
 
 impl<W: RawWriterInterface> SDSSWriter<W> {
     pub fn open_create_with_header(file: &str, header: SDSSHeader) -> IoResult<Self> {
-        let mut w = W::open_create(file)?;
+        let mut w = W::fopen_create(file)?;
         w.fwrite_all(header.get0_sr())?;
         w.fwrite_all(header.get1_dr_0_mdr())?;
         w.fwrite_all(header.get1_dr_1_vhr_0())?;
@@ -102,3 +116,57 @@ impl<W: RawWriterInterface> SDSSWriter<W> {
         self.fsync_write(num.be().repr())
     }
 }
+
+/*
+    Read interface
+*/
+
+pub trait RawReaderInterface: Sized {
+    fn fopen(fname: &str) -> IoResult<Self>;
+    fn fread_exact_seek(&mut self, buf: &mut [u8]) -> IoResult<()>;
+    fn fread_to_end(&mut self, buf: &mut Vec<u8>) -> IoResult<()>;
+}
+
+impl RawReaderInterface for File {
+    fn fopen(fname: &str) -> IoResult<Self> {
+        File::open(fname)
+    }
+    fn fread_exact_seek(&mut self, buf: &mut [u8]) -> IoResult<()> {
+        self.read_exact(buf)?;
+        let _ = self.seek(SeekFrom::Start(buf.len() as _))?;
+        Ok(())
+    }
+    fn fread_to_end(&mut self, buf: &mut Vec<u8>) -> IoResult<()> {
+        match self.read_to_end(buf) {
+            Ok(_) => Ok(()),
+            Err(e) => Err(e),
+        }
+    }
+}
+
+pub struct FileBuffered {
+    cursor: usize,
+    base: Vec<u8>,
+}
+
+impl RawReaderInterface for FileBuffered {
+    fn fopen(name: &str) -> IoResult<Self> {
+        Ok(Self {
+            base: fs::read(name)?,
+            cursor: 0,
+        })
+    }
+    fn fread_exact_seek(&mut self, buf: &mut [u8]) -> IoResult<()> {
+        let l = self.base[self.cursor..].len();
+        if l >= buf.len() {
+            self.cursor += buf.len();
+            Ok(buf.copy_from_slice(&self.base[self.cursor..]))
+        } else {
+            Err(ErrorKind::UnexpectedEof.into())
+        }
+    }
+    fn fread_to_end(&mut self, buf: &mut Vec<u8>) -> IoResult<()> {
+        buf.extend_from_slice(&self.base[self.cursor..]);
+        Ok(())
+    }
+}

From beb4d16ab47af45147415622e5d551ce359eb21b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 23 May 2023 11:25:23 -0700
Subject: [PATCH 195/310] Simplify header decode impl and fix MDR endian bug

---
 server/src/engine/mem/mod.rs                |  12 ++-
 server/src/engine/mem/stackop.rs            |  84 ++++++++++++++++
 server/src/engine/storage/header.rs         | 100 +++++++++++++++-----
 server/src/engine/storage/macros.rs         |  65 -------------
 server/src/engine/storage/mod.rs            |   2 -
 server/src/engine/storage/v1/header_impl.rs |  99 ++++++++++++-------
 6 files changed, 235 insertions(+), 127 deletions(-)
 create mode 100644 server/src/engine/mem/stackop.rs
 delete mode 100644 server/src/engine/storage/macros.rs

diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index 4c318e9d..8c1a2b1f 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -25,6 +25,7 @@
 */
 
 mod astr;
+mod stackop;
 mod uarray;
 mod vinline;
 mod word;
@@ -32,10 +33,13 @@ mod word;
 #[cfg(test)]
 mod tests;
 // re-exports
-pub use astr::AStr;
-pub use uarray::UArray;
-pub use vinline::VInline;
-pub use word::{DwordNN, DwordQN, QwordNNNN, TwordNNN, WordIO, ZERO_BLOCK};
+pub use {
+    astr::AStr,
+    stackop::ByteStack,
+    uarray::UArray,
+    vinline::VInline,
+    word::{DwordNN, DwordQN, QwordNNNN, TwordNNN, WordIO, ZERO_BLOCK},
+};
 // imports
 use std::alloc::{self, Layout};
 
diff --git a/server/src/engine/mem/stackop.rs b/server/src/engine/mem/stackop.rs
new file mode 100644
index 00000000..11a6fa63
--- /dev/null
+++ b/server/src/engine/mem/stackop.rs
@@ -0,0 +1,84 @@
+/*
+ * Created on Tue May 23 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub struct ByteStack<const N: usize> {
+    array: [u8; N],
+}
+
+impl<const N: usize> ByteStack<N> {
+    #[inline(always)]
+    pub const fn data_copy(&self) -> [u8; N] {
+        self.array
+    }
+    #[inline(always)]
+    pub const fn new(array: [u8; N]) -> Self {
+        Self { array }
+    }
+    #[inline(always)]
+    pub const fn zeroed() -> Self {
+        Self::new([0u8; N])
+    }
+    #[inline(always)]
+    pub const fn slice(&self) -> &[u8] {
+        &self.array
+    }
+    #[inline(always)]
+    pub const fn read_byte(&self, position: usize) -> u8 {
+        self.array[position]
+    }
+    #[inline(always)]
+    pub const fn read_word(&self, position: usize) -> u16 {
+        unsafe { core::mem::transmute([self.read_byte(position), self.read_byte(position + 1)]) }
+    }
+    #[inline(always)]
+    pub const fn read_dword(&self, position: usize) -> u32 {
+        unsafe {
+            core::mem::transmute([
+                self.read_word(position),
+                self.read_word(position + sizeof!(u16)),
+            ])
+        }
+    }
+    #[inline(always)]
+    pub const fn read_qword(&self, position: usize) -> u64 {
+        unsafe {
+            core::mem::transmute([
+                self.read_dword(position),
+                self.read_dword(position + sizeof!(u32)),
+            ])
+        }
+    }
+    #[inline(always)]
+    pub const fn read_xmmword(&self, position: usize) -> u128 {
+        unsafe {
+            core::mem::transmute([
+                self.read_qword(position),
+                self.read_qword(position + sizeof!(u64)),
+            ])
+        }
+    }
+}
diff --git a/server/src/engine/storage/header.rs b/server/src/engine/storage/header.rs
index 7c3215db..98790f96 100644
--- a/server/src/engine/storage/header.rs
+++ b/server/src/engine/storage/header.rs
@@ -56,7 +56,7 @@
     ☢ HEADS UP: Static record is always little endian ☢
 */
 
-use super::versions::HeaderVersion;
+use {super::versions::HeaderVersion, crate::engine::mem::ByteStack};
 
 /// magic
 const SR0_MAGIC: u64 = 0x4F48534159414E21;
@@ -264,10 +264,52 @@ impl HostPointerWidth {
     }
 }
 
+#[derive(Debug, PartialEq)]
+pub struct StaticRecordUVDecoded {
+    header_version: HeaderVersion,
+    ptr_width: HostPointerWidth,
+    endian: HostEndian,
+    arch: HostArch,
+    os: HostOS,
+}
+
+impl StaticRecordUVDecoded {
+    pub const fn new(
+        header_version: HeaderVersion,
+        ptr_width: HostPointerWidth,
+        endian: HostEndian,
+        arch: HostArch,
+        os: HostOS,
+    ) -> Self {
+        Self {
+            header_version,
+            ptr_width,
+            endian,
+            arch,
+            os,
+        }
+    }
+    pub fn header_version(&self) -> HeaderVersion {
+        self.header_version
+    }
+    pub fn ptr_width(&self) -> HostPointerWidth {
+        self.ptr_width
+    }
+    pub fn endian(&self) -> HostEndian {
+        self.endian
+    }
+    pub fn arch(&self) -> HostArch {
+        self.arch
+    }
+    pub fn os(&self) -> HostOS {
+        self.os
+    }
+}
+
 #[derive(Debug, PartialEq)]
 /// The static record
 pub struct StaticRecordUV {
-    data: [u8; 16],
+    data: ByteStack<16>,
 }
 
 impl StaticRecordUV {
@@ -299,7 +341,9 @@ impl StaticRecordUV {
         data[sizeof!(u64, 2) - 3] = sr3_endian.value_u8();
         data[sizeof!(u64, 2) - 2] = sr4_arch.value_u8();
         data[sizeof!(u64, 2) - 1] = sr5_os.value_u8();
-        Self { data }
+        Self {
+            data: ByteStack::new(data),
+        }
     }
     #[inline(always)]
     pub const fn create(sr1_version: HeaderVersion) -> Self {
@@ -308,19 +352,22 @@ impl StaticRecordUV {
     /// Decode and validate a SR
     ///
     /// WARNING: NOT CONTEXTUAL! VALIDATE YOUR OWN STUFF!
-    pub fn decode(data: [u8; 16]) -> Option<Self> {
+    pub fn decode(data: [u8; 16]) -> Option<StaticRecordUVDecoded> {
         let _ = Self::_ENSURE;
-        let slf = Self { data };
+        let slf = Self {
+            data: ByteStack::new(data),
+        };
         // p0: magic; the magic HAS to be the same
-        if u64::from_le(slf.read_qword(Self::OFFSET_P0)) != SR0_MAGIC {
+        if u64::from_le(slf.data.read_qword(Self::OFFSET_P0)) != SR0_MAGIC {
             return None;
         }
-        let sr2_ptr = HostPointerWidth::try_new_with_val(slf.read_byte(Self::OFFSET_P2))?; // p2: ptr width
-        let sr3_endian = HostEndian::try_new_with_val(slf.read_byte(Self::OFFSET_P3))?; // p3: endian
-        let sr4_arch = HostArch::try_new_with_val(slf.read_byte(Self::OFFSET_P4))?; // p4: arch
-        let sr5_os = HostOS::try_new_with_val(slf.read_byte(Self::OFFSET_P5))?; // p5: os
-        Some(Self::new(
-            HeaderVersion::__new(u32::from_le(slf.read_dword(Self::OFFSET_P1))),
+        let sr1_header_version = HeaderVersion::__new(slf.data.read_dword(Self::OFFSET_P1));
+        let sr2_ptr = HostPointerWidth::try_new_with_val(slf.data.read_byte(Self::OFFSET_P2))?; // p2: ptr width
+        let sr3_endian = HostEndian::try_new_with_val(slf.data.read_byte(Self::OFFSET_P3))?; // p3: endian
+        let sr4_arch = HostArch::try_new_with_val(slf.data.read_byte(Self::OFFSET_P4))?; // p4: arch
+        let sr5_os = HostOS::try_new_with_val(slf.data.read_byte(Self::OFFSET_P5))?; // p5: os
+        Some(StaticRecordUVDecoded::new(
+            sr1_header_version,
             sr2_ptr,
             sr3_endian,
             sr4_arch,
@@ -331,30 +378,37 @@ impl StaticRecordUV {
 
 impl StaticRecordUV {
     pub const fn get_ref(&self) -> &[u8] {
-        &self.data
+        self.data.slice()
     }
     pub const fn read_p0_magic(&self) -> u64 {
-        self.read_qword(Self::OFFSET_P0)
+        self.data.read_qword(Self::OFFSET_P0)
     }
     pub const fn read_p1_header_version(&self) -> HeaderVersion {
-        HeaderVersion::__new(self.read_dword(Self::OFFSET_P1))
+        HeaderVersion::__new(self.data.read_dword(Self::OFFSET_P1))
     }
     pub const fn read_p2_ptr_width(&self) -> HostPointerWidth {
-        HostPointerWidth::new_with_val(self.read_byte(Self::OFFSET_P2))
+        HostPointerWidth::new_with_val(self.data.read_byte(Self::OFFSET_P2))
     }
     pub const fn read_p3_endian(&self) -> HostEndian {
-        HostEndian::new_with_val(self.read_byte(Self::OFFSET_P3))
+        HostEndian::new_with_val(self.data.read_byte(Self::OFFSET_P3))
     }
     pub const fn read_p4_arch(&self) -> HostArch {
-        HostArch::new_with_val(self.read_byte(Self::OFFSET_P4))
+        HostArch::new_with_val(self.data.read_byte(Self::OFFSET_P4))
     }
     pub const fn read_p5_os(&self) -> HostOS {
-        HostOS::new_with_val(self.read_byte(Self::OFFSET_P5))
+        HostOS::new_with_val(self.data.read_byte(Self::OFFSET_P5))
+    }
+    pub const fn decoded(&self) -> StaticRecordUVDecoded {
+        StaticRecordUVDecoded::new(
+            self.read_p1_header_version(),
+            self.read_p2_ptr_width(),
+            self.read_p3_endian(),
+            self.read_p4_arch(),
+            self.read_p5_os(),
+        )
     }
 }
 
-impl_stack_read_primitives!(unsafe impl for StaticRecordUV {});
-
 #[test]
 fn test_static_record() {
     let static_record = StaticRecordUV::create(super::versions::v1::V1_HEADER_VERSION);
@@ -372,6 +426,6 @@ fn test_static_record() {
 #[test]
 fn test_static_record_encode_decode() {
     let static_record = StaticRecordUV::create(super::versions::v1::V1_HEADER_VERSION);
-    let static_record_decoded = StaticRecordUV::decode(static_record.data).unwrap();
-    assert_eq!(static_record, static_record_decoded);
+    let static_record_decoded = StaticRecordUV::decode(static_record.data.data_copy()).unwrap();
+    assert_eq!(static_record.decoded(), static_record_decoded);
 }
diff --git a/server/src/engine/storage/macros.rs b/server/src/engine/storage/macros.rs
deleted file mode 100644
index 38b0be82..00000000
--- a/server/src/engine/storage/macros.rs
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Created on Thu May 18 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-// HACK(@ohsayan): until const traits are stable, this is the silly stuff we have to resort to
-macro_rules! impl_stack_read_primitives {
-    (unsafe impl for $ty:ty {}) => {
-        impl $ty {
-            const fn read_byte(&self, position: usize) -> u8 {
-                self.data[position]
-            }
-            const fn read_word(&self, position: usize) -> u16 {
-                unsafe {
-                    core::mem::transmute([self.read_byte(position), self.read_byte(position + 1)])
-                }
-            }
-            const fn read_dword(&self, position: usize) -> u32 {
-                unsafe {
-                    core::mem::transmute([
-                        self.read_word(position),
-                        self.read_word(position + sizeof!(u16)),
-                    ])
-                }
-            }
-            const fn read_qword(&self, position: usize) -> u64 {
-                unsafe {
-                    core::mem::transmute([
-                        self.read_dword(position),
-                        self.read_dword(position + sizeof!(u32)),
-                    ])
-                }
-            }
-            const fn read_xmmword(&self, position: usize) -> u128 {
-                unsafe {
-                    core::mem::transmute([
-                        self.read_qword(position),
-                        self.read_qword(position + sizeof!(u64)),
-                    ])
-                }
-            }
-        }
-    };
-}
diff --git a/server/src/engine/storage/mod.rs b/server/src/engine/storage/mod.rs
index 5a869195..a42cdeb8 100644
--- a/server/src/engine/storage/mod.rs
+++ b/server/src/engine/storage/mod.rs
@@ -26,8 +26,6 @@
 
 //! Implementations of the Skytable Disk Storage Subsystem (SDSS)
 
-#[macro_use]
-mod macros;
 mod header;
 mod versions;
 // impls
diff --git a/server/src/engine/storage/v1/header_impl.rs b/server/src/engine/storage/v1/header_impl.rs
index 8b3519a5..9b4b3ff1 100644
--- a/server/src/engine/storage/v1/header_impl.rs
+++ b/server/src/engine/storage/v1/header_impl.rs
@@ -52,9 +52,12 @@
  * Note: The entire part of the header is little endian encoded
 */
 
-use crate::engine::storage::{
-    header::StaticRecordUV,
-    versions::{self, DriverVersion, ServerVersion},
+use crate::engine::{
+    mem::ByteStack,
+    storage::{
+        header::StaticRecordUV,
+        versions::{self, DriverVersion, ServerVersion},
+    },
 };
 
 /// Static record
@@ -135,25 +138,27 @@ impl FileSpecifierVersion {
 }
 
 pub struct MetadataRecord {
-    data: [u8; 32],
+    data: ByteStack<32>,
 }
 
-impl_stack_read_primitives!(unsafe impl for MetadataRecord {});
-
 impl MetadataRecord {
     /// Decodes a given metadata record, validating all data for correctness.
     ///
     /// WARNING: That means you need to do contextual validation! This function is not aware of any context
     pub fn decode(data: [u8; 32]) -> Option<Self> {
-        let slf = Self { data };
+        let slf = Self {
+            data: ByteStack::new(data),
+        };
         let server_version =
-            ServerVersion::__new(u64::from_le(slf.read_qword(Self::MDR_OFFSET_P0)));
+            ServerVersion::__new(u64::from_le(slf.data.read_qword(Self::MDR_OFFSET_P0)));
         let driver_version =
-            DriverVersion::__new(u64::from_le(slf.read_qword(Self::MDR_OFFSET_P1)));
-        let file_scope = FileScope::try_new(u64::from_le(slf.read_qword(Self::MDR_OFFSET_P2)))?;
-        let file_spec = FileSpecifier::try_new(u32::from_le(slf.read_dword(Self::MDR_OFFSET_P3)))?;
+            DriverVersion::__new(u64::from_le(slf.data.read_qword(Self::MDR_OFFSET_P1)));
+        let file_scope =
+            FileScope::try_new(u64::from_le(slf.data.read_qword(Self::MDR_OFFSET_P2)))?;
+        let file_spec =
+            FileSpecifier::try_new(u32::from_le(slf.data.read_dword(Self::MDR_OFFSET_P3)))?;
         let file_spec_id =
-            FileSpecifierVersion::__new(u32::from_le(slf.read_dword(Self::MDR_OFFSET_P4)));
+            FileSpecifierVersion::__new(u32::from_le(slf.data.read_dword(Self::MDR_OFFSET_P4)));
         Some(Self::new_full(
             server_version,
             driver_version,
@@ -186,8 +191,12 @@ impl MetadataRecord {
         let driver_version = driver_version.little_endian();
         let file_scope = scope.value_qword().to_le_bytes();
         // specifier + specifier ID
-        let file_specifier_and_id: u64 =
-            unsafe { core::mem::transmute([specifier.value_u8() as u32, specifier_id.0]) };
+        let file_specifier_and_id: u64 = unsafe {
+            core::mem::transmute([
+                (specifier.value_u8() as u32).to_le(),
+                specifier_id.0.to_le(),
+            ])
+        };
         let file_specifier_and_id = file_specifier_and_id.to_le_bytes();
         while i < sizeof!(u64) {
             ret[i] = server_version[i];
@@ -196,7 +205,9 @@ impl MetadataRecord {
             ret[i + sizeof!(u64, 3)] = file_specifier_and_id[i];
             i += 1;
         }
-        Self { data: ret }
+        Self {
+            data: ByteStack::new(ret),
+        }
     }
     pub const fn new(
         scope: FileScope,
@@ -215,19 +226,19 @@ impl MetadataRecord {
 
 impl MetadataRecord {
     pub const fn read_p0_server_version(&self) -> ServerVersion {
-        ServerVersion::__new(self.read_qword(Self::MDR_OFFSET_P0))
+        ServerVersion::__new(self.data.read_qword(Self::MDR_OFFSET_P0))
     }
     pub const fn read_p1_driver_version(&self) -> DriverVersion {
-        DriverVersion::__new(self.read_qword(Self::MDR_OFFSET_P1))
+        DriverVersion::__new(self.data.read_qword(Self::MDR_OFFSET_P1))
     }
     pub const fn read_p2_file_scope(&self) -> FileScope {
-        FileScope::new(self.read_qword(Self::MDR_OFFSET_P2))
+        FileScope::new(self.data.read_qword(Self::MDR_OFFSET_P2))
     }
     pub const fn read_p3_file_spec(&self) -> FileSpecifier {
-        FileSpecifier::new(self.read_dword(Self::MDR_OFFSET_P3))
+        FileSpecifier::new(self.data.read_dword(Self::MDR_OFFSET_P3))
     }
     pub const fn read_p4_file_spec_version(&self) -> FileSpecifierVersion {
-        FileSpecifierVersion(self.read_dword(Self::MDR_OFFSET_P4))
+        FileSpecifierVersion(self.data.read_dword(Self::MDR_OFFSET_P4))
     }
 }
 
@@ -270,13 +281,35 @@ impl HostRunMode {
 
 type VHRConstSection = [u8; 56];
 
-impl_stack_read_primitives!(unsafe impl for VariableHostRecord {});
-
 pub struct VariableHostRecord {
-    data: VHRConstSection,
+    data: ByteStack<{ sizeof!(VHRConstSection) }>,
     host_name: Box<[u8]>,
 }
 
+impl VariableHostRecord {
+    /// Decodes and validates the [`VHRConstSection`] of a [`VariableHostRecord`]. Use the returned result to construct this
+    pub fn decode(
+        data: VHRConstSection,
+    ) -> Option<(ByteStack<{ sizeof!(VHRConstSection) }>, usize)> {
+        let s = ByteStack::new(data);
+        let host_epoch_time = s.read_xmmword(Self::VHR_OFFSET_P0);
+        if host_epoch_time > crate::util::os::get_epoch_time() {
+            // and what? we have a file from the future. Einstein says hi. (ok, maybe the host time is incorrect)
+            return None;
+        }
+        let _host_uptime = s.read_xmmword(Self::VHR_OFFSET_P1);
+        let _host_setting_version_id = s.read_dword(Self::VHR_OFFSET_P2A);
+        let _host_setting_run_mode = s.read_dword(Self::VHR_OFFSET_P2B);
+        let _host_startup_counter = s.read_qword(Self::VHR_OFFSET_P3);
+        let host_name_length = s.read_qword(Self::VHR_OFFSET_P4);
+        if host_name_length as usize > usize::MAX {
+            // too large for us to load. per DNS standards this shouldn't be more than 255 but who knows, some people like it wild
+            return None;
+        }
+        Some((s, host_name_length as usize))
+    }
+}
+
 impl VariableHostRecord {
     const VHR_OFFSET_P0: usize = 0;
     const VHR_OFFSET_P1: usize = sizeof!(u128);
@@ -304,7 +337,7 @@ impl VariableHostRecord {
         variable_record_fl[40..48].copy_from_slice(&p3_host_startup_counter.to_le_bytes());
         variable_record_fl[48..56].copy_from_slice(&(p4_host_name_length as u64).to_le_bytes());
         Self {
-            data: variable_record_fl,
+            data: ByteStack::new(variable_record_fl),
             host_name: p5_host_name,
         }
     }
@@ -329,22 +362,22 @@ impl VariableHostRecord {
 
 impl VariableHostRecord {
     pub const fn read_p0_epoch_time(&self) -> u128 {
-        self.read_xmmword(Self::VHR_OFFSET_P0)
+        self.data.read_xmmword(Self::VHR_OFFSET_P0)
     }
     pub const fn read_p1_uptime(&self) -> u128 {
-        self.read_xmmword(Self::VHR_OFFSET_P1)
+        self.data.read_xmmword(Self::VHR_OFFSET_P1)
     }
     pub const fn read_p2a_setting_version_id(&self) -> u32 {
-        self.read_dword(Self::VHR_OFFSET_P2A)
+        self.data.read_dword(Self::VHR_OFFSET_P2A)
     }
     pub const fn read_p2b_run_mode(&self) -> HostRunMode {
-        HostRunMode::new_with_val(self.read_dword(Self::VHR_OFFSET_P2B) as u8)
+        HostRunMode::new_with_val(self.data.read_dword(Self::VHR_OFFSET_P2B) as u8)
     }
     pub const fn read_p3_startup_counter(&self) -> u64 {
-        self.read_qword(Self::VHR_OFFSET_P3)
+        self.data.read_qword(Self::VHR_OFFSET_P3)
     }
     pub const fn read_p4_host_name_length(&self) -> u64 {
-        self.read_qword(Self::VHR_OFFSET_P4)
+        self.data.read_qword(Self::VHR_OFFSET_P4)
     }
     pub fn read_p5_host_name(&self) -> &[u8] {
         &self.host_name
@@ -368,7 +401,7 @@ impl SDSSHeader {
             sr,
             dr_0_mdr,
             dr_1_vhr: VariableHostRecord {
-                data: dr_1_vhr_const_section,
+                data: ByteStack::new(dr_1_vhr_const_section),
                 host_name: dr_1_vhr_host_name,
             },
         }
@@ -401,10 +434,10 @@ impl SDSSHeader {
         self.sr.base.get_ref()
     }
     pub fn get1_dr_0_mdr(&self) -> &[u8] {
-        &self.dr_0_mdr.data
+        self.dr_0_mdr.data.slice()
     }
     pub fn get1_dr_1_vhr_0(&self) -> &[u8] {
-        &self.dr_1_vhr.data
+        self.dr_1_vhr.data.slice()
     }
     pub fn get1_dr_1_vhr_1(&self) -> &[u8] {
         self.dr_1_vhr.host_name.as_ref()

From 901150f98fec674d8d0181230bd4a2e7549fbdaa Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 24 May 2023 10:40:25 -0700
Subject: [PATCH 196/310] Impl basic header enc/dec methods

---
 server/src/engine/storage/header.rs         |  44 ++-
 server/src/engine/storage/v1/header_impl.rs | 397 +++++++++++++++-----
 server/src/engine/storage/v1/rw.rs          |  49 ++-
 3 files changed, 371 insertions(+), 119 deletions(-)

diff --git a/server/src/engine/storage/header.rs b/server/src/engine/storage/header.rs
index 98790f96..28ef2ae3 100644
--- a/server/src/engine/storage/header.rs
+++ b/server/src/engine/storage/header.rs
@@ -265,7 +265,7 @@ impl HostPointerWidth {
 }
 
 #[derive(Debug, PartialEq)]
-pub struct StaticRecordUVDecoded {
+pub struct StaticRecordUV {
     header_version: HeaderVersion,
     ptr_width: HostPointerWidth,
     endian: HostEndian,
@@ -273,7 +273,7 @@ pub struct StaticRecordUVDecoded {
     os: HostOS,
 }
 
-impl StaticRecordUVDecoded {
+impl StaticRecordUV {
     pub const fn new(
         header_version: HeaderVersion,
         ptr_width: HostPointerWidth,
@@ -289,30 +289,39 @@ impl StaticRecordUVDecoded {
             os,
         }
     }
-    pub fn header_version(&self) -> HeaderVersion {
+    pub const fn header_version(&self) -> HeaderVersion {
         self.header_version
     }
-    pub fn ptr_width(&self) -> HostPointerWidth {
+    pub const fn ptr_width(&self) -> HostPointerWidth {
         self.ptr_width
     }
-    pub fn endian(&self) -> HostEndian {
+    pub const fn endian(&self) -> HostEndian {
         self.endian
     }
-    pub fn arch(&self) -> HostArch {
+    pub const fn arch(&self) -> HostArch {
         self.arch
     }
-    pub fn os(&self) -> HostOS {
+    pub const fn os(&self) -> HostOS {
         self.os
     }
+    pub const fn encode(&self) -> StaticRecordUVRaw {
+        StaticRecordUVRaw::new(
+            self.header_version(),
+            self.ptr_width(),
+            self.endian(),
+            self.arch(),
+            self.os(),
+        )
+    }
 }
 
 #[derive(Debug, PartialEq)]
 /// The static record
-pub struct StaticRecordUV {
+pub struct StaticRecordUVRaw {
     data: ByteStack<16>,
 }
 
-impl StaticRecordUV {
+impl StaticRecordUVRaw {
     const OFFSET_P0: usize = 0;
     const OFFSET_P1: usize = sizeof!(u64);
     const OFFSET_P2: usize = Self::OFFSET_P1 + sizeof!(u32);
@@ -352,7 +361,7 @@ impl StaticRecordUV {
     /// Decode and validate a SR
     ///
     /// WARNING: NOT CONTEXTUAL! VALIDATE YOUR OWN STUFF!
-    pub fn decode(data: [u8; 16]) -> Option<StaticRecordUVDecoded> {
+    pub fn decode_from_bytes(data: [u8; 16]) -> Option<StaticRecordUV> {
         let _ = Self::_ENSURE;
         let slf = Self {
             data: ByteStack::new(data),
@@ -366,7 +375,7 @@ impl StaticRecordUV {
         let sr3_endian = HostEndian::try_new_with_val(slf.data.read_byte(Self::OFFSET_P3))?; // p3: endian
         let sr4_arch = HostArch::try_new_with_val(slf.data.read_byte(Self::OFFSET_P4))?; // p4: arch
         let sr5_os = HostOS::try_new_with_val(slf.data.read_byte(Self::OFFSET_P5))?; // p5: os
-        Some(StaticRecordUVDecoded::new(
+        Some(StaticRecordUV::new(
             sr1_header_version,
             sr2_ptr,
             sr3_endian,
@@ -376,7 +385,7 @@ impl StaticRecordUV {
     }
 }
 
-impl StaticRecordUV {
+impl StaticRecordUVRaw {
     pub const fn get_ref(&self) -> &[u8] {
         self.data.slice()
     }
@@ -398,8 +407,8 @@ impl StaticRecordUV {
     pub const fn read_p5_os(&self) -> HostOS {
         HostOS::new_with_val(self.data.read_byte(Self::OFFSET_P5))
     }
-    pub const fn decoded(&self) -> StaticRecordUVDecoded {
-        StaticRecordUVDecoded::new(
+    pub const fn decoded(&self) -> StaticRecordUV {
+        StaticRecordUV::new(
             self.read_p1_header_version(),
             self.read_p2_ptr_width(),
             self.read_p3_endian(),
@@ -411,7 +420,7 @@ impl StaticRecordUV {
 
 #[test]
 fn test_static_record() {
-    let static_record = StaticRecordUV::create(super::versions::v1::V1_HEADER_VERSION);
+    let static_record = StaticRecordUVRaw::create(super::versions::v1::V1_HEADER_VERSION);
     assert_eq!(static_record.read_p0_magic(), SR0_MAGIC);
     assert_eq!(
         static_record.read_p1_header_version(),
@@ -425,7 +434,8 @@ fn test_static_record() {
 
 #[test]
 fn test_static_record_encode_decode() {
-    let static_record = StaticRecordUV::create(super::versions::v1::V1_HEADER_VERSION);
-    let static_record_decoded = StaticRecordUV::decode(static_record.data.data_copy()).unwrap();
+    let static_record = StaticRecordUVRaw::create(super::versions::v1::V1_HEADER_VERSION);
+    let static_record_decoded =
+        StaticRecordUVRaw::decode_from_bytes(static_record.data.data_copy()).unwrap();
     assert_eq!(static_record.decoded(), static_record_decoded);
 }
diff --git a/server/src/engine/storage/v1/header_impl.rs b/server/src/engine/storage/v1/header_impl.rs
index 9b4b3ff1..fe68c8c8 100644
--- a/server/src/engine/storage/v1/header_impl.rs
+++ b/server/src/engine/storage/v1/header_impl.rs
@@ -44,33 +44,61 @@
  * |        +--------------------------------------------+        |
  * |        +--------------------------------------------+        |
  * |        |                                            |        |
- * |        |            VARIABLE HOST RECORD            |        |
+ * |        |             GENESIS HOST RECORD            |        |
  * |        |                  >56B                      |        |
  * |        +--------------------------------------------+        |
+ * |                                                              |
+ * +--------------------------------------------------------------+
+ * +--------------------------------------------------------------+
+ * |                     RUNTIME HOST RECORD                      |
+ * |                           >56B                               |
  * +--------------------------------------------------------------+
- *
  * Note: The entire part of the header is little endian encoded
 */
 
 use crate::engine::{
     mem::ByteStack,
     storage::{
-        header::StaticRecordUV,
+        header::{StaticRecordUV, StaticRecordUVRaw},
         versions::{self, DriverVersion, ServerVersion},
     },
 };
 
-/// Static record
 pub struct StaticRecord {
-    base: StaticRecordUV,
+    sr: StaticRecordUV,
 }
 
 impl StaticRecord {
+    pub const fn new(sr: StaticRecordUV) -> Self {
+        Self { sr }
+    }
+    pub const fn encode(&self) -> StaticRecordRaw {
+        StaticRecordRaw {
+            base: self.sr.encode(),
+        }
+    }
+    pub const fn sr(&self) -> &StaticRecordUV {
+        &self.sr
+    }
+}
+
+/// Static record
+pub struct StaticRecordRaw {
+    base: StaticRecordUVRaw,
+}
+
+impl StaticRecordRaw {
     pub const fn new() -> Self {
         Self {
-            base: StaticRecordUV::create(versions::v1::V1_HEADER_VERSION),
+            base: StaticRecordUVRaw::create(versions::v1::V1_HEADER_VERSION),
         }
     }
+    pub const fn empty_buffer() -> [u8; sizeof!(Self)] {
+        [0u8; sizeof!(Self)]
+    }
+    pub fn decode_from_bytes(buf: [u8; sizeof!(Self)]) -> Option<StaticRecord> {
+        StaticRecordUVRaw::decode_from_bytes(buf).map(StaticRecord::new)
+    }
 }
 
 /*
@@ -138,28 +166,74 @@ impl FileSpecifierVersion {
 }
 
 pub struct MetadataRecord {
-    data: ByteStack<32>,
+    server_version: ServerVersion,
+    driver_version: DriverVersion,
+    file_scope: FileScope,
+    file_spec: FileSpecifier,
+    file_spec_id: FileSpecifierVersion,
 }
 
 impl MetadataRecord {
+    pub const fn new(
+        server_version: ServerVersion,
+        driver_version: DriverVersion,
+        file_scope: FileScope,
+        file_spec: FileSpecifier,
+        file_spec_id: FileSpecifierVersion,
+    ) -> Self {
+        Self {
+            server_version,
+            driver_version,
+            file_scope,
+            file_spec,
+            file_spec_id,
+        }
+    }
+    pub const fn server_version(&self) -> ServerVersion {
+        self.server_version
+    }
+    pub const fn driver_version(&self) -> DriverVersion {
+        self.driver_version
+    }
+    pub const fn file_scope(&self) -> FileScope {
+        self.file_scope
+    }
+    pub const fn file_spec(&self) -> FileSpecifier {
+        self.file_spec
+    }
+    pub const fn file_spec_id(&self) -> FileSpecifierVersion {
+        self.file_spec_id
+    }
+    pub const fn encode(&self) -> MetadataRecordRaw {
+        MetadataRecordRaw::new_full(
+            self.server_version(),
+            self.driver_version(),
+            self.file_scope(),
+            self.file_spec(),
+            self.file_spec_id(),
+        )
+    }
+}
+
+pub struct MetadataRecordRaw {
+    data: ByteStack<32>,
+}
+
+impl MetadataRecordRaw {
     /// Decodes a given metadata record, validating all data for correctness.
     ///
     /// WARNING: That means you need to do contextual validation! This function is not aware of any context
-    pub fn decode(data: [u8; 32]) -> Option<Self> {
-        let slf = Self {
-            data: ByteStack::new(data),
-        };
+    pub fn decode_from_bytes(data: [u8; 32]) -> Option<MetadataRecord> {
+        let data = ByteStack::new(data);
         let server_version =
-            ServerVersion::__new(u64::from_le(slf.data.read_qword(Self::MDR_OFFSET_P0)));
+            ServerVersion::__new(u64::from_le(data.read_qword(Self::MDR_OFFSET_P0)));
         let driver_version =
-            DriverVersion::__new(u64::from_le(slf.data.read_qword(Self::MDR_OFFSET_P1)));
-        let file_scope =
-            FileScope::try_new(u64::from_le(slf.data.read_qword(Self::MDR_OFFSET_P2)))?;
-        let file_spec =
-            FileSpecifier::try_new(u32::from_le(slf.data.read_dword(Self::MDR_OFFSET_P3)))?;
+            DriverVersion::__new(u64::from_le(data.read_qword(Self::MDR_OFFSET_P1)));
+        let file_scope = FileScope::try_new(u64::from_le(data.read_qword(Self::MDR_OFFSET_P2)))?;
+        let file_spec = FileSpecifier::try_new(u32::from_le(data.read_dword(Self::MDR_OFFSET_P3)))?;
         let file_spec_id =
-            FileSpecifierVersion::__new(u32::from_le(slf.data.read_dword(Self::MDR_OFFSET_P4)));
-        Some(Self::new_full(
+            FileSpecifierVersion::__new(u32::from_le(data.read_dword(Self::MDR_OFFSET_P4)));
+        Some(MetadataRecord::new(
             server_version,
             driver_version,
             file_scope,
@@ -169,13 +243,16 @@ impl MetadataRecord {
     }
 }
 
-impl MetadataRecord {
+impl MetadataRecordRaw {
     const MDR_OFFSET_P0: usize = 0;
     const MDR_OFFSET_P1: usize = sizeof!(u64);
     const MDR_OFFSET_P2: usize = Self::MDR_OFFSET_P1 + sizeof!(u64);
     const MDR_OFFSET_P3: usize = Self::MDR_OFFSET_P2 + sizeof!(u64);
     const MDR_OFFSET_P4: usize = Self::MDR_OFFSET_P3 + sizeof!(u32);
     const _ENSURE: () = assert!(Self::MDR_OFFSET_P4 == (sizeof!(Self) - sizeof!(u32)));
+    pub const fn empty_buffer() -> [u8; sizeof!(Self)] {
+        [0u8; sizeof!(Self)]
+    }
     pub const fn new_full(
         server_version: ServerVersion,
         driver_version: DriverVersion,
@@ -224,7 +301,7 @@ impl MetadataRecord {
     }
 }
 
-impl MetadataRecord {
+impl MetadataRecordRaw {
     pub const fn read_p0_server_version(&self) -> ServerVersion {
         ServerVersion::__new(self.data.read_qword(Self::MDR_OFFSET_P0))
     }
@@ -245,7 +322,7 @@ impl MetadataRecord {
 /*
     Dynamic Record (2/2)
     ---
-    Variable record (?B; > 56B):
+    Host record (?B; > 56B):
     - 16B: Host epoch time in nanoseconds
     - 16B: Host uptime in nanoseconds
     - 08B:
@@ -264,14 +341,14 @@ pub enum HostRunMode {
 }
 
 impl HostRunMode {
-    pub const fn try_new_with_val(v: u8) -> Option<Self> {
+    pub const fn try_new_with_val(v: u32) -> Option<Self> {
         Some(match v {
             0 => Self::Dev,
             1 => Self::Prod,
             _ => return None,
         })
     }
-    pub const fn new_with_val(v: u8) -> Self {
+    pub const fn new_with_val(v: u32) -> Self {
         match Self::try_new_with_val(v) {
             Some(v) => v,
             None => panic!("unknown hostrunmode"),
@@ -279,45 +356,138 @@ impl HostRunMode {
     }
 }
 
-type VHRConstSection = [u8; 56];
+type HRConstSectionRaw = [u8; 56];
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct HostRecord {
+    hr_cr: HRConstSection,
+    host_name: Box<[u8]>,
+}
+
+impl HostRecord {
+    pub fn new(hr_cr: HRConstSection, host_name: Box<[u8]>) -> Self {
+        Self { hr_cr, host_name }
+    }
+    pub fn hr_cr(&self) -> &HRConstSection {
+        &self.hr_cr
+    }
+    pub fn host_name(&self) -> &[u8] {
+        self.host_name.as_ref()
+    }
+    pub fn encode(&self) -> HostRecordRaw {
+        HostRecordRaw::new(
+            self.hr_cr().host_epoch_time(),
+            self.hr_cr().host_uptime(),
+            self.hr_cr().host_setting_version_id(),
+            self.hr_cr().host_run_mode(),
+            self.hr_cr().host_startup_counter(),
+            self.host_name().into(),
+        )
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct HRConstSection {
+    host_epoch_time: u128,
+    host_uptime: u128,
+    host_setting_version_id: u32,
+    host_run_mode: HostRunMode,
+    host_startup_counter: u64,
+}
+
+impl HRConstSection {
+    pub const fn new(
+        host_epoch_time: u128,
+        host_uptime: u128,
+        host_setting_version_id: u32,
+        host_run_mode: HostRunMode,
+        host_startup_counter: u64,
+    ) -> Self {
+        Self {
+            host_epoch_time,
+            host_uptime,
+            host_setting_version_id,
+            host_run_mode,
+            host_startup_counter,
+        }
+    }
+    pub const fn host_epoch_time(&self) -> u128 {
+        self.host_epoch_time
+    }
+    pub const fn host_uptime(&self) -> u128 {
+        self.host_uptime
+    }
+    pub const fn host_setting_version_id(&self) -> u32 {
+        self.host_setting_version_id
+    }
+    pub const fn host_run_mode(&self) -> HostRunMode {
+        self.host_run_mode
+    }
+    pub const fn host_startup_counter(&self) -> u64 {
+        self.host_startup_counter
+    }
+}
 
-pub struct VariableHostRecord {
-    data: ByteStack<{ sizeof!(VHRConstSection) }>,
+pub struct HostRecordRaw {
+    data: ByteStack<{ sizeof!(HRConstSectionRaw) }>,
     host_name: Box<[u8]>,
 }
 
-impl VariableHostRecord {
-    /// Decodes and validates the [`VHRConstSection`] of a [`VariableHostRecord`]. Use the returned result to construct this
-    pub fn decode(
-        data: VHRConstSection,
-    ) -> Option<(ByteStack<{ sizeof!(VHRConstSection) }>, usize)> {
+impl HostRecordRaw {
+    pub const fn empty_buffer_const_section() -> [u8; sizeof!(HRConstSectionRaw)] {
+        [0u8; sizeof!(HRConstSectionRaw)]
+    }
+    /// Decodes and validates the [`HRConstSection`] of a [`HostRecord`]. Use the returned result to construct this
+    pub fn decode_from_bytes_const_sec(data: HRConstSectionRaw) -> Option<(HRConstSection, usize)> {
         let s = ByteStack::new(data);
-        let host_epoch_time = s.read_xmmword(Self::VHR_OFFSET_P0);
+        let host_epoch_time = s.read_xmmword(Self::HR_OFFSET_P0);
         if host_epoch_time > crate::util::os::get_epoch_time() {
             // and what? we have a file from the future. Einstein says hi. (ok, maybe the host time is incorrect)
             return None;
         }
-        let _host_uptime = s.read_xmmword(Self::VHR_OFFSET_P1);
-        let _host_setting_version_id = s.read_dword(Self::VHR_OFFSET_P2A);
-        let _host_setting_run_mode = s.read_dword(Self::VHR_OFFSET_P2B);
-        let _host_startup_counter = s.read_qword(Self::VHR_OFFSET_P3);
-        let host_name_length = s.read_qword(Self::VHR_OFFSET_P4);
+        let host_uptime = s.read_xmmword(Self::HR_OFFSET_P1);
+        let host_setting_version_id = s.read_dword(Self::HR_OFFSET_P2A);
+        let host_setting_run_mode =
+            HostRunMode::try_new_with_val(s.read_dword(Self::HR_OFFSET_P2B))?;
+        let host_startup_counter = s.read_qword(Self::HR_OFFSET_P3);
+        let host_name_length = s.read_qword(Self::HR_OFFSET_P4);
         if host_name_length as usize > usize::MAX {
             // too large for us to load. per DNS standards this shouldn't be more than 255 but who knows, some people like it wild
             return None;
         }
-        Some((s, host_name_length as usize))
+        Some((
+            HRConstSection::new(
+                host_epoch_time,
+                host_uptime,
+                host_setting_version_id,
+                host_setting_run_mode,
+                host_startup_counter,
+            ),
+            host_name_length as usize,
+        ))
+    }
+    pub fn decoded(&self) -> HostRecord {
+        HostRecord::new(
+            HRConstSection::new(
+                self.read_p0_epoch_time(),
+                self.read_p1_uptime(),
+                self.read_p2a_setting_version_id(),
+                self.read_p2b_run_mode(),
+                self.read_p3_startup_counter(),
+            ),
+            self.host_name.clone(),
+        )
     }
 }
 
-impl VariableHostRecord {
-    const VHR_OFFSET_P0: usize = 0;
-    const VHR_OFFSET_P1: usize = sizeof!(u128);
-    const VHR_OFFSET_P2A: usize = Self::VHR_OFFSET_P1 + sizeof!(u128);
-    const VHR_OFFSET_P2B: usize = Self::VHR_OFFSET_P2A + sizeof!(u32);
-    const VHR_OFFSET_P3: usize = Self::VHR_OFFSET_P2B + sizeof!(u32);
-    const VHR_OFFSET_P4: usize = Self::VHR_OFFSET_P3 + sizeof!(u64);
-    const _ENSURE: () = assert!(Self::VHR_OFFSET_P4 == sizeof!(VHRConstSection) - sizeof!(u64));
+impl HostRecordRaw {
+    const HR_OFFSET_P0: usize = 0;
+    const HR_OFFSET_P1: usize = sizeof!(u128);
+    const HR_OFFSET_P2A: usize = Self::HR_OFFSET_P1 + sizeof!(u128);
+    const HR_OFFSET_P2B: usize = Self::HR_OFFSET_P2A + sizeof!(u32);
+    const HR_OFFSET_P3: usize = Self::HR_OFFSET_P2B + sizeof!(u32);
+    const HR_OFFSET_P4: usize = Self::HR_OFFSET_P3 + sizeof!(u64);
+    const _ENSURE: () = assert!(Self::HR_OFFSET_P4 == sizeof!(HRConstSectionRaw) - sizeof!(u64));
     pub fn new(
         p0_host_epoch_time: u128,
         p1_host_uptime: u128,
@@ -328,16 +498,16 @@ impl VariableHostRecord {
     ) -> Self {
         let _ = Self::_ENSURE;
         let p4_host_name_length = p5_host_name.len();
-        let mut variable_record_fl = [0u8; 56];
-        variable_record_fl[0..16].copy_from_slice(&p0_host_epoch_time.to_le_bytes());
-        variable_record_fl[16..32].copy_from_slice(&p1_host_uptime.to_le_bytes());
-        variable_record_fl[32..36].copy_from_slice(&p2a_host_setting_version_id.to_le_bytes());
-        variable_record_fl[36..40]
+        let mut host_record_fl = [0u8; 56];
+        host_record_fl[0..16].copy_from_slice(&p0_host_epoch_time.to_le_bytes());
+        host_record_fl[16..32].copy_from_slice(&p1_host_uptime.to_le_bytes());
+        host_record_fl[32..36].copy_from_slice(&p2a_host_setting_version_id.to_le_bytes());
+        host_record_fl[36..40]
             .copy_from_slice(&(p2b_host_run_mode.value_u8() as u32).to_le_bytes());
-        variable_record_fl[40..48].copy_from_slice(&p3_host_startup_counter.to_le_bytes());
-        variable_record_fl[48..56].copy_from_slice(&(p4_host_name_length as u64).to_le_bytes());
+        host_record_fl[40..48].copy_from_slice(&p3_host_startup_counter.to_le_bytes());
+        host_record_fl[48..56].copy_from_slice(&(p4_host_name_length as u64).to_le_bytes());
         Self {
-            data: ByteStack::new(variable_record_fl),
+            data: ByteStack::new(host_record_fl),
             host_name: p5_host_name,
         }
     }
@@ -360,24 +530,24 @@ impl VariableHostRecord {
     }
 }
 
-impl VariableHostRecord {
+impl HostRecordRaw {
     pub const fn read_p0_epoch_time(&self) -> u128 {
-        self.data.read_xmmword(Self::VHR_OFFSET_P0)
+        self.data.read_xmmword(Self::HR_OFFSET_P0)
     }
     pub const fn read_p1_uptime(&self) -> u128 {
-        self.data.read_xmmword(Self::VHR_OFFSET_P1)
+        self.data.read_xmmword(Self::HR_OFFSET_P1)
     }
     pub const fn read_p2a_setting_version_id(&self) -> u32 {
-        self.data.read_dword(Self::VHR_OFFSET_P2A)
+        self.data.read_dword(Self::HR_OFFSET_P2A)
     }
     pub const fn read_p2b_run_mode(&self) -> HostRunMode {
-        HostRunMode::new_with_val(self.data.read_dword(Self::VHR_OFFSET_P2B) as u8)
+        HostRunMode::new_with_val(self.data.read_dword(Self::HR_OFFSET_P2B))
     }
     pub const fn read_p3_startup_counter(&self) -> u64 {
-        self.data.read_qword(Self::VHR_OFFSET_P3)
+        self.data.read_qword(Self::HR_OFFSET_P3)
     }
     pub const fn read_p4_host_name_length(&self) -> u64 {
-        self.data.read_qword(Self::VHR_OFFSET_P4)
+        self.data.read_qword(Self::HR_OFFSET_P4)
     }
     pub fn read_p5_host_name(&self) -> &[u8] {
         &self.host_name
@@ -386,23 +556,54 @@ impl VariableHostRecord {
 
 pub struct SDSSHeader {
     sr: StaticRecord,
-    dr_0_mdr: MetadataRecord,
-    dr_1_vhr: VariableHostRecord,
+    mdr: MetadataRecord,
+    hr: HostRecord,
 }
 
 impl SDSSHeader {
+    pub const fn new(sr: StaticRecord, mdr: MetadataRecord, hr: HostRecord) -> Self {
+        Self { sr, mdr, hr }
+    }
+    pub const fn sr(&self) -> &StaticRecord {
+        &self.sr
+    }
+    pub const fn mdr(&self) -> &MetadataRecord {
+        &self.mdr
+    }
+    pub const fn hr(&self) -> &HostRecord {
+        &self.hr
+    }
+    pub fn encode(&self) -> SDSSHeaderRaw {
+        SDSSHeaderRaw::new_full(self.sr.encode(), self.mdr.encode(), self.hr.encode())
+    }
+}
+
+pub struct SDSSHeaderRaw {
+    sr: StaticRecordRaw,
+    dr_0_mdr: MetadataRecordRaw,
+    dr_1_hr: HostRecordRaw,
+}
+
+impl SDSSHeaderRaw {
+    pub fn new_full(sr: StaticRecordRaw, mdr: MetadataRecordRaw, hr: HostRecordRaw) -> Self {
+        Self {
+            sr,
+            dr_0_mdr: mdr,
+            dr_1_hr: hr,
+        }
+    }
     pub fn new(
-        sr: StaticRecord,
-        dr_0_mdr: MetadataRecord,
-        dr_1_vhr_const_section: VHRConstSection,
-        dr_1_vhr_host_name: Box<[u8]>,
+        sr: StaticRecordRaw,
+        dr_0_mdr: MetadataRecordRaw,
+        dr_1_hr_const_section: HRConstSectionRaw,
+        dr_1_hr_host_name: Box<[u8]>,
     ) -> Self {
         Self {
             sr,
             dr_0_mdr,
-            dr_1_vhr: VariableHostRecord {
-                data: ByteStack::new(dr_1_vhr_const_section),
-                host_name: dr_1_vhr_host_name,
+            dr_1_hr: HostRecordRaw {
+                data: ByteStack::new(dr_1_hr_const_section),
+                host_name: dr_1_hr_host_name,
             },
         }
     }
@@ -410,23 +611,23 @@ impl SDSSHeader {
         mdr_file_scope: FileScope,
         mdr_file_specifier: FileSpecifier,
         mdr_file_specifier_id: FileSpecifierVersion,
-        vhr_host_setting_id: u32,
-        vhr_host_run_mode: HostRunMode,
-        vhr_host_startup_counter: u64,
-        vhr_host_name: Box<[u8]>,
+        hr_host_setting_id: u32,
+        hr_host_run_mode: HostRunMode,
+        hr_host_startup_counter: u64,
+        hr_host_name: Box<[u8]>,
     ) -> Self {
         Self {
-            sr: StaticRecord::new(),
-            dr_0_mdr: MetadataRecord::new(
+            sr: StaticRecordRaw::new(),
+            dr_0_mdr: MetadataRecordRaw::new(
                 mdr_file_scope,
                 mdr_file_specifier,
                 mdr_file_specifier_id,
             ),
-            dr_1_vhr: VariableHostRecord::new_auto(
-                vhr_host_setting_id,
-                vhr_host_run_mode,
-                vhr_host_startup_counter,
-                vhr_host_name,
+            dr_1_hr: HostRecordRaw::new_auto(
+                hr_host_setting_id,
+                hr_host_run_mode,
+                hr_host_startup_counter,
+                hr_host_name,
             ),
         }
     }
@@ -436,23 +637,23 @@ impl SDSSHeader {
     pub fn get1_dr_0_mdr(&self) -> &[u8] {
         self.dr_0_mdr.data.slice()
     }
-    pub fn get1_dr_1_vhr_0(&self) -> &[u8] {
-        self.dr_1_vhr.data.slice()
+    pub fn get1_dr_1_hr_0(&self) -> &[u8] {
+        self.dr_1_hr.data.slice()
     }
-    pub fn get1_dr_1_vhr_1(&self) -> &[u8] {
-        self.dr_1_vhr.host_name.as_ref()
+    pub fn get1_dr_1_hr_1(&self) -> &[u8] {
+        self.dr_1_hr.host_name.as_ref()
     }
     pub fn calculate_header_size(&self) -> usize {
-        Self::calculate_fixed_header_size() + self.dr_1_vhr.host_name.len()
+        Self::calculate_fixed_header_size() + self.dr_1_hr.host_name.len()
     }
     pub const fn calculate_fixed_header_size() -> usize {
-        sizeof!(StaticRecord) + sizeof!(MetadataRecord) + sizeof!(VHRConstSection)
+        sizeof!(StaticRecordRaw) + sizeof!(MetadataRecordRaw) + sizeof!(HRConstSectionRaw)
     }
 }
 
 #[test]
 fn test_metadata_record_encode_decode() {
-    let md = MetadataRecord::new(
+    let md = MetadataRecordRaw::new(
         FileScope::TransactionLog,
         FileSpecifier::GNSTxnLog,
         FileSpecifierVersion(1),
@@ -465,7 +666,7 @@ fn test_metadata_record_encode_decode() {
 }
 
 #[test]
-fn test_variable_host_record_encode_decode() {
+fn test_host_record_encode_decode() {
     const HOST_UPTIME: u128 = u128::MAX - 434324903;
     const HOST_SETTING_VERSION_ID: u32 = 245;
     const HOST_RUN_MODE: HostRunMode = HostRunMode::Prod;
@@ -476,7 +677,7 @@ fn test_variable_host_record_encode_decode() {
         .duration_since(UNIX_EPOCH)
         .unwrap()
         .as_nanos();
-    let vhr = VariableHostRecord::new(
+    let hr = HostRecordRaw::new(
         time,
         HOST_UPTIME,
         HOST_SETTING_VERSION_ID,
@@ -484,11 +685,11 @@ fn test_variable_host_record_encode_decode() {
         HOST_STARTUP_COUNTER,
         HOST_NAME.as_bytes().to_owned().into_boxed_slice(),
     );
-    assert_eq!(vhr.read_p0_epoch_time(), time);
-    assert_eq!(vhr.read_p1_uptime(), HOST_UPTIME);
-    assert_eq!(vhr.read_p2a_setting_version_id(), HOST_SETTING_VERSION_ID);
-    assert_eq!(vhr.read_p2b_run_mode(), HOST_RUN_MODE);
-    assert_eq!(vhr.read_p3_startup_counter(), HOST_STARTUP_COUNTER);
-    assert_eq!(vhr.read_p4_host_name_length(), HOST_NAME.len() as u64);
-    assert_eq!(vhr.read_p5_host_name(), HOST_NAME.as_bytes());
+    assert_eq!(hr.read_p0_epoch_time(), time);
+    assert_eq!(hr.read_p1_uptime(), HOST_UPTIME);
+    assert_eq!(hr.read_p2a_setting_version_id(), HOST_SETTING_VERSION_ID);
+    assert_eq!(hr.read_p2b_run_mode(), HOST_RUN_MODE);
+    assert_eq!(hr.read_p3_startup_counter(), HOST_STARTUP_COUNTER);
+    assert_eq!(hr.read_p4_host_name_length(), HOST_NAME.len() as u64);
+    assert_eq!(hr.read_p5_host_name(), HOST_NAME.as_bytes());
 }
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 33fdffb0..bc0d88ac 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -24,9 +24,14 @@
  *
 */
 
+use crate::engine::storage::versions;
+
 use {
-    super::header_impl::SDSSHeader,
+    super::header_impl::SDSSHeaderRaw,
     crate::{
+        engine::storage::v1::header_impl::{
+            HostRecord, HostRecordRaw, MetadataRecordRaw, SDSSHeader, StaticRecordRaw,
+        },
         util::{ByteRepr, NumericRepr},
         IoResult,
     },
@@ -40,6 +45,9 @@ use {
 pub enum SDSSError {
     SRVersionMismatch,
     IoError(IoError),
+    CorruptedHeaderSR,
+    CorruptedHeaderMDR,
+    CorruptedHeaderHR,
 }
 
 impl From<IoError> for SDSSError {
@@ -93,12 +101,12 @@ pub struct SDSSWriter<W> {
 }
 
 impl<W: RawWriterInterface> SDSSWriter<W> {
-    pub fn open_create_with_header(file: &str, header: SDSSHeader) -> IoResult<Self> {
+    pub fn open_create_with_header(file: &str, header: SDSSHeaderRaw) -> IoResult<Self> {
         let mut w = W::fopen_create(file)?;
         w.fwrite_all(header.get0_sr())?;
         w.fwrite_all(header.get1_dr_0_mdr())?;
-        w.fwrite_all(header.get1_dr_1_vhr_0())?;
-        w.fwrite_all(header.get1_dr_1_vhr_1())?;
+        w.fwrite_all(header.get1_dr_1_hr_0())?;
+        w.fwrite_all(header.get1_dr_1_hr_1())?;
         w.fsync_all()?;
         Ok(Self { writer: w })
     }
@@ -170,3 +178,36 @@ impl RawReaderInterface for FileBuffered {
         Ok(())
     }
 }
+
+pub struct SDSSReader<R> {
+    reader: R,
+    header: SDSSHeader,
+}
+
+impl<R: RawReaderInterface> SDSSReader<R> {
+    pub fn open(f: &str) -> SDSSResult<Self> {
+        let mut r = R::fopen(f)?;
+        let mut sr = StaticRecordRaw::empty_buffer();
+        let mut mdr = MetadataRecordRaw::empty_buffer();
+        let mut hr_0_const = HostRecordRaw::empty_buffer_const_section();
+        r.fread_exact_seek(&mut sr)?;
+        r.fread_exact_seek(&mut mdr)?;
+        r.fread_exact_seek(&mut hr_0_const)?;
+        let sr = StaticRecordRaw::decode_from_bytes(sr).ok_or(SDSSError::CorruptedHeaderSR)?;
+        let mdr = MetadataRecordRaw::decode_from_bytes(mdr).ok_or(SDSSError::CorruptedHeaderMDR)?;
+        let (hr_const, hostname_len) = HostRecordRaw::decode_from_bytes_const_sec(hr_0_const)
+            .ok_or(SDSSError::CorruptedHeaderHR)?;
+        let mut host_name = vec![0u8; hostname_len].into_boxed_slice();
+        r.fread_exact_seek(&mut host_name)?;
+        if (sr.sr().header_version() != versions::v1::V1_HEADER_VERSION)
+            || (mdr.driver_version() != versions::v1::V1_DRIVER_VERSION)
+            || (mdr.server_version() != versions::v1::V1_SERVER_VERSION)
+        {
+            return Err(SDSSError::SRVersionMismatch);
+        }
+        Ok(Self {
+            reader: r,
+            header: SDSSHeader::new(sr, mdr, HostRecord::new(hr_const, host_name)),
+        })
+    }
+}

From e126c826f947677b656bf0fc9e250e1a67be92c2 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 25 May 2023 18:26:46 +0000
Subject: [PATCH 197/310] Add new dynamic record definition and remove rw

---
 .../src/engine/storage/v1/header_impl/dr.rs   | 373 ++++++++++++++++++
 .../v1/{header_impl.rs => header_impl/gr.rs}  | 276 +------------
 .../src/engine/storage/v1/header_impl/mod.rs  | 266 +++++++++++++
 .../src/engine/storage/v1/header_impl/sr.rs   |  68 ++++
 server/src/engine/storage/v1/rw.rs            | 188 ---------
 5 files changed, 727 insertions(+), 444 deletions(-)
 create mode 100644 server/src/engine/storage/v1/header_impl/dr.rs
 rename server/src/engine/storage/v1/{header_impl.rs => header_impl/gr.rs} (64%)
 create mode 100644 server/src/engine/storage/v1/header_impl/mod.rs
 create mode 100644 server/src/engine/storage/v1/header_impl/sr.rs

diff --git a/server/src/engine/storage/v1/header_impl/dr.rs b/server/src/engine/storage/v1/header_impl/dr.rs
new file mode 100644
index 00000000..1f530ec9
--- /dev/null
+++ b/server/src/engine/storage/v1/header_impl/dr.rs
@@ -0,0 +1,373 @@
+/*
+ * Created on Thu May 25 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    mem::ByteStack,
+    storage::{
+        header::{HostArch, HostEndian, HostOS, HostPointerWidth},
+        v1::header_impl::FileSpecifierVersion,
+        versions::{DriverVersion, ServerVersion},
+    },
+};
+
+/*
+    Dynamic record (1/2): Host signature
+    ---
+    - 8B: Server version
+    - 8B: Driver version
+    - 4B: File specifier ID
+    - 1B: Endian
+    - 1B: Pointer width
+    - 1B: Arch
+    - 1B: OS
+*/
+
+#[derive(Debug, PartialEq)]
+pub struct DRHostSignature {
+    server_version: ServerVersion,
+    driver_version: DriverVersion,
+    file_specifier_version: FileSpecifierVersion,
+    endian: HostEndian,
+    ptr_width: HostPointerWidth,
+    arch: HostArch,
+    os: HostOS,
+}
+
+impl DRHostSignature {
+    pub const fn new(
+        server_version: ServerVersion,
+        driver_version: DriverVersion,
+        file_specifier_version: FileSpecifierVersion,
+        endian: HostEndian,
+        ptr_width: HostPointerWidth,
+        arch: HostArch,
+        os: HostOS,
+    ) -> Self {
+        Self {
+            server_version,
+            driver_version,
+            file_specifier_version,
+            endian,
+            ptr_width,
+            arch,
+            os,
+        }
+    }
+    pub const fn server_version(&self) -> ServerVersion {
+        self.server_version
+    }
+    pub const fn driver_version(&self) -> DriverVersion {
+        self.driver_version
+    }
+    pub const fn file_specifier_version(&self) -> FileSpecifierVersion {
+        self.file_specifier_version
+    }
+    pub const fn endian(&self) -> HostEndian {
+        self.endian
+    }
+    pub const fn ptr_width(&self) -> HostPointerWidth {
+        self.ptr_width
+    }
+    pub const fn arch(&self) -> HostArch {
+        self.arch
+    }
+    pub const fn os(&self) -> HostOS {
+        self.os
+    }
+    pub const fn encoded(&self) -> DRHostSignatureRaw {
+        DRHostSignatureRaw::new_full(
+            self.server_version(),
+            self.driver_version(),
+            self.file_specifier_version(),
+            self.endian(),
+            self.ptr_width(),
+            self.arch(),
+            self.os(),
+        )
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub struct DRHostSignatureRaw {
+    data: ByteStack<24>,
+}
+
+impl DRHostSignatureRaw {
+    const DRHS_OFFSET_P0: usize = 0;
+    const DRHS_OFFSET_P1: usize = sizeof!(u64);
+    const DRHS_OFFSET_P2: usize = Self::DRHS_OFFSET_P1 + sizeof!(u64);
+    const DRHS_OFFSET_P3: usize = Self::DRHS_OFFSET_P2 + sizeof!(u32);
+    const DRHS_OFFSET_P4: usize = Self::DRHS_OFFSET_P3 + 1;
+    const DRHS_OFFSET_P5: usize = Self::DRHS_OFFSET_P4 + 1;
+    const DRHS_OFFSET_P6: usize = Self::DRHS_OFFSET_P5 + 1;
+    const _ENSURE: () = assert!(Self::DRHS_OFFSET_P6 == sizeof!(Self) - 1);
+    pub const fn new(
+        server_version: ServerVersion,
+        driver_version: DriverVersion,
+        file_specifier_id: FileSpecifierVersion,
+    ) -> Self {
+        Self::new_full(
+            server_version,
+            driver_version,
+            file_specifier_id,
+            HostEndian::new(),
+            HostPointerWidth::new(),
+            HostArch::new(),
+            HostOS::new(),
+        )
+    }
+    pub const fn new_full(
+        server_version: ServerVersion,
+        driver_version: DriverVersion,
+        file_specifier_id: FileSpecifierVersion,
+        endian: HostEndian,
+        ptr_width: HostPointerWidth,
+        arch: HostArch,
+        os: HostOS,
+    ) -> Self {
+        let _ = Self::_ENSURE;
+        let bytes: [u8; 24] = unsafe {
+            let [qw_a, qw_b]: [u64; 2] = core::mem::transmute([
+                server_version.little_endian(),
+                driver_version.little_endian(),
+            ]);
+            let dw: u32 = core::mem::transmute([
+                endian.value_u8(),
+                ptr_width.value_u8(),
+                arch.value_u8(),
+                os.value_u8(),
+            ]);
+            let qw_c: u64 = core::mem::transmute([(file_specifier_id.0.to_le(), dw.to_le())]);
+            core::mem::transmute([qw_a, qw_b, qw_c])
+        };
+        Self {
+            data: ByteStack::new(bytes),
+        }
+    }
+}
+
+impl DRHostSignatureRaw {
+    pub const fn read_p0_server_version(&self) -> ServerVersion {
+        ServerVersion::__new(self.data.read_qword(Self::DRHS_OFFSET_P0))
+    }
+    pub const fn read_p1_driver_version(&self) -> DriverVersion {
+        DriverVersion::__new(self.data.read_qword(Self::DRHS_OFFSET_P1))
+    }
+    pub const fn read_p2_file_specifier_id(&self) -> FileSpecifierVersion {
+        FileSpecifierVersion::__new(self.data.read_dword(Self::DRHS_OFFSET_P2))
+    }
+    pub const fn read_p3_endian(&self) -> HostEndian {
+        HostEndian::new_with_val(self.data.read_byte(Self::DRHS_OFFSET_P3))
+    }
+    pub const fn read_p4_pointer_width(&self) -> HostPointerWidth {
+        HostPointerWidth::new_with_val(self.data.read_byte(Self::DRHS_OFFSET_P4))
+    }
+    pub const fn read_p5_arch(&self) -> HostArch {
+        HostArch::new_with_val(self.data.read_byte(Self::DRHS_OFFSET_P5))
+    }
+    pub const fn read_p6_os(&self) -> HostOS {
+        HostOS::new_with_val(self.data.read_byte(Self::DRHS_OFFSET_P6))
+    }
+    pub const fn encoded(&self) -> DRHostSignature {
+        DRHostSignature::new(
+            self.read_p0_server_version(),
+            self.read_p1_driver_version(),
+            self.read_p2_file_specifier_id(),
+            self.read_p3_endian(),
+            self.read_p4_pointer_width(),
+            self.read_p5_arch(),
+            self.read_p6_os(),
+        )
+    }
+}
+
+/*
+    Dynamic record (2/2): Runtime signature
+    ---
+    - 8B: Dynamic record modify count
+    - 16B: Host epoch time
+    - 16B: Host uptime
+    - 8B: Host name length
+    - ?B: Host name
+*/
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct DRRuntimeSignature {
+    rt_signature_fixed: DRRuntimeSignatureFixed,
+    host_name: Box<[u8]>,
+}
+
+impl DRRuntimeSignature {
+    pub fn new(fixed: DRRuntimeSignatureFixed, host_name: Box<[u8]>) -> Self {
+        Self {
+            rt_signature_fixed: fixed,
+            host_name,
+        }
+    }
+    pub const fn rt_signature_fixed(&self) -> &DRRuntimeSignatureFixed {
+        &self.rt_signature_fixed
+    }
+    pub fn host_name(&self) -> &[u8] {
+        self.host_name.as_ref()
+    }
+    pub fn into_encoded(self) -> DRRuntimeSignatureRaw {
+        let len = self.host_name.len();
+        DRRuntimeSignatureRaw::new_with_sections(
+            self.host_name,
+            self.rt_signature_fixed.encoded(len),
+        )
+    }
+    pub fn encoded(&self) -> DRRuntimeSignatureRaw {
+        self.clone().into_encoded()
+    }
+}
+
+pub struct DRRuntimeSignatureRaw {
+    rt_signature: DRRuntimeSignatureFixedRaw,
+    pub(super) host_name: Box<[u8]>,
+}
+
+impl DRRuntimeSignatureRaw {
+    pub fn new(host_name: Box<[u8]>, modify_count: u64) -> Self {
+        Self {
+            rt_signature: DRRuntimeSignatureFixedRaw::new(modify_count, host_name.len()),
+            host_name,
+        }
+    }
+    pub fn new_with_sections(host_name: Box<[u8]>, fixed: DRRuntimeSignatureFixedRaw) -> Self {
+        Self {
+            rt_signature: fixed,
+            host_name,
+        }
+    }
+    pub fn decode(
+        data: [u8; sizeof!(DRRuntimeSignatureFixedRaw)],
+    ) -> Option<(usize, DRRuntimeSignatureFixed)> {
+        let s = ByteStack::new(data);
+        let modify_count = u64::from_le(s.read_qword(DRRuntimeSignatureFixedRaw::DRRS_OFFSET_P0));
+        let epoch_time = u128::from_le(s.read_xmmword(DRRuntimeSignatureFixedRaw::DRRS_OFFSET_P1));
+        let uptime = u128::from_le(s.read_xmmword(DRRuntimeSignatureFixedRaw::DRRS_OFFSET_P2));
+        let host_name_length =
+            u64::from_le(s.read_qword(DRRuntimeSignatureFixedRaw::DRRS_OFFSET_P3));
+        if epoch_time > crate::util::os::get_epoch_time() || host_name_length > usize::MAX as u64 {
+            // damn, this file is from the future; I WISH EVERYONE HAD NTP SYNC GRRRR
+            // or, we have a bad host name. like, what?
+            return None;
+        }
+        Some((
+            host_name_length as _,
+            DRRuntimeSignatureFixed::new(modify_count, epoch_time, uptime),
+        ))
+    }
+    pub const fn runtime_signature(&self) -> &DRRuntimeSignatureFixedRaw {
+        &self.rt_signature
+    }
+    pub fn name(&self) -> &[u8] {
+        &self.host_name
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct DRRuntimeSignatureFixed {
+    modify_count: u64,
+    epoch_time: u128,
+    uptime: u128,
+}
+
+impl DRRuntimeSignatureFixed {
+    pub const fn new(modify_count: u64, epoch_time: u128, uptime: u128) -> Self {
+        Self {
+            modify_count,
+            epoch_time,
+            uptime,
+        }
+    }
+    pub const fn modify_count(&self) -> u64 {
+        self.modify_count
+    }
+    pub const fn epoch_time(&self) -> u128 {
+        self.epoch_time
+    }
+    pub const fn uptime(&self) -> u128 {
+        self.uptime
+    }
+    pub fn encoded(&self, host_name_length: usize) -> DRRuntimeSignatureFixedRaw {
+        DRRuntimeSignatureFixedRaw::new_full(
+            self.modify_count(),
+            self.epoch_time(),
+            self.uptime(),
+            host_name_length,
+        )
+    }
+}
+
+pub struct DRRuntimeSignatureFixedRaw {
+    data: ByteStack<48>,
+}
+
+impl DRRuntimeSignatureFixedRaw {
+    const DRRS_OFFSET_P0: usize = 0;
+    const DRRS_OFFSET_P1: usize = sizeof!(u64);
+    const DRRS_OFFSET_P2: usize = Self::DRRS_OFFSET_P1 + sizeof!(u128);
+    const DRRS_OFFSET_P3: usize = Self::DRRS_OFFSET_P2 + sizeof!(u128);
+    const _ENSURE: () = assert!(Self::DRRS_OFFSET_P3 == sizeof!(Self) - 8);
+    pub fn new_full(
+        modify_count: u64,
+        epoch_time: u128,
+        uptime: u128,
+        host_name_length: usize,
+    ) -> Self {
+        let _ = Self::_ENSURE;
+        let mut data = [0u8; sizeof!(Self)];
+        data[0..8].copy_from_slice(&modify_count.to_le_bytes());
+        data[8..24].copy_from_slice(&epoch_time.to_le_bytes());
+        data[24..40].copy_from_slice(&uptime.to_le_bytes());
+        data[40..48].copy_from_slice(&(host_name_length as u64).to_le_bytes());
+        Self {
+            data: ByteStack::new(data),
+        }
+    }
+    pub fn new(modify_count: u64, host_name_length: usize) -> Self {
+        Self::new_full(
+            modify_count,
+            crate::util::os::get_epoch_time(),
+            crate::util::os::get_uptime(),
+            host_name_length,
+        )
+    }
+    pub const fn read_p0_modify_count(&self) -> u64 {
+        self.data.read_qword(Self::DRRS_OFFSET_P0)
+    }
+    pub const fn read_p1_epoch_time(&self) -> u128 {
+        self.data.read_xmmword(Self::DRRS_OFFSET_P1)
+    }
+    pub const fn read_p2_uptime(&self) -> u128 {
+        self.data.read_xmmword(Self::DRRS_OFFSET_P2)
+    }
+    pub const fn read_p3_host_name_length(&self) -> u64 {
+        self.data.read_qword(Self::DRRS_OFFSET_P3)
+    }
+}
diff --git a/server/src/engine/storage/v1/header_impl.rs b/server/src/engine/storage/v1/header_impl/gr.rs
similarity index 64%
rename from server/src/engine/storage/v1/header_impl.rs
rename to server/src/engine/storage/v1/header_impl/gr.rs
index fe68c8c8..42fdd4df 100644
--- a/server/src/engine/storage/v1/header_impl.rs
+++ b/server/src/engine/storage/v1/header_impl/gr.rs
@@ -1,5 +1,5 @@
 /*
- * Created on Mon May 15 2023
+ * Created on Thu May 25 2023
  *
  * This file is a part of Skytable
  * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@@ -24,85 +24,16 @@
  *
 */
 
-/*
- * SDSS Header layout:
- *
- * +--------------------------------------------------------------+
- * |                                                              |
- * |                        STATIC RECORD                         |
- * |                            128B                              |
- * +--------------------------------------------------------------+
- * +--------------------------------------------------------------+
- * |                                                              |
- * |                                                              |
- * |                       DYNAMIC RECORD                         |
- * |                         (256+56+?)B                          |
- * |        +--------------------------------------------+        |
- * |        |                                            |        |
- * |        |              METADATA RECORD               |        |
- * |        |                   256B                     |        |
- * |        +--------------------------------------------+        |
- * |        +--------------------------------------------+        |
- * |        |                                            |        |
- * |        |             GENESIS HOST RECORD            |        |
- * |        |                  >56B                      |        |
- * |        +--------------------------------------------+        |
- * |                                                              |
- * +--------------------------------------------------------------+
- * +--------------------------------------------------------------+
- * |                     RUNTIME HOST RECORD                      |
- * |                           >56B                               |
- * +--------------------------------------------------------------+
- * Note: The entire part of the header is little endian encoded
-*/
-
 use crate::engine::{
     mem::ByteStack,
     storage::{
-        header::{StaticRecordUV, StaticRecordUVRaw},
+        v1::header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
         versions::{self, DriverVersion, ServerVersion},
     },
 };
 
-pub struct StaticRecord {
-    sr: StaticRecordUV,
-}
-
-impl StaticRecord {
-    pub const fn new(sr: StaticRecordUV) -> Self {
-        Self { sr }
-    }
-    pub const fn encode(&self) -> StaticRecordRaw {
-        StaticRecordRaw {
-            base: self.sr.encode(),
-        }
-    }
-    pub const fn sr(&self) -> &StaticRecordUV {
-        &self.sr
-    }
-}
-
-/// Static record
-pub struct StaticRecordRaw {
-    base: StaticRecordUVRaw,
-}
-
-impl StaticRecordRaw {
-    pub const fn new() -> Self {
-        Self {
-            base: StaticRecordUVRaw::create(versions::v1::V1_HEADER_VERSION),
-        }
-    }
-    pub const fn empty_buffer() -> [u8; sizeof!(Self)] {
-        [0u8; sizeof!(Self)]
-    }
-    pub fn decode_from_bytes(buf: [u8; sizeof!(Self)]) -> Option<StaticRecord> {
-        StaticRecordUVRaw::decode_from_bytes(buf).map(StaticRecord::new)
-    }
-}
-
 /*
-    Dynamic record (1/2)
+    Genesis record (1/2)
     ---
     Metadata record (8B x 3 + (4B x 2)):
     +----------+----------+----------+---------+
@@ -112,59 +43,7 @@ impl StaticRecordRaw {
     0, 63
 */
 
-/// The file scope
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
-pub enum FileScope {
-    TransactionLog = 0,
-    TransactionLogCompacted = 1,
-}
-
-impl FileScope {
-    pub const fn try_new(id: u64) -> Option<Self> {
-        Some(match id {
-            0 => Self::TransactionLog,
-            1 => Self::TransactionLogCompacted,
-            _ => return None,
-        })
-    }
-    pub const fn new(id: u64) -> Self {
-        match Self::try_new(id) {
-            Some(v) => v,
-            None => panic!("unknown filescope"),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
-#[repr(u8)]
-pub enum FileSpecifier {
-    GNSTxnLog = 0,
-}
-
-impl FileSpecifier {
-    pub const fn try_new(v: u32) -> Option<Self> {
-        Some(match v {
-            0 => Self::GNSTxnLog,
-            _ => return None,
-        })
-    }
-    pub const fn new(v: u32) -> Self {
-        match Self::try_new(v) {
-            Some(v) => v,
-            _ => panic!("unknown filespecifier"),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-pub struct FileSpecifierVersion(u32);
-impl FileSpecifierVersion {
-    pub const fn __new(v: u32) -> Self {
-        Self(v)
-    }
-}
-
+#[derive(Debug, PartialEq)]
 pub struct MetadataRecord {
     server_version: ServerVersion,
     driver_version: DriverVersion,
@@ -216,7 +95,7 @@ impl MetadataRecord {
 }
 
 pub struct MetadataRecordRaw {
-    data: ByteStack<32>,
+    pub(super) data: ByteStack<32>,
 }
 
 impl MetadataRecordRaw {
@@ -320,7 +199,7 @@ impl MetadataRecordRaw {
 }
 
 /*
-    Dynamic Record (2/2)
+    Genesis Record (2/2)
     ---
     Host record (?B; > 56B):
     - 16B: Host epoch time in nanoseconds
@@ -333,30 +212,7 @@ impl MetadataRecordRaw {
     - ??B: Host name
 */
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
-#[repr(u8)]
-pub enum HostRunMode {
-    Dev = 0,
-    Prod = 1,
-}
-
-impl HostRunMode {
-    pub const fn try_new_with_val(v: u32) -> Option<Self> {
-        Some(match v {
-            0 => Self::Dev,
-            1 => Self::Prod,
-            _ => return None,
-        })
-    }
-    pub const fn new_with_val(v: u32) -> Self {
-        match Self::try_new_with_val(v) {
-            Some(v) => v,
-            None => panic!("unknown hostrunmode"),
-        }
-    }
-}
-
-type HRConstSectionRaw = [u8; 56];
+pub type HRConstSectionRaw = [u8; 56];
 
 #[derive(Debug, PartialEq, Clone)]
 pub struct HostRecord {
@@ -429,8 +285,8 @@ impl HRConstSection {
 }
 
 pub struct HostRecordRaw {
-    data: ByteStack<{ sizeof!(HRConstSectionRaw) }>,
-    host_name: Box<[u8]>,
+    pub(super) data: ByteStack<{ sizeof!(HRConstSectionRaw) }>,
+    pub(super) host_name: Box<[u8]>,
 }
 
 impl HostRecordRaw {
@@ -499,13 +355,18 @@ impl HostRecordRaw {
         let _ = Self::_ENSURE;
         let p4_host_name_length = p5_host_name.len();
         let mut host_record_fl = [0u8; 56];
-        host_record_fl[0..16].copy_from_slice(&p0_host_epoch_time.to_le_bytes());
-        host_record_fl[16..32].copy_from_slice(&p1_host_uptime.to_le_bytes());
-        host_record_fl[32..36].copy_from_slice(&p2a_host_setting_version_id.to_le_bytes());
-        host_record_fl[36..40]
+        host_record_fl[Self::HR_OFFSET_P0..Self::HR_OFFSET_P1]
+            .copy_from_slice(&p0_host_epoch_time.to_le_bytes());
+        host_record_fl[Self::HR_OFFSET_P1..Self::HR_OFFSET_P2A]
+            .copy_from_slice(&p1_host_uptime.to_le_bytes());
+        host_record_fl[Self::HR_OFFSET_P2A..Self::HR_OFFSET_P2B]
+            .copy_from_slice(&p2a_host_setting_version_id.to_le_bytes());
+        host_record_fl[Self::HR_OFFSET_P2B..Self::HR_OFFSET_P3]
             .copy_from_slice(&(p2b_host_run_mode.value_u8() as u32).to_le_bytes());
-        host_record_fl[40..48].copy_from_slice(&p3_host_startup_counter.to_le_bytes());
-        host_record_fl[48..56].copy_from_slice(&(p4_host_name_length as u64).to_le_bytes());
+        host_record_fl[Self::HR_OFFSET_P3..Self::HR_OFFSET_P4]
+            .copy_from_slice(&p3_host_startup_counter.to_le_bytes());
+        host_record_fl[Self::HR_OFFSET_P4..]
+            .copy_from_slice(&(p4_host_name_length as u64).to_le_bytes());
         Self {
             data: ByteStack::new(host_record_fl),
             host_name: p5_host_name,
@@ -554,103 +415,6 @@ impl HostRecordRaw {
     }
 }
 
-pub struct SDSSHeader {
-    sr: StaticRecord,
-    mdr: MetadataRecord,
-    hr: HostRecord,
-}
-
-impl SDSSHeader {
-    pub const fn new(sr: StaticRecord, mdr: MetadataRecord, hr: HostRecord) -> Self {
-        Self { sr, mdr, hr }
-    }
-    pub const fn sr(&self) -> &StaticRecord {
-        &self.sr
-    }
-    pub const fn mdr(&self) -> &MetadataRecord {
-        &self.mdr
-    }
-    pub const fn hr(&self) -> &HostRecord {
-        &self.hr
-    }
-    pub fn encode(&self) -> SDSSHeaderRaw {
-        SDSSHeaderRaw::new_full(self.sr.encode(), self.mdr.encode(), self.hr.encode())
-    }
-}
-
-pub struct SDSSHeaderRaw {
-    sr: StaticRecordRaw,
-    dr_0_mdr: MetadataRecordRaw,
-    dr_1_hr: HostRecordRaw,
-}
-
-impl SDSSHeaderRaw {
-    pub fn new_full(sr: StaticRecordRaw, mdr: MetadataRecordRaw, hr: HostRecordRaw) -> Self {
-        Self {
-            sr,
-            dr_0_mdr: mdr,
-            dr_1_hr: hr,
-        }
-    }
-    pub fn new(
-        sr: StaticRecordRaw,
-        dr_0_mdr: MetadataRecordRaw,
-        dr_1_hr_const_section: HRConstSectionRaw,
-        dr_1_hr_host_name: Box<[u8]>,
-    ) -> Self {
-        Self {
-            sr,
-            dr_0_mdr,
-            dr_1_hr: HostRecordRaw {
-                data: ByteStack::new(dr_1_hr_const_section),
-                host_name: dr_1_hr_host_name,
-            },
-        }
-    }
-    pub fn init(
-        mdr_file_scope: FileScope,
-        mdr_file_specifier: FileSpecifier,
-        mdr_file_specifier_id: FileSpecifierVersion,
-        hr_host_setting_id: u32,
-        hr_host_run_mode: HostRunMode,
-        hr_host_startup_counter: u64,
-        hr_host_name: Box<[u8]>,
-    ) -> Self {
-        Self {
-            sr: StaticRecordRaw::new(),
-            dr_0_mdr: MetadataRecordRaw::new(
-                mdr_file_scope,
-                mdr_file_specifier,
-                mdr_file_specifier_id,
-            ),
-            dr_1_hr: HostRecordRaw::new_auto(
-                hr_host_setting_id,
-                hr_host_run_mode,
-                hr_host_startup_counter,
-                hr_host_name,
-            ),
-        }
-    }
-    pub fn get0_sr(&self) -> &[u8] {
-        self.sr.base.get_ref()
-    }
-    pub fn get1_dr_0_mdr(&self) -> &[u8] {
-        self.dr_0_mdr.data.slice()
-    }
-    pub fn get1_dr_1_hr_0(&self) -> &[u8] {
-        self.dr_1_hr.data.slice()
-    }
-    pub fn get1_dr_1_hr_1(&self) -> &[u8] {
-        self.dr_1_hr.host_name.as_ref()
-    }
-    pub fn calculate_header_size(&self) -> usize {
-        Self::calculate_fixed_header_size() + self.dr_1_hr.host_name.len()
-    }
-    pub const fn calculate_fixed_header_size() -> usize {
-        sizeof!(StaticRecordRaw) + sizeof!(MetadataRecordRaw) + sizeof!(HRConstSectionRaw)
-    }
-}
-
 #[test]
 fn test_metadata_record_encode_decode() {
     let md = MetadataRecordRaw::new(
diff --git a/server/src/engine/storage/v1/header_impl/mod.rs b/server/src/engine/storage/v1/header_impl/mod.rs
new file mode 100644
index 00000000..2ddf4bba
--- /dev/null
+++ b/server/src/engine/storage/v1/header_impl/mod.rs
@@ -0,0 +1,266 @@
+/*
+ * Created on Mon May 15 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/*
+ * SDSS Header layout:
+ *
+ * +--------------------------------------------------------------+
+ * |                                                              |
+ * |                        STATIC RECORD                         |
+ * |                            128B                              |
+ * +--------------------------------------------------------------+
+ * +--------------------------------------------------------------+
+ * |                                                              |
+ * |                                                              |
+ * |                       GENESIS RECORD                         |
+ * |                         (256+56+?)B                          |
+ * |        +--------------------------------------------+        |
+ * |        |                                            |        |
+ * |        |              METADATA RECORD               |        |
+ * |        |                   256B                     |        |
+ * |        +--------------------------------------------+        |
+ * |        +--------------------------------------------+        |
+ * |        |                                            |        |
+ * |        |               HOST RECORD                  |        |
+ * |        |                  >56B                      |        |
+ * |        +--------------------------------------------+        |
+ * |                                                              |
+ * +--------------------------------------------------------------+
+ * +--------------------------------------------------------------+
+ * |                       DYNAMIC RECORD                         |
+ * |                           >56B                               |
+ * +--------------------------------------------------------------+
+ * Note: The entire part of the header is little endian encoded
+*/
+
+// (1) sr
+mod sr;
+// (2) gr
+mod gr;
+// (3) dr
+mod dr;
+
+use crate::engine::mem::ByteStack;
+
+/// The file scope
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+pub enum FileScope {
+    TransactionLog = 0,
+    TransactionLogCompacted = 1,
+}
+
+impl FileScope {
+    pub const fn try_new(id: u64) -> Option<Self> {
+        Some(match id {
+            0 => Self::TransactionLog,
+            1 => Self::TransactionLogCompacted,
+            _ => return None,
+        })
+    }
+    pub const fn new(id: u64) -> Self {
+        match Self::try_new(id) {
+            Some(v) => v,
+            None => panic!("unknown filescope"),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+#[repr(u8)]
+pub enum FileSpecifier {
+    GNSTxnLog = 0,
+}
+
+impl FileSpecifier {
+    pub const fn try_new(v: u32) -> Option<Self> {
+        Some(match v {
+            0 => Self::GNSTxnLog,
+            _ => return None,
+        })
+    }
+    pub const fn new(v: u32) -> Self {
+        match Self::try_new(v) {
+            Some(v) => v,
+            _ => panic!("unknown filespecifier"),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct FileSpecifierVersion(u32);
+impl FileSpecifierVersion {
+    pub const fn __new(v: u32) -> Self {
+        Self(v)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+#[repr(u8)]
+pub enum HostRunMode {
+    Dev = 0,
+    Prod = 1,
+}
+
+impl HostRunMode {
+    pub const fn try_new_with_val(v: u32) -> Option<Self> {
+        Some(match v {
+            0 => Self::Dev,
+            1 => Self::Prod,
+            _ => return None,
+        })
+    }
+    pub const fn new_with_val(v: u32) -> Self {
+        match Self::try_new_with_val(v) {
+            Some(v) => v,
+            None => panic!("unknown hostrunmode"),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub struct SDSSHeader {
+    // static record
+    sr: sr::StaticRecord,
+    // genesis record
+    gr_mdr: gr::MetadataRecord,
+    gr_hr: gr::HostRecord,
+    // dynamic record
+    dr_hs: dr::DRHostSignature,
+    dr_rs: dr::DRRuntimeSignature,
+}
+
+impl SDSSHeader {
+    pub const fn new(
+        sr: sr::StaticRecord,
+        gr_mdr: gr::MetadataRecord,
+        gr_hr: gr::HostRecord,
+        dr_hs: dr::DRHostSignature,
+        dr_rs: dr::DRRuntimeSignature,
+    ) -> Self {
+        Self {
+            sr,
+            gr_mdr,
+            gr_hr,
+            dr_hs,
+            dr_rs,
+        }
+    }
+    pub fn encode(&self) -> SDSSHeaderRaw {
+        SDSSHeaderRaw::new_full(
+            self.sr.encode(),
+            self.gr_mdr.encode(),
+            self.gr_hr.encode(),
+            self.dr_hs().encoded(),
+            self.dr_rs().encoded(),
+        )
+    }
+    pub fn sr(&self) -> &sr::StaticRecord {
+        &self.sr
+    }
+    pub fn gr_mdr(&self) -> &gr::MetadataRecord {
+        &self.gr_mdr
+    }
+    pub fn gr_hr(&self) -> &gr::HostRecord {
+        &self.gr_hr
+    }
+    pub fn dr_hs(&self) -> &dr::DRHostSignature {
+        &self.dr_hs
+    }
+    pub fn dr_rs(&self) -> &dr::DRRuntimeSignature {
+        &self.dr_rs
+    }
+}
+
+pub struct SDSSHeaderRaw {
+    sr: sr::StaticRecordRaw,
+    gr_0_mdr: gr::MetadataRecordRaw,
+    gr_1_hr: gr::HostRecordRaw,
+    dr_0_hs: dr::DRHostSignatureRaw,
+    dr_1_rs: dr::DRRuntimeSignatureRaw,
+}
+
+impl SDSSHeaderRaw {
+    pub fn new_full(
+        sr: sr::StaticRecordRaw,
+        gr_mdr: gr::MetadataRecordRaw,
+        gr_hr: gr::HostRecordRaw,
+        dr_hs: dr::DRHostSignatureRaw,
+        dr_rs: dr::DRRuntimeSignatureRaw,
+    ) -> Self {
+        Self {
+            sr,
+            gr_0_mdr: gr_mdr,
+            gr_1_hr: gr_hr,
+            dr_0_hs: dr_hs,
+            dr_1_rs: dr_rs,
+        }
+    }
+    pub fn new(
+        sr: sr::StaticRecordRaw,
+        gr_0_mdr: gr::MetadataRecordRaw,
+        gr_1_hr_const_section: gr::HRConstSectionRaw,
+        gr_1_hr_host_name: Box<[u8]>,
+        dr_hs: dr::DRHostSignatureRaw,
+        dr_rs_const: dr::DRRuntimeSignatureFixedRaw,
+        dr_rs_host_name: Box<[u8]>,
+    ) -> Self {
+        Self {
+            sr,
+            gr_0_mdr,
+            gr_1_hr: gr::HostRecordRaw {
+                data: ByteStack::new(gr_1_hr_const_section),
+                host_name: gr_1_hr_host_name,
+            },
+            dr_0_hs: dr_hs,
+            dr_1_rs: dr::DRRuntimeSignatureRaw::new_with_sections(dr_rs_host_name, dr_rs_const),
+        }
+    }
+    pub fn get0_sr(&self) -> &[u8] {
+        self.sr.base.get_ref()
+    }
+    pub fn get1_dr_0_mdr(&self) -> &[u8] {
+        self.gr_0_mdr.data.slice()
+    }
+    pub fn get1_dr_1_hr_0(&self) -> &[u8] {
+        self.gr_1_hr.data.slice()
+    }
+    pub fn get1_dr_1_hr_1(&self) -> &[u8] {
+        self.gr_1_hr.host_name.as_ref()
+    }
+    pub fn calculate_header_size(&self) -> usize {
+        Self::calculate_fixed_header_size()
+            + self.gr_1_hr.host_name.len()
+            + self.dr_1_rs.host_name.len()
+    }
+    pub const fn calculate_fixed_header_size() -> usize {
+        sizeof!(sr::StaticRecordRaw)
+            + sizeof!(gr::MetadataRecordRaw)
+            + sizeof!(gr::HRConstSectionRaw)
+            + sizeof!(dr::DRHostSignatureRaw)
+            + sizeof!(dr::DRRuntimeSignatureFixedRaw)
+    }
+}
diff --git a/server/src/engine/storage/v1/header_impl/sr.rs b/server/src/engine/storage/v1/header_impl/sr.rs
new file mode 100644
index 00000000..a98f2af9
--- /dev/null
+++ b/server/src/engine/storage/v1/header_impl/sr.rs
@@ -0,0 +1,68 @@
+/*
+ * Created on Thu May 25 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::storage::{
+    header::{StaticRecordUV, StaticRecordUVRaw},
+    versions,
+};
+
+#[derive(Debug, PartialEq)]
+pub struct StaticRecord {
+    sr: StaticRecordUV,
+}
+
+impl StaticRecord {
+    pub const fn new(sr: StaticRecordUV) -> Self {
+        Self { sr }
+    }
+    pub const fn encode(&self) -> StaticRecordRaw {
+        StaticRecordRaw {
+            base: self.sr.encode(),
+        }
+    }
+    pub const fn sr(&self) -> &StaticRecordUV {
+        &self.sr
+    }
+}
+
+/// Static record
+pub struct StaticRecordRaw {
+    pub(super) base: StaticRecordUVRaw,
+}
+
+impl StaticRecordRaw {
+    pub const fn new() -> Self {
+        Self {
+            base: StaticRecordUVRaw::create(versions::v1::V1_HEADER_VERSION),
+        }
+    }
+    pub const fn empty_buffer() -> [u8; sizeof!(Self)] {
+        [0u8; sizeof!(Self)]
+    }
+    pub fn decode_from_bytes(buf: [u8; sizeof!(Self)]) -> Option<StaticRecord> {
+        StaticRecordUVRaw::decode_from_bytes(buf).map(StaticRecord::new)
+    }
+}
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index bc0d88ac..1dc078d2 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -23,191 +23,3 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.
  *
 */
-
-use crate::engine::storage::versions;
-
-use {
-    super::header_impl::SDSSHeaderRaw,
-    crate::{
-        engine::storage::v1::header_impl::{
-            HostRecord, HostRecordRaw, MetadataRecordRaw, SDSSHeader, StaticRecordRaw,
-        },
-        util::{ByteRepr, NumericRepr},
-        IoResult,
-    },
-    std::{
-        fs::{self, File, OpenOptions},
-        io::{Error as IoError, ErrorKind, Read, Seek, SeekFrom, Write},
-    },
-};
-
-#[derive(Debug)]
-pub enum SDSSError {
-    SRVersionMismatch,
-    IoError(IoError),
-    CorruptedHeaderSR,
-    CorruptedHeaderMDR,
-    CorruptedHeaderHR,
-}
-
-impl From<IoError> for SDSSError {
-    fn from(e: IoError) -> Self {
-        Self::IoError(e)
-    }
-}
-
-pub type SDSSResult<T> = Result<T, SDSSError>;
-
-/*
-    Writer interface
-*/
-
-pub trait RawWriterInterface: Sized {
-    fn fopen_truncated(fname: &str) -> IoResult<Self>;
-    fn fopen_create(fname: &str) -> IoResult<Self>;
-    fn fwrite_all(&mut self, bytes: &[u8]) -> IoResult<()>;
-    fn fsync_all(&mut self) -> IoResult<()>;
-}
-
-impl RawWriterInterface for File {
-    fn fopen_truncated(fname: &str) -> IoResult<Self> {
-        OpenOptions::new()
-            .write(true)
-            .truncate(true)
-            .create(false)
-            .open(fname)
-    }
-
-    fn fopen_create(fname: &str) -> IoResult<Self> {
-        File::create(fname)
-    }
-
-    fn fwrite_all(&mut self, bytes: &[u8]) -> IoResult<()> {
-        Write::write_all(self, bytes)
-    }
-
-    fn fsync_all(&mut self) -> IoResult<()> {
-        // FIXME(@ohsayan): too slow? maybe fdatasync only?
-        File::sync_all(self)
-    }
-}
-
-/*
-    Writer
-*/
-
-pub struct SDSSWriter<W> {
-    writer: W,
-}
-
-impl<W: RawWriterInterface> SDSSWriter<W> {
-    pub fn open_create_with_header(file: &str, header: SDSSHeaderRaw) -> IoResult<Self> {
-        let mut w = W::fopen_create(file)?;
-        w.fwrite_all(header.get0_sr())?;
-        w.fwrite_all(header.get1_dr_0_mdr())?;
-        w.fwrite_all(header.get1_dr_1_hr_0())?;
-        w.fwrite_all(header.get1_dr_1_hr_1())?;
-        w.fsync_all()?;
-        Ok(Self { writer: w })
-    }
-    pub fn fsync_write<D: ?Sized + ByteRepr>(&mut self, data: &D) -> IoResult<()> {
-        self.writer.fwrite_all(data.repr())?;
-        self.writer.fsync_all()
-    }
-    pub fn newrite_numeric(&mut self, num: impl NumericRepr) -> IoResult<()> {
-        self.fsync_write(num.repr())
-    }
-    pub fn lewrite_numeric(&mut self, num: impl NumericRepr) -> IoResult<()> {
-        self.fsync_write(num.le().repr())
-    }
-    pub fn bewrite_numeric(&mut self, num: impl NumericRepr) -> IoResult<()> {
-        self.fsync_write(num.be().repr())
-    }
-}
-
-/*
-    Read interface
-*/
-
-pub trait RawReaderInterface: Sized {
-    fn fopen(fname: &str) -> IoResult<Self>;
-    fn fread_exact_seek(&mut self, buf: &mut [u8]) -> IoResult<()>;
-    fn fread_to_end(&mut self, buf: &mut Vec<u8>) -> IoResult<()>;
-}
-
-impl RawReaderInterface for File {
-    fn fopen(fname: &str) -> IoResult<Self> {
-        File::open(fname)
-    }
-    fn fread_exact_seek(&mut self, buf: &mut [u8]) -> IoResult<()> {
-        self.read_exact(buf)?;
-        let _ = self.seek(SeekFrom::Start(buf.len() as _))?;
-        Ok(())
-    }
-    fn fread_to_end(&mut self, buf: &mut Vec<u8>) -> IoResult<()> {
-        match self.read_to_end(buf) {
-            Ok(_) => Ok(()),
-            Err(e) => Err(e),
-        }
-    }
-}
-
-pub struct FileBuffered {
-    cursor: usize,
-    base: Vec<u8>,
-}
-
-impl RawReaderInterface for FileBuffered {
-    fn fopen(name: &str) -> IoResult<Self> {
-        Ok(Self {
-            base: fs::read(name)?,
-            cursor: 0,
-        })
-    }
-    fn fread_exact_seek(&mut self, buf: &mut [u8]) -> IoResult<()> {
-        let l = self.base[self.cursor..].len();
-        if l >= buf.len() {
-            self.cursor += buf.len();
-            Ok(buf.copy_from_slice(&self.base[self.cursor..]))
-        } else {
-            Err(ErrorKind::UnexpectedEof.into())
-        }
-    }
-    fn fread_to_end(&mut self, buf: &mut Vec<u8>) -> IoResult<()> {
-        buf.extend_from_slice(&self.base[self.cursor..]);
-        Ok(())
-    }
-}
-
-pub struct SDSSReader<R> {
-    reader: R,
-    header: SDSSHeader,
-}
-
-impl<R: RawReaderInterface> SDSSReader<R> {
-    pub fn open(f: &str) -> SDSSResult<Self> {
-        let mut r = R::fopen(f)?;
-        let mut sr = StaticRecordRaw::empty_buffer();
-        let mut mdr = MetadataRecordRaw::empty_buffer();
-        let mut hr_0_const = HostRecordRaw::empty_buffer_const_section();
-        r.fread_exact_seek(&mut sr)?;
-        r.fread_exact_seek(&mut mdr)?;
-        r.fread_exact_seek(&mut hr_0_const)?;
-        let sr = StaticRecordRaw::decode_from_bytes(sr).ok_or(SDSSError::CorruptedHeaderSR)?;
-        let mdr = MetadataRecordRaw::decode_from_bytes(mdr).ok_or(SDSSError::CorruptedHeaderMDR)?;
-        let (hr_const, hostname_len) = HostRecordRaw::decode_from_bytes_const_sec(hr_0_const)
-            .ok_or(SDSSError::CorruptedHeaderHR)?;
-        let mut host_name = vec![0u8; hostname_len].into_boxed_slice();
-        r.fread_exact_seek(&mut host_name)?;
-        if (sr.sr().header_version() != versions::v1::V1_HEADER_VERSION)
-            || (mdr.driver_version() != versions::v1::V1_DRIVER_VERSION)
-            || (mdr.server_version() != versions::v1::V1_SERVER_VERSION)
-        {
-            return Err(SDSSError::SRVersionMismatch);
-        }
-        Ok(Self {
-            reader: r,
-            header: SDSSHeader::new(sr, mdr, HostRecord::new(hr_const, host_name)),
-        })
-    }
-}

From 45c93eac3c2bf77227eb4ed7b19f654350f4763b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 26 May 2023 08:59:07 +0000
Subject: [PATCH 198/310] Redefine SDSS Header records to have fixed layout

---
 .../src/engine/storage/v1/header_impl/dr.rs   | 281 ++++++++------
 .../src/engine/storage/v1/header_impl/gr.rs   | 354 ++++++++----------
 .../src/engine/storage/v1/header_impl/mod.rs  |  69 ++--
 .../src/engine/storage/v1/header_impl/sr.rs   |   2 +-
 server/src/util/mod.rs                        |  14 +
 5 files changed, 371 insertions(+), 349 deletions(-)

diff --git a/server/src/engine/storage/v1/header_impl/dr.rs b/server/src/engine/storage/v1/header_impl/dr.rs
index 1f530ec9..008287c8 100644
--- a/server/src/engine/storage/v1/header_impl/dr.rs
+++ b/server/src/engine/storage/v1/header_impl/dr.rs
@@ -24,13 +24,16 @@
  *
 */
 
-use crate::engine::{
-    mem::ByteStack,
-    storage::{
-        header::{HostArch, HostEndian, HostOS, HostPointerWidth},
-        v1::header_impl::FileSpecifierVersion,
-        versions::{DriverVersion, ServerVersion},
+use crate::{
+    engine::{
+        mem::ByteStack,
+        storage::{
+            header::{HostArch, HostEndian, HostOS, HostPointerWidth},
+            v1::header_impl::FileSpecifierVersion,
+            versions::{DriverVersion, ServerVersion},
+        },
     },
+    util,
 };
 
 /*
@@ -56,6 +59,39 @@ pub struct DRHostSignature {
     os: HostOS,
 }
 
+impl DRHostSignature {
+    /// Decode the [`DRHostSignature`] from the given bytes
+    ///
+    /// **☢ WARNING ☢: This only decodes; it doesn't validate expected values!**
+    pub fn decode(bytes: [u8; sizeof!(DRHostSignatureRaw)]) -> Option<Self> {
+        let ns = ByteStack::new(bytes);
+        let server_version = ServerVersion::__new(u64::from_le(
+            ns.read_qword(DRHostSignatureRaw::DRHS_OFFSET_P0),
+        ));
+        let driver_version = DriverVersion::__new(u64::from_le(
+            ns.read_qword(DRHostSignatureRaw::DRHS_OFFSET_P1),
+        ));
+        let file_specifier_id = FileSpecifierVersion::__new(u32::from_le(
+            ns.read_dword(DRHostSignatureRaw::DRHS_OFFSET_P2),
+        ));
+        let endian =
+            HostEndian::try_new_with_val(ns.read_byte(DRHostSignatureRaw::DRHS_OFFSET_P3))?;
+        let ptr_width =
+            HostPointerWidth::try_new_with_val(ns.read_byte(DRHostSignatureRaw::DRHS_OFFSET_P4))?;
+        let arch = HostArch::try_new_with_val(ns.read_byte(DRHostSignatureRaw::DRHS_OFFSET_P5))?;
+        let os = HostOS::try_new_with_val(ns.read_byte(DRHostSignatureRaw::DRHS_OFFSET_P6))?;
+        Some(Self::new(
+            server_version,
+            driver_version,
+            file_specifier_id,
+            endian,
+            ptr_width,
+            arch,
+            os,
+        ))
+    }
+}
+
 impl DRHostSignature {
     pub const fn new(
         server_version: ServerVersion,
@@ -191,7 +227,7 @@ impl DRHostSignatureRaw {
     pub const fn read_p6_os(&self) -> HostOS {
         HostOS::new_with_val(self.data.read_byte(Self::DRHS_OFFSET_P6))
     }
-    pub const fn encoded(&self) -> DRHostSignature {
+    pub const fn decoded(&self) -> DRHostSignature {
         DRHostSignature::new(
             self.read_p0_server_version(),
             self.read_p1_driver_version(),
@@ -210,99 +246,60 @@ impl DRHostSignatureRaw {
     - 8B: Dynamic record modify count
     - 16B: Host epoch time
     - 16B: Host uptime
-    - 8B: Host name length
-    - ?B: Host name
+    - 1B: Host name length
+    - 255B: Host name (nulled)
+    = 296B
 */
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq)]
 pub struct DRRuntimeSignature {
-    rt_signature_fixed: DRRuntimeSignatureFixed,
-    host_name: Box<[u8]>,
+    modify_count: u64,
+    epoch_time: u128,
+    host_uptime: u128,
+    host_name_length: u8,
+    host_name_raw: [u8; 255],
 }
 
 impl DRRuntimeSignature {
-    pub fn new(fixed: DRRuntimeSignatureFixed, host_name: Box<[u8]>) -> Self {
-        Self {
-            rt_signature_fixed: fixed,
-            host_name,
-        }
-    }
-    pub const fn rt_signature_fixed(&self) -> &DRRuntimeSignatureFixed {
-        &self.rt_signature_fixed
-    }
-    pub fn host_name(&self) -> &[u8] {
-        self.host_name.as_ref()
-    }
-    pub fn into_encoded(self) -> DRRuntimeSignatureRaw {
-        let len = self.host_name.len();
-        DRRuntimeSignatureRaw::new_with_sections(
-            self.host_name,
-            self.rt_signature_fixed.encoded(len),
-        )
-    }
-    pub fn encoded(&self) -> DRRuntimeSignatureRaw {
-        self.clone().into_encoded()
-    }
-}
-
-pub struct DRRuntimeSignatureRaw {
-    rt_signature: DRRuntimeSignatureFixedRaw,
-    pub(super) host_name: Box<[u8]>,
-}
-
-impl DRRuntimeSignatureRaw {
-    pub fn new(host_name: Box<[u8]>, modify_count: u64) -> Self {
-        Self {
-            rt_signature: DRRuntimeSignatureFixedRaw::new(modify_count, host_name.len()),
-            host_name,
-        }
-    }
-    pub fn new_with_sections(host_name: Box<[u8]>, fixed: DRRuntimeSignatureFixedRaw) -> Self {
-        Self {
-            rt_signature: fixed,
-            host_name,
-        }
-    }
-    pub fn decode(
-        data: [u8; sizeof!(DRRuntimeSignatureFixedRaw)],
-    ) -> Option<(usize, DRRuntimeSignatureFixed)> {
-        let s = ByteStack::new(data);
-        let modify_count = u64::from_le(s.read_qword(DRRuntimeSignatureFixedRaw::DRRS_OFFSET_P0));
-        let epoch_time = u128::from_le(s.read_xmmword(DRRuntimeSignatureFixedRaw::DRRS_OFFSET_P1));
-        let uptime = u128::from_le(s.read_xmmword(DRRuntimeSignatureFixedRaw::DRRS_OFFSET_P2));
-        let host_name_length =
-            u64::from_le(s.read_qword(DRRuntimeSignatureFixedRaw::DRRS_OFFSET_P3));
-        if epoch_time > crate::util::os::get_epoch_time() || host_name_length > usize::MAX as u64 {
-            // damn, this file is from the future; I WISH EVERYONE HAD NTP SYNC GRRRR
-            // or, we have a bad host name. like, what?
-            return None;
+    pub fn decode(bytes: [u8; sizeof!(DRRuntimeSignatureRaw)]) -> Option<Self> {
+        let bytes = ByteStack::new(bytes);
+        // check
+        let modify_count = u64::from_le(bytes.read_qword(DRRuntimeSignatureRaw::DRRS_OFFSET_P0));
+        let epoch_time = u128::from_le(bytes.read_xmmword(DRRuntimeSignatureRaw::DRRS_OFFSET_P1));
+        let host_uptime = u128::from_le(bytes.read_xmmword(DRRuntimeSignatureRaw::DRRS_OFFSET_P2));
+        let host_name_length = bytes.read_byte(DRRuntimeSignatureRaw::DRRS_OFFSET_P3);
+        let host_name_raw =
+            util::copy_slice_to_array(&bytes.slice()[DRRuntimeSignatureRaw::DRRS_OFFSET_P4..]);
+        if cfg!(debug_assertions) {
+            assert_eq!(
+                255 - host_name_raw.iter().filter(|b| **b == 0u8).count(),
+                host_name_length as _
+            );
         }
-        Some((
-            host_name_length as _,
-            DRRuntimeSignatureFixed::new(modify_count, epoch_time, uptime),
-        ))
-    }
-    pub const fn runtime_signature(&self) -> &DRRuntimeSignatureFixedRaw {
-        &self.rt_signature
-    }
-    pub fn name(&self) -> &[u8] {
-        &self.host_name
+        Some(Self {
+            modify_count,
+            epoch_time,
+            host_uptime,
+            host_name_length,
+            host_name_raw,
+        })
     }
 }
 
-#[derive(Debug, PartialEq, Clone)]
-pub struct DRRuntimeSignatureFixed {
-    modify_count: u64,
-    epoch_time: u128,
-    uptime: u128,
-}
-
-impl DRRuntimeSignatureFixed {
-    pub const fn new(modify_count: u64, epoch_time: u128, uptime: u128) -> Self {
+impl DRRuntimeSignature {
+    pub const fn new(
+        modify_count: u64,
+        epoch_time: u128,
+        host_uptime: u128,
+        host_name_length: u8,
+        host_name_raw: [u8; 255],
+    ) -> Self {
         Self {
             modify_count,
             epoch_time,
-            uptime,
+            host_uptime,
+            host_name_length,
+            host_name_raw,
         }
     }
     pub const fn modify_count(&self) -> u64 {
@@ -311,51 +308,69 @@ impl DRRuntimeSignatureFixed {
     pub const fn epoch_time(&self) -> u128 {
         self.epoch_time
     }
-    pub const fn uptime(&self) -> u128 {
-        self.uptime
+    pub const fn host_uptime(&self) -> u128 {
+        self.host_uptime
+    }
+    pub const fn host_name_length(&self) -> u8 {
+        self.host_name_length
     }
-    pub fn encoded(&self, host_name_length: usize) -> DRRuntimeSignatureFixedRaw {
-        DRRuntimeSignatureFixedRaw::new_full(
+    pub const fn host_name_raw(&self) -> [u8; 255] {
+        self.host_name_raw
+    }
+    pub fn host_name(&self) -> &[u8] {
+        &self.host_name_raw[..self.host_name_length() as usize]
+    }
+    pub fn encoded(&self) -> DRRuntimeSignatureRaw {
+        DRRuntimeSignatureRaw::new(
             self.modify_count(),
             self.epoch_time(),
-            self.uptime(),
-            host_name_length,
+            self.host_uptime(),
+            self.host_name_length(),
+            self.host_name_raw(),
         )
     }
 }
 
-pub struct DRRuntimeSignatureFixedRaw {
-    data: ByteStack<48>,
+#[derive(Debug, PartialEq, Clone)]
+pub struct DRRuntimeSignatureRaw {
+    data: ByteStack<296>,
 }
 
-impl DRRuntimeSignatureFixedRaw {
+impl DRRuntimeSignatureRaw {
     const DRRS_OFFSET_P0: usize = 0;
     const DRRS_OFFSET_P1: usize = sizeof!(u64);
     const DRRS_OFFSET_P2: usize = Self::DRRS_OFFSET_P1 + sizeof!(u128);
     const DRRS_OFFSET_P3: usize = Self::DRRS_OFFSET_P2 + sizeof!(u128);
-    const _ENSURE: () = assert!(Self::DRRS_OFFSET_P3 == sizeof!(Self) - 8);
-    pub fn new_full(
+    const DRRS_OFFSET_P4: usize = Self::DRRS_OFFSET_P3 + 1;
+    const _ENSURE: () = assert!(Self::DRRS_OFFSET_P4 == sizeof!(Self) - 255);
+    pub fn new(
         modify_count: u64,
-        epoch_time: u128,
-        uptime: u128,
-        host_name_length: usize,
+        host_epoch_time: u128,
+        host_uptime: u128,
+        host_name_length: u8,
+        host_name: [u8; 255],
     ) -> Self {
         let _ = Self::_ENSURE;
-        let mut data = [0u8; sizeof!(Self)];
-        data[0..8].copy_from_slice(&modify_count.to_le_bytes());
-        data[8..24].copy_from_slice(&epoch_time.to_le_bytes());
-        data[24..40].copy_from_slice(&uptime.to_le_bytes());
-        data[40..48].copy_from_slice(&(host_name_length as u64).to_le_bytes());
+        let mut data = [0u8; 296];
+        data[Self::DRRS_OFFSET_P0..Self::DRRS_OFFSET_P1]
+            .copy_from_slice(&modify_count.to_le_bytes());
+        data[Self::DRRS_OFFSET_P1..Self::DRRS_OFFSET_P2]
+            .copy_from_slice(&host_epoch_time.to_le_bytes());
+        data[Self::DRRS_OFFSET_P2..Self::DRRS_OFFSET_P3]
+            .copy_from_slice(&host_uptime.to_le_bytes());
+        data[Self::DRRS_OFFSET_P3] = host_name_length;
+        data[Self::DRRS_OFFSET_P4..].copy_from_slice(&host_name);
         Self {
             data: ByteStack::new(data),
         }
     }
-    pub fn new(modify_count: u64, host_name_length: usize) -> Self {
-        Self::new_full(
-            modify_count,
-            crate::util::os::get_epoch_time(),
-            crate::util::os::get_uptime(),
-            host_name_length,
+    pub fn decoded(&self) -> DRRuntimeSignature {
+        DRRuntimeSignature::new(
+            self.read_p0_modify_count(),
+            self.read_p1_epoch_time(),
+            self.read_p2_uptime(),
+            self.read_p3_host_name_length() as _,
+            util::copy_slice_to_array(self.read_p4_host_name_raw_null()),
         )
     }
     pub const fn read_p0_modify_count(&self) -> u64 {
@@ -367,7 +382,45 @@ impl DRRuntimeSignatureFixedRaw {
     pub const fn read_p2_uptime(&self) -> u128 {
         self.data.read_xmmword(Self::DRRS_OFFSET_P2)
     }
-    pub const fn read_p3_host_name_length(&self) -> u64 {
-        self.data.read_qword(Self::DRRS_OFFSET_P3)
+    pub const fn read_p3_host_name_length(&self) -> usize {
+        self.data.read_byte(Self::DRRS_OFFSET_P3) as _
+    }
+    pub fn read_p4_host_name_raw_null(&self) -> &[u8] {
+        &self.data.slice()[Self::DRRS_OFFSET_P4..]
+    }
+    pub fn read_host_name(&self) -> &[u8] {
+        &self.data.slice()
+            [Self::DRRS_OFFSET_P4..Self::DRRS_OFFSET_P4 + self.read_p3_host_name_length()]
     }
 }
+
+#[test]
+fn test_dr_host_signature_encode_decode() {
+    const TARGET: DRHostSignature = DRHostSignature::new(
+        crate::engine::storage::versions::v1::V1_SERVER_VERSION,
+        crate::engine::storage::versions::v1::V1_DRIVER_VERSION,
+        FileSpecifierVersion::__new(u32::MAX - 3),
+        HostEndian::new(),
+        HostPointerWidth::new(),
+        HostArch::new(),
+        HostOS::new(),
+    );
+    let encoded = TARGET.encoded();
+    let decoded = encoded.decoded();
+    assert_eq!(decoded, TARGET);
+}
+
+#[test]
+fn test_dr_runtime_signature_encoded_decode() {
+    const TARGET: DRRuntimeSignature = DRRuntimeSignature::new(
+        u64::MAX - 3,
+        u128::MAX - u32::MAX as u128,
+        u128::MAX - u32::MAX as u128,
+        "skycloud".len() as _,
+        util::copy_str_to_array("skycloud"),
+    );
+    let encoded = TARGET.encoded();
+    let decoded = encoded.decoded();
+    assert_eq!(decoded, TARGET);
+    assert_eq!(decoded.host_name(), b"skycloud");
+}
diff --git a/server/src/engine/storage/v1/header_impl/gr.rs b/server/src/engine/storage/v1/header_impl/gr.rs
index 42fdd4df..8e0a9352 100644
--- a/server/src/engine/storage/v1/header_impl/gr.rs
+++ b/server/src/engine/storage/v1/header_impl/gr.rs
@@ -24,12 +24,15 @@
  *
 */
 
-use crate::engine::{
-    mem::ByteStack,
-    storage::{
-        v1::header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
-        versions::{self, DriverVersion, ServerVersion},
+use crate::{
+    engine::{
+        mem::ByteStack,
+        storage::{
+            v1::header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
+            versions::{self, DriverVersion, ServerVersion},
+        },
     },
+    util,
 };
 
 /*
@@ -44,7 +47,7 @@ use crate::engine::{
 */
 
 #[derive(Debug, PartialEq)]
-pub struct MetadataRecord {
+pub struct GRMetadataRecord {
     server_version: ServerVersion,
     driver_version: DriverVersion,
     file_scope: FileScope,
@@ -52,7 +55,7 @@ pub struct MetadataRecord {
     file_spec_id: FileSpecifierVersion,
 }
 
-impl MetadataRecord {
+impl GRMetadataRecord {
     pub const fn new(
         server_version: ServerVersion,
         driver_version: DriverVersion,
@@ -83,8 +86,8 @@ impl MetadataRecord {
     pub const fn file_spec_id(&self) -> FileSpecifierVersion {
         self.file_spec_id
     }
-    pub const fn encode(&self) -> MetadataRecordRaw {
-        MetadataRecordRaw::new_full(
+    pub const fn encoded(&self) -> GRMetadataRecordRaw {
+        GRMetadataRecordRaw::new_full(
             self.server_version(),
             self.driver_version(),
             self.file_scope(),
@@ -94,15 +97,15 @@ impl MetadataRecord {
     }
 }
 
-pub struct MetadataRecordRaw {
+pub struct GRMetadataRecordRaw {
     pub(super) data: ByteStack<32>,
 }
 
-impl MetadataRecordRaw {
+impl GRMetadataRecordRaw {
     /// Decodes a given metadata record, validating all data for correctness.
     ///
-    /// WARNING: That means you need to do contextual validation! This function is not aware of any context
-    pub fn decode_from_bytes(data: [u8; 32]) -> Option<MetadataRecord> {
+    /// **☢ WARNING ☢: This only decodes; it doesn't validate expected values!**
+    pub fn decode(data: [u8; 32]) -> Option<GRMetadataRecord> {
         let data = ByteStack::new(data);
         let server_version =
             ServerVersion::__new(u64::from_le(data.read_qword(Self::MDR_OFFSET_P0)));
@@ -112,7 +115,7 @@ impl MetadataRecordRaw {
         let file_spec = FileSpecifier::try_new(u32::from_le(data.read_dword(Self::MDR_OFFSET_P3)))?;
         let file_spec_id =
             FileSpecifierVersion::__new(u32::from_le(data.read_dword(Self::MDR_OFFSET_P4)));
-        Some(MetadataRecord::new(
+        Some(GRMetadataRecord::new(
             server_version,
             driver_version,
             file_scope,
@@ -122,7 +125,7 @@ impl MetadataRecordRaw {
     }
 }
 
-impl MetadataRecordRaw {
+impl GRMetadataRecordRaw {
     const MDR_OFFSET_P0: usize = 0;
     const MDR_OFFSET_P1: usize = sizeof!(u64);
     const MDR_OFFSET_P2: usize = Self::MDR_OFFSET_P1 + sizeof!(u64);
@@ -180,7 +183,7 @@ impl MetadataRecordRaw {
     }
 }
 
-impl MetadataRecordRaw {
+impl GRMetadataRecordRaw {
     pub const fn read_p0_server_version(&self) -> ServerVersion {
         ServerVersion::__new(self.data.read_qword(Self::MDR_OFFSET_P0))
     }
@@ -208,216 +211,182 @@ impl MetadataRecordRaw {
      - 04B: Host setting version ID
      - 04B: Host run mode
     - 08B: Host startup counter
-    - 08B: Host name length
-    - ??B: Host name
+    - 01B: Host name length
+    - 255B: Host name
+    = 304B
 */
 
-pub type HRConstSectionRaw = [u8; 56];
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct HostRecord {
-    hr_cr: HRConstSection,
-    host_name: Box<[u8]>,
+#[derive(Debug, PartialEq)]
+pub struct GRHostRecord {
+    epoch_time: u128,
+    uptime: u128,
+    setting_version: u32,
+    run_mode: HostRunMode,
+    startup_counter: u64,
+    hostname_len: u8,
+    hostname_raw: [u8; 255],
 }
 
-impl HostRecord {
-    pub fn new(hr_cr: HRConstSection, host_name: Box<[u8]>) -> Self {
-        Self { hr_cr, host_name }
-    }
-    pub fn hr_cr(&self) -> &HRConstSection {
-        &self.hr_cr
-    }
-    pub fn host_name(&self) -> &[u8] {
-        self.host_name.as_ref()
-    }
-    pub fn encode(&self) -> HostRecordRaw {
-        HostRecordRaw::new(
-            self.hr_cr().host_epoch_time(),
-            self.hr_cr().host_uptime(),
-            self.hr_cr().host_setting_version_id(),
-            self.hr_cr().host_run_mode(),
-            self.hr_cr().host_startup_counter(),
-            self.host_name().into(),
-        )
+impl GRHostRecord {
+    pub fn decode(bytes: [u8; sizeof!(GRHostRecordRaw)]) -> Option<Self> {
+        let ns = ByteStack::new(bytes);
+        let epoch_time = u128::from_le(ns.read_xmmword(GRHostRecordRaw::GRHR_OFFSET_P0));
+        let uptime = u128::from_le(ns.read_xmmword(GRHostRecordRaw::GRHR_OFFSET_P1));
+        let setting_version = u32::from_le(ns.read_dword(GRHostRecordRaw::GRHR_OFFSET_P2));
+        let run_mode = HostRunMode::try_new_with_val(u32::from_le(
+            ns.read_dword(GRHostRecordRaw::GRHR_OFFSET_P3),
+        ))?;
+        let startup_counter = u64::from_le(ns.read_qword(GRHostRecordRaw::GRHR_OFFSET_P4));
+        let host_name_len = ns.read_byte(GRHostRecordRaw::GRHR_OFFSET_P5);
+        let host_name_raw =
+            util::copy_slice_to_array(&ns.slice()[GRHostRecordRaw::GRHR_OFFSET_P6..]);
+        Some(Self::new(
+            epoch_time,
+            uptime,
+            setting_version,
+            run_mode,
+            startup_counter,
+            host_name_len,
+            host_name_raw,
+        ))
     }
 }
 
-#[derive(Debug, PartialEq, Clone)]
-pub struct HRConstSection {
-    host_epoch_time: u128,
-    host_uptime: u128,
-    host_setting_version_id: u32,
-    host_run_mode: HostRunMode,
-    host_startup_counter: u64,
-}
-
-impl HRConstSection {
+impl GRHostRecord {
     pub const fn new(
-        host_epoch_time: u128,
-        host_uptime: u128,
-        host_setting_version_id: u32,
-        host_run_mode: HostRunMode,
-        host_startup_counter: u64,
+        epoch_time: u128,
+        uptime: u128,
+        setting_version: u32,
+        run_mode: HostRunMode,
+        startup_counter: u64,
+        hostname_len: u8,
+        hostname: [u8; 255],
     ) -> Self {
         Self {
-            host_epoch_time,
-            host_uptime,
-            host_setting_version_id,
-            host_run_mode,
-            host_startup_counter,
+            epoch_time,
+            uptime,
+            setting_version,
+            run_mode,
+            startup_counter,
+            hostname_len,
+            hostname_raw: hostname,
         }
     }
-    pub const fn host_epoch_time(&self) -> u128 {
-        self.host_epoch_time
+    pub fn epoch_time(&self) -> u128 {
+        self.epoch_time
     }
-    pub const fn host_uptime(&self) -> u128 {
-        self.host_uptime
+    pub fn uptime(&self) -> u128 {
+        self.uptime
     }
-    pub const fn host_setting_version_id(&self) -> u32 {
-        self.host_setting_version_id
+    pub fn setting_version(&self) -> u32 {
+        self.setting_version
     }
-    pub const fn host_run_mode(&self) -> HostRunMode {
-        self.host_run_mode
+    pub fn run_mode(&self) -> HostRunMode {
+        self.run_mode
     }
-    pub const fn host_startup_counter(&self) -> u64 {
-        self.host_startup_counter
+    pub fn startup_counter(&self) -> u64 {
+        self.startup_counter
     }
-}
-
-pub struct HostRecordRaw {
-    pub(super) data: ByteStack<{ sizeof!(HRConstSectionRaw) }>,
-    pub(super) host_name: Box<[u8]>,
-}
-
-impl HostRecordRaw {
-    pub const fn empty_buffer_const_section() -> [u8; sizeof!(HRConstSectionRaw)] {
-        [0u8; sizeof!(HRConstSectionRaw)]
-    }
-    /// Decodes and validates the [`HRConstSection`] of a [`HostRecord`]. Use the returned result to construct this
-    pub fn decode_from_bytes_const_sec(data: HRConstSectionRaw) -> Option<(HRConstSection, usize)> {
-        let s = ByteStack::new(data);
-        let host_epoch_time = s.read_xmmword(Self::HR_OFFSET_P0);
-        if host_epoch_time > crate::util::os::get_epoch_time() {
-            // and what? we have a file from the future. Einstein says hi. (ok, maybe the host time is incorrect)
-            return None;
-        }
-        let host_uptime = s.read_xmmword(Self::HR_OFFSET_P1);
-        let host_setting_version_id = s.read_dword(Self::HR_OFFSET_P2A);
-        let host_setting_run_mode =
-            HostRunMode::try_new_with_val(s.read_dword(Self::HR_OFFSET_P2B))?;
-        let host_startup_counter = s.read_qword(Self::HR_OFFSET_P3);
-        let host_name_length = s.read_qword(Self::HR_OFFSET_P4);
-        if host_name_length as usize > usize::MAX {
-            // too large for us to load. per DNS standards this shouldn't be more than 255 but who knows, some people like it wild
-            return None;
-        }
-        Some((
-            HRConstSection::new(
-                host_epoch_time,
-                host_uptime,
-                host_setting_version_id,
-                host_setting_run_mode,
-                host_startup_counter,
-            ),
-            host_name_length as usize,
-        ))
+    pub fn hostname_len(&self) -> u8 {
+        self.hostname_len
     }
-    pub fn decoded(&self) -> HostRecord {
-        HostRecord::new(
-            HRConstSection::new(
-                self.read_p0_epoch_time(),
-                self.read_p1_uptime(),
-                self.read_p2a_setting_version_id(),
-                self.read_p2b_run_mode(),
-                self.read_p3_startup_counter(),
-            ),
-            self.host_name.clone(),
+    pub fn hostname_raw(&self) -> [u8; 255] {
+        self.hostname_raw
+    }
+    pub fn encoded(&self) -> GRHostRecordRaw {
+        GRHostRecordRaw::new(
+            self.epoch_time(),
+            self.uptime(),
+            self.setting_version(),
+            self.run_mode(),
+            self.startup_counter(),
+            self.hostname_len(),
+            self.hostname_raw(),
         )
     }
 }
 
-impl HostRecordRaw {
-    const HR_OFFSET_P0: usize = 0;
-    const HR_OFFSET_P1: usize = sizeof!(u128);
-    const HR_OFFSET_P2A: usize = Self::HR_OFFSET_P1 + sizeof!(u128);
-    const HR_OFFSET_P2B: usize = Self::HR_OFFSET_P2A + sizeof!(u32);
-    const HR_OFFSET_P3: usize = Self::HR_OFFSET_P2B + sizeof!(u32);
-    const HR_OFFSET_P4: usize = Self::HR_OFFSET_P3 + sizeof!(u64);
-    const _ENSURE: () = assert!(Self::HR_OFFSET_P4 == sizeof!(HRConstSectionRaw) - sizeof!(u64));
+#[derive(Debug, PartialEq)]
+pub struct GRHostRecordRaw {
+    pub(super) data: ByteStack<304>,
+}
+
+impl GRHostRecordRaw {
+    const GRHR_OFFSET_P0: usize = 0;
+    const GRHR_OFFSET_P1: usize = sizeof!(u128);
+    const GRHR_OFFSET_P2: usize = Self::GRHR_OFFSET_P1 + sizeof!(u128);
+    const GRHR_OFFSET_P3: usize = Self::GRHR_OFFSET_P2 + sizeof!(u32);
+    const GRHR_OFFSET_P4: usize = Self::GRHR_OFFSET_P3 + sizeof!(u32);
+    const GRHR_OFFSET_P5: usize = Self::GRHR_OFFSET_P4 + sizeof!(u64);
+    const GRHR_OFFSET_P6: usize = Self::GRHR_OFFSET_P5 + 1;
+    const _ENSURE: () = assert!(Self::GRHR_OFFSET_P6 == sizeof!(Self) - 255);
     pub fn new(
-        p0_host_epoch_time: u128,
-        p1_host_uptime: u128,
-        p2a_host_setting_version_id: u32,
-        p2b_host_run_mode: HostRunMode,
-        p3_host_startup_counter: u64,
-        p5_host_name: Box<[u8]>,
+        p0_epoch_time: u128,
+        p1_uptime: u128,
+        p2_setting_version: u32,
+        p3_run_mode: HostRunMode,
+        p4_host_startup_counter: u64,
+        p5_host_name_length: u8,
+        p6_host_name_raw: [u8; 255],
     ) -> Self {
         let _ = Self::_ENSURE;
-        let p4_host_name_length = p5_host_name.len();
-        let mut host_record_fl = [0u8; 56];
-        host_record_fl[Self::HR_OFFSET_P0..Self::HR_OFFSET_P1]
-            .copy_from_slice(&p0_host_epoch_time.to_le_bytes());
-        host_record_fl[Self::HR_OFFSET_P1..Self::HR_OFFSET_P2A]
-            .copy_from_slice(&p1_host_uptime.to_le_bytes());
-        host_record_fl[Self::HR_OFFSET_P2A..Self::HR_OFFSET_P2B]
-            .copy_from_slice(&p2a_host_setting_version_id.to_le_bytes());
-        host_record_fl[Self::HR_OFFSET_P2B..Self::HR_OFFSET_P3]
-            .copy_from_slice(&(p2b_host_run_mode.value_u8() as u32).to_le_bytes());
-        host_record_fl[Self::HR_OFFSET_P3..Self::HR_OFFSET_P4]
-            .copy_from_slice(&p3_host_startup_counter.to_le_bytes());
-        host_record_fl[Self::HR_OFFSET_P4..]
-            .copy_from_slice(&(p4_host_name_length as u64).to_le_bytes());
+        let mut data = [0u8; sizeof!(Self)];
+        data[Self::GRHR_OFFSET_P0..Self::GRHR_OFFSET_P1]
+            .copy_from_slice(&p0_epoch_time.to_le_bytes());
+        data[Self::GRHR_OFFSET_P1..Self::GRHR_OFFSET_P2].copy_from_slice(&p1_uptime.to_le_bytes());
+        data[Self::GRHR_OFFSET_P2..Self::GRHR_OFFSET_P3]
+            .copy_from_slice(&p2_setting_version.to_le_bytes());
+        data[Self::GRHR_OFFSET_P3..Self::GRHR_OFFSET_P4]
+            .copy_from_slice(&(p3_run_mode.value_u8() as u32).to_le_bytes());
+        data[Self::GRHR_OFFSET_P4..Self::GRHR_OFFSET_P5]
+            .copy_from_slice(&p4_host_startup_counter.to_le_bytes());
+        data[Self::GRHR_OFFSET_P5] = p5_host_name_length;
+        data[Self::GRHR_OFFSET_P6..].copy_from_slice(&p6_host_name_raw);
         Self {
-            data: ByteStack::new(host_record_fl),
-            host_name: p5_host_name,
+            data: ByteStack::new(data),
         }
     }
-    pub fn new_auto(
-        p2a_host_setting_version_id: u32,
-        p2b_host_run_mode: HostRunMode,
-        p3_host_startup_counter: u64,
-        p5_host_name: Box<[u8]>,
-    ) -> Self {
-        let p0_host_epoch_time = crate::util::os::get_epoch_time();
-        let p1_host_uptime = crate::util::os::get_uptime();
-        Self::new(
-            p0_host_epoch_time,
-            p1_host_uptime,
-            p2a_host_setting_version_id,
-            p2b_host_run_mode,
-            p3_host_startup_counter,
-            p5_host_name,
-        )
-    }
-}
-
-impl HostRecordRaw {
     pub const fn read_p0_epoch_time(&self) -> u128 {
-        self.data.read_xmmword(Self::HR_OFFSET_P0)
+        self.data.read_xmmword(Self::GRHR_OFFSET_P0)
     }
     pub const fn read_p1_uptime(&self) -> u128 {
-        self.data.read_xmmword(Self::HR_OFFSET_P1)
+        self.data.read_xmmword(Self::GRHR_OFFSET_P1)
     }
-    pub const fn read_p2a_setting_version_id(&self) -> u32 {
-        self.data.read_dword(Self::HR_OFFSET_P2A)
+    pub const fn read_p2_setting_version_id(&self) -> u32 {
+        self.data.read_dword(Self::GRHR_OFFSET_P2)
     }
-    pub const fn read_p2b_run_mode(&self) -> HostRunMode {
-        HostRunMode::new_with_val(self.data.read_dword(Self::HR_OFFSET_P2B))
+    pub const fn read_p3_run_mode(&self) -> HostRunMode {
+        HostRunMode::new_with_val(self.data.read_dword(Self::GRHR_OFFSET_P3))
     }
-    pub const fn read_p3_startup_counter(&self) -> u64 {
-        self.data.read_qword(Self::HR_OFFSET_P3)
+    pub const fn read_p4_startup_counter(&self) -> u64 {
+        self.data.read_qword(Self::GRHR_OFFSET_P4)
     }
-    pub const fn read_p4_host_name_length(&self) -> u64 {
-        self.data.read_qword(Self::HR_OFFSET_P4)
+    pub const fn read_p5_host_name_length(&self) -> usize {
+        self.data.read_byte(Self::GRHR_OFFSET_P5) as _
     }
-    pub fn read_p5_host_name(&self) -> &[u8] {
-        &self.host_name
+    pub fn read_p6_host_name_raw(&self) -> &[u8] {
+        &self.data.slice()[Self::GRHR_OFFSET_P6..]
+    }
+    pub fn read_host_name(&self) -> &[u8] {
+        &self.data.slice()
+            [Self::GRHR_OFFSET_P6..Self::GRHR_OFFSET_P6 + self.read_p5_host_name_length()]
+    }
+    pub fn decoded(&self) -> GRHostRecord {
+        GRHostRecord::new(
+            self.read_p0_epoch_time(),
+            self.read_p1_uptime(),
+            self.read_p2_setting_version_id(),
+            self.read_p3_run_mode(),
+            self.read_p4_startup_counter(),
+            self.read_p5_host_name_length() as _,
+            util::copy_slice_to_array(self.read_p6_host_name_raw()),
+        )
     }
 }
 
 #[test]
 fn test_metadata_record_encode_decode() {
-    let md = MetadataRecordRaw::new(
+    let md = GRMetadataRecordRaw::new(
         FileScope::TransactionLog,
         FileSpecifier::GNSTxnLog,
         FileSpecifierVersion(1),
@@ -441,19 +410,20 @@ fn test_host_record_encode_decode() {
         .duration_since(UNIX_EPOCH)
         .unwrap()
         .as_nanos();
-    let hr = HostRecordRaw::new(
+    let hr = GRHostRecordRaw::new(
         time,
         HOST_UPTIME,
         HOST_SETTING_VERSION_ID,
         HOST_RUN_MODE,
         HOST_STARTUP_COUNTER,
-        HOST_NAME.as_bytes().to_owned().into_boxed_slice(),
+        HOST_NAME.len() as _,
+        crate::util::copy_str_to_array(HOST_NAME),
     );
     assert_eq!(hr.read_p0_epoch_time(), time);
     assert_eq!(hr.read_p1_uptime(), HOST_UPTIME);
-    assert_eq!(hr.read_p2a_setting_version_id(), HOST_SETTING_VERSION_ID);
-    assert_eq!(hr.read_p2b_run_mode(), HOST_RUN_MODE);
-    assert_eq!(hr.read_p3_startup_counter(), HOST_STARTUP_COUNTER);
-    assert_eq!(hr.read_p4_host_name_length(), HOST_NAME.len() as u64);
-    assert_eq!(hr.read_p5_host_name(), HOST_NAME.as_bytes());
+    assert_eq!(hr.read_p2_setting_version_id(), HOST_SETTING_VERSION_ID);
+    assert_eq!(hr.read_p3_run_mode(), HOST_RUN_MODE);
+    assert_eq!(hr.read_p4_startup_counter(), HOST_STARTUP_COUNTER);
+    assert_eq!(hr.read_p5_host_name_length(), HOST_NAME.len());
+    assert_eq!(hr.read_host_name(), HOST_NAME.as_bytes());
 }
diff --git a/server/src/engine/storage/v1/header_impl/mod.rs b/server/src/engine/storage/v1/header_impl/mod.rs
index 2ddf4bba..1dc30486 100644
--- a/server/src/engine/storage/v1/header_impl/mod.rs
+++ b/server/src/engine/storage/v1/header_impl/mod.rs
@@ -63,8 +63,6 @@ mod gr;
 // (3) dr
 mod dr;
 
-use crate::engine::mem::ByteStack;
-
 /// The file scope
 #[repr(u8)]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
@@ -146,8 +144,8 @@ pub struct SDSSHeader {
     // static record
     sr: sr::StaticRecord,
     // genesis record
-    gr_mdr: gr::MetadataRecord,
-    gr_hr: gr::HostRecord,
+    gr_mdr: gr::GRMetadataRecord,
+    gr_hr: gr::GRHostRecord,
     // dynamic record
     dr_hs: dr::DRHostSignature,
     dr_rs: dr::DRRuntimeSignature,
@@ -156,8 +154,8 @@ pub struct SDSSHeader {
 impl SDSSHeader {
     pub const fn new(
         sr: sr::StaticRecord,
-        gr_mdr: gr::MetadataRecord,
-        gr_hr: gr::HostRecord,
+        gr_mdr: gr::GRMetadataRecord,
+        gr_hr: gr::GRHostRecord,
         dr_hs: dr::DRHostSignature,
         dr_rs: dr::DRRuntimeSignature,
     ) -> Self {
@@ -169,22 +167,13 @@ impl SDSSHeader {
             dr_rs,
         }
     }
-    pub fn encode(&self) -> SDSSHeaderRaw {
-        SDSSHeaderRaw::new_full(
-            self.sr.encode(),
-            self.gr_mdr.encode(),
-            self.gr_hr.encode(),
-            self.dr_hs().encoded(),
-            self.dr_rs().encoded(),
-        )
-    }
     pub fn sr(&self) -> &sr::StaticRecord {
         &self.sr
     }
-    pub fn gr_mdr(&self) -> &gr::MetadataRecord {
+    pub fn gr_mdr(&self) -> &gr::GRMetadataRecord {
         &self.gr_mdr
     }
-    pub fn gr_hr(&self) -> &gr::HostRecord {
+    pub fn gr_hr(&self) -> &gr::GRHostRecord {
         &self.gr_hr
     }
     pub fn dr_hs(&self) -> &dr::DRHostSignature {
@@ -193,12 +182,21 @@ impl SDSSHeader {
     pub fn dr_rs(&self) -> &dr::DRRuntimeSignature {
         &self.dr_rs
     }
+    pub fn encode(&self) -> SDSSHeaderRaw {
+        SDSSHeaderRaw::new_full(
+            self.sr.encode(),
+            self.gr_mdr().encoded(),
+            self.gr_hr().encoded(),
+            self.dr_hs().encoded(),
+            self.dr_rs().encoded(),
+        )
+    }
 }
 
 pub struct SDSSHeaderRaw {
     sr: sr::StaticRecordRaw,
-    gr_0_mdr: gr::MetadataRecordRaw,
-    gr_1_hr: gr::HostRecordRaw,
+    gr_0_mdr: gr::GRMetadataRecordRaw,
+    gr_1_hr: gr::GRHostRecordRaw,
     dr_0_hs: dr::DRHostSignatureRaw,
     dr_1_rs: dr::DRRuntimeSignatureRaw,
 }
@@ -206,8 +204,8 @@ pub struct SDSSHeaderRaw {
 impl SDSSHeaderRaw {
     pub fn new_full(
         sr: sr::StaticRecordRaw,
-        gr_mdr: gr::MetadataRecordRaw,
-        gr_hr: gr::HostRecordRaw,
+        gr_mdr: gr::GRMetadataRecordRaw,
+        gr_hr: gr::GRHostRecordRaw,
         dr_hs: dr::DRHostSignatureRaw,
         dr_rs: dr::DRRuntimeSignatureRaw,
     ) -> Self {
@@ -221,22 +219,17 @@ impl SDSSHeaderRaw {
     }
     pub fn new(
         sr: sr::StaticRecordRaw,
-        gr_0_mdr: gr::MetadataRecordRaw,
-        gr_1_hr_const_section: gr::HRConstSectionRaw,
-        gr_1_hr_host_name: Box<[u8]>,
+        gr_0_mdr: gr::GRMetadataRecordRaw,
+        gr_1_hr: gr::GRHostRecordRaw,
         dr_hs: dr::DRHostSignatureRaw,
-        dr_rs_const: dr::DRRuntimeSignatureFixedRaw,
-        dr_rs_host_name: Box<[u8]>,
+        dr_rs: dr::DRRuntimeSignatureRaw,
     ) -> Self {
         Self {
             sr,
             gr_0_mdr,
-            gr_1_hr: gr::HostRecordRaw {
-                data: ByteStack::new(gr_1_hr_const_section),
-                host_name: gr_1_hr_host_name,
-            },
+            gr_1_hr,
             dr_0_hs: dr_hs,
-            dr_1_rs: dr::DRRuntimeSignatureRaw::new_with_sections(dr_rs_host_name, dr_rs_const),
+            dr_1_rs: dr_rs,
         }
     }
     pub fn get0_sr(&self) -> &[u8] {
@@ -248,19 +241,11 @@ impl SDSSHeaderRaw {
     pub fn get1_dr_1_hr_0(&self) -> &[u8] {
         self.gr_1_hr.data.slice()
     }
-    pub fn get1_dr_1_hr_1(&self) -> &[u8] {
-        self.gr_1_hr.host_name.as_ref()
-    }
-    pub fn calculate_header_size(&self) -> usize {
-        Self::calculate_fixed_header_size()
-            + self.gr_1_hr.host_name.len()
-            + self.dr_1_rs.host_name.len()
-    }
     pub const fn calculate_fixed_header_size() -> usize {
         sizeof!(sr::StaticRecordRaw)
-            + sizeof!(gr::MetadataRecordRaw)
-            + sizeof!(gr::HRConstSectionRaw)
+            + sizeof!(gr::GRMetadataRecordRaw)
+            + sizeof!(gr::GRHostRecordRaw)
             + sizeof!(dr::DRHostSignatureRaw)
-            + sizeof!(dr::DRRuntimeSignatureFixedRaw)
+            + sizeof!(dr::DRRuntimeSignatureRaw)
     }
 }
diff --git a/server/src/engine/storage/v1/header_impl/sr.rs b/server/src/engine/storage/v1/header_impl/sr.rs
index a98f2af9..33b3d246 100644
--- a/server/src/engine/storage/v1/header_impl/sr.rs
+++ b/server/src/engine/storage/v1/header_impl/sr.rs
@@ -62,7 +62,7 @@ impl StaticRecordRaw {
     pub const fn empty_buffer() -> [u8; sizeof!(Self)] {
         [0u8; sizeof!(Self)]
     }
-    pub fn decode_from_bytes(buf: [u8; sizeof!(Self)]) -> Option<StaticRecord> {
+    pub fn decode(buf: [u8; sizeof!(Self)]) -> Option<StaticRecord> {
         StaticRecordUVRaw::decode_from_bytes(buf).map(StaticRecord::new)
     }
 }
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index f29f920c..19a1b280 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -362,3 +362,17 @@ macro_rules! byte_repr_impls {
 }
 
 byte_repr_impls!(u8, i8, u16, i16, u32, i32, u64, i64, u128, i128, usize, isize);
+
+pub const fn copy_slice_to_array<const N: usize>(bytes: &[u8]) -> [u8; N] {
+    assert!(bytes.len() <= N);
+    let mut data = [0u8; N];
+    let mut i = 0;
+    while i < bytes.len() {
+        data[i] = bytes[i];
+        i += 1;
+    }
+    data
+}
+pub const fn copy_str_to_array<const N: usize>(str: &str) -> [u8; N] {
+    copy_slice_to_array(str.as_bytes())
+}

From 27566827294ed18d9ab3ba9e915016205fc98b7b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 29 May 2023 15:48:30 +0000
Subject: [PATCH 199/310] Implement auto generation for header objects

---
 server/src/engine/storage/header.rs           |   2 +-
 .../src/engine/storage/v1/header_impl/dr.rs   |  19 +-
 .../src/engine/storage/v1/header_impl/gr.rs   |  25 +++
 .../src/engine/storage/v1/header_impl/mod.rs  |  27 ++-
 .../src/engine/storage/v1/header_impl/sr.rs   |  13 +-
 server/src/util/os.rs                         | 202 ++++++++++++++----
 6 files changed, 230 insertions(+), 58 deletions(-)

diff --git a/server/src/engine/storage/header.rs b/server/src/engine/storage/header.rs
index 28ef2ae3..86accef5 100644
--- a/server/src/engine/storage/header.rs
+++ b/server/src/engine/storage/header.rs
@@ -304,7 +304,7 @@ impl StaticRecordUV {
     pub const fn os(&self) -> HostOS {
         self.os
     }
-    pub const fn encode(&self) -> StaticRecordUVRaw {
+    pub const fn encoded(&self) -> StaticRecordUVRaw {
         StaticRecordUVRaw::new(
             self.header_version(),
             self.ptr_width(),
diff --git a/server/src/engine/storage/v1/header_impl/dr.rs b/server/src/engine/storage/v1/header_impl/dr.rs
index 008287c8..9b952943 100644
--- a/server/src/engine/storage/v1/header_impl/dr.rs
+++ b/server/src/engine/storage/v1/header_impl/dr.rs
@@ -30,7 +30,7 @@ use crate::{
         storage::{
             header::{HostArch, HostEndian, HostOS, HostPointerWidth},
             v1::header_impl::FileSpecifierVersion,
-            versions::{DriverVersion, ServerVersion},
+            versions::{self, DriverVersion, ServerVersion},
         },
     },
     util,
@@ -160,6 +160,13 @@ impl DRHostSignatureRaw {
     const DRHS_OFFSET_P5: usize = Self::DRHS_OFFSET_P4 + 1;
     const DRHS_OFFSET_P6: usize = Self::DRHS_OFFSET_P5 + 1;
     const _ENSURE: () = assert!(Self::DRHS_OFFSET_P6 == sizeof!(Self) - 1);
+    pub const fn new_auto(file_specifier_version: FileSpecifierVersion) -> Self {
+        Self::new(
+            versions::v1::V1_SERVER_VERSION,
+            versions::v1::V1_DRIVER_VERSION,
+            file_specifier_version,
+        )
+    }
     pub const fn new(
         server_version: ServerVersion,
         driver_version: DriverVersion,
@@ -343,6 +350,16 @@ impl DRRuntimeSignatureRaw {
     const DRRS_OFFSET_P3: usize = Self::DRRS_OFFSET_P2 + sizeof!(u128);
     const DRRS_OFFSET_P4: usize = Self::DRRS_OFFSET_P3 + 1;
     const _ENSURE: () = assert!(Self::DRRS_OFFSET_P4 == sizeof!(Self) - 255);
+    pub fn new_auto(modify_count: u64) -> Self {
+        let hostname = crate::util::os::get_hostname();
+        Self::new(
+            modify_count,
+            crate::util::os::get_epoch_time(),
+            crate::util::os::get_uptime(),
+            hostname.len(),
+            hostname.raw(),
+        )
+    }
     pub fn new(
         modify_count: u64,
         host_epoch_time: u128,
diff --git a/server/src/engine/storage/v1/header_impl/gr.rs b/server/src/engine/storage/v1/header_impl/gr.rs
index 8e0a9352..c40adb9f 100644
--- a/server/src/engine/storage/v1/header_impl/gr.rs
+++ b/server/src/engine/storage/v1/header_impl/gr.rs
@@ -135,6 +135,19 @@ impl GRMetadataRecordRaw {
     pub const fn empty_buffer() -> [u8; sizeof!(Self)] {
         [0u8; sizeof!(Self)]
     }
+    pub const fn new_auto(
+        scope: FileScope,
+        specifier: FileSpecifier,
+        specifier_id: FileSpecifierVersion,
+    ) -> Self {
+        Self::new_full(
+            versions::v1::V1_SERVER_VERSION,
+            versions::v1::V1_DRIVER_VERSION,
+            scope,
+            specifier,
+            specifier_id,
+        )
+    }
     pub const fn new_full(
         server_version: ServerVersion,
         driver_version: DriverVersion,
@@ -320,6 +333,18 @@ impl GRHostRecordRaw {
     const GRHR_OFFSET_P5: usize = Self::GRHR_OFFSET_P4 + sizeof!(u64);
     const GRHR_OFFSET_P6: usize = Self::GRHR_OFFSET_P5 + 1;
     const _ENSURE: () = assert!(Self::GRHR_OFFSET_P6 == sizeof!(Self) - 255);
+    pub fn new_auto(setting_version: u32, run_mode: HostRunMode, startup_counter: u64) -> Self {
+        let hostname = crate::util::os::get_hostname();
+        Self::new(
+            crate::util::os::get_epoch_time(),
+            crate::util::os::get_uptime(),
+            setting_version,
+            run_mode,
+            startup_counter,
+            hostname.len(),
+            hostname.raw(),
+        )
+    }
     pub fn new(
         p0_epoch_time: u128,
         p1_uptime: u128,
diff --git a/server/src/engine/storage/v1/header_impl/mod.rs b/server/src/engine/storage/v1/header_impl/mod.rs
index 1dc30486..63f03f77 100644
--- a/server/src/engine/storage/v1/header_impl/mod.rs
+++ b/server/src/engine/storage/v1/header_impl/mod.rs
@@ -182,9 +182,9 @@ impl SDSSHeader {
     pub fn dr_rs(&self) -> &dr::DRRuntimeSignature {
         &self.dr_rs
     }
-    pub fn encode(&self) -> SDSSHeaderRaw {
+    pub fn encoded(&self) -> SDSSHeaderRaw {
         SDSSHeaderRaw::new_full(
-            self.sr.encode(),
+            self.sr.encoded(),
             self.gr_mdr().encoded(),
             self.gr_hr().encoded(),
             self.dr_hs().encoded(),
@@ -202,6 +202,27 @@ pub struct SDSSHeaderRaw {
 }
 
 impl SDSSHeaderRaw {
+    pub fn new_auto(
+        gr_mdr_scope: FileScope,
+        gr_mdr_specifier: FileSpecifier,
+        gr_mdr_specifier_id: FileSpecifierVersion,
+        gr_hr_setting_version: u32,
+        gr_hr_run_mode: HostRunMode,
+        gr_hr_startup_counter: u64,
+        dr_rts_modify_count: u64,
+    ) -> Self {
+        Self::new_full(
+            sr::StaticRecordRaw::new_auto(),
+            gr::GRMetadataRecordRaw::new_auto(gr_mdr_scope, gr_mdr_specifier, gr_mdr_specifier_id),
+            gr::GRHostRecordRaw::new_auto(
+                gr_hr_setting_version,
+                gr_hr_run_mode,
+                gr_hr_startup_counter,
+            ),
+            dr::DRHostSignatureRaw::new_auto(gr_mdr_specifier_id),
+            dr::DRRuntimeSignatureRaw::new_auto(dr_rts_modify_count),
+        )
+    }
     pub fn new_full(
         sr: sr::StaticRecordRaw,
         gr_mdr: gr::GRMetadataRecordRaw,
@@ -241,7 +262,7 @@ impl SDSSHeaderRaw {
     pub fn get1_dr_1_hr_0(&self) -> &[u8] {
         self.gr_1_hr.data.slice()
     }
-    pub const fn calculate_fixed_header_size() -> usize {
+    pub const fn header_size() -> usize {
         sizeof!(sr::StaticRecordRaw)
             + sizeof!(gr::GRMetadataRecordRaw)
             + sizeof!(gr::GRHostRecordRaw)
diff --git a/server/src/engine/storage/v1/header_impl/sr.rs b/server/src/engine/storage/v1/header_impl/sr.rs
index 33b3d246..4400fc41 100644
--- a/server/src/engine/storage/v1/header_impl/sr.rs
+++ b/server/src/engine/storage/v1/header_impl/sr.rs
@@ -38,9 +38,9 @@ impl StaticRecord {
     pub const fn new(sr: StaticRecordUV) -> Self {
         Self { sr }
     }
-    pub const fn encode(&self) -> StaticRecordRaw {
+    pub const fn encoded(&self) -> StaticRecordRaw {
         StaticRecordRaw {
-            base: self.sr.encode(),
+            base: self.sr.encoded(),
         }
     }
     pub const fn sr(&self) -> &StaticRecordUV {
@@ -54,10 +54,11 @@ pub struct StaticRecordRaw {
 }
 
 impl StaticRecordRaw {
-    pub const fn new() -> Self {
-        Self {
-            base: StaticRecordUVRaw::create(versions::v1::V1_HEADER_VERSION),
-        }
+    pub const fn new_auto() -> Self {
+        Self::new(StaticRecordUVRaw::create(versions::v1::V1_HEADER_VERSION))
+    }
+    pub const fn new(base: StaticRecordUVRaw) -> Self {
+        Self { base }
     }
     pub const fn empty_buffer() -> [u8; sizeof!(Self)] {
         [0u8; sizeof!(Self)]
diff --git a/server/src/util/os.rs b/server/src/util/os.rs
index 7c2ccc78..28c2d7fc 100644
--- a/server/src/util/os.rs
+++ b/server/src/util/os.rs
@@ -264,7 +264,7 @@ pub fn dirsize(path: impl AsRef<Path>) -> IoResult<u64> {
 
 /// Returns the current system uptime in milliseconds
 pub fn get_uptime() -> u128 {
-    uptime().unwrap()
+    uptime_impl::uptime().unwrap()
 }
 
 /// Returns the current epoch time in nanoseconds
@@ -275,58 +275,166 @@ pub fn get_epoch_time() -> u128 {
         .as_nanos()
 }
 
-#[cfg(target_os = "linux")]
-fn uptime() -> std::io::Result<u128> {
-    let mut sysinfo: libc::sysinfo = unsafe { std::mem::zeroed() };
-    let res = unsafe { libc::sysinfo(&mut sysinfo) };
-    if res == 0 {
-        Ok(sysinfo.uptime as u128 * 1_000)
-    } else {
-        Err(std::io::Error::last_os_error())
-    }
+/// Returns the hostname
+pub fn get_hostname() -> hostname_impl::Hostname {
+    hostname_impl::Hostname::get()
 }
 
-#[cfg(any(
-    target_os = "macos",
-    target_os = "freebsd",
-    target_os = "openbsd",
-    target_os = "netbsd"
-))]
-fn uptime() -> std::io::Result<u128> {
-    use libc::{c_void, size_t, sysctl, timeval};
-    use std::ptr;
-
-    let mib = [libc::CTL_KERN, libc::KERN_BOOTTIME];
-    let mut boottime = timeval {
-        tv_sec: 0,
-        tv_usec: 0,
-    };
-    let mut size = std::mem::size_of::<libc::timeval>() as size_t;
-
-    let result = unsafe {
-        sysctl(
-            // this cast is fine. sysctl only needs to access the ptr to array base (read)
-            &mib as *const _ as *mut _,
-            2,
-            &mut boottime as *mut timeval as *mut c_void,
-            &mut size,
-            ptr::null_mut(),
-            0,
-        )
-    };
+mod uptime_impl {
+    #[cfg(target_os = "linux")]
+    pub(super) fn uptime() -> std::io::Result<u128> {
+        let mut sysinfo: libc::sysinfo = unsafe { std::mem::zeroed() };
+        let res = unsafe { libc::sysinfo(&mut sysinfo) };
+        if res == 0 {
+            Ok(sysinfo.uptime as u128 * 1_000)
+        } else {
+            Err(std::io::Error::last_os_error())
+        }
+    }
+
+    #[cfg(any(
+        target_os = "macos",
+        target_os = "freebsd",
+        target_os = "openbsd",
+        target_os = "netbsd"
+    ))]
+    pub(super) fn uptime() -> std::io::Result<u128> {
+        use libc::{c_void, size_t, sysctl, timeval};
+        use std::ptr;
+
+        let mib = [libc::CTL_KERN, libc::KERN_BOOTTIME];
+        let mut boottime = timeval {
+            tv_sec: 0,
+            tv_usec: 0,
+        };
+        let mut size = std::mem::size_of::<libc::timeval>() as size_t;
+
+        let result = unsafe {
+            sysctl(
+                // this cast is fine. sysctl only needs to access the ptr to array base (read)
+                &mib as *const _ as *mut _,
+                2,
+                &mut boottime as *mut timeval as *mut c_void,
+                &mut size,
+                ptr::null_mut(),
+                0,
+            )
+        };
 
-    if result == 0 {
-        let current_time = unsafe { libc::time(ptr::null_mut()) };
-        let uptime_secs = current_time - boottime.tv_sec;
-        Ok((uptime_secs as u128) * 1_000)
-    } else {
-        Err(std::io::Error::last_os_error())
+        if result == 0 {
+            let current_time = unsafe { libc::time(ptr::null_mut()) };
+            let uptime_secs = current_time - boottime.tv_sec;
+            Ok((uptime_secs as u128) * 1_000)
+        } else {
+            Err(std::io::Error::last_os_error())
+        }
+    }
+
+    #[cfg(target_os = "windows")]
+    pub(super) fn uptime() -> std::io::Result<u128> {
+        Ok(unsafe { winapi::um::sysinfoapi::GetTickCount64() } as u128)
     }
 }
 
-#[cfg(target_os = "windows")]
-fn uptime() -> std::io::Result<u128> {
-    Ok(unsafe { winapi::um::sysinfoapi::GetTickCount64() } as u128)
+mod hostname_impl {
+    use std::ffi::CStr;
+
+    pub struct Hostname {
+        len: u8,
+        raw: [u8; 255],
+    }
+
+    impl Hostname {
+        pub fn get() -> Self {
+            get_hostname()
+        }
+        unsafe fn new_from_raw_buf(buf: &[u8; 256]) -> Self {
+            let mut raw = [0u8; 255];
+            raw.copy_from_slice(&buf[..255]);
+            Self {
+                len: CStr::from_ptr(buf.as_ptr().cast()).to_bytes().len() as _,
+                raw,
+            }
+        }
+        pub fn as_str(&self) -> &str {
+            unsafe {
+                core::str::from_utf8_unchecked(core::slice::from_raw_parts(
+                    self.raw.as_ptr(),
+                    self.len as _,
+                ))
+            }
+        }
+        pub fn raw(&self) -> [u8; 255] {
+            self.raw
+        }
+        pub fn len(&self) -> u8 {
+            self.len
+        }
+    }
+
+    #[cfg(target_family = "unix")]
+    fn get_hostname() -> Hostname {
+        use libc::gethostname;
+
+        let mut buf: [u8; 256] = [0; 256];
+        unsafe {
+            gethostname(buf.as_mut_ptr().cast(), buf.len());
+            Hostname::new_from_raw_buf(&buf)
+        }
+    }
+
+    #[cfg(target_family = "windows")]
+    fn get_hostname() -> (usize, [u8; 255]) {
+        use winapi::shared::minwindef::DWORD;
+        use winapi::um::sysinfoapi::GetComputerNameA;
+
+        let mut buf: [u8; 256] = [0; 256];
+        let mut size: DWORD = buf.len() as u32;
+
+        unsafe {
+            GetComputerNameA(buf.as_mut_ptr().cast(), &mut size);
+            Hostname::new_from_raw_buf(&buf)
+        }
+    }
+
+    #[cfg(test)]
+    mod test {
+        use std::process::Command;
+
+        #[cfg(target_os = "linux")]
+        fn test_get_hostname() -> Result<String, Box<dyn std::error::Error>> {
+            let output = Command::new("sh")
+                .arg("-c")
+                .arg("hostname")
+                .output()?;
+            let hostname = String::from_utf8_lossy(&output.stdout).trim().to_string();
+            Ok(hostname)
+        }
+        #[cfg(target_os = "windows")]
+        fn test_get_hostname() -> Result<String, Box<dyn std::error::Error>> {
+            let output = Command::new("cmd")
+                .args(&["/C", "hostname"])
+                .output()?;
+            let hostname = String::from_utf8_lossy(&output.stdout).trim().to_string();
+            Ok(hostname)
+        }
+                
+        #[cfg(target_os = "macos")]
+        fn test_get_hostname() -> Result<String, Box<dyn std::error::Error>> {
+            let output = Command::new("sh")
+                .arg("-c")
+                .arg("scutil --get LocalHostName")
+                .output()?;
+            let hostname = String::from_utf8_lossy(&output.stdout).trim().to_string();
+            Ok(hostname)
+        }
+        
+        #[test]
+        fn t_get_hostname() {
+            let hostname_from_cmd = test_get_hostname().unwrap();
+            assert_eq!(hostname_from_cmd.as_str(), super::Hostname::get().as_str());
+        }
+    }
 }
 
 #[test]

From 17b07897e5547357687c18b1a74d12fa61bf8060 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 30 May 2023 05:34:36 +0000
Subject: [PATCH 200/310] Implement `StartStop`

---
 server/src/engine/storage/v1/mod.rs        |  46 +++++++
 server/src/engine/storage/v1/start_stop.rs | 150 +++++++++++++++++++++
 server/src/util/macros.rs                  |  47 +++++++
 server/src/util/os.rs                      |  61 ++++-----
 4 files changed, 272 insertions(+), 32 deletions(-)
 create mode 100644 server/src/engine/storage/v1/start_stop.rs

diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index ad07016f..18725f21 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -26,3 +26,49 @@
 
 mod header_impl;
 mod rw;
+mod start_stop;
+
+use std::io::Error as IoError;
+
+pub type SDSSResult<T> = Result<T, SDSSError>;
+
+pub trait SDSSErrorContext {
+    type ExtraData;
+    fn with_extra(self, extra: Self::ExtraData) -> SDSSError;
+}
+
+impl SDSSErrorContext for IoError {
+    type ExtraData = &'static str;
+    fn with_extra(self, extra: Self::ExtraData) -> SDSSError {
+        SDSSError::IoErrorExtra(self, extra)
+    }
+}
+
+#[derive(Debug)]
+pub enum SDSSError {
+    IoError(IoError),
+    IoErrorExtra(IoError, &'static str),
+    CorruptedFile(&'static str),
+    StartupError(&'static str),
+}
+
+impl SDSSError {
+    pub const fn corrupted_file(fname: &'static str) -> Self {
+        Self::CorruptedFile(fname)
+    }
+    pub const fn ioerror_extra(error: IoError, extra: &'static str) -> Self {
+        Self::IoErrorExtra(error, extra)
+    }
+    pub fn with_ioerror_extra(self, extra: &'static str) -> Self {
+        match self {
+            Self::IoError(ioe) => Self::IoErrorExtra(ioe, extra),
+            x => x,
+        }
+    }
+}
+
+impl From<IoError> for SDSSError {
+    fn from(e: IoError) -> Self {
+        Self::IoError(e)
+    }
+}
diff --git a/server/src/engine/storage/v1/start_stop.rs b/server/src/engine/storage/v1/start_stop.rs
new file mode 100644
index 00000000..54bbe3b4
--- /dev/null
+++ b/server/src/engine/storage/v1/start_stop.rs
@@ -0,0 +1,150 @@
+/*
+ * Created on Mon May 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{SDSSError, SDSSErrorContext, SDSSResult},
+    crate::util::os,
+    std::{
+        fs::File,
+        io::{ErrorKind, Read, Write},
+    },
+};
+
+#[cfg(not(test))]
+const START_FILE: &'static str = ".start";
+#[cfg(test)]
+const START_FILE: &'static str = ".start_testmode";
+#[cfg(not(test))]
+const STOP_FILE: &'static str = ".stop";
+#[cfg(test)]
+const STOP_FILE: &'static str = ".stop_testmode";
+
+const EMSG_FAILED_WRITE_START_FILE: &str =
+    concat_str_to_str!("failed to write to `", START_FILE, "` file");
+const EMSG_FAILED_WRITE_STOP_FILE: &str =
+    concat_str_to_str!("failed to write to `", STOP_FILE, "` file");
+const EMSG_FAILED_OPEN_START_FILE: &str =
+    concat_str_to_str!("failed to open `", START_FILE, "` file");
+const EMSG_FAILED_OPEN_STOP_FILE: &str =
+    concat_str_to_str!("failed to open `", STOP_FILE, "` file");
+const EMSG_FAILED_VERIFY: &str = concat_str_to_str!(
+    "failed to verify `",
+    START_FILE,
+    concat_str_to_str!("` and `", STOP_FILE, "` timestamps")
+);
+
+#[derive(Debug)]
+pub struct StartStop {
+    begin: u128,
+    stop_file: File,
+}
+
+#[derive(Debug)]
+enum ReadNX {
+    Created(File),
+    Read(File, u128),
+}
+
+impl ReadNX {
+    const fn created(&self) -> bool {
+        matches!(self, Self::Created(_))
+    }
+    fn file_mut(&mut self) -> &mut File {
+        match self {
+            Self::Created(ref mut f) => f,
+            Self::Read(ref mut f, _) => f,
+        }
+    }
+    fn into_file(self) -> File {
+        match self {
+            Self::Created(f) => f,
+            Self::Read(f, _) => f,
+        }
+    }
+}
+
+impl StartStop {
+    fn read_time_file(f: &str, create_new_if_nx: bool) -> SDSSResult<ReadNX> {
+        let mut f = match File::options().write(true).read(true).open(f) {
+            Ok(f) => f,
+            Err(e) if e.kind() == ErrorKind::NotFound && create_new_if_nx => {
+                let f = File::create(f)?;
+                return Ok(ReadNX::Created(f));
+            }
+            Err(e) => return Err(e.into()),
+        };
+        let len = f.metadata().map(|m| m.len())?;
+        if len != sizeof!(u128) as u64 {
+            return Err(SDSSError::corrupted_file(START_FILE));
+        }
+        let mut buf = [0u8; sizeof!(u128)];
+        f.read_exact(&mut buf)?;
+        Ok(ReadNX::Read(f, u128::from_le_bytes(buf)))
+    }
+    pub fn terminate(mut self) -> SDSSResult<()> {
+        self.stop_file
+            .write_all(self.begin.to_le_bytes().as_ref())
+            .map_err(|e| e.with_extra(EMSG_FAILED_WRITE_STOP_FILE))
+    }
+    pub fn verify_and_start() -> SDSSResult<Self> {
+        // read start file
+        let mut start_file = Self::read_time_file(START_FILE, true)
+            .map_err(|e| e.with_ioerror_extra(EMSG_FAILED_OPEN_START_FILE))?;
+        // read stop file
+        let stop_file = Self::read_time_file(STOP_FILE, start_file.created())
+            .map_err(|e| e.with_ioerror_extra(EMSG_FAILED_OPEN_STOP_FILE))?;
+        // read current time
+        let ctime = os::get_epoch_time();
+        match (&start_file, &stop_file) {
+            (ReadNX::Read(_, time_start), ReadNX::Read(_, time_stop))
+                if time_start == time_stop => {}
+            (ReadNX::Created(_), ReadNX::Created(_)) => {}
+            _ => return Err(SDSSError::StartupError(EMSG_FAILED_VERIFY)),
+        }
+        start_file
+            .file_mut()
+            .write_all(&ctime.to_le_bytes())
+            .map_err(|e| e.with_extra(EMSG_FAILED_WRITE_START_FILE))?;
+        Ok(Self {
+            stop_file: stop_file.into_file(),
+            begin: ctime,
+        })
+    }
+}
+
+#[test]
+fn verify_test() {
+    let x = || -> SDSSResult<()> {
+        let ss = StartStop::verify_and_start()?;
+        ss.terminate()?;
+        let ss = StartStop::verify_and_start()?;
+        ss.terminate()?;
+        std::fs::remove_file(START_FILE)?;
+        std::fs::remove_file(STOP_FILE)?;
+        Ok(())
+    };
+    x().unwrap();
+}
diff --git a/server/src/util/macros.rs b/server/src/util/macros.rs
index d23fd723..d5099b2b 100644
--- a/server/src/util/macros.rs
+++ b/server/src/util/macros.rs
@@ -303,3 +303,50 @@ macro_rules! is_64b {
         cfg!(target_pointer_width = "64")
     };
 }
+
+#[macro_export]
+macro_rules! concat_array_to_array {
+    ($a:expr, $b:expr) => {{
+        const BUFFER_A: [u8; $a.len()] = crate::util::copy_slice_to_array($a);
+        const BUFFER_B: [u8; $b.len()] = crate::util::copy_slice_to_array($b);
+        const BUFFER: [u8; BUFFER_A.len() + BUFFER_B.len()] = unsafe {
+            // UNSAFE(@ohsayan): safe because align = 1
+            core::mem::transmute((BUFFER_A, BUFFER_B))
+        };
+        BUFFER
+    }};
+    ($a:expr, $b:expr, $c:expr) => {{
+        const LA: usize = $a.len() + $b.len();
+        const LB: usize = LA + $c.len();
+        const S_1: [u8; LA] = concat_array_to_array!($a, $b);
+        const S_2: [u8; LB] = concat_array_to_array!(&S_1, $c);
+        S_2
+    }};
+}
+
+#[macro_export]
+macro_rules! concat_str_to_array {
+    ($a:expr, $b:expr) => {
+        concat_array_to_array!($a.as_bytes(), $b.as_bytes())
+    };
+    ($a:expr, $b:expr, $c:expr) => {{
+        concat_array_to_array!($a.as_bytes(), $b.as_bytes(), $c.as_bytes())
+    }};
+}
+
+#[macro_export]
+macro_rules! concat_str_to_str {
+    ($a:expr, $b:expr) => {{
+        const BUFFER: [u8; ::core::primitive::str::len($a) + ::core::primitive::str::len($b)] =
+            concat_str_to_array!($a, $b);
+        const STATIC_BUFFER: &[u8] = &BUFFER;
+        unsafe {
+            // UNSAFE(@ohsayan): all good because of restriction to str
+            core::str::from_utf8_unchecked(&STATIC_BUFFER)
+        }
+    }};
+    ($a:expr, $b:expr, $c:expr) => {{
+        const A: &str = concat_str_to_str!($a, $b);
+        concat_str_to_str!(A, $c)
+    }};
+}
diff --git a/server/src/util/os.rs b/server/src/util/os.rs
index 28c2d7fc..c379bbf0 100644
--- a/server/src/util/os.rs
+++ b/server/src/util/os.rs
@@ -384,15 +384,19 @@ mod hostname_impl {
     }
 
     #[cfg(target_family = "windows")]
-    fn get_hostname() -> (usize, [u8; 255]) {
+    fn get_hostname() -> Hostname {
         use winapi::shared::minwindef::DWORD;
-        use winapi::um::sysinfoapi::GetComputerNameA;
+        use winapi::um::sysinfoapi::{self, GetComputerNameExA};
 
         let mut buf: [u8; 256] = [0; 256];
         let mut size: DWORD = buf.len() as u32;
 
         unsafe {
-            GetComputerNameA(buf.as_mut_ptr().cast(), &mut size);
+            GetComputerNameExA(
+                sysinfoapi::ComputerNamePhysicalDnsHostname,
+                buf.as_mut_ptr().cast(),
+                &mut size,
+            );
             Hostname::new_from_raw_buf(&buf)
         }
     }
@@ -401,38 +405,31 @@ mod hostname_impl {
     mod test {
         use std::process::Command;
 
-        #[cfg(target_os = "linux")]
-        fn test_get_hostname() -> Result<String, Box<dyn std::error::Error>> {
-            let output = Command::new("sh")
-                .arg("-c")
-                .arg("hostname")
-                .output()?;
-            let hostname = String::from_utf8_lossy(&output.stdout).trim().to_string();
-            Ok(hostname)
-        }
-        #[cfg(target_os = "windows")]
-        fn test_get_hostname() -> Result<String, Box<dyn std::error::Error>> {
-            let output = Command::new("cmd")
-                .args(&["/C", "hostname"])
-                .output()?;
-            let hostname = String::from_utf8_lossy(&output.stdout).trim().to_string();
-            Ok(hostname)
+        fn test_get_hostname() -> String {
+            let x = if cfg!(target_os = "windows") {
+                // Windows command to get hostname
+                Command::new("cmd")
+                    .args(&["/C", "hostname"])
+                    .output()
+                    .expect("Failed to execute command")
+                    .stdout
+            } else {
+                // Unix command to get hostname
+                Command::new("uname")
+                    .args(&["-n"])
+                    .output()
+                    .expect("Failed to execute command")
+                    .stdout
+            };
+            String::from_utf8_lossy(&x).trim().to_string()
         }
-                
-        #[cfg(target_os = "macos")]
-        fn test_get_hostname() -> Result<String, Box<dyn std::error::Error>> {
-            let output = Command::new("sh")
-                .arg("-c")
-                .arg("scutil --get LocalHostName")
-                .output()?;
-            let hostname = String::from_utf8_lossy(&output.stdout).trim().to_string();
-            Ok(hostname)
-        }
-        
+
         #[test]
         fn t_get_hostname() {
-            let hostname_from_cmd = test_get_hostname().unwrap();
-            assert_eq!(hostname_from_cmd.as_str(), super::Hostname::get().as_str());
+            assert_eq!(
+                test_get_hostname().as_str(),
+                super::Hostname::get().as_str()
+            );
         }
     }
 }

From 9091db6bd33ea91d7c138b6bb003997da640b8d2 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 31 Jul 2023 07:52:12 +0000
Subject: [PATCH 201/310] Impl SDSS Writer

---
 server/src/engine/storage/header.rs           |   4 +-
 .../src/engine/storage/v1/header_impl/dr.rs   |  16 +-
 .../src/engine/storage/v1/header_impl/gr.rs   |   7 +-
 .../src/engine/storage/v1/header_impl/mod.rs  |  32 +++-
 .../src/engine/storage/v1/header_impl/sr.rs   |   3 +-
 server/src/engine/storage/v1/mod.rs           |   7 +
 server/src/engine/storage/v1/rw.rs            | 126 ++++++++++++++-
 server/src/engine/storage/v1/tests.rs         | 151 ++++++++++++++++++
 server/src/engine/storage/v1/txn.rs           |  38 +++++
 server/src/engine/sync/atm.rs                 |   1 +
 server/src/engine/sync/cell.rs                | 106 +++++++++++-
 server/src/engine/sync/mod.rs                 |  33 ++++
 12 files changed, 509 insertions(+), 15 deletions(-)
 create mode 100644 server/src/engine/storage/v1/tests.rs
 create mode 100644 server/src/engine/storage/v1/txn.rs

diff --git a/server/src/engine/storage/header.rs b/server/src/engine/storage/header.rs
index 86accef5..2cf9371e 100644
--- a/server/src/engine/storage/header.rs
+++ b/server/src/engine/storage/header.rs
@@ -264,7 +264,7 @@ impl HostPointerWidth {
     }
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct StaticRecordUV {
     header_version: HeaderVersion,
     ptr_width: HostPointerWidth,
@@ -315,7 +315,7 @@ impl StaticRecordUV {
     }
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 /// The static record
 pub struct StaticRecordUVRaw {
     data: ByteStack<16>,
diff --git a/server/src/engine/storage/v1/header_impl/dr.rs b/server/src/engine/storage/v1/header_impl/dr.rs
index 9b952943..d7d7647e 100644
--- a/server/src/engine/storage/v1/header_impl/dr.rs
+++ b/server/src/engine/storage/v1/header_impl/dr.rs
@@ -48,7 +48,7 @@ use crate::{
     - 1B: OS
 */
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct DRHostSignature {
     server_version: ServerVersion,
     driver_version: DriverVersion,
@@ -146,9 +146,9 @@ impl DRHostSignature {
     }
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct DRHostSignatureRaw {
-    data: ByteStack<24>,
+    pub(super) data: ByteStack<24>,
 }
 
 impl DRHostSignatureRaw {
@@ -258,7 +258,7 @@ impl DRHostSignatureRaw {
     = 296B
 */
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct DRRuntimeSignature {
     modify_count: u64,
     epoch_time: u128,
@@ -336,11 +336,17 @@ impl DRRuntimeSignature {
             self.host_name_raw(),
         )
     }
+    pub fn set_modify_count(&mut self, new: u64) {
+        self.modify_count = new;
+    }
+    pub fn bump_modify_count(&mut self) {
+        self.modify_count += 1;
+    }
 }
 
 #[derive(Debug, PartialEq, Clone)]
 pub struct DRRuntimeSignatureRaw {
-    data: ByteStack<296>,
+    pub(super) data: ByteStack<296>,
 }
 
 impl DRRuntimeSignatureRaw {
diff --git a/server/src/engine/storage/v1/header_impl/gr.rs b/server/src/engine/storage/v1/header_impl/gr.rs
index c40adb9f..072574e5 100644
--- a/server/src/engine/storage/v1/header_impl/gr.rs
+++ b/server/src/engine/storage/v1/header_impl/gr.rs
@@ -46,7 +46,7 @@ use crate::{
     0, 63
 */
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct GRMetadataRecord {
     server_version: ServerVersion,
     driver_version: DriverVersion,
@@ -97,6 +97,7 @@ impl GRMetadataRecord {
     }
 }
 
+#[derive(Clone)]
 pub struct GRMetadataRecordRaw {
     pub(super) data: ByteStack<32>,
 }
@@ -229,7 +230,7 @@ impl GRMetadataRecordRaw {
     = 304B
 */
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct GRHostRecord {
     epoch_time: u128,
     uptime: u128,
@@ -319,7 +320,7 @@ impl GRHostRecord {
     }
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct GRHostRecordRaw {
     pub(super) data: ByteStack<304>,
 }
diff --git a/server/src/engine/storage/v1/header_impl/mod.rs b/server/src/engine/storage/v1/header_impl/mod.rs
index 63f03f77..6b58a93c 100644
--- a/server/src/engine/storage/v1/header_impl/mod.rs
+++ b/server/src/engine/storage/v1/header_impl/mod.rs
@@ -56,6 +56,8 @@
  * Note: The entire part of the header is little endian encoded
 */
 
+use crate::util::copy_slice_to_array as cp;
+
 // (1) sr
 mod sr;
 // (2) gr
@@ -139,7 +141,7 @@ impl HostRunMode {
     }
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct SDSSHeader {
     // static record
     sr: sr::StaticRecord,
@@ -182,6 +184,9 @@ impl SDSSHeader {
     pub fn dr_rs(&self) -> &dr::DRRuntimeSignature {
         &self.dr_rs
     }
+    pub fn dr_rs_mut(&mut self) -> &mut dr::DRRuntimeSignature {
+        &mut self.dr_rs
+    }
     pub fn encoded(&self) -> SDSSHeaderRaw {
         SDSSHeaderRaw::new_full(
             self.sr.encoded(),
@@ -193,6 +198,7 @@ impl SDSSHeader {
     }
 }
 
+#[derive(Clone)]
 pub struct SDSSHeaderRaw {
     sr: sr::StaticRecordRaw,
     gr_0_mdr: gr::GRMetadataRecordRaw,
@@ -202,6 +208,11 @@ pub struct SDSSHeaderRaw {
 }
 
 impl SDSSHeaderRaw {
+    const OFFSET_SR0: usize = 0;
+    const OFFSET_SR1: usize = sizeof!(sr::StaticRecordRaw);
+    const OFFSET_SR2: usize = Self::OFFSET_SR1 + sizeof!(gr::GRMetadataRecordRaw);
+    const OFFSET_SR3: usize = Self::OFFSET_SR2 + sizeof!(gr::GRHostRecordRaw);
+    const OFFSET_SR4: usize = Self::OFFSET_SR3 + sizeof!(dr::DRHostSignatureRaw);
     pub fn new_auto(
         gr_mdr_scope: FileScope,
         gr_mdr_specifier: FileSpecifier,
@@ -269,4 +280,23 @@ impl SDSSHeaderRaw {
             + sizeof!(dr::DRHostSignatureRaw)
             + sizeof!(dr::DRRuntimeSignatureRaw)
     }
+    pub fn array(&self) -> [u8; Self::header_size()] {
+        let mut data = [0u8; Self::header_size()];
+        data[Self::OFFSET_SR0..Self::OFFSET_SR1].copy_from_slice(self.sr.base.get_ref());
+        data[Self::OFFSET_SR1..Self::OFFSET_SR2].copy_from_slice(self.gr_0_mdr.data.slice());
+        data[Self::OFFSET_SR2..Self::OFFSET_SR3].copy_from_slice(self.gr_1_hr.data.slice());
+        data[Self::OFFSET_SR3..Self::OFFSET_SR4].copy_from_slice(self.dr_0_hs.data.slice());
+        data[Self::OFFSET_SR4..].copy_from_slice(self.dr_1_rs.data.slice());
+        data
+    }
+    /// **☢ WARNING ☢: This only decodes; it doesn't validate expected values!**
+    pub fn decode(slice: [u8; Self::header_size()]) -> Option<SDSSHeader> {
+        let sr = sr::StaticRecordRaw::decode(cp(&slice[Self::OFFSET_SR0..Self::OFFSET_SR1]))?;
+        let gr_mdr =
+            gr::GRMetadataRecordRaw::decode(cp(&slice[Self::OFFSET_SR1..Self::OFFSET_SR2]))?;
+        let gr_hr = gr::GRHostRecord::decode(cp(&slice[Self::OFFSET_SR2..Self::OFFSET_SR3]))?;
+        let dr_sig = dr::DRHostSignature::decode(cp(&slice[Self::OFFSET_SR3..Self::OFFSET_SR4]))?;
+        let dr_rt = dr::DRRuntimeSignature::decode(cp(&slice[Self::OFFSET_SR4..]))?;
+        Some(SDSSHeader::new(sr, gr_mdr, gr_hr, dr_sig, dr_rt))
+    }
 }
diff --git a/server/src/engine/storage/v1/header_impl/sr.rs b/server/src/engine/storage/v1/header_impl/sr.rs
index 4400fc41..46645c21 100644
--- a/server/src/engine/storage/v1/header_impl/sr.rs
+++ b/server/src/engine/storage/v1/header_impl/sr.rs
@@ -29,7 +29,7 @@ use crate::engine::storage::{
     versions,
 };
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct StaticRecord {
     sr: StaticRecordUV,
 }
@@ -49,6 +49,7 @@ impl StaticRecord {
 }
 
 /// Static record
+#[derive(Clone)]
 pub struct StaticRecordRaw {
     pub(super) base: StaticRecordUVRaw,
 }
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index 18725f21..f2009547 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -24,9 +24,15 @@
  *
 */
 
+// raw
 mod header_impl;
+// impls
 mod rw;
 mod start_stop;
+mod txn;
+// test
+#[cfg(test)]
+mod tests;
 
 use std::io::Error as IoError;
 
@@ -50,6 +56,7 @@ pub enum SDSSError {
     IoErrorExtra(IoError, &'static str),
     CorruptedFile(&'static str),
     StartupError(&'static str),
+    CorruptedHeader,
 }
 
 impl SDSSError {
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 1dc078d2..6ba25de7 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -1,5 +1,5 @@
 /*
- * Created on Fri May 19 2023
+ * Created on Tue Jul 23 2023
  *
  * This file is a part of Skytable
  * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@@ -23,3 +23,127 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.
  *
 */
+
+use {
+    super::{
+        header_impl::{
+            FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode, SDSSHeader, SDSSHeaderRaw,
+        },
+        SDSSResult,
+    },
+    crate::engine::storage::v1::SDSSError,
+    std::{
+        fs::File,
+        io::{Read, Write},
+    },
+};
+
+#[derive(Debug)]
+/// Log whether
+pub enum FileOpen<F> {
+    Created(F),
+    Existing(F, SDSSHeader),
+}
+
+#[derive(Debug)]
+pub enum RawFileOpen<F> {
+    Created(F),
+    Existing(F),
+}
+
+pub trait RawFileIOInterface: Sized {
+    fn fopen_or_create_rw(file_path: &str) -> SDSSResult<RawFileOpen<Self>>;
+    fn fread_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()>;
+    fn fwrite_all(&mut self, bytes: &[u8]) -> SDSSResult<()>;
+    fn fsync_all(&mut self) -> SDSSResult<()>;
+}
+
+impl RawFileIOInterface for File {
+    fn fopen_or_create_rw(file_path: &str) -> SDSSResult<RawFileOpen<Self>> {
+        let f = File::options()
+            .create(true)
+            .read(true)
+            .write(true)
+            .open(file_path)?;
+        let md = f.metadata()?;
+        if md.created()? == md.modified()? {
+            return Ok(RawFileOpen::Created(f));
+        } else {
+            return Ok(RawFileOpen::Existing(f));
+        }
+    }
+    fn fread_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()> {
+        self.read_exact(buf)?;
+        Ok(())
+    }
+    fn fwrite_all(&mut self, bytes: &[u8]) -> SDSSResult<()> {
+        self.write_all(bytes)?;
+        Ok(())
+    }
+    fn fsync_all(&mut self) -> SDSSResult<()> {
+        self.sync_all()?;
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+pub struct SDSSFileIO<F> {
+    f: F,
+}
+
+impl<F: RawFileIOInterface> SDSSFileIO<F> {
+    pub fn open_or_create_perm_rw(
+        file_path: &str,
+        file_scope: FileScope,
+        file_specifier: FileSpecifier,
+        file_specifier_version: FileSpecifierVersion,
+        host_setting_version: u32,
+        host_run_mode: HostRunMode,
+        host_startup_counter: u64,
+    ) -> SDSSResult<FileOpen<Self>> {
+        let f = F::fopen_or_create_rw(file_path)?;
+        match f {
+            RawFileOpen::Created(f) => {
+                // since this file was just created, we need to append the header
+                let data = SDSSHeaderRaw::new_auto(
+                    file_scope,
+                    file_specifier,
+                    file_specifier_version,
+                    host_setting_version,
+                    host_run_mode,
+                    host_startup_counter,
+                    0,
+                )
+                .array();
+                let mut f = Self::_new(f);
+                f.fsynced_write(&data)?;
+                Ok(FileOpen::Created(f))
+            }
+            RawFileOpen::Existing(mut f) => {
+                // this is an existing file. decoded the header
+                let mut header_raw = [0u8; SDSSHeaderRaw::header_size()];
+                f.fread_exact(&mut header_raw)?;
+                let header = SDSSHeaderRaw::decode(header_raw).ok_or(SDSSError::CorruptedHeader)?;
+                // since we updated this file, let us update the header
+                let mut new_header = header.clone();
+                new_header.dr_rs_mut().bump_modify_count();
+                let mut f = Self::_new(f);
+                f.fsynced_write(new_header.encoded().array().as_ref())?;
+                Ok(FileOpen::Existing(f, header))
+            }
+        }
+    }
+}
+
+impl<F: RawFileIOInterface> SDSSFileIO<F> {
+    fn _new(f: F) -> Self {
+        Self { f }
+    }
+    pub fn fsynced_write(&mut self, data: &[u8]) -> SDSSResult<()> {
+        self.f.fwrite_all(data)?;
+        self.f.fsync_all()
+    }
+    pub fn read_to_buffer(&mut self, buffer: &mut [u8]) -> SDSSResult<()> {
+        self.f.fread_exact(buffer)
+    }
+}
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
new file mode 100644
index 00000000..901be0c7
--- /dev/null
+++ b/server/src/engine/storage/v1/tests.rs
@@ -0,0 +1,151 @@
+/*
+ * Created on Thu Jul 23 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{
+        rw::{RawFileIOInterface, RawFileOpen},
+        SDSSError, SDSSResult,
+    },
+    crate::engine::sync::cell::Lazy,
+    parking_lot::RwLock,
+    std::{
+        collections::{hash_map::Entry, HashMap},
+        io::{ErrorKind, Read, Write},
+    },
+};
+
+static VFS: Lazy<
+    RwLock<HashMap<String, VirtualFile>>,
+    fn() -> RwLock<HashMap<String, VirtualFile>>,
+> = Lazy::new(|| RwLock::new(HashMap::new()));
+
+fn vfs<T>(fname: &str, mut func: impl FnMut(&mut VirtualFile) -> SDSSResult<T>) -> SDSSResult<T> {
+    let mut vfs = VFS.write();
+    let f = vfs
+        .get_mut(fname)
+        .ok_or(SDSSError::from(std::io::Error::from(ErrorKind::NotFound)))?;
+    func(f)
+}
+
+struct VirtualFile {
+    read: bool,
+    write: bool,
+    data: Vec<u8>,
+}
+
+impl VirtualFile {
+    fn new(read: bool, write: bool, data: Vec<u8>) -> Self {
+        Self { read, write, data }
+    }
+    fn rw(data: Vec<u8>) -> Self {
+        Self::new(true, true, data)
+    }
+    fn w(data: Vec<u8>) -> Self {
+        Self::new(false, true, data)
+    }
+    fn r(data: Vec<u8>) -> Self {
+        Self::new(true, false, data)
+    }
+}
+
+struct VirtualFileInterface(Box<str>);
+
+impl RawFileIOInterface for VirtualFileInterface {
+    fn fopen_or_create_rw(file_path: &str) -> SDSSResult<RawFileOpen<Self>> {
+        match VFS.write().entry(file_path.to_owned()) {
+            Entry::Occupied(_) => Ok(RawFileOpen::Existing(Self(file_path.into()))),
+            Entry::Vacant(ve) => {
+                ve.insert(VirtualFile::rw(vec![]));
+                Ok(RawFileOpen::Created(Self(file_path.into())))
+            }
+        }
+    }
+    fn fread_exact(&mut self, buf: &mut [u8]) -> super::SDSSResult<()> {
+        vfs(&self.0, |f| {
+            assert!(f.read);
+            f.data.as_slice().read_exact(buf)?;
+            Ok(())
+        })
+    }
+    fn fwrite_all(&mut self, bytes: &[u8]) -> super::SDSSResult<()> {
+        vfs(&self.0, |f| {
+            assert!(f.write);
+            f.data.write_all(bytes)?;
+            Ok(())
+        })
+    }
+    fn fsync_all(&mut self) -> super::SDSSResult<()> {
+        Ok(())
+    }
+}
+
+mod rw {
+    use {
+        super::VirtualFileInterface,
+        crate::engine::storage::v1::{
+            header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
+            rw::{FileOpen, SDSSFileIO},
+        },
+    };
+
+    #[test]
+    fn create_delete() {
+        let f = SDSSFileIO::<VirtualFileInterface>::open_or_create_perm_rw(
+            "hello_world.db-tlog",
+            FileScope::TransactionLogCompacted,
+            FileSpecifier::GNSTxnLog,
+            FileSpecifierVersion::__new(0),
+            0,
+            HostRunMode::Prod,
+            0,
+        )
+        .unwrap();
+        match f {
+            FileOpen::Existing(_, _) => panic!(),
+            FileOpen::Created(_) => {}
+        };
+        let open = SDSSFileIO::<VirtualFileInterface>::open_or_create_perm_rw(
+            "hello_world.db-tlog",
+            FileScope::TransactionLogCompacted,
+            FileSpecifier::GNSTxnLog,
+            FileSpecifierVersion::__new(0),
+            0,
+            HostRunMode::Prod,
+            0,
+        )
+        .unwrap();
+        let h = match open {
+            FileOpen::Existing(_, header) => header,
+            _ => panic!(),
+        };
+        assert_eq!(h.gr_mdr().file_scope(), FileScope::TransactionLogCompacted);
+        assert_eq!(h.gr_mdr().file_spec(), FileSpecifier::GNSTxnLog);
+        assert_eq!(h.gr_mdr().file_spec_id(), FileSpecifierVersion::__new(0));
+        assert_eq!(h.gr_hr().run_mode(), HostRunMode::Prod);
+        assert_eq!(h.gr_hr().setting_version(), 0);
+        assert_eq!(h.gr_hr().startup_counter(), 0);
+    }
+}
diff --git a/server/src/engine/storage/v1/txn.rs b/server/src/engine/storage/v1/txn.rs
new file mode 100644
index 00000000..8547dfe5
--- /dev/null
+++ b/server/src/engine/storage/v1/txn.rs
@@ -0,0 +1,38 @@
+/*
+ * Created on Thu Jul 23 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/*
+  +----------------+------------------------------+-----------------+------------------+--------------------+
+  | EVENT ID (16B) | EVENT SOURCE + METADATA (8B) | EVENT MD5 (16B)  | PAYLOAD LEN (8B) | EVENT PAYLOAD (?B) |
+  +----------------+------------------------------+-----------------+------------------+--------------------+
+  Event ID:
+  - The atomically incrementing event ID (for future scale we have 16B; it's like the ZFS situation haha)
+  - Event source (1B) + 7B padding (for future metadata)
+  - Event MD5; yeah, it's "not as strong as" SHA256 but I've chosen to have it here (since it's sometimes faster and further on,
+    we already sum the entire log)
+  - Payload len: the size of the pyload
+  - Payload: the payload
+*/
diff --git a/server/src/engine/sync/atm.rs b/server/src/engine/sync/atm.rs
index bffae52a..ad7ac2fa 100644
--- a/server/src/engine/sync/atm.rs
+++ b/server/src/engine/sync/atm.rs
@@ -35,6 +35,7 @@ pub const ORD_RLX: Ordering = Ordering::Relaxed;
 pub const ORD_ACQ: Ordering = Ordering::Acquire;
 pub const ORD_REL: Ordering = Ordering::Release;
 pub const ORD_ACR: Ordering = Ordering::AcqRel;
+pub const ORD_SEQ: Ordering = Ordering::SeqCst;
 
 type CxResult<'g, T, P> = Result<Shared<'g, T>, CompareExchangeError<'g, T, P>>;
 
diff --git a/server/src/engine/sync/cell.rs b/server/src/engine/sync/cell.rs
index 21d722fa..ef2581fa 100644
--- a/server/src/engine/sync/cell.rs
+++ b/server/src/engine/sync/cell.rs
@@ -25,11 +25,113 @@
 */
 
 use {
-    super::atm::{upin, Atomic, Guard, Owned, Shared, ORD_REL},
-    core::{marker::PhantomData, ops::Deref},
+    super::{
+        atm::{upin, Atomic, Guard, Owned, Shared, ORD_ACQ, ORD_REL, ORD_SEQ},
+        Backoff,
+    },
+    core::{
+        marker::PhantomData,
+        mem,
+        ops::Deref,
+        ptr,
+        sync::atomic::{AtomicBool, AtomicPtr},
+    },
     parking_lot::{Mutex, MutexGuard},
 };
 
+/// A lazily intialized, or _call by need_ value
+#[derive(Debug)]
+pub struct Lazy<T, F> {
+    /// the value (null at first)
+    value: AtomicPtr<T>,
+    /// the function that will init the value
+    init_func: F,
+    /// is some thread trying to initialize the value
+    init_state: AtomicBool,
+}
+
+impl<T, F> Lazy<T, F> {
+    pub const fn new(init_func: F) -> Self {
+        Self {
+            value: AtomicPtr::new(ptr::null_mut()),
+            init_func,
+            init_state: AtomicBool::new(false),
+        }
+    }
+}
+
+impl<T, F> Deref for Lazy<T, F>
+where
+    F: Fn() -> T,
+{
+    type Target = T;
+    fn deref(&self) -> &Self::Target {
+        let value_ptr = self.value.load(ORD_ACQ);
+        if !value_ptr.is_null() {
+            // the value has already been initialized, return
+            unsafe {
+                // UNSAFE(@ohsayan): We've just asserted that the value is not null
+                return &*value_ptr;
+            }
+        }
+        // it's null, so it's useless
+
+        // hold on until someone is trying to init
+        let backoff = Backoff::new();
+        while self
+            .init_state
+            .compare_exchange(false, true, ORD_SEQ, ORD_SEQ)
+            .is_err()
+        {
+            // wait until the other thread finishes
+            backoff.snooze();
+        }
+        /*
+         see the value before the last store. while we were one the loop,
+         some other thread could have initialized it already
+        */
+        let value_ptr = self.value.load(ORD_ACQ);
+        if !value_ptr.is_null() {
+            // no more init, someone initialized it already
+            assert!(self.init_state.swap(false, ORD_SEQ));
+            unsafe {
+                // UNSAFE(@ohsayan): We've already loaded the value checked
+                // that it isn't null
+                &*value_ptr
+            }
+        } else {
+            // so no one cared to initialize the value in between
+            // fine, we'll init it
+            let value = (self.init_func)();
+            let value_ptr = Box::into_raw(Box::new(value));
+            // now swap out the older value and check it for sanity
+            assert!(self.value.swap(value_ptr, ORD_SEQ).is_null());
+            // set trying to init flag to false
+            assert!(self.init_state.swap(false, ORD_SEQ));
+            unsafe {
+                // UNSAFE(@ohsayan): We just initialized the value ourselves
+                // so it is not null!
+                &*value_ptr
+            }
+        }
+    }
+}
+
+impl<T, F> Drop for Lazy<T, F> {
+    fn drop(&mut self) {
+        if mem::needs_drop::<T>() {
+            // this needs drop
+            let value_ptr = self.value.load(ORD_ACQ);
+            if !value_ptr.is_null() {
+                unsafe {
+                    // UNSAFE(@ohsayan): We've just checked if the value is null or not
+                    mem::drop(Box::from_raw(value_ptr))
+                }
+            }
+        }
+    }
+}
+
 /// A [`TMCell`] provides atomic reads and serialized writes; the `static` is a CB hack
 #[derive(Debug)]
 pub struct TMCell<T: 'static> {
diff --git a/server/src/engine/sync/mod.rs b/server/src/engine/sync/mod.rs
index 7670d162..fc18e702 100644
--- a/server/src/engine/sync/mod.rs
+++ b/server/src/engine/sync/mod.rs
@@ -27,3 +27,36 @@
 pub(super) mod atm;
 pub(super) mod cell;
 pub(super) mod smart;
+
+use std::{cell::Cell, hint::spin_loop, thread};
+
+/// Type to perform exponential backoff
+pub struct Backoff {
+    cur: Cell<u8>,
+}
+
+impl Backoff {
+    const MAX_SPIN: u8 = 6;
+    const MAX_YIELD: u8 = 8;
+    pub fn new() -> Self {
+        Self { cur: Cell::new(0) }
+    }
+    /// Spin a few times, giving way to the CPU but if we have spun too many times,
+    /// then block by yielding to the OS scheduler. This will **eventually block**
+    /// if we spin more than the set `MAX_SPIN`
+    pub fn snooze(&self) {
+        if self.cur.get() <= Self::MAX_SPIN {
+            // we can still spin (exp)
+            for _ in 0..1 << self.cur.get() {
+                spin_loop();
+            }
+        } else {
+            // nope, yield to scheduler
+            thread::yield_now();
+        }
+        if self.cur.get() <= Self::MAX_YIELD {
+            // bump current step
+            self.cur.set(self.cur.get() + 1)
+        }
+    }
+}

From 55f53456f892d3d62a4aaaadbfc8f3fac34f4f1d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 31 Jul 2023 07:52:21 +0000
Subject: [PATCH 202/310] Implement transaction reader

---
 Cargo.lock                            |  16 ++
 server/Cargo.toml                     |   1 +
 server/src/engine/storage/v1/mod.rs   |   2 +
 server/src/engine/storage/v1/rw.rs    |   7 +
 server/src/engine/storage/v1/tests.rs |   3 +
 server/src/engine/storage/v1/txn.rs   | 207 +++++++++++++++++++++++++-
 server/src/util/mod.rs                |  16 ++
 7 files changed, 247 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2b76bbcf..6e7ef45e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -303,6 +303,21 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crc"
+version = "3.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe"
+dependencies = [
+ "crc-catalog",
+]
+
+[[package]]
+name = "crc-catalog"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484"
+
 [[package]]
 name = "crc32fast"
 version = "1.3.2"
@@ -1394,6 +1409,7 @@ dependencies = [
  "cc",
  "chrono",
  "clap",
+ "crc",
  "crossbeam-epoch",
  "env_logger",
  "hashbrown",
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 45b26e9a..13db3206 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -28,6 +28,7 @@ tokio-openssl = "0.6.3"
 toml = "0.5.9"
 base64 = "0.13.0"
 uuid = { version = "1.2.2", features = ["v4", "fast-rng", "macro-diagnostics"] }
+crc = "3.0.1"
 
 [target.'cfg(all(not(target_env = "msvc"), not(miri)))'.dependencies]
 # external deps
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index f2009547..d4b14534 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -57,6 +57,8 @@ pub enum SDSSError {
     CorruptedFile(&'static str),
     StartupError(&'static str),
     CorruptedHeader,
+    TransactionLogEntryCorrupted,
+    TransactionLogCorrupted,
 }
 
 impl SDSSError {
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 6ba25de7..527e7e11 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -56,6 +56,7 @@ pub trait RawFileIOInterface: Sized {
     fn fread_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()>;
     fn fwrite_all(&mut self, bytes: &[u8]) -> SDSSResult<()>;
     fn fsync_all(&mut self) -> SDSSResult<()>;
+    fn flen(&self) -> SDSSResult<u64>;
 }
 
 impl RawFileIOInterface for File {
@@ -84,6 +85,9 @@ impl RawFileIOInterface for File {
         self.sync_all()?;
         Ok(())
     }
+    fn flen(&self) -> SDSSResult<u64> {
+        Ok(self.metadata()?.len())
+    }
 }
 
 #[derive(Debug)]
@@ -146,4 +150,7 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
     pub fn read_to_buffer(&mut self, buffer: &mut [u8]) -> SDSSResult<()> {
         self.f.fread_exact(buffer)
     }
+    pub fn file_length(&self) -> SDSSResult<u64> {
+        self.f.flen()
+    }
 }
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index 901be0c7..998b352f 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -100,6 +100,9 @@ impl RawFileIOInterface for VirtualFileInterface {
     fn fsync_all(&mut self) -> super::SDSSResult<()> {
         Ok(())
     }
+    fn flen(&self) -> SDSSResult<u64> {
+        vfs(&self.0, |f| Ok(f.data.len() as _))
+    }
 }
 
 mod rw {
diff --git a/server/src/engine/storage/v1/txn.rs b/server/src/engine/storage/v1/txn.rs
index 8547dfe5..7c812f4e 100644
--- a/server/src/engine/storage/v1/txn.rs
+++ b/server/src/engine/storage/v1/txn.rs
@@ -25,14 +25,211 @@
 */
 
 /*
-  +----------------+------------------------------+-----------------+------------------+--------------------+
-  | EVENT ID (16B) | EVENT SOURCE + METADATA (8B) | EVENT MD5 (16B)  | PAYLOAD LEN (8B) | EVENT PAYLOAD (?B) |
-  +----------------+------------------------------+-----------------+------------------+--------------------+
+  +----------------+------------------------------+------------------+------------------+--------------------+
+  | EVENT ID (16B) | EVENT SOURCE + METADATA (8B) | EVENT CRC32 (4B) | PAYLOAD LEN (8B) | EVENT PAYLOAD (?B) |
+  +----------------+------------------------------+------------------+------------------+--------------------+
   Event ID:
   - The atomically incrementing event ID (for future scale we have 16B; it's like the ZFS situation haha)
   - Event source (1B) + 7B padding (for future metadata)
-  - Event MD5; yeah, it's "not as strong as" SHA256 but I've chosen to have it here (since it's sometimes faster and further on,
-    we already sum the entire log)
+  - Event CRC32
   - Payload len: the size of the pyload
   - Payload: the payload
 */
+
+use {
+    super::{
+        rw::{RawFileIOInterface, SDSSFileIO},
+        SDSSError, SDSSResult,
+    },
+    crate::util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy},
+    std::marker::PhantomData,
+};
+
+/// The transaction adapter
+pub trait TransactionLogAdapter {
+    /// The transaction event
+    type TransactionEvent;
+    /// The global state, which we want to modify on decoding the event
+    type GlobalState;
+    /// Encode a transaction event into a blob
+    fn encode(event: Self::TransactionEvent) -> Box<[u8]>;
+    /// Decode a transaction event and apply it to the global state
+    fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> SDSSResult<()>;
+}
+
+pub struct TxnLogEntryMetadata {
+    event_id: u128,
+    event_source_md: u64,
+    event_crc: u32,
+    event_payload_len: u64,
+}
+
+impl TxnLogEntryMetadata {
+    const SIZE: usize = sizeof!(u128) + sizeof!(u64) + sizeof!(u32) + sizeof!(u64);
+    const P0: usize = 0;
+    const P1: usize = sizeof!(u128);
+    const P2: usize = Self::P1 + sizeof!(u64);
+    const P3: usize = Self::P2 + sizeof!(u32);
+    pub const fn new(
+        event_id: u128,
+        event_source_md: u64,
+        event_crc: u32,
+        event_payload_len: u64,
+    ) -> Self {
+        Self {
+            event_id,
+            event_source_md,
+            event_crc,
+            event_payload_len,
+        }
+    }
+    /// Encodes the log entry metadata
+    pub const fn encoded(&self) -> [u8; TxnLogEntryMetadata::SIZE] {
+        let mut encoded = [0u8; TxnLogEntryMetadata::SIZE];
+        encoded = copy_a_into_b(self.event_id.to_le_bytes(), encoded, Self::P0);
+        encoded = copy_a_into_b(self.event_source_md.to_le_bytes(), encoded, Self::P1);
+        encoded = copy_a_into_b(self.event_crc.to_le_bytes(), encoded, Self::P2);
+        encoded = copy_a_into_b(self.event_payload_len.to_le_bytes(), encoded, Self::P3);
+        encoded
+    }
+    /// Decodes the log entry metadata (essentially a simply type transmutation)
+    pub fn decode(data: [u8; TxnLogEntryMetadata::SIZE]) -> Self {
+        Self::new(
+            u128::from_le_bytes(memcpy(&data[..Self::P1])),
+            u64::from_le_bytes(memcpy(&data[Self::P1..Self::P2])),
+            u32::from_le_bytes(memcpy(&data[Self::P2..Self::P3])),
+            u64::from_le_bytes(memcpy(&data[Self::P3..])),
+        )
+    }
+}
+
+/*
+    Event source:
+    * * * * _ * * * *
+
+    b1 (s+d): event source (unset -> driver, set -> server)
+    b* -> unused. MUST be unset
+    b8 (d):
+        - unset: closed log
+*/
+pub enum EventSourceMarker {
+    ServerStandard,
+    DriverClosed,
+}
+
+impl EventSourceMarker {
+    const SERVER_STD: u64 = 1 << 63;
+    const DRIVER_CLOSED: u64 = 0;
+}
+
+impl TxnLogEntryMetadata {
+    pub const fn is_server_event(&self) -> bool {
+        self.event_source_md == EventSourceMarker::SERVER_STD
+    }
+    pub const fn is_driver_event(&self) -> bool {
+        self.event_source_md <= 1
+    }
+    pub const fn event_source_marker(&self) -> Option<EventSourceMarker> {
+        Some(match self.event_source_md {
+            EventSourceMarker::DRIVER_CLOSED => EventSourceMarker::DriverClosed,
+            EventSourceMarker::SERVER_STD => EventSourceMarker::ServerStandard,
+            _ => return None,
+        })
+    }
+}
+
+#[derive(Debug)]
+pub struct TransactionLogReader<TA, LF> {
+    log_file: SDSSFileIO<LF>,
+    evid: u64,
+    closed: bool,
+    remaining_bytes: u64,
+    _m: PhantomData<TA>,
+}
+
+impl<TA: TransactionLogAdapter, LF: RawFileIOInterface> TransactionLogReader<TA, LF> {
+    pub fn new(log_file: SDSSFileIO<LF>) -> SDSSResult<Self> {
+        let log_size = log_file.file_length()?;
+        Ok(Self {
+            log_file,
+            evid: 0,
+            closed: false,
+            remaining_bytes: log_size,
+            _m: PhantomData,
+        })
+    }
+    /// Read the next event and apply it to the global state
+    pub fn rapply_next_event(&mut self, gs: &TA::GlobalState) -> SDSSResult<()> {
+        self._incr_evid();
+        // read metadata
+        let mut raw_txn_log_row_md = [0u8; TxnLogEntryMetadata::SIZE];
+        self.log_file.read_to_buffer(&mut raw_txn_log_row_md)?;
+        let event_metadata = TxnLogEntryMetadata::decode(raw_txn_log_row_md);
+        /*
+            verify metadata and read bytes into buffer, verify sum
+        */
+        // verify md
+        let event_src_marker = event_metadata.event_source_marker();
+        let okay = (self.evid == (event_metadata.event_id as _))
+            & event_src_marker.is_some()
+            & (event_metadata.event_payload_len < (isize::MAX as u64))
+            & self.has_remaining_bytes(event_metadata.event_payload_len);
+        if compiler::unlikely(!okay) {
+            return Err(SDSSError::TransactionLogEntryCorrupted);
+        }
+        let event_is_zero =
+            (event_metadata.event_crc == 0) & (event_metadata.event_payload_len == 0);
+        let event_src_marker = event_src_marker.unwrap();
+        match event_src_marker {
+            EventSourceMarker::ServerStandard => {}
+            EventSourceMarker::DriverClosed if event_is_zero => {
+                // expect last entry
+                if self.end_of_file() {
+                    self.closed = true;
+                    // good
+                    return Ok(());
+                } else {
+                    return Err(SDSSError::TransactionLogCorrupted);
+                }
+            }
+            _ => return Err(SDSSError::TransactionLogEntryCorrupted),
+        }
+        // read bytes
+        let mut payload_data_block = vec![0u8; event_metadata.event_payload_len as usize];
+        self.log_file.read_to_buffer(&mut payload_data_block)?;
+        self._record_bytes_read(event_metadata.event_payload_len as _);
+        // verify sum
+        const CRC: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
+        let actual_sum = CRC.checksum(&payload_data_block);
+        if compiler::likely(actual_sum == event_metadata.event_crc) {
+            // great, the sums match
+            TA::decode_and_update_state(&payload_data_block, gs)?;
+            Ok(())
+        } else {
+            Err(SDSSError::TransactionLogEntryCorrupted)
+        }
+    }
+    /// Read and apply all events in the given log file to the global state, returning the open file
+    pub fn scroll(file: SDSSFileIO<LF>, gs: &TA::GlobalState) -> SDSSResult<SDSSFileIO<LF>> {
+        let mut slf = Self::new(file)?;
+        while !slf.end_of_file() {
+            slf.rapply_next_event(gs)?;
+        }
+        Ok(slf.log_file)
+    }
+}
+
+impl<TA, LF> TransactionLogReader<TA, LF> {
+    fn _incr_evid(&mut self) {
+        self.evid += 1;
+    }
+    fn _record_bytes_read(&mut self, cnt: usize) {
+        self.remaining_bytes -= cnt as u64;
+    }
+    fn has_remaining_bytes(&self, size: u64) -> bool {
+        self.remaining_bytes >= size
+    }
+    fn end_of_file(&self) -> bool {
+        self.remaining_bytes == 0
+    }
+}
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index 19a1b280..98783b33 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -376,3 +376,19 @@ pub const fn copy_slice_to_array<const N: usize>(bytes: &[u8]) -> [u8; N] {
 pub const fn copy_str_to_array<const N: usize>(str: &str) -> [u8; N] {
     copy_slice_to_array(str.as_bytes())
 }
+/// Copy the elements of a into b, beginning the copy at `pos`
+pub const fn copy_a_into_b<const M: usize, const N: usize>(
+    a: [u8; M],
+    mut b: [u8; N],
+    mut pos: usize,
+) -> [u8; N] {
+    assert!(M <= N);
+    assert!(pos < N);
+    let mut i = 0;
+    while i < M {
+        b[pos] = a[pos];
+        i += 1;
+        pos += 1;
+    }
+    b
+}

From 843ff05d856beca4b6ea7ba6d1e741658849d683 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 31 Jul 2023 07:52:24 +0000
Subject: [PATCH 203/310] Impl txn log writer

---
 server/src/engine/storage/v1/rw.rs    | 17 ++++++++-
 server/src/engine/storage/v1/tests.rs | 28 ++++++++++++--
 server/src/engine/storage/v1/txn.rs   | 55 ++++++++++++++++++++++++++-
 3 files changed, 95 insertions(+), 5 deletions(-)

diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 527e7e11..954bab6d 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -34,7 +34,7 @@ use {
     crate::engine::storage::v1::SDSSError,
     std::{
         fs::File,
-        io::{Read, Write},
+        io::{Read, Seek, SeekFrom, Write},
     },
 };
 
@@ -56,6 +56,7 @@ pub trait RawFileIOInterface: Sized {
     fn fread_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()>;
     fn fwrite_all(&mut self, bytes: &[u8]) -> SDSSResult<()>;
     fn fsync_all(&mut self) -> SDSSResult<()>;
+    fn fseek_ahead(&mut self, by: usize) -> SDSSResult<()>;
     fn flen(&self) -> SDSSResult<u64>;
 }
 
@@ -88,6 +89,10 @@ impl RawFileIOInterface for File {
     fn flen(&self) -> SDSSResult<u64> {
         Ok(self.metadata()?.len())
     }
+    fn fseek_ahead(&mut self, by: usize) -> SDSSResult<()> {
+        self.seek(SeekFrom::Start(by as _))?;
+        Ok(())
+    }
 }
 
 #[derive(Debug)]
@@ -143,6 +148,13 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
     fn _new(f: F) -> Self {
         Self { f }
     }
+    pub fn unfsynced_write(&mut self, data: &[u8]) -> SDSSResult<()> {
+        self.f.fwrite_all(data)
+    }
+    pub fn fsync_all(&mut self) -> SDSSResult<()> {
+        self.f.fsync_all()?;
+        Ok(())
+    }
     pub fn fsynced_write(&mut self, data: &[u8]) -> SDSSResult<()> {
         self.f.fwrite_all(data)?;
         self.f.fsync_all()
@@ -153,4 +165,7 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
     pub fn file_length(&self) -> SDSSResult<u64> {
         self.f.flen()
     }
+    pub fn seek_ahead(&mut self, by: usize) -> SDSSResult<()> {
+        self.f.fseek_ahead(by)
+    }
 }
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index 998b352f..3754bc8a 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -51,6 +51,7 @@ fn vfs<T>(fname: &str, mut func: impl FnMut(&mut VirtualFile) -> SDSSResult<T>)
 }
 
 struct VirtualFile {
+    pos: u64,
     read: bool,
     write: bool,
     data: Vec<u8>,
@@ -58,7 +59,12 @@ struct VirtualFile {
 
 impl VirtualFile {
     fn new(read: bool, write: bool, data: Vec<u8>) -> Self {
-        Self { read, write, data }
+        Self {
+            read,
+            write,
+            data,
+            pos: 0,
+        }
     }
     fn rw(data: Vec<u8>) -> Self {
         Self::new(true, true, data)
@@ -69,6 +75,16 @@ impl VirtualFile {
     fn r(data: Vec<u8>) -> Self {
         Self::new(true, false, data)
     }
+    fn seek_forward(&mut self, by: usize) {
+        self.pos += by as u64;
+        assert!(self.pos <= self.data.len() as u64);
+    }
+    fn data(&self) -> &[u8] {
+        &self.data[self.pos as usize..]
+    }
+    fn data_mut(&mut self) -> &mut [u8] {
+        &mut self.data[self.pos as usize..]
+    }
 }
 
 struct VirtualFileInterface(Box<str>);
@@ -86,14 +102,14 @@ impl RawFileIOInterface for VirtualFileInterface {
     fn fread_exact(&mut self, buf: &mut [u8]) -> super::SDSSResult<()> {
         vfs(&self.0, |f| {
             assert!(f.read);
-            f.data.as_slice().read_exact(buf)?;
+            f.data().read_exact(buf)?;
             Ok(())
         })
     }
     fn fwrite_all(&mut self, bytes: &[u8]) -> super::SDSSResult<()> {
         vfs(&self.0, |f| {
             assert!(f.write);
-            f.data.write_all(bytes)?;
+            f.data_mut().write_all(bytes)?;
             Ok(())
         })
     }
@@ -103,6 +119,12 @@ impl RawFileIOInterface for VirtualFileInterface {
     fn flen(&self) -> SDSSResult<u64> {
         vfs(&self.0, |f| Ok(f.data.len() as _))
     }
+    fn fseek_ahead(&mut self, by: usize) -> SDSSResult<()> {
+        vfs(&self.0, |f| {
+            f.seek_forward(by);
+            Ok(())
+        })
+    }
 }
 
 mod rw {
diff --git a/server/src/engine/storage/v1/txn.rs b/server/src/engine/storage/v1/txn.rs
index 7c812f4e..e1b90fd7 100644
--- a/server/src/engine/storage/v1/txn.rs
+++ b/server/src/engine/storage/v1/txn.rs
@@ -45,6 +45,8 @@ use {
     std::marker::PhantomData,
 };
 
+const CRC: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
+
 /// The transaction adapter
 pub trait TransactionLogAdapter {
     /// The transaction event
@@ -199,7 +201,6 @@ impl<TA: TransactionLogAdapter, LF: RawFileIOInterface> TransactionLogReader<TA,
         self.log_file.read_to_buffer(&mut payload_data_block)?;
         self._record_bytes_read(event_metadata.event_payload_len as _);
         // verify sum
-        const CRC: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
         let actual_sum = CRC.checksum(&payload_data_block);
         if compiler::likely(actual_sum == event_metadata.event_crc) {
             // great, the sums match
@@ -233,3 +234,55 @@ impl<TA, LF> TransactionLogReader<TA, LF> {
         self.remaining_bytes == 0
     }
 }
+
+pub struct TransactionLogWriter<LF, TA> {
+    /// the txn log file
+    log_file: SDSSFileIO<LF>,
+    /// the id of the **next** transaction
+    id: u64,
+    _m: PhantomData<TA>,
+}
+
+impl<LF: RawFileIOInterface, TA: TransactionLogAdapter> TransactionLogWriter<LF, TA> {
+    pub fn new(
+        mut log_file: SDSSFileIO<LF>,
+        last_txn_id: u64,
+        last_size: usize,
+    ) -> SDSSResult<Self> {
+        log_file.seek_ahead(last_size)?;
+        Ok(Self {
+            log_file,
+            id: last_txn_id,
+            _m: PhantomData,
+        })
+    }
+    pub fn append_event(&mut self, event: TA::TransactionEvent) -> SDSSResult<()> {
+        let encoded = TA::encode(event);
+        let md = TxnLogEntryMetadata::new(
+            self._incr_id() as u128,
+            EventSourceMarker::SERVER_STD,
+            CRC.checksum(&encoded),
+            encoded.len() as u64,
+        )
+        .encoded();
+        self.log_file.unfsynced_write(&md)?;
+        self.log_file.unfsynced_write(&encoded)?;
+        self.log_file.fsync_all()?;
+        Ok(())
+    }
+    pub fn close_log(mut self) -> SDSSResult<()> {
+        let crc = CRC.checksum(EventSourceMarker::DRIVER_CLOSED.to_le_bytes().as_ref());
+        let id = self._incr_id() as u128;
+        self.log_file.fsynced_write(
+            &TxnLogEntryMetadata::new(id, EventSourceMarker::DRIVER_CLOSED, crc, 0).encoded(),
+        )
+    }
+}
+
+impl<LF, TA> TransactionLogWriter<LF, TA> {
+    fn _incr_id(&mut self) -> u64 {
+        let current = self.id;
+        self.id += 1;
+        current
+    }
+}

From b0ef30853c91e27185159ff8c0484ea415ce8616 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 31 Jul 2023 07:52:27 +0000
Subject: [PATCH 204/310] Upgrade deps

---
 Cargo.lock               | 528 +++++++++++++++++++++++++++------------
 cli/Cargo.toml           |   6 +-
 harness/Cargo.toml       |   8 +-
 libstress/Cargo.toml     |   4 +-
 server/Cargo.toml        |  30 +--
 server/src/auth/keys.rs  |   8 +-
 server/src/config/mod.rs |   4 +-
 sky-bench/Cargo.toml     |   6 +-
 sky-macros/Cargo.toml    |   6 +-
 sky-migrate/Cargo.toml   |   2 +-
 stress-test/Cargo.toml   |   6 +-
 11 files changed, 413 insertions(+), 195 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 6e7ef45e..7e22d582 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,15 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "addr2line"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
+dependencies = [
+ "gimli",
+]
+
 [[package]]
 name = "adler"
 version = "1.0.2"
@@ -10,32 +19,20 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
 [[package]]
 name = "aes"
-version = "0.7.5"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8"
+checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2"
 dependencies = [
  "cfg-if",
- "cipher 0.3.0",
+ "cipher",
  "cpufeatures",
- "opaque-debug",
-]
-
-[[package]]
-name = "ahash"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
-dependencies = [
- "getrandom",
- "once_cell",
- "version_check",
 ]
 
 [[package]]
 name = "ahash"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf6ccdb167abbf410dcb915cabd428929d7f6a04980b54a11f26a39f1c7f7107"
+checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
 dependencies = [
  "cfg-if",
  "getrandom",
@@ -52,6 +49,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "android-tzdata"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
+
 [[package]]
 name = "android_system_properties"
 version = "0.1.5"
@@ -78,7 +81,7 @@ checksum = "689894c2db1ea643a50834b999abf1c110887402542955ff5451dab8f861f9ed"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -98,12 +101,33 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "backtrace"
+version = "0.3.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if",
+ "libc",
+ "miniz_oxide 0.7.1",
+ "object",
+ "rustc-demangle",
+]
+
 [[package]]
 name = "base64"
 version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
+[[package]]
+name = "base64"
+version = "0.21.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
+
 [[package]]
 name = "base64ct"
 version = "1.5.3"
@@ -138,6 +162,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitflags"
+version = "2.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
+
 [[package]]
 name = "block-buffer"
 version = "0.10.3"
@@ -154,7 +184,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e412e2cd0f2b2d93e02543ceae7917b3c70331573df19ee046bcbc35e45e87d7"
 dependencies = [
  "byteorder",
- "cipher 0.4.3",
+ "cipher",
 ]
 
 [[package]]
@@ -171,9 +201,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
 name = "bytes"
-version = "1.3.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c"
+checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
 
 [[package]]
 name = "bzip2"
@@ -198,9 +228,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.78"
+version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
 dependencies = [
  "jobserver",
 ]
@@ -213,28 +243,19 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.23"
+version = "0.4.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
+checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5"
 dependencies = [
+ "android-tzdata",
  "iana-time-zone",
  "js-sys",
- "num-integer",
  "num-traits",
  "time 0.1.45",
  "wasm-bindgen",
  "winapi",
 ]
 
-[[package]]
-name = "cipher"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7"
-dependencies = [
- "generic-array",
-]
-
 [[package]]
 name = "cipher"
 version = "0.4.3"
@@ -253,7 +274,7 @@ checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
 dependencies = [
  "ansi_term",
  "atty",
- "bitflags",
+ "bitflags 1.3.2",
  "strsim",
  "textwrap",
  "unicode-width",
@@ -329,9 +350,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.6"
+version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
+checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
 dependencies = [
  "cfg-if",
  "crossbeam-utils",
@@ -350,9 +371,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.13"
+version = "0.9.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
+checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
 dependencies = [
  "autocfg",
  "cfg-if",
@@ -372,11 +393,11 @@ dependencies = [
 
 [[package]]
 name = "crossterm"
-version = "0.25.0"
+version = "0.26.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67"
+checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "crossterm_winapi",
  "libc",
  "mio",
@@ -429,7 +450,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "scratch",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -446,7 +467,7 @@ checksum = "65e07508b90551e610910fa648a1878991d367064997a596135b86df30daf07e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -501,17 +522,23 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
 
 [[package]]
 name = "env_logger"
-version = "0.9.3"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7"
+checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"
 dependencies = [
- "atty",
  "humantime",
+ "is-terminal",
  "log",
  "regex",
  "termcolor",
 ]
 
+[[package]]
+name = "equivalent"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1"
+
 [[package]]
 name = "errno"
 version = "0.2.8"
@@ -523,6 +550,17 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "errno"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "errno-dragonfly"
 version = "0.1.2"
@@ -550,8 +588,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bb21c69b9fea5e15dbc1049e4b77145dd0ba1c84019c488102de0dc4ea4b0a27"
 dependencies = [
  "cfg-if",
- "rustix",
- "windows-sys",
+ "rustix 0.36.7",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]
@@ -561,7 +599,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841"
 dependencies = [
  "crc32fast",
- "miniz_oxide",
+ "miniz_oxide 0.6.2",
 ]
 
 [[package]]
@@ -641,6 +679,12 @@ dependencies = [
  "wasi 0.11.0+wasi-snapshot-preview1",
 ]
 
+[[package]]
+name = "gimli"
+version = "0.27.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
+
 [[package]]
 name = "harness"
 version = "0.1.0"
@@ -656,13 +700,19 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.12.3"
+version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
 dependencies = [
- "ahash 0.7.6",
+ "ahash",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.19"
@@ -681,6 +731,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "hermit-abi"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
+
 [[package]]
 name = "hmac"
 version = "0.12.1"
@@ -720,6 +776,16 @@ dependencies = [
  "cxx-build",
 ]
 
+[[package]]
+name = "indexmap"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.0",
+]
+
 [[package]]
 name = "inout"
 version = "0.1.3"
@@ -736,7 +802,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e"
 dependencies = [
  "libc",
- "windows-sys",
+ "windows-sys 0.42.0",
+]
+
+[[package]]
+name = "is-terminal"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
+dependencies = [
+ "hermit-abi 0.3.2",
+ "rustix 0.38.3",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -792,9 +869,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.139"
+version = "0.2.147"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
+checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
 
 [[package]]
 name = "libsky"
@@ -825,6 +902,12 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
+
 [[package]]
 name = "lock_api"
 version = "0.4.9"
@@ -852,9 +935,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
 [[package]]
 name = "memoffset"
-version = "0.7.1"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
 dependencies = [
  "autocfg",
 ]
@@ -868,16 +951,25 @@ dependencies = [
  "adler",
 ]
 
+[[package]]
+name = "miniz_oxide"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
+dependencies = [
+ "adler",
+]
+
 [[package]]
 name = "mio"
-version = "0.8.5"
+version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
+checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"
 dependencies = [
  "libc",
  "log",
  "wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -896,7 +988,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4"
 dependencies = [
  "autocfg",
- "bitflags",
+ "bitflags 1.3.2",
  "cfg-if",
  "libc",
 ]
@@ -910,16 +1002,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "num-integer"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
-dependencies = [
- "autocfg",
- "num-traits",
-]
-
 [[package]]
 name = "num-traits"
 version = "0.2.15"
@@ -940,24 +1022,27 @@ dependencies = [
 ]
 
 [[package]]
-name = "once_cell"
-version = "1.17.0"
+name = "object"
+version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
+checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
+dependencies = [
+ "memchr",
+]
 
 [[package]]
-name = "opaque-debug"
-version = "0.3.0"
+name = "once_cell"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
+checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
 
 [[package]]
 name = "openssl"
-version = "0.10.45"
+version = "0.10.55"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1"
+checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "cfg-if",
  "foreign-types",
  "libc",
@@ -974,7 +1059,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -988,11 +1073,10 @@ dependencies = [
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.80"
+version = "0.9.90"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7"
+checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6"
 dependencies = [
- "autocfg",
  "cc",
  "libc",
  "openssl-src",
@@ -1020,7 +1104,7 @@ dependencies = [
  "libc",
  "redox_syscall",
  "smallvec",
- "windows-sys",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]
@@ -1066,9 +1150,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
 
 [[package]]
 name = "powershell_script"
-version = "1.0.4"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54bde2e1a395c0aee9423072d781610da37b7b120edf17d4da99f83d04f2cd54"
+checksum = "bef8336090917f3d3a044256bc0e5c51d5420e5d09dfa1df4868083c5231a454"
 
 [[package]]
 name = "ppv-lite86"
@@ -1078,18 +1162,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.50"
+version = "1.0.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2"
+checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.23"
+version = "1.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
+checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
 dependencies = [
  "proc-macro2",
 ]
@@ -1147,9 +1231,9 @@ dependencies = [
 
 [[package]]
 name = "rayon"
-version = "1.6.1"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7"
+checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
 dependencies = [
  "either",
  "rayon-core",
@@ -1157,9 +1241,9 @@ dependencies = [
 
 [[package]]
 name = "rayon-core"
-version = "1.10.1"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"
+checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
 dependencies = [
  "crossbeam-channel",
  "crossbeam-deque",
@@ -1173,7 +1257,7 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c70c3bf2f0b523cc6f2420446e6fca7e14db888cbec907991df043a4d6d2e1e"
 dependencies = [
- "base64",
+ "base64 0.13.1",
  "blowfish",
  "getrandom",
 ]
@@ -1184,7 +1268,7 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]
 
 [[package]]
@@ -1215,18 +1299,37 @@ version = "0.6.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
 
+[[package]]
+name = "rustc-demangle"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
+
 [[package]]
 name = "rustix"
 version = "0.36.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03"
 dependencies = [
- "bitflags",
- "errno",
+ "bitflags 1.3.2",
+ "errno 0.2.8",
  "io-lifetimes",
  "libc",
- "linux-raw-sys",
- "windows-sys",
+ "linux-raw-sys 0.1.4",
+ "windows-sys 0.42.0",
+]
+
+[[package]]
+name = "rustix"
+version = "0.38.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac5ffa1efe7548069688cd7028f32591853cd7b5b756d41bcffd2353e4fc75b4"
+dependencies = [
+ "bitflags 2.3.3",
+ "errno 0.3.1",
+ "libc",
+ "linux-raw-sys 0.4.3",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -1235,7 +1338,7 @@ version = "10.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1e83c32c3f3c33b08496e0d1df9ea8c64d39adb8eb36a1ebb1440c690697aef"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "cfg-if",
  "clipboard-win",
  "dirs-next",
@@ -1281,35 +1384,44 @@ checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2"
 
 [[package]]
 name = "serde"
-version = "1.0.152"
+version = "1.0.167"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
+checksum = "7daf513456463b42aa1d94cff7e0c24d682b429f020b9afa4f5ba5c40a22b237"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.152"
+version = "1.0.167"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
+checksum = "b69b106b68bc8054f0e974e70d19984040f8a5cf9215ca82626ea4853f82c4b9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.23",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.91"
+version = "1.0.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"
+checksum = "0f1e14e89be7aa4c4b78bdbdc9eb5bf8517829a600ae8eaa39a6e1d960b5185c"
 dependencies = [
  "itoa",
  "ryu",
  "serde",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "sha1"
 version = "0.10.5"
@@ -1395,15 +1507,15 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rand",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
 name = "skyd"
 version = "0.8.0"
 dependencies = [
- "ahash 0.8.2",
- "base64",
+ "ahash",
+ "base64 0.21.2",
  "bincode",
  "bytes",
  "cc",
@@ -1412,7 +1524,7 @@ dependencies = [
  "crc",
  "crossbeam-epoch",
  "env_logger",
- "hashbrown",
+ "hashbrown 0.13.2",
  "jemallocator",
  "libc",
  "libsky",
@@ -1485,9 +1597,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
 [[package]]
 name = "socket2"
-version = "0.4.7"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd"
+checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
 dependencies = [
  "libc",
  "winapi",
@@ -1527,9 +1639,20 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
 [[package]]
 name = "syn"
-version = "1.0.107"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
+checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1538,9 +1661,9 @@ dependencies = [
 
 [[package]]
 name = "sysinfo"
-version = "0.26.9"
+version = "0.29.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c18a6156d1f27a9592ee18c1a846ca8dd5c258b7179fc193ae87c74ebb666f5"
+checksum = "751e810399bba86e9326f5762b7f32ac5a085542df78da6a78d94e07d14d7c11"
 dependencies = [
  "cfg-if",
  "core-foundation-sys",
@@ -1586,7 +1709,7 @@ checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -1606,10 +1729,8 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"
 dependencies = [
- "itoa",
  "serde",
  "time-core",
- "time-macros",
 ]
 
 [[package]]
@@ -1618,25 +1739,16 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
 
-[[package]]
-name = "time-macros"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2"
-dependencies = [
- "time-core",
-]
-
 [[package]]
 name = "tokio"
-version = "1.24.2"
+version = "1.29.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb"
+checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da"
 dependencies = [
  "autocfg",
+ "backtrace",
  "bytes",
  "libc",
- "memchr",
  "mio",
  "num_cpus",
  "parking_lot",
@@ -1644,18 +1756,18 @@ dependencies = [
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "windows-sys",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "1.8.2"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
+checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -1672,11 +1784,36 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.5.11"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.19.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"
+checksum = "c500344a19072298cd05a7224b3c0c629348b78692bf48466c5238656e315a78"
 dependencies = [
+ "indexmap",
  "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow",
 ]
 
 [[package]]
@@ -1711,9 +1848,9 @@ checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
 
 [[package]]
 name = "uuid"
-version = "1.2.2"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c"
+checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be"
 dependencies = [
  "getrandom",
  "rand",
@@ -1722,13 +1859,13 @@ dependencies = [
 
 [[package]]
 name = "uuid-macro-internal"
-version = "1.2.2"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73bc89f2894593e665241e0052c3791999e6787b7c4831daa0a5c2e637e276d8"
+checksum = "8614dda80b9075fbca36bc31b58d1447715b1236af98dee21db521c47a0cc2c0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.23",
 ]
 
 [[package]]
@@ -1782,7 +1919,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
  "wasm-bindgen-shared",
 ]
 
@@ -1804,7 +1941,7 @@ checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -1852,13 +1989,37 @@ version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
 dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
+ "windows_aarch64_gnullvm 0.42.1",
+ "windows_aarch64_msvc 0.42.1",
+ "windows_i686_gnu 0.42.1",
+ "windows_i686_msvc 0.42.1",
+ "windows_x86_64_gnu 0.42.1",
+ "windows_x86_64_gnullvm 0.42.1",
+ "windows_x86_64_msvc 0.42.1",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.0",
+ "windows_aarch64_msvc 0.48.0",
+ "windows_i686_gnu 0.48.0",
+ "windows_i686_msvc 0.48.0",
+ "windows_x86_64_gnu 0.48.0",
+ "windows_x86_64_gnullvm 0.48.0",
+ "windows_x86_64_msvc 0.48.0",
 ]
 
 [[package]]
@@ -1867,42 +2028,93 @@ version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
 
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
 
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
 
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
+
+[[package]]
+name = "winnow"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9482fe6ceabdf32f3966bfdd350ba69256a97c30253dc616fe0005af24f164e"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "yaml-rust"
 version = "0.3.5"
@@ -1911,9 +2123,9 @@ checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992"
 
 [[package]]
 name = "zip"
-version = "0.6.3"
+version = "0.6.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "537ce7411d25e54e8ae21a7ce0b15840e7bfcff15b51d697ec3266cc76bdf080"
+checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261"
 dependencies = [
  "aes",
  "byteorder",
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index d042cec9..033c7108 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -14,8 +14,8 @@ skytable = { git = "https://github.com/skytable/client-rust", branch = "next", f
   "aio-sslv",
 ], default-features = false }
 # external deps
-tokio = { version = "1.21.0", features = ["full"] }
+tokio = { version = "1.28.1", features = ["full"] }
 clap = { version = "2", features = ["yaml"] }
-rustyline = "10.0.0"
-crossterm = "0.25.0"
+rustyline = "10.1.1"
+crossterm = "0.26.1"
 lazy_static = "1.4.0"
diff --git a/harness/Cargo.toml b/harness/Cargo.toml
index 8b5d12ca..caab1961 100644
--- a/harness/Cargo.toml
+++ b/harness/Cargo.toml
@@ -12,8 +12,8 @@ skytable = { git = "https://github.com/skytable/client-rust.git", features = [
 ], default-features = false }
 libsky = { path = "../libsky" }
 # external deps
-env_logger = "0.9.0"
+env_logger = "0.10.0"
 log = "0.4.17"
-zip = { version = "0.6.2", features = ["deflate"] }
-powershell_script = "1.0.4"
-openssl = { version = "0.10.41", features = ["vendored"] }
+zip = { version = "0.6.6", features = ["deflate"] }
+powershell_script = "1.1.0"
+openssl = { version = "0.10.52", features = ["vendored"] }
diff --git a/libstress/Cargo.toml b/libstress/Cargo.toml
index dcf62665..99f89c99 100644
--- a/libstress/Cargo.toml
+++ b/libstress/Cargo.toml
@@ -8,7 +8,7 @@ edition = "2021"
 
 [dependencies]
 # external deps
-crossbeam-channel = "0.5.6"
-rayon = "1.5.3"
+crossbeam-channel = "0.5.8"
+rayon = "1.7.0"
 log = "0.4.17"
 rand = "0.8.5"
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 13db3206..1e5e2cc3 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -11,23 +11,23 @@ libsky = { path = "../libsky" }
 sky_macros = { path = "../sky-macros" }
 rcrypt = "0.4.0"
 # external deps
-ahash = "0.8.0"
-bytes = "1.2.1"
-chrono = "0.4.22"
+ahash = "0.8.3"
+bytes = "1.4.0"
+chrono = "0.4.24"
 clap = { version = "2", features = ["yaml"] }
-env_logger = "0.9.0"
-hashbrown = { version = "0.12.3", features = ["raw"] }
+env_logger = "0.10.0"
+hashbrown = { version = "0.13.2", features = ["raw"] }
 log = "0.4.17"
-openssl = { version = "0.10.41", features = ["vendored"] }
-crossbeam-epoch = { version = "0.9.13" }
+openssl = { version = "0.10.52", features = ["vendored"] }
+crossbeam-epoch = { version = "0.9.14" }
 parking_lot = "0.12.1"
 regex = "1.6.0"
-serde = { version = "1.0.144", features = ["derive"] }
-tokio = { version = "1.21.0", features = ["full"] }
+serde = { version = "1.0.163", features = ["derive"] }
+tokio = { version = "1.28.1", features = ["full"] }
 tokio-openssl = "0.6.3"
-toml = "0.5.9"
-base64 = "0.13.0"
-uuid = { version = "1.2.2", features = ["v4", "fast-rng", "macro-diagnostics"] }
+toml = "0.7.4"
+base64 = "0.21.2"
+uuid = { version = "1.3.3", features = ["v4", "fast-rng", "macro-diagnostics"] }
 crc = "3.0.1"
 
 [target.'cfg(all(not(target_env = "msvc"), not(miri)))'.dependencies]
@@ -39,11 +39,11 @@ winapi = { version = "0.3.9", features = ["fileapi"] }
 
 [target.'cfg(unix)'.dependencies]
 # external deps
-libc = "0.2.132"
+libc = "0.2.144"
 
 [target.'cfg(unix)'.build-dependencies]
 # external deps
-cc = "1.0.73"
+cc = "1.0.79"
 
 [dev-dependencies]
 # internal deps
@@ -55,7 +55,7 @@ skytable = { git = "https://github.com/skytable/client-rust", features = [
 # external deps
 bincode = "1.3.3"
 rand = "0.8.5"
-tokio = { version = "1.21.0", features = ["test-util"] }
+tokio = { version = "1.28.1", features = ["test-util"] }
 
 [features]
 nightly = []
diff --git a/server/src/auth/keys.rs b/server/src/auth/keys.rs
index f1f61732..9eccd89c 100644
--- a/server/src/auth/keys.rs
+++ b/server/src/auth/keys.rs
@@ -27,10 +27,16 @@
 use {
     super::provider::{Authkey, AUTHKEY_SIZE},
     crate::corestore::array::Array,
+    base64::{
+        alphabet::BCRYPT,
+        engine::{GeneralPurpose, GeneralPurposeConfig},
+        Engine,
+    },
 };
 
 type AuthkeyArray = Array<u8, { AUTHKEY_SIZE }>;
 const RAN_BYTES_SIZE: usize = 40;
+const BASE64: GeneralPurpose = GeneralPurpose::new(&BCRYPT, GeneralPurposeConfig::new());
 
 /// Return a "human readable key" and the "authbytes" that can be stored
 /// safely. To do this:
@@ -41,7 +47,7 @@ const RAN_BYTES_SIZE: usize = 40;
 pub fn generate_full() -> (String, Authkey) {
     let mut bytes: [u8; RAN_BYTES_SIZE] = [0u8; RAN_BYTES_SIZE];
     openssl::rand::rand_bytes(&mut bytes).unwrap();
-    let ret = base64::encode_config(&bytes, base64::BCRYPT);
+    let ret = BASE64.encode(&bytes);
     let hash = rcrypt::hash(&ret, rcrypt::DEFAULT_COST).unwrap();
     let store_in_db = unsafe {
         let mut array = AuthkeyArray::new();
diff --git a/server/src/config/mod.rs b/server/src/config/mod.rs
index 1d3d7b61..7fe2634c 100644
--- a/server/src/config/mod.rs
+++ b/server/src/config/mod.rs
@@ -672,8 +672,8 @@ pub fn get_config() -> Result<ConfigType, ConfigError> {
 
     // get config from file
     let cfg_from_file = if let Some(file) = matches.value_of("config") {
-        let file = fs::read(file)?;
-        let cfg_file: ConfigFile = toml::from_slice(&file)?;
+        let file = fs::read_to_string(file)?;
+        let cfg_file: ConfigFile = toml::from_str(&file)?;
         Some(cfgfile::from_file(cfg_file))
     } else {
         None
diff --git a/sky-bench/Cargo.toml b/sky-bench/Cargo.toml
index bebf9941..a795012d 100644
--- a/sky-bench/Cargo.toml
+++ b/sky-bench/Cargo.toml
@@ -16,8 +16,8 @@ libstress = { path = "../libstress" }
 # external deps
 clap = { version = "2", features = ["yaml"] }
 log = "0.4.17"
-env_logger = "0.9.0"
+env_logger = "0.10.0"
 devtimer = "4.0.1"
-serde = { version = "1.0.144", features = ["derive"] }
-serde_json = "1.0.85"
+serde = { version = "1.0.163", features = ["derive"] }
+serde_json = "1.0.96"
 rand = "0.8.5"
diff --git a/sky-macros/Cargo.toml b/sky-macros/Cargo.toml
index a492e47c..ba17531f 100644
--- a/sky-macros/Cargo.toml
+++ b/sky-macros/Cargo.toml
@@ -11,7 +11,7 @@ proc-macro = true
 
 [dependencies]
 # external deps
-proc-macro2 = "1.0.43"
-quote = "1.0.21"
+proc-macro2 = "1.0.58"
+quote = "1.0.27"
 rand = "0.8.5"
-syn = { version = "1.0.99", features = ["full"] }
+syn = { version = "1.0.109", features = ["full"] }
diff --git a/sky-migrate/Cargo.toml b/sky-migrate/Cargo.toml
index 76552b05..c55e919e 100644
--- a/sky-migrate/Cargo.toml
+++ b/sky-migrate/Cargo.toml
@@ -8,7 +8,7 @@ edition = "2021"
 
 [dependencies]
 skytable = { git = "https://github.com/skytable/client-rust.git" }
-env_logger = "0.9.0"
+env_logger = "0.10.0"
 bincode = "1.3.3"
 log = "0.4.17"
 clap = { version = "2", features = ["yaml"] }
diff --git a/stress-test/Cargo.toml b/stress-test/Cargo.toml
index e3572db0..7b5bf6d6 100644
--- a/stress-test/Cargo.toml
+++ b/stress-test/Cargo.toml
@@ -14,8 +14,8 @@ skytable = { git = "https://github.com/skytable/client-rust.git", branch = "next
 ] }
 devtimer = "4.0.1"
 # external deps
-sysinfo = "0.26.2"
-env_logger = "0.9.0"
+sysinfo = "0.29.0"
+env_logger = "0.10.0"
 log = "0.4.17"
 rand = "0.8.5"
-crossbeam-channel = "0.5.6"
+crossbeam-channel = "0.5.8"

From 24c3b0f8a7f0a371946f789747054d2947529a7b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 31 Jul 2023 07:52:30 +0000
Subject: [PATCH 205/310] Fix test suite file writers

---
 server/src/config/tests.rs            | 2 +-
 server/src/engine/storage/v1/tests.rs | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/server/src/config/tests.rs b/server/src/config/tests.rs
index 0b644be7..2e425c87 100644
--- a/server/src/config/tests.rs
+++ b/server/src/config/tests.rs
@@ -831,7 +831,7 @@ mod modeset_de {
         let e = toml::from_str::<Example>(toml).unwrap_err();
         assert_eq!(
             e.to_string(),
-            "Bad value `superuser` for modeset for key `mode` at line 1 column 6"
+            "TOML parse error at line 1, column 6\n  |\n1 | mode=\"superuser\"\n  |      ^^^^^^^^^^^\nBad value `superuser` for modeset\n"
         );
     }
 }
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index 3754bc8a..e1f2bba0 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -109,7 +109,11 @@ impl RawFileIOInterface for VirtualFileInterface {
     fn fwrite_all(&mut self, bytes: &[u8]) -> super::SDSSResult<()> {
         vfs(&self.0, |f| {
             assert!(f.write);
-            f.data_mut().write_all(bytes)?;
+            if f.data.len() < bytes.len() {
+                f.data.extend(bytes);
+            } else {
+                f.data_mut().write_all(bytes)?;
+            }
             Ok(())
         })
     }

From 710fd79e647ada5bd0245f3512f1a9053868eebe Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 31 Jul 2023 07:52:33 +0000
Subject: [PATCH 206/310] Add methods to verify header data

---
 .../src/engine/storage/v1/header_impl/dr.rs   | 29 +++++++++--
 .../src/engine/storage/v1/header_impl/gr.rs   | 41 +++++++++++++--
 .../src/engine/storage/v1/header_impl/mod.rs  | 40 +++++++++++---
 .../src/engine/storage/v1/header_impl/sr.rs   | 17 +++++-
 server/src/engine/storage/v1/mod.rs           |  5 ++
 server/src/engine/storage/v1/rw.rs            | 13 +++--
 server/src/engine/storage/v1/tests.rs         |  6 +--
 server/src/engine/storage/v1/txn.rs           | 52 +++++++++++++++----
 8 files changed, 171 insertions(+), 32 deletions(-)

diff --git a/server/src/engine/storage/v1/header_impl/dr.rs b/server/src/engine/storage/v1/header_impl/dr.rs
index d7d7647e..f80eea2b 100644
--- a/server/src/engine/storage/v1/header_impl/dr.rs
+++ b/server/src/engine/storage/v1/header_impl/dr.rs
@@ -29,7 +29,7 @@ use crate::{
         mem::ByteStack,
         storage::{
             header::{HostArch, HostEndian, HostOS, HostPointerWidth},
-            v1::header_impl::FileSpecifierVersion,
+            v1::{header_impl::FileSpecifierVersion, SDSSError, SDSSResult},
             versions::{self, DriverVersion, ServerVersion},
         },
     },
@@ -59,11 +59,26 @@ pub struct DRHostSignature {
     os: HostOS,
 }
 
+impl DRHostSignature {
+    pub fn verify(&self, expected_file_specifier_version: FileSpecifierVersion) -> SDSSResult<()> {
+        if self.server_version() != versions::v1::V1_SERVER_VERSION {
+            return Err(SDSSError::ServerVersionMismatch);
+        }
+        if self.driver_version() != versions::v1::V1_DRIVER_VERSION {
+            return Err(SDSSError::DriverVersionMismatch);
+        }
+        if self.file_specifier_version() != expected_file_specifier_version {
+            return Err(SDSSError::HeaderDataMismatch);
+        }
+        Ok(())
+    }
+}
+
 impl DRHostSignature {
     /// Decode the [`DRHostSignature`] from the given bytes
     ///
     /// **☢ WARNING ☢: This only decodes; it doesn't validate expected values!**
-    pub fn decode(bytes: [u8; sizeof!(DRHostSignatureRaw)]) -> Option<Self> {
+    pub fn decode_noverify(bytes: [u8; sizeof!(DRHostSignatureRaw)]) -> Option<Self> {
         let ns = ByteStack::new(bytes);
         let server_version = ServerVersion::__new(u64::from_le(
             ns.read_qword(DRHostSignatureRaw::DRHS_OFFSET_P0),
@@ -268,7 +283,15 @@ pub struct DRRuntimeSignature {
 }
 
 impl DRRuntimeSignature {
-    pub fn decode(bytes: [u8; sizeof!(DRRuntimeSignatureRaw)]) -> Option<Self> {
+    pub fn verify(&self) -> SDSSResult<()> {
+        let et = util::os::get_epoch_time();
+        if self.epoch_time() > et || self.host_uptime() > et {
+            // a file from the future?
+            return Err(SDSSError::TimeConflict);
+        }
+        Ok(())
+    }
+    pub fn decode_noverify(bytes: [u8; sizeof!(DRRuntimeSignatureRaw)]) -> Option<Self> {
         let bytes = ByteStack::new(bytes);
         // check
         let modify_count = u64::from_le(bytes.read_qword(DRRuntimeSignatureRaw::DRRS_OFFSET_P0));
diff --git a/server/src/engine/storage/v1/header_impl/gr.rs b/server/src/engine/storage/v1/header_impl/gr.rs
index 072574e5..1565dbfd 100644
--- a/server/src/engine/storage/v1/header_impl/gr.rs
+++ b/server/src/engine/storage/v1/header_impl/gr.rs
@@ -28,7 +28,10 @@ use crate::{
     engine::{
         mem::ByteStack,
         storage::{
-            v1::header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
+            v1::{
+                header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
+                SDSSError, SDSSResult,
+            },
             versions::{self, DriverVersion, ServerVersion},
         },
     },
@@ -55,6 +58,30 @@ pub struct GRMetadataRecord {
     file_spec_id: FileSpecifierVersion,
 }
 
+impl GRMetadataRecord {
+    pub fn verify(
+        &self,
+        expected_file_scope: FileScope,
+        expected_file_specifier: FileSpecifier,
+        expected_file_specifier_version: FileSpecifierVersion,
+    ) -> SDSSResult<()> {
+        if self.server_version() != versions::v1::V1_SERVER_VERSION {
+            return Err(SDSSError::ServerVersionMismatch);
+        }
+        if self.driver_version() != versions::v1::V1_DRIVER_VERSION {
+            return Err(SDSSError::DriverVersionMismatch);
+        }
+        let okay = self.file_scope() == expected_file_scope
+            && self.file_spec() == expected_file_specifier
+            && self.file_spec_id() == expected_file_specifier_version;
+        if okay {
+            Ok(())
+        } else {
+            Err(SDSSError::HeaderDataMismatch)
+        }
+    }
+}
+
 impl GRMetadataRecord {
     pub const fn new(
         server_version: ServerVersion,
@@ -106,7 +133,7 @@ impl GRMetadataRecordRaw {
     /// Decodes a given metadata record, validating all data for correctness.
     ///
     /// **☢ WARNING ☢: This only decodes; it doesn't validate expected values!**
-    pub fn decode(data: [u8; 32]) -> Option<GRMetadataRecord> {
+    pub fn decode_noverify(data: [u8; 32]) -> Option<GRMetadataRecord> {
         let data = ByteStack::new(data);
         let server_version =
             ServerVersion::__new(u64::from_le(data.read_qword(Self::MDR_OFFSET_P0)));
@@ -242,7 +269,15 @@ pub struct GRHostRecord {
 }
 
 impl GRHostRecord {
-    pub fn decode(bytes: [u8; sizeof!(GRHostRecordRaw)]) -> Option<Self> {
+    /// Verified: N/A
+    /// To verify: N/A
+    pub fn verify(&self) -> SDSSResult<()> {
+        Ok(())
+    }
+}
+
+impl GRHostRecord {
+    pub fn decode_noverify(bytes: [u8; sizeof!(GRHostRecordRaw)]) -> Option<Self> {
         let ns = ByteStack::new(bytes);
         let epoch_time = u128::from_le(ns.read_xmmword(GRHostRecordRaw::GRHR_OFFSET_P0));
         let uptime = u128::from_le(ns.read_xmmword(GRHostRecordRaw::GRHR_OFFSET_P1));
diff --git a/server/src/engine/storage/v1/header_impl/mod.rs b/server/src/engine/storage/v1/header_impl/mod.rs
index 6b58a93c..75e6bb9e 100644
--- a/server/src/engine/storage/v1/header_impl/mod.rs
+++ b/server/src/engine/storage/v1/header_impl/mod.rs
@@ -58,6 +58,8 @@
 
 use crate::util::copy_slice_to_array as cp;
 
+use super::SDSSResult;
+
 // (1) sr
 mod sr;
 // (2) gr
@@ -153,6 +155,26 @@ pub struct SDSSHeader {
     dr_rs: dr::DRRuntimeSignature,
 }
 
+impl SDSSHeader {
+    pub fn verify(
+        &self,
+        expected_file_scope: FileScope,
+        expected_file_specifier: FileSpecifier,
+        expected_file_specifier_version: FileSpecifierVersion,
+    ) -> SDSSResult<()> {
+        self.sr().verify()?;
+        self.gr_mdr().verify(
+            expected_file_scope,
+            expected_file_specifier,
+            expected_file_specifier_version,
+        )?;
+        self.gr_hr().verify()?;
+        self.dr_hs().verify(expected_file_specifier_version)?;
+        self.dr_rs().verify()?;
+        Ok(())
+    }
+}
+
 impl SDSSHeader {
     pub const fn new(
         sr: sr::StaticRecord,
@@ -290,13 +312,17 @@ impl SDSSHeaderRaw {
         data
     }
     /// **☢ WARNING ☢: This only decodes; it doesn't validate expected values!**
-    pub fn decode(slice: [u8; Self::header_size()]) -> Option<SDSSHeader> {
-        let sr = sr::StaticRecordRaw::decode(cp(&slice[Self::OFFSET_SR0..Self::OFFSET_SR1]))?;
-        let gr_mdr =
-            gr::GRMetadataRecordRaw::decode(cp(&slice[Self::OFFSET_SR1..Self::OFFSET_SR2]))?;
-        let gr_hr = gr::GRHostRecord::decode(cp(&slice[Self::OFFSET_SR2..Self::OFFSET_SR3]))?;
-        let dr_sig = dr::DRHostSignature::decode(cp(&slice[Self::OFFSET_SR3..Self::OFFSET_SR4]))?;
-        let dr_rt = dr::DRRuntimeSignature::decode(cp(&slice[Self::OFFSET_SR4..]))?;
+    pub fn decode_noverify(slice: [u8; Self::header_size()]) -> Option<SDSSHeader> {
+        let sr =
+            sr::StaticRecordRaw::decode_noverify(cp(&slice[Self::OFFSET_SR0..Self::OFFSET_SR1]))?;
+        let gr_mdr = gr::GRMetadataRecordRaw::decode_noverify(cp(
+            &slice[Self::OFFSET_SR1..Self::OFFSET_SR2]
+        ))?;
+        let gr_hr =
+            gr::GRHostRecord::decode_noverify(cp(&slice[Self::OFFSET_SR2..Self::OFFSET_SR3]))?;
+        let dr_sig =
+            dr::DRHostSignature::decode_noverify(cp(&slice[Self::OFFSET_SR3..Self::OFFSET_SR4]))?;
+        let dr_rt = dr::DRRuntimeSignature::decode_noverify(cp(&slice[Self::OFFSET_SR4..]))?;
         Some(SDSSHeader::new(sr, gr_mdr, gr_hr, dr_sig, dr_rt))
     }
 }
diff --git a/server/src/engine/storage/v1/header_impl/sr.rs b/server/src/engine/storage/v1/header_impl/sr.rs
index 46645c21..3dcdffb2 100644
--- a/server/src/engine/storage/v1/header_impl/sr.rs
+++ b/server/src/engine/storage/v1/header_impl/sr.rs
@@ -26,6 +26,7 @@
 
 use crate::engine::storage::{
     header::{StaticRecordUV, StaticRecordUVRaw},
+    v1::{SDSSError, SDSSResult},
     versions,
 };
 
@@ -34,6 +35,20 @@ pub struct StaticRecord {
     sr: StaticRecordUV,
 }
 
+impl StaticRecord {
+    /// Verified:
+    /// - header version
+    ///
+    /// Need to verify: N/A
+    pub fn verify(&self) -> SDSSResult<()> {
+        if self.sr().header_version() == versions::v1::V1_HEADER_VERSION {
+            Ok(())
+        } else {
+            return Err(SDSSError::HeaderVersionMismatch);
+        }
+    }
+}
+
 impl StaticRecord {
     pub const fn new(sr: StaticRecordUV) -> Self {
         Self { sr }
@@ -64,7 +79,7 @@ impl StaticRecordRaw {
     pub const fn empty_buffer() -> [u8; sizeof!(Self)] {
         [0u8; sizeof!(Self)]
     }
-    pub fn decode(buf: [u8; sizeof!(Self)]) -> Option<StaticRecord> {
+    pub fn decode_noverify(buf: [u8; sizeof!(Self)]) -> Option<StaticRecord> {
         StaticRecordUVRaw::decode_from_bytes(buf).map(StaticRecord::new)
     }
 }
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index d4b14534..2e9931a7 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -59,6 +59,11 @@ pub enum SDSSError {
     CorruptedHeader,
     TransactionLogEntryCorrupted,
     TransactionLogCorrupted,
+    HeaderVersionMismatch,
+    DriverVersionMismatch,
+    ServerVersionMismatch,
+    HeaderDataMismatch,
+    TimeConflict,
 }
 
 impl SDSSError {
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 954bab6d..2c15b3d8 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -56,7 +56,7 @@ pub trait RawFileIOInterface: Sized {
     fn fread_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()>;
     fn fwrite_all(&mut self, bytes: &[u8]) -> SDSSResult<()>;
     fn fsync_all(&mut self) -> SDSSResult<()>;
-    fn fseek_ahead(&mut self, by: usize) -> SDSSResult<()>;
+    fn fseek_ahead(&mut self, by: u64) -> SDSSResult<()>;
     fn flen(&self) -> SDSSResult<u64>;
 }
 
@@ -89,8 +89,8 @@ impl RawFileIOInterface for File {
     fn flen(&self) -> SDSSResult<u64> {
         Ok(self.metadata()?.len())
     }
-    fn fseek_ahead(&mut self, by: usize) -> SDSSResult<()> {
-        self.seek(SeekFrom::Start(by as _))?;
+    fn fseek_ahead(&mut self, by: u64) -> SDSSResult<()> {
+        self.seek(SeekFrom::Start(by))?;
         Ok(())
     }
 }
@@ -132,7 +132,10 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
                 // this is an existing file. decoded the header
                 let mut header_raw = [0u8; SDSSHeaderRaw::header_size()];
                 f.fread_exact(&mut header_raw)?;
-                let header = SDSSHeaderRaw::decode(header_raw).ok_or(SDSSError::CorruptedHeader)?;
+                let header =
+                    SDSSHeaderRaw::decode_noverify(header_raw).ok_or(SDSSError::CorruptedHeader)?;
+                // now validate the header
+                header.verify(file_scope, file_specifier, file_specifier_version)?;
                 // since we updated this file, let us update the header
                 let mut new_header = header.clone();
                 new_header.dr_rs_mut().bump_modify_count();
@@ -165,7 +168,7 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
     pub fn file_length(&self) -> SDSSResult<u64> {
         self.f.flen()
     }
-    pub fn seek_ahead(&mut self, by: usize) -> SDSSResult<()> {
+    pub fn seek_ahead(&mut self, by: u64) -> SDSSResult<()> {
         self.f.fseek_ahead(by)
     }
 }
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index e1f2bba0..ce0e15e4 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -75,8 +75,8 @@ impl VirtualFile {
     fn r(data: Vec<u8>) -> Self {
         Self::new(true, false, data)
     }
-    fn seek_forward(&mut self, by: usize) {
-        self.pos += by as u64;
+    fn seek_forward(&mut self, by: u64) {
+        self.pos += by;
         assert!(self.pos <= self.data.len() as u64);
     }
     fn data(&self) -> &[u8] {
@@ -123,7 +123,7 @@ impl RawFileIOInterface for VirtualFileInterface {
     fn flen(&self) -> SDSSResult<u64> {
         vfs(&self.0, |f| Ok(f.data.len() as _))
     }
-    fn fseek_ahead(&mut self, by: usize) -> SDSSResult<()> {
+    fn fseek_ahead(&mut self, by: u64) -> SDSSResult<()> {
         vfs(&self.0, |f| {
             f.seek_forward(by);
             Ok(())
diff --git a/server/src/engine/storage/v1/txn.rs b/server/src/engine/storage/v1/txn.rs
index e1b90fd7..d9410551 100644
--- a/server/src/engine/storage/v1/txn.rs
+++ b/server/src/engine/storage/v1/txn.rs
@@ -38,15 +38,45 @@
 
 use {
     super::{
-        rw::{RawFileIOInterface, SDSSFileIO},
+        header_impl::{FileSpecifierVersion, HostRunMode},
+        rw::{FileOpen, RawFileIOInterface, SDSSFileIO},
         SDSSError, SDSSResult,
     },
-    crate::util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy},
+    crate::{
+        engine::storage::v1::header_impl::{FileScope, FileSpecifier},
+        util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy},
+    },
     std::marker::PhantomData,
 };
 
 const CRC: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
 
+pub fn open_log<TA: TransactionLogAdapter, LF: RawFileIOInterface>(
+    log_file_name: &str,
+    log_kind: FileSpecifier,
+    log_kind_version: FileSpecifierVersion,
+    host_setting_version: u32,
+    host_run_mode: HostRunMode,
+    host_startup_counter: u64,
+    gs: &TA::GlobalState,
+) -> SDSSResult<TransactionLogWriter<LF, TA>> {
+    let f = SDSSFileIO::<LF>::open_or_create_perm_rw(
+        log_file_name,
+        FileScope::TransactionLog,
+        log_kind,
+        log_kind_version,
+        host_setting_version,
+        host_run_mode,
+        host_startup_counter,
+    )?;
+    let file = match f {
+        FileOpen::Created(f) => return TransactionLogWriter::new(f, 0, 0),
+        FileOpen::Existing(file, _) => file,
+    };
+    let (file, size, last_txn) = TransactionLogReader::<TA, LF>::scroll(file, gs)?;
+    TransactionLogWriter::new(file, size, last_txn)
+}
+
 /// The transaction adapter
 pub trait TransactionLogAdapter {
     /// The transaction event
@@ -143,6 +173,7 @@ impl TxnLogEntryMetadata {
 #[derive(Debug)]
 pub struct TransactionLogReader<TA, LF> {
     log_file: SDSSFileIO<LF>,
+    log_size: u64,
     evid: u64,
     closed: bool,
     remaining_bytes: u64,
@@ -154,6 +185,7 @@ impl<TA: TransactionLogAdapter, LF: RawFileIOInterface> TransactionLogReader<TA,
         let log_size = log_file.file_length()?;
         Ok(Self {
             log_file,
+            log_size,
             evid: 0,
             closed: false,
             remaining_bytes: log_size,
@@ -210,13 +242,17 @@ impl<TA: TransactionLogAdapter, LF: RawFileIOInterface> TransactionLogReader<TA,
             Err(SDSSError::TransactionLogEntryCorrupted)
         }
     }
-    /// Read and apply all events in the given log file to the global state, returning the open file
-    pub fn scroll(file: SDSSFileIO<LF>, gs: &TA::GlobalState) -> SDSSResult<SDSSFileIO<LF>> {
+    /// Read and apply all events in the given log file to the global state, returning the
+    /// (open file, log size, last event ID)
+    pub fn scroll(
+        file: SDSSFileIO<LF>,
+        gs: &TA::GlobalState,
+    ) -> SDSSResult<(SDSSFileIO<LF>, u64, u64)> {
         let mut slf = Self::new(file)?;
         while !slf.end_of_file() {
             slf.rapply_next_event(gs)?;
         }
-        Ok(slf.log_file)
+        Ok((slf.log_file, slf.log_size, slf.evid))
     }
 }
 
@@ -244,11 +280,7 @@ pub struct TransactionLogWriter<LF, TA> {
 }
 
 impl<LF: RawFileIOInterface, TA: TransactionLogAdapter> TransactionLogWriter<LF, TA> {
-    pub fn new(
-        mut log_file: SDSSFileIO<LF>,
-        last_txn_id: u64,
-        last_size: usize,
-    ) -> SDSSResult<Self> {
+    pub fn new(mut log_file: SDSSFileIO<LF>, last_size: u64, last_txn_id: u64) -> SDSSResult<Self> {
         log_file.seek_ahead(last_size)?;
         Ok(Self {
             log_file,

From dc4afdc257fc9ba2a65e0321b1a48dc2ce1fc8bf Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 31 Jul 2023 07:52:36 +0000
Subject: [PATCH 207/310] Add mock txn impl and fix txn impls

---
 server/src/config/tests.rs                    |   1 -
 .../src/engine/storage/v1/header_impl/mod.rs  |   4 +
 server/src/engine/storage/v1/rw.rs            |   9 +-
 server/src/engine/storage/v1/tests.rs         | 181 +++++++++++++++++-
 server/src/engine/storage/v1/txn.rs           |  45 +++--
 server/src/util/mod.rs                        |   8 +-
 server/src/util/test_utils.rs                 |  14 ++
 7 files changed, 226 insertions(+), 36 deletions(-)

diff --git a/server/src/config/tests.rs b/server/src/config/tests.rs
index 2e425c87..ca06e29e 100644
--- a/server/src/config/tests.rs
+++ b/server/src/config/tests.rs
@@ -626,7 +626,6 @@ mod try_from_config_source_impls {
         has_mutated: bool,
     ) {
         let mut mutated = false;
-        dbg!(new.is_present(), is_present);
         assert_eq!(new.is_present(), is_present);
         assert_eq!(new.mutate_failed(&mut default, &mut mutated), mutate_failed);
         assert_eq!(mutated, has_mutated);
diff --git a/server/src/engine/storage/v1/header_impl/mod.rs b/server/src/engine/storage/v1/header_impl/mod.rs
index 75e6bb9e..4e2fa051 100644
--- a/server/src/engine/storage/v1/header_impl/mod.rs
+++ b/server/src/engine/storage/v1/header_impl/mod.rs
@@ -95,12 +95,16 @@ impl FileScope {
 #[repr(u8)]
 pub enum FileSpecifier {
     GNSTxnLog = 0,
+    #[cfg(test)]
+    TestTransactionLog = 1,
 }
 
 impl FileSpecifier {
     pub const fn try_new(v: u32) -> Option<Self> {
         Some(match v {
             0 => Self::GNSTxnLog,
+            #[cfg(test)]
+            1 => Self::TestTransactionLog,
             _ => return None,
         })
     }
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 2c15b3d8..c59cf9a8 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -68,10 +68,10 @@ impl RawFileIOInterface for File {
             .write(true)
             .open(file_path)?;
         let md = f.metadata()?;
-        if md.created()? == md.modified()? {
-            return Ok(RawFileOpen::Created(f));
+        if md.len() == 0 {
+            Ok(RawFileOpen::Created(f))
         } else {
-            return Ok(RawFileOpen::Existing(f));
+            Ok(RawFileOpen::Existing(f))
         }
     }
     fn fread_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()> {
@@ -101,6 +101,7 @@ pub struct SDSSFileIO<F> {
 }
 
 impl<F: RawFileIOInterface> SDSSFileIO<F> {
+    /// **IMPORTANT: File position: end-of-header-section**
     pub fn open_or_create_perm_rw(
         file_path: &str,
         file_scope: FileScope,
@@ -140,7 +141,9 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
                 let mut new_header = header.clone();
                 new_header.dr_rs_mut().bump_modify_count();
                 let mut f = Self::_new(f);
+                f.seek_ahead(0)?;
                 f.fsynced_write(new_header.encoded().array().as_ref())?;
+                f.seek_ahead(SDSSHeaderRaw::header_size() as _)?;
                 Ok(FileOpen::Existing(f, header))
             }
         }
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index ce0e15e4..97b65c6c 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -85,14 +85,34 @@ impl VirtualFile {
     fn data_mut(&mut self) -> &mut [u8] {
         &mut self.data[self.pos as usize..]
     }
+    fn close(&mut self) {
+        self.pos = 0;
+        self.read = false;
+        self.write = false;
+    }
 }
 
 struct VirtualFileInterface(Box<str>);
 
+impl Drop for VirtualFileInterface {
+    fn drop(&mut self) {
+        vfs(&self.0, |f| {
+            f.close();
+            Ok(())
+        })
+        .unwrap();
+    }
+}
+
 impl RawFileIOInterface for VirtualFileInterface {
     fn fopen_or_create_rw(file_path: &str) -> SDSSResult<RawFileOpen<Self>> {
         match VFS.write().entry(file_path.to_owned()) {
-            Entry::Occupied(_) => Ok(RawFileOpen::Existing(Self(file_path.into()))),
+            Entry::Occupied(mut oe) => {
+                let file_md = oe.get_mut();
+                file_md.read = true;
+                file_md.write = true;
+                Ok(RawFileOpen::Existing(Self(file_path.into())))
+            }
             Entry::Vacant(ve) => {
                 ve.insert(VirtualFile::rw(vec![]));
                 Ok(RawFileOpen::Created(Self(file_path.into())))
@@ -131,18 +151,18 @@ impl RawFileIOInterface for VirtualFileInterface {
     }
 }
 
+type VirtualFS = VirtualFileInterface;
+type RealFS = std::fs::File;
+
 mod rw {
-    use {
-        super::VirtualFileInterface,
-        crate::engine::storage::v1::{
-            header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
-            rw::{FileOpen, SDSSFileIO},
-        },
+    use crate::engine::storage::v1::{
+        header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
+        rw::{FileOpen, SDSSFileIO},
     };
 
     #[test]
     fn create_delete() {
-        let f = SDSSFileIO::<VirtualFileInterface>::open_or_create_perm_rw(
+        let f = SDSSFileIO::<super::VirtualFS>::open_or_create_perm_rw(
             "hello_world.db-tlog",
             FileScope::TransactionLogCompacted,
             FileSpecifier::GNSTxnLog,
@@ -156,7 +176,7 @@ mod rw {
             FileOpen::Existing(_, _) => panic!(),
             FileOpen::Created(_) => {}
         };
-        let open = SDSSFileIO::<VirtualFileInterface>::open_or_create_perm_rw(
+        let open = SDSSFileIO::<super::VirtualFS>::open_or_create_perm_rw(
             "hello_world.db-tlog",
             FileScope::TransactionLogCompacted,
             FileSpecifier::GNSTxnLog,
@@ -178,3 +198,146 @@ mod rw {
         assert_eq!(h.gr_hr().startup_counter(), 0);
     }
 }
+
+mod tx {
+    use crate::engine::storage::v1::header_impl::{
+        FileSpecifier, FileSpecifierVersion, HostRunMode,
+    };
+
+    type FileInterface = super::RealFS;
+
+    use {
+        crate::{
+            engine::storage::v1::{
+                txn::{self, TransactionLogAdapter, TransactionLogWriter},
+                SDSSError, SDSSResult,
+            },
+            util,
+        },
+        std::cell::RefCell,
+    };
+    pub struct Database {
+        data: RefCell<[u8; 10]>,
+    }
+    impl Database {
+        fn copy_data(&self) -> [u8; 10] {
+            *self.data.borrow()
+        }
+        fn new() -> Self {
+            Self {
+                data: RefCell::new([0; 10]),
+            }
+        }
+        fn reset(&self) {
+            *self.data.borrow_mut() = [0; 10];
+        }
+        fn txn_reset(
+            &self,
+            txn_writer: &mut TransactionLogWriter<FileInterface, DatabaseTxnAdapter>,
+        ) -> SDSSResult<()> {
+            self.reset();
+            txn_writer.append_event(TxEvent::Reset)
+        }
+        fn set(&self, pos: usize, val: u8) {
+            self.data.borrow_mut()[pos] = val;
+        }
+        fn txn_set(
+            &self,
+            pos: usize,
+            val: u8,
+            txn_writer: &mut TransactionLogWriter<FileInterface, DatabaseTxnAdapter>,
+        ) -> SDSSResult<()> {
+            self.set(pos, val);
+            txn_writer.append_event(TxEvent::Set(pos, val))
+        }
+    }
+    pub enum TxEvent {
+        Reset,
+        Set(usize, u8),
+    }
+    #[derive(Debug)]
+    pub struct DatabaseTxnAdapter;
+    impl TransactionLogAdapter for DatabaseTxnAdapter {
+        type TransactionEvent = TxEvent;
+
+        type GlobalState = Database;
+
+        fn encode(event: Self::TransactionEvent) -> Box<[u8]> {
+            /*
+                [1B: opcode][8B:Index][1B: New value]
+            */
+            let opcode = match event {
+                TxEvent::Reset => 0u8,
+                TxEvent::Set(_, _) => 1u8,
+            };
+            let index = match event {
+                TxEvent::Reset => 0u64,
+                TxEvent::Set(index, _) => index as u64,
+            };
+            let new_value = match event {
+                TxEvent::Reset => 0,
+                TxEvent::Set(_, val) => val,
+            };
+            let mut ret = Vec::with_capacity(10);
+            ret.push(opcode);
+            ret.extend(index.to_le_bytes());
+            ret.push(new_value);
+            ret.into_boxed_slice()
+        }
+
+        fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> SDSSResult<()> {
+            if payload.len() != 10 {
+                return Err(SDSSError::CorruptedFile("testtxn.log"));
+            }
+            let opcode = payload[0];
+            let index = u64::from_le_bytes(util::copy_slice_to_array(&payload[1..9]));
+            let new_value = payload[9];
+            match opcode {
+                0 if index == 0 && new_value == 0 => gs.reset(),
+                1 if index < 10 && index < isize::MAX as u64 => gs.set(index as usize, new_value),
+                _ => return Err(SDSSError::TransactionLogEntryCorrupted),
+            }
+            Ok(())
+        }
+    }
+
+    #[test]
+    fn two_set() {
+        // create log
+        let db1 = Database::new();
+        let x = || -> SDSSResult<()> {
+            let mut log = txn::open_log(
+                "testtxn.log",
+                FileSpecifier::TestTransactionLog,
+                FileSpecifierVersion::__new(0),
+                0,
+                HostRunMode::Prod,
+                1,
+                &db1,
+            )?;
+            db1.txn_set(0, 20, &mut log)?;
+            db1.txn_set(9, 21, &mut log)?;
+            log.close_log()
+        };
+        x().unwrap();
+        // backup original data
+        let original_data = db1.copy_data();
+        // restore log
+        let empty_db2 = Database::new();
+        {
+            let log = txn::open_log::<DatabaseTxnAdapter, FileInterface>(
+                "testtxn.log",
+                FileSpecifier::TestTransactionLog,
+                FileSpecifierVersion::__new(0),
+                0,
+                HostRunMode::Prod,
+                1,
+                &empty_db2,
+            )
+            .unwrap();
+            log.close_log().unwrap();
+        }
+        assert_eq!(original_data, empty_db2.copy_data());
+        std::fs::remove_file("testtxn.log").unwrap();
+    }
+}
diff --git a/server/src/engine/storage/v1/txn.rs b/server/src/engine/storage/v1/txn.rs
index d9410551..4fce4b97 100644
--- a/server/src/engine/storage/v1/txn.rs
+++ b/server/src/engine/storage/v1/txn.rs
@@ -38,7 +38,7 @@
 
 use {
     super::{
-        header_impl::{FileSpecifierVersion, HostRunMode},
+        header_impl::{FileSpecifierVersion, HostRunMode, SDSSHeaderRaw},
         rw::{FileOpen, RawFileIOInterface, SDSSFileIO},
         SDSSError, SDSSResult,
     },
@@ -51,7 +51,10 @@ use {
 
 const CRC: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
 
-pub fn open_log<TA: TransactionLogAdapter, LF: RawFileIOInterface>(
+pub fn open_log<
+    TA: TransactionLogAdapter + core::fmt::Debug,
+    LF: RawFileIOInterface + core::fmt::Debug,
+>(
     log_file_name: &str,
     log_kind: FileSpecifier,
     log_kind_version: FileSpecifierVersion,
@@ -70,11 +73,11 @@ pub fn open_log<TA: TransactionLogAdapter, LF: RawFileIOInterface>(
         host_startup_counter,
     )?;
     let file = match f {
-        FileOpen::Created(f) => return TransactionLogWriter::new(f, 0, 0),
+        FileOpen::Created(f) => return TransactionLogWriter::new(f, 0),
         FileOpen::Existing(file, _) => file,
     };
-    let (file, size, last_txn) = TransactionLogReader::<TA, LF>::scroll(file, gs)?;
-    TransactionLogWriter::new(file, size, last_txn)
+    let (file, last_txn) = TransactionLogReader::<TA, LF>::scroll(file, gs)?;
+    TransactionLogWriter::new(file, last_txn)
 }
 
 /// The transaction adapter
@@ -89,6 +92,7 @@ pub trait TransactionLogAdapter {
     fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> SDSSResult<()>;
 }
 
+#[derive(Debug)]
 pub struct TxnLogEntryMetadata {
     event_id: u128,
     event_source_md: u64,
@@ -182,7 +186,7 @@ pub struct TransactionLogReader<TA, LF> {
 
 impl<TA: TransactionLogAdapter, LF: RawFileIOInterface> TransactionLogReader<TA, LF> {
     pub fn new(log_file: SDSSFileIO<LF>) -> SDSSResult<Self> {
-        let log_size = log_file.file_length()?;
+        let log_size = log_file.file_length()? - SDSSHeaderRaw::header_size() as u64;
         Ok(Self {
             log_file,
             log_size,
@@ -194,10 +198,10 @@ impl<TA: TransactionLogAdapter, LF: RawFileIOInterface> TransactionLogReader<TA,
     }
     /// Read the next event and apply it to the global state
     pub fn rapply_next_event(&mut self, gs: &TA::GlobalState) -> SDSSResult<()> {
-        self._incr_evid();
         // read metadata
         let mut raw_txn_log_row_md = [0u8; TxnLogEntryMetadata::SIZE];
         self.log_file.read_to_buffer(&mut raw_txn_log_row_md)?;
+        self._record_bytes_read(36);
         let event_metadata = TxnLogEntryMetadata::decode(raw_txn_log_row_md);
         /*
             verify metadata and read bytes into buffer, verify sum
@@ -226,7 +230,9 @@ impl<TA: TransactionLogAdapter, LF: RawFileIOInterface> TransactionLogReader<TA,
                     return Err(SDSSError::TransactionLogCorrupted);
                 }
             }
-            _ => return Err(SDSSError::TransactionLogEntryCorrupted),
+            EventSourceMarker::DriverClosed => {
+                return Err(SDSSError::TransactionLogEntryCorrupted);
+            }
         }
         // read bytes
         let mut payload_data_block = vec![0u8; event_metadata.event_payload_len as usize];
@@ -235,6 +241,7 @@ impl<TA: TransactionLogAdapter, LF: RawFileIOInterface> TransactionLogReader<TA,
         // verify sum
         let actual_sum = CRC.checksum(&payload_data_block);
         if compiler::likely(actual_sum == event_metadata.event_crc) {
+            self._incr_evid();
             // great, the sums match
             TA::decode_and_update_state(&payload_data_block, gs)?;
             Ok(())
@@ -242,17 +249,17 @@ impl<TA: TransactionLogAdapter, LF: RawFileIOInterface> TransactionLogReader<TA,
             Err(SDSSError::TransactionLogEntryCorrupted)
         }
     }
-    /// Read and apply all events in the given log file to the global state, returning the
-    /// (open file, log size, last event ID)
-    pub fn scroll(
-        file: SDSSFileIO<LF>,
-        gs: &TA::GlobalState,
-    ) -> SDSSResult<(SDSSFileIO<LF>, u64, u64)> {
+    /// Read and apply all events in the given log file to the global state, returning the (open file, last event ID)
+    pub fn scroll(file: SDSSFileIO<LF>, gs: &TA::GlobalState) -> SDSSResult<(SDSSFileIO<LF>, u64)> {
         let mut slf = Self::new(file)?;
         while !slf.end_of_file() {
             slf.rapply_next_event(gs)?;
         }
-        Ok((slf.log_file, slf.log_size, slf.evid))
+        if slf.closed {
+            Ok((slf.log_file, slf.evid))
+        } else {
+            Err(SDSSError::TransactionLogCorrupted)
+        }
     }
 }
 
@@ -280,8 +287,9 @@ pub struct TransactionLogWriter<LF, TA> {
 }
 
 impl<LF: RawFileIOInterface, TA: TransactionLogAdapter> TransactionLogWriter<LF, TA> {
-    pub fn new(mut log_file: SDSSFileIO<LF>, last_size: u64, last_txn_id: u64) -> SDSSResult<Self> {
-        log_file.seek_ahead(last_size)?;
+    pub fn new(mut log_file: SDSSFileIO<LF>, last_txn_id: u64) -> SDSSResult<Self> {
+        let l = log_file.file_length()?;
+        log_file.seek_ahead(l)?;
         Ok(Self {
             log_file,
             id: last_txn_id,
@@ -303,10 +311,9 @@ impl<LF: RawFileIOInterface, TA: TransactionLogAdapter> TransactionLogWriter<LF,
         Ok(())
     }
     pub fn close_log(mut self) -> SDSSResult<()> {
-        let crc = CRC.checksum(EventSourceMarker::DRIVER_CLOSED.to_le_bytes().as_ref());
         let id = self._incr_id() as u128;
         self.log_file.fsynced_write(
-            &TxnLogEntryMetadata::new(id, EventSourceMarker::DRIVER_CLOSED, crc, 0).encoded(),
+            &TxnLogEntryMetadata::new(id, EventSourceMarker::DRIVER_CLOSED, 0, 0).encoded(),
         )
     }
 }
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index 98783b33..8d971c4c 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -378,17 +378,17 @@ pub const fn copy_str_to_array<const N: usize>(str: &str) -> [u8; N] {
 }
 /// Copy the elements of a into b, beginning the copy at `pos`
 pub const fn copy_a_into_b<const M: usize, const N: usize>(
-    a: [u8; M],
-    mut b: [u8; N],
+    from: [u8; M],
+    mut to: [u8; N],
     mut pos: usize,
 ) -> [u8; N] {
     assert!(M <= N);
     assert!(pos < N);
     let mut i = 0;
     while i < M {
-        b[pos] = a[pos];
+        to[pos] = from[i];
         i += 1;
         pos += 1;
     }
-    b
+    to
 }
diff --git a/server/src/util/test_utils.rs b/server/src/util/test_utils.rs
index f35f1034..01d80524 100644
--- a/server/src/util/test_utils.rs
+++ b/server/src/util/test_utils.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use std::io::Read;
+
 use {
     rand::{
         distributions::{uniform::SampleUniform, Alphanumeric},
@@ -36,6 +38,18 @@ use {
     },
 };
 
+pub fn wait_for_key(msg: &str) {
+    use std::io::{self, Write};
+    print!("{msg}");
+    let x = || -> std::io::Result<()> {
+        io::stdout().flush()?;
+        let mut key = [0u8; 1];
+        io::stdin().read_exact(&mut key)?;
+        Ok(())
+    };
+    x().unwrap();
+}
+
 // TODO(@ohsayan): Use my own PRNG algo here. Maybe my quadratic one?
 
 /// Generates a random boolean based on Bernoulli distributions

From c1f8e2d1bd1ad69b1141d21c697fe17580dd3702 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 31 Jul 2023 07:52:40 +0000
Subject: [PATCH 208/310] Remove the delete entry once validated

---
 server/src/engine/storage/v1/rw.rs    | 14 ++++-
 server/src/engine/storage/v1/tests.rs | 91 ++++++++++++++++++++-------
 server/src/engine/storage/v1/txn.rs   | 28 +++++++--
 3 files changed, 104 insertions(+), 29 deletions(-)

diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index c59cf9a8..6d9022e8 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -58,6 +58,7 @@ pub trait RawFileIOInterface: Sized {
     fn fsync_all(&mut self) -> SDSSResult<()>;
     fn fseek_ahead(&mut self, by: u64) -> SDSSResult<()>;
     fn flen(&self) -> SDSSResult<u64>;
+    fn flen_set(&mut self, to: u64) -> SDSSResult<()>;
 }
 
 impl RawFileIOInterface for File {
@@ -93,6 +94,10 @@ impl RawFileIOInterface for File {
         self.seek(SeekFrom::Start(by))?;
         Ok(())
     }
+    fn flen_set(&mut self, to: u64) -> SDSSResult<()> {
+        self.set_len(to)?;
+        Ok(())
+    }
 }
 
 #[derive(Debug)]
@@ -141,9 +146,9 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
                 let mut new_header = header.clone();
                 new_header.dr_rs_mut().bump_modify_count();
                 let mut f = Self::_new(f);
-                f.seek_ahead(0)?;
+                f.seek_from_start(0)?;
                 f.fsynced_write(new_header.encoded().array().as_ref())?;
-                f.seek_ahead(SDSSHeaderRaw::header_size() as _)?;
+                f.seek_from_start(SDSSHeaderRaw::header_size() as _)?;
                 Ok(FileOpen::Existing(f, header))
             }
         }
@@ -171,7 +176,10 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
     pub fn file_length(&self) -> SDSSResult<u64> {
         self.f.flen()
     }
-    pub fn seek_ahead(&mut self, by: u64) -> SDSSResult<()> {
+    pub fn seek_from_start(&mut self, by: u64) -> SDSSResult<()> {
         self.f.fseek_ahead(by)
     }
+    pub fn trim_file_to(&mut self, to: u64) -> SDSSResult<()> {
+        self.f.flen_set(to)
+    }
 }
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index 97b65c6c..e19c82a4 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -149,6 +149,12 @@ impl RawFileIOInterface for VirtualFileInterface {
             Ok(())
         })
     }
+    fn flen_set(&mut self, to: u64) -> SDSSResult<()> {
+        vfs(&self.0, |f| {
+            f.data.drain(f.data.len() - to as usize..);
+            Ok(())
+        })
+    }
 }
 
 type VirtualFS = VirtualFileInterface;
@@ -301,20 +307,27 @@ mod tx {
         }
     }
 
+    fn open_log(
+        log_name: &str,
+        db: &Database,
+    ) -> SDSSResult<TransactionLogWriter<FileInterface, DatabaseTxnAdapter>> {
+        txn::open_log::<DatabaseTxnAdapter, FileInterface>(
+            log_name,
+            FileSpecifier::TestTransactionLog,
+            FileSpecifierVersion::__new(0),
+            0,
+            HostRunMode::Prod,
+            1,
+            &db,
+        )
+    }
+
     #[test]
-    fn two_set() {
+    fn first_boot_second_readonly() {
         // create log
         let db1 = Database::new();
         let x = || -> SDSSResult<()> {
-            let mut log = txn::open_log(
-                "testtxn.log",
-                FileSpecifier::TestTransactionLog,
-                FileSpecifierVersion::__new(0),
-                0,
-                HostRunMode::Prod,
-                1,
-                &db1,
-            )?;
+            let mut log = open_log("testtxn.log", &db1)?;
             db1.txn_set(0, 20, &mut log)?;
             db1.txn_set(9, 21, &mut log)?;
             log.close_log()
@@ -324,20 +337,54 @@ mod tx {
         let original_data = db1.copy_data();
         // restore log
         let empty_db2 = Database::new();
-        {
-            let log = txn::open_log::<DatabaseTxnAdapter, FileInterface>(
-                "testtxn.log",
-                FileSpecifier::TestTransactionLog,
-                FileSpecifierVersion::__new(0),
-                0,
-                HostRunMode::Prod,
-                1,
-                &empty_db2,
-            )
+        open_log("testtxn.log", &empty_db2)
+            .unwrap()
+            .close_log()
             .unwrap();
-            log.close_log().unwrap();
-        }
         assert_eq!(original_data, empty_db2.copy_data());
         std::fs::remove_file("testtxn.log").unwrap();
     }
+    #[test]
+    fn oneboot_mod_twoboot_mod_thirdboot_read() {
+        // first boot: set all to 1
+        let db1 = Database::new();
+        let x = || -> SDSSResult<()> {
+            let mut log = open_log("duatxn.db-tlog", &db1)?;
+            for i in 0..10 {
+                db1.txn_set(i, 1, &mut log)?;
+            }
+            log.close_log()
+        };
+        x().unwrap();
+        let bkp_db1 = db1.copy_data();
+        drop(db1);
+        // second boot
+        let db2 = Database::new();
+        let x = || -> SDSSResult<()> {
+            let mut log = open_log("duatxn.db-tlog", &db2)?;
+            assert_eq!(bkp_db1, db2.copy_data());
+            for i in 0..10 {
+                let current_val = db2.data.borrow()[i];
+                db2.txn_set(i, current_val + i as u8, &mut log)?;
+            }
+            log.close_log()
+        };
+        x().unwrap();
+        let bkp_db2 = db2.copy_data();
+        drop(db2);
+        // third boot
+        let db3 = Database::new();
+        let log = open_log("duatxn.db-tlog", &db3).unwrap();
+        log.close_log().unwrap();
+        assert_eq!(bkp_db2, db3.copy_data());
+        assert_eq!(
+            db3.copy_data(),
+            (1..=10)
+                .into_iter()
+                .map(u8::from)
+                .collect::<Box<[u8]>>()
+                .as_ref()
+        );
+        std::fs::remove_file("duatxn.db-tlog").unwrap();
+    }
 }
diff --git a/server/src/engine/storage/v1/txn.rs b/server/src/engine/storage/v1/txn.rs
index 4fce4b97..404c0cb3 100644
--- a/server/src/engine/storage/v1/txn.rs
+++ b/server/src/engine/storage/v1/txn.rs
@@ -201,7 +201,7 @@ impl<TA: TransactionLogAdapter, LF: RawFileIOInterface> TransactionLogReader<TA,
         // read metadata
         let mut raw_txn_log_row_md = [0u8; TxnLogEntryMetadata::SIZE];
         self.log_file.read_to_buffer(&mut raw_txn_log_row_md)?;
-        self._record_bytes_read(36);
+        self._record_bytes_read(TxnLogEntryMetadata::SIZE);
         let event_metadata = TxnLogEntryMetadata::decode(raw_txn_log_row_md);
         /*
             verify metadata and read bytes into buffer, verify sum
@@ -223,6 +223,16 @@ impl<TA: TransactionLogAdapter, LF: RawFileIOInterface> TransactionLogReader<TA,
             EventSourceMarker::DriverClosed if event_is_zero => {
                 // expect last entry
                 if self.end_of_file() {
+                    // remove the delete entry
+                    self.log_file.seek_from_start(
+                        self.log_size - TxnLogEntryMetadata::SIZE as u64
+                            + SDSSHeaderRaw::header_size() as u64,
+                    )?;
+                    self.log_file.trim_file_to(
+                        self.log_size - TxnLogEntryMetadata::SIZE as u64
+                            + SDSSHeaderRaw::header_size() as u64,
+                    )?;
+                    self.log_file.fsync_all()?;
                     self.closed = true;
                     // good
                     return Ok(());
@@ -284,16 +294,18 @@ pub struct TransactionLogWriter<LF, TA> {
     /// the id of the **next** transaction
     id: u64,
     _m: PhantomData<TA>,
+    closed: bool,
 }
 
 impl<LF: RawFileIOInterface, TA: TransactionLogAdapter> TransactionLogWriter<LF, TA> {
     pub fn new(mut log_file: SDSSFileIO<LF>, last_txn_id: u64) -> SDSSResult<Self> {
-        let l = log_file.file_length()?;
-        log_file.seek_ahead(l)?;
+        let log_size = log_file.file_length()?;
+        log_file.seek_from_start(log_size)?;
         Ok(Self {
             log_file,
             id: last_txn_id,
             _m: PhantomData,
+            closed: false,
         })
     }
     pub fn append_event(&mut self, event: TA::TransactionEvent) -> SDSSResult<()> {
@@ -311,10 +323,12 @@ impl<LF: RawFileIOInterface, TA: TransactionLogAdapter> TransactionLogWriter<LF,
         Ok(())
     }
     pub fn close_log(mut self) -> SDSSResult<()> {
+        self.closed = true;
         let id = self._incr_id() as u128;
         self.log_file.fsynced_write(
             &TxnLogEntryMetadata::new(id, EventSourceMarker::DRIVER_CLOSED, 0, 0).encoded(),
-        )
+        )?;
+        Ok(())
     }
 }
 
@@ -325,3 +339,9 @@ impl<LF, TA> TransactionLogWriter<LF, TA> {
         current
     }
 }
+
+impl<LF, TA> Drop for TransactionLogWriter<LF, TA> {
+    fn drop(&mut self) {
+        assert!(self.closed, "log not closed");
+    }
+}

From e2b81ac77f09ac2374ca1038c4a0517c4ddaebe5 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 31 Jul 2023 07:52:49 +0000
Subject: [PATCH 209/310] Upgrade deps

---
 Cargo.lock                      | 667 +++++++++++---------------------
 cli/Cargo.toml                  |   4 +-
 cli/src/argparse.rs             |   8 +-
 harness/Cargo.toml              |   4 +-
 libstress/Cargo.toml            |   2 +-
 server/Cargo.toml               |  24 +-
 server/src/corestore/map/mod.rs |   2 +-
 sky-bench/Cargo.toml            |   6 +-
 sky-macros/Cargo.toml           |   4 +-
 sky-migrate/Cargo.toml          |   2 +-
 stress-test/Cargo.toml          |   4 +-
 11 files changed, 259 insertions(+), 468 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7e22d582..9f4a718a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -42,13 +42,19 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "0.7.20"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
+checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
 dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "allocator-api2"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
+
 [[package]]
 name = "android-tzdata"
 version = "0.1.1"
@@ -75,13 +81,13 @@ dependencies = [
 
 [[package]]
 name = "async-trait"
-version = "0.1.62"
+version = "0.1.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "689894c2db1ea643a50834b999abf1c110887402542955ff5451dab8f861f9ed"
+checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.27",
 ]
 
 [[package]]
@@ -111,7 +117,7 @@ dependencies = [
  "cc",
  "cfg-if",
  "libc",
- "miniz_oxide 0.7.1",
+ "miniz_oxide",
  "object",
  "rustc-demangle",
 ]
@@ -130,15 +136,15 @@ checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
 
 [[package]]
 name = "base64ct"
-version = "1.5.3"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf"
+checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
 
 [[package]]
 name = "bb8"
-version = "0.8.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1627eccf3aa91405435ba240be23513eeca466b5dc33866422672264de061582"
+checksum = "98b4b0f25f18bcdc3ac72bdb486ed0acf7e185221fd4dc985bc15db5800b0ba2"
 dependencies = [
  "async-trait",
  "futures-channel",
@@ -170,9 +176,9 @@ checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
 
 [[package]]
 name = "block-buffer"
-version = "0.10.3"
+version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
 dependencies = [
  "generic-array",
 ]
@@ -189,9 +195,9 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.12.0"
+version = "3.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
+checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
 
 [[package]]
 name = "byteorder"
@@ -258,9 +264,9 @@ dependencies = [
 
 [[package]]
 name = "cipher"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e"
+checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
 dependencies = [
  "crypto-common",
  "inout",
@@ -293,16 +299,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "codespan-reporting"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
-dependencies = [
- "termcolor",
- "unicode-width",
-]
-
 [[package]]
 name = "constant_time_eq"
 version = "0.1.5"
@@ -311,15 +307,15 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
 
 [[package]]
 name = "core-foundation-sys"
-version = "0.8.3"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
+checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.5"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320"
+checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1"
 dependencies = [
  "libc",
 ]
@@ -360,9 +356,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-deque"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
+checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
 dependencies = [
  "cfg-if",
  "crossbeam-epoch",
@@ -384,9 +380,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.14"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
+checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
 dependencies = [
  "cfg-if",
 ]
@@ -409,9 +405,9 @@ dependencies = [
 
 [[package]]
 name = "crossterm_winapi"
-version = "0.9.0"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c"
+checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
 dependencies = [
  "winapi",
 ]
@@ -426,50 +422,6 @@ dependencies = [
  "typenum",
 ]
 
-[[package]]
-name = "cxx"
-version = "1.0.87"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b61a7545f753a88bcbe0a70de1fcc0221e10bfc752f576754fa91e663db1622e"
-dependencies = [
- "cc",
- "cxxbridge-flags",
- "cxxbridge-macro",
- "link-cplusplus",
-]
-
-[[package]]
-name = "cxx-build"
-version = "1.0.87"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f464457d494b5ed6905c63b0c4704842aba319084a0a3561cdc1359536b53200"
-dependencies = [
- "cc",
- "codespan-reporting",
- "once_cell",
- "proc-macro2",
- "quote",
- "scratch",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "cxxbridge-flags"
-version = "1.0.87"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43c7119ce3a3701ed81aca8410b9acf6fc399d2629d057b87e2efa4e63a3aaea"
-
-[[package]]
-name = "cxxbridge-macro"
-version = "1.0.87"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65e07508b90551e610910fa648a1878991d367064997a596135b86df30daf07e"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "devtimer"
 version = "4.0.1"
@@ -478,41 +430,20 @@ checksum = "907339959a92f6b98846570500c0a567c9aecbb3871cef00561eb5d20d47b7c1"
 
 [[package]]
 name = "digest"
-version = "0.10.6"
+version = "0.10.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
  "block-buffer",
  "crypto-common",
  "subtle",
 ]
 
-[[package]]
-name = "dirs-next"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
-dependencies = [
- "cfg-if",
- "dirs-sys-next",
-]
-
-[[package]]
-name = "dirs-sys-next"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
-dependencies = [
- "libc",
- "redox_users",
- "winapi",
-]
-
 [[package]]
 name = "either"
-version = "1.8.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
+checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 
 [[package]]
 name = "endian-type"
@@ -535,20 +466,9 @@ dependencies = [
 
 [[package]]
 name = "equivalent"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1"
-
-[[package]]
-name = "errno"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
-dependencies = [
- "errno-dragonfly",
- "libc",
- "winapi",
-]
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 
 [[package]]
 name = "errno"
@@ -558,7 +478,7 @@ checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
 dependencies = [
  "errno-dragonfly",
  "libc",
- "windows-sys 0.48.0",
+ "windows-sys",
 ]
 
 [[package]]
@@ -583,23 +503,23 @@ dependencies = [
 
 [[package]]
 name = "fd-lock"
-version = "3.0.8"
+version = "3.0.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb21c69b9fea5e15dbc1049e4b77145dd0ba1c84019c488102de0dc4ea4b0a27"
+checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5"
 dependencies = [
  "cfg-if",
- "rustix 0.36.7",
- "windows-sys 0.42.0",
+ "rustix",
+ "windows-sys",
 ]
 
 [[package]]
 name = "flate2"
-version = "1.0.25"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841"
+checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743"
 dependencies = [
  "crc32fast",
- "miniz_oxide 0.6.2",
+ "miniz_oxide",
 ]
 
 [[package]]
@@ -617,38 +537,32 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
 
-[[package]]
-name = "fs_extra"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
-
 [[package]]
 name = "futures-channel"
-version = "0.3.25"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
+checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
 dependencies = [
  "futures-core",
 ]
 
 [[package]]
 name = "futures-core"
-version = "0.3.25"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
+checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
 
 [[package]]
 name = "futures-task"
-version = "0.3.25"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"
+checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
 
 [[package]]
 name = "futures-util"
-version = "0.3.25"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
+checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -660,9 +574,9 @@ dependencies = [
 
 [[package]]
 name = "generic-array"
-version = "0.14.6"
+version = "0.14.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
 dependencies = [
  "typenum",
  "version_check",
@@ -670,9 +584,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.8"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
+checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
 dependencies = [
  "cfg-if",
  "libc",
@@ -700,19 +614,14 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.13.2"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
+checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
 dependencies = [
  "ahash",
+ "allocator-api2",
 ]
 
-[[package]]
-name = "hashbrown"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
-
 [[package]]
 name = "hermit-abi"
 version = "0.1.19"
@@ -722,15 +631,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "hermit-abi"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "hermit-abi"
 version = "0.3.2"
@@ -746,6 +646,15 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "home"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb"
+dependencies = [
+ "windows-sys",
+]
+
 [[package]]
 name = "humantime"
 version = "2.1.0"
@@ -754,26 +663,25 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.53"
+version = "0.1.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
+checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
  "iana-time-zone-haiku",
  "js-sys",
  "wasm-bindgen",
- "winapi",
+ "windows",
 ]
 
 [[package]]
 name = "iana-time-zone-haiku"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
 dependencies = [
- "cxx",
- "cxx-build",
+ "cc",
 ]
 
 [[package]]
@@ -783,7 +691,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
 dependencies = [
  "equivalent",
- "hashbrown 0.14.0",
+ "hashbrown",
 ]
 
 [[package]]
@@ -795,16 +703,6 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "io-lifetimes"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e"
-dependencies = [
- "libc",
- "windows-sys 0.42.0",
-]
-
 [[package]]
 name = "is-terminal"
 version = "0.4.9"
@@ -812,24 +710,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
 dependencies = [
  "hermit-abi 0.3.2",
- "rustix 0.38.3",
- "windows-sys 0.48.0",
+ "rustix",
+ "windows-sys",
 ]
 
 [[package]]
 name = "itoa"
-version = "1.0.5"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
+checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
 
 [[package]]
 name = "jemalloc-sys"
-version = "0.5.2+5.3.0-patched"
+version = "0.5.3+5.3.0-patched"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "134163979b6eed9564c98637b710b40979939ba351f59952708234ea11b5f3f8"
+checksum = "f9bd5d616ea7ed58b571b2e209a65759664d7fb021a0819d7a790afc67e47ca1"
 dependencies = [
  "cc",
- "fs_extra",
  "libc",
 ]
 
@@ -845,18 +742,18 @@ dependencies = [
 
 [[package]]
 name = "jobserver"
-version = "0.1.25"
+version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b"
+checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
 dependencies = [
  "libc",
 ]
 
 [[package]]
 name = "js-sys"
-version = "0.3.60"
+version = "0.3.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
+checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -887,21 +784,6 @@ dependencies = [
  "rayon",
 ]
 
-[[package]]
-name = "link-cplusplus"
-version = "1.0.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"
-dependencies = [
- "cc",
-]
-
-[[package]]
-name = "linux-raw-sys"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
-
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.3"
@@ -910,9 +792,9 @@ checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
 
 [[package]]
 name = "lock_api"
-version = "0.4.9"
+version = "0.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"
+checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16"
 dependencies = [
  "autocfg",
  "scopeguard",
@@ -920,12 +802,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.17"
+version = "0.4.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
-dependencies = [
- "cfg-if",
-]
+checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
 
 [[package]]
 name = "memchr"
@@ -942,15 +821,6 @@ dependencies = [
  "autocfg",
 ]
 
-[[package]]
-name = "miniz_oxide"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa"
-dependencies = [
- "adler",
-]
-
 [[package]]
 name = "miniz_oxide"
 version = "0.7.1"
@@ -969,7 +839,7 @@ dependencies = [
  "libc",
  "log",
  "wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys 0.48.0",
+ "windows-sys",
 ]
 
 [[package]]
@@ -983,41 +853,41 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.25.1"
+version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4"
+checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a"
 dependencies = [
- "autocfg",
  "bitflags 1.3.2",
  "cfg-if",
  "libc",
+ "static_assertions",
 ]
 
 [[package]]
 name = "ntapi"
-version = "0.4.0"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc51db7b362b205941f71232e56c625156eb9a929f8cf74a428fd5bc094a4afc"
+checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
 dependencies = [
  "winapi",
 ]
 
 [[package]]
 name = "num-traits"
-version = "0.2.15"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
 dependencies = [
  "autocfg",
 ]
 
 [[package]]
 name = "num_cpus"
-version = "1.15.0"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi 0.2.6",
+ "hermit-abi 0.3.2",
  "libc",
 ]
 
@@ -1032,9 +902,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.17.0"
+version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
+checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
 [[package]]
 name = "openssl"
@@ -1053,20 +923,20 @@ dependencies = [
 
 [[package]]
 name = "openssl-macros"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.27",
 ]
 
 [[package]]
 name = "openssl-src"
-version = "111.24.0+1.1.1s"
+version = "111.26.0+1.1.1u"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd"
+checksum = "efc62c9f12b22b8f5208c23a7200a442b2e5999f8bdf80233852122b5a4f6f37"
 dependencies = [
  "cc",
 ]
@@ -1096,15 +966,15 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.6"
+version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf"
+checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
 dependencies = [
  "cfg-if",
  "libc",
  "redox_syscall",
  "smallvec",
- "windows-sys 0.42.0",
+ "windows-targets",
 ]
 
 [[package]]
@@ -1132,9 +1002,9 @@ dependencies = [
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.9"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
+checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57"
 
 [[package]]
 name = "pin-utils"
@@ -1144,9 +1014,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pkg-config"
-version = "0.3.26"
+version = "0.3.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
+checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
 
 [[package]]
 name = "powershell_script"
@@ -1162,18 +1032,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.63"
+version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb"
+checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.29"
+version = "1.0.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
+checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965"
 dependencies = [
  "proc-macro2",
 ]
@@ -1264,29 +1134,30 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.16"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
 dependencies = [
  "bitflags 1.3.2",
 ]
 
 [[package]]
-name = "redox_users"
-version = "0.4.3"
+name = "regex"
+version = "1.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
+checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
 dependencies = [
- "getrandom",
- "redox_syscall",
- "thiserror",
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
 ]
 
 [[package]]
-name = "regex"
-version = "1.7.1"
+name = "regex-automata"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
+checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1295,9 +1166,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.28"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
+checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
 
 [[package]]
 name = "rustc-demangle"
@@ -1307,42 +1178,28 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
 
 [[package]]
 name = "rustix"
-version = "0.36.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03"
-dependencies = [
- "bitflags 1.3.2",
- "errno 0.2.8",
- "io-lifetimes",
- "libc",
- "linux-raw-sys 0.1.4",
- "windows-sys 0.42.0",
-]
-
-[[package]]
-name = "rustix"
-version = "0.38.3"
+version = "0.38.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac5ffa1efe7548069688cd7028f32591853cd7b5b756d41bcffd2353e4fc75b4"
+checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5"
 dependencies = [
  "bitflags 2.3.3",
- "errno 0.3.1",
+ "errno",
  "libc",
- "linux-raw-sys 0.4.3",
- "windows-sys 0.48.0",
+ "linux-raw-sys",
+ "windows-sys",
 ]
 
 [[package]]
 name = "rustyline"
-version = "10.1.1"
+version = "12.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1e83c32c3f3c33b08496e0d1df9ea8c64d39adb8eb36a1ebb1440c690697aef"
+checksum = "994eca4bca05c87e86e15d90fc7a91d1be64b4482b38cb2d27474568fe7c9db9"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags 2.3.3",
  "cfg-if",
  "clipboard-win",
- "dirs-next",
  "fd-lock",
+ "home",
  "libc",
  "log",
  "memchr",
@@ -1357,56 +1214,50 @@ dependencies = [
 
 [[package]]
 name = "ryu"
-version = "1.0.12"
+version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
+checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
 
 [[package]]
 name = "scheduled-thread-pool"
-version = "0.2.6"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf"
+checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"
 dependencies = [
  "parking_lot",
 ]
 
 [[package]]
 name = "scopeguard"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
-
-[[package]]
-name = "scratch"
-version = "1.0.3"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
 [[package]]
 name = "serde"
-version = "1.0.167"
+version = "1.0.175"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7daf513456463b42aa1d94cff7e0c24d682b429f020b9afa4f5ba5c40a22b237"
+checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.167"
+version = "1.0.175"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b69b106b68bc8054f0e974e70d19984040f8a5cf9215ca82626ea4853f82c4b9"
+checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.27",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.100"
+version = "1.0.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f1e14e89be7aa4c4b78bdbdc9eb5bf8517829a600ae8eaa39a6e1d960b5185c"
+checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b"
 dependencies = [
  "itoa",
  "ryu",
@@ -1435,9 +1286,9 @@ dependencies = [
 
 [[package]]
 name = "sha2"
-version = "0.10.6"
+version = "0.10.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"
+checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -1446,9 +1297,9 @@ dependencies = [
 
 [[package]]
 name = "signal-hook"
-version = "0.3.14"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d"
+checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801"
 dependencies = [
  "libc",
  "signal-hook-registry",
@@ -1467,9 +1318,9 @@ dependencies = [
 
 [[package]]
 name = "signal-hook-registry"
-version = "1.4.0"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0"
+checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
 dependencies = [
  "libc",
 ]
@@ -1524,7 +1375,7 @@ dependencies = [
  "crc",
  "crossbeam-epoch",
  "env_logger",
- "hashbrown 0.13.2",
+ "hashbrown",
  "jemallocator",
  "libc",
  "libsky",
@@ -1561,7 +1412,7 @@ dependencies = [
 [[package]]
 name = "skytable"
 version = "0.8.0"
-source = "git+https://github.com/skytable/client-rust?branch=next#c57e46e6ddce57c221e41cd038e7eb8296d2d732"
+source = "git+https://github.com/skytable/client-rust?branch=next#a55fbdc964e34c75c99404ea2395d03fd302daee"
 dependencies = [
  "async-trait",
  "bb8",
@@ -1575,25 +1426,25 @@ dependencies = [
 [[package]]
 name = "skytable"
 version = "0.8.0"
-source = "git+https://github.com/skytable/client-rust.git#c57e46e6ddce57c221e41cd038e7eb8296d2d732"
+source = "git+https://github.com/skytable/client-rust.git#a55fbdc964e34c75c99404ea2395d03fd302daee"
 dependencies = [
  "r2d2",
 ]
 
 [[package]]
 name = "slab"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef"
+checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d"
 dependencies = [
  "autocfg",
 ]
 
 [[package]]
 name = "smallvec"
-version = "1.10.0"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
+checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
 
 [[package]]
 name = "socket2"
@@ -1605,6 +1456,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "str-buf"
 version = "1.0.6"
@@ -1633,9 +1490,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
 
 [[package]]
 name = "subtle"
-version = "2.4.1"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
+checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
 
 [[package]]
 name = "syn"
@@ -1650,9 +1507,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.23"
+version = "2.0.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737"
+checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1661,9 +1518,9 @@ dependencies = [
 
 [[package]]
 name = "sysinfo"
-version = "0.29.4"
+version = "0.29.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "751e810399bba86e9326f5762b7f32ac5a085542df78da6a78d94e07d14d7c11"
+checksum = "c7cb97a5a85a136d84e75d5c3cf89655090602efb1be0d8d5337b7e386af2908"
 dependencies = [
  "cfg-if",
  "core-foundation-sys",
@@ -1692,26 +1549,6 @@ dependencies = [
  "unicode-width",
 ]
 
-[[package]]
-name = "thiserror"
-version = "1.0.38"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "1.0.38"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "time"
 version = "0.1.45"
@@ -1725,9 +1562,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.17"
+version = "0.3.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"
+checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446"
 dependencies = [
  "serde",
  "time-core",
@@ -1735,9 +1572,9 @@ dependencies = [
 
 [[package]]
 name = "time-core"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
+checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
 
 [[package]]
 name = "tokio"
@@ -1756,7 +1593,7 @@ dependencies = [
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "windows-sys 0.48.0",
+ "windows-sys",
 ]
 
 [[package]]
@@ -1767,7 +1604,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.27",
 ]
 
 [[package]]
@@ -1805,9 +1642,9 @@ dependencies = [
 
 [[package]]
 name = "toml_edit"
-version = "0.19.12"
+version = "0.19.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c500344a19072298cd05a7224b3c0c629348b78692bf48466c5238656e315a78"
+checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a"
 dependencies = [
  "indexmap",
  "serde",
@@ -1824,15 +1661,15 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.6"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
+checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.10.0"
+version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a"
+checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
 
 [[package]]
 name = "unicode-width"
@@ -1842,15 +1679,15 @@ checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
 
 [[package]]
 name = "utf8parse"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
+checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
 
 [[package]]
 name = "uuid"
-version = "1.4.0"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be"
+checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d"
 dependencies = [
  "getrandom",
  "rand",
@@ -1859,13 +1696,13 @@ dependencies = [
 
 [[package]]
 name = "uuid-macro-internal"
-version = "1.4.0"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8614dda80b9075fbca36bc31b58d1447715b1236af98dee21db521c47a0cc2c0"
+checksum = "f7e1ba1f333bd65ce3c9f27de592fcbc256dafe3af2717f56d7c87761fbaccf4"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.23",
+ "syn 2.0.27",
 ]
 
 [[package]]
@@ -1900,9 +1737,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.83"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
+checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342"
 dependencies = [
  "cfg-if",
  "wasm-bindgen-macro",
@@ -1910,24 +1747,24 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.83"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
+checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd"
 dependencies = [
  "bumpalo",
  "log",
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.27",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.83"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
+checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -1935,22 +1772,22 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.83"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
+checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.27",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.83"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
+checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
 
 [[package]]
 name = "winapi"
@@ -1984,18 +1821,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
-name = "windows-sys"
-version = "0.42.0"
+name = "windows"
+version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
+checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
 dependencies = [
- "windows_aarch64_gnullvm 0.42.1",
- "windows_aarch64_msvc 0.42.1",
- "windows_i686_gnu 0.42.1",
- "windows_i686_msvc 0.42.1",
- "windows_x86_64_gnu 0.42.1",
- "windows_x86_64_gnullvm 0.42.1",
- "windows_x86_64_msvc 0.42.1",
+ "windows-targets",
 ]
 
 [[package]]
@@ -2013,93 +1844,51 @@ version = "0.48.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
 dependencies = [
- "windows_aarch64_gnullvm 0.48.0",
- "windows_aarch64_msvc 0.48.0",
- "windows_i686_gnu 0.48.0",
- "windows_i686_msvc 0.48.0",
- "windows_x86_64_gnu 0.48.0",
- "windows_x86_64_gnullvm 0.48.0",
- "windows_x86_64_msvc 0.48.0",
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
 ]
 
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.42.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
-
 [[package]]
 name = "windows_aarch64_gnullvm"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
 
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.42.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
-
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
 
-[[package]]
-name = "windows_i686_gnu"
-version = "0.42.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
-
 [[package]]
 name = "windows_i686_gnu"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
 
-[[package]]
-name = "windows_i686_msvc"
-version = "0.42.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
-
 [[package]]
 name = "windows_i686_msvc"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
 
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.42.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
-
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
 
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.42.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
-
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
 
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.42.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
-
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.48.0"
@@ -2108,9 +1897,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
 
 [[package]]
 name = "winnow"
-version = "0.4.8"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9482fe6ceabdf32f3966bfdd350ba69256a97c30253dc616fe0005af24f164e"
+checksum = "81fac9742fd1ad1bd9643b991319f72dd031016d44b77039a26977eb667141e7"
 dependencies = [
  "memchr",
 ]
@@ -2137,7 +1926,7 @@ dependencies = [
  "hmac",
  "pbkdf2",
  "sha1",
- "time 0.3.17",
+ "time 0.3.23",
  "zstd",
 ]
 
@@ -2162,9 +1951,9 @@ dependencies = [
 
 [[package]]
 name = "zstd-sys"
-version = "2.0.5+zstd.1.5.2"
+version = "2.0.8+zstd.1.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edc50ffce891ad571e9f9afe5039c4837bede781ac4bb13052ed7ae695518596"
+checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c"
 dependencies = [
  "cc",
  "libc",
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index 033c7108..0402ce65 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -14,8 +14,8 @@ skytable = { git = "https://github.com/skytable/client-rust", branch = "next", f
   "aio-sslv",
 ], default-features = false }
 # external deps
-tokio = { version = "1.28.1", features = ["full"] }
+tokio = { version = "1.29.1", features = ["full"] }
 clap = { version = "2", features = ["yaml"] }
-rustyline = "10.1.1"
+rustyline = "12.0.0"
 crossterm = "0.26.1"
 lazy_static = "1.4.0"
diff --git a/cli/src/argparse.rs b/cli/src/argparse.rs
index e8a3f0b3..12ad8e80 100644
--- a/cli/src/argparse.rs
+++ b/cli/src/argparse.rs
@@ -32,7 +32,7 @@ use {
         terminal::{Clear, ClearType},
     },
     libsky::{URL, VERSION},
-    rustyline::{config::Configurer, error::ReadlineError, Editor},
+    rustyline::{config::Configurer, error::ReadlineError, DefaultEditor},
     skytable::{Pipeline, Query},
     std::{io::stdout, process},
 };
@@ -110,12 +110,14 @@ pub async fn start_repl() {
         },
         None => 2003,
     };
-    let mut editor = match Editor::<()>::new() {
+    let mut editor = match DefaultEditor::new() {
         Ok(e) => e,
         Err(e) => fatal!("Editor init error: {}", e),
     };
     editor.set_auto_add_history(true);
-    editor.set_history_ignore_dups(true);
+    editor
+        .set_history_ignore_dups(true)
+        .unwrap_or_else(|_| fatal!("couldn't set up terminal"));
     editor.bind_sequence(
         rustyline::KeyEvent(
             rustyline::KeyCode::BracketedPasteStart,
diff --git a/harness/Cargo.toml b/harness/Cargo.toml
index caab1961..246f9243 100644
--- a/harness/Cargo.toml
+++ b/harness/Cargo.toml
@@ -13,7 +13,7 @@ skytable = { git = "https://github.com/skytable/client-rust.git", features = [
 libsky = { path = "../libsky" }
 # external deps
 env_logger = "0.10.0"
-log = "0.4.17"
+log = "0.4.19"
 zip = { version = "0.6.6", features = ["deflate"] }
 powershell_script = "1.1.0"
-openssl = { version = "0.10.52", features = ["vendored"] }
+openssl = { version = "0.10.55", features = ["vendored"] }
diff --git a/libstress/Cargo.toml b/libstress/Cargo.toml
index 99f89c99..dc66438b 100644
--- a/libstress/Cargo.toml
+++ b/libstress/Cargo.toml
@@ -10,5 +10,5 @@ edition = "2021"
 # external deps
 crossbeam-channel = "0.5.8"
 rayon = "1.7.0"
-log = "0.4.17"
+log = "0.4.19"
 rand = "0.8.5"
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 1e5e2cc3..565c2997 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -13,21 +13,21 @@ rcrypt = "0.4.0"
 # external deps
 ahash = "0.8.3"
 bytes = "1.4.0"
-chrono = "0.4.24"
+chrono = "0.4.26"
 clap = { version = "2", features = ["yaml"] }
 env_logger = "0.10.0"
-hashbrown = { version = "0.13.2", features = ["raw"] }
-log = "0.4.17"
-openssl = { version = "0.10.52", features = ["vendored"] }
-crossbeam-epoch = { version = "0.9.14" }
+hashbrown = { version = "0.14.0", features = ["raw"] }
+log = "0.4.19"
+openssl = { version = "0.10.55", features = ["vendored"] }
+crossbeam-epoch = { version = "0.9.15" }
 parking_lot = "0.12.1"
-regex = "1.6.0"
-serde = { version = "1.0.163", features = ["derive"] }
-tokio = { version = "1.28.1", features = ["full"] }
+regex = "1.9.1"
+serde = { version = "1.0.175", features = ["derive"] }
+tokio = { version = "1.29.1", features = ["full"] }
 tokio-openssl = "0.6.3"
-toml = "0.7.4"
+toml = "0.7.6"
 base64 = "0.21.2"
-uuid = { version = "1.3.3", features = ["v4", "fast-rng", "macro-diagnostics"] }
+uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics"] }
 crc = "3.0.1"
 
 [target.'cfg(all(not(target_env = "msvc"), not(miri)))'.dependencies]
@@ -39,7 +39,7 @@ winapi = { version = "0.3.9", features = ["fileapi"] }
 
 [target.'cfg(unix)'.dependencies]
 # external deps
-libc = "0.2.144"
+libc = "0.2.147"
 
 [target.'cfg(unix)'.build-dependencies]
 # external deps
@@ -55,7 +55,7 @@ skytable = { git = "https://github.com/skytable/client-rust", features = [
 # external deps
 bincode = "1.3.3"
 rand = "0.8.5"
-tokio = { version = "1.28.1", features = ["test-util"] }
+tokio = { version = "1.29.1", features = ["test-util"] }
 
 [features]
 nightly = []
diff --git a/server/src/corestore/map/mod.rs b/server/src/corestore/map/mod.rs
index a7c0f600..8427b2bb 100644
--- a/server/src/corestore/map/mod.rs
+++ b/server/src/corestore/map/mod.rs
@@ -277,7 +277,7 @@ where
                 Some(bucket) => {
                     let (kptr, vptr) = bucket.as_ref();
                     if f(kptr, vptr) {
-                        Some(lowtable.remove(bucket))
+                        Some(lowtable.remove(bucket).0)
                     } else {
                         None
                     }
diff --git a/sky-bench/Cargo.toml b/sky-bench/Cargo.toml
index a795012d..cf6ed8f5 100644
--- a/sky-bench/Cargo.toml
+++ b/sky-bench/Cargo.toml
@@ -15,9 +15,9 @@ skytable = { git = "https://github.com/skytable/client-rust.git", features = [
 libstress = { path = "../libstress" }
 # external deps
 clap = { version = "2", features = ["yaml"] }
-log = "0.4.17"
+log = "0.4.19"
 env_logger = "0.10.0"
 devtimer = "4.0.1"
-serde = { version = "1.0.163", features = ["derive"] }
-serde_json = "1.0.96"
+serde = { version = "1.0.175", features = ["derive"] }
+serde_json = "1.0.103"
 rand = "0.8.5"
diff --git a/sky-macros/Cargo.toml b/sky-macros/Cargo.toml
index ba17531f..b2052679 100644
--- a/sky-macros/Cargo.toml
+++ b/sky-macros/Cargo.toml
@@ -11,7 +11,7 @@ proc-macro = true
 
 [dependencies]
 # external deps
-proc-macro2 = "1.0.58"
-quote = "1.0.27"
+proc-macro2 = "1.0.66"
+quote = "1.0.32"
 rand = "0.8.5"
 syn = { version = "1.0.109", features = ["full"] }
diff --git a/sky-migrate/Cargo.toml b/sky-migrate/Cargo.toml
index c55e919e..30dfd28f 100644
--- a/sky-migrate/Cargo.toml
+++ b/sky-migrate/Cargo.toml
@@ -10,5 +10,5 @@ edition = "2021"
 skytable = { git = "https://github.com/skytable/client-rust.git" }
 env_logger = "0.10.0"
 bincode = "1.3.3"
-log = "0.4.17"
+log = "0.4.19"
 clap = { version = "2", features = ["yaml"] }
diff --git a/stress-test/Cargo.toml b/stress-test/Cargo.toml
index 7b5bf6d6..6211e517 100644
--- a/stress-test/Cargo.toml
+++ b/stress-test/Cargo.toml
@@ -14,8 +14,8 @@ skytable = { git = "https://github.com/skytable/client-rust.git", branch = "next
 ] }
 devtimer = "4.0.1"
 # external deps
-sysinfo = "0.29.0"
+sysinfo = "0.29.6"
 env_logger = "0.10.0"
-log = "0.4.17"
+log = "0.4.19"
 rand = "0.8.5"
 crossbeam-channel = "0.5.8"

From 263e70905d7c10fbe4da5943d10920493c80d563 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 31 Jul 2023 14:31:57 +0000
Subject: [PATCH 210/310] Rewrite journal: recovery impl and driver init

On initialization the driver will no longer attempt to truncate the
close event but instead use a truly AO style and utilize a new
directive.

We also now have a basic externally triggered on write and automatically
trigger on read recovery system in place. However, there are several
scopes of improvement to the implementation, such as tolerating even
more inconsistencies in the data.
---
 server/src/engine/core/dml/upd.rs             |   2 +-
 server/src/engine/core/model/alt.rs           |   2 +-
 .../src/engine/storage/v1/header_impl/gr.rs   |   4 +-
 .../src/engine/storage/v1/header_impl/mod.rs  |   4 +-
 server/src/engine/storage/v1/journal.rs       | 448 ++++++++++++++++++
 server/src/engine/storage/v1/mod.rs           |   7 +-
 server/src/engine/storage/v1/rw.rs            |   7 +
 server/src/engine/storage/v1/tests.rs         |  35 +-
 server/src/engine/storage/v1/txn.rs           | 347 --------------
 server/src/util/mod.rs                        |  16 +
 10 files changed, 500 insertions(+), 372 deletions(-)
 create mode 100644 server/src/engine/storage/v1/journal.rs
 delete mode 100644 server/src/engine/storage/v1/txn.rs

diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index d15810a6..24237033 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -238,7 +238,7 @@ pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()>
         let irm = mdl.intent_read_model();
         let g = sync::atm::cpin();
         let Some(row) = mdl.primary_index().select(key, &g) else {
-            return Err(DatabaseError::DmlEntryNotFound)
+            return Err(DatabaseError::DmlEntryNotFound);
         };
         let mut row_data_wl = row.d_data().write();
         let mut rollback_now = false;
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index c7ac8d7f..82163a44 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -214,7 +214,7 @@ impl<'a> AlterPlan<'a> {
             // actually parse the new layer
             okay &= props.is_empty();
             let Some(new_parsed_layer) = Layer::get_layer(&ty) else {
-                return Err(DatabaseError::DdlModelAlterBadTypedef)
+                return Err(DatabaseError::DdlModelAlterBadTypedef);
             };
             match (
                 current_layer.tag.tag_selector(),
diff --git a/server/src/engine/storage/v1/header_impl/gr.rs b/server/src/engine/storage/v1/header_impl/gr.rs
index 1565dbfd..22653a07 100644
--- a/server/src/engine/storage/v1/header_impl/gr.rs
+++ b/server/src/engine/storage/v1/header_impl/gr.rs
@@ -448,13 +448,13 @@ impl GRHostRecordRaw {
 #[test]
 fn test_metadata_record_encode_decode() {
     let md = GRMetadataRecordRaw::new(
-        FileScope::TransactionLog,
+        FileScope::Journal,
         FileSpecifier::GNSTxnLog,
         FileSpecifierVersion(1),
     );
     assert_eq!(md.read_p0_server_version(), versions::v1::V1_SERVER_VERSION);
     assert_eq!(md.read_p1_driver_version(), versions::v1::V1_DRIVER_VERSION);
-    assert_eq!(md.read_p2_file_scope(), FileScope::TransactionLog);
+    assert_eq!(md.read_p2_file_scope(), FileScope::Journal);
     assert_eq!(md.read_p3_file_spec(), FileSpecifier::GNSTxnLog);
     assert_eq!(md.read_p4_file_spec_version(), FileSpecifierVersion(1));
 }
diff --git a/server/src/engine/storage/v1/header_impl/mod.rs b/server/src/engine/storage/v1/header_impl/mod.rs
index 4e2fa051..abec2a20 100644
--- a/server/src/engine/storage/v1/header_impl/mod.rs
+++ b/server/src/engine/storage/v1/header_impl/mod.rs
@@ -71,14 +71,14 @@ mod dr;
 #[repr(u8)]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
 pub enum FileScope {
-    TransactionLog = 0,
+    Journal = 0,
     TransactionLogCompacted = 1,
 }
 
 impl FileScope {
     pub const fn try_new(id: u64) -> Option<Self> {
         Some(match id {
-            0 => Self::TransactionLog,
+            0 => Self::Journal,
             1 => Self::TransactionLogCompacted,
             _ => return None,
         })
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
new file mode 100644
index 00000000..6d2864a4
--- /dev/null
+++ b/server/src/engine/storage/v1/journal.rs
@@ -0,0 +1,448 @@
+/*
+ * Created on Sat Jul 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/*
+  +----------------+------------------------------+------------------+------------------+--------------------+
+  | EVENT ID (16B) | EVENT SOURCE + METADATA (8B) | EVENT CRC32 (4B) | PAYLOAD LEN (8B) | EVENT PAYLOAD (?B) |
+  +----------------+------------------------------+------------------+------------------+--------------------+
+  Event ID:
+  - The atomically incrementing event ID (for future scale we have 16B; it's like the ZFS situation haha)
+  - Event source (1B) + 7B padding (for future metadata)
+  - Event CRC32
+  - Payload len: the size of the pyload
+  - Payload: the payload
+
+
+  Notes on error tolerance:
+  - FIXME(@ohsayan): we currently expect atleast 36 bytes of the signature to be present. this is not great
+  - FIXME(@ohsayan): we will probably (naively) need to dynamically reposition the cursor in case the metadata is corrupted as well
+*/
+
+use {
+    super::{
+        header_impl::{FileSpecifierVersion, HostRunMode, SDSSHeaderRaw},
+        rw::{FileOpen, RawFileIOInterface, SDSSFileIO},
+        SDSSError, SDSSResult,
+    },
+    crate::{
+        engine::storage::v1::header_impl::{FileScope, FileSpecifier},
+        util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy, Threshold},
+    },
+    std::marker::PhantomData,
+};
+
+const CRC: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
+const RECOVERY_BLOCK_AUTO_THRESHOLD: usize = 5;
+
+pub fn open_journal<
+    TA: JournalAdapter + core::fmt::Debug,
+    LF: RawFileIOInterface + core::fmt::Debug,
+>(
+    log_file_name: &str,
+    log_kind: FileSpecifier,
+    log_kind_version: FileSpecifierVersion,
+    host_setting_version: u32,
+    host_run_mode: HostRunMode,
+    host_startup_counter: u64,
+    gs: &TA::GlobalState,
+) -> SDSSResult<JournalWriter<LF, TA>> {
+    let f = SDSSFileIO::<LF>::open_or_create_perm_rw(
+        log_file_name,
+        FileScope::Journal,
+        log_kind,
+        log_kind_version,
+        host_setting_version,
+        host_run_mode,
+        host_startup_counter,
+    )?;
+    let file = match f {
+        FileOpen::Created(f) => return JournalWriter::new(f, 0, true),
+        FileOpen::Existing(file, _) => file,
+    };
+    let (file, last_txn) = JournalReader::<TA, LF>::scroll(file, gs)?;
+    JournalWriter::new(file, last_txn, false)
+}
+
+/// The journal adapter
+pub trait JournalAdapter {
+    /// enable/disable automated recovery algorithms
+    const RECOVERY_PLUGIN: bool;
+    /// The journal event
+    type JournalEvent;
+    /// The global state, which we want to modify on decoding the event
+    type GlobalState;
+    /// Encode a journal event into a blob
+    fn encode(event: Self::JournalEvent) -> Box<[u8]>;
+    /// Decode a journal event and apply it to the global state
+    fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> SDSSResult<()>;
+}
+
+#[derive(Debug)]
+pub struct JournalEntryMetadata {
+    event_id: u128,
+    event_source_md: u64,
+    event_crc: u32,
+    event_payload_len: u64,
+}
+
+impl JournalEntryMetadata {
+    const SIZE: usize = sizeof!(u128) + sizeof!(u64) + sizeof!(u32) + sizeof!(u64);
+    const P0: usize = 0;
+    const P1: usize = sizeof!(u128);
+    const P2: usize = Self::P1 + sizeof!(u64);
+    const P3: usize = Self::P2 + sizeof!(u32);
+    pub const fn new(
+        event_id: u128,
+        event_source_md: u64,
+        event_crc: u32,
+        event_payload_len: u64,
+    ) -> Self {
+        Self {
+            event_id,
+            event_source_md,
+            event_crc,
+            event_payload_len,
+        }
+    }
+    /// Encodes the log entry metadata
+    pub const fn encoded(&self) -> [u8; JournalEntryMetadata::SIZE] {
+        let mut encoded = [0u8; JournalEntryMetadata::SIZE];
+        encoded = copy_a_into_b(self.event_id.to_le_bytes(), encoded, Self::P0);
+        encoded = copy_a_into_b(self.event_source_md.to_le_bytes(), encoded, Self::P1);
+        encoded = copy_a_into_b(self.event_crc.to_le_bytes(), encoded, Self::P2);
+        encoded = copy_a_into_b(self.event_payload_len.to_le_bytes(), encoded, Self::P3);
+        encoded
+    }
+    /// Decodes the log entry metadata (essentially a simply type transmutation)
+    pub fn decode(data: [u8; JournalEntryMetadata::SIZE]) -> Self {
+        Self::new(
+            u128::from_le_bytes(memcpy(&data[..Self::P1])),
+            u64::from_le_bytes(memcpy(&data[Self::P1..Self::P2])),
+            u32::from_le_bytes(memcpy(&data[Self::P2..Self::P3])),
+            u64::from_le_bytes(memcpy(&data[Self::P3..])),
+        )
+    }
+}
+
+/*
+    Event source:
+    * * * * _ * * * *
+
+    b1 (s+d): event source (unset -> driver, set -> server)
+    b* -> unused. MUST be unset
+    b7 (d):
+        - set: [recovery] reverse journal event
+    b8 (d):
+        - unset: closed log
+        - set: reopened log
+*/
+
+pub enum EventSourceMarker {
+    ServerStandard,
+    DriverClosed,
+    RecoveryReverseLastJournal,
+    DriverReopened,
+}
+
+impl EventSourceMarker {
+    const SERVER_STD: u64 = 1 << 63;
+    const DRIVER_CLOSED: u64 = 0;
+    const DRIVER_REOPENED: u64 = 1;
+    const RECOVERY_REVERSE_LAST_JOURNAL: u64 = 2;
+}
+
+impl JournalEntryMetadata {
+    pub const fn is_server_event(&self) -> bool {
+        self.event_source_md == EventSourceMarker::SERVER_STD
+    }
+    pub const fn is_driver_event(&self) -> bool {
+        self.event_source_md <= 1
+    }
+    pub const fn event_source_marker(&self) -> Option<EventSourceMarker> {
+        Some(match self.event_source_md {
+            EventSourceMarker::SERVER_STD => EventSourceMarker::ServerStandard,
+            EventSourceMarker::DRIVER_CLOSED => EventSourceMarker::DriverClosed,
+            EventSourceMarker::DRIVER_REOPENED => EventSourceMarker::DriverReopened,
+            EventSourceMarker::RECOVERY_REVERSE_LAST_JOURNAL => {
+                EventSourceMarker::RecoveryReverseLastJournal
+            }
+            _ => return None,
+        })
+    }
+}
+
+#[derive(Debug)]
+pub struct JournalReader<TA, LF> {
+    log_file: SDSSFileIO<LF>,
+    log_size: u64,
+    evid: u64,
+    closed: bool,
+    remaining_bytes: u64,
+    _m: PhantomData<TA>,
+}
+
+impl<TA: JournalAdapter, LF: RawFileIOInterface> JournalReader<TA, LF> {
+    pub fn new(log_file: SDSSFileIO<LF>) -> SDSSResult<Self> {
+        let log_size = log_file.file_length()? - SDSSHeaderRaw::header_size() as u64;
+        Ok(Self {
+            log_file,
+            log_size,
+            evid: 0,
+            closed: false,
+            remaining_bytes: log_size,
+            _m: PhantomData,
+        })
+    }
+    /// Read the next event and apply it to the global state
+    pub fn rapply_next_event(&mut self, gs: &TA::GlobalState) -> SDSSResult<()> {
+        // read metadata
+        let mut en_jrnl_md = [0u8; JournalEntryMetadata::SIZE];
+        self.logfile_read_into_buffer(&mut en_jrnl_md)?; // FIXME(@ohsayan): increase tolerance to not just payload
+        let entry_metadata = JournalEntryMetadata::decode(en_jrnl_md);
+        /*
+            validate metadata:
+            - evid
+            - sourcemd
+            - sum
+            - len < alloc cap; FIXME(@ohsayan): more sensitive via alloc?
+        */
+        if self.evid != entry_metadata.event_id as u64 {
+            // the only case when this happens is when the journal faults at runtime with a write zero (or some other error when no bytes were written)
+            self.remaining_bytes += JournalEntryMetadata::SIZE as u64;
+            // move back cursor
+            let new_cursor = self.log_file.retrieve_cursor()? - JournalEntryMetadata::SIZE as u64;
+            self.log_file.seek_from_start(new_cursor)?;
+            return self.try_recover_journal_strategy_simple_reverse();
+        }
+        match entry_metadata
+            .event_source_marker()
+            .ok_or(SDSSError::JournalLogEntryCorrupted)?
+        {
+            EventSourceMarker::ServerStandard => {}
+            EventSourceMarker::DriverClosed => {
+                // is this a real close?
+                if self.end_of_file() {
+                    self.closed = true;
+                    return Ok(());
+                } else {
+                    return self.handle_driver_reopen();
+                }
+            }
+            EventSourceMarker::DriverReopened | EventSourceMarker::RecoveryReverseLastJournal => {
+                // these two are only taken in close and error paths (respectively) so we shouldn't see them here; this is bad
+                // two special directives in the middle of nowhere? incredible
+                return Err(SDSSError::JournalCorrupted);
+            }
+        }
+        // read payload
+        if compiler::unlikely(!self.has_remaining_bytes(entry_metadata.event_payload_len)) {
+            return compiler::cold_call(|| self.try_recover_journal_strategy_simple_reverse());
+        }
+        let mut payload = vec![0; entry_metadata.event_payload_len as usize];
+        self.logfile_read_into_buffer(&mut payload)?; // exit jump -> we checked if enough data is there, but the read failed so this is not our business
+        if compiler::unlikely(CRC.checksum(&payload) != entry_metadata.event_crc) {
+            return compiler::cold_call(|| self.try_recover_journal_strategy_simple_reverse());
+        }
+        if compiler::unlikely(TA::decode_and_update_state(&payload, gs).is_err()) {
+            return compiler::cold_call(|| self.try_recover_journal_strategy_simple_reverse());
+        }
+        self._incr_evid();
+        Ok(())
+    }
+    /// handle a driver reopen (IMPORTANT: every event is unique so this must be called BEFORE the ID is incremented)
+    fn handle_driver_reopen(&mut self) -> SDSSResult<()> {
+        if self.has_remaining_bytes(JournalEntryMetadata::SIZE as _) {
+            let mut reopen_block = [0u8; JournalEntryMetadata::SIZE];
+            self.logfile_read_into_buffer(&mut reopen_block)?; // exit jump -> not our business since we have checked flen and if it changes due to user intervention, that's a you problem
+            let md = JournalEntryMetadata::decode(reopen_block);
+            if (md.event_id as u64 == self.evid)
+                & (md.event_crc == 0)
+                & (md.event_payload_len == 0)
+                & (md.event_source_md == EventSourceMarker::DRIVER_REOPENED)
+            {
+                self._incr_evid();
+                Ok(())
+            } else {
+                // FIXME(@ohsayan): tolerate loss in this directive too
+                Err(SDSSError::JournalCorrupted)
+            }
+        } else {
+            Err(SDSSError::JournalCorrupted)
+        }
+    }
+    #[cold] // FIXME(@ohsayan): how bad can prod systems be? (clue: pretty bad, so look for possible changes)
+    #[inline(never)]
+    /// attempt to recover the journal using the reverse directive (simple strategy)
+    /// IMPORTANT: every event is unique so this must be called BEFORE the ID is incremented (remember that we only increment
+    /// once we **sucessfully** finish processing a normal (aka server event origin) event and not a non-normal branch)
+    fn try_recover_journal_strategy_simple_reverse(&mut self) -> SDSSResult<()> {
+        debug_assert!(TA::RECOVERY_PLUGIN, "recovery plugin not enabled");
+        let mut threshold = Threshold::<RECOVERY_BLOCK_AUTO_THRESHOLD>::new();
+        while threshold.not_busted() & self.has_remaining_bytes(JournalEntryMetadata::SIZE as _) {
+            self.__record_read_bytes(JournalEntryMetadata::SIZE); // FIXME(@ohsayan): don't assume read length?
+            let mut entry_buf = [0u8; JournalEntryMetadata::SIZE];
+            if self.log_file.read_to_buffer(&mut entry_buf).is_err() {
+                threshold.bust_one();
+                continue;
+            }
+            let entry = JournalEntryMetadata::decode(entry_buf);
+            let okay = (entry.event_id == self.evid as u128)
+                & (entry.event_crc == 0)
+                & (entry.event_payload_len == 0)
+                & (entry.event_source_md == EventSourceMarker::RECOVERY_REVERSE_LAST_JOURNAL);
+            if okay {
+                return Ok(());
+            }
+            self._incr_evid();
+            threshold.bust_one();
+        }
+        Err(SDSSError::JournalCorrupted)
+    }
+    /// Read and apply all events in the given log file to the global state, returning the (open file, last event ID)
+    pub fn scroll(file: SDSSFileIO<LF>, gs: &TA::GlobalState) -> SDSSResult<(SDSSFileIO<LF>, u64)> {
+        let mut slf = Self::new(file)?;
+        while !slf.end_of_file() {
+            slf.rapply_next_event(gs)?;
+        }
+        if slf.closed {
+            Ok((slf.log_file, slf.evid))
+        } else {
+            Err(SDSSError::JournalCorrupted)
+        }
+    }
+}
+
+impl<TA, LF> JournalReader<TA, LF> {
+    fn _incr_evid(&mut self) {
+        self.evid += 1;
+    }
+    fn __record_read_bytes(&mut self, cnt: usize) {
+        self.remaining_bytes -= cnt as u64;
+    }
+    fn has_remaining_bytes(&self, size: u64) -> bool {
+        self.remaining_bytes >= size
+    }
+    fn end_of_file(&self) -> bool {
+        self.remaining_bytes == 0
+    }
+}
+
+impl<TA, LF: RawFileIOInterface> JournalReader<TA, LF> {
+    fn logfile_read_into_buffer(&mut self, buf: &mut [u8]) -> SDSSResult<()> {
+        if !self.has_remaining_bytes(buf.len() as _) {
+            // do this right here to avoid another syscall
+            return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof).into());
+        }
+        self.log_file.read_to_buffer(buf)?;
+        self.__record_read_bytes(buf.len());
+        Ok(())
+    }
+}
+
+pub struct JournalWriter<LF, TA> {
+    /// the txn log file
+    log_file: SDSSFileIO<LF>,
+    /// the id of the **next** journal
+    id: u64,
+    _m: PhantomData<TA>,
+    closed: bool,
+}
+
+impl<LF: RawFileIOInterface, TA: JournalAdapter> JournalWriter<LF, TA> {
+    pub fn new(mut log_file: SDSSFileIO<LF>, last_txn_id: u64, new: bool) -> SDSSResult<Self> {
+        let log_size = log_file.file_length()?;
+        log_file.seek_from_start(log_size)?; // avoid jumbling with headers
+        let mut slf = Self {
+            log_file,
+            id: last_txn_id,
+            _m: PhantomData,
+            closed: false,
+        };
+        if !new {
+            // IMPORTANT: don't forget this; otherwise the journal reader will report a false error!
+            slf.append_journal_reopen()?;
+        }
+        Ok(slf)
+    }
+    pub fn append_event(&mut self, event: TA::JournalEvent) -> SDSSResult<()> {
+        let encoded = TA::encode(event);
+        let md = JournalEntryMetadata::new(
+            self._incr_id() as u128,
+            EventSourceMarker::SERVER_STD,
+            CRC.checksum(&encoded),
+            encoded.len() as u64,
+        )
+        .encoded();
+        self.log_file.unfsynced_write(&md)?;
+        self.log_file.unfsynced_write(&encoded)?;
+        self.log_file.fsync_all()?;
+        Ok(())
+    }
+}
+
+impl<LF: RawFileIOInterface, TA> JournalWriter<LF, TA> {
+    pub fn appendrec_journal_reverse_entry(&mut self) -> SDSSResult<()> {
+        let mut threshold = Threshold::<RECOVERY_BLOCK_AUTO_THRESHOLD>::new();
+        let mut entry =
+            JournalEntryMetadata::new(0, EventSourceMarker::RECOVERY_REVERSE_LAST_JOURNAL, 0, 0);
+        while threshold.not_busted() {
+            entry.event_id = self._incr_id() as u128;
+            if self.log_file.fsynced_write(&entry.encoded()).is_ok() {
+                return Ok(());
+            }
+            threshold.bust_one();
+        }
+        Err(SDSSError::JournalWRecoveryStageOneFailCritical)
+    }
+    pub fn append_journal_reopen(&mut self) -> SDSSResult<()> {
+        let id = self._incr_id() as u128;
+        self.log_file.fsynced_write(
+            &JournalEntryMetadata::new(id, EventSourceMarker::DRIVER_REOPENED, 0, 0).encoded(),
+        )
+    }
+    pub fn append_journal_close_and_close(mut self) -> SDSSResult<()> {
+        self.closed = true;
+        let id = self._incr_id() as u128;
+        self.log_file.fsynced_write(
+            &JournalEntryMetadata::new(id, EventSourceMarker::DRIVER_CLOSED, 0, 0).encoded(),
+        )?;
+        Ok(())
+    }
+}
+
+impl<LF, TA> JournalWriter<LF, TA> {
+    fn _incr_id(&mut self) -> u64 {
+        let current = self.id;
+        self.id += 1;
+        current
+    }
+}
+
+impl<LF, TA> Drop for JournalWriter<LF, TA> {
+    fn drop(&mut self) {
+        assert!(self.closed, "log not closed");
+    }
+}
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index 2e9931a7..8c10fd13 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -27,9 +27,9 @@
 // raw
 mod header_impl;
 // impls
+mod journal;
 mod rw;
 mod start_stop;
-mod txn;
 // test
 #[cfg(test)]
 mod tests;
@@ -57,13 +57,14 @@ pub enum SDSSError {
     CorruptedFile(&'static str),
     StartupError(&'static str),
     CorruptedHeader,
-    TransactionLogEntryCorrupted,
-    TransactionLogCorrupted,
+    JournalLogEntryCorrupted,
+    JournalCorrupted,
     HeaderVersionMismatch,
     DriverVersionMismatch,
     ServerVersionMismatch,
     HeaderDataMismatch,
     TimeConflict,
+    JournalWRecoveryStageOneFailCritical,
 }
 
 impl SDSSError {
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 6d9022e8..f5b9dcf7 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -59,6 +59,7 @@ pub trait RawFileIOInterface: Sized {
     fn fseek_ahead(&mut self, by: u64) -> SDSSResult<()>;
     fn flen(&self) -> SDSSResult<u64>;
     fn flen_set(&mut self, to: u64) -> SDSSResult<()>;
+    fn fcursor(&mut self) -> SDSSResult<u64>;
 }
 
 impl RawFileIOInterface for File {
@@ -98,6 +99,9 @@ impl RawFileIOInterface for File {
         self.set_len(to)?;
         Ok(())
     }
+    fn fcursor(&mut self) -> SDSSResult<u64> {
+        self.stream_position().map_err(From::from)
+    }
 }
 
 #[derive(Debug)]
@@ -182,4 +186,7 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
     pub fn trim_file_to(&mut self, to: u64) -> SDSSResult<()> {
         self.f.flen_set(to)
     }
+    pub fn retrieve_cursor(&mut self) -> SDSSResult<u64> {
+        self.f.fcursor()
+    }
 }
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index e19c82a4..c8c3bd60 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -1,5 +1,5 @@
 /*
- * Created on Thu Jul 23 2023
+ * Created on Sat Jul 29 2023
  *
  * This file is a part of Skytable
  * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@@ -155,6 +155,9 @@ impl RawFileIOInterface for VirtualFileInterface {
             Ok(())
         })
     }
+    fn fcursor(&mut self) -> SDSSResult<u64> {
+        vfs(&self.0, |f| Ok(f.pos))
+    }
 }
 
 type VirtualFS = VirtualFileInterface;
@@ -215,7 +218,7 @@ mod tx {
     use {
         crate::{
             engine::storage::v1::{
-                txn::{self, TransactionLogAdapter, TransactionLogWriter},
+                journal::{self, JournalAdapter, JournalWriter},
                 SDSSError, SDSSResult,
             },
             util,
@@ -239,7 +242,7 @@ mod tx {
         }
         fn txn_reset(
             &self,
-            txn_writer: &mut TransactionLogWriter<FileInterface, DatabaseTxnAdapter>,
+            txn_writer: &mut JournalWriter<FileInterface, DatabaseTxnAdapter>,
         ) -> SDSSResult<()> {
             self.reset();
             txn_writer.append_event(TxEvent::Reset)
@@ -251,7 +254,7 @@ mod tx {
             &self,
             pos: usize,
             val: u8,
-            txn_writer: &mut TransactionLogWriter<FileInterface, DatabaseTxnAdapter>,
+            txn_writer: &mut JournalWriter<FileInterface, DatabaseTxnAdapter>,
         ) -> SDSSResult<()> {
             self.set(pos, val);
             txn_writer.append_event(TxEvent::Set(pos, val))
@@ -263,12 +266,12 @@ mod tx {
     }
     #[derive(Debug)]
     pub struct DatabaseTxnAdapter;
-    impl TransactionLogAdapter for DatabaseTxnAdapter {
-        type TransactionEvent = TxEvent;
-
+    impl JournalAdapter for DatabaseTxnAdapter {
+        const RECOVERY_PLUGIN: bool = false;
+        type JournalEvent = TxEvent;
         type GlobalState = Database;
 
-        fn encode(event: Self::TransactionEvent) -> Box<[u8]> {
+        fn encode(event: Self::JournalEvent) -> Box<[u8]> {
             /*
                 [1B: opcode][8B:Index][1B: New value]
             */
@@ -301,7 +304,7 @@ mod tx {
             match opcode {
                 0 if index == 0 && new_value == 0 => gs.reset(),
                 1 if index < 10 && index < isize::MAX as u64 => gs.set(index as usize, new_value),
-                _ => return Err(SDSSError::TransactionLogEntryCorrupted),
+                _ => return Err(SDSSError::JournalLogEntryCorrupted),
             }
             Ok(())
         }
@@ -310,8 +313,8 @@ mod tx {
     fn open_log(
         log_name: &str,
         db: &Database,
-    ) -> SDSSResult<TransactionLogWriter<FileInterface, DatabaseTxnAdapter>> {
-        txn::open_log::<DatabaseTxnAdapter, FileInterface>(
+    ) -> SDSSResult<JournalWriter<FileInterface, DatabaseTxnAdapter>> {
+        journal::open_journal::<DatabaseTxnAdapter, FileInterface>(
             log_name,
             FileSpecifier::TestTransactionLog,
             FileSpecifierVersion::__new(0),
@@ -330,7 +333,7 @@ mod tx {
             let mut log = open_log("testtxn.log", &db1)?;
             db1.txn_set(0, 20, &mut log)?;
             db1.txn_set(9, 21, &mut log)?;
-            log.close_log()
+            log.append_journal_close_and_close()
         };
         x().unwrap();
         // backup original data
@@ -339,7 +342,7 @@ mod tx {
         let empty_db2 = Database::new();
         open_log("testtxn.log", &empty_db2)
             .unwrap()
-            .close_log()
+            .append_journal_close_and_close()
             .unwrap();
         assert_eq!(original_data, empty_db2.copy_data());
         std::fs::remove_file("testtxn.log").unwrap();
@@ -353,7 +356,7 @@ mod tx {
             for i in 0..10 {
                 db1.txn_set(i, 1, &mut log)?;
             }
-            log.close_log()
+            log.append_journal_close_and_close()
         };
         x().unwrap();
         let bkp_db1 = db1.copy_data();
@@ -367,7 +370,7 @@ mod tx {
                 let current_val = db2.data.borrow()[i];
                 db2.txn_set(i, current_val + i as u8, &mut log)?;
             }
-            log.close_log()
+            log.append_journal_close_and_close()
         };
         x().unwrap();
         let bkp_db2 = db2.copy_data();
@@ -375,7 +378,7 @@ mod tx {
         // third boot
         let db3 = Database::new();
         let log = open_log("duatxn.db-tlog", &db3).unwrap();
-        log.close_log().unwrap();
+        log.append_journal_close_and_close().unwrap();
         assert_eq!(bkp_db2, db3.copy_data());
         assert_eq!(
             db3.copy_data(),
diff --git a/server/src/engine/storage/v1/txn.rs b/server/src/engine/storage/v1/txn.rs
deleted file mode 100644
index 404c0cb3..00000000
--- a/server/src/engine/storage/v1/txn.rs
+++ /dev/null
@@ -1,347 +0,0 @@
-/*
- * Created on Thu Jul 23 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
-  +----------------+------------------------------+------------------+------------------+--------------------+
-  | EVENT ID (16B) | EVENT SOURCE + METADATA (8B) | EVENT CRC32 (4B) | PAYLOAD LEN (8B) | EVENT PAYLOAD (?B) |
-  +----------------+------------------------------+------------------+------------------+--------------------+
-  Event ID:
-  - The atomically incrementing event ID (for future scale we have 16B; it's like the ZFS situation haha)
-  - Event source (1B) + 7B padding (for future metadata)
-  - Event CRC32
-  - Payload len: the size of the pyload
-  - Payload: the payload
-*/
-
-use {
-    super::{
-        header_impl::{FileSpecifierVersion, HostRunMode, SDSSHeaderRaw},
-        rw::{FileOpen, RawFileIOInterface, SDSSFileIO},
-        SDSSError, SDSSResult,
-    },
-    crate::{
-        engine::storage::v1::header_impl::{FileScope, FileSpecifier},
-        util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy},
-    },
-    std::marker::PhantomData,
-};
-
-const CRC: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
-
-pub fn open_log<
-    TA: TransactionLogAdapter + core::fmt::Debug,
-    LF: RawFileIOInterface + core::fmt::Debug,
->(
-    log_file_name: &str,
-    log_kind: FileSpecifier,
-    log_kind_version: FileSpecifierVersion,
-    host_setting_version: u32,
-    host_run_mode: HostRunMode,
-    host_startup_counter: u64,
-    gs: &TA::GlobalState,
-) -> SDSSResult<TransactionLogWriter<LF, TA>> {
-    let f = SDSSFileIO::<LF>::open_or_create_perm_rw(
-        log_file_name,
-        FileScope::TransactionLog,
-        log_kind,
-        log_kind_version,
-        host_setting_version,
-        host_run_mode,
-        host_startup_counter,
-    )?;
-    let file = match f {
-        FileOpen::Created(f) => return TransactionLogWriter::new(f, 0),
-        FileOpen::Existing(file, _) => file,
-    };
-    let (file, last_txn) = TransactionLogReader::<TA, LF>::scroll(file, gs)?;
-    TransactionLogWriter::new(file, last_txn)
-}
-
-/// The transaction adapter
-pub trait TransactionLogAdapter {
-    /// The transaction event
-    type TransactionEvent;
-    /// The global state, which we want to modify on decoding the event
-    type GlobalState;
-    /// Encode a transaction event into a blob
-    fn encode(event: Self::TransactionEvent) -> Box<[u8]>;
-    /// Decode a transaction event and apply it to the global state
-    fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> SDSSResult<()>;
-}
-
-#[derive(Debug)]
-pub struct TxnLogEntryMetadata {
-    event_id: u128,
-    event_source_md: u64,
-    event_crc: u32,
-    event_payload_len: u64,
-}
-
-impl TxnLogEntryMetadata {
-    const SIZE: usize = sizeof!(u128) + sizeof!(u64) + sizeof!(u32) + sizeof!(u64);
-    const P0: usize = 0;
-    const P1: usize = sizeof!(u128);
-    const P2: usize = Self::P1 + sizeof!(u64);
-    const P3: usize = Self::P2 + sizeof!(u32);
-    pub const fn new(
-        event_id: u128,
-        event_source_md: u64,
-        event_crc: u32,
-        event_payload_len: u64,
-    ) -> Self {
-        Self {
-            event_id,
-            event_source_md,
-            event_crc,
-            event_payload_len,
-        }
-    }
-    /// Encodes the log entry metadata
-    pub const fn encoded(&self) -> [u8; TxnLogEntryMetadata::SIZE] {
-        let mut encoded = [0u8; TxnLogEntryMetadata::SIZE];
-        encoded = copy_a_into_b(self.event_id.to_le_bytes(), encoded, Self::P0);
-        encoded = copy_a_into_b(self.event_source_md.to_le_bytes(), encoded, Self::P1);
-        encoded = copy_a_into_b(self.event_crc.to_le_bytes(), encoded, Self::P2);
-        encoded = copy_a_into_b(self.event_payload_len.to_le_bytes(), encoded, Self::P3);
-        encoded
-    }
-    /// Decodes the log entry metadata (essentially a simply type transmutation)
-    pub fn decode(data: [u8; TxnLogEntryMetadata::SIZE]) -> Self {
-        Self::new(
-            u128::from_le_bytes(memcpy(&data[..Self::P1])),
-            u64::from_le_bytes(memcpy(&data[Self::P1..Self::P2])),
-            u32::from_le_bytes(memcpy(&data[Self::P2..Self::P3])),
-            u64::from_le_bytes(memcpy(&data[Self::P3..])),
-        )
-    }
-}
-
-/*
-    Event source:
-    * * * * _ * * * *
-
-    b1 (s+d): event source (unset -> driver, set -> server)
-    b* -> unused. MUST be unset
-    b8 (d):
-        - unset: closed log
-*/
-pub enum EventSourceMarker {
-    ServerStandard,
-    DriverClosed,
-}
-
-impl EventSourceMarker {
-    const SERVER_STD: u64 = 1 << 63;
-    const DRIVER_CLOSED: u64 = 0;
-}
-
-impl TxnLogEntryMetadata {
-    pub const fn is_server_event(&self) -> bool {
-        self.event_source_md == EventSourceMarker::SERVER_STD
-    }
-    pub const fn is_driver_event(&self) -> bool {
-        self.event_source_md <= 1
-    }
-    pub const fn event_source_marker(&self) -> Option<EventSourceMarker> {
-        Some(match self.event_source_md {
-            EventSourceMarker::DRIVER_CLOSED => EventSourceMarker::DriverClosed,
-            EventSourceMarker::SERVER_STD => EventSourceMarker::ServerStandard,
-            _ => return None,
-        })
-    }
-}
-
-#[derive(Debug)]
-pub struct TransactionLogReader<TA, LF> {
-    log_file: SDSSFileIO<LF>,
-    log_size: u64,
-    evid: u64,
-    closed: bool,
-    remaining_bytes: u64,
-    _m: PhantomData<TA>,
-}
-
-impl<TA: TransactionLogAdapter, LF: RawFileIOInterface> TransactionLogReader<TA, LF> {
-    pub fn new(log_file: SDSSFileIO<LF>) -> SDSSResult<Self> {
-        let log_size = log_file.file_length()? - SDSSHeaderRaw::header_size() as u64;
-        Ok(Self {
-            log_file,
-            log_size,
-            evid: 0,
-            closed: false,
-            remaining_bytes: log_size,
-            _m: PhantomData,
-        })
-    }
-    /// Read the next event and apply it to the global state
-    pub fn rapply_next_event(&mut self, gs: &TA::GlobalState) -> SDSSResult<()> {
-        // read metadata
-        let mut raw_txn_log_row_md = [0u8; TxnLogEntryMetadata::SIZE];
-        self.log_file.read_to_buffer(&mut raw_txn_log_row_md)?;
-        self._record_bytes_read(TxnLogEntryMetadata::SIZE);
-        let event_metadata = TxnLogEntryMetadata::decode(raw_txn_log_row_md);
-        /*
-            verify metadata and read bytes into buffer, verify sum
-        */
-        // verify md
-        let event_src_marker = event_metadata.event_source_marker();
-        let okay = (self.evid == (event_metadata.event_id as _))
-            & event_src_marker.is_some()
-            & (event_metadata.event_payload_len < (isize::MAX as u64))
-            & self.has_remaining_bytes(event_metadata.event_payload_len);
-        if compiler::unlikely(!okay) {
-            return Err(SDSSError::TransactionLogEntryCorrupted);
-        }
-        let event_is_zero =
-            (event_metadata.event_crc == 0) & (event_metadata.event_payload_len == 0);
-        let event_src_marker = event_src_marker.unwrap();
-        match event_src_marker {
-            EventSourceMarker::ServerStandard => {}
-            EventSourceMarker::DriverClosed if event_is_zero => {
-                // expect last entry
-                if self.end_of_file() {
-                    // remove the delete entry
-                    self.log_file.seek_from_start(
-                        self.log_size - TxnLogEntryMetadata::SIZE as u64
-                            + SDSSHeaderRaw::header_size() as u64,
-                    )?;
-                    self.log_file.trim_file_to(
-                        self.log_size - TxnLogEntryMetadata::SIZE as u64
-                            + SDSSHeaderRaw::header_size() as u64,
-                    )?;
-                    self.log_file.fsync_all()?;
-                    self.closed = true;
-                    // good
-                    return Ok(());
-                } else {
-                    return Err(SDSSError::TransactionLogCorrupted);
-                }
-            }
-            EventSourceMarker::DriverClosed => {
-                return Err(SDSSError::TransactionLogEntryCorrupted);
-            }
-        }
-        // read bytes
-        let mut payload_data_block = vec![0u8; event_metadata.event_payload_len as usize];
-        self.log_file.read_to_buffer(&mut payload_data_block)?;
-        self._record_bytes_read(event_metadata.event_payload_len as _);
-        // verify sum
-        let actual_sum = CRC.checksum(&payload_data_block);
-        if compiler::likely(actual_sum == event_metadata.event_crc) {
-            self._incr_evid();
-            // great, the sums match
-            TA::decode_and_update_state(&payload_data_block, gs)?;
-            Ok(())
-        } else {
-            Err(SDSSError::TransactionLogEntryCorrupted)
-        }
-    }
-    /// Read and apply all events in the given log file to the global state, returning the (open file, last event ID)
-    pub fn scroll(file: SDSSFileIO<LF>, gs: &TA::GlobalState) -> SDSSResult<(SDSSFileIO<LF>, u64)> {
-        let mut slf = Self::new(file)?;
-        while !slf.end_of_file() {
-            slf.rapply_next_event(gs)?;
-        }
-        if slf.closed {
-            Ok((slf.log_file, slf.evid))
-        } else {
-            Err(SDSSError::TransactionLogCorrupted)
-        }
-    }
-}
-
-impl<TA, LF> TransactionLogReader<TA, LF> {
-    fn _incr_evid(&mut self) {
-        self.evid += 1;
-    }
-    fn _record_bytes_read(&mut self, cnt: usize) {
-        self.remaining_bytes -= cnt as u64;
-    }
-    fn has_remaining_bytes(&self, size: u64) -> bool {
-        self.remaining_bytes >= size
-    }
-    fn end_of_file(&self) -> bool {
-        self.remaining_bytes == 0
-    }
-}
-
-pub struct TransactionLogWriter<LF, TA> {
-    /// the txn log file
-    log_file: SDSSFileIO<LF>,
-    /// the id of the **next** transaction
-    id: u64,
-    _m: PhantomData<TA>,
-    closed: bool,
-}
-
-impl<LF: RawFileIOInterface, TA: TransactionLogAdapter> TransactionLogWriter<LF, TA> {
-    pub fn new(mut log_file: SDSSFileIO<LF>, last_txn_id: u64) -> SDSSResult<Self> {
-        let log_size = log_file.file_length()?;
-        log_file.seek_from_start(log_size)?;
-        Ok(Self {
-            log_file,
-            id: last_txn_id,
-            _m: PhantomData,
-            closed: false,
-        })
-    }
-    pub fn append_event(&mut self, event: TA::TransactionEvent) -> SDSSResult<()> {
-        let encoded = TA::encode(event);
-        let md = TxnLogEntryMetadata::new(
-            self._incr_id() as u128,
-            EventSourceMarker::SERVER_STD,
-            CRC.checksum(&encoded),
-            encoded.len() as u64,
-        )
-        .encoded();
-        self.log_file.unfsynced_write(&md)?;
-        self.log_file.unfsynced_write(&encoded)?;
-        self.log_file.fsync_all()?;
-        Ok(())
-    }
-    pub fn close_log(mut self) -> SDSSResult<()> {
-        self.closed = true;
-        let id = self._incr_id() as u128;
-        self.log_file.fsynced_write(
-            &TxnLogEntryMetadata::new(id, EventSourceMarker::DRIVER_CLOSED, 0, 0).encoded(),
-        )?;
-        Ok(())
-    }
-}
-
-impl<LF, TA> TransactionLogWriter<LF, TA> {
-    fn _incr_id(&mut self) -> u64 {
-        let current = self.id;
-        self.id += 1;
-        current
-    }
-}
-
-impl<LF, TA> Drop for TransactionLogWriter<LF, TA> {
-    fn drop(&mut self) {
-        assert!(self.closed, "log not closed");
-    }
-}
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index 8d971c4c..bd69fba5 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -392,3 +392,19 @@ pub const fn copy_a_into_b<const M: usize, const N: usize>(
     }
     to
 }
+
+pub struct Threshold<const N: usize> {
+    now: usize,
+}
+
+impl<const N: usize> Threshold<N> {
+    pub const fn new() -> Self {
+        Self { now: 0 }
+    }
+    pub fn bust_one(&mut self) {
+        self.now += 1;
+    }
+    pub fn not_busted(&self) -> bool {
+        self.now < N
+    }
+}

From 3d2e4f20148fcfdc4b9ee50efb87f4a14ebcdd86 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 4 Aug 2023 15:15:25 +0000
Subject: [PATCH 211/310] Simplify metadict semantics

---
 server/src/engine/core/model/alt.rs           |  4 +-
 server/src/engine/core/space.rs               | 14 +++----
 .../src/engine/data/{md_dict.rs => dict.rs}   | 42 ++++++++++++-------
 server/src/engine/data/mod.rs                 |  4 +-
 server/src/engine/data/tests/md_dict_tests.rs | 32 +++++++-------
 server/src/engine/ql/ddl/syn.rs               | 10 ++---
 server/src/engine/ql/tests.rs                 | 14 +++----
 7 files changed, 65 insertions(+), 55 deletions(-)
 rename server/src/engine/data/{md_dict.rs => dict.rs} (83%)

diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index 82163a44..0136cdb6 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -79,9 +79,9 @@ fn no_field(mr: &IWModel, new: &str) -> bool {
     !mr.fields().st_contains(new)
 }
 
-fn check_nullable(props: &mut HashMap<Box<str>, Option<DictEntryGeneric>>) -> DatabaseResult<bool> {
+fn check_nullable(props: &mut HashMap<Box<str>, DictEntryGeneric>) -> DatabaseResult<bool> {
     match props.remove("nullable") {
-        Some(Some(DictEntryGeneric::Lit(b))) if b.kind() == TagClass::Bool => Ok(b.bool()),
+        Some(DictEntryGeneric::Lit(b)) if b.kind() == TagClass::Bool => Ok(b.bool()),
         Some(_) => Err(DatabaseError::DdlModelAlterBadProperty),
         None => Ok(false),
     }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 8d9e0e0e..4750c4aa 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -27,7 +27,7 @@
 use {
     crate::engine::{
         core::{model::ModelData, RWLIdx},
-        data::{md_dict, DictEntryGeneric, MetaDict},
+        data::{dict, DictEntryGeneric, MetaDict},
         error::{DatabaseError, DatabaseResult},
         idx::{IndexST, STIndex},
         ql::ddl::{alt::AlterSpace, crt::CreateSpace, drop::DropSpace},
@@ -123,8 +123,8 @@ impl Space {
         let space_name = space_name.to_string().into_boxed_str();
         // check env
         let env = match props.remove(SpaceMeta::KEY_ENV) {
-            Some(Some(DictEntryGeneric::Map(m))) if props.is_empty() => m,
-            Some(None) | None if props.is_empty() => IndexST::default(),
+            Some(DictEntryGeneric::Map(m)) if props.is_empty() => m,
+            Some(DictEntryGeneric::Null) | None if props.is_empty() => IndexST::default(),
             _ => {
                 return Err(DatabaseError::DdlSpaceBadProperty);
             }
@@ -135,7 +135,7 @@ impl Space {
                 IndexST::default(),
                 SpaceMeta::with_env(
                     // FIXME(@ohsayan): see this is bad. attempt to do it at AST build time
-                    md_dict::rflatten_metadata(env),
+                    dict::rflatten_metadata(env),
                 ),
             ),
         })
@@ -164,12 +164,12 @@ impl Space {
         gns.with_space(&space_name, |space| {
             let mut space_env = space.meta.env.write();
             match updated_props.remove(SpaceMeta::KEY_ENV) {
-                Some(Some(DictEntryGeneric::Map(env))) if updated_props.is_empty() => {
-                    if !md_dict::rmerge_metadata(&mut space_env, env) {
+                Some(DictEntryGeneric::Map(env)) if updated_props.is_empty() => {
+                    if !dict::rmerge_metadata(&mut space_env, env) {
                         return Err(DatabaseError::DdlSpaceBadProperty);
                     }
                 }
-                Some(None) if updated_props.is_empty() => space_env.clear(),
+                Some(DictEntryGeneric::Null) if updated_props.is_empty() => space_env.clear(),
                 None => {}
                 _ => return Err(DatabaseError::DdlSpaceBadProperty),
             }
diff --git a/server/src/engine/data/md_dict.rs b/server/src/engine/data/dict.rs
similarity index 83%
rename from server/src/engine/data/md_dict.rs
rename to server/src/engine/data/dict.rs
index bb06af98..148af9a6 100644
--- a/server/src/engine/data/md_dict.rs
+++ b/server/src/engine/data/dict.rs
@@ -37,18 +37,28 @@ use {
 
 /*
     dict kinds: one is from a generic parse while the other one is the stored metadata
+    - MetaDict -> only non-null values
+    - DictGeneric -> null allowed
 */
 
 /// A metadata dictionary
 pub type MetaDict = HashMap<Box<str>, MetaDictEntry>;
 /// A generic dictionary built from scratch from syntactical elements
-pub type DictGeneric = HashMap<Box<str>, Option<DictEntryGeneric>>;
+pub type DictGeneric = HashMap<Box<str>, DictEntryGeneric>;
 
 #[derive(Debug, PartialEq)]
 /// A generic dict entry: either a literal or a recursive dictionary
 pub enum DictEntryGeneric {
+    /// A literal
     Lit(Datacell),
+    /// A map
     Map(DictGeneric),
+    /// A null
+    ///
+    /// This distinction is important because we currently only allow [`Datacell`] to store information about the intialization state (as it should be)
+    /// and it can't distinct between "null-but-concrete-types" (that's the task of our engine: to enforce the schema but it's not DC's task). Hence this
+    /// specific flag tells us that this is null (and neither a lit nor a map)
+    Null,
 }
 
 #[derive(Debug, PartialEq)]
@@ -80,17 +90,16 @@ pub fn rflatten_metadata(new: DictGeneric) -> MetaDict {
 
 fn _rflatten_metadata(new: DictGeneric, empty: &mut MetaDict) {
     for (key, val) in new {
-        if let Some(v) = val {
-            match v {
-                DictEntryGeneric::Lit(l) => {
-                    empty.insert(key, MetaDictEntry::Data(l));
-                }
-                DictEntryGeneric::Map(m) => {
-                    let mut rnew = MetaDict::new();
-                    _rflatten_metadata(m, &mut rnew);
-                    empty.insert(key, MetaDictEntry::Map(rnew));
-                }
+        match val {
+            DictEntryGeneric::Lit(l) => {
+                empty.insert(key, MetaDictEntry::Data(l));
+            }
+            DictEntryGeneric::Map(m) => {
+                let mut rnew = MetaDict::new();
+                _rflatten_metadata(m, &mut rnew);
+                empty.insert(key, MetaDictEntry::Map(rnew));
             }
+            DictEntryGeneric::Null => {}
         }
     }
 }
@@ -144,9 +153,9 @@ fn rmerge_metadata_prepare_patch(
     let mut okay = true;
     while new.len() != 0 && okay {
         let (key, new_entry) = new.next().unwrap();
-        match (current.get(&key), new_entry) {
+        match (current.get(&key), &new_entry) {
             // non-null -> non-null: merge flatten update
-            (Some(this_current), Some(new_entry)) => {
+            (Some(this_current), DictEntryGeneric::Lit(_) | DictEntryGeneric::Map(_)) => {
                 okay &= {
                     match (this_current, new_entry) {
                         (MetaDictEntry::Data(this_data), DictEntryGeneric::Lit(new_data))
@@ -177,7 +186,7 @@ fn rmerge_metadata_prepare_patch(
                 };
             }
             // null -> non-null: flatten insert
-            (None, Some(new_entry)) => match new_entry {
+            (None, DictEntryGeneric::Lit(_) | DictEntryGeneric::Map(_)) => match new_entry {
                 DictEntryGeneric::Lit(d) => {
                     let _ = patch.0.insert(key, Some(MetaDictPatchEntry::Data(d)));
                 }
@@ -188,12 +197,13 @@ fn rmerge_metadata_prepare_patch(
                         .0
                         .insert(key, Some(MetaDictPatchEntry::Map(this_patch)));
                 }
+                _ => unreachable!(),
             },
             // non-null -> null: remove
-            (Some(_), None) => {
+            (Some(_), DictEntryGeneric::Null) => {
                 patch.0.insert(key, None);
             }
-            (None, None) => {
+            (None, DictEntryGeneric::Null) => {
                 // ignore
             }
         }
diff --git a/server/src/engine/data/mod.rs b/server/src/engine/data/mod.rs
index 499f6092..082c08b2 100644
--- a/server/src/engine/data/mod.rs
+++ b/server/src/engine/data/mod.rs
@@ -27,11 +27,11 @@
 #[macro_use]
 mod macros;
 pub mod cell;
+pub mod dict;
 pub mod lit;
-pub mod md_dict;
 pub mod spec;
 pub mod tag;
 #[cfg(test)]
 mod tests;
 
-pub use md_dict::{DictEntryGeneric, DictGeneric, MetaDict};
+pub use dict::{DictEntryGeneric, DictGeneric, MetaDict};
diff --git a/server/src/engine/data/tests/md_dict_tests.rs b/server/src/engine/data/tests/md_dict_tests.rs
index 467cc862..1110b8d2 100644
--- a/server/src/engine/data/tests/md_dict_tests.rs
+++ b/server/src/engine/data/tests/md_dict_tests.rs
@@ -26,19 +26,19 @@
 
 use crate::engine::data::{
     cell::Datacell,
-    md_dict::{self, DictEntryGeneric, DictGeneric, MetaDict, MetaDictEntry},
+    dict::{self, DictEntryGeneric, DictGeneric, MetaDict, MetaDictEntry},
 };
 
 #[test]
 fn t_simple_flatten() {
     let generic_dict: DictGeneric = into_dict! {
-        "a_valid_key" => Some(DictEntryGeneric::Lit(100u64.into())),
-        "a_null_key" => None,
+        "a_valid_key" => DictEntryGeneric::Lit(100u64.into()),
+        "a_null_key" => DictEntryGeneric::Null,
     };
     let expected: MetaDict = into_dict!(
         "a_valid_key" => Datacell::new_uint(100)
     );
-    let ret = md_dict::rflatten_metadata(generic_dict);
+    let ret = dict::rflatten_metadata(generic_dict);
     assert_eq!(ret, expected);
 }
 
@@ -50,15 +50,15 @@ fn t_simple_patch() {
         "z" => Datacell::new_sint(-100),
     };
     let new: DictGeneric = into_dict! {
-        "a" => Some(Datacell::new_uint(1).into()),
-        "b" => Some(Datacell::new_uint(2).into()),
-        "z" => None,
+        "a" => Datacell::new_uint(1),
+        "b" => Datacell::new_uint(2),
+        "z" => DictEntryGeneric::Null,
     };
     let expected: MetaDict = into_dict! {
         "a" => Datacell::new_uint(1),
         "b" => Datacell::new_uint(2),
     };
-    assert!(md_dict::rmerge_metadata(&mut current, new));
+    assert!(dict::rmerge_metadata(&mut current, new));
     assert_eq!(current, expected);
 }
 
@@ -71,11 +71,11 @@ fn t_bad_patch() {
     };
     let backup = current.clone();
     let new: DictGeneric = into_dict! {
-        "a" => Some(Datacell::new_uint(1).into()),
-        "b" => Some(Datacell::new_uint(2).into()),
-        "z" => Some(Datacell::new_str("omg".into()).into()),
+        "a" => Datacell::new_uint(1),
+        "b" => Datacell::new_uint(2),
+        "z" => Datacell::new_str("omg".into()),
     };
-    assert!(!md_dict::rmerge_metadata(&mut current, new));
+    assert!(!dict::rmerge_metadata(&mut current, new));
     assert_eq!(current, backup);
 }
 
@@ -94,10 +94,10 @@ fn patch_null_out_dict() {
         "b" => Datacell::new_uint(3),
     };
     let new: DictGeneric = into_dict! {
-        "a" => Some(Datacell::new_uint(2).into()),
-        "b" => Some(Datacell::new_uint(3).into()),
-        "z" => None,
+        "a" => Datacell::new_uint(2),
+        "b" => Datacell::new_uint(3),
+        "z" => DictEntryGeneric::Null,
     };
-    assert!(md_dict::rmerge_metadata(&mut current, new));
+    assert!(dict::rmerge_metadata(&mut current, new));
     assert_eq!(current, expected);
 }
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 6c019be9..bdc5edce 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -46,7 +46,7 @@
 
 use crate::{
     engine::{
-        data::DictGeneric,
+        data::dict::{DictEntryGeneric, DictGeneric},
         error::{LangError, LangResult},
         ql::{
             ast::{QueryData, State},
@@ -151,10 +151,10 @@ where
             }
             (tok, DictFoldState::LIT_OR_OB) if state.can_read_lit_from(tok) => {
                 // found lit
-                let v = Some(unsafe {
+                let v = unsafe {
                     // UNSAFE(@ohsayan): verified at guard
                     state.read_lit_unchecked_from(tok).into()
-                });
+                };
                 state.poison_if_not(
                     dict.insert(
                         unsafe {
@@ -176,7 +176,7 @@ where
                             // UNSAFE(@ohsayan): we only switch to this when we've already read in a key
                             key.take().as_str().into()
                         },
-                        None,
+                        DictEntryGeneric::Null,
                     )
                     .is_none(),
                 );
@@ -193,7 +193,7 @@ where
                             // UNSAFE(@ohsayan): correct again because whenever we hit an expression position, we've already read in a key (ident)
                             key.take().as_str().into()
                         },
-                        Some(ndict.into()),
+                        DictEntryGeneric::Map(ndict),
                     )
                     .is_none(),
                 );
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 96310c80..28051312 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -76,24 +76,24 @@ fn nullable_datatype(v: impl NullableData<Datacell>) -> Datacell {
 }
 
 pub trait NullableDictEntry {
-    fn data(self) -> Option<crate::engine::data::DictEntryGeneric>;
+    fn data(self) -> crate::engine::data::DictEntryGeneric;
 }
 
 impl NullableDictEntry for Null {
-    fn data(self) -> Option<crate::engine::data::DictEntryGeneric> {
-        None
+    fn data(self) -> crate::engine::data::DictEntryGeneric {
+        crate::engine::data::DictEntryGeneric::Null
     }
 }
 
 impl<'a> NullableDictEntry for crate::engine::data::lit::Lit<'a> {
-    fn data(self) -> Option<crate::engine::data::DictEntryGeneric> {
-        Some(crate::engine::data::DictEntryGeneric::from(self.as_ir()))
+    fn data(self) -> crate::engine::data::DictEntryGeneric {
+        crate::engine::data::DictEntryGeneric::from(self.as_ir())
     }
 }
 
 impl NullableDictEntry for crate::engine::data::DictGeneric {
-    fn data(self) -> Option<crate::engine::data::DictEntryGeneric> {
-        Some(crate::engine::data::DictEntryGeneric::Map(self))
+    fn data(self) -> crate::engine::data::DictEntryGeneric {
+        crate::engine::data::DictEntryGeneric::Map(self)
     }
 }
 

From 7346035a27df0a8377e1d9b0192c04106f1614eb Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 9 Aug 2023 18:25:31 +0000
Subject: [PATCH 212/310] Impl basic hl inf for persist

---
 server/src/engine/data/cell.rs                |   2 +-
 .../src/engine/storage/v1/header_impl/dr.rs   |   8 +-
 .../src/engine/storage/v1/header_impl/gr.rs   |   6 +-
 .../src/engine/storage/v1/header_impl/sr.rs   |   2 +-
 server/src/engine/storage/v1/inf.rs           | 221 ++++++++++++++++++
 server/src/engine/storage/v1/mod.rs           |  40 +++-
 server/src/engine/storage/v1/rw.rs            |  43 +++-
 server/src/util/mod.rs                        |  35 +++
 8 files changed, 339 insertions(+), 18 deletions(-)
 create mode 100644 server/src/engine/storage/v1/inf.rs

diff --git a/server/src/engine/data/cell.rs b/server/src/engine/data/cell.rs
index fc9c8e59..1ce99a2d 100644
--- a/server/src/engine/data/cell.rs
+++ b/server/src/engine/data/cell.rs
@@ -276,7 +276,7 @@ impl<const N: usize> From<[Datacell; N]> for Datacell {
 }
 
 impl Datacell {
-    pub fn tag(&self) -> CUTag {
+    pub const fn tag(&self) -> CUTag {
         self.tag
     }
     pub fn kind(&self) -> TagClass {
diff --git a/server/src/engine/storage/v1/header_impl/dr.rs b/server/src/engine/storage/v1/header_impl/dr.rs
index f80eea2b..8ec3cff8 100644
--- a/server/src/engine/storage/v1/header_impl/dr.rs
+++ b/server/src/engine/storage/v1/header_impl/dr.rs
@@ -62,13 +62,13 @@ pub struct DRHostSignature {
 impl DRHostSignature {
     pub fn verify(&self, expected_file_specifier_version: FileSpecifierVersion) -> SDSSResult<()> {
         if self.server_version() != versions::v1::V1_SERVER_VERSION {
-            return Err(SDSSError::ServerVersionMismatch);
+            return Err(SDSSError::HeaderDecodeServerVersionMismatch);
         }
         if self.driver_version() != versions::v1::V1_DRIVER_VERSION {
-            return Err(SDSSError::DriverVersionMismatch);
+            return Err(SDSSError::HeaderDecodeDriverVersionMismatch);
         }
         if self.file_specifier_version() != expected_file_specifier_version {
-            return Err(SDSSError::HeaderDataMismatch);
+            return Err(SDSSError::HeaderDecodeDataMismatch);
         }
         Ok(())
     }
@@ -287,7 +287,7 @@ impl DRRuntimeSignature {
         let et = util::os::get_epoch_time();
         if self.epoch_time() > et || self.host_uptime() > et {
             // a file from the future?
-            return Err(SDSSError::TimeConflict);
+            return Err(SDSSError::HeaderTimeConflict);
         }
         Ok(())
     }
diff --git a/server/src/engine/storage/v1/header_impl/gr.rs b/server/src/engine/storage/v1/header_impl/gr.rs
index 22653a07..e7a0a076 100644
--- a/server/src/engine/storage/v1/header_impl/gr.rs
+++ b/server/src/engine/storage/v1/header_impl/gr.rs
@@ -66,10 +66,10 @@ impl GRMetadataRecord {
         expected_file_specifier_version: FileSpecifierVersion,
     ) -> SDSSResult<()> {
         if self.server_version() != versions::v1::V1_SERVER_VERSION {
-            return Err(SDSSError::ServerVersionMismatch);
+            return Err(SDSSError::HeaderDecodeServerVersionMismatch);
         }
         if self.driver_version() != versions::v1::V1_DRIVER_VERSION {
-            return Err(SDSSError::DriverVersionMismatch);
+            return Err(SDSSError::HeaderDecodeDriverVersionMismatch);
         }
         let okay = self.file_scope() == expected_file_scope
             && self.file_spec() == expected_file_specifier
@@ -77,7 +77,7 @@ impl GRMetadataRecord {
         if okay {
             Ok(())
         } else {
-            Err(SDSSError::HeaderDataMismatch)
+            Err(SDSSError::HeaderDecodeDataMismatch)
         }
     }
 }
diff --git a/server/src/engine/storage/v1/header_impl/sr.rs b/server/src/engine/storage/v1/header_impl/sr.rs
index 3dcdffb2..40b591f1 100644
--- a/server/src/engine/storage/v1/header_impl/sr.rs
+++ b/server/src/engine/storage/v1/header_impl/sr.rs
@@ -44,7 +44,7 @@ impl StaticRecord {
         if self.sr().header_version() == versions::v1::V1_HEADER_VERSION {
             Ok(())
         } else {
-            return Err(SDSSError::HeaderVersionMismatch);
+            return Err(SDSSError::HeaderDecodeHeaderVersionMismatch);
         }
     }
 }
diff --git a/server/src/engine/storage/v1/inf.rs b/server/src/engine/storage/v1/inf.rs
new file mode 100644
index 00000000..3c4d29f3
--- /dev/null
+++ b/server/src/engine/storage/v1/inf.rs
@@ -0,0 +1,221 @@
+/*
+ * Created on Fri Aug 04 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+//! High level interfaces
+
+use {
+    crate::{
+        engine::{
+            data::{
+                dict::DictEntryGeneric,
+                tag::{DataTag, TagClass},
+            },
+            idx::{AsKey, AsValue},
+            storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult},
+        },
+        util::EndianQW,
+    },
+    std::collections::HashMap,
+};
+
+type VecU8 = Vec<u8>;
+
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, sky_macros::EnumMethods)]
+#[repr(u8)]
+/// Disambiguation for data
+pub enum PersistDictEntryDscr {
+    Null = 0,
+    Bool = 1,
+    UnsignedInt = 2,
+    SignedInt = 3,
+    Float = 4,
+    Bin = 5,
+    Str = 6,
+    List = 7,
+    Dict = 8,
+}
+
+impl PersistDictEntryDscr {
+    /// translates the tag class definition into the dscr definition
+    pub const fn translate_from_class(class: TagClass) -> Self {
+        unsafe { core::mem::transmute(class.d() + 1) }
+    }
+    pub fn new_from_dict_gen_entry(e: &DictEntryGeneric) -> Self {
+        match e {
+            DictEntryGeneric::Null => Self::Null,
+            DictEntryGeneric::Map(_) => Self::Dict,
+            DictEntryGeneric::Lit(dc) => Self::translate_from_class(dc.tag().tag_class()),
+        }
+    }
+    /// The data in question is null (well, can we call that data afterall?)
+    pub const fn is_null(&self) -> bool {
+        self.value_u8() == Self::Null.value_u8()
+    }
+    /// The data in question is a scalar
+    pub const fn is_scalar(&self) -> bool {
+        self.value_u8() <= Self::Float.value_u8()
+    }
+    /// The data is composite
+    pub const fn is_composite(&self) -> bool {
+        self.value_u8() > Self::Float.value_u8()
+    }
+    /// Recursive data
+    pub const fn is_recursive(&self) -> bool {
+        self.value_u8() >= Self::List.value_u8()
+    }
+}
+
+/*
+    spec
+*/
+
+/// metadata spec for a persist dict
+pub trait PersistDictEntryMetadata {
+    /// Verify the state of scanner to ensure that it complies with the metadata
+    fn verify_with_src(&self, scanner: &BufferedScanner) -> bool;
+}
+
+/// spec for a persist dict
+pub trait PersistDict {
+    /// type of key
+    type Key: AsKey;
+    /// type of value
+    type Value: AsValue;
+    /// metadata type
+    type Metadata: PersistDictEntryMetadata;
+    /// enc coupled (packed enc)
+    const ENC_COUPLED: bool;
+    /// during dec, ignore failure of the metadata parse (IMP: NOT the metadata src verification but the
+    /// validity of the metadata itself) because it is handled later
+    const DEC_ENTRYMD_INFALLIBLE: bool;
+    /// dec coupled (packed dec)
+    const DEC_COUPLED: bool;
+    /// during dec, verify the md directly with the src instead of handing it over to the dec helpers
+    const DEC_VERIFY_MD_WITH_SRC_STANDALONE: bool;
+    // meta
+    /// pretest for pre-entry stage
+    fn metadec_pretest_routine(scanner: &BufferedScanner) -> bool;
+    /// pretest for entry stage
+    fn metadec_pretest_entry(scanner: &BufferedScanner) -> bool;
+    /// enc md for an entry
+    fn enc_entry_metadata(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);
+    /// dec the entry metadata
+    unsafe fn dec_entry_metadata(scanner: &mut BufferedScanner) -> Option<Self::Metadata>;
+    // entry (coupled)
+    /// enc a packed entry
+    fn enc_entry_coupled(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);
+    /// dec a packed entry
+    fn dec_entry_coupled(
+        scanner: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> Option<(Self::Key, Self::Value)>;
+    // entry (non-packed)
+    /// enc key for a normal entry
+    fn enc_key(buf: &mut VecU8, key: &Self::Key);
+    /// dec normal entry key
+    fn dec_key(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Key>;
+    /// enc val for a normal entry
+    fn enc_val(buf: &mut VecU8, val: &Self::Value);
+    /// dec normal entry val
+    fn dec_val(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Value>;
+}
+
+/*
+    blanket
+*/
+
+pub fn encode_dict<Pd: PersistDict>(dict: &HashMap<Pd::Key, Pd::Value>) -> Vec<u8> {
+    let mut buf = Vec::new();
+    buf.extend(dict.len().u64_bytes_le());
+    for (key, val) in dict {
+        Pd::enc_entry_metadata(&mut buf, key, val);
+        if Pd::ENC_COUPLED {
+            Pd::enc_entry_coupled(&mut buf, key, val);
+        } else {
+            Pd::enc_key(&mut buf, key);
+            Pd::enc_val(&mut buf, val);
+        }
+    }
+    buf
+}
+
+pub fn decode_dict<Pd: PersistDict>(
+    scanner: &mut BufferedScanner,
+) -> SDSSResult<HashMap<Pd::Key, Pd::Value>> {
+    if Pd::metadec_pretest_routine(scanner) & scanner.has_left(sizeof!(u64)) {
+        return Err(SDSSError::InternalDecodeStructureCorrupted);
+    }
+    let dict_len = unsafe {
+        // UNSAFE(@ohsayan): pretest
+        scanner.next_u64_le() as usize
+    };
+    let mut dict = HashMap::with_capacity(dict_len);
+    while Pd::metadec_pretest_entry(scanner) {
+        let md = unsafe {
+            // UNSAFE(@ohsayan): this is compeletely because of the entry pretest
+            match Pd::dec_entry_metadata(scanner) {
+                Some(dec) => dec,
+                None => {
+                    if Pd::DEC_ENTRYMD_INFALLIBLE {
+                        impossible!()
+                    } else {
+                        return Err(SDSSError::InternalDecodeStructureCorrupted);
+                    }
+                }
+            }
+        };
+        if Pd::DEC_VERIFY_MD_WITH_SRC_STANDALONE && !md.verify_with_src(scanner) {
+            return Err(SDSSError::InternalDecodeStructureCorrupted);
+        }
+        let k;
+        let v;
+        if Pd::DEC_COUPLED {
+            match Pd::dec_entry_coupled(scanner, md) {
+                Some((_k, _v)) => {
+                    k = _k;
+                    v = _v;
+                }
+                None => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
+            }
+        } else {
+            match (Pd::dec_key(scanner, &md), Pd::dec_val(scanner, &md)) {
+                (Some(_k), Some(_v)) => {
+                    k = _k;
+                    v = _v;
+                }
+                _ => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
+            }
+        }
+        if dict.insert(k, v).is_some() {
+            return Err(SDSSError::InternalDecodeStructureIllegalData);
+        }
+    }
+    if dict.len() == dict_len {
+        Ok(dict)
+    } else {
+        Err(SDSSError::InternalDecodeStructureIllegalData)
+    }
+}
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index 8c10fd13..d65e817f 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -29,6 +29,8 @@ mod header_impl;
 // impls
 mod journal;
 mod rw;
+// hl
+mod inf;
 mod start_stop;
 // test
 #[cfg(test)]
@@ -52,19 +54,43 @@ impl SDSSErrorContext for IoError {
 
 #[derive(Debug)]
 pub enum SDSSError {
+    // IO errors
+    /// An IO err
     IoError(IoError),
+    /// An IO err with extra ctx
     IoErrorExtra(IoError, &'static str),
+    /// A corrupted file
     CorruptedFile(&'static str),
+    // process errors
     StartupError(&'static str),
-    CorruptedHeader,
+    // header
+    /// The entire header is corrupted
+    HeaderDecodeCorruptedHeader,
+    /// The header versions don't match
+    HeaderDecodeHeaderVersionMismatch,
+    /// The driver versions don't match
+    HeaderDecodeDriverVersionMismatch,
+    /// The server versions don't match
+    HeaderDecodeServerVersionMismatch,
+    /// Expected header values were not matched with the current header
+    HeaderDecodeDataMismatch,
+    /// The time in the [header/dynrec/rtsig] is in the future
+    HeaderTimeConflict,
+    // journal
+    /// While attempting to handle a basic failure (such as adding a journal entry), the recovery engine ran into an exceptional
+    /// situation where it failed to make a necessary repair the log
+    JournalWRecoveryStageOneFailCritical,
+    /// An entry in the journal is corrupted
     JournalLogEntryCorrupted,
+    /// The structure of the journal is corrupted
     JournalCorrupted,
-    HeaderVersionMismatch,
-    DriverVersionMismatch,
-    ServerVersionMismatch,
-    HeaderDataMismatch,
-    TimeConflict,
-    JournalWRecoveryStageOneFailCritical,
+    // internal file structures
+    /// While attempting to decode a structure in an internal segment of a file, the storage engine ran into a possibly irrecoverable error
+    InternalDecodeStructureCorrupted,
+    /// the payload (non-static) part of a structure in an internal segment of a file is corrupted
+    InternalDecodeStructureCorruptedPayload,
+    /// the data for an internal structure was decoded but is logically invalid
+    InternalDecodeStructureIllegalData,
 }
 
 impl SDSSError {
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index f5b9dcf7..ca42b93b 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -35,6 +35,7 @@ use {
     std::{
         fs::File,
         io::{Read, Seek, SeekFrom, Write},
+        ptr,
     },
 };
 
@@ -142,8 +143,8 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
                 // this is an existing file. decoded the header
                 let mut header_raw = [0u8; SDSSHeaderRaw::header_size()];
                 f.fread_exact(&mut header_raw)?;
-                let header =
-                    SDSSHeaderRaw::decode_noverify(header_raw).ok_or(SDSSError::CorruptedHeader)?;
+                let header = SDSSHeaderRaw::decode_noverify(header_raw)
+                    .ok_or(SDSSError::HeaderDecodeCorruptedHeader)?;
                 // now validate the header
                 header.verify(file_scope, file_specifier, file_specifier_version)?;
                 // since we updated this file, let us update the header
@@ -190,3 +191,41 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
         self.f.fcursor()
     }
 }
+
+pub struct BufferedScanner<'a> {
+    d: &'a [u8],
+    i: usize,
+}
+
+impl<'a> BufferedScanner<'a> {
+    pub const fn new(d: &'a [u8]) -> Self {
+        Self { d, i: 0 }
+    }
+    pub const fn remaining(&self) -> usize {
+        self.d.len() - self.i
+    }
+    pub const fn consumed(&self) -> usize {
+        self.i
+    }
+    pub const fn cursor(&self) -> usize {
+        self.i
+    }
+    pub(crate) fn has_left(&self, sizeof: usize) -> bool {
+        self.remaining() >= sizeof
+    }
+    unsafe fn _cursor(&self) -> *const u8 {
+        self.d.as_ptr().add(self.i)
+    }
+    pub fn eof(&self) -> bool {
+        self.remaining() == 0
+    }
+}
+
+impl<'a> BufferedScanner<'a> {
+    pub unsafe fn next_u64_le(&mut self) -> u64 {
+        let mut b = [0u8; sizeof!(u64)];
+        ptr::copy_nonoverlapping(self._cursor(), b.as_mut_ptr(), sizeof!(u64));
+        self.i += sizeof!(u64);
+        u64::from_le_bytes(b)
+    }
+}
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index bd69fba5..c3f2d360 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -408,3 +408,38 @@ impl<const N: usize> Threshold<N> {
         self.now < N
     }
 }
+
+pub trait EndianQW {
+    fn u64_bytes_le(&self) -> [u8; 8];
+    fn u64_bytes_be(&self) -> [u8; 8];
+    fn u64_bytes_ne(&self) -> [u8; 8] {
+        if cfg!(target_endian = "big") {
+            self.u64_bytes_be()
+        } else {
+            self.u64_bytes_le()
+        }
+    }
+}
+
+pub trait EndianDW {
+    fn u32_bytes_le(&self) -> [u8; 8];
+    fn u32_bytes_be(&self) -> [u8; 8];
+    fn u32_bytes_ne(&self) -> [u8; 8] {
+        if cfg!(target_endian = "big") {
+            self.u32_bytes_be()
+        } else {
+            self.u32_bytes_le()
+        }
+    }
+}
+
+macro_rules! impl_endian {
+    ($($ty:ty),*) => {
+        $(impl EndianQW for $ty {
+            fn u64_bytes_le(&self) -> [u8; 8] { (*self as u64).to_le_bytes() }
+            fn u64_bytes_be(&self) -> [u8; 8] { (*self as u64).to_le_bytes() }
+        })*
+    }
+}
+
+impl_endian!(u8, i8, u16, i16, u32, i32, u64, i64, usize, isize);

From b6d5ad1c7535487c6be7c6c4d55de4a35b9a5dd7 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 10 Aug 2023 08:25:12 +0000
Subject: [PATCH 213/310] Add basic impl for `DictGeneric`

---
 server/src/engine/storage/v1/inf.rs | 171 +++++++++++++++++++++++++---
 server/src/engine/storage/v1/rw.rs  |  23 +++-
 2 files changed, 178 insertions(+), 16 deletions(-)

diff --git a/server/src/engine/storage/v1/inf.rs b/server/src/engine/storage/v1/inf.rs
index 3c4d29f3..c43834c9 100644
--- a/server/src/engine/storage/v1/inf.rs
+++ b/server/src/engine/storage/v1/inf.rs
@@ -30,15 +30,16 @@ use {
     crate::{
         engine::{
             data::{
-                dict::DictEntryGeneric,
+                cell::Datacell,
+                dict::{DictEntryGeneric, DictGeneric},
                 tag::{DataTag, TagClass},
             },
             idx::{AsKey, AsValue},
             storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult},
         },
-        util::EndianQW,
+        util::{copy_slice_to_array as memcpy, EndianQW},
     },
-    std::collections::HashMap,
+    std::{cmp, collections::HashMap, mem},
 };
 
 type VecU8 = Vec<u8>;
@@ -123,12 +124,15 @@ pub trait PersistDict {
     /// enc md for an entry
     fn enc_entry_metadata(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);
     /// dec the entry metadata
+    /// SAFETY: Must have passed entry pretest
     unsafe fn dec_entry_metadata(scanner: &mut BufferedScanner) -> Option<Self::Metadata>;
     // entry (coupled)
     /// enc a packed entry
     fn enc_entry_coupled(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);
     /// dec a packed entry
-    fn dec_entry_coupled(
+    /// SAFETY: must have verified metadata with src (unless explicitly skipped with the `DEC_VERIFY_MD_WITH_SRC_STANDALONE`)
+    /// flag
+    unsafe fn dec_entry_coupled(
         scanner: &mut BufferedScanner,
         md: Self::Metadata,
     ) -> Option<(Self::Key, Self::Value)>;
@@ -136,11 +140,15 @@ pub trait PersistDict {
     /// enc key for a normal entry
     fn enc_key(buf: &mut VecU8, key: &Self::Key);
     /// dec normal entry key
-    fn dec_key(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Key>;
+    /// SAFETY: must have verified metadata with src (unless explicitly skipped with the `DEC_VERIFY_MD_WITH_SRC_STANDALONE`)
+    /// flag
+    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Key>;
     /// enc val for a normal entry
     fn enc_val(buf: &mut VecU8, val: &Self::Value);
     /// dec normal entry val
-    fn dec_val(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Value>;
+    /// SAFETY: must have verified metadata with src (unless explicitly skipped with the `DEC_VERIFY_MD_WITH_SRC_STANDALONE`)
+    /// flag
+    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Value>;
 }
 
 /*
@@ -148,18 +156,22 @@ pub trait PersistDict {
 */
 
 pub fn encode_dict<Pd: PersistDict>(dict: &HashMap<Pd::Key, Pd::Value>) -> Vec<u8> {
-    let mut buf = Vec::new();
+    let mut v = vec![];
+    _encode_dict::<Pd>(&mut v, dict);
+    v
+}
+
+fn _encode_dict<Pd: PersistDict>(buf: &mut VecU8, dict: &HashMap<Pd::Key, Pd::Value>) {
     buf.extend(dict.len().u64_bytes_le());
     for (key, val) in dict {
-        Pd::enc_entry_metadata(&mut buf, key, val);
+        Pd::enc_entry_metadata(buf, key, val);
         if Pd::ENC_COUPLED {
-            Pd::enc_entry_coupled(&mut buf, key, val);
+            Pd::enc_entry_coupled(buf, key, val);
         } else {
-            Pd::enc_key(&mut buf, key);
-            Pd::enc_val(&mut buf, val);
+            Pd::enc_key(buf, key);
+            Pd::enc_val(buf, val);
         }
     }
-    buf
 }
 
 pub fn decode_dict<Pd: PersistDict>(
@@ -193,7 +205,10 @@ pub fn decode_dict<Pd: PersistDict>(
         let k;
         let v;
         if Pd::DEC_COUPLED {
-            match Pd::dec_entry_coupled(scanner, md) {
+            match unsafe {
+                // UNSAFE(@ohsayan): verified metadata
+                Pd::dec_entry_coupled(scanner, md)
+            } {
                 Some((_k, _v)) => {
                     k = _k;
                     v = _v;
@@ -201,7 +216,10 @@ pub fn decode_dict<Pd: PersistDict>(
                 None => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
             }
         } else {
-            match (Pd::dec_key(scanner, &md), Pd::dec_val(scanner, &md)) {
+            match unsafe {
+                // UNSAFE(@ohsayan): verified metadata
+                (Pd::dec_key(scanner, &md), Pd::dec_val(scanner, &md))
+            } {
                 (Some(_k), Some(_v)) => {
                     k = _k;
                     v = _v;
@@ -219,3 +237,128 @@ pub fn decode_dict<Pd: PersistDict>(
         Err(SDSSError::InternalDecodeStructureIllegalData)
     }
 }
+
+/*
+    impls
+*/
+
+pub struct DGEntryMD {
+    klen: usize,
+    dscr: u8,
+}
+
+impl DGEntryMD {
+    fn decode(data: [u8; 9]) -> Self {
+        Self {
+            klen: u64::from_le_bytes(memcpy(&data[..8])) as usize,
+            dscr: data[8],
+        }
+    }
+    fn encode(klen: usize, dscr: u8) -> [u8; 9] {
+        let mut ret = [0u8; 9];
+        ret[..8].copy_from_slice(&klen.u64_bytes_le());
+        ret[8] = dscr;
+        ret
+    }
+}
+
+impl PersistDictEntryMetadata for DGEntryMD {
+    fn verify_with_src(&self, scanner: &BufferedScanner) -> bool {
+        static EXPECT_ATLEAST: [u8; 4] = [0, 1, 8, 8]; // PAD to align
+        let lbound_rem = self.klen + EXPECT_ATLEAST[cmp::min(self.dscr, 4) as usize] as usize;
+        scanner.has_left(lbound_rem) & (self.dscr <= PersistDictEntryDscr::Dict.value_u8())
+    }
+}
+
+impl PersistDict for DictGeneric {
+    type Key = Box<str>;
+    type Value = DictEntryGeneric;
+    type Metadata = DGEntryMD;
+    const ENC_COUPLED: bool = true;
+    const DEC_ENTRYMD_INFALLIBLE: bool = true;
+    const DEC_COUPLED: bool = false;
+    const DEC_VERIFY_MD_WITH_SRC_STANDALONE: bool = true;
+    fn metadec_pretest_routine(_: &BufferedScanner) -> bool {
+        true
+    }
+    fn metadec_pretest_entry(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u64) + 1)
+    }
+    fn enc_entry_metadata(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) {
+        buf.extend(key.len().u64_bytes_le());
+    }
+    unsafe fn dec_entry_metadata(scanner: &mut BufferedScanner) -> Option<Self::Metadata> {
+        Some(Self::Metadata::decode(scanner.next_chunk()))
+    }
+    fn enc_entry_coupled(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
+        match val {
+            DictEntryGeneric::Null => {
+                buf.push(PersistDictEntryDscr::Null.value_u8());
+                buf.extend(key.as_bytes())
+            }
+            DictEntryGeneric::Map(map) => {
+                buf.push(PersistDictEntryDscr::Dict.value_u8());
+                buf.extend(key.as_bytes());
+                _encode_dict::<Self>(buf, map);
+            }
+            DictEntryGeneric::Lit(dc) => {
+                buf.push(
+                    PersistDictEntryDscr::translate_from_class(dc.tag().tag_class()).value_u8()
+                        * (!dc.is_null() as u8),
+                );
+                buf.extend(key.as_bytes());
+                fn encode_element(buf: &mut VecU8, dc: &Datacell) {
+                    unsafe {
+                        use TagClass::*;
+                        match dc.tag().tag_class() {
+                            Bool if dc.is_init() => buf.push(dc.read_bool() as u8),
+                            Bool => {}
+                            UnsignedInt | SignedInt | Float => {
+                                buf.extend(dc.read_uint().to_le_bytes())
+                            }
+                            Str | Bin => {
+                                let slc = dc.read_bin();
+                                buf.extend(slc.len().u64_bytes_le());
+                                buf.extend(slc);
+                            }
+                            List => {
+                                let lst = dc.read_list().read();
+                                buf.extend(lst.len().u64_bytes_le());
+                                for item in lst.iter() {
+                                    encode_element(buf, item);
+                                }
+                            }
+                        }
+                    }
+                }
+                encode_element(buf, dc);
+            }
+        }
+    }
+    unsafe fn dec_entry_coupled(
+        _: &mut BufferedScanner,
+        _: Self::Metadata,
+    ) -> Option<(Self::Key, Self::Value)> {
+        unimplemented!()
+    }
+    fn enc_key(_: &mut VecU8, _: &Self::Key) {
+        unimplemented!()
+    }
+    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Key> {
+        String::from_utf8(scanner.next_chunk_variable(md.klen).to_owned())
+            .map(|s| s.into_boxed_str())
+            .ok()
+    }
+    fn enc_val(_: &mut VecU8, _: &Self::Value) {
+        unimplemented!()
+    }
+    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Value> {
+        unsafe fn decode_element(
+            _: &mut BufferedScanner,
+            _: PersistDictEntryDscr,
+        ) -> Option<DictEntryGeneric> {
+            todo!()
+        }
+        decode_element(scanner, mem::transmute(md.dscr))
+    }
+}
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index ca42b93b..5e015411 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -35,7 +35,7 @@ use {
     std::{
         fs::File,
         io::{Read, Seek, SeekFrom, Write},
-        ptr,
+        ptr, slice,
     },
 };
 
@@ -219,13 +219,32 @@ impl<'a> BufferedScanner<'a> {
     pub fn eof(&self) -> bool {
         self.remaining() == 0
     }
+    unsafe fn _incr(&mut self, by: usize) {
+        self.i += by;
+    }
 }
 
 impl<'a> BufferedScanner<'a> {
     pub unsafe fn next_u64_le(&mut self) -> u64 {
         let mut b = [0u8; sizeof!(u64)];
         ptr::copy_nonoverlapping(self._cursor(), b.as_mut_ptr(), sizeof!(u64));
-        self.i += sizeof!(u64);
+        self._incr(sizeof!(u64));
         u64::from_le_bytes(b)
     }
+    pub unsafe fn next_chunk<const N: usize>(&mut self) -> [u8; N] {
+        let mut b = [0u8; N];
+        ptr::copy_nonoverlapping(self._cursor(), b.as_mut_ptr(), N);
+        self._incr(N);
+        b
+    }
+    pub unsafe fn next_chunk_variable(&mut self, size: usize) -> &[u8] {
+        let r = slice::from_raw_parts(self._cursor(), size);
+        self._incr(size);
+        r
+    }
+    pub unsafe fn next_byte(&mut self) -> u8 {
+        let r = *self._cursor();
+        self._incr(1);
+        r
+    }
 }

From bd3225693fe40542bd44aac77113a370004e04c8 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 10 Aug 2023 13:14:56 +0000
Subject: [PATCH 214/310] Upgrade deps

---
 Cargo.lock             | 115 ++++++++++++++++++++++-------------------
 server/Cargo.toml      |  12 ++---
 sky-bench/Cargo.toml   |   4 +-
 stress-test/Cargo.toml |   2 +-
 4 files changed, 70 insertions(+), 63 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 9f4a718a..42b12021 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -42,9 +42,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "1.0.2"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
+checksum = "86b8f9420f797f2d9e935edf629310eb938a0d839f984e25327f3c7eed22300c"
 dependencies = [
  "memchr",
 ]
@@ -87,7 +87,7 @@ checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -234,11 +234,12 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.79"
+version = "1.0.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
+checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01"
 dependencies = [
  "jobserver",
+ "libc",
 ]
 
 [[package]]
@@ -422,6 +423,12 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "deranged"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929"
+
 [[package]]
 name = "devtimer"
 version = "4.0.1"
@@ -472,9 +479,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 
 [[package]]
 name = "errno"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
+checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
 dependencies = [
  "errno-dragonfly",
  "libc",
@@ -722,9 +729,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
 
 [[package]]
 name = "jemalloc-sys"
-version = "0.5.3+5.3.0-patched"
+version = "0.5.4+5.3.0-patched"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9bd5d616ea7ed58b571b2e209a65759664d7fb021a0819d7a790afc67e47ca1"
+checksum = "ac6c1946e1cea1788cbfde01c993b52a10e2da07f4bac608228d1bed20bfebf2"
 dependencies = [
  "cc",
  "libc",
@@ -732,9 +739,9 @@ dependencies = [
 
 [[package]]
 name = "jemallocator"
-version = "0.5.0"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16c2514137880c52b0b4822b563fadd38257c1f380858addb74a400889696ea6"
+checksum = "a0de374a9f8e63150e6f5e8a60cc14c668226d7a347d8aee1a45766e3c4dd3bc"
 dependencies = [
  "jemalloc-sys",
  "libc",
@@ -786,9 +793,9 @@ dependencies = [
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.3"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
+checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
 
 [[package]]
 name = "lock_api"
@@ -908,9 +915,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
 [[package]]
 name = "openssl"
-version = "0.10.55"
+version = "0.10.56"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d"
+checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e"
 dependencies = [
  "bitflags 1.3.2",
  "cfg-if",
@@ -929,23 +936,23 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
 name = "openssl-src"
-version = "111.26.0+1.1.1u"
+version = "111.27.0+1.1.1v"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efc62c9f12b22b8f5208c23a7200a442b2e5999f8bdf80233852122b5a4f6f37"
+checksum = "06e8f197c82d7511c5b014030c9b1efeda40d7d5f99d23b4ceed3524a5e63f02"
 dependencies = [
  "cc",
 ]
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.90"
+version = "0.9.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6"
+checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac"
 dependencies = [
  "cc",
  "libc",
@@ -1002,9 +1009,9 @@ dependencies = [
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.10"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57"
+checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05"
 
 [[package]]
 name = "pin-utils"
@@ -1143,9 +1150,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.9.1"
+version = "1.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
+checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1155,9 +1162,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.3.3"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310"
+checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1178,9 +1185,9 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
 
 [[package]]
 name = "rustix"
-version = "0.38.4"
+version = "0.38.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5"
+checksum = "172891ebdceb05aa0005f533a6cbfca599ddd7d966f6f5d4d9b2e70478e70399"
 dependencies = [
  "bitflags 2.3.3",
  "errno",
@@ -1235,29 +1242,29 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
 [[package]]
 name = "serde"
-version = "1.0.175"
+version = "1.0.183"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b"
+checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.175"
+version = "1.0.183"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4"
+checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.103"
+version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b"
+checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
 dependencies = [
  "itoa",
  "ryu",
@@ -1448,12 +1455,12 @@ checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
 
 [[package]]
 name = "socket2"
-version = "0.4.9"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
+checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877"
 dependencies = [
  "libc",
- "winapi",
+ "windows-sys",
 ]
 
 [[package]]
@@ -1507,9 +1514,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.27"
+version = "2.0.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0"
+checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1518,9 +1525,9 @@ dependencies = [
 
 [[package]]
 name = "sysinfo"
-version = "0.29.6"
+version = "0.29.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7cb97a5a85a136d84e75d5c3cf89655090602efb1be0d8d5337b7e386af2908"
+checksum = "165d6d8539689e3d3bc8b98ac59541e1f21c7de7c85d60dc80e43ae0ed2113db"
 dependencies = [
  "cfg-if",
  "core-foundation-sys",
@@ -1562,10 +1569,11 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.23"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446"
+checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea"
 dependencies = [
+ "deranged",
  "serde",
  "time-core",
 ]
@@ -1578,11 +1586,10 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
 
 [[package]]
 name = "tokio"
-version = "1.29.1"
+version = "1.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da"
+checksum = "2d3ce25f50619af8b0aec2eb23deebe84249e19e2ddd393a6e16e3300a6dadfd"
 dependencies = [
- "autocfg",
  "backtrace",
  "bytes",
  "libc",
@@ -1604,7 +1611,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -1702,7 +1709,7 @@ checksum = "f7e1ba1f333bd65ce3c9f27de592fcbc256dafe3af2717f56d7c87761fbaccf4"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -1756,7 +1763,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
  "wasm-bindgen-shared",
 ]
 
@@ -1778,7 +1785,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.27",
+ "syn 2.0.28",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -1897,9 +1904,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
 
 [[package]]
 name = "winnow"
-version = "0.5.0"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81fac9742fd1ad1bd9643b991319f72dd031016d44b77039a26977eb667141e7"
+checksum = "acaaa1190073b2b101e15083c38ee8ec891b5e05cbee516521e94ec008f61e64"
 dependencies = [
  "memchr",
 ]
@@ -1926,7 +1933,7 @@ dependencies = [
  "hmac",
  "pbkdf2",
  "sha1",
- "time 0.3.23",
+ "time 0.3.25",
  "zstd",
 ]
 
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 565c2997..284cb58d 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -18,12 +18,12 @@ clap = { version = "2", features = ["yaml"] }
 env_logger = "0.10.0"
 hashbrown = { version = "0.14.0", features = ["raw"] }
 log = "0.4.19"
-openssl = { version = "0.10.55", features = ["vendored"] }
+openssl = { version = "0.10.56", features = ["vendored"] }
 crossbeam-epoch = { version = "0.9.15" }
 parking_lot = "0.12.1"
 regex = "1.9.1"
-serde = { version = "1.0.175", features = ["derive"] }
-tokio = { version = "1.29.1", features = ["full"] }
+serde = { version = "1.0.183", features = ["derive"] }
+tokio = { version = "1.30.0", features = ["full"] }
 tokio-openssl = "0.6.3"
 toml = "0.7.6"
 base64 = "0.21.2"
@@ -32,7 +32,7 @@ crc = "3.0.1"
 
 [target.'cfg(all(not(target_env = "msvc"), not(miri)))'.dependencies]
 # external deps
-jemallocator = "0.5.0"
+jemallocator = "0.5.4"
 [target.'cfg(target_os = "windows")'.dependencies]
 # external deps
 winapi = { version = "0.3.9", features = ["fileapi"] }
@@ -43,7 +43,7 @@ libc = "0.2.147"
 
 [target.'cfg(unix)'.build-dependencies]
 # external deps
-cc = "1.0.79"
+cc = "1.0.82"
 
 [dev-dependencies]
 # internal deps
@@ -55,7 +55,7 @@ skytable = { git = "https://github.com/skytable/client-rust", features = [
 # external deps
 bincode = "1.3.3"
 rand = "0.8.5"
-tokio = { version = "1.29.1", features = ["test-util"] }
+tokio = { version = "1.30.0", features = ["test-util"] }
 
 [features]
 nightly = []
diff --git a/sky-bench/Cargo.toml b/sky-bench/Cargo.toml
index cf6ed8f5..8726792b 100644
--- a/sky-bench/Cargo.toml
+++ b/sky-bench/Cargo.toml
@@ -18,6 +18,6 @@ clap = { version = "2", features = ["yaml"] }
 log = "0.4.19"
 env_logger = "0.10.0"
 devtimer = "4.0.1"
-serde = { version = "1.0.175", features = ["derive"] }
-serde_json = "1.0.103"
+serde = { version = "1.0.183", features = ["derive"] }
+serde_json = "1.0.104"
 rand = "0.8.5"
diff --git a/stress-test/Cargo.toml b/stress-test/Cargo.toml
index 6211e517..f05bb998 100644
--- a/stress-test/Cargo.toml
+++ b/stress-test/Cargo.toml
@@ -14,7 +14,7 @@ skytable = { git = "https://github.com/skytable/client-rust.git", branch = "next
 ] }
 devtimer = "4.0.1"
 # external deps
-sysinfo = "0.29.6"
+sysinfo = "0.29.7"
 env_logger = "0.10.0"
 log = "0.4.19"
 rand = "0.8.5"

From 026b5ef0d2bc9537cea69689c53193504411d7ea Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 11 Aug 2023 14:17:04 +0000
Subject: [PATCH 215/310] Implement dict dec and resolve null disambiguation

The previous logic of using a different null was extremely confusing,
and we essentially ended up creating "two different null types."

That has been removed and we have implemented the dict dec methods and
rectified the enc methods.
---
 server/src/corestore/rc.rs                    |   2 +-
 server/src/engine/core/space.rs               |   7 +-
 server/src/engine/data/cell.rs                |   6 +
 server/src/engine/data/dict.rs                |  98 +++++++-------
 server/src/engine/data/tag.rs                 |   2 +-
 server/src/engine/data/tests/md_dict_tests.rs |   6 +-
 server/src/engine/ql/ddl/syn.rs               |   7 +-
 server/src/engine/ql/tests.rs                 |   2 +-
 server/src/engine/storage/v1/inf.rs           | 121 +++++++++++++++---
 server/src/storage/v1/tests.rs                |  13 +-
 10 files changed, 176 insertions(+), 88 deletions(-)

diff --git a/server/src/corestore/rc.rs b/server/src/corestore/rc.rs
index 080ececa..08439fbb 100644
--- a/server/src/corestore/rc.rs
+++ b/server/src/corestore/rc.rs
@@ -163,7 +163,7 @@ impl AsRef<[u8]> for SharedSlice {
 impl Borrow<[u8]> for SharedSlice {
     #[inline(always)]
     fn borrow(&self) -> &[u8] {
-        self.as_slice().borrow()
+        self.as_slice()
     }
 }
 
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 4750c4aa..c311780b 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -124,7 +124,8 @@ impl Space {
         // check env
         let env = match props.remove(SpaceMeta::KEY_ENV) {
             Some(DictEntryGeneric::Map(m)) if props.is_empty() => m,
-            Some(DictEntryGeneric::Null) | None if props.is_empty() => IndexST::default(),
+            Some(DictEntryGeneric::Lit(l)) if l.is_null() => IndexST::default(),
+            None if props.is_empty() => IndexST::default(),
             _ => {
                 return Err(DatabaseError::DdlSpaceBadProperty);
             }
@@ -169,7 +170,9 @@ impl Space {
                         return Err(DatabaseError::DdlSpaceBadProperty);
                     }
                 }
-                Some(DictEntryGeneric::Null) if updated_props.is_empty() => space_env.clear(),
+                Some(DictEntryGeneric::Lit(l)) if updated_props.is_empty() & l.is_null() => {
+                    space_env.clear()
+                }
                 None => {}
                 _ => return Err(DatabaseError::DdlSpaceBadProperty),
             }
diff --git a/server/src/engine/data/cell.rs b/server/src/engine/data/cell.rs
index 1ce99a2d..95017988 100644
--- a/server/src/engine/data/cell.rs
+++ b/server/src/engine/data/cell.rs
@@ -202,6 +202,12 @@ impl Datacell {
     pub fn list(&self) -> &RwLock<Vec<Self>> {
         self.try_list().unwrap()
     }
+    pub unsafe fn new_qw(qw: u64, tag: CUTag) -> Datacell {
+        Self::new(
+            tag,
+            DataRaw::word(SpecialPaddedWord::store(qw).dwordqn_promote()),
+        )
+    }
 }
 
 direct_from! {
diff --git a/server/src/engine/data/dict.rs b/server/src/engine/data/dict.rs
index 148af9a6..fbab8ef0 100644
--- a/server/src/engine/data/dict.rs
+++ b/server/src/engine/data/dict.rs
@@ -53,12 +53,6 @@ pub enum DictEntryGeneric {
     Lit(Datacell),
     /// A map
     Map(DictGeneric),
-    /// A null
-    ///
-    /// This distinction is important because we currently only allow [`Datacell`] to store information about the intialization state (as it should be)
-    /// and it can't distinct between "null-but-concrete-types" (that's the task of our engine: to enforce the schema but it's not DC's task). Hence this
-    /// specific flag tells us that this is null (and neither a lit nor a map)
-    Null,
 }
 
 #[derive(Debug, PartialEq)]
@@ -91,15 +85,15 @@ pub fn rflatten_metadata(new: DictGeneric) -> MetaDict {
 fn _rflatten_metadata(new: DictGeneric, empty: &mut MetaDict) {
     for (key, val) in new {
         match val {
-            DictEntryGeneric::Lit(l) => {
+            DictEntryGeneric::Lit(l) if l.is_init() => {
                 empty.insert(key, MetaDictEntry::Data(l));
             }
+            DictEntryGeneric::Lit(_) => {}
             DictEntryGeneric::Map(m) => {
                 let mut rnew = MetaDict::new();
                 _rflatten_metadata(m, &mut rnew);
                 empty.insert(key, MetaDictEntry::Map(rnew));
             }
-            DictEntryGeneric::Null => {}
         }
     }
 }
@@ -153,57 +147,55 @@ fn rmerge_metadata_prepare_patch(
     let mut okay = true;
     while new.len() != 0 && okay {
         let (key, new_entry) = new.next().unwrap();
-        match (current.get(&key), &new_entry) {
+        match (current.get(&key), new_entry) {
             // non-null -> non-null: merge flatten update
-            (Some(this_current), DictEntryGeneric::Lit(_) | DictEntryGeneric::Map(_)) => {
-                okay &= {
-                    match (this_current, new_entry) {
-                        (MetaDictEntry::Data(this_data), DictEntryGeneric::Lit(new_data))
-                            if this_data.kind() == new_data.kind() =>
-                        {
-                            patch
-                                .0
-                                .insert(key, Some(MetaDictPatchEntry::Data(new_data)));
-                            true
-                        }
-                        (
-                            MetaDictEntry::Map(this_recursive_data),
-                            DictEntryGeneric::Map(new_recursive_data),
-                        ) => {
-                            let mut this_patch = MetaDictPatch::default();
-                            let okay = rmerge_metadata_prepare_patch(
-                                this_recursive_data,
-                                new_recursive_data,
-                                &mut this_patch,
-                            );
-                            patch
-                                .0
-                                .insert(key, Some(MetaDictPatchEntry::Map(this_patch)));
-                            okay
-                        }
-                        _ => false,
-                    }
-                };
-            }
-            // null -> non-null: flatten insert
-            (None, DictEntryGeneric::Lit(_) | DictEntryGeneric::Map(_)) => match new_entry {
-                DictEntryGeneric::Lit(d) => {
-                    let _ = patch.0.insert(key, Some(MetaDictPatchEntry::Data(d)));
-                }
-                DictEntryGeneric::Map(m) => {
-                    let mut this_patch = MetaDictPatch::default();
-                    okay &= rmerge_metadata_prepare_patch(&into_dict!(), m, &mut this_patch);
-                    let _ = patch
+            (Some(MetaDictEntry::Data(this_data)), DictEntryGeneric::Lit(new_data))
+                if new_data.is_init() =>
+            {
+                if this_data.kind() == new_data.kind() {
+                    patch
                         .0
-                        .insert(key, Some(MetaDictPatchEntry::Map(this_patch)));
+                        .insert(key, Some(MetaDictPatchEntry::Data(new_data)));
+                } else {
+                    okay = false;
                 }
-                _ => unreachable!(),
-            },
+            }
+            (Some(MetaDictEntry::Data(_)), DictEntryGeneric::Map(_)) => {
+                okay = false;
+            }
+            (
+                Some(MetaDictEntry::Map(this_recursive_data)),
+                DictEntryGeneric::Map(new_recursive_data),
+            ) => {
+                let mut this_patch = MetaDictPatch::default();
+                let _okay = rmerge_metadata_prepare_patch(
+                    this_recursive_data,
+                    new_recursive_data,
+                    &mut this_patch,
+                );
+                patch
+                    .0
+                    .insert(key, Some(MetaDictPatchEntry::Map(this_patch)));
+                okay &= _okay;
+            }
+            // null -> non-null: flatten insert
+            (None, DictEntryGeneric::Lit(l)) if l.is_init() => {
+                let _ = patch.0.insert(key, Some(MetaDictPatchEntry::Data(l)));
+            }
+            (None, DictEntryGeneric::Map(m)) => {
+                let mut this_patch = MetaDictPatch::default();
+                okay &= rmerge_metadata_prepare_patch(&into_dict!(), m, &mut this_patch);
+                let _ = patch
+                    .0
+                    .insert(key, Some(MetaDictPatchEntry::Map(this_patch)));
+            }
             // non-null -> null: remove
-            (Some(_), DictEntryGeneric::Null) => {
+            (Some(_), DictEntryGeneric::Lit(l)) => {
+                debug_assert!(l.is_null());
                 patch.0.insert(key, None);
             }
-            (None, DictEntryGeneric::Null) => {
+            (None, DictEntryGeneric::Lit(l)) => {
+                debug_assert!(l.is_null());
                 // ignore
             }
         }
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
index 03d1c9cc..01d143fa 100644
--- a/server/src/engine/data/tag.rs
+++ b/server/src/engine/data/tag.rs
@@ -173,7 +173,7 @@ macro_rules! cutag {
 }
 
 impl CUTag {
-    const fn new(class: TagClass, unique: TagUnique) -> Self {
+    pub const fn new(class: TagClass, unique: TagUnique) -> Self {
         Self { class, unique }
     }
 }
diff --git a/server/src/engine/data/tests/md_dict_tests.rs b/server/src/engine/data/tests/md_dict_tests.rs
index 1110b8d2..c0c3adf4 100644
--- a/server/src/engine/data/tests/md_dict_tests.rs
+++ b/server/src/engine/data/tests/md_dict_tests.rs
@@ -33,7 +33,7 @@ use crate::engine::data::{
 fn t_simple_flatten() {
     let generic_dict: DictGeneric = into_dict! {
         "a_valid_key" => DictEntryGeneric::Lit(100u64.into()),
-        "a_null_key" => DictEntryGeneric::Null,
+        "a_null_key" => Datacell::null(),
     };
     let expected: MetaDict = into_dict!(
         "a_valid_key" => Datacell::new_uint(100)
@@ -52,7 +52,7 @@ fn t_simple_patch() {
     let new: DictGeneric = into_dict! {
         "a" => Datacell::new_uint(1),
         "b" => Datacell::new_uint(2),
-        "z" => DictEntryGeneric::Null,
+        "z" => Datacell::null(),
     };
     let expected: MetaDict = into_dict! {
         "a" => Datacell::new_uint(1),
@@ -96,7 +96,7 @@ fn patch_null_out_dict() {
     let new: DictGeneric = into_dict! {
         "a" => Datacell::new_uint(2),
         "b" => Datacell::new_uint(3),
-        "z" => DictEntryGeneric::Null,
+        "z" => Datacell::null(),
     };
     assert!(dict::rmerge_metadata(&mut current, new));
     assert_eq!(current, expected);
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index bdc5edce..89b8ca48 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -46,7 +46,10 @@
 
 use crate::{
     engine::{
-        data::dict::{DictEntryGeneric, DictGeneric},
+        data::{
+            cell::Datacell,
+            dict::{DictEntryGeneric, DictGeneric},
+        },
         error::{LangError, LangResult},
         ql::{
             ast::{QueryData, State},
@@ -176,7 +179,7 @@ where
                             // UNSAFE(@ohsayan): we only switch to this when we've already read in a key
                             key.take().as_str().into()
                         },
-                        DictEntryGeneric::Null,
+                        DictEntryGeneric::Lit(Datacell::null()),
                     )
                     .is_none(),
                 );
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 28051312..0ec76fe1 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -81,7 +81,7 @@ pub trait NullableDictEntry {
 
 impl NullableDictEntry for Null {
     fn data(self) -> crate::engine::data::DictEntryGeneric {
-        crate::engine::data::DictEntryGeneric::Null
+        crate::engine::data::DictEntryGeneric::Lit(Datacell::null())
     }
 }
 
diff --git a/server/src/engine/storage/v1/inf.rs b/server/src/engine/storage/v1/inf.rs
index c43834c9..84b13a54 100644
--- a/server/src/engine/storage/v1/inf.rs
+++ b/server/src/engine/storage/v1/inf.rs
@@ -32,7 +32,7 @@ use {
             data::{
                 cell::Datacell,
                 dict::{DictEntryGeneric, DictGeneric},
-                tag::{DataTag, TagClass},
+                tag::{CUTag, DataTag, TagClass, TagUnique},
             },
             idx::{AsKey, AsValue},
             storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult},
@@ -62,11 +62,13 @@ pub enum PersistDictEntryDscr {
 impl PersistDictEntryDscr {
     /// translates the tag class definition into the dscr definition
     pub const fn translate_from_class(class: TagClass) -> Self {
-        unsafe { core::mem::transmute(class.d() + 1) }
+        unsafe { Self::from_raw(class.d() + 1) }
+    }
+    pub const unsafe fn from_raw(v: u8) -> Self {
+        core::mem::transmute(v)
     }
     pub fn new_from_dict_gen_entry(e: &DictEntryGeneric) -> Self {
         match e {
-            DictEntryGeneric::Null => Self::Null,
             DictEntryGeneric::Map(_) => Self::Dict,
             DictEntryGeneric::Lit(dc) => Self::translate_from_class(dc.tag().tag_class()),
         }
@@ -87,6 +89,10 @@ impl PersistDictEntryDscr {
     pub const fn is_recursive(&self) -> bool {
         self.value_u8() >= Self::List.value_u8()
     }
+    fn into_class(&self) -> TagClass {
+        debug_assert!(*self != Self::Null);
+        unsafe { mem::transmute(self.value_u8() - 1) }
+    }
 }
 
 /*
@@ -177,7 +183,7 @@ fn _encode_dict<Pd: PersistDict>(buf: &mut VecU8, dict: &HashMap<Pd::Key, Pd::Va
 pub fn decode_dict<Pd: PersistDict>(
     scanner: &mut BufferedScanner,
 ) -> SDSSResult<HashMap<Pd::Key, Pd::Value>> {
-    if Pd::metadec_pretest_routine(scanner) & scanner.has_left(sizeof!(u64)) {
+    if !(Pd::metadec_pretest_routine(scanner) & scanner.has_left(sizeof!(u64))) {
         return Err(SDSSError::InternalDecodeStructureCorrupted);
     }
     let dict_len = unsafe {
@@ -185,7 +191,7 @@ pub fn decode_dict<Pd: PersistDict>(
         scanner.next_u64_le() as usize
     };
     let mut dict = HashMap::with_capacity(dict_len);
-    while Pd::metadec_pretest_entry(scanner) {
+    while Pd::metadec_pretest_entry(scanner) & (dict.len() < dict_len) {
         let md = unsafe {
             // UNSAFE(@ohsayan): this is compeletely because of the entry pretest
             match Pd::dec_entry_metadata(scanner) {
@@ -224,7 +230,9 @@ pub fn decode_dict<Pd: PersistDict>(
                     k = _k;
                     v = _v;
                 }
-                _ => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
+                _ => {
+                    return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+                }
             }
         }
         if dict.insert(k, v).is_some() {
@@ -265,7 +273,7 @@ impl DGEntryMD {
 impl PersistDictEntryMetadata for DGEntryMD {
     fn verify_with_src(&self, scanner: &BufferedScanner) -> bool {
         static EXPECT_ATLEAST: [u8; 4] = [0, 1, 8, 8]; // PAD to align
-        let lbound_rem = self.klen + EXPECT_ATLEAST[cmp::min(self.dscr, 4) as usize] as usize;
+        let lbound_rem = self.klen + EXPECT_ATLEAST[cmp::min(self.dscr, 3) as usize] as usize;
         scanner.has_left(lbound_rem) & (self.dscr <= PersistDictEntryDscr::Dict.value_u8())
     }
 }
@@ -292,10 +300,6 @@ impl PersistDict for DictGeneric {
     }
     fn enc_entry_coupled(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
         match val {
-            DictEntryGeneric::Null => {
-                buf.push(PersistDictEntryDscr::Null.value_u8());
-                buf.extend(key.as_bytes())
-            }
             DictEntryGeneric::Map(map) => {
                 buf.push(PersistDictEntryDscr::Dict.value_u8());
                 buf.extend(key.as_bytes());
@@ -354,11 +358,98 @@ impl PersistDict for DictGeneric {
     }
     unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Value> {
         unsafe fn decode_element(
-            _: &mut BufferedScanner,
-            _: PersistDictEntryDscr,
+            scanner: &mut BufferedScanner,
+            dscr: PersistDictEntryDscr,
+            dg_top_element: bool,
         ) -> Option<DictEntryGeneric> {
-            todo!()
+            let r = match dscr {
+                PersistDictEntryDscr::Null => DictEntryGeneric::Lit(Datacell::null()),
+                PersistDictEntryDscr::Bool => {
+                    DictEntryGeneric::Lit(Datacell::new_bool(scanner.next_byte() == 1))
+                }
+                PersistDictEntryDscr::UnsignedInt
+                | PersistDictEntryDscr::SignedInt
+                | PersistDictEntryDscr::Float => DictEntryGeneric::Lit(Datacell::new_qw(
+                    scanner.next_u64_le(),
+                    CUTag::new(
+                        dscr.into_class(),
+                        [
+                            TagUnique::UnsignedInt,
+                            TagUnique::SignedInt,
+                            TagUnique::Illegal,
+                            TagUnique::Illegal, // pad
+                        ][(dscr.value_u8() - 2) as usize],
+                    ),
+                )),
+                PersistDictEntryDscr::Str | PersistDictEntryDscr::Bin => {
+                    let slc_len = scanner.next_u64_le() as usize;
+                    if !scanner.has_left(slc_len) {
+                        return None;
+                    }
+                    let slc = scanner.next_chunk_variable(slc_len);
+                    DictEntryGeneric::Lit(if dscr == PersistDictEntryDscr::Str {
+                        if core::str::from_utf8(slc).is_err() {
+                            return None;
+                        }
+                        Datacell::new_str(
+                            String::from_utf8_unchecked(slc.to_owned()).into_boxed_str(),
+                        )
+                    } else {
+                        Datacell::new_bin(slc.to_owned().into_boxed_slice())
+                    })
+                }
+                PersistDictEntryDscr::List => {
+                    let list_len = scanner.next_u64_le() as usize;
+                    let mut v = Vec::with_capacity(list_len);
+                    while (!scanner.eof()) & (v.len() < list_len) {
+                        let dscr = scanner.next_byte();
+                        if dscr > PersistDictEntryDscr::Dict.value_u8() {
+                            return None;
+                        }
+                        v.push(
+                            match decode_element(
+                                scanner,
+                                PersistDictEntryDscr::from_raw(dscr),
+                                false,
+                            ) {
+                                Some(DictEntryGeneric::Lit(l)) => l,
+                                None => return None,
+                                _ => unreachable!("found top-level dict item in datacell"),
+                            },
+                        );
+                    }
+                    if v.len() == list_len {
+                        DictEntryGeneric::Lit(Datacell::new_list(v))
+                    } else {
+                        return None;
+                    }
+                }
+                PersistDictEntryDscr::Dict => {
+                    if dg_top_element {
+                        DictEntryGeneric::Map(decode_dict::<DictGeneric>(scanner).ok()?)
+                    } else {
+                        unreachable!("found top-level dict item in datacell")
+                    }
+                }
+            };
+            Some(r)
         }
-        decode_element(scanner, mem::transmute(md.dscr))
+        decode_element(scanner, PersistDictEntryDscr::from_raw(md.dscr), true)
     }
 }
+
+#[test]
+fn t_dict() {
+    let dict: DictGeneric = into_dict! {
+        "hello" => Datacell::new_str("world".into()),
+        "omg a null?" => Datacell::null(),
+        "a big fat dict" => DictEntryGeneric::Map(into_dict!(
+            "with a value" => Datacell::new_uint(1002),
+            "and a null" => Datacell::null(),
+        ))
+    };
+    let encoded = encode_dict::<DictGeneric>(&dict);
+    let mut scanner = BufferedScanner::new(&encoded);
+    let decoded = decode_dict::<DictGeneric>(&mut scanner).unwrap();
+    assert_eq!(dict, decoded);
+}
diff --git a/server/src/storage/v1/tests.rs b/server/src/storage/v1/tests.rs
index ca306d12..de891ed9 100644
--- a/server/src/storage/v1/tests.rs
+++ b/server/src/storage/v1/tests.rs
@@ -566,7 +566,6 @@ mod list_tests {
     use super::{de, se};
     use crate::corestore::{htable::Coremap, SharedSlice};
     use crate::kvengine::LockedVec;
-    use core::ops::Deref;
     use parking_lot::RwLock;
     #[test]
     fn test_list_se_de() {
@@ -613,13 +612,7 @@ mod list_tests {
         se::raw_serialize_list_map(&mymap, &mut v).unwrap();
         let de = de::deserialize_list_map(&v).unwrap();
         assert_eq!(de.len(), 1);
-        let mykey_value = de
-            .get("mykey".as_bytes())
-            .unwrap()
-            .value()
-            .deref()
-            .read()
-            .clone();
+        let mykey_value = de.get("mykey".as_bytes()).unwrap().value().read().clone();
         assert_eq!(
             mykey_value,
             vals.into_inner()
@@ -642,14 +635,14 @@ mod list_tests {
         let de = de::deserialize_list_map(&v).unwrap();
         assert_eq!(de.len(), 2);
         assert_eq!(
-            de.get(&key1).unwrap().value().deref().read().clone(),
+            de.get(&key1).unwrap().value().read().clone(),
             val1.into_inner()
                 .into_iter()
                 .map(SharedSlice::from)
                 .collect::<Vec<SharedSlice>>()
         );
         assert_eq!(
-            de.get(&key2).unwrap().value().deref().read().clone(),
+            de.get(&key2).unwrap().value().read().clone(),
             val2.into_inner()
                 .into_iter()
                 .map(SharedSlice::from)

From 29457c6b89248707616b4e5ba8b48f3c977b7dd2 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 13 Aug 2023 14:59:22 +0000
Subject: [PATCH 216/310] Impl enc/dec routines for internal structures

---
 server/src/engine/core/mod.rs                 |   2 +-
 server/src/engine/core/model/mod.rs           |  12 +-
 server/src/engine/core/tests/ddl_model/crt.rs |   8 +-
 server/src/engine/data/tag.rs                 |  48 +++++++
 server/src/engine/mem/vinline.rs              |   2 +-
 .../engine/storage/v1/{inf.rs => inf/mod.rs}  | 135 ++++++++++++++++--
 server/src/engine/storage/v1/inf/tests.rs     |  66 +++++++++
 7 files changed, 246 insertions(+), 27 deletions(-)
 rename server/src/engine/storage/v1/{inf.rs => inf/mod.rs} (80%)
 create mode 100644 server/src/engine/storage/v1/inf/tests.rs

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index d12b7f2f..2c042518 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -26,7 +26,7 @@
 
 mod dml;
 mod index;
-mod model;
+pub(in crate::engine) mod model;
 pub(in crate::engine) mod query_meta;
 mod space;
 mod util;
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index e5e14640..413a8b45 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -324,7 +324,6 @@ impl Field {
 #[derive(Debug, PartialEq, Clone, Copy)]
 pub struct Layer {
     tag: FullTag,
-    config: [usize; 2],
 }
 
 impl Layer {
@@ -376,19 +375,18 @@ impl Layer {
     pub fn tag(&self) -> FullTag {
         self.tag
     }
-    #[cfg(test)]
-    pub fn new_test(tag: FullTag, config: [usize; 2]) -> Self {
-        Self::new(tag, config)
+    pub fn new_empty_props(tag: FullTag) -> Self {
+        Self::new(tag)
     }
     #[inline(always)]
     fn compute_index(&self, dc: &Datacell) -> usize {
         self.tag.tag_class().word() * (dc.is_null() as usize)
     }
-    const fn new(tag: FullTag, config: [usize; 2]) -> Self {
-        Self { tag, config }
+    const fn new(tag: FullTag) -> Self {
+        Self { tag }
     }
     const fn empty(tag: FullTag) -> Self {
-        Self::new(tag, [0; 2])
+        Self::new(tag)
     }
     fn hf(key: &[u8], v: [u8; 7]) -> u16 {
         let mut tot = 0;
diff --git a/server/src/engine/core/tests/ddl_model/crt.rs b/server/src/engine/core/tests/ddl_model/crt.rs
index 72e16501..4f6a2276 100644
--- a/server/src/engine/core/tests/ddl_model/crt.rs
+++ b/server/src/engine/core/tests/ddl_model/crt.rs
@@ -48,8 +48,8 @@ mod validation {
                 .cloned()
                 .collect::<Vec<Field>>(),
             [
-                Field::new([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
-                Field::new([Layer::new_test(FullTag::BIN, [0; 2])].into(), false)
+                Field::new([Layer::new_empty_props(FullTag::STR)].into(), false),
+                Field::new([Layer::new_empty_props(FullTag::BIN)].into(), false)
             ]
         );
         assert_eq!(
@@ -72,8 +72,8 @@ mod validation {
                 .cloned()
                 .collect::<Vec<Field>>(),
             [
-                Field::new([Layer::new_test(FullTag::BIN, [0; 2])].into(), false),
-                Field::new([Layer::new_test(FullTag::STR, [0; 2])].into(), false),
+                Field::new([Layer::new_empty_props(FullTag::BIN)].into(), false),
+                Field::new([Layer::new_empty_props(FullTag::STR)].into(), false),
             ]
         );
         assert_eq!(
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
index 01d143fa..53c4067e 100644
--- a/server/src/engine/data/tag.rs
+++ b/server/src/engine/data/tag.rs
@@ -62,6 +62,54 @@ pub enum TagSelector {
     List = 13,
 }
 
+impl TagSelector {
+    pub const fn max_dscr() -> u8 {
+        Self::List.d()
+    }
+    pub const fn into_full(self) -> FullTag {
+        FullTag::new(self.tag_class(), self, self.tag_unique())
+    }
+    pub const unsafe fn from_raw(v: u8) -> Self {
+        core::mem::transmute(v)
+    }
+    pub const fn tag_unique(&self) -> TagUnique {
+        [
+            TagUnique::Illegal,
+            TagUnique::UnsignedInt,
+            TagUnique::UnsignedInt,
+            TagUnique::UnsignedInt,
+            TagUnique::UnsignedInt,
+            TagUnique::SignedInt,
+            TagUnique::SignedInt,
+            TagUnique::SignedInt,
+            TagUnique::SignedInt,
+            TagUnique::Illegal,
+            TagUnique::Illegal,
+            TagUnique::Bin,
+            TagUnique::Str,
+            TagUnique::Illegal,
+        ][self.d() as usize]
+    }
+    pub const fn tag_class(&self) -> TagClass {
+        [
+            TagClass::Bool,
+            TagClass::UnsignedInt,
+            TagClass::UnsignedInt,
+            TagClass::UnsignedInt,
+            TagClass::UnsignedInt,
+            TagClass::SignedInt,
+            TagClass::SignedInt,
+            TagClass::SignedInt,
+            TagClass::SignedInt,
+            TagClass::Float,
+            TagClass::Float,
+            TagClass::Bin,
+            TagClass::Str,
+            TagClass::List,
+        ][self.d() as usize]
+    }
+}
+
 #[repr(u8)]
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord)]
 pub enum TagUnique {
diff --git a/server/src/engine/mem/vinline.rs b/server/src/engine/mem/vinline.rs
index 04552860..191bad18 100644
--- a/server/src/engine/mem/vinline.rs
+++ b/server/src/engine/mem/vinline.rs
@@ -293,7 +293,7 @@ impl<T, const N: usize> Extend<T> for VInline<N, T> {
 }
 
 #[cfg(test)]
-impl<T, const N: usize> From<[T; N]> for VInline<N, T> {
+impl<T, const M: usize, const N: usize> From<[T; N]> for VInline<M, T> {
     fn from(a: [T; N]) -> Self {
         a.into_iter().collect()
     }
diff --git a/server/src/engine/storage/v1/inf.rs b/server/src/engine/storage/v1/inf/mod.rs
similarity index 80%
rename from server/src/engine/storage/v1/inf.rs
rename to server/src/engine/storage/v1/inf/mod.rs
index 84b13a54..a04a4433 100644
--- a/server/src/engine/storage/v1/inf.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -26,6 +26,9 @@
 
 //! High level interfaces
 
+#[cfg(test)]
+mod tests;
+
 use {
     crate::{
         engine::{
@@ -99,6 +102,37 @@ impl PersistDictEntryDscr {
     spec
 */
 
+pub fn enc<Obj: PersistObjectHlIO>(buf: &mut VecU8, obj: &Obj) {
+    obj.pe_obj_hlio_enc(buf)
+}
+
+pub fn enc_buf<Obj: PersistObjectHlIO>(obj: &Obj) -> Vec<u8> {
+    let mut buf = vec![];
+    enc(&mut buf, obj);
+    buf
+}
+
+pub fn dec<Obj: PersistObjectHlIO>(scanner: &mut BufferedScanner) -> SDSSResult<Obj::DType> {
+    if Obj::pe_obj_hlio_dec_ver(scanner) {
+        unsafe { Ok(Obj::pe_obj_hlio_dec(scanner).unwrap_unchecked()) }
+    } else {
+        Err(SDSSError::InternalDecodeStructureCorrupted)
+    }
+}
+
+pub fn dec_buf<Obj: PersistObjectHlIO>(buf: &[u8]) -> SDSSResult<Obj::DType> {
+    let mut scanner = BufferedScanner::new(buf);
+    dec::<Obj>(&mut scanner)
+}
+
+/// Any object that can persist
+pub trait PersistObjectHlIO: Sized {
+    type DType;
+    fn pe_obj_hlio_enc(&self, buf: &mut VecU8);
+    fn pe_obj_hlio_dec_ver(scanner: &BufferedScanner) -> bool;
+    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::DType>;
+}
+
 /// metadata spec for a persist dict
 pub trait PersistDictEntryMetadata {
     /// Verify the state of scanner to ensure that it complies with the metadata
@@ -247,7 +281,7 @@ pub fn decode_dict<Pd: PersistDict>(
 }
 
 /*
-    impls
+    dict impls
 */
 
 pub struct DGEntryMD {
@@ -438,18 +472,91 @@ impl PersistDict for DictGeneric {
     }
 }
 
-#[test]
-fn t_dict() {
-    let dict: DictGeneric = into_dict! {
-        "hello" => Datacell::new_str("world".into()),
-        "omg a null?" => Datacell::null(),
-        "a big fat dict" => DictEntryGeneric::Map(into_dict!(
-            "with a value" => Datacell::new_uint(1002),
-            "and a null" => Datacell::null(),
+/*
+    persist obj impls
+*/
+
+use crate::engine::{
+    core::model::{Field, Layer},
+    data::tag::{FullTag, TagSelector},
+    mem::VInline,
+};
+
+struct POByteBlockFullTag(FullTag);
+
+impl PersistObjectHlIO for POByteBlockFullTag {
+    type DType = FullTag;
+    fn pe_obj_hlio_enc(&self, buf: &mut VecU8) {
+        buf.extend(self.0.tag_selector().d().u64_bytes_le())
+    }
+    fn pe_obj_hlio_dec_ver(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u64))
+    }
+    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<FullTag> {
+        let dscr = scanner.next_u64_le();
+        if dscr > TagSelector::max_dscr() as u64 {
+            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+        }
+        Ok(TagSelector::from_raw(dscr as u8).into_full())
+    }
+}
+
+impl PersistObjectHlIO for Layer {
+    type DType = Layer;
+    fn pe_obj_hlio_enc(&self, buf: &mut VecU8) {
+        // [8B: type sig][8B: empty property set]
+        POByteBlockFullTag(self.tag()).pe_obj_hlio_enc(buf);
+        buf.extend(0u64.to_le_bytes());
+    }
+    fn pe_obj_hlio_dec_ver(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u64) * 2)
+    }
+    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::DType> {
+        let type_sel = scanner.next_u64_le();
+        let prop_set_arity = scanner.next_u64_le();
+        if (type_sel > TagSelector::List.d() as u64) | (prop_set_arity != 0) {
+            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+        }
+        Ok(Layer::new_empty_props(
+            TagSelector::from_raw(type_sel as u8).into_full(),
         ))
-    };
-    let encoded = encode_dict::<DictGeneric>(&dict);
-    let mut scanner = BufferedScanner::new(&encoded);
-    let decoded = decode_dict::<DictGeneric>(&mut scanner).unwrap();
-    assert_eq!(dict, decoded);
+    }
+}
+
+impl PersistObjectHlIO for Field {
+    type DType = Self;
+    fn pe_obj_hlio_enc(&self, buf: &mut VecU8) {
+        // [null][prop_c][layer_c]
+        buf.push(self.is_nullable() as u8);
+        buf.extend(0u64.to_le_bytes());
+        buf.extend(self.layers().len().u64_bytes_le());
+        for layer in self.layers() {
+            PersistObjectHlIO::pe_obj_hlio_enc(layer, buf);
+        }
+    }
+    fn pe_obj_hlio_dec_ver(scanner: &BufferedScanner) -> bool {
+        scanner.has_left((sizeof!(u64) * 2) + 1)
+    }
+    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::DType> {
+        let nullable = scanner.next_byte();
+        let prop_c = scanner.next_u64_le();
+        let layer_cnt = scanner.next_u64_le();
+        let mut layers = VInline::new();
+        let mut fin = false;
+        while (!scanner.eof())
+            & (layers.len() as u64 != layer_cnt)
+            & (Layer::pe_obj_hlio_dec_ver(scanner))
+            & !fin
+        {
+            let l = Layer::pe_obj_hlio_dec(scanner)?;
+            fin = l.tag().tag_class() != TagClass::List;
+            layers.push(l);
+        }
+        let field = Field::new(layers, nullable == 1);
+        if (field.layers().len() as u64 == layer_cnt) & (nullable <= 1) & (prop_c == 0) & fin {
+            Ok(field)
+        } else {
+            Err(SDSSError::InternalDecodeStructureCorrupted)
+        }
+    }
 }
diff --git a/server/src/engine/storage/v1/inf/tests.rs b/server/src/engine/storage/v1/inf/tests.rs
new file mode 100644
index 00000000..95d3a252
--- /dev/null
+++ b/server/src/engine/storage/v1/inf/tests.rs
@@ -0,0 +1,66 @@
+/*
+ * Created on Sun Aug 13 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    core::model::{Field, Layer},
+    data::{
+        cell::Datacell,
+        dict::{DictEntryGeneric, DictGeneric},
+    },
+    storage::v1::rw::BufferedScanner,
+};
+
+#[test]
+fn dict() {
+    let dict: DictGeneric = into_dict! {
+        "hello" => Datacell::new_str("world".into()),
+        "omg a null?" => Datacell::null(),
+        "a big fat dict" => DictEntryGeneric::Map(into_dict!(
+            "with a value" => Datacell::new_uint(1002),
+            "and a null" => Datacell::null(),
+        ))
+    };
+    let encoded = super::encode_dict::<DictGeneric>(&dict);
+    let mut scanner = BufferedScanner::new(&encoded);
+    let decoded = super::decode_dict::<DictGeneric>(&mut scanner).unwrap();
+    assert_eq!(dict, decoded);
+}
+
+#[test]
+fn layer() {
+    let layer = Layer::list();
+    let encoded = super::enc_buf(&layer);
+    let dec = super::dec_buf::<Layer>(&encoded).unwrap();
+    assert_eq!(layer, dec);
+}
+
+#[test]
+fn field() {
+    let field = Field::new([Layer::list(), Layer::uint64()].into(), true);
+    let encoded = super::enc_buf(&field);
+    let dec = super::dec_buf::<Field>(&encoded).unwrap();
+    assert_eq!(field, dec);
+}

From a619ea635c870a1167adceabffe575c3a8a69ec2 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 14 Aug 2023 14:38:02 +0000
Subject: [PATCH 217/310] Always assign UUIDs

---
 server/src/engine/core/model/mod.rs           | 10 +++-
 server/src/engine/core/space.rs               | 22 +++++++--
 server/src/engine/core/tests/ddl_model/alt.rs | 12 ++++-
 server/src/engine/core/tests/ddl_model/mod.rs | 17 +++++--
 .../src/engine/core/tests/ddl_space/alter.rs  | 16 ++++---
 .../src/engine/core/tests/ddl_space/create.rs |  4 +-
 server/src/engine/core/tests/ddl_space/mod.rs | 14 ++++--
 server/src/engine/data/mod.rs                 |  2 +
 server/src/engine/data/uuid.rs                | 47 +++++++++++++++++++
 9 files changed, 118 insertions(+), 26 deletions(-)
 create mode 100644 server/src/engine/data/uuid.rs

diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 413a8b45..0c938b1c 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -37,6 +37,7 @@ use {
         data::{
             cell::Datacell,
             tag::{DataTag, FullTag, TagClass, TagSelector},
+            uuid::Uuid,
         },
         error::{DatabaseError, DatabaseResult},
         idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
@@ -55,6 +56,7 @@ pub(in crate::engine::core) type Fields = IndexSTSeqCns<Box<str>, Field>;
 
 #[derive(Debug)]
 pub struct ModelData {
+    uuid: Uuid,
     p_key: Box<str>,
     p_tag: FullTag,
     fields: UnsafeCell<Fields>,
@@ -68,11 +70,16 @@ impl PartialEq for ModelData {
     fn eq(&self, m: &Self) -> bool {
         let mdl1 = self.intent_read_model();
         let mdl2 = m.intent_read_model();
-        self.p_key == m.p_key && self.p_tag == m.p_tag && mdl1.fields() == mdl2.fields()
+        ((self.p_key == m.p_key) & (self.p_tag == m.p_tag))
+            && self.uuid == m.uuid
+            && mdl1.fields() == mdl2.fields()
     }
 }
 
 impl ModelData {
+    pub fn get_uuid(&self) -> Uuid {
+        self.uuid
+    }
     pub fn p_key(&self) -> &str {
         &self.p_key
     }
@@ -157,6 +164,7 @@ impl ModelData {
             let tag = fields.st_get(last_pk).unwrap().layers()[0].tag;
             if tag.tag_unique().is_unique() {
                 return Ok(Self {
+                    uuid: Uuid::new(),
                     p_key: last_pk.into(),
                     p_tag: tag,
                     fields: UnsafeCell::new(fields),
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index c311780b..88736ead 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -27,7 +27,7 @@
 use {
     crate::engine::{
         core::{model::ModelData, RWLIdx},
-        data::{dict, DictEntryGeneric, MetaDict},
+        data::{dict, uuid::Uuid, DictEntryGeneric, MetaDict},
         error::{DatabaseError, DatabaseResult},
         idx::{IndexST, STIndex},
         ql::ddl::{alt::AlterSpace, crt::CreateSpace, drop::DropSpace},
@@ -38,6 +38,7 @@ use {
 #[derive(Debug)]
 /// A space with the model namespace
 pub struct Space {
+    uuid: Uuid,
     mns: RWLIdx<Box<str>, ModelData>,
     pub(super) meta: SpaceMeta,
 }
@@ -85,6 +86,9 @@ impl Space {
             Err(DatabaseError::DdlModelAlreadyExists)
         }
     }
+    pub fn get_uuid(&self) -> Uuid {
+        self.uuid
+    }
     pub(super) fn models(&self) -> &RWLIdx<Box<str>, ModelData> {
         &self.mns
     }
@@ -103,15 +107,23 @@ impl Space {
 
 impl Space {
     pub fn empty() -> Self {
-        Space::new(Default::default(), SpaceMeta::with_env(into_dict! {}))
+        Space::new_auto(Default::default(), SpaceMeta::with_env(into_dict! {}))
     }
     #[inline(always)]
-    pub fn new(mns: IndexST<Box<str>, ModelData>, meta: SpaceMeta) -> Self {
+    pub fn new_auto(mns: IndexST<Box<str>, ModelData>, meta: SpaceMeta) -> Self {
         Self {
+            uuid: Uuid::new(),
             mns: RWLIdx::new(mns),
             meta,
         }
     }
+    pub fn new_with_uuid(mns: IndexST<Box<str>, ModelData>, meta: SpaceMeta, uuid: Uuid) -> Self {
+        Self {
+            uuid,
+            meta,
+            mns: RwLock::new(mns),
+        }
+    }
     #[inline]
     /// Validate a `create` stmt
     fn process_create(
@@ -132,7 +144,7 @@ impl Space {
         };
         Ok(ProcedureCreate {
             space_name,
-            space: Self::new(
+            space: Self::new_auto(
                 IndexST::default(),
                 SpaceMeta::with_env(
                     // FIXME(@ohsayan): see this is bad. attempt to do it at AST build time
@@ -211,6 +223,6 @@ impl PartialEq for Space {
     fn eq(&self, other: &Self) -> bool {
         let self_mns = self.mns.read();
         let other_mns = other.mns.read();
-        self.meta == other.meta && *self_mns == *other_mns
+        self.meta == other.meta && *self_mns == *other_mns && self.uuid == other.uuid
     }
 }
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index adc1f6f7..de832f74 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -53,7 +53,13 @@ fn exec_plan(
     plan: &str,
     f: impl Fn(&ModelData),
 ) -> DatabaseResult<()> {
-    exec_create(gns, model, new_space)?;
+    let mdl_name = exec_create(gns, model, new_space)?;
+    let prev_uuid = {
+        let gns = gns.spaces().read();
+        let space = gns.get("myspace").unwrap();
+        let space_read = space.models().read();
+        space_read.get(mdl_name.as_str()).unwrap().get_uuid()
+    };
     let tok = lex_insecure(plan.as_bytes()).unwrap();
     let alter = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
     let (_space, model_name) = alter.model.into_full().unwrap();
@@ -61,7 +67,9 @@ fn exec_plan(
     let gns_read = gns.spaces().read();
     let space = gns_read.st_get("myspace").unwrap();
     let model = space.models().read();
-    f(model.st_get(model_name.as_str()).unwrap());
+    let model = model.st_get(model_name.as_str()).unwrap();
+    assert_eq!(prev_uuid, model.get_uuid());
+    f(model);
     Ok(())
 }
 
diff --git a/server/src/engine/core/tests/ddl_model/mod.rs b/server/src/engine/core/tests/ddl_model/mod.rs
index dfebbba0..73ae4e95 100644
--- a/server/src/engine/core/tests/ddl_model/mod.rs
+++ b/server/src/engine/core/tests/ddl_model/mod.rs
@@ -32,7 +32,11 @@ use crate::engine::{
     core::{model::ModelData, space::Space, GlobalNS},
     error::DatabaseResult,
     idx::STIndex,
-    ql::{ast::parse_ast_node_full, ddl::crt::CreateModel, tests::lex_insecure},
+    ql::{
+        ast::{parse_ast_node_full, Entity},
+        ddl::crt::CreateModel,
+        tests::lex_insecure,
+    },
 };
 
 fn create(s: &str) -> DatabaseResult<ModelData> {
@@ -45,21 +49,24 @@ pub fn exec_create(
     gns: &GlobalNS,
     create_stmt: &str,
     create_new_space: bool,
-) -> DatabaseResult<()> {
+) -> DatabaseResult<String> {
     let tok = lex_insecure(create_stmt.as_bytes()).unwrap();
     let create_model = parse_ast_node_full::<CreateModel>(&tok[2..]).unwrap();
+    let name = match create_model.model_name {
+        Entity::Single(tbl) | Entity::Full(_, tbl) => tbl.to_string(),
+    };
     if create_new_space {
         gns.test_new_empty_space(&create_model.model_name.into_full().unwrap().0);
     }
-    ModelData::exec_create(gns, create_model)
+    ModelData::exec_create(gns, create_model).map(|_| name)
 }
 
 pub fn exec_create_new_space(gns: &GlobalNS, create_stmt: &str) -> DatabaseResult<()> {
-    exec_create(gns, create_stmt, true)
+    exec_create(gns, create_stmt, true).map(|_| ())
 }
 
 pub fn exec_create_no_create(gns: &GlobalNS, create_stmt: &str) -> DatabaseResult<()> {
-    exec_create(gns, create_stmt, false)
+    exec_create(gns, create_stmt, false).map(|_| ())
 }
 
 fn with_space(gns: &GlobalNS, space_name: &str, f: impl Fn(&Space)) {
diff --git a/server/src/engine/core/tests/ddl_space/alter.rs b/server/src/engine/core/tests/ddl_space/alter.rs
index 190d4fa8..4e58e648 100644
--- a/server/src/engine/core/tests/ddl_space/alter.rs
+++ b/server/src/engine/core/tests/ddl_space/alter.rs
@@ -36,18 +36,20 @@ use crate::engine::{
 #[test]
 fn alter_add_prop_env_var() {
     let gns = GlobalNS::empty();
-    super::exec_create_empty_verify(&gns, "create space myspace");
+    let uuid = super::exec_create_empty_verify(&gns, "create space myspace").unwrap();
     super::exec_alter_and_verify(
         &gns,
         "alter space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
+            let space = space.unwrap();
             assert_eq!(
-                space.unwrap(),
-                &Space::new(
+                space,
+                &Space::new_with_uuid(
                     into_dict!(),
-                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint(100)))
+                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint(100))),
+                    uuid
                 )
-            )
+            );
         },
     )
 }
@@ -71,7 +73,7 @@ fn alter_update_prop_env_var() {
         |space| {
             assert_eq!(
                 space.unwrap(),
-                &Space::new(
+                &Space::new_auto(
                     into_dict!(),
                     SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint(200)))
                 )
@@ -99,7 +101,7 @@ fn alter_remove_prop_env_var() {
         |space| {
             assert_eq!(
                 space.unwrap(),
-                &Space::new(into_dict!(), SpaceMeta::with_env(into_dict!()))
+                &Space::new_auto(into_dict!(), SpaceMeta::with_env(into_dict!()))
             )
         },
     )
diff --git a/server/src/engine/core/tests/ddl_space/create.rs b/server/src/engine/core/tests/ddl_space/create.rs
index 707a6305..a38ea866 100644
--- a/server/src/engine/core/tests/ddl_space/create.rs
+++ b/server/src/engine/core/tests/ddl_space/create.rs
@@ -36,7 +36,7 @@ use crate::engine::{
 #[test]
 fn exec_create_space_simple() {
     let gns = GlobalNS::empty();
-    super::exec_create_empty_verify(&gns, "create space myspace");
+    super::exec_create_empty_verify(&gns, "create space myspace").unwrap();
 }
 
 #[test]
@@ -54,7 +54,7 @@ fn exec_create_space_with_env() {
         |space| {
             assert_eq!(
                 space.unwrap(),
-                &Space::new(
+                &Space::new_auto(
                     into_dict! {},
                     SpaceMeta::with_env(into_dict! {
                         "MAX_MODELS" => Datacell::new_uint(100)
diff --git a/server/src/engine/core/tests/ddl_space/mod.rs b/server/src/engine/core/tests/ddl_space/mod.rs
index 0ca2f61a..3a1139dc 100644
--- a/server/src/engine/core/tests/ddl_space/mod.rs
+++ b/server/src/engine/core/tests/ddl_space/mod.rs
@@ -40,8 +40,8 @@ use crate::engine::{
 fn exec_verify(
     gns: &GlobalNS,
     query: &str,
-    exec: impl Fn(&GlobalNS, Statement<'_>) -> (DatabaseResult<()>, Box<str>),
-    verify: impl Fn(DatabaseResult<&Space>),
+    mut exec: impl FnMut(&GlobalNS, Statement<'_>) -> (DatabaseResult<()>, Box<str>),
+    mut verify: impl FnMut(DatabaseResult<&Space>),
 ) {
     let tok = lex(query.as_bytes()).unwrap();
     let ast_node = compile_test(&tok).unwrap();
@@ -68,7 +68,7 @@ fn exec_alter_and_verify(gns: &GlobalNS, tok: &str, verify: impl Fn(DatabaseResu
 }
 
 /// Creates a space using the given tokens and allows the caller to verify it
-fn exec_create_and_verify(gns: &GlobalNS, tok: &str, verify: impl Fn(DatabaseResult<&Space>)) {
+fn exec_create_and_verify(gns: &GlobalNS, tok: &str, verify: impl FnMut(DatabaseResult<&Space>)) {
     exec_verify(
         gns,
         tok,
@@ -83,8 +83,14 @@ fn exec_create_and_verify(gns: &GlobalNS, tok: &str, verify: impl Fn(DatabaseRes
 }
 
 /// Creates an empty space with the given tokens
-fn exec_create_empty_verify(gns: &GlobalNS, tok: &str) {
+fn exec_create_empty_verify(
+    gns: &GlobalNS,
+    tok: &str,
+) -> DatabaseResult<crate::engine::data::uuid::Uuid> {
+    let mut name = None;
     self::exec_create_and_verify(gns, tok, |space| {
         assert_eq!(space.unwrap(), &Space::empty());
+        name = Some(space.unwrap().get_uuid());
     });
+    Ok(name.unwrap())
 }
diff --git a/server/src/engine/data/mod.rs b/server/src/engine/data/mod.rs
index 082c08b2..aaa35691 100644
--- a/server/src/engine/data/mod.rs
+++ b/server/src/engine/data/mod.rs
@@ -31,6 +31,8 @@ pub mod dict;
 pub mod lit;
 pub mod spec;
 pub mod tag;
+pub mod uuid;
+// test
 #[cfg(test)]
 mod tests;
 
diff --git a/server/src/engine/data/uuid.rs b/server/src/engine/data/uuid.rs
new file mode 100644
index 00000000..354f09a2
--- /dev/null
+++ b/server/src/engine/data/uuid.rs
@@ -0,0 +1,47 @@
+/*
+ * Created on Mon Aug 14 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct Uuid {
+    data: uuid::Uuid,
+}
+
+impl Uuid {
+    pub fn new() -> Self {
+        Self {
+            data: uuid::Uuid::new_v4(),
+        }
+    }
+    pub fn as_slice(&self) -> &[u8] {
+        self.data.as_bytes()
+    }
+}
+
+impl ToString for Uuid {
+    fn to_string(&self) -> String {
+        self.data.to_string()
+    }
+}

From 2937cc7fbe8fc0c264543cf04b58c875a4a473c1 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 16 Aug 2023 06:11:08 +0000
Subject: [PATCH 218/310] Simplify enc/dec spec traits

---
 server/src/engine/core/dml/ins.rs             |   4 +-
 server/src/engine/core/dml/mod.rs             |   4 +-
 server/src/engine/core/mod.rs                 |   4 +-
 server/src/engine/core/model/alt.rs           |   6 +-
 server/src/engine/core/model/delta.rs         |   8 +-
 server/src/engine/core/model/mod.rs           |  10 +-
 server/src/engine/core/space.rs               |  14 +-
 server/src/engine/core/tests/ddl_model/alt.rs |   6 +-
 server/src/engine/core/tests/ddl_model/mod.rs |  10 +-
 server/src/engine/core/tests/dml/mod.rs       |   4 +-
 server/src/engine/storage/v1/inf/mod.rs       | 342 ++++++++++--------
 server/src/engine/storage/v1/inf/tests.rs     |  15 +-
 12 files changed, 230 insertions(+), 197 deletions(-)

diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index 394800c3..bc3cd29e 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -27,7 +27,7 @@
 use crate::engine::{
     core::{
         index::{DcFieldIndex, PrimaryIndexKey},
-        model::{Fields, ModelData},
+        model::{Fields, Model},
         GlobalNS,
     },
     error::{DatabaseError, DatabaseResult},
@@ -54,7 +54,7 @@ pub fn insert(gns: &GlobalNS, insert: InsertStatement) -> DatabaseResult<()> {
 
 // TODO(@ohsayan): optimize null case
 fn prepare_insert(
-    model: &ModelData,
+    model: &Model,
     fields: &Fields,
     insert: InsertData,
 ) -> DatabaseResult<(PrimaryIndexKey, DcFieldIndex)> {
diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index ab149447..65f0f384 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -31,7 +31,7 @@ mod upd;
 
 use crate::{
     engine::{
-        core::model::ModelData,
+        core::model::Model,
         data::{lit::LitIR, spec::DataspecMeta1D, tag::DataTag},
         error::{DatabaseError, DatabaseResult},
         ql::dml::WhereClause,
@@ -43,7 +43,7 @@ use crate::{
 pub use upd::collect_trace_path as update_flow_trace;
 pub use {del::delete, ins::insert, sel::select_custom, upd::update};
 
-impl ModelData {
+impl Model {
     pub(self) fn resolve_where<'a>(
         &self,
         where_clause: &mut WhereClause<'a>,
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 2c042518..8b6f31b1 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -35,7 +35,7 @@ mod util;
 mod tests;
 // imports
 use {
-    self::{model::ModelData, util::EntityLocator},
+    self::{model::Model, util::EntityLocator},
     crate::engine::{
         core::space::Space,
         error::{DatabaseError, DatabaseResult},
@@ -82,7 +82,7 @@ impl GlobalNS {
     }
     pub fn with_model<'a, T, E, F>(&self, entity: E, f: F) -> DatabaseResult<T>
     where
-        F: FnOnce(&ModelData) -> DatabaseResult<T>,
+        F: FnOnce(&Model) -> DatabaseResult<T>,
         E: 'a + EntityLocator<'a>,
     {
         entity
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index 0136cdb6..f4d16ff9 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    super::{Field, IWModel, Layer, ModelData},
+    super::{Field, IWModel, Layer, Model},
     crate::{
         engine::{
             core::GlobalNS,
@@ -89,7 +89,7 @@ fn check_nullable(props: &mut HashMap<Box<str>, DictEntryGeneric>) -> DatabaseRe
 
 impl<'a> AlterPlan<'a> {
     pub fn fdeltas(
-        mv: &ModelData,
+        mv: &Model,
         wm: &IWModel,
         AlterModel { model, kind }: AlterModel<'a>,
     ) -> DatabaseResult<AlterPlan<'a>> {
@@ -246,7 +246,7 @@ impl<'a> AlterPlan<'a> {
     }
 }
 
-impl ModelData {
+impl Model {
     pub fn exec_alter(gns: &GlobalNS, alter: AlterModel) -> DatabaseResult<()> {
         gns.with_model(alter.model, |model| {
             // make intent
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index b86257a5..a69af383 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    super::{Fields, ModelData},
+    super::{Fields, Model},
     parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard},
     std::{
         collections::btree_map::{BTreeMap, Range},
@@ -62,7 +62,7 @@ pub struct IRModelSMData<'a> {
 }
 
 impl<'a> IRModelSMData<'a> {
-    pub fn new(m: &'a ModelData) -> Self {
+    pub fn new(m: &'a Model) -> Self {
         let rmodel = m.sync_matrix().v_priv_model_alter.read();
         let mdata = m.sync_matrix().v_priv_data_new_or_revise.read();
         Self {
@@ -86,7 +86,7 @@ pub struct IRModel<'a> {
 }
 
 impl<'a> IRModel<'a> {
-    pub fn new(m: &'a ModelData) -> Self {
+    pub fn new(m: &'a Model) -> Self {
         Self {
             rmodel: m.sync_matrix().v_priv_model_alter.read(),
             fields: unsafe {
@@ -107,7 +107,7 @@ pub struct IWModel<'a> {
 }
 
 impl<'a> IWModel<'a> {
-    pub fn new(m: &'a ModelData) -> Self {
+    pub fn new(m: &'a Model) -> Self {
         Self {
             wmodel: m.sync_matrix().v_priv_model_alter.write(),
             fields: unsafe {
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 0c938b1c..2c94798e 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -55,7 +55,7 @@ pub(in crate::engine::core) use self::delta::{DeltaKind, DeltaState, DeltaVersio
 pub(in crate::engine::core) type Fields = IndexSTSeqCns<Box<str>, Field>;
 
 #[derive(Debug)]
-pub struct ModelData {
+pub struct Model {
     uuid: Uuid,
     p_key: Box<str>,
     p_tag: FullTag,
@@ -66,7 +66,7 @@ pub struct ModelData {
 }
 
 #[cfg(test)]
-impl PartialEq for ModelData {
+impl PartialEq for Model {
     fn eq(&self, m: &Self) -> bool {
         let mdl1 = self.intent_read_model();
         let mdl2 = m.intent_read_model();
@@ -76,7 +76,7 @@ impl PartialEq for ModelData {
     }
 }
 
-impl ModelData {
+impl Model {
     pub fn get_uuid(&self) -> Uuid {
         self.uuid
     }
@@ -129,7 +129,7 @@ impl ModelData {
     }
 }
 
-impl ModelData {
+impl Model {
     pub fn process_create(
         CreateModel {
             model_name: _,
@@ -178,7 +178,7 @@ impl ModelData {
     }
 }
 
-impl ModelData {
+impl Model {
     pub fn exec_create(gns: &super::GlobalNS, stmt: CreateModel) -> DatabaseResult<()> {
         let (space_name, model_name) = stmt.model_name.parse_entity()?;
         let model = Self::process_create(stmt)?;
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 88736ead..6f251e51 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -26,7 +26,7 @@
 
 use {
     crate::engine::{
-        core::{model::ModelData, RWLIdx},
+        core::{model::Model, RWLIdx},
         data::{dict, uuid::Uuid, DictEntryGeneric, MetaDict},
         error::{DatabaseError, DatabaseResult},
         idx::{IndexST, STIndex},
@@ -39,7 +39,7 @@ use {
 /// A space with the model namespace
 pub struct Space {
     uuid: Uuid,
-    mns: RWLIdx<Box<str>, ModelData>,
+    mns: RWLIdx<Box<str>, Model>,
     pub(super) meta: SpaceMeta,
 }
 
@@ -75,7 +75,7 @@ impl ProcedureCreate {
 }
 
 impl Space {
-    pub fn _create_model(&self, name: &str, model: ModelData) -> DatabaseResult<()> {
+    pub fn _create_model(&self, name: &str, model: Model) -> DatabaseResult<()> {
         if self
             .mns
             .write()
@@ -89,13 +89,13 @@ impl Space {
     pub fn get_uuid(&self) -> Uuid {
         self.uuid
     }
-    pub(super) fn models(&self) -> &RWLIdx<Box<str>, ModelData> {
+    pub(super) fn models(&self) -> &RWLIdx<Box<str>, Model> {
         &self.mns
     }
     pub fn with_model<T>(
         &self,
         model: &str,
-        f: impl FnOnce(&ModelData) -> DatabaseResult<T>,
+        f: impl FnOnce(&Model) -> DatabaseResult<T>,
     ) -> DatabaseResult<T> {
         let mread = self.mns.read();
         let Some(model) = mread.st_get(model) else {
@@ -110,14 +110,14 @@ impl Space {
         Space::new_auto(Default::default(), SpaceMeta::with_env(into_dict! {}))
     }
     #[inline(always)]
-    pub fn new_auto(mns: IndexST<Box<str>, ModelData>, meta: SpaceMeta) -> Self {
+    pub fn new_auto(mns: IndexST<Box<str>, Model>, meta: SpaceMeta) -> Self {
         Self {
             uuid: Uuid::new(),
             mns: RWLIdx::new(mns),
             meta,
         }
     }
-    pub fn new_with_uuid(mns: IndexST<Box<str>, ModelData>, meta: SpaceMeta, uuid: Uuid) -> Self {
+    pub fn new_with_uuid(mns: IndexST<Box<str>, Model>, meta: SpaceMeta, uuid: Uuid) -> Self {
         Self {
             uuid,
             meta,
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index de832f74..53cc1656 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -26,7 +26,7 @@
 
 use crate::engine::{
     core::{
-        model::{alt::AlterPlan, ModelData},
+        model::{alt::AlterPlan, Model},
         tests::ddl_model::{create, exec_create},
         GlobalNS,
     },
@@ -51,7 +51,7 @@ fn exec_plan(
     new_space: bool,
     model: &str,
     plan: &str,
-    f: impl Fn(&ModelData),
+    f: impl Fn(&Model),
 ) -> DatabaseResult<()> {
     let mdl_name = exec_create(gns, model, new_space)?;
     let prev_uuid = {
@@ -63,7 +63,7 @@ fn exec_plan(
     let tok = lex_insecure(plan.as_bytes()).unwrap();
     let alter = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
     let (_space, model_name) = alter.model.into_full().unwrap();
-    ModelData::exec_alter(gns, alter)?;
+    Model::exec_alter(gns, alter)?;
     let gns_read = gns.spaces().read();
     let space = gns_read.st_get("myspace").unwrap();
     let model = space.models().read();
diff --git a/server/src/engine/core/tests/ddl_model/mod.rs b/server/src/engine/core/tests/ddl_model/mod.rs
index 73ae4e95..7ae3f75a 100644
--- a/server/src/engine/core/tests/ddl_model/mod.rs
+++ b/server/src/engine/core/tests/ddl_model/mod.rs
@@ -29,7 +29,7 @@ mod crt;
 mod layer;
 
 use crate::engine::{
-    core::{model::ModelData, space::Space, GlobalNS},
+    core::{model::Model, space::Space, GlobalNS},
     error::DatabaseResult,
     idx::STIndex,
     ql::{
@@ -39,10 +39,10 @@ use crate::engine::{
     },
 };
 
-fn create(s: &str) -> DatabaseResult<ModelData> {
+fn create(s: &str) -> DatabaseResult<Model> {
     let tok = lex_insecure(s.as_bytes()).unwrap();
     let create_model = parse_ast_node_full(&tok[2..]).unwrap();
-    ModelData::process_create(create_model)
+    Model::process_create(create_model)
 }
 
 pub fn exec_create(
@@ -58,7 +58,7 @@ pub fn exec_create(
     if create_new_space {
         gns.test_new_empty_space(&create_model.model_name.into_full().unwrap().0);
     }
-    ModelData::exec_create(gns, create_model).map(|_| name)
+    Model::exec_create(gns, create_model).map(|_| name)
 }
 
 pub fn exec_create_new_space(gns: &GlobalNS, create_stmt: &str) -> DatabaseResult<()> {
@@ -75,7 +75,7 @@ fn with_space(gns: &GlobalNS, space_name: &str, f: impl Fn(&Space)) {
     f(space);
 }
 
-fn with_model(gns: &GlobalNS, space_id: &str, model_name: &str, f: impl Fn(&ModelData)) {
+fn with_model(gns: &GlobalNS, space_id: &str, model_name: &str, f: impl Fn(&Model)) {
     with_space(gns, space_id, |space| {
         let space_rl = space.models().read();
         let model = space_rl.st_get(model_name).unwrap();
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index 4857c339..dc710ed9 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -30,7 +30,7 @@ mod select;
 mod update;
 
 use crate::engine::{
-    core::{dml, index::Row, model::ModelData, GlobalNS},
+    core::{dml, index::Row, model::Model, GlobalNS},
     data::{cell::Datacell, lit::LitIR},
     error::DatabaseResult,
     ql::{
@@ -47,7 +47,7 @@ fn _exec_only_create_space_model(gns: &GlobalNS, model: &str) -> DatabaseResult<
     }
     let lex_create_model = lex_insecure(model.as_bytes()).unwrap();
     let stmt_create_model = parse_ast_node_full(&lex_create_model[2..]).unwrap();
-    ModelData::exec_create(gns, stmt_create_model)
+    Model::exec_create(gns, stmt_create_model)
 }
 
 fn _exec_only_insert<T>(
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index a04a4433..a39746f2 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -34,7 +34,7 @@ use {
         engine::{
             data::{
                 cell::Datacell,
-                dict::{DictEntryGeneric, DictGeneric},
+                dict::DictEntryGeneric,
                 tag::{CUTag, DataTag, TagClass, TagUnique},
             },
             idx::{AsKey, AsValue},
@@ -42,7 +42,7 @@ use {
         },
         util::{copy_slice_to_array as memcpy, EndianQW},
     },
-    std::{cmp, collections::HashMap, mem},
+    std::{cmp, collections::HashMap, marker::PhantomData, mem},
 };
 
 type VecU8 = Vec<u8>;
@@ -102,136 +102,163 @@ impl PersistDictEntryDscr {
     spec
 */
 
-pub fn enc<Obj: PersistObjectHlIO>(buf: &mut VecU8, obj: &Obj) {
-    obj.pe_obj_hlio_enc(buf)
+/// Specification for any object that can be persisted
+pub trait PersistObjectHlIO {
+    /// the actual type (we can have wrappers)
+    type Type;
+    /// enc routine
+    fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type);
+    /// verify the src to see if we can atleast start the routine
+    fn pe_obj_hlio_dec_ver(scanner: &BufferedScanner) -> bool;
+    /// dec routine
+    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Type>;
 }
 
-pub fn enc_buf<Obj: PersistObjectHlIO>(obj: &Obj) -> Vec<u8> {
+/// enc the given object into a new buffer
+pub fn enc<Obj: PersistObjectHlIO>(obj: &Obj::Type) -> VecU8 {
     let mut buf = vec![];
-    enc(&mut buf, obj);
+    Obj::pe_obj_hlio_enc(&mut buf, obj);
     buf
 }
 
-pub fn dec<Obj: PersistObjectHlIO>(scanner: &mut BufferedScanner) -> SDSSResult<Obj::DType> {
+/// enc the object into the given buffer
+pub fn enc_into_buf<Obj: PersistObjectHlIO>(buf: &mut VecU8, obj: &Obj::Type) {
+    Obj::pe_obj_hlio_enc(buf, obj)
+}
+
+/// enc the object into the given buffer
+pub fn enc_self_into_buf<Obj: PersistObjectHlIO<Type = Obj>>(buf: &mut VecU8, obj: &Obj) {
+    Obj::pe_obj_hlio_enc(buf, obj)
+}
+
+/// enc the object into a new buffer
+pub fn enc_self<Obj: PersistObjectHlIO<Type = Obj>>(obj: &Obj) -> VecU8 {
+    enc::<Obj>(obj)
+}
+
+/// dec the object
+pub fn dec<Obj: PersistObjectHlIO>(scanner: &mut BufferedScanner) -> SDSSResult<Obj::Type> {
     if Obj::pe_obj_hlio_dec_ver(scanner) {
-        unsafe { Ok(Obj::pe_obj_hlio_dec(scanner).unwrap_unchecked()) }
+        unsafe { Obj::pe_obj_hlio_dec(scanner) }
     } else {
         Err(SDSSError::InternalDecodeStructureCorrupted)
     }
 }
 
-pub fn dec_buf<Obj: PersistObjectHlIO>(buf: &[u8]) -> SDSSResult<Obj::DType> {
-    let mut scanner = BufferedScanner::new(buf);
-    dec::<Obj>(&mut scanner)
-}
-
-/// Any object that can persist
-pub trait PersistObjectHlIO: Sized {
-    type DType;
-    fn pe_obj_hlio_enc(&self, buf: &mut VecU8);
-    fn pe_obj_hlio_dec_ver(scanner: &BufferedScanner) -> bool;
-    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::DType>;
+/// dec the object
+pub fn dec_self<Obj: PersistObjectHlIO<Type = Obj>>(
+    scanner: &mut BufferedScanner,
+) -> SDSSResult<Obj> {
+    dec::<Obj>(scanner)
 }
 
-/// metadata spec for a persist dict
-pub trait PersistDictEntryMetadata {
-    /// Verify the state of scanner to ensure that it complies with the metadata
+/// metadata spec for a persist map entry
+pub trait PersistMapEntryMD {
     fn verify_with_src(&self, scanner: &BufferedScanner) -> bool;
 }
 
-/// spec for a persist dict
-pub trait PersistDict {
-    /// type of key
+/// specification for a persist map
+pub trait PersistMapSpec {
+    /// metadata type
+    type Metadata: PersistMapEntryMD;
+    /// key type (NOTE: set this to the true key type; handle any differences using the spec unless you have an entirely different
+    /// wrapper type)
     type Key: AsKey;
-    /// type of value
+    /// value type (NOTE: see [`PersistMapSpec::Key`])
     type Value: AsValue;
-    /// metadata type
-    type Metadata: PersistDictEntryMetadata;
-    /// enc coupled (packed enc)
+    /// coupled enc
     const ENC_COUPLED: bool;
-    /// during dec, ignore failure of the metadata parse (IMP: NOT the metadata src verification but the
-    /// validity of the metadata itself) because it is handled later
-    const DEC_ENTRYMD_INFALLIBLE: bool;
-    /// dec coupled (packed dec)
+    /// coupled dec
     const DEC_COUPLED: bool;
-    /// during dec, verify the md directly with the src instead of handing it over to the dec helpers
-    const DEC_VERIFY_MD_WITH_SRC_STANDALONE: bool;
-    // meta
-    /// pretest for pre-entry stage
-    fn metadec_pretest_routine(scanner: &BufferedScanner) -> bool;
-    /// pretest for entry stage
-    fn metadec_pretest_entry(scanner: &BufferedScanner) -> bool;
-    /// enc md for an entry
-    fn enc_entry_metadata(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);
-    /// dec the entry metadata
-    /// SAFETY: Must have passed entry pretest
-    unsafe fn dec_entry_metadata(scanner: &mut BufferedScanner) -> Option<Self::Metadata>;
-    // entry (coupled)
-    /// enc a packed entry
-    fn enc_entry_coupled(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);
-    /// dec a packed entry
-    /// SAFETY: must have verified metadata with src (unless explicitly skipped with the `DEC_VERIFY_MD_WITH_SRC_STANDALONE`)
-    /// flag
-    unsafe fn dec_entry_coupled(
-        scanner: &mut BufferedScanner,
-        md: Self::Metadata,
-    ) -> Option<(Self::Key, Self::Value)>;
-    // entry (non-packed)
-    /// enc key for a normal entry
+    /// once pretests pass, the metadata dec is infallible
+    const META_INFALLIBLE_MD_PARSE: bool;
+    /// verify the src using the given metadata
+    const META_VERIFY_BEFORE_DEC: bool;
+    // collection meta
+    /// pretest before jmp to routine for entire collection
+    fn meta_dec_collection_pretest(scanner: &BufferedScanner) -> bool;
+    /// pretest before jmp to entry dec routine
+    fn meta_dec_entry_pretest(scanner: &BufferedScanner) -> bool;
+    // entry meta
+    /// enc the entry meta
+    fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);
+    /// dec the entry meta
+    /// SAFETY: ensure that all pretests have passed (we expect the caller to not be stupid)
+    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::Metadata>;
+    // independent packing
+    /// enc key (non-packed)
     fn enc_key(buf: &mut VecU8, key: &Self::Key);
-    /// dec normal entry key
-    /// SAFETY: must have verified metadata with src (unless explicitly skipped with the `DEC_VERIFY_MD_WITH_SRC_STANDALONE`)
-    /// flag
+    /// enc val (non-packed)
+    fn enc_val(buf: &mut VecU8, key: &Self::Value);
+    /// dec key (non-packed)
     unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Key>;
-    /// enc val for a normal entry
-    fn enc_val(buf: &mut VecU8, val: &Self::Value);
-    /// dec normal entry val
-    /// SAFETY: must have verified metadata with src (unless explicitly skipped with the `DEC_VERIFY_MD_WITH_SRC_STANDALONE`)
-    /// flag
+    /// dec val (non-packed)
     unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Value>;
+    // coupled packing
+    /// entry packed enc
+    fn enc_entry(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);
+    /// entry packed dec
+    unsafe fn dec_entry(
+        scanner: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> Option<(Self::Key, Self::Value)>;
 }
 
 /*
     blanket
 */
 
-pub fn encode_dict<Pd: PersistDict>(dict: &HashMap<Pd::Key, Pd::Value>) -> Vec<u8> {
-    let mut v = vec![];
-    _encode_dict::<Pd>(&mut v, dict);
-    v
+/// This is more of a lazy hack than anything sensible. Just implement a spec and then use this wrapper for any enc/dec operations
+pub struct PersistMapImpl<M: PersistMapSpec>(PhantomData<M>);
+
+impl<M: PersistMapSpec> PersistObjectHlIO for PersistMapImpl<M> {
+    type Type = HashMap<M::Key, M::Value>;
+    fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) {
+        enc_dict_into_buffer::<M>(buf, v)
+    }
+    fn pe_obj_hlio_dec_ver(_: &BufferedScanner) -> bool {
+        true // handled by the dec impl
+    }
+    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Type> {
+        dec_dict::<M>(scanner)
+    }
 }
 
-fn _encode_dict<Pd: PersistDict>(buf: &mut VecU8, dict: &HashMap<Pd::Key, Pd::Value>) {
-    buf.extend(dict.len().u64_bytes_le());
-    for (key, val) in dict {
-        Pd::enc_entry_metadata(buf, key, val);
-        if Pd::ENC_COUPLED {
-            Pd::enc_entry_coupled(buf, key, val);
+/// Encode the dict into the given buffer
+pub fn enc_dict_into_buffer<PM: PersistMapSpec>(
+    buf: &mut VecU8,
+    map: &HashMap<PM::Key, PM::Value>,
+) {
+    buf.extend(map.len().u64_bytes_le());
+    for (key, val) in map {
+        PM::entry_md_enc(buf, key, val);
+        if PM::ENC_COUPLED {
+            PM::enc_entry(buf, key, val);
         } else {
-            Pd::enc_key(buf, key);
-            Pd::enc_val(buf, val);
+            PM::enc_key(buf, key);
+            PM::enc_val(buf, val);
         }
     }
 }
 
-pub fn decode_dict<Pd: PersistDict>(
+/// Decode the dict using the given buffered scanner
+pub fn dec_dict<PM: PersistMapSpec>(
     scanner: &mut BufferedScanner,
-) -> SDSSResult<HashMap<Pd::Key, Pd::Value>> {
-    if !(Pd::metadec_pretest_routine(scanner) & scanner.has_left(sizeof!(u64))) {
+) -> SDSSResult<HashMap<PM::Key, PM::Value>> {
+    if !(PM::meta_dec_collection_pretest(scanner) & scanner.has_left(sizeof!(u64))) {
         return Err(SDSSError::InternalDecodeStructureCorrupted);
     }
-    let dict_len = unsafe {
+    let size = unsafe {
         // UNSAFE(@ohsayan): pretest
         scanner.next_u64_le() as usize
     };
-    let mut dict = HashMap::with_capacity(dict_len);
-    while Pd::metadec_pretest_entry(scanner) & (dict.len() < dict_len) {
+    let mut dict = HashMap::with_capacity(size);
+    while PM::meta_dec_entry_pretest(scanner) & (dict.len() != size) {
         let md = unsafe {
-            // UNSAFE(@ohsayan): this is compeletely because of the entry pretest
-            match Pd::dec_entry_metadata(scanner) {
-                Some(dec) => dec,
+            match PM::entry_md_dec(scanner) {
+                Some(v) => v,
                 None => {
-                    if Pd::DEC_ENTRYMD_INFALLIBLE {
+                    if PM::META_INFALLIBLE_MD_PARSE {
                         impossible!()
                     } else {
                         return Err(SDSSError::InternalDecodeStructureCorrupted);
@@ -239,41 +266,37 @@ pub fn decode_dict<Pd: PersistDict>(
                 }
             }
         };
-        if Pd::DEC_VERIFY_MD_WITH_SRC_STANDALONE && !md.verify_with_src(scanner) {
+        if PM::META_VERIFY_BEFORE_DEC && !md.verify_with_src(scanner) {
             return Err(SDSSError::InternalDecodeStructureCorrupted);
         }
-        let k;
-        let v;
-        if Pd::DEC_COUPLED {
-            match unsafe {
-                // UNSAFE(@ohsayan): verified metadata
-                Pd::dec_entry_coupled(scanner, md)
-            } {
-                Some((_k, _v)) => {
-                    k = _k;
-                    v = _v;
-                }
-                None => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
-            }
-        } else {
-            match unsafe {
-                // UNSAFE(@ohsayan): verified metadata
-                (Pd::dec_key(scanner, &md), Pd::dec_val(scanner, &md))
-            } {
-                (Some(_k), Some(_v)) => {
-                    k = _k;
-                    v = _v;
+        let key;
+        let val;
+        unsafe {
+            if PM::DEC_COUPLED {
+                match PM::dec_entry(scanner, md) {
+                    Some((_k, _v)) => {
+                        key = _k;
+                        val = _v;
+                    }
+                    None => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
                 }
-                _ => {
-                    return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+            } else {
+                let _k = PM::dec_key(scanner, &md);
+                let _v = PM::dec_val(scanner, &md);
+                match (_k, _v) {
+                    (Some(_k), Some(_v)) => {
+                        key = _k;
+                        val = _v;
+                    }
+                    _ => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
                 }
             }
         }
-        if dict.insert(k, v).is_some() {
+        if dict.insert(key, val).is_some() {
             return Err(SDSSError::InternalDecodeStructureIllegalData);
         }
     }
-    if dict.len() == dict_len {
+    if dict.len() == size {
         Ok(dict)
     } else {
         Err(SDSSError::InternalDecodeStructureIllegalData)
@@ -281,21 +304,27 @@ pub fn decode_dict<Pd: PersistDict>(
 }
 
 /*
-    dict impls
+    impls
 */
 
-pub struct DGEntryMD {
-    klen: usize,
+/// generic dict spec (simple spec for [DictGeneric](crate::engine::data::dict::DictGeneric))
+pub struct GenericDictSpec;
+/// generic dict entry metadata
+pub struct GenericDictEntryMD {
     dscr: u8,
+    klen: usize,
 }
 
-impl DGEntryMD {
+impl GenericDictEntryMD {
+    /// decode md (no need for any validation since that has to be handled later and can only produce incorrect results
+    /// if unsafe code is used to translate an incorrect dscr)
     fn decode(data: [u8; 9]) -> Self {
         Self {
             klen: u64::from_le_bytes(memcpy(&data[..8])) as usize,
             dscr: data[8],
         }
     }
+    /// encode md
     fn encode(klen: usize, dscr: u8) -> [u8; 9] {
         let mut ret = [0u8; 9];
         ret[..8].copy_from_slice(&klen.u64_bytes_le());
@@ -304,7 +333,7 @@ impl DGEntryMD {
     }
 }
 
-impl PersistDictEntryMetadata for DGEntryMD {
+impl PersistMapEntryMD for GenericDictEntryMD {
     fn verify_with_src(&self, scanner: &BufferedScanner) -> bool {
         static EXPECT_ATLEAST: [u8; 4] = [0, 1, 8, 8]; // PAD to align
         let lbound_rem = self.klen + EXPECT_ATLEAST[cmp::min(self.dscr, 3) as usize] as usize;
@@ -312,32 +341,32 @@ impl PersistDictEntryMetadata for DGEntryMD {
     }
 }
 
-impl PersistDict for DictGeneric {
+impl PersistMapSpec for GenericDictSpec {
     type Key = Box<str>;
     type Value = DictEntryGeneric;
-    type Metadata = DGEntryMD;
-    const ENC_COUPLED: bool = true;
-    const DEC_ENTRYMD_INFALLIBLE: bool = true;
+    type Metadata = GenericDictEntryMD;
     const DEC_COUPLED: bool = false;
-    const DEC_VERIFY_MD_WITH_SRC_STANDALONE: bool = true;
-    fn metadec_pretest_routine(_: &BufferedScanner) -> bool {
+    const ENC_COUPLED: bool = true;
+    const META_INFALLIBLE_MD_PARSE: bool = true;
+    const META_VERIFY_BEFORE_DEC: bool = true;
+    fn meta_dec_collection_pretest(_: &BufferedScanner) -> bool {
         true
     }
-    fn metadec_pretest_entry(scanner: &BufferedScanner) -> bool {
+    fn meta_dec_entry_pretest(scanner: &BufferedScanner) -> bool {
         scanner.has_left(sizeof!(u64) + 1)
     }
-    fn enc_entry_metadata(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) {
+    fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) {
         buf.extend(key.len().u64_bytes_le());
     }
-    unsafe fn dec_entry_metadata(scanner: &mut BufferedScanner) -> Option<Self::Metadata> {
+    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::Metadata> {
         Some(Self::Metadata::decode(scanner.next_chunk()))
     }
-    fn enc_entry_coupled(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
+    fn enc_entry(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
         match val {
             DictEntryGeneric::Map(map) => {
                 buf.push(PersistDictEntryDscr::Dict.value_u8());
                 buf.extend(key.as_bytes());
-                _encode_dict::<Self>(buf, map);
+                enc_dict_into_buffer::<Self>(buf, map);
             }
             DictEntryGeneric::Lit(dc) => {
                 buf.push(
@@ -373,23 +402,11 @@ impl PersistDict for DictGeneric {
             }
         }
     }
-    unsafe fn dec_entry_coupled(
-        _: &mut BufferedScanner,
-        _: Self::Metadata,
-    ) -> Option<(Self::Key, Self::Value)> {
-        unimplemented!()
-    }
-    fn enc_key(_: &mut VecU8, _: &Self::Key) {
-        unimplemented!()
-    }
     unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Key> {
         String::from_utf8(scanner.next_chunk_variable(md.klen).to_owned())
             .map(|s| s.into_boxed_str())
             .ok()
     }
-    fn enc_val(_: &mut VecU8, _: &Self::Value) {
-        unimplemented!()
-    }
     unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Value> {
         unsafe fn decode_element(
             scanner: &mut BufferedScanner,
@@ -460,7 +477,7 @@ impl PersistDict for DictGeneric {
                 }
                 PersistDictEntryDscr::Dict => {
                     if dg_top_element {
-                        DictEntryGeneric::Map(decode_dict::<DictGeneric>(scanner).ok()?)
+                        DictEntryGeneric::Map(dec_dict::<GenericDictSpec>(scanner).ok()?)
                     } else {
                         unreachable!("found top-level dict item in datacell")
                     }
@@ -470,6 +487,19 @@ impl PersistDict for DictGeneric {
         }
         decode_element(scanner, PersistDictEntryDscr::from_raw(md.dscr), true)
     }
+    // not implemented
+    fn enc_key(_: &mut VecU8, _: &Self::Key) {
+        unimplemented!()
+    }
+    fn enc_val(_: &mut VecU8, _: &Self::Value) {
+        unimplemented!()
+    }
+    unsafe fn dec_entry(
+        _: &mut BufferedScanner,
+        _: Self::Metadata,
+    ) -> Option<(Self::Key, Self::Value)> {
+        unimplemented!()
+    }
 }
 
 /*
@@ -485,9 +515,9 @@ use crate::engine::{
 struct POByteBlockFullTag(FullTag);
 
 impl PersistObjectHlIO for POByteBlockFullTag {
-    type DType = FullTag;
-    fn pe_obj_hlio_enc(&self, buf: &mut VecU8) {
-        buf.extend(self.0.tag_selector().d().u64_bytes_le())
+    type Type = FullTag;
+    fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
+        buf.extend(slf.tag_selector().d().u64_bytes_le())
     }
     fn pe_obj_hlio_dec_ver(scanner: &BufferedScanner) -> bool {
         scanner.has_left(sizeof!(u64))
@@ -502,16 +532,16 @@ impl PersistObjectHlIO for POByteBlockFullTag {
 }
 
 impl PersistObjectHlIO for Layer {
-    type DType = Layer;
-    fn pe_obj_hlio_enc(&self, buf: &mut VecU8) {
+    type Type = Layer;
+    fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
         // [8B: type sig][8B: empty property set]
-        POByteBlockFullTag(self.tag()).pe_obj_hlio_enc(buf);
+        POByteBlockFullTag::pe_obj_hlio_enc(buf, &slf.tag());
         buf.extend(0u64.to_le_bytes());
     }
     fn pe_obj_hlio_dec_ver(scanner: &BufferedScanner) -> bool {
         scanner.has_left(sizeof!(u64) * 2)
     }
-    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::DType> {
+    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Type> {
         let type_sel = scanner.next_u64_le();
         let prop_set_arity = scanner.next_u64_le();
         if (type_sel > TagSelector::List.d() as u64) | (prop_set_arity != 0) {
@@ -524,20 +554,20 @@ impl PersistObjectHlIO for Layer {
 }
 
 impl PersistObjectHlIO for Field {
-    type DType = Self;
-    fn pe_obj_hlio_enc(&self, buf: &mut VecU8) {
+    type Type = Self;
+    fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
         // [null][prop_c][layer_c]
-        buf.push(self.is_nullable() as u8);
+        buf.push(slf.is_nullable() as u8);
         buf.extend(0u64.to_le_bytes());
-        buf.extend(self.layers().len().u64_bytes_le());
-        for layer in self.layers() {
-            PersistObjectHlIO::pe_obj_hlio_enc(layer, buf);
+        buf.extend(slf.layers().len().u64_bytes_le());
+        for layer in slf.layers() {
+            Layer::pe_obj_hlio_enc(buf, layer);
         }
     }
     fn pe_obj_hlio_dec_ver(scanner: &BufferedScanner) -> bool {
         scanner.has_left((sizeof!(u64) * 2) + 1)
     }
-    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::DType> {
+    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Type> {
         let nullable = scanner.next_byte();
         let prop_c = scanner.next_u64_le();
         let layer_cnt = scanner.next_u64_le();
diff --git a/server/src/engine/storage/v1/inf/tests.rs b/server/src/engine/storage/v1/inf/tests.rs
index 95d3a252..47fa8091 100644
--- a/server/src/engine/storage/v1/inf/tests.rs
+++ b/server/src/engine/storage/v1/inf/tests.rs
@@ -43,24 +43,27 @@ fn dict() {
             "and a null" => Datacell::null(),
         ))
     };
-    let encoded = super::encode_dict::<DictGeneric>(&dict);
+    let encoded = super::enc::<super::PersistMapImpl<super::GenericDictSpec>>(&dict);
     let mut scanner = BufferedScanner::new(&encoded);
-    let decoded = super::decode_dict::<DictGeneric>(&mut scanner).unwrap();
+    let decoded =
+        super::dec::<super::PersistMapImpl<super::GenericDictSpec>>(&mut scanner).unwrap();
     assert_eq!(dict, decoded);
 }
 
 #[test]
 fn layer() {
     let layer = Layer::list();
-    let encoded = super::enc_buf(&layer);
-    let dec = super::dec_buf::<Layer>(&encoded).unwrap();
+    let encoded = super::enc_self(&layer);
+    let mut scanner = BufferedScanner::new(&encoded);
+    let dec = super::dec_self::<Layer>(&mut scanner).unwrap();
     assert_eq!(layer, dec);
 }
 
 #[test]
 fn field() {
     let field = Field::new([Layer::list(), Layer::uint64()].into(), true);
-    let encoded = super::enc_buf(&field);
-    let dec = super::dec_buf::<Field>(&encoded).unwrap();
+    let encoded = super::enc_self(&field);
+    let mut scanner = BufferedScanner::new(&encoded);
+    let dec = super::dec_self::<Field>(&mut scanner).unwrap();
     assert_eq!(field, dec);
 }

From 369abe9a2226c6da3b870d5de3316cb1498d16d7 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 16 Aug 2023 09:34:49 +0000
Subject: [PATCH 219/310] Allow persist objects to have custom metadata

---
 server/src/engine/storage/v1/inf/map.rs   | 334 ++++++++++++++
 server/src/engine/storage/v1/inf/mod.rs   | 509 +++++-----------------
 server/src/engine/storage/v1/inf/obj.rs   | 184 ++++++++
 server/src/engine/storage/v1/inf/tests.rs |   5 +-
 server/src/engine/storage/v1/rw.rs        |   3 +
 5 files changed, 622 insertions(+), 413 deletions(-)
 create mode 100644 server/src/engine/storage/v1/inf/map.rs
 create mode 100644 server/src/engine/storage/v1/inf/obj.rs

diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
new file mode 100644
index 00000000..aaeec366
--- /dev/null
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -0,0 +1,334 @@
+/*
+ * Created on Wed Aug 16 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{
+        dec_md, PersistDictEntryDscr, PersistMapSpec, PersistObjectHlIO, PersistObjectMD, VecU8,
+        VoidMetadata,
+    },
+    crate::{
+        engine::{
+            data::{
+                cell::Datacell,
+                dict::DictEntryGeneric,
+                tag::{CUTag, DataTag, TagClass, TagUnique},
+            },
+            storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult},
+        },
+        util::{copy_slice_to_array as memcpy, EndianQW},
+    },
+    core::marker::PhantomData,
+    std::{cmp, collections::HashMap},
+};
+
+/// This is more of a lazy hack than anything sensible. Just implement a spec and then use this wrapper for any enc/dec operations
+pub struct PersistMapImpl<M: PersistMapSpec>(PhantomData<M>);
+
+impl<M: PersistMapSpec> PersistObjectHlIO for PersistMapImpl<M> {
+    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
+    type Type = HashMap<M::Key, M::Value>;
+    type Metadata = VoidMetadata;
+    fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) {
+        enc_dict_into_buffer::<M>(buf, v)
+    }
+    unsafe fn pe_obj_hlio_dec(
+        scanner: &mut BufferedScanner,
+        _: VoidMetadata,
+    ) -> SDSSResult<Self::Type> {
+        dec_dict::<M>(scanner)
+    }
+}
+
+/// Encode the dict into the given buffer
+pub fn enc_dict_into_buffer<PM: PersistMapSpec>(
+    buf: &mut VecU8,
+    map: &HashMap<PM::Key, PM::Value>,
+) {
+    buf.extend(map.len().u64_bytes_le());
+    for (key, val) in map {
+        PM::entry_md_enc(buf, key, val);
+        if PM::ENC_COUPLED {
+            PM::enc_entry(buf, key, val);
+        } else {
+            PM::enc_key(buf, key);
+            PM::enc_val(buf, val);
+        }
+    }
+}
+
+/// Decode the dict using the given buffered scanner
+pub fn dec_dict<PM: PersistMapSpec>(
+    scanner: &mut BufferedScanner,
+) -> SDSSResult<HashMap<PM::Key, PM::Value>> {
+    if !(PM::meta_dec_collection_pretest(scanner) & scanner.has_left(sizeof!(u64))) {
+        return Err(SDSSError::InternalDecodeStructureCorrupted);
+    }
+    let size = unsafe {
+        // UNSAFE(@ohsayan): pretest
+        scanner.next_u64_le() as usize
+    };
+    let mut dict = HashMap::with_capacity(size);
+    while PM::meta_dec_entry_pretest(scanner) & (dict.len() != size) {
+        let md = dec_md::<PM::Metadata>(scanner)?;
+        if PM::META_VERIFY_BEFORE_DEC && !md.pretest_src_for_object_dec(scanner) {
+            return Err(SDSSError::InternalDecodeStructureCorrupted);
+        }
+        let key;
+        let val;
+        unsafe {
+            if PM::DEC_COUPLED {
+                match PM::dec_entry(scanner, md) {
+                    Some((_k, _v)) => {
+                        key = _k;
+                        val = _v;
+                    }
+                    None => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
+                }
+            } else {
+                let _k = PM::dec_key(scanner, &md);
+                let _v = PM::dec_val(scanner, &md);
+                match (_k, _v) {
+                    (Some(_k), Some(_v)) => {
+                        key = _k;
+                        val = _v;
+                    }
+                    _ => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
+                }
+            }
+        }
+        if dict.insert(key, val).is_some() {
+            return Err(SDSSError::InternalDecodeStructureIllegalData);
+        }
+    }
+    if dict.len() == size {
+        Ok(dict)
+    } else {
+        Err(SDSSError::InternalDecodeStructureIllegalData)
+    }
+}
+
+/// generic dict spec (simple spec for [DictGeneric](crate::engine::data::dict::DictGeneric))
+pub struct GenericDictSpec;
+
+/// generic dict entry metadata
+pub struct GenericDictEntryMD {
+    pub(crate) dscr: u8,
+    pub(crate) klen: usize,
+}
+
+impl GenericDictEntryMD {
+    /// decode md (no need for any validation since that has to be handled later and can only produce incorrect results
+    /// if unsafe code is used to translate an incorrect dscr)
+    pub(crate) fn decode(data: [u8; 9]) -> Self {
+        Self {
+            klen: u64::from_le_bytes(memcpy(&data[..8])) as usize,
+            dscr: data[8],
+        }
+    }
+    /// encode md
+    pub(crate) fn encode(klen: usize, dscr: u8) -> [u8; 9] {
+        let mut ret = [0u8; 9];
+        ret[..8].copy_from_slice(&klen.u64_bytes_le());
+        ret[8] = dscr;
+        ret
+    }
+}
+
+impl PersistObjectMD for GenericDictEntryMD {
+    const MD_DEC_INFALLIBLE: bool = true;
+    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
+        scanner.has_left((sizeof!(u64) * 2) + 1)
+    }
+    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
+        Some(Self::decode(scanner.next_chunk()))
+    }
+    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool {
+        static EXPECT_ATLEAST: [u8; 4] = [0, 1, 8, 8]; // PAD to align
+        let lbound_rem = self.klen + EXPECT_ATLEAST[cmp::min(self.dscr, 3) as usize] as usize;
+        scanner.has_left(lbound_rem) & (self.dscr <= PersistDictEntryDscr::Dict.value_u8())
+    }
+}
+
+impl PersistMapSpec for GenericDictSpec {
+    type Key = Box<str>;
+    type Value = DictEntryGeneric;
+    type Metadata = GenericDictEntryMD;
+    const DEC_COUPLED: bool = false;
+    const ENC_COUPLED: bool = true;
+    const META_VERIFY_BEFORE_DEC: bool = true;
+    fn meta_dec_collection_pretest(_: &BufferedScanner) -> bool {
+        true
+    }
+    fn meta_dec_entry_pretest(scanner: &BufferedScanner) -> bool {
+        // we just need to see if we can decode the entry metadata
+        Self::Metadata::pretest_src_for_metadata_dec(scanner)
+    }
+    fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) {
+        buf.extend(key.len().u64_bytes_le());
+    }
+    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::Metadata> {
+        Some(Self::Metadata::decode(scanner.next_chunk()))
+    }
+    fn enc_entry(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
+        match val {
+            DictEntryGeneric::Map(map) => {
+                buf.push(PersistDictEntryDscr::Dict.value_u8());
+                buf.extend(key.as_bytes());
+                enc_dict_into_buffer::<Self>(buf, map);
+            }
+            DictEntryGeneric::Lit(dc) => {
+                buf.push(
+                    PersistDictEntryDscr::translate_from_class(dc.tag().tag_class()).value_u8()
+                        * (!dc.is_null() as u8),
+                );
+                buf.extend(key.as_bytes());
+                fn encode_element(buf: &mut VecU8, dc: &Datacell) {
+                    unsafe {
+                        use TagClass::*;
+                        match dc.tag().tag_class() {
+                            Bool if dc.is_init() => buf.push(dc.read_bool() as u8),
+                            Bool => {}
+                            UnsignedInt | SignedInt | Float => {
+                                buf.extend(dc.read_uint().to_le_bytes())
+                            }
+                            Str | Bin => {
+                                let slc = dc.read_bin();
+                                buf.extend(slc.len().u64_bytes_le());
+                                buf.extend(slc);
+                            }
+                            List => {
+                                let lst = dc.read_list().read();
+                                buf.extend(lst.len().u64_bytes_le());
+                                for item in lst.iter() {
+                                    encode_element(buf, item);
+                                }
+                            }
+                        }
+                    }
+                }
+                encode_element(buf, dc);
+            }
+        }
+    }
+    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Key> {
+        String::from_utf8(scanner.next_chunk_variable(md.klen).to_owned())
+            .map(|s| s.into_boxed_str())
+            .ok()
+    }
+    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Value> {
+        unsafe fn decode_element(
+            scanner: &mut BufferedScanner,
+            dscr: PersistDictEntryDscr,
+            dg_top_element: bool,
+        ) -> Option<DictEntryGeneric> {
+            let r = match dscr {
+                PersistDictEntryDscr::Null => DictEntryGeneric::Lit(Datacell::null()),
+                PersistDictEntryDscr::Bool => {
+                    DictEntryGeneric::Lit(Datacell::new_bool(scanner.next_byte() == 1))
+                }
+                PersistDictEntryDscr::UnsignedInt
+                | PersistDictEntryDscr::SignedInt
+                | PersistDictEntryDscr::Float => DictEntryGeneric::Lit(Datacell::new_qw(
+                    scanner.next_u64_le(),
+                    CUTag::new(
+                        dscr.into_class(),
+                        [
+                            TagUnique::UnsignedInt,
+                            TagUnique::SignedInt,
+                            TagUnique::Illegal,
+                            TagUnique::Illegal, // pad
+                        ][(dscr.value_u8() - 2) as usize],
+                    ),
+                )),
+                PersistDictEntryDscr::Str | PersistDictEntryDscr::Bin => {
+                    let slc_len = scanner.next_u64_le() as usize;
+                    if !scanner.has_left(slc_len) {
+                        return None;
+                    }
+                    let slc = scanner.next_chunk_variable(slc_len);
+                    DictEntryGeneric::Lit(if dscr == PersistDictEntryDscr::Str {
+                        if core::str::from_utf8(slc).is_err() {
+                            return None;
+                        }
+                        Datacell::new_str(
+                            String::from_utf8_unchecked(slc.to_owned()).into_boxed_str(),
+                        )
+                    } else {
+                        Datacell::new_bin(slc.to_owned().into_boxed_slice())
+                    })
+                }
+                PersistDictEntryDscr::List => {
+                    let list_len = scanner.next_u64_le() as usize;
+                    let mut v = Vec::with_capacity(list_len);
+                    while (!scanner.eof()) & (v.len() < list_len) {
+                        let dscr = scanner.next_byte();
+                        if dscr > PersistDictEntryDscr::Dict.value_u8() {
+                            return None;
+                        }
+                        v.push(
+                            match decode_element(
+                                scanner,
+                                PersistDictEntryDscr::from_raw(dscr),
+                                false,
+                            ) {
+                                Some(DictEntryGeneric::Lit(l)) => l,
+                                None => return None,
+                                _ => unreachable!("found top-level dict item in datacell"),
+                            },
+                        );
+                    }
+                    if v.len() == list_len {
+                        DictEntryGeneric::Lit(Datacell::new_list(v))
+                    } else {
+                        return None;
+                    }
+                }
+                PersistDictEntryDscr::Dict => {
+                    if dg_top_element {
+                        DictEntryGeneric::Map(dec_dict::<GenericDictSpec>(scanner).ok()?)
+                    } else {
+                        unreachable!("found top-level dict item in datacell")
+                    }
+                }
+            };
+            Some(r)
+        }
+        decode_element(scanner, PersistDictEntryDscr::from_raw(md.dscr), true)
+    }
+    // not implemented
+    fn enc_key(_: &mut VecU8, _: &Self::Key) {
+        unimplemented!()
+    }
+    fn enc_val(_: &mut VecU8, _: &Self::Value) {
+        unimplemented!()
+    }
+    unsafe fn dec_entry(
+        _: &mut BufferedScanner,
+        _: Self::Metadata,
+    ) -> Option<(Self::Key, Self::Value)> {
+        unimplemented!()
+    }
+}
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index a39746f2..dfb5345a 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -26,23 +26,22 @@
 
 //! High level interfaces
 
+mod map;
+mod obj;
+// tests
 #[cfg(test)]
 mod tests;
 
 use {
-    crate::{
-        engine::{
-            data::{
-                cell::Datacell,
-                dict::DictEntryGeneric,
-                tag::{CUTag, DataTag, TagClass, TagUnique},
-            },
-            idx::{AsKey, AsValue},
-            storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult},
+    crate::engine::{
+        data::{
+            dict::DictEntryGeneric,
+            tag::{DataTag, TagClass},
         },
-        util::{copy_slice_to_array as memcpy, EndianQW},
+        idx::{AsKey, AsValue},
+        storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult},
     },
-    std::{cmp, collections::HashMap, marker::PhantomData, mem},
+    std::mem,
 };
 
 type VecU8 = Vec<u8>;
@@ -99,19 +98,93 @@ impl PersistDictEntryDscr {
 }
 
 /*
-    spec
+    md spec
+*/
+
+/// metadata spec for a persist map entry
+pub trait PersistObjectMD: Sized {
+    /// set to true if decode is infallible once the MD payload has been verified
+    const MD_DEC_INFALLIBLE: bool;
+    /// returns true if the current buffered source can be used to decode the metadata (self)
+    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool;
+    /// returns true if per the metadata and the current buffered source, the target object in question can be decoded
+    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool;
+    /// decode the metadata
+    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self>;
+}
+
+pub struct SimpleSizeMD<const N: usize>;
+
+impl<const N: usize> PersistObjectMD for SimpleSizeMD<N> {
+    const MD_DEC_INFALLIBLE: bool = true;
+    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(N)
+    }
+    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
+        true
+    }
+    unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
+        Some(Self)
+    }
+}
+
+/// For wrappers and other complicated metadata handling, set this to the metadata type
+pub struct VoidMetadata;
+
+impl PersistObjectMD for VoidMetadata {
+    const MD_DEC_INFALLIBLE: bool = true;
+    fn pretest_src_for_metadata_dec(_: &BufferedScanner) -> bool {
+        true
+    }
+    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
+        true
+    }
+    unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
+        Some(Self)
+    }
+}
+
+fn dec_md<Md: PersistObjectMD>(scanner: &mut BufferedScanner) -> SDSSResult<Md> {
+    if Md::pretest_src_for_metadata_dec(scanner) {
+        unsafe {
+            match Md::dec_md_payload(scanner) {
+                Some(md) => Ok(md),
+                None => {
+                    if Md::MD_DEC_INFALLIBLE {
+                        impossible!()
+                    } else {
+                        Err(SDSSError::InternalDecodeStructureCorrupted)
+                    }
+                }
+            }
+        }
+    } else {
+        Err(SDSSError::InternalDecodeStructureCorrupted)
+    }
+}
+
+/*
+    obj spec
 */
 
 /// Specification for any object that can be persisted
+///
+/// To actuall enc/dec any object, use functions (and their derivatives) [`enc`] and [`dec`]
 pub trait PersistObjectHlIO {
+    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool;
     /// the actual type (we can have wrappers)
     type Type;
+    /// the metadata type (use this to verify the buffered source)
+    type Metadata: PersistObjectMD;
     /// enc routine
+    ///
+    /// METADATA: handle yourself
     fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type);
-    /// verify the src to see if we can atleast start the routine
-    fn pe_obj_hlio_dec_ver(scanner: &BufferedScanner) -> bool;
     /// dec routine
-    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Type>;
+    unsafe fn pe_obj_hlio_dec(
+        scanner: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> SDSSResult<Self::Type>;
 }
 
 /// enc the given object into a new buffer
@@ -138,8 +211,12 @@ pub fn enc_self<Obj: PersistObjectHlIO<Type = Obj>>(obj: &Obj) -> VecU8 {
 
 /// dec the object
 pub fn dec<Obj: PersistObjectHlIO>(scanner: &mut BufferedScanner) -> SDSSResult<Obj::Type> {
-    if Obj::pe_obj_hlio_dec_ver(scanner) {
-        unsafe { Obj::pe_obj_hlio_dec(scanner) }
+    if Obj::Metadata::pretest_src_for_metadata_dec(scanner) {
+        let md = dec_md::<Obj::Metadata>(scanner)?;
+        if Obj::ALWAYS_VERIFY_PAYLOAD_USING_MD && !md.pretest_src_for_object_dec(scanner) {
+            return Err(SDSSError::InternalDecodeStructureCorrupted);
+        }
+        unsafe { Obj::pe_obj_hlio_dec(scanner, md) }
     } else {
         Err(SDSSError::InternalDecodeStructureCorrupted)
     }
@@ -152,15 +229,14 @@ pub fn dec_self<Obj: PersistObjectHlIO<Type = Obj>>(
     dec::<Obj>(scanner)
 }
 
-/// metadata spec for a persist map entry
-pub trait PersistMapEntryMD {
-    fn verify_with_src(&self, scanner: &BufferedScanner) -> bool;
-}
+/*
+    map spec
+*/
 
 /// specification for a persist map
 pub trait PersistMapSpec {
     /// metadata type
-    type Metadata: PersistMapEntryMD;
+    type Metadata: PersistObjectMD;
     /// key type (NOTE: set this to the true key type; handle any differences using the spec unless you have an entirely different
     /// wrapper type)
     type Key: AsKey;
@@ -170,8 +246,6 @@ pub trait PersistMapSpec {
     const ENC_COUPLED: bool;
     /// coupled dec
     const DEC_COUPLED: bool;
-    /// once pretests pass, the metadata dec is infallible
-    const META_INFALLIBLE_MD_PARSE: bool;
     /// verify the src using the given metadata
     const META_VERIFY_BEFORE_DEC: bool;
     // collection meta
@@ -203,390 +277,3 @@ pub trait PersistMapSpec {
         md: Self::Metadata,
     ) -> Option<(Self::Key, Self::Value)>;
 }
-
-/*
-    blanket
-*/
-
-/// This is more of a lazy hack than anything sensible. Just implement a spec and then use this wrapper for any enc/dec operations
-pub struct PersistMapImpl<M: PersistMapSpec>(PhantomData<M>);
-
-impl<M: PersistMapSpec> PersistObjectHlIO for PersistMapImpl<M> {
-    type Type = HashMap<M::Key, M::Value>;
-    fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) {
-        enc_dict_into_buffer::<M>(buf, v)
-    }
-    fn pe_obj_hlio_dec_ver(_: &BufferedScanner) -> bool {
-        true // handled by the dec impl
-    }
-    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Type> {
-        dec_dict::<M>(scanner)
-    }
-}
-
-/// Encode the dict into the given buffer
-pub fn enc_dict_into_buffer<PM: PersistMapSpec>(
-    buf: &mut VecU8,
-    map: &HashMap<PM::Key, PM::Value>,
-) {
-    buf.extend(map.len().u64_bytes_le());
-    for (key, val) in map {
-        PM::entry_md_enc(buf, key, val);
-        if PM::ENC_COUPLED {
-            PM::enc_entry(buf, key, val);
-        } else {
-            PM::enc_key(buf, key);
-            PM::enc_val(buf, val);
-        }
-    }
-}
-
-/// Decode the dict using the given buffered scanner
-pub fn dec_dict<PM: PersistMapSpec>(
-    scanner: &mut BufferedScanner,
-) -> SDSSResult<HashMap<PM::Key, PM::Value>> {
-    if !(PM::meta_dec_collection_pretest(scanner) & scanner.has_left(sizeof!(u64))) {
-        return Err(SDSSError::InternalDecodeStructureCorrupted);
-    }
-    let size = unsafe {
-        // UNSAFE(@ohsayan): pretest
-        scanner.next_u64_le() as usize
-    };
-    let mut dict = HashMap::with_capacity(size);
-    while PM::meta_dec_entry_pretest(scanner) & (dict.len() != size) {
-        let md = unsafe {
-            match PM::entry_md_dec(scanner) {
-                Some(v) => v,
-                None => {
-                    if PM::META_INFALLIBLE_MD_PARSE {
-                        impossible!()
-                    } else {
-                        return Err(SDSSError::InternalDecodeStructureCorrupted);
-                    }
-                }
-            }
-        };
-        if PM::META_VERIFY_BEFORE_DEC && !md.verify_with_src(scanner) {
-            return Err(SDSSError::InternalDecodeStructureCorrupted);
-        }
-        let key;
-        let val;
-        unsafe {
-            if PM::DEC_COUPLED {
-                match PM::dec_entry(scanner, md) {
-                    Some((_k, _v)) => {
-                        key = _k;
-                        val = _v;
-                    }
-                    None => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
-                }
-            } else {
-                let _k = PM::dec_key(scanner, &md);
-                let _v = PM::dec_val(scanner, &md);
-                match (_k, _v) {
-                    (Some(_k), Some(_v)) => {
-                        key = _k;
-                        val = _v;
-                    }
-                    _ => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
-                }
-            }
-        }
-        if dict.insert(key, val).is_some() {
-            return Err(SDSSError::InternalDecodeStructureIllegalData);
-        }
-    }
-    if dict.len() == size {
-        Ok(dict)
-    } else {
-        Err(SDSSError::InternalDecodeStructureIllegalData)
-    }
-}
-
-/*
-    impls
-*/
-
-/// generic dict spec (simple spec for [DictGeneric](crate::engine::data::dict::DictGeneric))
-pub struct GenericDictSpec;
-/// generic dict entry metadata
-pub struct GenericDictEntryMD {
-    dscr: u8,
-    klen: usize,
-}
-
-impl GenericDictEntryMD {
-    /// decode md (no need for any validation since that has to be handled later and can only produce incorrect results
-    /// if unsafe code is used to translate an incorrect dscr)
-    fn decode(data: [u8; 9]) -> Self {
-        Self {
-            klen: u64::from_le_bytes(memcpy(&data[..8])) as usize,
-            dscr: data[8],
-        }
-    }
-    /// encode md
-    fn encode(klen: usize, dscr: u8) -> [u8; 9] {
-        let mut ret = [0u8; 9];
-        ret[..8].copy_from_slice(&klen.u64_bytes_le());
-        ret[8] = dscr;
-        ret
-    }
-}
-
-impl PersistMapEntryMD for GenericDictEntryMD {
-    fn verify_with_src(&self, scanner: &BufferedScanner) -> bool {
-        static EXPECT_ATLEAST: [u8; 4] = [0, 1, 8, 8]; // PAD to align
-        let lbound_rem = self.klen + EXPECT_ATLEAST[cmp::min(self.dscr, 3) as usize] as usize;
-        scanner.has_left(lbound_rem) & (self.dscr <= PersistDictEntryDscr::Dict.value_u8())
-    }
-}
-
-impl PersistMapSpec for GenericDictSpec {
-    type Key = Box<str>;
-    type Value = DictEntryGeneric;
-    type Metadata = GenericDictEntryMD;
-    const DEC_COUPLED: bool = false;
-    const ENC_COUPLED: bool = true;
-    const META_INFALLIBLE_MD_PARSE: bool = true;
-    const META_VERIFY_BEFORE_DEC: bool = true;
-    fn meta_dec_collection_pretest(_: &BufferedScanner) -> bool {
-        true
-    }
-    fn meta_dec_entry_pretest(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u64) + 1)
-    }
-    fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) {
-        buf.extend(key.len().u64_bytes_le());
-    }
-    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::Metadata> {
-        Some(Self::Metadata::decode(scanner.next_chunk()))
-    }
-    fn enc_entry(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
-        match val {
-            DictEntryGeneric::Map(map) => {
-                buf.push(PersistDictEntryDscr::Dict.value_u8());
-                buf.extend(key.as_bytes());
-                enc_dict_into_buffer::<Self>(buf, map);
-            }
-            DictEntryGeneric::Lit(dc) => {
-                buf.push(
-                    PersistDictEntryDscr::translate_from_class(dc.tag().tag_class()).value_u8()
-                        * (!dc.is_null() as u8),
-                );
-                buf.extend(key.as_bytes());
-                fn encode_element(buf: &mut VecU8, dc: &Datacell) {
-                    unsafe {
-                        use TagClass::*;
-                        match dc.tag().tag_class() {
-                            Bool if dc.is_init() => buf.push(dc.read_bool() as u8),
-                            Bool => {}
-                            UnsignedInt | SignedInt | Float => {
-                                buf.extend(dc.read_uint().to_le_bytes())
-                            }
-                            Str | Bin => {
-                                let slc = dc.read_bin();
-                                buf.extend(slc.len().u64_bytes_le());
-                                buf.extend(slc);
-                            }
-                            List => {
-                                let lst = dc.read_list().read();
-                                buf.extend(lst.len().u64_bytes_le());
-                                for item in lst.iter() {
-                                    encode_element(buf, item);
-                                }
-                            }
-                        }
-                    }
-                }
-                encode_element(buf, dc);
-            }
-        }
-    }
-    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Key> {
-        String::from_utf8(scanner.next_chunk_variable(md.klen).to_owned())
-            .map(|s| s.into_boxed_str())
-            .ok()
-    }
-    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Value> {
-        unsafe fn decode_element(
-            scanner: &mut BufferedScanner,
-            dscr: PersistDictEntryDscr,
-            dg_top_element: bool,
-        ) -> Option<DictEntryGeneric> {
-            let r = match dscr {
-                PersistDictEntryDscr::Null => DictEntryGeneric::Lit(Datacell::null()),
-                PersistDictEntryDscr::Bool => {
-                    DictEntryGeneric::Lit(Datacell::new_bool(scanner.next_byte() == 1))
-                }
-                PersistDictEntryDscr::UnsignedInt
-                | PersistDictEntryDscr::SignedInt
-                | PersistDictEntryDscr::Float => DictEntryGeneric::Lit(Datacell::new_qw(
-                    scanner.next_u64_le(),
-                    CUTag::new(
-                        dscr.into_class(),
-                        [
-                            TagUnique::UnsignedInt,
-                            TagUnique::SignedInt,
-                            TagUnique::Illegal,
-                            TagUnique::Illegal, // pad
-                        ][(dscr.value_u8() - 2) as usize],
-                    ),
-                )),
-                PersistDictEntryDscr::Str | PersistDictEntryDscr::Bin => {
-                    let slc_len = scanner.next_u64_le() as usize;
-                    if !scanner.has_left(slc_len) {
-                        return None;
-                    }
-                    let slc = scanner.next_chunk_variable(slc_len);
-                    DictEntryGeneric::Lit(if dscr == PersistDictEntryDscr::Str {
-                        if core::str::from_utf8(slc).is_err() {
-                            return None;
-                        }
-                        Datacell::new_str(
-                            String::from_utf8_unchecked(slc.to_owned()).into_boxed_str(),
-                        )
-                    } else {
-                        Datacell::new_bin(slc.to_owned().into_boxed_slice())
-                    })
-                }
-                PersistDictEntryDscr::List => {
-                    let list_len = scanner.next_u64_le() as usize;
-                    let mut v = Vec::with_capacity(list_len);
-                    while (!scanner.eof()) & (v.len() < list_len) {
-                        let dscr = scanner.next_byte();
-                        if dscr > PersistDictEntryDscr::Dict.value_u8() {
-                            return None;
-                        }
-                        v.push(
-                            match decode_element(
-                                scanner,
-                                PersistDictEntryDscr::from_raw(dscr),
-                                false,
-                            ) {
-                                Some(DictEntryGeneric::Lit(l)) => l,
-                                None => return None,
-                                _ => unreachable!("found top-level dict item in datacell"),
-                            },
-                        );
-                    }
-                    if v.len() == list_len {
-                        DictEntryGeneric::Lit(Datacell::new_list(v))
-                    } else {
-                        return None;
-                    }
-                }
-                PersistDictEntryDscr::Dict => {
-                    if dg_top_element {
-                        DictEntryGeneric::Map(dec_dict::<GenericDictSpec>(scanner).ok()?)
-                    } else {
-                        unreachable!("found top-level dict item in datacell")
-                    }
-                }
-            };
-            Some(r)
-        }
-        decode_element(scanner, PersistDictEntryDscr::from_raw(md.dscr), true)
-    }
-    // not implemented
-    fn enc_key(_: &mut VecU8, _: &Self::Key) {
-        unimplemented!()
-    }
-    fn enc_val(_: &mut VecU8, _: &Self::Value) {
-        unimplemented!()
-    }
-    unsafe fn dec_entry(
-        _: &mut BufferedScanner,
-        _: Self::Metadata,
-    ) -> Option<(Self::Key, Self::Value)> {
-        unimplemented!()
-    }
-}
-
-/*
-    persist obj impls
-*/
-
-use crate::engine::{
-    core::model::{Field, Layer},
-    data::tag::{FullTag, TagSelector},
-    mem::VInline,
-};
-
-struct POByteBlockFullTag(FullTag);
-
-impl PersistObjectHlIO for POByteBlockFullTag {
-    type Type = FullTag;
-    fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
-        buf.extend(slf.tag_selector().d().u64_bytes_le())
-    }
-    fn pe_obj_hlio_dec_ver(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u64))
-    }
-    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<FullTag> {
-        let dscr = scanner.next_u64_le();
-        if dscr > TagSelector::max_dscr() as u64 {
-            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
-        }
-        Ok(TagSelector::from_raw(dscr as u8).into_full())
-    }
-}
-
-impl PersistObjectHlIO for Layer {
-    type Type = Layer;
-    fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
-        // [8B: type sig][8B: empty property set]
-        POByteBlockFullTag::pe_obj_hlio_enc(buf, &slf.tag());
-        buf.extend(0u64.to_le_bytes());
-    }
-    fn pe_obj_hlio_dec_ver(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u64) * 2)
-    }
-    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Type> {
-        let type_sel = scanner.next_u64_le();
-        let prop_set_arity = scanner.next_u64_le();
-        if (type_sel > TagSelector::List.d() as u64) | (prop_set_arity != 0) {
-            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
-        }
-        Ok(Layer::new_empty_props(
-            TagSelector::from_raw(type_sel as u8).into_full(),
-        ))
-    }
-}
-
-impl PersistObjectHlIO for Field {
-    type Type = Self;
-    fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
-        // [null][prop_c][layer_c]
-        buf.push(slf.is_nullable() as u8);
-        buf.extend(0u64.to_le_bytes());
-        buf.extend(slf.layers().len().u64_bytes_le());
-        for layer in slf.layers() {
-            Layer::pe_obj_hlio_enc(buf, layer);
-        }
-    }
-    fn pe_obj_hlio_dec_ver(scanner: &BufferedScanner) -> bool {
-        scanner.has_left((sizeof!(u64) * 2) + 1)
-    }
-    unsafe fn pe_obj_hlio_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Type> {
-        let nullable = scanner.next_byte();
-        let prop_c = scanner.next_u64_le();
-        let layer_cnt = scanner.next_u64_le();
-        let mut layers = VInline::new();
-        let mut fin = false;
-        while (!scanner.eof())
-            & (layers.len() as u64 != layer_cnt)
-            & (Layer::pe_obj_hlio_dec_ver(scanner))
-            & !fin
-        {
-            let l = Layer::pe_obj_hlio_dec(scanner)?;
-            fin = l.tag().tag_class() != TagClass::List;
-            layers.push(l);
-        }
-        let field = Field::new(layers, nullable == 1);
-        if (field.layers().len() as u64 == layer_cnt) & (nullable <= 1) & (prop_c == 0) & fin {
-            Ok(field)
-        } else {
-            Err(SDSSError::InternalDecodeStructureCorrupted)
-        }
-    }
-}
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
new file mode 100644
index 00000000..70376626
--- /dev/null
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -0,0 +1,184 @@
+/*
+ * Created on Wed Aug 16 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{dec_md, PersistObjectHlIO, PersistObjectMD, SimpleSizeMD, VecU8},
+    crate::{
+        engine::{
+            core::model::{Field, Layer},
+            data::tag::{DataTag, FullTag, TagClass, TagSelector},
+            mem::VInline,
+            storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult},
+        },
+        util::EndianQW,
+    },
+};
+
+struct POByteBlockFullTag(FullTag);
+
+impl PersistObjectHlIO for POByteBlockFullTag {
+    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
+    type Type = FullTag;
+    type Metadata = SimpleSizeMD<{ sizeof!(u64) }>;
+    fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
+        buf.extend(slf.tag_selector().d().u64_bytes_le())
+    }
+    unsafe fn pe_obj_hlio_dec(
+        scanner: &mut BufferedScanner,
+        _: Self::Metadata,
+    ) -> SDSSResult<FullTag> {
+        let dscr = scanner.next_u64_le();
+        if dscr > TagSelector::max_dscr() as u64 {
+            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+        }
+        Ok(TagSelector::from_raw(dscr as u8).into_full())
+    }
+}
+
+#[derive(Debug)]
+pub struct LayerMD {
+    type_selector: u64,
+    prop_set_arity: u64,
+}
+
+impl LayerMD {
+    const fn new(type_selector: u64, prop_set_arity: u64) -> Self {
+        Self {
+            type_selector,
+            prop_set_arity,
+        }
+    }
+}
+
+impl PersistObjectMD for LayerMD {
+    const MD_DEC_INFALLIBLE: bool = true;
+    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u64) * 2)
+    }
+    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
+        true
+    }
+    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
+        Some(Self::new(
+            u64::from_le_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()),
+        ))
+    }
+}
+
+impl PersistObjectHlIO for Layer {
+    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
+    type Type = Layer;
+    type Metadata = LayerMD;
+    fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
+        // [8B: type sig][8B: empty property set]
+        POByteBlockFullTag::pe_obj_hlio_enc(buf, &slf.tag());
+        buf.extend(0u64.to_le_bytes());
+    }
+    unsafe fn pe_obj_hlio_dec(
+        _: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> SDSSResult<Self::Type> {
+        if (md.type_selector > TagSelector::List.d() as u64) | (md.prop_set_arity != 0) {
+            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+        }
+        Ok(Layer::new_empty_props(
+            TagSelector::from_raw(md.type_selector as u8).into_full(),
+        ))
+    }
+}
+
+pub struct FieldMD {
+    prop_c: u64,
+    layer_c: u64,
+    null: u8,
+}
+
+impl FieldMD {
+    const fn new(prop_c: u64, layer_c: u64, null: u8) -> Self {
+        Self {
+            prop_c,
+            layer_c,
+            null,
+        }
+    }
+}
+
+impl PersistObjectMD for FieldMD {
+    const MD_DEC_INFALLIBLE: bool = true;
+    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
+        scanner.has_left((sizeof!(u64) * 2) + 1)
+    }
+    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
+        // nothing here really; we can't help much with the stuff ahead
+        true
+    }
+    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
+        Some(Self::new(
+            u64::from_le_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()),
+            scanner.next_byte(),
+        ))
+    }
+}
+
+impl PersistObjectHlIO for Field {
+    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
+    type Type = Self;
+    type Metadata = FieldMD;
+    fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
+        // [prop_c][layer_c][null]
+        buf.extend(0u64.to_le_bytes());
+        buf.extend(slf.layers().len().u64_bytes_le());
+        buf.push(slf.is_nullable() as u8);
+        for layer in slf.layers() {
+            Layer::pe_obj_hlio_enc(buf, layer);
+        }
+    }
+    unsafe fn pe_obj_hlio_dec(
+        scanner: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> SDSSResult<Self::Type> {
+        let mut layers = VInline::new();
+        let mut fin = false;
+        while (!scanner.eof())
+            & (layers.len() as u64 != md.layer_c)
+            & (<Layer as PersistObjectHlIO>::Metadata::pretest_src_for_metadata_dec(scanner))
+            & !fin
+        {
+            let layer_md = dec_md(scanner)?;
+            let l = Layer::pe_obj_hlio_dec(scanner, layer_md)?;
+            fin = l.tag().tag_class() != TagClass::List;
+            layers.push(l);
+        }
+        let field = Field::new(layers, md.null == 1);
+        if (field.layers().len() as u64 == md.layer_c) & (md.null <= 1) & (md.prop_c == 0) & fin {
+            Ok(field)
+        } else {
+            Err(SDSSError::InternalDecodeStructureCorrupted)
+        }
+    }
+}
diff --git a/server/src/engine/storage/v1/inf/tests.rs b/server/src/engine/storage/v1/inf/tests.rs
index 47fa8091..5c0b5665 100644
--- a/server/src/engine/storage/v1/inf/tests.rs
+++ b/server/src/engine/storage/v1/inf/tests.rs
@@ -43,10 +43,11 @@ fn dict() {
             "and a null" => Datacell::null(),
         ))
     };
-    let encoded = super::enc::<super::PersistMapImpl<super::GenericDictSpec>>(&dict);
+    let encoded = super::enc::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(&dict);
     let mut scanner = BufferedScanner::new(&encoded);
     let decoded =
-        super::dec::<super::PersistMapImpl<super::GenericDictSpec>>(&mut scanner).unwrap();
+        super::dec::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(&mut scanner)
+            .unwrap();
     assert_eq!(dict, decoded);
 }
 
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 5e015411..1d2c6f01 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -222,6 +222,9 @@ impl<'a> BufferedScanner<'a> {
     unsafe fn _incr(&mut self, by: usize) {
         self.i += by;
     }
+    pub fn current(&self) -> &[u8] {
+        &self.d[self.i..]
+    }
 }
 
 impl<'a> BufferedScanner<'a> {

From 550a39ad99e3cc01c98b0034186ce907be6fa0c7 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 16 Aug 2023 14:22:32 +0000
Subject: [PATCH 220/310] Fix space DDL tests

---
 server/src/engine/core/space.rs               |  3 +
 .../src/engine/core/tests/ddl_space/alter.rs  | 65 ++++++++------
 .../src/engine/core/tests/ddl_space/create.rs | 36 ++++----
 server/src/engine/core/tests/ddl_space/mod.rs | 84 +++++++------------
 server/src/engine/macros.rs                   |  1 +
 server/src/engine/storage/v1/inf/map.rs       |  5 +-
 server/src/engine/storage/v1/inf/mod.rs       | 32 ++++---
 server/src/engine/storage/v1/inf/obj.rs       |  5 +-
 8 files changed, 123 insertions(+), 108 deletions(-)

diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 6f251e51..2fc8b741 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -109,6 +109,9 @@ impl Space {
     pub fn empty() -> Self {
         Space::new_auto(Default::default(), SpaceMeta::with_env(into_dict! {}))
     }
+    pub fn empty_with_uuid(uuid: Uuid) -> Self {
+        Space::new_with_uuid(Default::default(), SpaceMeta::with_env(into_dict!()), uuid)
+    }
     #[inline(always)]
     pub fn new_auto(mns: IndexST<Box<str>, Model>, meta: SpaceMeta) -> Self {
         Self {
diff --git a/server/src/engine/core/tests/ddl_space/alter.rs b/server/src/engine/core/tests/ddl_space/alter.rs
index 4e58e648..65cb7e44 100644
--- a/server/src/engine/core/tests/ddl_space/alter.rs
+++ b/server/src/engine/core/tests/ddl_space/alter.rs
@@ -36,101 +36,112 @@ use crate::engine::{
 #[test]
 fn alter_add_prop_env_var() {
     let gns = GlobalNS::empty();
-    let uuid = super::exec_create_empty_verify(&gns, "create space myspace").unwrap();
-    super::exec_alter_and_verify(
+    super::exec_create_alter(
         &gns,
+        "create space myspace",
         "alter space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
-            let space = space.unwrap();
             assert_eq!(
                 space,
                 &Space::new_with_uuid(
                     into_dict!(),
                     SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint(100))),
-                    uuid
+                    space.get_uuid()
                 )
             );
         },
     )
+    .unwrap();
 }
 
 #[test]
 fn alter_update_prop_env_var() {
     let gns = GlobalNS::empty();
-    super::exec_create_and_verify(
+    let uuid = super::exec_create(
         &gns,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
             assert_eq!(
-                space.unwrap().meta.env.read().get("MY_NEW_PROP").unwrap(),
+                space.meta.env.read().get("MY_NEW_PROP").unwrap(),
                 &(Datacell::new_uint(100).into())
             )
         },
-    );
-    super::exec_alter_and_verify(
+    )
+    .unwrap();
+    super::exec_alter(
         &gns,
         "alter space myspace with { env: { MY_NEW_PROP: 200 } }",
         |space| {
             assert_eq!(
-                space.unwrap(),
-                &Space::new_auto(
+                space,
+                &Space::new_with_uuid(
                     into_dict!(),
-                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint(200)))
+                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint(200))),
+                    uuid,
                 )
             )
         },
     )
+    .unwrap();
 }
 
 #[test]
 fn alter_remove_prop_env_var() {
     let gns = GlobalNS::empty();
-    super::exec_create_and_verify(
+    let uuid = super::exec_create(
         &gns,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
             assert_eq!(
-                space.unwrap().meta.env.read().get("MY_NEW_PROP").unwrap(),
+                space.meta.env.read().get("MY_NEW_PROP").unwrap(),
                 &(Datacell::new_uint(100).into())
             )
         },
-    );
-    super::exec_alter_and_verify(
+    )
+    .unwrap();
+    super::exec_alter(
         &gns,
         "alter space myspace with { env: { MY_NEW_PROP: null } }",
         |space| {
             assert_eq!(
-                space.unwrap(),
-                &Space::new_auto(into_dict!(), SpaceMeta::with_env(into_dict!()))
+                space,
+                &Space::new_with_uuid(into_dict!(), SpaceMeta::with_env(into_dict!()), uuid)
             )
         },
     )
+    .unwrap();
 }
 
 #[test]
 fn alter_nx() {
     let gns = GlobalNS::empty();
-    super::exec_alter_and_verify(
-        &gns,
-        "alter space myspace with { env: { MY_NEW_PROP: 100 } }",
-        |space| assert_eq!(space.unwrap_err(), DatabaseError::DdlSpaceNotFound),
-    )
+    assert_eq!(
+        super::exec_alter(
+            &gns,
+            "alter space myspace with { env: { MY_NEW_PROP: 100 } }",
+            |_| {},
+        )
+        .unwrap_err(),
+        DatabaseError::DdlSpaceNotFound
+    );
 }
 
 #[test]
 fn alter_remove_all_env() {
     let gns = GlobalNS::empty();
-    super::exec_create_and_verify(
+    let uuid = super::exec_create(
         &gns,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
             assert_eq!(
-                space.unwrap().meta.env.read().get("MY_NEW_PROP").unwrap(),
+                space.meta.env.read().get("MY_NEW_PROP").unwrap(),
                 &(Datacell::new_uint(100).into())
             )
         },
-    );
-    super::exec_alter_and_verify(&gns, "alter space myspace with { env: null }", |space| {
-        assert_eq!(space.unwrap(), &Space::empty())
+    )
+    .unwrap();
+    super::exec_alter(&gns, "alter space myspace with { env: null }", |space| {
+        assert_eq!(space, &Space::empty_with_uuid(uuid))
     })
+    .unwrap();
 }
diff --git a/server/src/engine/core/tests/ddl_space/create.rs b/server/src/engine/core/tests/ddl_space/create.rs
index a38ea866..e66f4926 100644
--- a/server/src/engine/core/tests/ddl_space/create.rs
+++ b/server/src/engine/core/tests/ddl_space/create.rs
@@ -36,13 +36,16 @@ use crate::engine::{
 #[test]
 fn exec_create_space_simple() {
     let gns = GlobalNS::empty();
-    super::exec_create_empty_verify(&gns, "create space myspace").unwrap();
+    super::exec_create(&gns, "create space myspace", |spc| {
+        assert!(spc.models().read().is_empty())
+    })
+    .unwrap();
 }
 
 #[test]
 fn exec_create_space_with_env() {
     let gns = GlobalNS::empty();
-    super::exec_create_and_verify(
+    super::exec_create(
         &gns,
         r#"
         create space myspace with {
@@ -53,34 +56,39 @@ fn exec_create_space_with_env() {
     "#,
         |space| {
             assert_eq!(
-                space.unwrap(),
-                &Space::new_auto(
+                space,
+                &Space::new_with_uuid(
                     into_dict! {},
                     SpaceMeta::with_env(into_dict! {
                         "MAX_MODELS" => Datacell::new_uint(100)
-                    })
+                    }),
+                    space.get_uuid()
                 )
             );
         },
     )
+    .unwrap();
 }
 
 #[test]
 fn exec_create_space_with_bad_env_type() {
     let gns = GlobalNS::empty();
-    super::exec_create_and_verify(&gns, "create space myspace with { env: 100 }", |space| {
-        assert_eq!(space.unwrap_err(), DatabaseError::DdlSpaceBadProperty);
-    });
+    assert_eq!(
+        super::exec_create(&gns, "create space myspace with { env: 100 }", |_| {}).unwrap_err(),
+        DatabaseError::DdlSpaceBadProperty
+    );
 }
 
 #[test]
 fn exec_create_space_with_random_property() {
     let gns = GlobalNS::empty();
-    super::exec_create_and_verify(
-        &gns,
-        "create space myspace with { i_am_blue_da_ba_dee: 100 }",
-        |space| {
-            assert_eq!(space.unwrap_err(), DatabaseError::DdlSpaceBadProperty);
-        },
+    assert_eq!(
+        super::exec_create(
+            &gns,
+            "create space myspace with { i_am_blue_da_ba_dee: 100 }",
+            |_| {}
+        )
+        .unwrap_err(),
+        DatabaseError::DdlSpaceBadProperty
     );
 }
diff --git a/server/src/engine/core/tests/ddl_space/mod.rs b/server/src/engine/core/tests/ddl_space/mod.rs
index 3a1139dc..b33babf1 100644
--- a/server/src/engine/core/tests/ddl_space/mod.rs
+++ b/server/src/engine/core/tests/ddl_space/mod.rs
@@ -29,68 +29,46 @@ mod create;
 
 use crate::engine::{
     core::{space::Space, GlobalNS},
+    data::uuid::Uuid,
     error::DatabaseResult,
-    idx::STIndex,
     ql::{
-        ast::{compile_test, Statement},
+        ast::{self},
         tests::lex_insecure as lex,
     },
 };
 
-fn exec_verify(
-    gns: &GlobalNS,
-    query: &str,
-    mut exec: impl FnMut(&GlobalNS, Statement<'_>) -> (DatabaseResult<()>, Box<str>),
-    mut verify: impl FnMut(DatabaseResult<&Space>),
-) {
-    let tok = lex(query.as_bytes()).unwrap();
-    let ast_node = compile_test(&tok).unwrap();
-    let (res, space_name) = exec(gns, ast_node);
-    let rl = gns.spaces().read();
-    let space_ref = rl.st_get(&space_name);
-    let r = res.map(|_| space_ref.unwrap());
-    verify(r);
-}
-
-/// Creates a space using the given tokens and allows the caller to verify it
-fn exec_alter_and_verify(gns: &GlobalNS, tok: &str, verify: impl Fn(DatabaseResult<&Space>)) {
-    exec_verify(
-        gns,
-        tok,
-        |gns, stmt| {
-            let space = extract_safe!(stmt, Statement::AlterSpace(s) => s);
-            let space_name = space.space_name;
-            let r = Space::exec_alter(&gns, space);
-            (r, space_name.boxed_str())
-        },
-        verify,
-    );
+fn exec_create(gns: &GlobalNS, create: &str, verify: impl Fn(&Space)) -> DatabaseResult<Uuid> {
+    let tok = lex(create.as_bytes()).unwrap();
+    let ast_node =
+        ast::parse_ast_node_full::<crate::engine::ql::ddl::crt::CreateSpace>(&tok[2..]).unwrap();
+    let name = ast_node.space_name;
+    Space::exec_create(gns, ast_node)?;
+    gns.with_space(&name, |space| {
+        verify(space);
+        Ok(space.get_uuid())
+    })
 }
 
-/// Creates a space using the given tokens and allows the caller to verify it
-fn exec_create_and_verify(gns: &GlobalNS, tok: &str, verify: impl FnMut(DatabaseResult<&Space>)) {
-    exec_verify(
-        gns,
-        tok,
-        |gns, stmt| {
-            let space = extract_safe!(stmt, Statement::CreateSpace(s) => s);
-            let space_name = space.space_name;
-            let r = Space::exec_create(&gns, space);
-            (r, space_name.boxed_str())
-        },
-        verify,
-    );
+fn exec_alter(gns: &GlobalNS, alter: &str, verify: impl Fn(&Space)) -> DatabaseResult<Uuid> {
+    let tok = lex(alter.as_bytes()).unwrap();
+    let ast_node =
+        ast::parse_ast_node_full::<crate::engine::ql::ddl::alt::AlterSpace>(&tok[2..]).unwrap();
+    let name = ast_node.space_name;
+    Space::exec_alter(gns, ast_node)?;
+    gns.with_space(&name, |space| {
+        verify(space);
+        Ok(space.get_uuid())
+    })
 }
 
-/// Creates an empty space with the given tokens
-fn exec_create_empty_verify(
+fn exec_create_alter(
     gns: &GlobalNS,
-    tok: &str,
-) -> DatabaseResult<crate::engine::data::uuid::Uuid> {
-    let mut name = None;
-    self::exec_create_and_verify(gns, tok, |space| {
-        assert_eq!(space.unwrap(), &Space::empty());
-        name = Some(space.unwrap().get_uuid());
-    });
-    Ok(name.unwrap())
+    crt: &str,
+    alt: &str,
+    verify_post_alt: impl Fn(&Space),
+) -> DatabaseResult<Uuid> {
+    let uuid_crt = exec_create(gns, crt, |_| {})?;
+    let uuid_alt = exec_alter(gns, alt, verify_post_alt)?;
+    assert_eq!(uuid_crt, uuid_alt);
+    Ok(uuid_alt)
 }
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index 13ba99ab..bb9ce76d 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -35,6 +35,7 @@ macro_rules! extract {
 }
 
 #[cfg(test)]
+#[allow(unused_macros)]
 macro_rules! extract_safe {
     ($src:expr, $what:pat => $ret:expr) => {
         if let $what = $src {
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index aaeec366..f90b33f2 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -92,7 +92,10 @@ pub fn dec_dict<PM: PersistMapSpec>(
     };
     let mut dict = HashMap::with_capacity(size);
     while PM::meta_dec_entry_pretest(scanner) & (dict.len() != size) {
-        let md = dec_md::<PM::Metadata>(scanner)?;
+        let md = unsafe {
+            // pretest
+            dec_md::<PM::Metadata, true>(scanner)?
+        };
         if PM::META_VERIFY_BEFORE_DEC && !md.pretest_src_for_object_dec(scanner) {
             return Err(SDSSError::InternalDecodeStructureCorrupted);
         }
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index dfb5345a..38e8448b 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -113,6 +113,7 @@ pub trait PersistObjectMD: Sized {
     unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self>;
 }
 
+/// Metadata for a simple size requirement
 pub struct SimpleSizeMD<const N: usize>;
 
 impl<const N: usize> PersistObjectMD for SimpleSizeMD<N> {
@@ -144,17 +145,21 @@ impl PersistObjectMD for VoidMetadata {
     }
 }
 
-fn dec_md<Md: PersistObjectMD>(scanner: &mut BufferedScanner) -> SDSSResult<Md> {
-    if Md::pretest_src_for_metadata_dec(scanner) {
-        unsafe {
-            match Md::dec_md_payload(scanner) {
-                Some(md) => Ok(md),
-                None => {
-                    if Md::MD_DEC_INFALLIBLE {
-                        impossible!()
-                    } else {
-                        Err(SDSSError::InternalDecodeStructureCorrupted)
-                    }
+/// Decode metadata
+///
+/// ## Safety
+/// unsafe because you need to set whether you've already verified the metadata or not
+unsafe fn dec_md<Md: PersistObjectMD, const ASSUME_PRETEST_PASS: bool>(
+    scanner: &mut BufferedScanner,
+) -> SDSSResult<Md> {
+    if ASSUME_PRETEST_PASS || Md::pretest_src_for_metadata_dec(scanner) {
+        match Md::dec_md_payload(scanner) {
+            Some(md) => Ok(md),
+            None => {
+                if Md::MD_DEC_INFALLIBLE {
+                    impossible!()
+                } else {
+                    Err(SDSSError::InternalDecodeStructureCorrupted)
                 }
             }
         }
@@ -212,7 +217,10 @@ pub fn enc_self<Obj: PersistObjectHlIO<Type = Obj>>(obj: &Obj) -> VecU8 {
 /// dec the object
 pub fn dec<Obj: PersistObjectHlIO>(scanner: &mut BufferedScanner) -> SDSSResult<Obj::Type> {
     if Obj::Metadata::pretest_src_for_metadata_dec(scanner) {
-        let md = dec_md::<Obj::Metadata>(scanner)?;
+        let md = unsafe {
+            // UNSAFE(@ohsaya): pretest
+            dec_md::<Obj::Metadata, true>(scanner)?
+        };
         if Obj::ALWAYS_VERIFY_PAYLOAD_USING_MD && !md.pretest_src_for_object_dec(scanner) {
             return Err(SDSSError::InternalDecodeStructureCorrupted);
         }
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 70376626..99689840 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -169,7 +169,10 @@ impl PersistObjectHlIO for Field {
             & (<Layer as PersistObjectHlIO>::Metadata::pretest_src_for_metadata_dec(scanner))
             & !fin
         {
-            let layer_md = dec_md(scanner)?;
+            let layer_md = unsafe {
+                // UNSAFE(@ohsayan): pretest
+                dec_md::<_, true>(scanner)?
+            };
             let l = Layer::pe_obj_hlio_dec(scanner, layer_md)?;
             fin = l.tag().tag_class() != TagClass::List;
             layers.push(l);

From 6e3f26ddbd84fd26146c9ad8b956d9f26ab3fb67 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 16 Aug 2023 16:21:40 +0000
Subject: [PATCH 221/310] Make enc/dec for dicts more generic

---
 server/src/engine/idx/mod.rs            |  2 +
 server/src/engine/idx/stdhm.rs          |  4 ++
 server/src/engine/idx/stord/mod.rs      |  4 ++
 server/src/engine/storage/v1/inf/map.rs | 54 +++++++++++++------------
 server/src/engine/storage/v1/inf/mod.rs | 13 +++---
 server/src/engine/storage/v1/inf/obj.rs | 19 ++++++++-
 6 files changed, 64 insertions(+), 32 deletions(-)

diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 211bcecf..013d6080 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -225,6 +225,8 @@ pub trait STIndex<K: ?Sized, V>: IndexBaseSpec {
     where
         Self: 'a,
         V: 'a;
+    /// returns the length of the idx
+    fn st_len(&self) -> usize;
     /// Attempts to compact the backing storage
     fn st_compact(&mut self) {}
     /// Clears all the entries in the STIndex
diff --git a/server/src/engine/idx/stdhm.rs b/server/src/engine/idx/stdhm.rs
index 4161d03b..0820ada7 100644
--- a/server/src/engine/idx/stdhm.rs
+++ b/server/src/engine/idx/stdhm.rs
@@ -95,6 +95,10 @@ where
         self.shrink_to_fit()
     }
 
+    fn st_len(&self) -> usize {
+        self.len()
+    }
+
     fn st_clear(&mut self) {
         self.clear()
     }
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index 67ae4a93..f54d02f2 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -585,6 +585,10 @@ where
         self.vacuum_full();
     }
 
+    fn st_len(&self) -> usize {
+        self.len()
+    }
+
     fn st_clear(&mut self) {
         self._clear()
     }
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index f90b33f2..c853fdad 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -35,21 +35,26 @@ use {
                 cell::Datacell,
                 dict::DictEntryGeneric,
                 tag::{CUTag, DataTag, TagClass, TagUnique},
+                DictGeneric,
             },
+            idx::{IndexBaseSpec, STIndex},
             storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult},
         },
         util::{copy_slice_to_array as memcpy, EndianQW},
     },
     core::marker::PhantomData,
-    std::{cmp, collections::HashMap},
+    std::cmp,
 };
 
 /// This is more of a lazy hack than anything sensible. Just implement a spec and then use this wrapper for any enc/dec operations
 pub struct PersistMapImpl<M: PersistMapSpec>(PhantomData<M>);
 
-impl<M: PersistMapSpec> PersistObjectHlIO for PersistMapImpl<M> {
+impl<M: PersistMapSpec> PersistObjectHlIO for PersistMapImpl<M>
+where
+    M::MapType: STIndex<M::Key, M::Value>,
+{
     const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
-    type Type = HashMap<M::Key, M::Value>;
+    type Type = M::MapType;
     type Metadata = VoidMetadata;
     fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) {
         enc_dict_into_buffer::<M>(buf, v)
@@ -63,12 +68,9 @@ impl<M: PersistMapSpec> PersistObjectHlIO for PersistMapImpl<M> {
 }
 
 /// Encode the dict into the given buffer
-pub fn enc_dict_into_buffer<PM: PersistMapSpec>(
-    buf: &mut VecU8,
-    map: &HashMap<PM::Key, PM::Value>,
-) {
-    buf.extend(map.len().u64_bytes_le());
-    for (key, val) in map {
+pub fn enc_dict_into_buffer<PM: PersistMapSpec>(buf: &mut VecU8, map: &PM::MapType) {
+    buf.extend(map.st_len().u64_bytes_le());
+    for (key, val) in map.st_iter_kv() {
         PM::entry_md_enc(buf, key, val);
         if PM::ENC_COUPLED {
             PM::enc_entry(buf, key, val);
@@ -80,9 +82,10 @@ pub fn enc_dict_into_buffer<PM: PersistMapSpec>(
 }
 
 /// Decode the dict using the given buffered scanner
-pub fn dec_dict<PM: PersistMapSpec>(
-    scanner: &mut BufferedScanner,
-) -> SDSSResult<HashMap<PM::Key, PM::Value>> {
+pub fn dec_dict<PM: PersistMapSpec>(scanner: &mut BufferedScanner) -> SDSSResult<PM::MapType>
+where
+    PM::MapType: STIndex<PM::Key, PM::Value>,
+{
     if !(PM::meta_dec_collection_pretest(scanner) & scanner.has_left(sizeof!(u64))) {
         return Err(SDSSError::InternalDecodeStructureCorrupted);
     }
@@ -90,11 +93,11 @@ pub fn dec_dict<PM: PersistMapSpec>(
         // UNSAFE(@ohsayan): pretest
         scanner.next_u64_le() as usize
     };
-    let mut dict = HashMap::with_capacity(size);
-    while PM::meta_dec_entry_pretest(scanner) & (dict.len() != size) {
+    let mut dict = PM::MapType::idx_init_cap(size);
+    while PM::meta_dec_entry_pretest(scanner) & (dict.st_len() != size) {
         let md = unsafe {
             // pretest
-            dec_md::<PM::Metadata, true>(scanner)?
+            dec_md::<PM::EntryMD, true>(scanner)?
         };
         if PM::META_VERIFY_BEFORE_DEC && !md.pretest_src_for_object_dec(scanner) {
             return Err(SDSSError::InternalDecodeStructureCorrupted);
@@ -122,11 +125,11 @@ pub fn dec_dict<PM: PersistMapSpec>(
                 }
             }
         }
-        if dict.insert(key, val).is_some() {
+        if !dict.st_insert(key, val) {
             return Err(SDSSError::InternalDecodeStructureIllegalData);
         }
     }
-    if dict.len() == size {
+    if dict.st_len() == size {
         Ok(dict)
     } else {
         Err(SDSSError::InternalDecodeStructureIllegalData)
@@ -163,7 +166,7 @@ impl GenericDictEntryMD {
 impl PersistObjectMD for GenericDictEntryMD {
     const MD_DEC_INFALLIBLE: bool = true;
     fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
-        scanner.has_left((sizeof!(u64) * 2) + 1)
+        scanner.has_left(sizeof!(u64, 2) + 1)
     }
     unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
         Some(Self::decode(scanner.next_chunk()))
@@ -176,9 +179,10 @@ impl PersistObjectMD for GenericDictEntryMD {
 }
 
 impl PersistMapSpec for GenericDictSpec {
+    type MapType = DictGeneric;
     type Key = Box<str>;
     type Value = DictEntryGeneric;
-    type Metadata = GenericDictEntryMD;
+    type EntryMD = GenericDictEntryMD;
     const DEC_COUPLED: bool = false;
     const ENC_COUPLED: bool = true;
     const META_VERIFY_BEFORE_DEC: bool = true;
@@ -187,13 +191,13 @@ impl PersistMapSpec for GenericDictSpec {
     }
     fn meta_dec_entry_pretest(scanner: &BufferedScanner) -> bool {
         // we just need to see if we can decode the entry metadata
-        Self::Metadata::pretest_src_for_metadata_dec(scanner)
+        Self::EntryMD::pretest_src_for_metadata_dec(scanner)
     }
     fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) {
         buf.extend(key.len().u64_bytes_le());
     }
-    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::Metadata> {
-        Some(Self::Metadata::decode(scanner.next_chunk()))
+    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::EntryMD> {
+        Some(Self::EntryMD::decode(scanner.next_chunk()))
     }
     fn enc_entry(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
         match val {
@@ -236,12 +240,12 @@ impl PersistMapSpec for GenericDictSpec {
             }
         }
     }
-    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Key> {
+    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Key> {
         String::from_utf8(scanner.next_chunk_variable(md.klen).to_owned())
             .map(|s| s.into_boxed_str())
             .ok()
     }
-    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Value> {
+    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value> {
         unsafe fn decode_element(
             scanner: &mut BufferedScanner,
             dscr: PersistDictEntryDscr,
@@ -330,7 +334,7 @@ impl PersistMapSpec for GenericDictSpec {
     }
     unsafe fn dec_entry(
         _: &mut BufferedScanner,
-        _: Self::Metadata,
+        _: Self::EntryMD,
     ) -> Option<(Self::Key, Self::Value)> {
         unimplemented!()
     }
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index 38e8448b..3320c533 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -26,6 +26,8 @@
 
 //! High level interfaces
 
+use crate::engine::idx::STIndex;
+
 mod map;
 mod obj;
 // tests
@@ -243,8 +245,9 @@ pub fn dec_self<Obj: PersistObjectHlIO<Type = Obj>>(
 
 /// specification for a persist map
 pub trait PersistMapSpec {
+    type MapType: STIndex<Self::Key, Self::Value>;
     /// metadata type
-    type Metadata: PersistObjectMD;
+    type EntryMD: PersistObjectMD;
     /// key type (NOTE: set this to the true key type; handle any differences using the spec unless you have an entirely different
     /// wrapper type)
     type Key: AsKey;
@@ -266,22 +269,22 @@ pub trait PersistMapSpec {
     fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);
     /// dec the entry meta
     /// SAFETY: ensure that all pretests have passed (we expect the caller to not be stupid)
-    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::Metadata>;
+    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::EntryMD>;
     // independent packing
     /// enc key (non-packed)
     fn enc_key(buf: &mut VecU8, key: &Self::Key);
     /// enc val (non-packed)
     fn enc_val(buf: &mut VecU8, key: &Self::Value);
     /// dec key (non-packed)
-    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Key>;
+    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Key>;
     /// dec val (non-packed)
-    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::Metadata) -> Option<Self::Value>;
+    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value>;
     // coupled packing
     /// entry packed enc
     fn enc_entry(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);
     /// entry packed dec
     unsafe fn dec_entry(
         scanner: &mut BufferedScanner,
-        md: Self::Metadata,
+        md: Self::EntryMD,
     ) -> Option<(Self::Key, Self::Value)>;
 }
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 99689840..4934a22b 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -37,6 +37,13 @@ use {
     },
 };
 
+/*
+    Full 8B tag block. Notes:
+    1. 7B at this moment is currently unused but there's a lot of additional flags that we might want to store here
+    2. If we end up deciding that this is indeed a waste of space, version this out and get rid of the 7B (or whatever we determine
+    to be the correct size.)
+*/
+
 struct POByteBlockFullTag(FullTag);
 
 impl PersistObjectHlIO for POByteBlockFullTag {
@@ -58,6 +65,10 @@ impl PersistObjectHlIO for POByteBlockFullTag {
     }
 }
 
+/*
+    layer
+*/
+
 #[derive(Debug)]
 pub struct LayerMD {
     type_selector: u64,
@@ -76,7 +87,7 @@ impl LayerMD {
 impl PersistObjectMD for LayerMD {
     const MD_DEC_INFALLIBLE: bool = true;
     fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u64) * 2)
+        scanner.has_left(sizeof!(u64, 2))
     }
     fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
         true
@@ -111,6 +122,10 @@ impl PersistObjectHlIO for Layer {
     }
 }
 
+/*
+    field
+*/
+
 pub struct FieldMD {
     prop_c: u64,
     layer_c: u64,
@@ -130,7 +145,7 @@ impl FieldMD {
 impl PersistObjectMD for FieldMD {
     const MD_DEC_INFALLIBLE: bool = true;
     fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
-        scanner.has_left((sizeof!(u64) * 2) + 1)
+        scanner.has_left(sizeof!(u64, 2) + 1)
     }
     fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
         // nothing here really; we can't help much with the stuff ahead

From 70552a5df42089b3cabf0560a92518755a9fff71 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 18 Aug 2023 14:56:12 +0000
Subject: [PATCH 222/310] Impl enc/dec for model

---
 server/src/engine/core/model/mod.rs       |  41 ++++++--
 server/src/engine/data/uuid.rs            |   8 ++
 server/src/engine/idx/mod.rs              |   2 +
 server/src/engine/idx/stord/mod.rs        |   4 +-
 server/src/engine/storage/v1/inf/map.rs   | 116 +++++++++++++++++++++-
 server/src/engine/storage/v1/inf/mod.rs   |   7 ++
 server/src/engine/storage/v1/inf/obj.rs   |  82 ++++++++++++++-
 server/src/engine/storage/v1/inf/tests.rs |  43 +++++++-
 8 files changed, 283 insertions(+), 20 deletions(-)

diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 2c94798e..0863d376 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -77,6 +77,26 @@ impl PartialEq for Model {
 }
 
 impl Model {
+    pub fn new(
+        uuid: Uuid,
+        p_key: Box<str>,
+        p_tag: FullTag,
+        fields: UnsafeCell<Fields>,
+        sync_matrix: ISyncMatrix,
+        data: PrimaryIndex,
+        delta: DeltaState,
+    ) -> Self {
+        Self {
+            uuid,
+            p_key,
+            p_tag,
+            fields,
+            sync_matrix,
+            data,
+            delta,
+        }
+    }
+
     pub fn get_uuid(&self) -> Uuid {
         self.uuid
     }
@@ -130,6 +150,17 @@ impl Model {
 }
 
 impl Model {
+    pub fn new_restore(uuid: Uuid, p_key: Box<str>, p_tag: FullTag, fields: Fields) -> Self {
+        Self::new(
+            uuid,
+            p_key,
+            p_tag,
+            UnsafeCell::new(fields),
+            ISyncMatrix::new(),
+            PrimaryIndex::new_empty(),
+            DeltaState::new_resolved(),
+        )
+    }
     pub fn process_create(
         CreateModel {
             model_name: _,
@@ -163,15 +194,7 @@ impl Model {
             let last_pk = last_pk.unwrap_or(fields.stseq_ord_key().next().unwrap());
             let tag = fields.st_get(last_pk).unwrap().layers()[0].tag;
             if tag.tag_unique().is_unique() {
-                return Ok(Self {
-                    uuid: Uuid::new(),
-                    p_key: last_pk.into(),
-                    p_tag: tag,
-                    fields: UnsafeCell::new(fields),
-                    sync_matrix: ISyncMatrix::new(),
-                    data: PrimaryIndex::new_empty(),
-                    delta: DeltaState::new_resolved(),
-                });
+                return Ok(Self::new_restore(Uuid::new(), last_pk.into(), tag, fields));
             }
         }
         Err(DatabaseError::DdlModelBadDefinition)
diff --git a/server/src/engine/data/uuid.rs b/server/src/engine/data/uuid.rs
index 354f09a2..154d51fc 100644
--- a/server/src/engine/data/uuid.rs
+++ b/server/src/engine/data/uuid.rs
@@ -38,6 +38,14 @@ impl Uuid {
     pub fn as_slice(&self) -> &[u8] {
         self.data.as_bytes()
     }
+    pub fn from_bytes(b: [u8; 16]) -> Self {
+        Self {
+            data: uuid::Uuid::from_u128_le(u128::from_le_bytes(b)),
+        }
+    }
+    pub fn to_le_bytes(self) -> [u8; 16] {
+        self.data.to_u128_le().to_le_bytes()
+    }
 }
 
 impl ToString for Uuid {
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 013d6080..abcb7719 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -39,6 +39,8 @@ use {
     core::{borrow::Borrow, hash::Hash},
 };
 
+pub use stord::iter::IndexSTSeqDllIterOrdKV;
+
 // re-exports
 pub type IndexSTSeqCns<K, V> = stord::IndexSTSeqDll<K, V, stord::config::ConservativeConfig<K, V>>;
 pub type IndexSTSeqLib<K, V> = stord::IndexSTSeqDll<K, V, stord::config::LiberalConfig<K, V>>;
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index f54d02f2..d053caf5 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -25,7 +25,7 @@
 */
 
 pub(super) mod config;
-mod iter;
+pub(super) mod iter;
 
 use {
     self::{
@@ -759,7 +759,7 @@ impl<K: AsKey, V: AsValue + PartialEq, C: Config<K, V>> PartialEq for IndexSTSeq
     fn eq(&self, other: &Self) -> bool {
         self.len() == other.len()
             && self
-                ._iter_unord_kv()
+                ._iter_ord_kv()
                 .all(|(k, v)| other._get(k).unwrap().eq(v))
     }
 }
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index c853fdad..f04853ae 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -26,18 +26,19 @@
 
 use {
     super::{
-        dec_md, PersistDictEntryDscr, PersistMapSpec, PersistObjectHlIO, PersistObjectMD, VecU8,
-        VoidMetadata,
+        dec_md, obj::FieldMD, PersistDictEntryDscr, PersistMapSpec, PersistObjectHlIO,
+        PersistObjectMD, VecU8, VoidMetadata,
     },
     crate::{
         engine::{
+            core::model::{Field, Layer},
             data::{
                 cell::Datacell,
                 dict::DictEntryGeneric,
                 tag::{CUTag, DataTag, TagClass, TagUnique},
                 DictGeneric,
             },
-            idx::{IndexBaseSpec, STIndex},
+            idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
             storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult},
         },
         util::{copy_slice_to_array as memcpy, EndianQW},
@@ -70,7 +71,7 @@ where
 /// Encode the dict into the given buffer
 pub fn enc_dict_into_buffer<PM: PersistMapSpec>(buf: &mut VecU8, map: &PM::MapType) {
     buf.extend(map.st_len().u64_bytes_le());
-    for (key, val) in map.st_iter_kv() {
+    for (key, val) in PM::_get_iter(map) {
         PM::entry_md_enc(buf, key, val);
         if PM::ENC_COUPLED {
             PM::enc_entry(buf, key, val);
@@ -179,6 +180,7 @@ impl PersistObjectMD for GenericDictEntryMD {
 }
 
 impl PersistMapSpec for GenericDictSpec {
+    type MapIter<'a> = std::collections::hash_map::Iter<'a, Box<str>, DictEntryGeneric>;
     type MapType = DictGeneric;
     type Key = Box<str>;
     type Value = DictEntryGeneric;
@@ -186,6 +188,9 @@ impl PersistMapSpec for GenericDictSpec {
     const DEC_COUPLED: bool = false;
     const ENC_COUPLED: bool = true;
     const META_VERIFY_BEFORE_DEC: bool = true;
+    fn _get_iter<'a>(map: &'a Self::MapType) -> Self::MapIter<'a> {
+        map.iter()
+    }
     fn meta_dec_collection_pretest(_: &BufferedScanner) -> bool {
         true
     }
@@ -339,3 +344,106 @@ impl PersistMapSpec for GenericDictSpec {
         unimplemented!()
     }
 }
+
+pub struct FieldMapSpec;
+pub struct FieldMapEntryMD {
+    field_id_l: u64,
+    field_prop_c: u64,
+    field_layer_c: u64,
+    null: u8,
+}
+
+impl FieldMapEntryMD {
+    const fn new(field_id_l: u64, field_prop_c: u64, field_layer_c: u64, null: u8) -> Self {
+        Self {
+            field_id_l,
+            field_prop_c,
+            field_layer_c,
+            null,
+        }
+    }
+}
+
+impl PersistObjectMD for FieldMapEntryMD {
+    const MD_DEC_INFALLIBLE: bool = true;
+
+    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u64, 3) + 1)
+    }
+
+    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool {
+        scanner.has_left(self.field_id_l as usize) // TODO(@ohsayan): we can enforce way more here such as atleast one field etc
+    }
+
+    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
+        Some(Self::new(
+            u64::from_le_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()),
+            scanner.next_byte(),
+        ))
+    }
+}
+
+impl PersistMapSpec for FieldMapSpec {
+    type MapIter<'a> = crate::engine::idx::IndexSTSeqDllIterOrdKV<'a, Box<str>, Field>;
+    type MapType = IndexSTSeqCns<Self::Key, Self::Value>;
+    type EntryMD = FieldMapEntryMD;
+    type Key = Box<str>;
+    type Value = Field;
+    const ENC_COUPLED: bool = false;
+    const DEC_COUPLED: bool = false;
+    const META_VERIFY_BEFORE_DEC: bool = true;
+    fn _get_iter<'a>(m: &'a Self::MapType) -> Self::MapIter<'a> {
+        m.stseq_ord_kv()
+    }
+    fn meta_dec_collection_pretest(_: &BufferedScanner) -> bool {
+        true
+    }
+    fn meta_dec_entry_pretest(scanner: &BufferedScanner) -> bool {
+        FieldMapEntryMD::pretest_src_for_metadata_dec(scanner)
+    }
+    fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
+        buf.extend(key.len().u64_bytes_le());
+        buf.extend(0u64.to_le_bytes()); // TODO(@ohsayan): props
+        buf.extend(val.layers().len().u64_bytes_le());
+        buf.push(val.is_nullable() as u8);
+    }
+    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::EntryMD> {
+        FieldMapEntryMD::dec_md_payload(scanner)
+    }
+    fn enc_key(buf: &mut VecU8, key: &Self::Key) {
+        buf.extend(key.as_bytes());
+    }
+    fn enc_val(buf: &mut VecU8, val: &Self::Value) {
+        for layer in val.layers() {
+            Layer::pe_obj_hlio_enc(buf, layer)
+        }
+    }
+    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Key> {
+        String::from_utf8(
+            scanner
+                .next_chunk_variable(md.field_id_l as usize)
+                .to_owned(),
+        )
+        .map(|v| v.into_boxed_str())
+        .ok()
+    }
+    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value> {
+        Field::pe_obj_hlio_dec(
+            scanner,
+            FieldMD::new(md.field_prop_c, md.field_layer_c, md.null),
+        )
+        .ok()
+    }
+    // unimplemented
+    fn enc_entry(_: &mut VecU8, _: &Self::Key, _: &Self::Value) {
+        unimplemented!()
+    }
+    unsafe fn dec_entry(
+        _: &mut BufferedScanner,
+        _: Self::EntryMD,
+    ) -> Option<(Self::Key, Self::Value)> {
+        unimplemented!()
+    }
+}
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index 3320c533..8035899e 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -245,7 +245,12 @@ pub fn dec_self<Obj: PersistObjectHlIO<Type = Obj>>(
 
 /// specification for a persist map
 pub trait PersistMapSpec {
+    /// map type
     type MapType: STIndex<Self::Key, Self::Value>;
+    /// map iter
+    type MapIter<'a>: Iterator<Item = (&'a Self::Key, &'a Self::Value)>
+    where
+        Self: 'a;
     /// metadata type
     type EntryMD: PersistObjectMD;
     /// key type (NOTE: set this to the true key type; handle any differences using the spec unless you have an entirely different
@@ -259,6 +264,8 @@ pub trait PersistMapSpec {
     const DEC_COUPLED: bool;
     /// verify the src using the given metadata
     const META_VERIFY_BEFORE_DEC: bool;
+    // collection misc
+    fn _get_iter<'a>(map: &'a Self::MapType) -> Self::MapIter<'a>;
     // collection meta
     /// pretest before jmp to routine for entire collection
     fn meta_dec_collection_pretest(scanner: &BufferedScanner) -> bool;
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 4934a22b..33504058 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -25,11 +25,14 @@
 */
 
 use {
-    super::{dec_md, PersistObjectHlIO, PersistObjectMD, SimpleSizeMD, VecU8},
+    super::{dec_md, map::FieldMapSpec, PersistObjectHlIO, PersistObjectMD, SimpleSizeMD, VecU8},
     crate::{
         engine::{
-            core::model::{Field, Layer},
-            data::tag::{DataTag, FullTag, TagClass, TagSelector},
+            core::model::{Field, Layer, Model},
+            data::{
+                tag::{DataTag, FullTag, TagClass, TagSelector},
+                uuid::Uuid,
+            },
             mem::VInline,
             storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult},
         },
@@ -133,7 +136,7 @@ pub struct FieldMD {
 }
 
 impl FieldMD {
-    const fn new(prop_c: u64, layer_c: u64, null: u8) -> Self {
+    pub(super) const fn new(prop_c: u64, layer_c: u64, null: u8) -> Self {
         Self {
             prop_c,
             layer_c,
@@ -200,3 +203,74 @@ impl PersistObjectHlIO for Field {
         }
     }
 }
+
+pub struct ModelLayout;
+pub struct ModelLayoutMD {
+    model_uuid: Uuid,
+    p_key_len: u64,
+    p_key_tag: u64,
+}
+
+impl ModelLayoutMD {
+    pub(super) const fn new(model_uuid: Uuid, p_key_len: u64, p_key_tag: u64) -> Self {
+        Self {
+            model_uuid,
+            p_key_len,
+            p_key_tag,
+        }
+    }
+}
+
+impl PersistObjectMD for ModelLayoutMD {
+    const MD_DEC_INFALLIBLE: bool = true;
+    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u64, 3) + sizeof!(u128)) // u64,3 since the fieldmap len is also there, but we don't handle it directly
+    }
+    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool {
+        scanner.has_left(self.p_key_len as usize)
+    }
+    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
+        Some(Self::new(
+            Uuid::from_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()),
+        ))
+    }
+}
+
+impl PersistObjectHlIO for ModelLayout {
+    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = true;
+    type Type = Model;
+    type Metadata = ModelLayoutMD;
+    fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) {
+        let irm = v.intent_read_model();
+        buf.extend(v.get_uuid().to_le_bytes());
+        buf.extend(v.p_key().len().u64_bytes_le());
+        buf.extend(v.p_tag().tag_selector().d().u64_bytes_le());
+        buf.extend(v.p_key().as_bytes());
+        super::map::enc_dict_into_buffer::<FieldMapSpec>(buf, irm.fields())
+    }
+    unsafe fn pe_obj_hlio_dec(
+        scanner: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> SDSSResult<Self::Type> {
+        let key = String::from_utf8(
+            scanner
+                .next_chunk_variable(md.p_key_len as usize)
+                .to_owned(),
+        )
+        .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)?;
+        let fieldmap = super::map::dec_dict::<FieldMapSpec>(scanner)?;
+        let ptag = if md.p_key_tag > TagSelector::max_dscr() as u64 {
+            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+        } else {
+            TagSelector::from_raw(md.p_key_tag as u8)
+        };
+        Ok(Model::new_restore(
+            md.model_uuid,
+            key.into_boxed_str(),
+            ptag.into_full(),
+            fieldmap,
+        ))
+    }
+}
diff --git a/server/src/engine/storage/v1/inf/tests.rs b/server/src/engine/storage/v1/inf/tests.rs
index 5c0b5665..19d48fc0 100644
--- a/server/src/engine/storage/v1/inf/tests.rs
+++ b/server/src/engine/storage/v1/inf/tests.rs
@@ -25,11 +25,14 @@
 */
 
 use crate::engine::{
-    core::model::{Field, Layer},
+    core::model::{Field, Layer, Model},
     data::{
         cell::Datacell,
         dict::{DictEntryGeneric, DictGeneric},
+        tag::TagSelector,
+        uuid::Uuid,
     },
+    idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
     storage::v1::rw::BufferedScanner,
 };
 
@@ -68,3 +71,41 @@ fn field() {
     let dec = super::dec_self::<Field>(&mut scanner).unwrap();
     assert_eq!(field, dec);
 }
+
+#[test]
+fn fieldmap() {
+    let mut fields = IndexSTSeqCns::<Box<str>, Field>::idx_init();
+    fields.st_insert("password".into(), Field::new([Layer::bin()].into(), false));
+    fields.st_insert(
+        "profile_pic".into(),
+        Field::new([Layer::bin()].into(), true),
+    );
+    let enc = super::enc::<super::map::PersistMapImpl<super::map::FieldMapSpec>>(&fields);
+    let mut scanner = BufferedScanner::new(&enc);
+    let dec =
+        super::dec::<super::map::PersistMapImpl<super::map::FieldMapSpec>>(&mut scanner).unwrap();
+    for ((orig_field_id, orig_field), (restored_field_id, restored_field)) in
+        fields.stseq_ord_kv().zip(dec.stseq_ord_kv())
+    {
+        assert_eq!(orig_field_id, restored_field_id);
+        assert_eq!(orig_field, restored_field);
+    }
+}
+
+#[test]
+fn model() {
+    let uuid = Uuid::new();
+    let model = Model::new_restore(
+        uuid,
+        "username".into(),
+        TagSelector::Str.into_full(),
+        into_dict! {
+            "password" => Field::new([Layer::bin()].into(), false),
+            "profile_pic" => Field::new([Layer::bin()].into(), true),
+        },
+    );
+    let enc = super::enc::<super::obj::ModelLayout>(&model);
+    let mut scanner = BufferedScanner::new(&enc);
+    let dec = super::dec::<super::obj::ModelLayout>(&mut scanner).unwrap();
+    assert_eq!(model, dec);
+}

From d8cabb9761c5a5e711c512f6d4c7052516bdd636 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 18 Aug 2023 16:53:00 +0000
Subject: [PATCH 223/310] Add enc/dec for space

---
 server/src/engine/core/mod.rs                 |   2 +-
 server/src/engine/core/model/alt.rs           |   2 +-
 server/src/engine/core/space.rs               |  19 ++-
 server/src/engine/data/dict.rs                | 113 +++++++-----------
 server/src/engine/data/mod.rs                 |   2 +-
 server/src/engine/data/tests/md_dict_tests.rs |  18 +--
 server/src/engine/ql/ddl/syn.rs               |   2 +-
 server/src/engine/ql/tests.rs                 |   2 +-
 server/src/engine/storage/v1/inf/map.rs       |  14 +--
 server/src/engine/storage/v1/inf/mod.rs       |   2 +-
 server/src/engine/storage/v1/inf/obj.rs       |  54 ++++++++-
 server/src/engine/storage/v1/inf/tests.rs     |  15 ++-
 12 files changed, 145 insertions(+), 100 deletions(-)

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 8b6f31b1..7e036e22 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -28,7 +28,7 @@ mod dml;
 mod index;
 pub(in crate::engine) mod model;
 pub(in crate::engine) mod query_meta;
-mod space;
+pub mod space;
 mod util;
 // test
 #[cfg(test)]
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index f4d16ff9..87868716 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -81,7 +81,7 @@ fn no_field(mr: &IWModel, new: &str) -> bool {
 
 fn check_nullable(props: &mut HashMap<Box<str>, DictEntryGeneric>) -> DatabaseResult<bool> {
     match props.remove("nullable") {
-        Some(DictEntryGeneric::Lit(b)) if b.kind() == TagClass::Bool => Ok(b.bool()),
+        Some(DictEntryGeneric::Data(b)) if b.kind() == TagClass::Bool => Ok(b.bool()),
         Some(_) => Err(DatabaseError::DdlModelAlterBadProperty),
         None => Ok(false),
     }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 2fc8b741..86169d7c 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -27,7 +27,7 @@
 use {
     crate::engine::{
         core::{model::Model, RWLIdx},
-        data::{dict, uuid::Uuid, DictEntryGeneric, MetaDict},
+        data::{dict, uuid::Uuid, DictEntryGeneric, DictGeneric},
         error::{DatabaseError, DatabaseResult},
         idx::{IndexST, STIndex},
         ql::ddl::{alt::AlterSpace, crt::CreateSpace, drop::DropSpace},
@@ -46,16 +46,19 @@ pub struct Space {
 #[derive(Debug, Default)]
 /// Space metadata
 pub struct SpaceMeta {
-    pub(super) env: RwLock<MetaDict>,
+    pub(super) env: RwLock<DictGeneric>,
 }
 
 impl SpaceMeta {
     pub const KEY_ENV: &str = "env";
-    pub fn with_env(env: MetaDict) -> Self {
+    pub fn with_env(env: DictGeneric) -> Self {
         Self {
             env: RWLIdx::new(env),
         }
     }
+    pub fn env(&self) -> &RwLock<DictGeneric> {
+        &self.env
+    }
 }
 
 #[derive(Debug)]
@@ -92,6 +95,9 @@ impl Space {
     pub(super) fn models(&self) -> &RWLIdx<Box<str>, Model> {
         &self.mns
     }
+    pub fn metadata(&self) -> &SpaceMeta {
+        &self.meta
+    }
     pub fn with_model<T>(
         &self,
         model: &str,
@@ -103,6 +109,9 @@ impl Space {
         };
         f(model)
     }
+    pub(crate) fn new_restore_empty(meta: SpaceMeta, uuid: Uuid) -> Space {
+        Self::new_with_uuid(Default::default(), meta, uuid)
+    }
 }
 
 impl Space {
@@ -139,7 +148,7 @@ impl Space {
         // check env
         let env = match props.remove(SpaceMeta::KEY_ENV) {
             Some(DictEntryGeneric::Map(m)) if props.is_empty() => m,
-            Some(DictEntryGeneric::Lit(l)) if l.is_null() => IndexST::default(),
+            Some(DictEntryGeneric::Data(l)) if l.is_null() => IndexST::default(),
             None if props.is_empty() => IndexST::default(),
             _ => {
                 return Err(DatabaseError::DdlSpaceBadProperty);
@@ -185,7 +194,7 @@ impl Space {
                         return Err(DatabaseError::DdlSpaceBadProperty);
                     }
                 }
-                Some(DictEntryGeneric::Lit(l)) if updated_props.is_empty() & l.is_null() => {
+                Some(DictEntryGeneric::Data(l)) if updated_props.is_empty() & l.is_null() => {
                     space_env.clear()
                 }
                 None => {}
diff --git a/server/src/engine/data/dict.rs b/server/src/engine/data/dict.rs
index fbab8ef0..21aecfc0 100644
--- a/server/src/engine/data/dict.rs
+++ b/server/src/engine/data/dict.rs
@@ -35,73 +35,51 @@ use {
     std::collections::HashMap,
 };
 
-/*
-    dict kinds: one is from a generic parse while the other one is the stored metadata
-    - MetaDict -> only non-null values
-    - DictGeneric -> null allowed
-*/
-
-/// A metadata dictionary
-pub type MetaDict = HashMap<Box<str>, MetaDictEntry>;
 /// A generic dictionary built from scratch from syntactical elements
 pub type DictGeneric = HashMap<Box<str>, DictEntryGeneric>;
 
 #[derive(Debug, PartialEq)]
+#[cfg_attr(test, derive(Clone))]
 /// A generic dict entry: either a literal or a recursive dictionary
 pub enum DictEntryGeneric {
     /// A literal
-    Lit(Datacell),
+    Data(Datacell),
     /// A map
     Map(DictGeneric),
 }
 
-#[derive(Debug, PartialEq)]
-#[cfg_attr(test, derive(Clone))]
-/// A metadata dictionary
-pub enum MetaDictEntry {
-    Data(Datacell),
-    Map(MetaDict),
-}
-
 /*
     patchsets
 */
 
 #[derive(Debug, PartialEq, Default)]
-struct MetaDictPatch(HashMap<Box<str>, Option<MetaDictPatchEntry>>);
+struct DictGenericPatch(HashMap<Box<str>, Option<DictGenericPatchEntry>>);
 #[derive(Debug, PartialEq)]
-enum MetaDictPatchEntry {
+enum DictGenericPatchEntry {
     Data(Datacell),
-    Map(MetaDictPatch),
+    Map(DictGenericPatch),
 }
 
-/// Recursively flatten a [`DictGeneric`] into a [`MetaDict`]
-pub fn rflatten_metadata(new: DictGeneric) -> MetaDict {
-    let mut empty = MetaDict::new();
-    _rflatten_metadata(new, &mut empty);
-    empty
+/// Accepts a dict with possible null values, and removes those null values
+pub fn rflatten_metadata(mut new: DictGeneric) -> DictGeneric {
+    _rflatten_metadata(&mut new);
+    new
 }
 
-fn _rflatten_metadata(new: DictGeneric, empty: &mut MetaDict) {
-    for (key, val) in new {
-        match val {
-            DictEntryGeneric::Lit(l) if l.is_init() => {
-                empty.insert(key, MetaDictEntry::Data(l));
-            }
-            DictEntryGeneric::Lit(_) => {}
-            DictEntryGeneric::Map(m) => {
-                let mut rnew = MetaDict::new();
-                _rflatten_metadata(m, &mut rnew);
-                empty.insert(key, MetaDictEntry::Map(rnew));
-            }
+fn _rflatten_metadata(new: &mut DictGeneric) {
+    new.retain(|_, v| match v {
+        DictEntryGeneric::Data(d) => d.is_init(),
+        DictEntryGeneric::Map(m) => {
+            _rflatten_metadata(m);
+            true
         }
-    }
+    });
 }
 
-/// Recursively merge a [`DictGeneric`] into a [`MetaDict`] with the use of an intermediary
+/// Recursively merge a [`DictGeneric`] into a [`DictGeneric`] with the use of an intermediary
 /// patchset to avoid inconsistent states
-pub fn rmerge_metadata(current: &mut MetaDict, new: DictGeneric) -> bool {
-    let mut patch = MetaDictPatch::default();
+pub fn rmerge_metadata(current: &mut DictGeneric, new: DictGeneric) -> bool {
+    let mut patch = DictGenericPatch::default();
     let current_ref = current as &_;
     let r = rmerge_metadata_prepare_patch(current_ref, new, &mut patch);
     if r {
@@ -110,15 +88,15 @@ pub fn rmerge_metadata(current: &mut MetaDict, new: DictGeneric) -> bool {
     r
 }
 
-fn merge_data_with_patch(current: &mut MetaDict, patch: MetaDictPatch) {
+fn merge_data_with_patch(current: &mut DictGeneric, patch: DictGenericPatch) {
     for (key, patch) in patch.0 {
         match patch {
-            Some(MetaDictPatchEntry::Data(d)) => {
-                current.st_upsert(key, MetaDictEntry::Data(d));
+            Some(DictGenericPatchEntry::Data(d)) => {
+                current.st_upsert(key, DictEntryGeneric::Data(d));
             }
-            Some(MetaDictPatchEntry::Map(m)) => match current.get_mut(&key) {
+            Some(DictGenericPatchEntry::Map(m)) => match current.get_mut(&key) {
                 Some(current_recursive) => match current_recursive {
-                    MetaDictEntry::Map(current_m) => {
+                    DictEntryGeneric::Map(current_m) => {
                         merge_data_with_patch(current_m, m);
                     }
                     _ => {
@@ -127,7 +105,7 @@ fn merge_data_with_patch(current: &mut MetaDict, patch: MetaDictPatch) {
                     }
                 },
                 None => {
-                    let mut new = MetaDict::new();
+                    let mut new = DictGeneric::new();
                     merge_data_with_patch(&mut new, m);
                 }
             },
@@ -139,9 +117,9 @@ fn merge_data_with_patch(current: &mut MetaDict, patch: MetaDictPatch) {
 }
 
 fn rmerge_metadata_prepare_patch(
-    current: &MetaDict,
+    current: &DictGeneric,
     new: DictGeneric,
-    patch: &mut MetaDictPatch,
+    patch: &mut DictGenericPatch,
 ) -> bool {
     let mut new = new.into_iter();
     let mut okay = true;
@@ -149,25 +127,25 @@ fn rmerge_metadata_prepare_patch(
         let (key, new_entry) = new.next().unwrap();
         match (current.get(&key), new_entry) {
             // non-null -> non-null: merge flatten update
-            (Some(MetaDictEntry::Data(this_data)), DictEntryGeneric::Lit(new_data))
+            (Some(DictEntryGeneric::Data(this_data)), DictEntryGeneric::Data(new_data))
                 if new_data.is_init() =>
             {
                 if this_data.kind() == new_data.kind() {
                     patch
                         .0
-                        .insert(key, Some(MetaDictPatchEntry::Data(new_data)));
+                        .insert(key, Some(DictGenericPatchEntry::Data(new_data)));
                 } else {
                     okay = false;
                 }
             }
-            (Some(MetaDictEntry::Data(_)), DictEntryGeneric::Map(_)) => {
+            (Some(DictEntryGeneric::Data(_)), DictEntryGeneric::Map(_)) => {
                 okay = false;
             }
             (
-                Some(MetaDictEntry::Map(this_recursive_data)),
+                Some(DictEntryGeneric::Map(this_recursive_data)),
                 DictEntryGeneric::Map(new_recursive_data),
             ) => {
-                let mut this_patch = MetaDictPatch::default();
+                let mut this_patch = DictGenericPatch::default();
                 let _okay = rmerge_metadata_prepare_patch(
                     this_recursive_data,
                     new_recursive_data,
@@ -175,26 +153,26 @@ fn rmerge_metadata_prepare_patch(
                 );
                 patch
                     .0
-                    .insert(key, Some(MetaDictPatchEntry::Map(this_patch)));
+                    .insert(key, Some(DictGenericPatchEntry::Map(this_patch)));
                 okay &= _okay;
             }
             // null -> non-null: flatten insert
-            (None, DictEntryGeneric::Lit(l)) if l.is_init() => {
-                let _ = patch.0.insert(key, Some(MetaDictPatchEntry::Data(l)));
+            (None, DictEntryGeneric::Data(l)) if l.is_init() => {
+                let _ = patch.0.insert(key, Some(DictGenericPatchEntry::Data(l)));
             }
             (None, DictEntryGeneric::Map(m)) => {
-                let mut this_patch = MetaDictPatch::default();
+                let mut this_patch = DictGenericPatch::default();
                 okay &= rmerge_metadata_prepare_patch(&into_dict!(), m, &mut this_patch);
                 let _ = patch
                     .0
-                    .insert(key, Some(MetaDictPatchEntry::Map(this_patch)));
+                    .insert(key, Some(DictGenericPatchEntry::Map(this_patch)));
             }
             // non-null -> null: remove
-            (Some(_), DictEntryGeneric::Lit(l)) => {
+            (Some(_), DictEntryGeneric::Data(l)) => {
                 debug_assert!(l.is_null());
                 patch.0.insert(key, None);
             }
-            (None, DictEntryGeneric::Lit(l)) => {
+            (None, DictEntryGeneric::Data(l)) => {
                 debug_assert!(l.is_null());
                 // ignore
             }
@@ -209,26 +187,19 @@ fn rmerge_metadata_prepare_patch(
 
 impl<'a> From<LitIR<'a>> for DictEntryGeneric {
     fn from(l: LitIR<'a>) -> Self {
-        Self::Lit(Datacell::from(l))
+        Self::Data(Datacell::from(l))
     }
 }
 
 impl<'a> From<Lit<'a>> for DictEntryGeneric {
     fn from(value: Lit<'a>) -> Self {
-        Self::Lit(Datacell::from(value))
+        Self::Data(Datacell::from(value))
     }
 }
 
 direct_from! {
     DictEntryGeneric => {
-        Datacell as Lit,
-        DictGeneric as Map,
-    }
-}
-
-direct_from! {
-    MetaDictEntry => {
         Datacell as Data,
-        MetaDict as Map,
+        DictGeneric as Map,
     }
 }
diff --git a/server/src/engine/data/mod.rs b/server/src/engine/data/mod.rs
index aaa35691..97dedc69 100644
--- a/server/src/engine/data/mod.rs
+++ b/server/src/engine/data/mod.rs
@@ -36,4 +36,4 @@ pub mod uuid;
 #[cfg(test)]
 mod tests;
 
-pub use dict::{DictEntryGeneric, DictGeneric, MetaDict};
+pub use dict::{DictEntryGeneric, DictGeneric};
diff --git a/server/src/engine/data/tests/md_dict_tests.rs b/server/src/engine/data/tests/md_dict_tests.rs
index c0c3adf4..f7e81fbd 100644
--- a/server/src/engine/data/tests/md_dict_tests.rs
+++ b/server/src/engine/data/tests/md_dict_tests.rs
@@ -26,16 +26,16 @@
 
 use crate::engine::data::{
     cell::Datacell,
-    dict::{self, DictEntryGeneric, DictGeneric, MetaDict, MetaDictEntry},
+    dict::{self, DictEntryGeneric, DictGeneric},
 };
 
 #[test]
 fn t_simple_flatten() {
     let generic_dict: DictGeneric = into_dict! {
-        "a_valid_key" => DictEntryGeneric::Lit(100u64.into()),
+        "a_valid_key" => DictEntryGeneric::Data(100u64.into()),
         "a_null_key" => Datacell::null(),
     };
-    let expected: MetaDict = into_dict!(
+    let expected: DictGeneric = into_dict!(
         "a_valid_key" => Datacell::new_uint(100)
     );
     let ret = dict::rflatten_metadata(generic_dict);
@@ -44,7 +44,7 @@ fn t_simple_flatten() {
 
 #[test]
 fn t_simple_patch() {
-    let mut current: MetaDict = into_dict! {
+    let mut current: DictGeneric = into_dict! {
         "a" => Datacell::new_uint(2),
         "b" => Datacell::new_uint(3),
         "z" => Datacell::new_sint(-100),
@@ -54,7 +54,7 @@ fn t_simple_patch() {
         "b" => Datacell::new_uint(2),
         "z" => Datacell::null(),
     };
-    let expected: MetaDict = into_dict! {
+    let expected: DictGeneric = into_dict! {
         "a" => Datacell::new_uint(1),
         "b" => Datacell::new_uint(2),
     };
@@ -64,7 +64,7 @@ fn t_simple_patch() {
 
 #[test]
 fn t_bad_patch() {
-    let mut current: MetaDict = into_dict! {
+    let mut current: DictGeneric = into_dict! {
         "a" => Datacell::new_uint(2),
         "b" => Datacell::new_uint(3),
         "z" => Datacell::new_sint(-100),
@@ -81,15 +81,15 @@ fn t_bad_patch() {
 
 #[test]
 fn patch_null_out_dict() {
-    let mut current: MetaDict = into_dict! {
+    let mut current: DictGeneric = into_dict! {
         "a" => Datacell::new_uint(2),
         "b" => Datacell::new_uint(3),
-        "z" => MetaDictEntry::Map(into_dict!(
+        "z" => DictEntryGeneric::Map(into_dict!(
             "c" => Datacell::new_uint(1),
             "d" => Datacell::new_uint(2)
         )),
     };
-    let expected: MetaDict = into_dict! {
+    let expected: DictGeneric = into_dict! {
         "a" => Datacell::new_uint(2),
         "b" => Datacell::new_uint(3),
     };
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 89b8ca48..feff7dab 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -179,7 +179,7 @@ where
                             // UNSAFE(@ohsayan): we only switch to this when we've already read in a key
                             key.take().as_str().into()
                         },
-                        DictEntryGeneric::Lit(Datacell::null()),
+                        DictEntryGeneric::Data(Datacell::null()),
                     )
                     .is_none(),
                 );
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 0ec76fe1..b5ecd462 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -81,7 +81,7 @@ pub trait NullableDictEntry {
 
 impl NullableDictEntry for Null {
     fn data(self) -> crate::engine::data::DictEntryGeneric {
-        crate::engine::data::DictEntryGeneric::Lit(Datacell::null())
+        crate::engine::data::DictEntryGeneric::Data(Datacell::null())
     }
 }
 
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index f04853ae..cf9112af 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -211,7 +211,7 @@ impl PersistMapSpec for GenericDictSpec {
                 buf.extend(key.as_bytes());
                 enc_dict_into_buffer::<Self>(buf, map);
             }
-            DictEntryGeneric::Lit(dc) => {
+            DictEntryGeneric::Data(dc) => {
                 buf.push(
                     PersistDictEntryDscr::translate_from_class(dc.tag().tag_class()).value_u8()
                         * (!dc.is_null() as u8),
@@ -257,13 +257,13 @@ impl PersistMapSpec for GenericDictSpec {
             dg_top_element: bool,
         ) -> Option<DictEntryGeneric> {
             let r = match dscr {
-                PersistDictEntryDscr::Null => DictEntryGeneric::Lit(Datacell::null()),
+                PersistDictEntryDscr::Null => DictEntryGeneric::Data(Datacell::null()),
                 PersistDictEntryDscr::Bool => {
-                    DictEntryGeneric::Lit(Datacell::new_bool(scanner.next_byte() == 1))
+                    DictEntryGeneric::Data(Datacell::new_bool(scanner.next_byte() == 1))
                 }
                 PersistDictEntryDscr::UnsignedInt
                 | PersistDictEntryDscr::SignedInt
-                | PersistDictEntryDscr::Float => DictEntryGeneric::Lit(Datacell::new_qw(
+                | PersistDictEntryDscr::Float => DictEntryGeneric::Data(Datacell::new_qw(
                     scanner.next_u64_le(),
                     CUTag::new(
                         dscr.into_class(),
@@ -281,7 +281,7 @@ impl PersistMapSpec for GenericDictSpec {
                         return None;
                     }
                     let slc = scanner.next_chunk_variable(slc_len);
-                    DictEntryGeneric::Lit(if dscr == PersistDictEntryDscr::Str {
+                    DictEntryGeneric::Data(if dscr == PersistDictEntryDscr::Str {
                         if core::str::from_utf8(slc).is_err() {
                             return None;
                         }
@@ -306,14 +306,14 @@ impl PersistMapSpec for GenericDictSpec {
                                 PersistDictEntryDscr::from_raw(dscr),
                                 false,
                             ) {
-                                Some(DictEntryGeneric::Lit(l)) => l,
+                                Some(DictEntryGeneric::Data(l)) => l,
                                 None => return None,
                                 _ => unreachable!("found top-level dict item in datacell"),
                             },
                         );
                     }
                     if v.len() == list_len {
-                        DictEntryGeneric::Lit(Datacell::new_list(v))
+                        DictEntryGeneric::Data(Datacell::new_list(v))
                     } else {
                         return None;
                     }
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index 8035899e..fe2702be 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -74,7 +74,7 @@ impl PersistDictEntryDscr {
     pub fn new_from_dict_gen_entry(e: &DictEntryGeneric) -> Self {
         match e {
             DictEntryGeneric::Map(_) => Self::Dict,
-            DictEntryGeneric::Lit(dc) => Self::translate_from_class(dc.tag().tag_class()),
+            DictEntryGeneric::Data(dc) => Self::translate_from_class(dc.tag().tag_class()),
         }
     }
     /// The data in question is null (well, can we call that data afterall?)
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 33504058..24af2954 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -28,7 +28,10 @@ use {
     super::{dec_md, map::FieldMapSpec, PersistObjectHlIO, PersistObjectMD, SimpleSizeMD, VecU8},
     crate::{
         engine::{
-            core::model::{Field, Layer, Model},
+            core::{
+                model::{Field, Layer, Model},
+                space::{Space, SpaceMeta},
+            },
             data::{
                 tag::{DataTag, FullTag, TagClass, TagSelector},
                 uuid::Uuid,
@@ -274,3 +277,52 @@ impl PersistObjectHlIO for ModelLayout {
         ))
     }
 }
+
+pub struct SpaceLayout;
+pub struct SpaceLayoutMD {
+    uuid: Uuid,
+}
+
+impl SpaceLayoutMD {
+    pub fn new(uuid: Uuid) -> Self {
+        Self { uuid }
+    }
+}
+
+impl PersistObjectMD for SpaceLayoutMD {
+    const MD_DEC_INFALLIBLE: bool = true;
+
+    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u128) + sizeof!(u64)) // u64 for props dict; we don't handle that directly
+    }
+    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
+        true
+    }
+    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
+        Some(Self::new(Uuid::from_bytes(scanner.next_chunk())))
+    }
+}
+
+impl PersistObjectHlIO for SpaceLayout {
+    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false; // no need, since the MD only handles the UUID
+    type Type = Space;
+    type Metadata = SpaceLayoutMD;
+    fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) {
+        buf.extend(v.get_uuid().to_le_bytes());
+        super::enc_into_buf::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(
+            buf,
+            &v.metadata().env().read(),
+        );
+    }
+    unsafe fn pe_obj_hlio_dec(
+        scanner: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> SDSSResult<Self::Type> {
+        let space_meta =
+            super::dec::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(scanner)?;
+        Ok(Space::new_restore_empty(
+            SpaceMeta::with_env(space_meta),
+            md.uuid,
+        ))
+    }
+}
diff --git a/server/src/engine/storage/v1/inf/tests.rs b/server/src/engine/storage/v1/inf/tests.rs
index 19d48fc0..bf7128c7 100644
--- a/server/src/engine/storage/v1/inf/tests.rs
+++ b/server/src/engine/storage/v1/inf/tests.rs
@@ -25,7 +25,10 @@
 */
 
 use crate::engine::{
-    core::model::{Field, Layer, Model},
+    core::{
+        model::{Field, Layer, Model},
+        space::{Space, SpaceMeta},
+    },
     data::{
         cell::Datacell,
         dict::{DictEntryGeneric, DictGeneric},
@@ -109,3 +112,13 @@ fn model() {
     let dec = super::dec::<super::obj::ModelLayout>(&mut scanner).unwrap();
     assert_eq!(model, dec);
 }
+
+#[test]
+fn space() {
+    let uuid = Uuid::new();
+    let space = Space::new_with_uuid(Default::default(), SpaceMeta::default(), uuid);
+    let enc = super::enc::<super::obj::SpaceLayout>(&space);
+    let mut scanner = BufferedScanner::new(&enc);
+    let dec = super::dec::<super::obj::SpaceLayout>(&mut scanner).unwrap();
+    assert_eq!(space, dec);
+}

From df61b627b481861f0172a1c36e74afd37dbde912 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 21 Aug 2023 06:28:16 +0000
Subject: [PATCH 224/310] Add preliminary event impls

---
 server/src/engine/mod.rs                |  1 +
 server/src/engine/storage/mod.rs        |  2 +-
 server/src/engine/storage/v1/journal.rs |  6 ++-
 server/src/engine/storage/v1/mod.rs     | 19 +++++++-
 server/src/engine/storage/v1/tests.rs   | 16 +++++--
 server/src/engine/txn/gns.rs            | 61 +++++++++++++++++++++++++
 server/src/engine/txn/mod.rs            | 42 +++++++++++++++++
 server/src/util/os.rs                   | 18 ++++++++
 8 files changed, 158 insertions(+), 7 deletions(-)
 create mode 100644 server/src/engine/txn/gns.rs
 create mode 100644 server/src/engine/txn/mod.rs

diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 0654b6c0..7bc029f4 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -36,3 +36,4 @@ mod mem;
 mod ql;
 mod storage;
 mod sync;
+mod txn;
diff --git a/server/src/engine/storage/mod.rs b/server/src/engine/storage/mod.rs
index a42cdeb8..c656c894 100644
--- a/server/src/engine/storage/mod.rs
+++ b/server/src/engine/storage/mod.rs
@@ -29,4 +29,4 @@
 mod header;
 mod versions;
 // impls
-mod v1;
+pub mod v1;
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index 6d2864a4..583f6547 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -94,10 +94,12 @@ pub trait JournalAdapter {
     type JournalEvent;
     /// The global state, which we want to modify on decoding the event
     type GlobalState;
+    /// The transactional impl that makes use of this journal, should define it's error type
+    type Error;
     /// Encode a journal event into a blob
     fn encode(event: Self::JournalEvent) -> Box<[u8]>;
     /// Decode a journal event and apply it to the global state
-    fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> SDSSResult<()>;
+    fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> Result<(), Self::Error>;
 }
 
 #[derive(Debug)]
@@ -232,7 +234,7 @@ impl<TA: JournalAdapter, LF: RawFileIOInterface> JournalReader<TA, LF> {
         if self.evid != entry_metadata.event_id as u64 {
             // the only case when this happens is when the journal faults at runtime with a write zero (or some other error when no bytes were written)
             self.remaining_bytes += JournalEntryMetadata::SIZE as u64;
-            // move back cursor
+            // move back cursor to see if we have a recovery block
             let new_cursor = self.log_file.retrieve_cursor()? - JournalEntryMetadata::SIZE as u64;
             self.log_file.seek_from_start(new_cursor)?;
             return self.try_recover_journal_strategy_simple_reverse();
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index d65e817f..e387d907 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -36,7 +36,10 @@ mod start_stop;
 #[cfg(test)]
 mod tests;
 
-use std::io::Error as IoError;
+// re-exports
+pub use journal::JournalAdapter;
+
+use crate::util::os::SysIOError as IoError;
 
 pub type SDSSResult<T> = Result<T, SDSSError>;
 
@@ -52,7 +55,15 @@ impl SDSSErrorContext for IoError {
     }
 }
 
+impl SDSSErrorContext for std::io::Error {
+    type ExtraData = &'static str;
+    fn with_extra(self, extra: Self::ExtraData) -> SDSSError {
+        SDSSError::IoErrorExtra(self.into(), extra)
+    }
+}
+
 #[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub enum SDSSError {
     // IO errors
     /// An IO err
@@ -113,3 +124,9 @@ impl From<IoError> for SDSSError {
         Self::IoError(e)
     }
 }
+
+impl From<std::io::Error> for SDSSError {
+    fn from(e: std::io::Error) -> Self {
+        Self::IoError(e.into())
+    }
+}
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index c8c3bd60..f86b8175 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -265,9 +265,19 @@ mod tx {
         Set(usize, u8),
     }
     #[derive(Debug)]
+    pub enum TxError {
+        SDSS(SDSSError),
+    }
+    direct_from! {
+        TxError => {
+            SDSSError as SDSS
+        }
+    }
+    #[derive(Debug)]
     pub struct DatabaseTxnAdapter;
     impl JournalAdapter for DatabaseTxnAdapter {
         const RECOVERY_PLUGIN: bool = false;
+        type Error = TxError;
         type JournalEvent = TxEvent;
         type GlobalState = Database;
 
@@ -294,9 +304,9 @@ mod tx {
             ret.into_boxed_slice()
         }
 
-        fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> SDSSResult<()> {
+        fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> Result<(), TxError> {
             if payload.len() != 10 {
-                return Err(SDSSError::CorruptedFile("testtxn.log"));
+                return Err(SDSSError::CorruptedFile("testtxn.log").into());
             }
             let opcode = payload[0];
             let index = u64::from_le_bytes(util::copy_slice_to_array(&payload[1..9]));
@@ -304,7 +314,7 @@ mod tx {
             match opcode {
                 0 if index == 0 && new_value == 0 => gs.reset(),
                 1 if index < 10 && index < isize::MAX as u64 => gs.set(index as usize, new_value),
-                _ => return Err(SDSSError::JournalLogEntryCorrupted),
+                _ => return Err(SDSSError::JournalLogEntryCorrupted.into()),
             }
             Ok(())
         }
diff --git a/server/src/engine/txn/gns.rs b/server/src/engine/txn/gns.rs
new file mode 100644
index 00000000..a34abe9e
--- /dev/null
+++ b/server/src/engine/txn/gns.rs
@@ -0,0 +1,61 @@
+/*
+ * Created on Sun Aug 20 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{TransactionError, TransactionResult},
+    crate::engine::{core::GlobalNS, storage::v1::JournalAdapter},
+};
+
+/*
+    journal implementor
+*/
+
+/// the journal adapter for DDL queries on the GNS
+pub struct GNSAdapter;
+
+impl JournalAdapter for GNSAdapter {
+    const RECOVERY_PLUGIN: bool = true;
+    type JournalEvent = GNSSuperEvent;
+    type GlobalState = GlobalNS;
+    type Error = TransactionError;
+    fn encode(GNSSuperEvent(b): Self::JournalEvent) -> Box<[u8]> {
+        b
+    }
+    fn decode_and_update_state(_: &[u8], _: &Self::GlobalState) -> TransactionResult<()> {
+        todo!()
+    }
+}
+
+/*
+    Events
+    ---
+    FIXME(@ohsayan): In the current impl, we unnecessarily use an intermediary buffer which we clearly don't need to (and also makes
+    pointless allocations). We need to fix this, but with a consistent API (and preferably not something like commit_*(...) unless
+    we have absolutely no other choice)
+*/
+
+// ah that stinging buffer
+pub struct GNSSuperEvent(Box<[u8]>);
diff --git a/server/src/engine/txn/mod.rs b/server/src/engine/txn/mod.rs
new file mode 100644
index 00000000..62745ada
--- /dev/null
+++ b/server/src/engine/txn/mod.rs
@@ -0,0 +1,42 @@
+/*
+ * Created on Sun Aug 20 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+pub mod gns;
+
+use super::storage::v1::SDSSError;
+pub type TransactionResult<T> = Result<T, TransactionError>;
+
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+pub enum TransactionError {
+    SDSSError(SDSSError),
+}
+
+direct_from! {
+    TransactionError => {
+        SDSSError as SDSSError
+    }
+}
diff --git a/server/src/util/os.rs b/server/src/util/os.rs
index c379bbf0..a043da62 100644
--- a/server/src/util/os.rs
+++ b/server/src/util/os.rs
@@ -39,6 +39,24 @@ use {
     },
 };
 
+#[derive(Debug)]
+#[repr(transparent)]
+/// A wrapper around [`std`]'s I/O [Error](std::io::Error) type for simplicity with equality
+pub struct SysIOError(std::io::Error);
+
+impl From<std::io::Error> for SysIOError {
+    fn from(e: std::io::Error) -> Self {
+        Self(e)
+    }
+}
+
+#[cfg(test)]
+impl PartialEq for SysIOError {
+    fn eq(&self, other: &Self) -> bool {
+        self.0.to_string() == other.0.to_string()
+    }
+}
+
 #[cfg(unix)]
 mod unix {
     use {

From d09df81757472d33e7710c72eda382f21356cbcf Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 21 Aug 2023 15:26:22 +0000
Subject: [PATCH 225/310] Unlink MD traits from map enc/dec impls

---
 server/src/engine/storage/v1/inf/map.rs |  87 +++++++++-----------
 server/src/engine/storage/v1/inf/md.rs  |  94 +++++++++++++++++++++
 server/src/engine/storage/v1/inf/mod.rs | 103 +++++-------------------
 server/src/engine/storage/v1/inf/obj.rs |  18 +++--
 4 files changed, 160 insertions(+), 142 deletions(-)
 create mode 100644 server/src/engine/storage/v1/inf/md.rs

diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index cf9112af..65b897ee 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -26,8 +26,7 @@
 
 use {
     super::{
-        dec_md, obj::FieldMD, PersistDictEntryDscr, PersistMapSpec, PersistObjectHlIO,
-        PersistObjectMD, VecU8, VoidMetadata,
+        md::VoidMetadata, obj::FieldMD, PersistDictEntryDscr, PersistMapSpec, PersistObject, VecU8,
     },
     crate::{
         engine::{
@@ -50,7 +49,7 @@ use {
 /// This is more of a lazy hack than anything sensible. Just implement a spec and then use this wrapper for any enc/dec operations
 pub struct PersistMapImpl<M: PersistMapSpec>(PhantomData<M>);
 
-impl<M: PersistMapSpec> PersistObjectHlIO for PersistMapImpl<M>
+impl<M: PersistMapSpec> PersistObject for PersistMapImpl<M>
 where
     M::MapType: STIndex<M::Key, M::Value>,
 {
@@ -87,7 +86,7 @@ pub fn dec_dict<PM: PersistMapSpec>(scanner: &mut BufferedScanner) -> SDSSResult
 where
     PM::MapType: STIndex<PM::Key, PM::Value>,
 {
-    if !(PM::meta_dec_collection_pretest(scanner) & scanner.has_left(sizeof!(u64))) {
+    if !(PM::pretest_collection(scanner) & scanner.has_left(sizeof!(u64))) {
         return Err(SDSSError::InternalDecodeStructureCorrupted);
     }
     let size = unsafe {
@@ -95,12 +94,21 @@ where
         scanner.next_u64_le() as usize
     };
     let mut dict = PM::MapType::idx_init_cap(size);
-    while PM::meta_dec_entry_pretest(scanner) & (dict.st_len() != size) {
+    while PM::pretest_entry_metadata(scanner) & (dict.st_len() != size) {
         let md = unsafe {
             // pretest
-            dec_md::<PM::EntryMD, true>(scanner)?
+            match PM::entry_md_dec(scanner) {
+                Some(md) => md,
+                None => {
+                    if PM::ENTRYMETA_DEC_INFALLIBLE {
+                        impossible!()
+                    } else {
+                        return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+                    }
+                }
+            }
         };
-        if PM::META_VERIFY_BEFORE_DEC && !md.pretest_src_for_object_dec(scanner) {
+        if PM::META_VERIFY_BEFORE_DEC && !PM::pretest_entry_data(scanner, &md) {
             return Err(SDSSError::InternalDecodeStructureCorrupted);
         }
         let key;
@@ -164,21 +172,6 @@ impl GenericDictEntryMD {
     }
 }
 
-impl PersistObjectMD for GenericDictEntryMD {
-    const MD_DEC_INFALLIBLE: bool = true;
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u64, 2) + 1)
-    }
-    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
-        Some(Self::decode(scanner.next_chunk()))
-    }
-    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool {
-        static EXPECT_ATLEAST: [u8; 4] = [0, 1, 8, 8]; // PAD to align
-        let lbound_rem = self.klen + EXPECT_ATLEAST[cmp::min(self.dscr, 3) as usize] as usize;
-        scanner.has_left(lbound_rem) & (self.dscr <= PersistDictEntryDscr::Dict.value_u8())
-    }
-}
-
 impl PersistMapSpec for GenericDictSpec {
     type MapIter<'a> = std::collections::hash_map::Iter<'a, Box<str>, DictEntryGeneric>;
     type MapType = DictGeneric;
@@ -188,15 +181,21 @@ impl PersistMapSpec for GenericDictSpec {
     const DEC_COUPLED: bool = false;
     const ENC_COUPLED: bool = true;
     const META_VERIFY_BEFORE_DEC: bool = true;
+    const ENTRYMETA_DEC_INFALLIBLE: bool = true;
     fn _get_iter<'a>(map: &'a Self::MapType) -> Self::MapIter<'a> {
         map.iter()
     }
-    fn meta_dec_collection_pretest(_: &BufferedScanner) -> bool {
+    fn pretest_collection(_: &BufferedScanner) -> bool {
         true
     }
-    fn meta_dec_entry_pretest(scanner: &BufferedScanner) -> bool {
+    fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool {
         // we just need to see if we can decode the entry metadata
-        Self::EntryMD::pretest_src_for_metadata_dec(scanner)
+        scanner.has_left(9)
+    }
+    fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool {
+        static EXPECT_ATLEAST: [u8; 4] = [0, 1, 8, 8]; // PAD to align
+        let lbound_rem = md.klen + EXPECT_ATLEAST[cmp::min(md.dscr, 3) as usize] as usize;
+        scanner.has_left(lbound_rem) & (md.dscr <= PersistDictEntryDscr::Dict.value_u8())
     }
     fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) {
         buf.extend(key.len().u64_bytes_le());
@@ -364,44 +363,27 @@ impl FieldMapEntryMD {
     }
 }
 
-impl PersistObjectMD for FieldMapEntryMD {
-    const MD_DEC_INFALLIBLE: bool = true;
-
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u64, 3) + 1)
-    }
-
-    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool {
-        scanner.has_left(self.field_id_l as usize) // TODO(@ohsayan): we can enforce way more here such as atleast one field etc
-    }
-
-    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
-        Some(Self::new(
-            u64::from_le_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()),
-            scanner.next_byte(),
-        ))
-    }
-}
-
 impl PersistMapSpec for FieldMapSpec {
     type MapIter<'a> = crate::engine::idx::IndexSTSeqDllIterOrdKV<'a, Box<str>, Field>;
     type MapType = IndexSTSeqCns<Self::Key, Self::Value>;
     type EntryMD = FieldMapEntryMD;
     type Key = Box<str>;
     type Value = Field;
+    const ENTRYMETA_DEC_INFALLIBLE: bool = true;
     const ENC_COUPLED: bool = false;
     const DEC_COUPLED: bool = false;
     const META_VERIFY_BEFORE_DEC: bool = true;
     fn _get_iter<'a>(m: &'a Self::MapType) -> Self::MapIter<'a> {
         m.stseq_ord_kv()
     }
-    fn meta_dec_collection_pretest(_: &BufferedScanner) -> bool {
+    fn pretest_collection(_: &BufferedScanner) -> bool {
         true
     }
-    fn meta_dec_entry_pretest(scanner: &BufferedScanner) -> bool {
-        FieldMapEntryMD::pretest_src_for_metadata_dec(scanner)
+    fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u64, 3) + 1)
+    }
+    fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool {
+        scanner.has_left(md.field_id_l as usize) // TODO(@ohsayan): we can enforce way more here such as atleast one field etc
     }
     fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
         buf.extend(key.len().u64_bytes_le());
@@ -410,7 +392,12 @@ impl PersistMapSpec for FieldMapSpec {
         buf.push(val.is_nullable() as u8);
     }
     unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::EntryMD> {
-        FieldMapEntryMD::dec_md_payload(scanner)
+        Some(FieldMapEntryMD::new(
+            u64::from_le_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()),
+            scanner.next_byte(),
+        ))
     }
     fn enc_key(buf: &mut VecU8, key: &Self::Key) {
         buf.extend(key.as_bytes());
diff --git a/server/src/engine/storage/v1/inf/md.rs b/server/src/engine/storage/v1/inf/md.rs
new file mode 100644
index 00000000..fd04bec1
--- /dev/null
+++ b/server/src/engine/storage/v1/inf/md.rs
@@ -0,0 +1,94 @@
+/*
+ * Created on Mon Aug 21 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::super::{rw::BufferedScanner, SDSSError, SDSSResult};
+
+/// metadata spec for a persist map entry
+pub trait PersistObjectMD: Sized {
+    /// set to true if decode is infallible once the MD payload has been verified
+    const MD_DEC_INFALLIBLE: bool;
+    /// returns true if the current buffered source can be used to decode the metadata (self)
+    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool;
+    /// returns true if per the metadata and the current buffered source, the target object in question can be decoded
+    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool;
+    /// decode the metadata
+    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self>;
+}
+
+/// Metadata for a simple size requirement
+pub struct SimpleSizeMD<const N: usize>;
+
+impl<const N: usize> PersistObjectMD for SimpleSizeMD<N> {
+    const MD_DEC_INFALLIBLE: bool = true;
+    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(N)
+    }
+    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
+        true
+    }
+    unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
+        Some(Self)
+    }
+}
+
+/// For wrappers and other complicated metadata handling, set this to the metadata type
+pub struct VoidMetadata;
+
+impl PersistObjectMD for VoidMetadata {
+    const MD_DEC_INFALLIBLE: bool = true;
+    fn pretest_src_for_metadata_dec(_: &BufferedScanner) -> bool {
+        true
+    }
+    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
+        true
+    }
+    unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
+        Some(Self)
+    }
+}
+
+/// Decode metadata
+///
+/// ## Safety
+/// unsafe because you need to set whether you've already verified the metadata or not
+pub(super) unsafe fn dec_md<Md: PersistObjectMD, const ASSUME_PRETEST_PASS: bool>(
+    scanner: &mut BufferedScanner,
+) -> SDSSResult<Md> {
+    if ASSUME_PRETEST_PASS || Md::pretest_src_for_metadata_dec(scanner) {
+        match Md::dec_md_payload(scanner) {
+            Some(md) => Ok(md),
+            None => {
+                if Md::MD_DEC_INFALLIBLE {
+                    impossible!()
+                } else {
+                    Err(SDSSError::InternalDecodeStructureCorrupted)
+                }
+            }
+        }
+    } else {
+        Err(SDSSError::InternalDecodeStructureCorrupted)
+    }
+}
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index fe2702be..f0432d07 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -29,12 +29,14 @@
 use crate::engine::idx::STIndex;
 
 mod map;
+mod md;
 mod obj;
 // tests
 #[cfg(test)]
 mod tests;
 
 use {
+    self::md::PersistObjectMD,
     crate::engine::{
         data::{
             dict::DictEntryGeneric,
@@ -99,77 +101,6 @@ impl PersistDictEntryDscr {
     }
 }
 
-/*
-    md spec
-*/
-
-/// metadata spec for a persist map entry
-pub trait PersistObjectMD: Sized {
-    /// set to true if decode is infallible once the MD payload has been verified
-    const MD_DEC_INFALLIBLE: bool;
-    /// returns true if the current buffered source can be used to decode the metadata (self)
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool;
-    /// returns true if per the metadata and the current buffered source, the target object in question can be decoded
-    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool;
-    /// decode the metadata
-    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self>;
-}
-
-/// Metadata for a simple size requirement
-pub struct SimpleSizeMD<const N: usize>;
-
-impl<const N: usize> PersistObjectMD for SimpleSizeMD<N> {
-    const MD_DEC_INFALLIBLE: bool = true;
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(N)
-    }
-    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
-        true
-    }
-    unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
-        Some(Self)
-    }
-}
-
-/// For wrappers and other complicated metadata handling, set this to the metadata type
-pub struct VoidMetadata;
-
-impl PersistObjectMD for VoidMetadata {
-    const MD_DEC_INFALLIBLE: bool = true;
-    fn pretest_src_for_metadata_dec(_: &BufferedScanner) -> bool {
-        true
-    }
-    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
-        true
-    }
-    unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
-        Some(Self)
-    }
-}
-
-/// Decode metadata
-///
-/// ## Safety
-/// unsafe because you need to set whether you've already verified the metadata or not
-unsafe fn dec_md<Md: PersistObjectMD, const ASSUME_PRETEST_PASS: bool>(
-    scanner: &mut BufferedScanner,
-) -> SDSSResult<Md> {
-    if ASSUME_PRETEST_PASS || Md::pretest_src_for_metadata_dec(scanner) {
-        match Md::dec_md_payload(scanner) {
-            Some(md) => Ok(md),
-            None => {
-                if Md::MD_DEC_INFALLIBLE {
-                    impossible!()
-                } else {
-                    Err(SDSSError::InternalDecodeStructureCorrupted)
-                }
-            }
-        }
-    } else {
-        Err(SDSSError::InternalDecodeStructureCorrupted)
-    }
-}
-
 /*
     obj spec
 */
@@ -177,12 +108,12 @@ unsafe fn dec_md<Md: PersistObjectMD, const ASSUME_PRETEST_PASS: bool>(
 /// Specification for any object that can be persisted
 ///
 /// To actuall enc/dec any object, use functions (and their derivatives) [`enc`] and [`dec`]
-pub trait PersistObjectHlIO {
+pub trait PersistObject {
     const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool;
     /// the actual type (we can have wrappers)
     type Type;
     /// the metadata type (use this to verify the buffered source)
-    type Metadata: PersistObjectMD;
+    type Metadata: md::PersistObjectMD;
     /// enc routine
     ///
     /// METADATA: handle yourself
@@ -195,33 +126,33 @@ pub trait PersistObjectHlIO {
 }
 
 /// enc the given object into a new buffer
-pub fn enc<Obj: PersistObjectHlIO>(obj: &Obj::Type) -> VecU8 {
+pub fn enc<Obj: PersistObject>(obj: &Obj::Type) -> VecU8 {
     let mut buf = vec![];
     Obj::pe_obj_hlio_enc(&mut buf, obj);
     buf
 }
 
 /// enc the object into the given buffer
-pub fn enc_into_buf<Obj: PersistObjectHlIO>(buf: &mut VecU8, obj: &Obj::Type) {
+pub fn enc_into_buf<Obj: PersistObject>(buf: &mut VecU8, obj: &Obj::Type) {
     Obj::pe_obj_hlio_enc(buf, obj)
 }
 
 /// enc the object into the given buffer
-pub fn enc_self_into_buf<Obj: PersistObjectHlIO<Type = Obj>>(buf: &mut VecU8, obj: &Obj) {
+pub fn enc_self_into_buf<Obj: PersistObject<Type = Obj>>(buf: &mut VecU8, obj: &Obj) {
     Obj::pe_obj_hlio_enc(buf, obj)
 }
 
 /// enc the object into a new buffer
-pub fn enc_self<Obj: PersistObjectHlIO<Type = Obj>>(obj: &Obj) -> VecU8 {
+pub fn enc_self<Obj: PersistObject<Type = Obj>>(obj: &Obj) -> VecU8 {
     enc::<Obj>(obj)
 }
 
 /// dec the object
-pub fn dec<Obj: PersistObjectHlIO>(scanner: &mut BufferedScanner) -> SDSSResult<Obj::Type> {
+pub fn dec<Obj: PersistObject>(scanner: &mut BufferedScanner) -> SDSSResult<Obj::Type> {
     if Obj::Metadata::pretest_src_for_metadata_dec(scanner) {
         let md = unsafe {
             // UNSAFE(@ohsaya): pretest
-            dec_md::<Obj::Metadata, true>(scanner)?
+            md::dec_md::<Obj::Metadata, true>(scanner)?
         };
         if Obj::ALWAYS_VERIFY_PAYLOAD_USING_MD && !md.pretest_src_for_object_dec(scanner) {
             return Err(SDSSError::InternalDecodeStructureCorrupted);
@@ -233,9 +164,7 @@ pub fn dec<Obj: PersistObjectHlIO>(scanner: &mut BufferedScanner) -> SDSSResult<
 }
 
 /// dec the object
-pub fn dec_self<Obj: PersistObjectHlIO<Type = Obj>>(
-    scanner: &mut BufferedScanner,
-) -> SDSSResult<Obj> {
+pub fn dec_self<Obj: PersistObject<Type = Obj>>(scanner: &mut BufferedScanner) -> SDSSResult<Obj> {
     dec::<Obj>(scanner)
 }
 
@@ -252,7 +181,7 @@ pub trait PersistMapSpec {
     where
         Self: 'a;
     /// metadata type
-    type EntryMD: PersistObjectMD;
+    type EntryMD;
     /// key type (NOTE: set this to the true key type; handle any differences using the spec unless you have an entirely different
     /// wrapper type)
     type Key: AsKey;
@@ -264,13 +193,17 @@ pub trait PersistMapSpec {
     const DEC_COUPLED: bool;
     /// verify the src using the given metadata
     const META_VERIFY_BEFORE_DEC: bool;
+    /// set to true if the entry meta, once pretested never fails to decode
+    const ENTRYMETA_DEC_INFALLIBLE: bool;
     // collection misc
     fn _get_iter<'a>(map: &'a Self::MapType) -> Self::MapIter<'a>;
     // collection meta
     /// pretest before jmp to routine for entire collection
-    fn meta_dec_collection_pretest(scanner: &BufferedScanner) -> bool;
+    fn pretest_collection(scanner: &BufferedScanner) -> bool;
     /// pretest before jmp to entry dec routine
-    fn meta_dec_entry_pretest(scanner: &BufferedScanner) -> bool;
+    fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool;
+    /// pretest the src before jmp to entry data dec routine
+    fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool;
     // entry meta
     /// enc the entry meta
     fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 24af2954..0046fc85 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -25,7 +25,11 @@
 */
 
 use {
-    super::{dec_md, map::FieldMapSpec, PersistObjectHlIO, PersistObjectMD, SimpleSizeMD, VecU8},
+    super::{
+        map::FieldMapSpec,
+        md::{dec_md, PersistObjectMD, SimpleSizeMD},
+        PersistObject, VecU8,
+    },
     crate::{
         engine::{
             core::{
@@ -52,7 +56,7 @@ use {
 
 struct POByteBlockFullTag(FullTag);
 
-impl PersistObjectHlIO for POByteBlockFullTag {
+impl PersistObject for POByteBlockFullTag {
     const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
     type Type = FullTag;
     type Metadata = SimpleSizeMD<{ sizeof!(u64) }>;
@@ -106,7 +110,7 @@ impl PersistObjectMD for LayerMD {
     }
 }
 
-impl PersistObjectHlIO for Layer {
+impl PersistObject for Layer {
     const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
     type Type = Layer;
     type Metadata = LayerMD;
@@ -166,7 +170,7 @@ impl PersistObjectMD for FieldMD {
     }
 }
 
-impl PersistObjectHlIO for Field {
+impl PersistObject for Field {
     const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
     type Type = Self;
     type Metadata = FieldMD;
@@ -187,7 +191,7 @@ impl PersistObjectHlIO for Field {
         let mut fin = false;
         while (!scanner.eof())
             & (layers.len() as u64 != md.layer_c)
-            & (<Layer as PersistObjectHlIO>::Metadata::pretest_src_for_metadata_dec(scanner))
+            & (<Layer as PersistObject>::Metadata::pretest_src_for_metadata_dec(scanner))
             & !fin
         {
             let layer_md = unsafe {
@@ -241,7 +245,7 @@ impl PersistObjectMD for ModelLayoutMD {
     }
 }
 
-impl PersistObjectHlIO for ModelLayout {
+impl PersistObject for ModelLayout {
     const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = true;
     type Type = Model;
     type Metadata = ModelLayoutMD;
@@ -303,7 +307,7 @@ impl PersistObjectMD for SpaceLayoutMD {
     }
 }
 
-impl PersistObjectHlIO for SpaceLayout {
+impl PersistObject for SpaceLayout {
     const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false; // no need, since the MD only handles the UUID
     type Type = Space;
     type Metadata = SpaceLayoutMD;

From 79b450160ef3594e7642643882b2daa17faec937 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 22 Aug 2023 06:10:52 +0000
Subject: [PATCH 226/310] Remove metadata spec and simplify obj enc/dec

---
 server/src/engine/core/model/mod.rs       |   4 +-
 server/src/engine/storage/v1/inf/map.rs   | 184 +++++++---------
 server/src/engine/storage/v1/inf/md.rs    |  94 --------
 server/src/engine/storage/v1/inf/mod.rs   | 161 ++++++++------
 server/src/engine/storage/v1/inf/obj.rs   | 255 +++++++++++-----------
 server/src/engine/storage/v1/inf/tests.rs |  64 +++---
 6 files changed, 349 insertions(+), 413 deletions(-)
 delete mode 100644 server/src/engine/storage/v1/inf/md.rs

diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 0863d376..727e800c 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -25,7 +25,7 @@
 */
 
 pub(super) mod alt;
-mod delta;
+pub(in crate::engine) mod delta;
 
 #[cfg(test)]
 use std::cell::RefCell;
@@ -52,7 +52,7 @@ use {
 };
 
 pub(in crate::engine::core) use self::delta::{DeltaKind, DeltaState, DeltaVersion};
-pub(in crate::engine::core) type Fields = IndexSTSeqCns<Box<str>, Field>;
+pub(in crate::engine) type Fields = IndexSTSeqCns<Box<str>, Field>;
 
 #[derive(Debug)]
 pub struct Model {
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index 65b897ee..766489dc 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -25,12 +25,10 @@
 */
 
 use {
-    super::{
-        md::VoidMetadata, obj::FieldMD, PersistDictEntryDscr, PersistMapSpec, PersistObject, VecU8,
-    },
+    super::{obj::FieldMD, PersistDictEntryDscr, PersistMapSpec, PersistObject, VecU8},
     crate::{
         engine::{
-            core::model::{Field, Layer},
+            core::model::Field,
             data::{
                 cell::Datacell,
                 dict::DictEntryGeneric,
@@ -46,103 +44,94 @@ use {
     std::cmp,
 };
 
+#[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord)]
+pub struct MapIndexSizeMD(pub(super) usize);
+
 /// This is more of a lazy hack than anything sensible. Just implement a spec and then use this wrapper for any enc/dec operations
-pub struct PersistMapImpl<M: PersistMapSpec>(PhantomData<M>);
+pub struct PersistMapImpl<'a, M: PersistMapSpec>(PhantomData<&'a M::MapType>);
 
-impl<M: PersistMapSpec> PersistObject for PersistMapImpl<M>
+impl<'a, M: PersistMapSpec> PersistObject for PersistMapImpl<'a, M>
 where
-    M::MapType: STIndex<M::Key, M::Value>,
+    M::MapType: 'a + STIndex<M::Key, M::Value>,
 {
-    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
-    type Type = M::MapType;
-    type Metadata = VoidMetadata;
-    fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) {
-        enc_dict_into_buffer::<M>(buf, v)
-    }
-    unsafe fn pe_obj_hlio_dec(
-        scanner: &mut BufferedScanner,
-        _: VoidMetadata,
-    ) -> SDSSResult<Self::Type> {
-        dec_dict::<M>(scanner)
+    type InputType = &'a M::MapType;
+    type OutputType = M::MapType;
+    type Metadata = MapIndexSizeMD;
+    fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u64))
+    }
+    fn pretest_can_dec_object(
+        s: &BufferedScanner,
+        MapIndexSizeMD(dict_size): &Self::Metadata,
+    ) -> bool {
+        M::pretest_collection_using_size(s, *dict_size)
+    }
+    fn meta_enc(buf: &mut VecU8, data: Self::InputType) {
+        buf.extend(data.st_len().u64_bytes_le());
+    }
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+        Ok(MapIndexSizeMD(
+            u64::from_le_bytes(scanner.next_chunk()) as usize
+        ))
     }
-}
-
-/// Encode the dict into the given buffer
-pub fn enc_dict_into_buffer<PM: PersistMapSpec>(buf: &mut VecU8, map: &PM::MapType) {
-    buf.extend(map.st_len().u64_bytes_le());
-    for (key, val) in PM::_get_iter(map) {
-        PM::entry_md_enc(buf, key, val);
-        if PM::ENC_COUPLED {
-            PM::enc_entry(buf, key, val);
-        } else {
-            PM::enc_key(buf, key);
-            PM::enc_val(buf, val);
+    fn obj_enc(buf: &mut VecU8, map: Self::InputType) {
+        for (key, val) in M::_get_iter(map) {
+            M::entry_md_enc(buf, key, val);
+            if M::ENC_COUPLED {
+                M::enc_entry(buf, key, val);
+            } else {
+                M::enc_key(buf, key);
+                M::enc_val(buf, val);
+            }
         }
     }
-}
-
-/// Decode the dict using the given buffered scanner
-pub fn dec_dict<PM: PersistMapSpec>(scanner: &mut BufferedScanner) -> SDSSResult<PM::MapType>
-where
-    PM::MapType: STIndex<PM::Key, PM::Value>,
-{
-    if !(PM::pretest_collection(scanner) & scanner.has_left(sizeof!(u64))) {
-        return Err(SDSSError::InternalDecodeStructureCorrupted);
-    }
-    let size = unsafe {
-        // UNSAFE(@ohsayan): pretest
-        scanner.next_u64_le() as usize
-    };
-    let mut dict = PM::MapType::idx_init_cap(size);
-    while PM::pretest_entry_metadata(scanner) & (dict.st_len() != size) {
-        let md = unsafe {
-            // pretest
-            match PM::entry_md_dec(scanner) {
-                Some(md) => md,
-                None => {
-                    if PM::ENTRYMETA_DEC_INFALLIBLE {
-                        impossible!()
-                    } else {
-                        return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
-                    }
-                }
+    unsafe fn obj_dec(
+        scanner: &mut BufferedScanner,
+        MapIndexSizeMD(dict_size): Self::Metadata,
+    ) -> SDSSResult<Self::OutputType> {
+        let mut dict = M::MapType::idx_init();
+        while M::pretest_entry_metadata(scanner) & (dict.st_len() != dict_size) {
+            let md = unsafe {
+                // UNSAFE(@ohsayan): +pretest
+                M::entry_md_dec(scanner)
+                    .ok_or(SDSSError::InternalDecodeStructureCorruptedPayload)?
+            };
+            if !M::pretest_entry_data(scanner, &md) {
+                return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
             }
-        };
-        if PM::META_VERIFY_BEFORE_DEC && !PM::pretest_entry_data(scanner, &md) {
-            return Err(SDSSError::InternalDecodeStructureCorrupted);
-        }
-        let key;
-        let val;
-        unsafe {
-            if PM::DEC_COUPLED {
-                match PM::dec_entry(scanner, md) {
-                    Some((_k, _v)) => {
-                        key = _k;
-                        val = _v;
+            let key;
+            let val;
+            unsafe {
+                if M::DEC_COUPLED {
+                    match M::dec_entry(scanner, md) {
+                        Some((_k, _v)) => {
+                            key = _k;
+                            val = _v;
+                        }
+                        None => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
                     }
-                    None => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
-                }
-            } else {
-                let _k = PM::dec_key(scanner, &md);
-                let _v = PM::dec_val(scanner, &md);
-                match (_k, _v) {
-                    (Some(_k), Some(_v)) => {
-                        key = _k;
-                        val = _v;
+                } else {
+                    let _k = M::dec_key(scanner, &md);
+                    let _v = M::dec_val(scanner, &md);
+                    match (_k, _v) {
+                        (Some(_k), Some(_v)) => {
+                            key = _k;
+                            val = _v;
+                        }
+                        _ => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
                     }
-                    _ => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
                 }
             }
+            if !dict.st_insert(key, val) {
+                return Err(SDSSError::InternalDecodeStructureIllegalData);
+            }
         }
-        if !dict.st_insert(key, val) {
-            return Err(SDSSError::InternalDecodeStructureIllegalData);
+        if dict.st_len() == dict_size {
+            Ok(dict)
+        } else {
+            Err(SDSSError::InternalDecodeStructureIllegalData)
         }
     }
-    if dict.st_len() == size {
-        Ok(dict)
-    } else {
-        Err(SDSSError::InternalDecodeStructureIllegalData)
-    }
 }
 
 /// generic dict spec (simple spec for [DictGeneric](crate::engine::data::dict::DictGeneric))
@@ -180,14 +169,9 @@ impl PersistMapSpec for GenericDictSpec {
     type EntryMD = GenericDictEntryMD;
     const DEC_COUPLED: bool = false;
     const ENC_COUPLED: bool = true;
-    const META_VERIFY_BEFORE_DEC: bool = true;
-    const ENTRYMETA_DEC_INFALLIBLE: bool = true;
     fn _get_iter<'a>(map: &'a Self::MapType) -> Self::MapIter<'a> {
         map.iter()
     }
-    fn pretest_collection(_: &BufferedScanner) -> bool {
-        true
-    }
     fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool {
         // we just need to see if we can decode the entry metadata
         scanner.has_left(9)
@@ -208,7 +192,7 @@ impl PersistMapSpec for GenericDictSpec {
             DictEntryGeneric::Map(map) => {
                 buf.push(PersistDictEntryDscr::Dict.value_u8());
                 buf.extend(key.as_bytes());
-                enc_dict_into_buffer::<Self>(buf, map);
+                <PersistMapImpl<Self> as PersistObject>::default_full_enc(buf, map);
             }
             DictEntryGeneric::Data(dc) => {
                 buf.push(
@@ -319,7 +303,12 @@ impl PersistMapSpec for GenericDictSpec {
                 }
                 PersistDictEntryDscr::Dict => {
                     if dg_top_element {
-                        DictEntryGeneric::Map(dec_dict::<GenericDictSpec>(scanner).ok()?)
+                        DictEntryGeneric::Map(
+                            <PersistMapImpl<GenericDictSpec> as PersistObject>::default_full_dec(
+                                scanner,
+                            )
+                            .ok()?,
+                        )
                     } else {
                         unreachable!("found top-level dict item in datacell")
                     }
@@ -369,16 +358,11 @@ impl PersistMapSpec for FieldMapSpec {
     type EntryMD = FieldMapEntryMD;
     type Key = Box<str>;
     type Value = Field;
-    const ENTRYMETA_DEC_INFALLIBLE: bool = true;
     const ENC_COUPLED: bool = false;
     const DEC_COUPLED: bool = false;
-    const META_VERIFY_BEFORE_DEC: bool = true;
     fn _get_iter<'a>(m: &'a Self::MapType) -> Self::MapIter<'a> {
         m.stseq_ord_kv()
     }
-    fn pretest_collection(_: &BufferedScanner) -> bool {
-        true
-    }
     fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool {
         scanner.has_left(sizeof!(u64, 3) + 1)
     }
@@ -404,7 +388,7 @@ impl PersistMapSpec for FieldMapSpec {
     }
     fn enc_val(buf: &mut VecU8, val: &Self::Value) {
         for layer in val.layers() {
-            Layer::pe_obj_hlio_enc(buf, layer)
+            super::obj::LayerRef::default_full_enc(buf, super::obj::LayerRef(layer))
         }
     }
     unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Key> {
@@ -417,7 +401,7 @@ impl PersistMapSpec for FieldMapSpec {
         .ok()
     }
     unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value> {
-        Field::pe_obj_hlio_dec(
+        super::obj::FieldRef::obj_dec(
             scanner,
             FieldMD::new(md.field_prop_c, md.field_layer_c, md.null),
         )
diff --git a/server/src/engine/storage/v1/inf/md.rs b/server/src/engine/storage/v1/inf/md.rs
deleted file mode 100644
index fd04bec1..00000000
--- a/server/src/engine/storage/v1/inf/md.rs
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Created on Mon Aug 21 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use super::super::{rw::BufferedScanner, SDSSError, SDSSResult};
-
-/// metadata spec for a persist map entry
-pub trait PersistObjectMD: Sized {
-    /// set to true if decode is infallible once the MD payload has been verified
-    const MD_DEC_INFALLIBLE: bool;
-    /// returns true if the current buffered source can be used to decode the metadata (self)
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool;
-    /// returns true if per the metadata and the current buffered source, the target object in question can be decoded
-    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool;
-    /// decode the metadata
-    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self>;
-}
-
-/// Metadata for a simple size requirement
-pub struct SimpleSizeMD<const N: usize>;
-
-impl<const N: usize> PersistObjectMD for SimpleSizeMD<N> {
-    const MD_DEC_INFALLIBLE: bool = true;
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(N)
-    }
-    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
-        true
-    }
-    unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
-        Some(Self)
-    }
-}
-
-/// For wrappers and other complicated metadata handling, set this to the metadata type
-pub struct VoidMetadata;
-
-impl PersistObjectMD for VoidMetadata {
-    const MD_DEC_INFALLIBLE: bool = true;
-    fn pretest_src_for_metadata_dec(_: &BufferedScanner) -> bool {
-        true
-    }
-    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
-        true
-    }
-    unsafe fn dec_md_payload(_: &mut BufferedScanner) -> Option<Self> {
-        Some(Self)
-    }
-}
-
-/// Decode metadata
-///
-/// ## Safety
-/// unsafe because you need to set whether you've already verified the metadata or not
-pub(super) unsafe fn dec_md<Md: PersistObjectMD, const ASSUME_PRETEST_PASS: bool>(
-    scanner: &mut BufferedScanner,
-) -> SDSSResult<Md> {
-    if ASSUME_PRETEST_PASS || Md::pretest_src_for_metadata_dec(scanner) {
-        match Md::dec_md_payload(scanner) {
-            Some(md) => Ok(md),
-            None => {
-                if Md::MD_DEC_INFALLIBLE {
-                    impossible!()
-                } else {
-                    Err(SDSSError::InternalDecodeStructureCorrupted)
-                }
-            }
-        }
-    } else {
-        Err(SDSSError::InternalDecodeStructureCorrupted)
-    }
-}
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index f0432d07..a74c1b11 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -29,14 +29,12 @@
 use crate::engine::idx::STIndex;
 
 mod map;
-mod md;
 mod obj;
 // tests
 #[cfg(test)]
 mod tests;
 
 use {
-    self::md::PersistObjectMD,
     crate::engine::{
         data::{
             dict::DictEntryGeneric,
@@ -105,69 +103,63 @@ impl PersistDictEntryDscr {
     obj spec
 */
 
-/// Specification for any object that can be persisted
-///
-/// To actuall enc/dec any object, use functions (and their derivatives) [`enc`] and [`dec`]
+/// Any object that can be persisted
 pub trait PersistObject {
-    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool;
-    /// the actual type (we can have wrappers)
-    type Type;
-    /// the metadata type (use this to verify the buffered source)
-    type Metadata: md::PersistObjectMD;
-    /// enc routine
+    // types
+    /// Input type for enc operations
+    type InputType: Copy;
+    /// Output type for dec operations
+    type OutputType;
+    /// Metadata type
+    type Metadata;
+    // pretest
+    /// Pretest to see if the src has the required data for metadata dec
+    fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool;
+    /// Pretest to see if the src has the required data for object dec
+    fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool;
+    // meta
+    /// metadata enc
+    fn meta_enc(buf: &mut VecU8, data: Self::InputType);
+    /// metadata dec
     ///
-    /// METADATA: handle yourself
-    fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type);
-    /// dec routine
-    unsafe fn pe_obj_hlio_dec(
-        scanner: &mut BufferedScanner,
-        md: Self::Metadata,
-    ) -> SDSSResult<Self::Type>;
-}
-
-/// enc the given object into a new buffer
-pub fn enc<Obj: PersistObject>(obj: &Obj::Type) -> VecU8 {
-    let mut buf = vec![];
-    Obj::pe_obj_hlio_enc(&mut buf, obj);
-    buf
-}
-
-/// enc the object into the given buffer
-pub fn enc_into_buf<Obj: PersistObject>(buf: &mut VecU8, obj: &Obj::Type) {
-    Obj::pe_obj_hlio_enc(buf, obj)
-}
-
-/// enc the object into the given buffer
-pub fn enc_self_into_buf<Obj: PersistObject<Type = Obj>>(buf: &mut VecU8, obj: &Obj) {
-    Obj::pe_obj_hlio_enc(buf, obj)
-}
-
-/// enc the object into a new buffer
-pub fn enc_self<Obj: PersistObject<Type = Obj>>(obj: &Obj) -> VecU8 {
-    enc::<Obj>(obj)
-}
-
-/// dec the object
-pub fn dec<Obj: PersistObject>(scanner: &mut BufferedScanner) -> SDSSResult<Obj::Type> {
-    if Obj::Metadata::pretest_src_for_metadata_dec(scanner) {
+    /// ## Safety
+    ///
+    /// Must pass the [`PersistObject::pretest_can_dec_metadata`] assertion
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata>;
+    // obj
+    /// obj enc
+    fn obj_enc(buf: &mut VecU8, data: Self::InputType);
+    /// obj dec
+    ///
+    /// ## Safety
+    ///
+    /// Must pass the [`PersistObject::pretest_can_dec_object`] assertion
+    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType>;
+    // default
+    /// Default routine to encode an object + its metadata
+    fn default_full_enc(buf: &mut VecU8, data: Self::InputType) {
+        Self::meta_enc(buf, data);
+        Self::obj_enc(buf, data);
+    }
+    /// Default routine to decode an object + its metadata (however, the metadata is used and not returned)
+    fn default_full_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::OutputType> {
+        if !Self::pretest_can_dec_metadata(scanner) {
+            return Err(SDSSError::InternalDecodeStructureCorrupted);
+        }
         let md = unsafe {
-            // UNSAFE(@ohsaya): pretest
-            md::dec_md::<Obj::Metadata, true>(scanner)?
+            // UNSAFE(@ohsayan): +pretest
+            Self::meta_dec(scanner)?
         };
-        if Obj::ALWAYS_VERIFY_PAYLOAD_USING_MD && !md.pretest_src_for_object_dec(scanner) {
-            return Err(SDSSError::InternalDecodeStructureCorrupted);
+        if !Self::pretest_can_dec_object(scanner, &md) {
+            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): +obj pretest
+            Self::obj_dec(scanner, md)
         }
-        unsafe { Obj::pe_obj_hlio_dec(scanner, md) }
-    } else {
-        Err(SDSSError::InternalDecodeStructureCorrupted)
     }
 }
 
-/// dec the object
-pub fn dec_self<Obj: PersistObject<Type = Obj>>(scanner: &mut BufferedScanner) -> SDSSResult<Obj> {
-    dec::<Obj>(scanner)
-}
-
 /*
     map spec
 */
@@ -191,15 +183,13 @@ pub trait PersistMapSpec {
     const ENC_COUPLED: bool;
     /// coupled dec
     const DEC_COUPLED: bool;
-    /// verify the src using the given metadata
-    const META_VERIFY_BEFORE_DEC: bool;
-    /// set to true if the entry meta, once pretested never fails to decode
-    const ENTRYMETA_DEC_INFALLIBLE: bool;
     // collection misc
     fn _get_iter<'a>(map: &'a Self::MapType) -> Self::MapIter<'a>;
     // collection meta
     /// pretest before jmp to routine for entire collection
-    fn pretest_collection(scanner: &BufferedScanner) -> bool;
+    fn pretest_collection_using_size(_: &BufferedScanner, _: usize) -> bool {
+        true
+    }
     /// pretest before jmp to entry dec routine
     fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool;
     /// pretest the src before jmp to entry data dec routine
@@ -228,3 +218,50 @@ pub trait PersistMapSpec {
         md: Self::EntryMD,
     ) -> Option<(Self::Key, Self::Value)>;
 }
+
+// enc
+pub mod enc {
+    use super::{map, PersistMapSpec, PersistObject, VecU8};
+    pub fn enc_full<Obj: PersistObject>(obj: Obj::InputType) -> Vec<u8> {
+        let mut v = vec![];
+        enc_full_into_buffer::<Obj>(&mut v, obj);
+        v
+    }
+    pub fn enc_full_into_buffer<Obj: PersistObject>(buf: &mut VecU8, obj: Obj::InputType) {
+        Obj::default_full_enc(buf, obj)
+    }
+    pub fn enc_dict_full<PM: PersistMapSpec>(dict: &PM::MapType) -> Vec<u8> {
+        let mut v = vec![];
+        enc_dict_full_into_buffer::<PM>(&mut v, dict);
+        v
+    }
+    pub fn enc_dict_full_into_buffer<PM: PersistMapSpec>(buf: &mut VecU8, dict: &PM::MapType) {
+        <map::PersistMapImpl<PM> as PersistObject>::default_full_enc(buf, dict)
+    }
+}
+
+// dec
+pub mod dec {
+    use {
+        super::{map, PersistMapSpec, PersistObject},
+        crate::engine::storage::v1::{rw::BufferedScanner, SDSSResult},
+    };
+    pub fn dec_full<Obj: PersistObject>(data: &[u8]) -> SDSSResult<Obj::OutputType> {
+        let mut scanner = BufferedScanner::new(data);
+        dec_full_from_scanner::<Obj>(&mut scanner)
+    }
+    pub fn dec_full_from_scanner<Obj: PersistObject>(
+        scanner: &mut BufferedScanner,
+    ) -> SDSSResult<Obj::OutputType> {
+        Obj::default_full_dec(scanner)
+    }
+    pub fn dec_dict_full<PM: PersistMapSpec>(data: &[u8]) -> SDSSResult<PM::MapType> {
+        let mut scanner = BufferedScanner::new(data);
+        dec_dict_full_from_scanner::<PM>(&mut scanner)
+    }
+    fn dec_dict_full_from_scanner<PM: PersistMapSpec>(
+        scanner: &mut BufferedScanner,
+    ) -> SDSSResult<PM::MapType> {
+        <map::PersistMapImpl<PM> as PersistObject>::default_full_dec(scanner)
+    }
+}
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 0046fc85..34e7e7c0 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -24,12 +24,10 @@
  *
 */
 
+use crate::engine::{core::model::delta::IRModel, data::DictGeneric};
+
 use {
-    super::{
-        map::FieldMapSpec,
-        md::{dec_md, PersistObjectMD, SimpleSizeMD},
-        PersistObject, VecU8,
-    },
+    super::{PersistObject, VecU8},
     crate::{
         engine::{
             core::{
@@ -37,7 +35,7 @@ use {
                 space::{Space, SpaceMeta},
             },
             data::{
-                tag::{DataTag, FullTag, TagClass, TagSelector},
+                tag::{DataTag, TagClass, TagSelector},
                 uuid::Uuid,
             },
             mem::VInline,
@@ -54,27 +52,6 @@ use {
     to be the correct size.)
 */
 
-struct POByteBlockFullTag(FullTag);
-
-impl PersistObject for POByteBlockFullTag {
-    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
-    type Type = FullTag;
-    type Metadata = SimpleSizeMD<{ sizeof!(u64) }>;
-    fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
-        buf.extend(slf.tag_selector().d().u64_bytes_le())
-    }
-    unsafe fn pe_obj_hlio_dec(
-        scanner: &mut BufferedScanner,
-        _: Self::Metadata,
-    ) -> SDSSResult<FullTag> {
-        let dscr = scanner.next_u64_le();
-        if dscr > TagSelector::max_dscr() as u64 {
-            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
-        }
-        Ok(TagSelector::from_raw(dscr as u8).into_full())
-    }
-}
-
 /*
     layer
 */
@@ -94,35 +71,35 @@ impl LayerMD {
     }
 }
 
-impl PersistObjectMD for LayerMD {
-    const MD_DEC_INFALLIBLE: bool = true;
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
+#[derive(Clone, Copy)]
+pub struct LayerRef<'a>(pub &'a Layer);
+impl<'a> From<&'a Layer> for LayerRef<'a> {
+    fn from(value: &'a Layer) -> Self {
+        Self(value)
+    }
+}
+impl<'a> PersistObject for LayerRef<'a> {
+    type InputType = LayerRef<'a>;
+    type OutputType = Layer;
+    type Metadata = LayerMD;
+    fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
         scanner.has_left(sizeof!(u64, 2))
     }
-    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
+    fn pretest_can_dec_object(_: &BufferedScanner, _: &Self::Metadata) -> bool {
         true
     }
-    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
-        Some(Self::new(
+    fn meta_enc(buf: &mut VecU8, LayerRef(layer): Self::InputType) {
+        buf.extend(layer.tag().tag_selector().d().u64_bytes_le());
+        buf.extend(0u64.to_le_bytes());
+    }
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+        Ok(LayerMD::new(
             u64::from_le_bytes(scanner.next_chunk()),
             u64::from_le_bytes(scanner.next_chunk()),
         ))
     }
-}
-
-impl PersistObject for Layer {
-    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
-    type Type = Layer;
-    type Metadata = LayerMD;
-    fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
-        // [8B: type sig][8B: empty property set]
-        POByteBlockFullTag::pe_obj_hlio_enc(buf, &slf.tag());
-        buf.extend(0u64.to_le_bytes());
-    }
-    unsafe fn pe_obj_hlio_dec(
-        _: &mut BufferedScanner,
-        md: Self::Metadata,
-    ) -> SDSSResult<Self::Type> {
+    fn obj_enc(_: &mut VecU8, _: Self::InputType) {}
+    unsafe fn obj_dec(_: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
         if (md.type_selector > TagSelector::List.d() as u64) | (md.prop_set_arity != 0) {
             return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
         }
@@ -152,53 +129,56 @@ impl FieldMD {
     }
 }
 
-impl PersistObjectMD for FieldMD {
-    const MD_DEC_INFALLIBLE: bool = true;
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
+pub struct FieldRef<'a>(&'a Field);
+impl<'a> From<&'a Field> for FieldRef<'a> {
+    fn from(f: &'a Field) -> Self {
+        Self(f)
+    }
+}
+impl<'a> PersistObject for FieldRef<'a> {
+    type InputType = &'a Field;
+    type OutputType = Field;
+    type Metadata = FieldMD;
+    fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
         scanner.has_left(sizeof!(u64, 2) + 1)
     }
-    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
-        // nothing here really; we can't help much with the stuff ahead
+    fn pretest_can_dec_object(_: &BufferedScanner, _: &Self::Metadata) -> bool {
         true
     }
-    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
-        Some(Self::new(
+    fn meta_enc(buf: &mut VecU8, slf: Self::InputType) {
+        // [prop_c][layer_c][null]
+        buf.extend(0u64.to_le_bytes());
+        buf.extend(slf.layers().len().u64_bytes_le());
+        buf.push(slf.is_nullable() as u8);
+    }
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+        Ok(FieldMD::new(
             u64::from_le_bytes(scanner.next_chunk()),
             u64::from_le_bytes(scanner.next_chunk()),
             scanner.next_byte(),
         ))
     }
-}
-
-impl PersistObject for Field {
-    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false;
-    type Type = Self;
-    type Metadata = FieldMD;
-    fn pe_obj_hlio_enc(buf: &mut VecU8, slf: &Self::Type) {
-        // [prop_c][layer_c][null]
-        buf.extend(0u64.to_le_bytes());
-        buf.extend(slf.layers().len().u64_bytes_le());
-        buf.push(slf.is_nullable() as u8);
+    fn obj_enc(buf: &mut VecU8, slf: Self::InputType) {
         for layer in slf.layers() {
-            Layer::pe_obj_hlio_enc(buf, layer);
+            LayerRef::default_full_enc(buf, LayerRef(layer));
         }
     }
-    unsafe fn pe_obj_hlio_dec(
+    unsafe fn obj_dec(
         scanner: &mut BufferedScanner,
         md: Self::Metadata,
-    ) -> SDSSResult<Self::Type> {
+    ) -> SDSSResult<Self::OutputType> {
         let mut layers = VInline::new();
         let mut fin = false;
         while (!scanner.eof())
             & (layers.len() as u64 != md.layer_c)
-            & (<Layer as PersistObject>::Metadata::pretest_src_for_metadata_dec(scanner))
+            & (LayerRef::pretest_can_dec_metadata(scanner))
             & !fin
         {
             let layer_md = unsafe {
                 // UNSAFE(@ohsayan): pretest
-                dec_md::<_, true>(scanner)?
+                LayerRef::meta_dec(scanner)?
             };
-            let l = Layer::pe_obj_hlio_dec(scanner, layer_md)?;
+            let l = LayerRef::obj_dec(scanner, layer_md)?;
             fin = l.tag().tag_class() != TagClass::List;
             layers.push(l);
         }
@@ -216,58 +196,78 @@ pub struct ModelLayoutMD {
     model_uuid: Uuid,
     p_key_len: u64,
     p_key_tag: u64,
+    field_c: u64,
 }
 
 impl ModelLayoutMD {
-    pub(super) const fn new(model_uuid: Uuid, p_key_len: u64, p_key_tag: u64) -> Self {
+    pub(super) const fn new(
+        model_uuid: Uuid,
+        p_key_len: u64,
+        p_key_tag: u64,
+        field_c: u64,
+    ) -> Self {
         Self {
             model_uuid,
             p_key_len,
             p_key_tag,
+            field_c,
         }
     }
 }
 
-impl PersistObjectMD for ModelLayoutMD {
-    const MD_DEC_INFALLIBLE: bool = true;
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u64, 3) + sizeof!(u128)) // u64,3 since the fieldmap len is also there, but we don't handle it directly
+#[derive(Clone, Copy)]
+pub struct ModelLayoutRef<'a>(pub(super) &'a Model, pub(super) &'a IRModel<'a>);
+impl<'a> From<(&'a Model, &'a IRModel<'a>)> for ModelLayoutRef<'a> {
+    fn from((mdl, irm): (&'a Model, &'a IRModel<'a>)) -> Self {
+        Self(mdl, irm)
+    }
+}
+impl<'a> PersistObject for ModelLayoutRef<'a> {
+    type InputType = ModelLayoutRef<'a>;
+    type OutputType = Model;
+    type Metadata = ModelLayoutMD;
+    fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u128) + sizeof!(u64, 3))
     }
-    fn pretest_src_for_object_dec(&self, scanner: &BufferedScanner) -> bool {
-        scanner.has_left(self.p_key_len as usize)
+    fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
+        scanner.has_left(md.p_key_len as usize)
+    }
+    fn meta_enc(buf: &mut VecU8, ModelLayoutRef(v, irm): Self::InputType) {
+        buf.extend(v.get_uuid().to_le_bytes());
+        buf.extend(v.p_key().len().u64_bytes_le());
+        buf.extend(v.p_tag().tag_selector().d().u64_bytes_le());
+        buf.extend(irm.fields().len().u64_bytes_le());
     }
-    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
-        Some(Self::new(
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+        Ok(ModelLayoutMD::new(
             Uuid::from_bytes(scanner.next_chunk()),
             u64::from_le_bytes(scanner.next_chunk()),
             u64::from_le_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()),
         ))
     }
-}
-
-impl PersistObject for ModelLayout {
-    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = true;
-    type Type = Model;
-    type Metadata = ModelLayoutMD;
-    fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) {
-        let irm = v.intent_read_model();
-        buf.extend(v.get_uuid().to_le_bytes());
-        buf.extend(v.p_key().len().u64_bytes_le());
-        buf.extend(v.p_tag().tag_selector().d().u64_bytes_le());
-        buf.extend(v.p_key().as_bytes());
-        super::map::enc_dict_into_buffer::<FieldMapSpec>(buf, irm.fields())
+    fn obj_enc(buf: &mut VecU8, ModelLayoutRef(mdl, irm): Self::InputType) {
+        buf.extend(mdl.p_key().as_bytes());
+        <super::map::PersistMapImpl<super::map::FieldMapSpec> as PersistObject>::obj_enc(
+            buf,
+            irm.fields(),
+        )
     }
-    unsafe fn pe_obj_hlio_dec(
+    unsafe fn obj_dec(
         scanner: &mut BufferedScanner,
         md: Self::Metadata,
-    ) -> SDSSResult<Self::Type> {
+    ) -> SDSSResult<Self::OutputType> {
         let key = String::from_utf8(
             scanner
                 .next_chunk_variable(md.p_key_len as usize)
                 .to_owned(),
         )
         .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)?;
-        let fieldmap = super::map::dec_dict::<FieldMapSpec>(scanner)?;
+        let fieldmap =
+            <super::map::PersistMapImpl<super::map::FieldMapSpec> as PersistObject>::obj_dec(
+                scanner,
+                super::map::MapIndexSizeMD(md.field_c as usize),
+            )?;
         let ptag = if md.p_key_tag > TagSelector::max_dscr() as u64 {
             return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
         } else {
@@ -285,45 +285,56 @@ impl PersistObject for ModelLayout {
 pub struct SpaceLayout;
 pub struct SpaceLayoutMD {
     uuid: Uuid,
+    prop_c: usize,
 }
 
 impl SpaceLayoutMD {
-    pub fn new(uuid: Uuid) -> Self {
-        Self { uuid }
+    pub fn new(uuid: Uuid, prop_c: usize) -> Self {
+        Self { uuid, prop_c }
     }
 }
 
-impl PersistObjectMD for SpaceLayoutMD {
-    const MD_DEC_INFALLIBLE: bool = true;
-
-    fn pretest_src_for_metadata_dec(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u128) + sizeof!(u64)) // u64 for props dict; we don't handle that directly
+#[derive(Clone, Copy)]
+pub struct SpaceLayoutRef<'a>(&'a Space, &'a DictGeneric);
+impl<'a> From<(&'a Space, &'a DictGeneric)> for SpaceLayoutRef<'a> {
+    fn from((spc, spc_meta): (&'a Space, &'a DictGeneric)) -> Self {
+        Self(spc, spc_meta)
     }
-    fn pretest_src_for_object_dec(&self, _: &BufferedScanner) -> bool {
+}
+impl<'a> PersistObject for SpaceLayoutRef<'a> {
+    type InputType = SpaceLayoutRef<'a>;
+    type OutputType = Space;
+    type Metadata = SpaceLayoutMD;
+    fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u128) + sizeof!(u64)) // u64 for props dict
+    }
+    fn pretest_can_dec_object(_: &BufferedScanner, _: &Self::Metadata) -> bool {
         true
     }
-    unsafe fn dec_md_payload(scanner: &mut BufferedScanner) -> Option<Self> {
-        Some(Self::new(Uuid::from_bytes(scanner.next_chunk())))
+    fn meta_enc(buf: &mut VecU8, SpaceLayoutRef(space, space_meta): Self::InputType) {
+        buf.extend(space.get_uuid().to_le_bytes());
+        buf.extend(space_meta.len().u64_bytes_le());
     }
-}
-
-impl PersistObject for SpaceLayout {
-    const ALWAYS_VERIFY_PAYLOAD_USING_MD: bool = false; // no need, since the MD only handles the UUID
-    type Type = Space;
-    type Metadata = SpaceLayoutMD;
-    fn pe_obj_hlio_enc(buf: &mut VecU8, v: &Self::Type) {
-        buf.extend(v.get_uuid().to_le_bytes());
-        super::enc_into_buf::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(
-            buf,
-            &v.metadata().env().read(),
-        );
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+        Ok(SpaceLayoutMD::new(
+            Uuid::from_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()) as usize,
+        ))
+    }
+    fn obj_enc(buf: &mut VecU8, SpaceLayoutRef(_, space_meta): Self::InputType) {
+        <super::map::PersistMapImpl<super::map::GenericDictSpec> as PersistObject>::obj_enc(
+            buf, space_meta,
+        )
     }
-    unsafe fn pe_obj_hlio_dec(
+    unsafe fn obj_dec(
         scanner: &mut BufferedScanner,
         md: Self::Metadata,
-    ) -> SDSSResult<Self::Type> {
+    ) -> SDSSResult<Self::OutputType> {
         let space_meta =
-            super::dec::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(scanner)?;
+            <super::map::PersistMapImpl<super::map::GenericDictSpec> as PersistObject>::obj_dec(
+                scanner,
+                super::map::MapIndexSizeMD(md.prop_c),
+            )?;
         Ok(Space::new_restore_empty(
             SpaceMeta::with_env(space_meta),
             md.uuid,
diff --git a/server/src/engine/storage/v1/inf/tests.rs b/server/src/engine/storage/v1/inf/tests.rs
index bf7128c7..87752e18 100644
--- a/server/src/engine/storage/v1/inf/tests.rs
+++ b/server/src/engine/storage/v1/inf/tests.rs
@@ -24,19 +24,21 @@
  *
 */
 
-use crate::engine::{
-    core::{
-        model::{Field, Layer, Model},
-        space::{Space, SpaceMeta},
-    },
-    data::{
-        cell::Datacell,
-        dict::{DictEntryGeneric, DictGeneric},
-        tag::TagSelector,
-        uuid::Uuid,
+use {
+    super::obj,
+    crate::engine::{
+        core::{
+            model::{Field, Layer, Model},
+            space::{Space, SpaceMeta},
+        },
+        data::{
+            cell::Datacell,
+            dict::{DictEntryGeneric, DictGeneric},
+            tag::TagSelector,
+            uuid::Uuid,
+        },
+        idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
     },
-    idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
-    storage::v1::rw::BufferedScanner,
 };
 
 #[test]
@@ -49,29 +51,24 @@ fn dict() {
             "and a null" => Datacell::null(),
         ))
     };
-    let encoded = super::enc::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(&dict);
-    let mut scanner = BufferedScanner::new(&encoded);
-    let decoded =
-        super::dec::<super::map::PersistMapImpl<super::map::GenericDictSpec>>(&mut scanner)
-            .unwrap();
+    let encoded = super::enc::enc_dict_full::<super::map::GenericDictSpec>(&dict);
+    let decoded = super::dec::dec_dict_full::<super::map::GenericDictSpec>(&encoded).unwrap();
     assert_eq!(dict, decoded);
 }
 
 #[test]
 fn layer() {
     let layer = Layer::list();
-    let encoded = super::enc_self(&layer);
-    let mut scanner = BufferedScanner::new(&encoded);
-    let dec = super::dec_self::<Layer>(&mut scanner).unwrap();
+    let encoded = super::enc::enc_full::<obj::LayerRef>(obj::LayerRef(&layer));
+    let dec = super::dec::dec_full::<obj::LayerRef>(&encoded).unwrap();
     assert_eq!(layer, dec);
 }
 
 #[test]
 fn field() {
     let field = Field::new([Layer::list(), Layer::uint64()].into(), true);
-    let encoded = super::enc_self(&field);
-    let mut scanner = BufferedScanner::new(&encoded);
-    let dec = super::dec_self::<Field>(&mut scanner).unwrap();
+    let encoded = super::enc::enc_full::<obj::FieldRef>((&field).into());
+    let dec = super::dec::dec_full::<obj::FieldRef>(&encoded).unwrap();
     assert_eq!(field, dec);
 }
 
@@ -83,10 +80,8 @@ fn fieldmap() {
         "profile_pic".into(),
         Field::new([Layer::bin()].into(), true),
     );
-    let enc = super::enc::<super::map::PersistMapImpl<super::map::FieldMapSpec>>(&fields);
-    let mut scanner = BufferedScanner::new(&enc);
-    let dec =
-        super::dec::<super::map::PersistMapImpl<super::map::FieldMapSpec>>(&mut scanner).unwrap();
+    let enc = super::enc::enc_dict_full::<super::map::FieldMapSpec>(&fields);
+    let dec = super::dec::dec_dict_full::<super::map::FieldMapSpec>(&enc).unwrap();
     for ((orig_field_id, orig_field), (restored_field_id, restored_field)) in
         fields.stseq_ord_kv().zip(dec.stseq_ord_kv())
     {
@@ -107,9 +102,9 @@ fn model() {
             "profile_pic" => Field::new([Layer::bin()].into(), true),
         },
     );
-    let enc = super::enc::<super::obj::ModelLayout>(&model);
-    let mut scanner = BufferedScanner::new(&enc);
-    let dec = super::dec::<super::obj::ModelLayout>(&mut scanner).unwrap();
+    let model_irm = model.intent_read_model();
+    let enc = super::enc::enc_full::<obj::ModelLayoutRef>(obj::ModelLayoutRef(&model, &model_irm));
+    let dec = super::dec::dec_full::<obj::ModelLayoutRef>(&enc).unwrap();
     assert_eq!(model, dec);
 }
 
@@ -117,8 +112,11 @@ fn model() {
 fn space() {
     let uuid = Uuid::new();
     let space = Space::new_with_uuid(Default::default(), SpaceMeta::default(), uuid);
-    let enc = super::enc::<super::obj::SpaceLayout>(&space);
-    let mut scanner = BufferedScanner::new(&enc);
-    let dec = super::dec::<super::obj::SpaceLayout>(&mut scanner).unwrap();
+    let space_meta_read = space.metadata().env().read();
+    let enc = super::enc::enc_full::<obj::SpaceLayoutRef>(obj::SpaceLayoutRef::from((
+        &space,
+        &*space_meta_read,
+    )));
+    let dec = super::dec::dec_full::<obj::SpaceLayoutRef>(&enc).unwrap();
     assert_eq!(space, dec);
 }

From 7eff973b9e7a48f985943b6b1234484d13c6d741 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 23 Aug 2023 00:10:53 +0000
Subject: [PATCH 227/310] Add `create space` txn impl

---
 server/src/engine/storage/v1/inf/map.rs |  4 +-
 server/src/engine/storage/v1/inf/mod.rs | 15 ++--
 server/src/engine/storage/v1/inf/obj.rs | 16 ++---
 server/src/engine/storage/v1/mod.rs     |  7 +-
 server/src/engine/txn/gns.rs            | 92 ++++++++++++++++++++++++-
 5 files changed, 111 insertions(+), 23 deletions(-)

diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index 766489dc..5ea9e05e 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -54,12 +54,10 @@ impl<'a, M: PersistMapSpec> PersistObject for PersistMapImpl<'a, M>
 where
     M::MapType: 'a + STIndex<M::Key, M::Value>,
 {
+    const METADATA_SIZE: usize = sizeof!(u64);
     type InputType = &'a M::MapType;
     type OutputType = M::MapType;
     type Metadata = MapIndexSizeMD;
-    fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u64))
-    }
     fn pretest_can_dec_object(
         s: &BufferedScanner,
         MapIndexSizeMD(dict_size): &Self::Metadata,
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index a74c1b11..c7f874e0 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -28,8 +28,8 @@
 
 use crate::engine::idx::STIndex;
 
-mod map;
-mod obj;
+pub mod map;
+pub mod obj;
 // tests
 #[cfg(test)]
 mod tests;
@@ -105,6 +105,9 @@ impl PersistDictEntryDscr {
 
 /// Any object that can be persisted
 pub trait PersistObject {
+    // const
+    /// Size of the metadata region
+    const METADATA_SIZE: usize;
     // types
     /// Input type for enc operations
     type InputType: Copy;
@@ -113,8 +116,10 @@ pub trait PersistObject {
     /// Metadata type
     type Metadata;
     // pretest
-    /// Pretest to see if the src has the required data for metadata dec
-    fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool;
+    /// Pretest to see if the src has the required data for metadata dec. Defaults to the metadata size
+    fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(Self::METADATA_SIZE)
+    }
     /// Pretest to see if the src has the required data for object dec
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool;
     // meta
@@ -222,6 +227,7 @@ pub trait PersistMapSpec {
 // enc
 pub mod enc {
     use super::{map, PersistMapSpec, PersistObject, VecU8};
+    // obj
     pub fn enc_full<Obj: PersistObject>(obj: Obj::InputType) -> Vec<u8> {
         let mut v = vec![];
         enc_full_into_buffer::<Obj>(&mut v, obj);
@@ -230,6 +236,7 @@ pub mod enc {
     pub fn enc_full_into_buffer<Obj: PersistObject>(buf: &mut VecU8, obj: Obj::InputType) {
         Obj::default_full_enc(buf, obj)
     }
+    // dict
     pub fn enc_dict_full<PM: PersistMapSpec>(dict: &PM::MapType) -> Vec<u8> {
         let mut v = vec![];
         enc_dict_full_into_buffer::<PM>(&mut v, dict);
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 34e7e7c0..fb857f9b 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -79,12 +79,10 @@ impl<'a> From<&'a Layer> for LayerRef<'a> {
     }
 }
 impl<'a> PersistObject for LayerRef<'a> {
+    const METADATA_SIZE: usize = sizeof!(u64, 2);
     type InputType = LayerRef<'a>;
     type OutputType = Layer;
     type Metadata = LayerMD;
-    fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u64, 2))
-    }
     fn pretest_can_dec_object(_: &BufferedScanner, _: &Self::Metadata) -> bool {
         true
     }
@@ -136,12 +134,10 @@ impl<'a> From<&'a Field> for FieldRef<'a> {
     }
 }
 impl<'a> PersistObject for FieldRef<'a> {
+    const METADATA_SIZE: usize = sizeof!(u64, 2) + 1;
     type InputType = &'a Field;
     type OutputType = Field;
     type Metadata = FieldMD;
-    fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u64, 2) + 1)
-    }
     fn pretest_can_dec_object(_: &BufferedScanner, _: &Self::Metadata) -> bool {
         true
     }
@@ -223,12 +219,10 @@ impl<'a> From<(&'a Model, &'a IRModel<'a>)> for ModelLayoutRef<'a> {
     }
 }
 impl<'a> PersistObject for ModelLayoutRef<'a> {
+    const METADATA_SIZE: usize = sizeof!(u128) + sizeof!(u64, 3);
     type InputType = ModelLayoutRef<'a>;
     type OutputType = Model;
     type Metadata = ModelLayoutMD;
-    fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u128) + sizeof!(u64, 3))
-    }
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
         scanner.has_left(md.p_key_len as usize)
     }
@@ -302,12 +296,10 @@ impl<'a> From<(&'a Space, &'a DictGeneric)> for SpaceLayoutRef<'a> {
     }
 }
 impl<'a> PersistObject for SpaceLayoutRef<'a> {
+    const METADATA_SIZE: usize = sizeof!(u128) + sizeof!(u64);
     type InputType = SpaceLayoutRef<'a>;
     type OutputType = Space;
     type Metadata = SpaceLayoutMD;
-    fn pretest_can_dec_metadata(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u128) + sizeof!(u64)) // u64 for props dict
-    }
     fn pretest_can_dec_object(_: &BufferedScanner, _: &Self::Metadata) -> bool {
         true
     }
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index e387d907..e8c51f5b 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -30,14 +30,17 @@ mod header_impl;
 mod journal;
 mod rw;
 // hl
-mod inf;
+pub mod inf;
 mod start_stop;
 // test
 #[cfg(test)]
 mod tests;
 
 // re-exports
-pub use journal::JournalAdapter;
+pub use {
+    journal::{open_journal, JournalAdapter, JournalWriter},
+    rw::BufferedScanner,
+};
 
 use crate::util::os::SysIOError as IoError;
 
diff --git a/server/src/engine/txn/gns.rs b/server/src/engine/txn/gns.rs
index a34abe9e..ce0d1cd8 100644
--- a/server/src/engine/txn/gns.rs
+++ b/server/src/engine/txn/gns.rs
@@ -26,7 +26,18 @@
 
 use {
     super::{TransactionError, TransactionResult},
-    crate::engine::{core::GlobalNS, storage::v1::JournalAdapter},
+    crate::{
+        engine::{
+            core::{space::Space, GlobalNS},
+            data::DictGeneric,
+            storage::v1::{
+                inf::{obj, PersistObject},
+                JournalAdapter, SDSSError,
+            },
+        },
+        util::EndianQW,
+    },
+    std::marker::PhantomData,
 };
 
 /*
@@ -55,7 +66,84 @@ impl JournalAdapter for GNSAdapter {
     FIXME(@ohsayan): In the current impl, we unnecessarily use an intermediary buffer which we clearly don't need to (and also makes
     pointless allocations). We need to fix this, but with a consistent API (and preferably not something like commit_*(...) unless
     we have absolutely no other choice)
+    ---
+    [OPC:2B][PAYLOAD]
 */
 
-// ah that stinging buffer
 pub struct GNSSuperEvent(Box<[u8]>);
+
+pub trait GNSEvent: PersistObject {
+    const OPC: u16;
+    type InputItem;
+}
+
+/*
+    create space
+*/
+
+pub struct CreateSpaceTxn<'a>(PhantomData<&'a ()>);
+#[derive(Clone, Copy)]
+pub struct CreateSpaceTxnCommitPL<'a> {
+    space_meta: &'a DictGeneric,
+    space_name: &'a str,
+    space: &'a Space,
+}
+pub struct CreateSpaceTxnRestorePL {
+    space_name: Box<str>,
+    space: Space,
+}
+pub struct CreateSpaceTxnMD {
+    space_name_l: u64,
+    space_meta: <obj::SpaceLayoutRef<'static> as PersistObject>::Metadata,
+}
+
+impl<'a> GNSEvent for CreateSpaceTxn<'a> {
+    const OPC: u16 = 0;
+    type InputItem = CreateSpaceTxnCommitPL<'a>;
+}
+
+impl<'a> PersistObject for CreateSpaceTxn<'a> {
+    const METADATA_SIZE: usize =
+        <obj::SpaceLayoutRef<'static> as PersistObject>::METADATA_SIZE + sizeof!(u64);
+    type InputType = CreateSpaceTxnCommitPL<'a>;
+    type OutputType = CreateSpaceTxnRestorePL;
+    type Metadata = CreateSpaceTxnMD;
+    fn pretest_can_dec_object(
+        scanner: &crate::engine::storage::v1::BufferedScanner,
+        md: &Self::Metadata,
+    ) -> bool {
+        scanner.has_left(md.space_name_l as usize)
+    }
+    fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.space_name.len().u64_bytes_le());
+        <obj::SpaceLayoutRef<'a> as PersistObject>::meta_enc(
+            buf,
+            obj::SpaceLayoutRef::from((data.space, data.space_meta)),
+        );
+    }
+    unsafe fn meta_dec(
+        scanner: &mut crate::engine::storage::v1::BufferedScanner,
+    ) -> crate::engine::storage::v1::SDSSResult<Self::Metadata> {
+        let space_name_l = u64::from_le_bytes(scanner.next_chunk());
+        let space_meta = <obj::SpaceLayoutRef as PersistObject>::meta_dec(scanner)?;
+        Ok(CreateSpaceTxnMD {
+            space_name_l,
+            space_meta,
+        })
+    }
+    fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.space_name.as_bytes());
+        <obj::SpaceLayoutRef as PersistObject>::meta_enc(buf, (data.space, data.space_meta).into());
+    }
+    unsafe fn obj_dec(
+        s: &mut crate::engine::storage::v1::BufferedScanner,
+        md: Self::Metadata,
+    ) -> crate::engine::storage::v1::SDSSResult<Self::OutputType> {
+        let space_name =
+            String::from_utf8(s.next_chunk_variable(md.space_name_l as usize).to_owned())
+                .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)?
+                .into_boxed_str();
+        let space = <obj::SpaceLayoutRef as PersistObject>::obj_dec(s, md.space_meta)?;
+        Ok(CreateSpaceTxnRestorePL { space_name, space })
+    }
+}

From b78824d6d0c050ccb065753ea3570db04f8e914f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 23 Aug 2023 06:46:36 +0000
Subject: [PATCH 228/310] Simplify gns event impls

---
 server/src/engine/core/mod.rs           |   2 +-
 server/src/engine/storage/v1/inf/map.rs |   2 +-
 server/src/engine/txn/gns.rs            | 149 --------------
 server/src/engine/txn/gns/mod.rs        |  90 ++++++++
 server/src/engine/txn/gns/space.rs      | 260 ++++++++++++++++++++++++
 server/src/engine/txn/mod.rs            |   3 +
 6 files changed, 355 insertions(+), 151 deletions(-)
 delete mode 100644 server/src/engine/txn/gns.rs
 create mode 100644 server/src/engine/txn/gns/mod.rs
 create mode 100644 server/src/engine/txn/gns/space.rs

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 7e036e22..b9fca120 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -55,7 +55,7 @@ pub struct GlobalNS {
 }
 
 impl GlobalNS {
-    pub(self) fn spaces(&self) -> &RWLIdx<Box<str>, Space> {
+    pub fn spaces(&self) -> &RWLIdx<Box<str>, Space> {
         &self.index_space
     }
     pub fn empty() -> Self {
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index 5ea9e05e..6cc9c1aa 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -45,7 +45,7 @@ use {
 };
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord)]
-pub struct MapIndexSizeMD(pub(super) usize);
+pub struct MapIndexSizeMD(pub usize);
 
 /// This is more of a lazy hack than anything sensible. Just implement a spec and then use this wrapper for any enc/dec operations
 pub struct PersistMapImpl<'a, M: PersistMapSpec>(PhantomData<&'a M::MapType>);
diff --git a/server/src/engine/txn/gns.rs b/server/src/engine/txn/gns.rs
deleted file mode 100644
index ce0d1cd8..00000000
--- a/server/src/engine/txn/gns.rs
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Created on Sun Aug 20 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{TransactionError, TransactionResult},
-    crate::{
-        engine::{
-            core::{space::Space, GlobalNS},
-            data::DictGeneric,
-            storage::v1::{
-                inf::{obj, PersistObject},
-                JournalAdapter, SDSSError,
-            },
-        },
-        util::EndianQW,
-    },
-    std::marker::PhantomData,
-};
-
-/*
-    journal implementor
-*/
-
-/// the journal adapter for DDL queries on the GNS
-pub struct GNSAdapter;
-
-impl JournalAdapter for GNSAdapter {
-    const RECOVERY_PLUGIN: bool = true;
-    type JournalEvent = GNSSuperEvent;
-    type GlobalState = GlobalNS;
-    type Error = TransactionError;
-    fn encode(GNSSuperEvent(b): Self::JournalEvent) -> Box<[u8]> {
-        b
-    }
-    fn decode_and_update_state(_: &[u8], _: &Self::GlobalState) -> TransactionResult<()> {
-        todo!()
-    }
-}
-
-/*
-    Events
-    ---
-    FIXME(@ohsayan): In the current impl, we unnecessarily use an intermediary buffer which we clearly don't need to (and also makes
-    pointless allocations). We need to fix this, but with a consistent API (and preferably not something like commit_*(...) unless
-    we have absolutely no other choice)
-    ---
-    [OPC:2B][PAYLOAD]
-*/
-
-pub struct GNSSuperEvent(Box<[u8]>);
-
-pub trait GNSEvent: PersistObject {
-    const OPC: u16;
-    type InputItem;
-}
-
-/*
-    create space
-*/
-
-pub struct CreateSpaceTxn<'a>(PhantomData<&'a ()>);
-#[derive(Clone, Copy)]
-pub struct CreateSpaceTxnCommitPL<'a> {
-    space_meta: &'a DictGeneric,
-    space_name: &'a str,
-    space: &'a Space,
-}
-pub struct CreateSpaceTxnRestorePL {
-    space_name: Box<str>,
-    space: Space,
-}
-pub struct CreateSpaceTxnMD {
-    space_name_l: u64,
-    space_meta: <obj::SpaceLayoutRef<'static> as PersistObject>::Metadata,
-}
-
-impl<'a> GNSEvent for CreateSpaceTxn<'a> {
-    const OPC: u16 = 0;
-    type InputItem = CreateSpaceTxnCommitPL<'a>;
-}
-
-impl<'a> PersistObject for CreateSpaceTxn<'a> {
-    const METADATA_SIZE: usize =
-        <obj::SpaceLayoutRef<'static> as PersistObject>::METADATA_SIZE + sizeof!(u64);
-    type InputType = CreateSpaceTxnCommitPL<'a>;
-    type OutputType = CreateSpaceTxnRestorePL;
-    type Metadata = CreateSpaceTxnMD;
-    fn pretest_can_dec_object(
-        scanner: &crate::engine::storage::v1::BufferedScanner,
-        md: &Self::Metadata,
-    ) -> bool {
-        scanner.has_left(md.space_name_l as usize)
-    }
-    fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        buf.extend(data.space_name.len().u64_bytes_le());
-        <obj::SpaceLayoutRef<'a> as PersistObject>::meta_enc(
-            buf,
-            obj::SpaceLayoutRef::from((data.space, data.space_meta)),
-        );
-    }
-    unsafe fn meta_dec(
-        scanner: &mut crate::engine::storage::v1::BufferedScanner,
-    ) -> crate::engine::storage::v1::SDSSResult<Self::Metadata> {
-        let space_name_l = u64::from_le_bytes(scanner.next_chunk());
-        let space_meta = <obj::SpaceLayoutRef as PersistObject>::meta_dec(scanner)?;
-        Ok(CreateSpaceTxnMD {
-            space_name_l,
-            space_meta,
-        })
-    }
-    fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        buf.extend(data.space_name.as_bytes());
-        <obj::SpaceLayoutRef as PersistObject>::meta_enc(buf, (data.space, data.space_meta).into());
-    }
-    unsafe fn obj_dec(
-        s: &mut crate::engine::storage::v1::BufferedScanner,
-        md: Self::Metadata,
-    ) -> crate::engine::storage::v1::SDSSResult<Self::OutputType> {
-        let space_name =
-            String::from_utf8(s.next_chunk_variable(md.space_name_l as usize).to_owned())
-                .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)?
-                .into_boxed_str();
-        let space = <obj::SpaceLayoutRef as PersistObject>::obj_dec(s, md.space_meta)?;
-        Ok(CreateSpaceTxnRestorePL { space_name, space })
-    }
-}
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
new file mode 100644
index 00000000..a3d7135a
--- /dev/null
+++ b/server/src/engine/txn/gns/mod.rs
@@ -0,0 +1,90 @@
+/*
+ * Created on Sun Aug 20 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::storage::v1::BufferedScanner;
+
+use {
+    super::{TransactionError, TransactionResult},
+    crate::engine::{
+        core::GlobalNS,
+        storage::v1::{
+            inf::{self, PersistObject},
+            JournalAdapter,
+        },
+    },
+};
+
+mod space;
+
+/*
+    journal implementor
+*/
+
+/// the journal adapter for DDL queries on the GNS
+pub struct GNSAdapter;
+
+impl JournalAdapter for GNSAdapter {
+    const RECOVERY_PLUGIN: bool = true;
+    type JournalEvent = GNSSuperEvent;
+    type GlobalState = GlobalNS;
+    type Error = TransactionError;
+    fn encode(GNSSuperEvent(b): Self::JournalEvent) -> Box<[u8]> {
+        b
+    }
+    fn decode_and_update_state(_: &[u8], _: &Self::GlobalState) -> TransactionResult<()> {
+        todo!()
+    }
+}
+
+/*
+    Events
+    ---
+    FIXME(@ohsayan): In the current impl, we unnecessarily use an intermediary buffer which we clearly don't need to (and also makes
+    pointless allocations). We need to fix this, but with a consistent API (and preferably not something like commit_*(...) unless
+    we have absolutely no other choice)
+    ---
+    [OPC:2B][PAYLOAD]
+*/
+
+pub struct GNSSuperEvent(Box<[u8]>);
+
+pub trait GNSEvent
+where
+    Self: PersistObject<InputType = Self::CommitType, OutputType = Self::RestoreType> + Sized,
+{
+    const OPC: u16;
+    type CommitType;
+    type RestoreType;
+    fn encode_super_event(commit: Self::CommitType) -> GNSSuperEvent {
+        GNSSuperEvent(inf::enc::enc_full::<Self>(commit).into_boxed_slice())
+    }
+    fn decode_from_super_event(
+        scanner: &mut BufferedScanner,
+    ) -> TransactionResult<Self::RestoreType> {
+        inf::dec::dec_full_from_scanner::<Self>(scanner).map_err(|e| e.into())
+    }
+    fn update_global_state(restore: Self::RestoreType, gns: &GlobalNS) -> TransactionResult<()>;
+}
diff --git a/server/src/engine/txn/gns/space.rs b/server/src/engine/txn/gns/space.rs
new file mode 100644
index 00000000..6dd39694
--- /dev/null
+++ b/server/src/engine/txn/gns/space.rs
@@ -0,0 +1,260 @@
+/*
+ * Created on Wed Aug 23 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{idx::STIndex, txn::TransactionError};
+
+use {
+    super::GNSEvent,
+    crate::{
+        engine::{
+            core::space::Space,
+            data::{uuid::Uuid, DictGeneric},
+            storage::v1::{
+                inf::{map, obj, PersistObject},
+                SDSSError,
+            },
+        },
+        util::EndianQW,
+    },
+    std::marker::PhantomData,
+};
+
+/*
+    create space
+*/
+
+pub struct CreateSpaceTxn<'a>(PhantomData<&'a ()>);
+
+#[derive(Clone, Copy)]
+pub struct CreateSpaceTxnCommitPL<'a> {
+    pub(crate) space_meta: &'a DictGeneric,
+    pub(crate) space_name: &'a str,
+    pub(crate) space: &'a Space,
+}
+
+pub struct CreateSpaceTxnRestorePL {
+    pub(crate) space_name: Box<str>,
+    pub(crate) space: Space,
+}
+
+pub struct CreateSpaceTxnMD {
+    pub(crate) space_name_l: u64,
+    pub(crate) space_meta: <obj::SpaceLayoutRef<'static> as PersistObject>::Metadata,
+}
+
+impl<'a> PersistObject for CreateSpaceTxn<'a> {
+    const METADATA_SIZE: usize =
+        <obj::SpaceLayoutRef<'static> as PersistObject>::METADATA_SIZE + sizeof!(u64);
+    type InputType = CreateSpaceTxnCommitPL<'a>;
+    type OutputType = CreateSpaceTxnRestorePL;
+    type Metadata = CreateSpaceTxnMD;
+    fn pretest_can_dec_object(
+        scanner: &crate::engine::storage::v1::BufferedScanner,
+        md: &Self::Metadata,
+    ) -> bool {
+        scanner.has_left(md.space_name_l as usize)
+    }
+    fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.space_name.len().u64_bytes_le());
+        <obj::SpaceLayoutRef<'a> as PersistObject>::meta_enc(
+            buf,
+            obj::SpaceLayoutRef::from((data.space, data.space_meta)),
+        );
+    }
+    unsafe fn meta_dec(
+        scanner: &mut crate::engine::storage::v1::BufferedScanner,
+    ) -> crate::engine::storage::v1::SDSSResult<Self::Metadata> {
+        let space_name_l = u64::from_le_bytes(scanner.next_chunk());
+        let space_meta = <obj::SpaceLayoutRef as PersistObject>::meta_dec(scanner)?;
+        Ok(CreateSpaceTxnMD {
+            space_name_l,
+            space_meta,
+        })
+    }
+    fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.space_name.as_bytes());
+        <obj::SpaceLayoutRef as PersistObject>::meta_enc(buf, (data.space, data.space_meta).into());
+    }
+    unsafe fn obj_dec(
+        s: &mut crate::engine::storage::v1::BufferedScanner,
+        md: Self::Metadata,
+    ) -> crate::engine::storage::v1::SDSSResult<Self::OutputType> {
+        let space_name =
+            String::from_utf8(s.next_chunk_variable(md.space_name_l as usize).to_owned())
+                .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)?
+                .into_boxed_str();
+        let space = <obj::SpaceLayoutRef as PersistObject>::obj_dec(s, md.space_meta)?;
+        Ok(CreateSpaceTxnRestorePL { space_name, space })
+    }
+}
+
+impl<'a> GNSEvent for CreateSpaceTxn<'a> {
+    const OPC: u16 = 0;
+    type CommitType = CreateSpaceTxnCommitPL<'a>;
+    type RestoreType = CreateSpaceTxnRestorePL;
+    fn update_global_state(
+        CreateSpaceTxnRestorePL { space_name, space }: CreateSpaceTxnRestorePL,
+        gns: &crate::engine::core::GlobalNS,
+    ) -> crate::engine::txn::TransactionResult<()> {
+        let mut wgns = gns.spaces().write();
+        if wgns.st_insert(space_name, space) {
+            Ok(())
+        } else {
+            Err(TransactionError::OnRestoreDataConflictAlreadyExists)
+        }
+    }
+}
+
+/*
+    alter space
+    ---
+    for now dump the entire meta
+*/
+
+pub struct AlterSpaceTxn<'a>(PhantomData<&'a ()>);
+pub struct AlterSpaceTxnMD {
+    uuid: Uuid,
+    space_name_l: u64,
+    dict_len: u64,
+}
+#[derive(Clone, Copy)]
+pub struct AlterSpaceTxnCommitPL<'a> {
+    space_uuid: Uuid,
+    space_name: &'a str,
+    space_meta: &'a DictGeneric,
+}
+pub struct AlterSpaceTxnRestorePL {
+    space_name: Box<str>,
+    space_meta: DictGeneric,
+}
+
+impl<'a> PersistObject for AlterSpaceTxn<'a> {
+    const METADATA_SIZE: usize = sizeof!(u64, 2) + sizeof!(u128);
+    type InputType = AlterSpaceTxnCommitPL<'a>;
+    type OutputType = AlterSpaceTxnRestorePL;
+    type Metadata = AlterSpaceTxnMD;
+    fn pretest_can_dec_object(
+        scanner: &crate::engine::storage::v1::BufferedScanner,
+        md: &Self::Metadata,
+    ) -> bool {
+        scanner.has_left(md.space_name_l as usize)
+    }
+    fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.space_uuid.to_le_bytes());
+        buf.extend(data.space_name.len().u64_bytes_le());
+        buf.extend(data.space_meta.len().u64_bytes_le());
+    }
+    unsafe fn meta_dec(
+        scanner: &mut crate::engine::storage::v1::BufferedScanner,
+    ) -> crate::engine::storage::v1::SDSSResult<Self::Metadata> {
+        Ok(AlterSpaceTxnMD {
+            uuid: Uuid::from_bytes(scanner.next_chunk()),
+            space_name_l: u64::from_le_bytes(scanner.next_chunk()),
+            dict_len: u64::from_le_bytes(scanner.next_chunk()),
+        })
+    }
+    fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.space_name.as_bytes());
+        <map::PersistMapImpl<map::GenericDictSpec> as PersistObject>::obj_enc(buf, data.space_meta);
+    }
+    unsafe fn obj_dec(
+        s: &mut crate::engine::storage::v1::BufferedScanner,
+        md: Self::Metadata,
+    ) -> crate::engine::storage::v1::SDSSResult<Self::OutputType> {
+        let space_name =
+            String::from_utf8(s.next_chunk_variable(md.space_name_l as usize).to_owned())
+                .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)?
+                .into_boxed_str();
+        let space_meta = <map::PersistMapImpl<map::GenericDictSpec> as PersistObject>::obj_dec(
+            s,
+            map::MapIndexSizeMD(md.dict_len as usize),
+        )?;
+        Ok(AlterSpaceTxnRestorePL {
+            space_name,
+            space_meta,
+        })
+    }
+}
+
+/*
+    drop space
+*/
+
+pub struct DropSpace<'a>(PhantomData<&'a ()>);
+pub struct DropSpaceTxnMD {
+    space_name_l: u64,
+    uuid: Uuid,
+}
+#[derive(Clone, Copy)]
+pub struct DropSpaceTxnCommitPL<'a> {
+    space_name: &'a str,
+    uuid: Uuid,
+}
+pub struct DropSpaceTxnRestorePL {
+    uuid: Uuid,
+    space_name: Box<str>,
+}
+
+impl<'a> PersistObject for DropSpace<'a> {
+    const METADATA_SIZE: usize = sizeof!(u128) + sizeof!(u64);
+    type InputType = DropSpaceTxnCommitPL<'a>;
+    type OutputType = DropSpaceTxnRestorePL;
+    type Metadata = DropSpaceTxnMD;
+    fn pretest_can_dec_object(
+        scanner: &crate::engine::storage::v1::BufferedScanner,
+        md: &Self::Metadata,
+    ) -> bool {
+        scanner.has_left(md.space_name_l as usize)
+    }
+    fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.space_name.len().u64_bytes_le());
+        buf.extend(data.uuid.to_le_bytes());
+    }
+    unsafe fn meta_dec(
+        scanner: &mut crate::engine::storage::v1::BufferedScanner,
+    ) -> crate::engine::storage::v1::SDSSResult<Self::Metadata> {
+        Ok(DropSpaceTxnMD {
+            space_name_l: u64::from_le_bytes(scanner.next_chunk()),
+            uuid: Uuid::from_bytes(scanner.next_chunk()),
+        })
+    }
+    fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.space_name.as_bytes());
+    }
+    unsafe fn obj_dec(
+        s: &mut crate::engine::storage::v1::BufferedScanner,
+        md: Self::Metadata,
+    ) -> crate::engine::storage::v1::SDSSResult<Self::OutputType> {
+        let space_name =
+            String::from_utf8(s.next_chunk_variable(md.space_name_l as usize).to_owned())
+                .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)?
+                .into_boxed_str();
+        Ok(DropSpaceTxnRestorePL {
+            uuid: md.uuid,
+            space_name,
+        })
+    }
+}
diff --git a/server/src/engine/txn/mod.rs b/server/src/engine/txn/mod.rs
index 62745ada..c7ba9657 100644
--- a/server/src/engine/txn/mod.rs
+++ b/server/src/engine/txn/mod.rs
@@ -33,6 +33,9 @@ pub type TransactionResult<T> = Result<T, TransactionError>;
 #[cfg_attr(test, derive(PartialEq))]
 pub enum TransactionError {
     SDSSError(SDSSError),
+    /// While restoring a certain item, a non-resolvable conflict was encountered in the global state, because the item was
+    /// already present (when it was expected to not be present)
+    OnRestoreDataConflictAlreadyExists,
 }
 
 direct_from! {

From 138753c4ad093bae760cf8d6cd5ed0d094b03ce9 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 23 Aug 2023 09:45:55 +0000
Subject: [PATCH 229/310] Add `create model` txn impl

---
 server/src/engine/storage/v1/inf/map.rs |  14 +-
 server/src/engine/storage/v1/inf/mod.rs |   7 +
 server/src/engine/storage/v1/inf/obj.rs |  12 +-
 server/src/engine/storage/v1/journal.rs |  10 ++
 server/src/engine/txn/gns/mod.rs        |  45 +++++-
 server/src/engine/txn/gns/model.rs      | 179 ++++++++++++++++++++++++
 server/src/engine/txn/gns/space.rs      | 163 ++++++++++++++-------
 server/src/engine/txn/mod.rs            |   4 +
 8 files changed, 363 insertions(+), 71 deletions(-)
 create mode 100644 server/src/engine/txn/gns/model.rs

diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index 6cc9c1aa..7fdc3b0a 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -36,7 +36,7 @@ use {
                 DictGeneric,
             },
             idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
-            storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult},
+            storage::v1::{inf, rw::BufferedScanner, SDSSError, SDSSResult},
         },
         util::{copy_slice_to_array as memcpy, EndianQW},
     },
@@ -227,7 +227,7 @@ impl PersistMapSpec for GenericDictSpec {
         }
     }
     unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Key> {
-        String::from_utf8(scanner.next_chunk_variable(md.klen).to_owned())
+        inf::dec::utils::decode_string(scanner, md.klen as usize)
             .map(|s| s.into_boxed_str())
             .ok()
     }
@@ -390,13 +390,9 @@ impl PersistMapSpec for FieldMapSpec {
         }
     }
     unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Key> {
-        String::from_utf8(
-            scanner
-                .next_chunk_variable(md.field_id_l as usize)
-                .to_owned(),
-        )
-        .map(|v| v.into_boxed_str())
-        .ok()
+        inf::dec::utils::decode_string(scanner, md.field_id_l as usize)
+            .map(|s| s.into_boxed_str())
+            .ok()
     }
     unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value> {
         super::obj::FieldRef::obj_dec(
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index c7f874e0..03892bab 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -271,4 +271,11 @@ pub mod dec {
     ) -> SDSSResult<PM::MapType> {
         <map::PersistMapImpl<PM> as PersistObject>::default_full_dec(scanner)
     }
+    pub mod utils {
+        use crate::engine::storage::v1::{BufferedScanner, SDSSError, SDSSResult};
+        pub unsafe fn decode_string(s: &mut BufferedScanner, len: usize) -> SDSSResult<String> {
+            String::from_utf8(s.next_chunk_variable(len).to_owned())
+                .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)
+        }
+    }
 }
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index fb857f9b..2942ff9a 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -39,7 +39,7 @@ use {
                 uuid::Uuid,
             },
             mem::VInline,
-            storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult},
+            storage::v1::{inf, rw::BufferedScanner, SDSSError, SDSSResult},
         },
         util::EndianQW,
     },
@@ -209,6 +209,9 @@ impl ModelLayoutMD {
             field_c,
         }
     }
+    pub fn p_key_len(&self) -> u64 {
+        self.p_key_len
+    }
 }
 
 #[derive(Clone, Copy)]
@@ -251,12 +254,7 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
         scanner: &mut BufferedScanner,
         md: Self::Metadata,
     ) -> SDSSResult<Self::OutputType> {
-        let key = String::from_utf8(
-            scanner
-                .next_chunk_variable(md.p_key_len as usize)
-                .to_owned(),
-        )
-        .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)?;
+        let key = inf::dec::utils::decode_string(scanner, md.p_key_len as usize)?;
         let fieldmap =
             <super::map::PersistMapImpl<super::map::FieldMapSpec> as PersistObject>::obj_dec(
                 scanner,
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index 583f6547..a72bf512 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -364,6 +364,7 @@ impl<TA, LF: RawFileIOInterface> JournalReader<TA, LF> {
     }
 }
 
+#[derive(Debug)]
 pub struct JournalWriter<LF, TA> {
     /// the txn log file
     log_file: SDSSFileIO<LF>,
@@ -403,6 +404,15 @@ impl<LF: RawFileIOInterface, TA: JournalAdapter> JournalWriter<LF, TA> {
         self.log_file.fsync_all()?;
         Ok(())
     }
+    pub fn append_event_with_recovery_plugin(&mut self, event: TA::JournalEvent) -> SDSSResult<()> {
+        debug_assert!(TA::RECOVERY_PLUGIN);
+        match self.append_event(event) {
+            Ok(()) => Ok(()),
+            Err(_) => {
+                return self.appendrec_journal_reverse_entry();
+            }
+        }
+    }
 }
 
 impl<LF: RawFileIOInterface, TA> JournalWriter<LF, TA> {
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index a3d7135a..f0b705a3 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -24,27 +24,53 @@
  *
 */
 
-use crate::engine::storage::v1::BufferedScanner;
-
 use {
     super::{TransactionError, TransactionResult},
     crate::engine::{
         core::GlobalNS,
         storage::v1::{
             inf::{self, PersistObject},
-            JournalAdapter,
+            BufferedScanner, JournalAdapter, JournalWriter,
         },
     },
+    std::fs::File,
 };
 
+mod model;
 mod space;
 
+// re-exports
+pub use {
+    model::CreateModelTxn,
+    space::{AlterSpaceTxn, CreateSpaceTxn, DropSpaceTxn},
+};
+
+#[derive(Debug)]
+/// The GNS transaction driver is used to handle DDL transactions
+pub struct GNSTransactionDriver {
+    journal: JournalWriter<File, GNSAdapter>,
+}
+
+impl GNSTransactionDriver {
+    /// Attempts to commit the given event into the journal, handling any possible recovery triggers and returning
+    /// errors (if any)
+    pub fn try_commit<GE: GNSEvent>(&mut self, gns_event: GE::CommitType) -> TransactionResult<()> {
+        let mut buf = vec![];
+        buf.extend(GE::OPC.to_le_bytes());
+        GE::encode_super_event(gns_event, &mut buf);
+        self.journal
+            .append_event_with_recovery_plugin(GNSSuperEvent(buf.into_boxed_slice()))?;
+        Ok(())
+    }
+}
+
 /*
     journal implementor
 */
 
 /// the journal adapter for DDL queries on the GNS
-pub struct GNSAdapter;
+#[derive(Debug)]
+struct GNSAdapter;
 
 impl JournalAdapter for GNSAdapter {
     const RECOVERY_PLUGIN: bool = true;
@@ -71,20 +97,27 @@ impl JournalAdapter for GNSAdapter {
 
 pub struct GNSSuperEvent(Box<[u8]>);
 
+/// Definition for an event in the GNS (DDL queries)
 pub trait GNSEvent
 where
     Self: PersistObject<InputType = Self::CommitType, OutputType = Self::RestoreType> + Sized,
 {
+    /// OPC for the event (unique)
     const OPC: u16;
+    /// Expected type for a commit
     type CommitType;
+    /// Expected type for a restore
     type RestoreType;
-    fn encode_super_event(commit: Self::CommitType) -> GNSSuperEvent {
-        GNSSuperEvent(inf::enc::enc_full::<Self>(commit).into_boxed_slice())
+    /// Encodes the event into the given buffer
+    fn encode_super_event(commit: Self::CommitType, buf: &mut Vec<u8>) {
+        inf::enc::enc_full_into_buffer::<Self>(buf, commit)
     }
+    /// Attempts to decode the event using the given scanner
     fn decode_from_super_event(
         scanner: &mut BufferedScanner,
     ) -> TransactionResult<Self::RestoreType> {
         inf::dec::dec_full_from_scanner::<Self>(scanner).map_err(|e| e.into())
     }
+    /// Update the global state from the restored event
     fn update_global_state(restore: Self::RestoreType, gns: &GlobalNS) -> TransactionResult<()>;
 }
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
new file mode 100644
index 00000000..77c9d6a2
--- /dev/null
+++ b/server/src/engine/txn/gns/model.rs
@@ -0,0 +1,179 @@
+/*
+ * Created on Wed Aug 23 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::GNSEvent,
+    crate::{
+        engine::{
+            core::model::{delta::IRModel, Model},
+            data::uuid::Uuid,
+            idx::STIndex,
+            storage::v1::{
+                inf::{self, obj, PersistObject},
+                BufferedScanner, SDSSResult,
+            },
+            txn::TransactionError,
+        },
+        util::EndianQW,
+    },
+    std::marker::PhantomData,
+};
+
+/*
+    create model
+*/
+
+/// Transaction for running a `create model ... (...) with {..}` query
+pub struct CreateModelTxn<'a>(PhantomData<&'a ()>);
+
+impl<'a> CreateModelTxn<'a> {
+    pub const fn new_commit(
+        space_name: &'a str,
+        space_uuid: Uuid,
+        model_name: &'a str,
+        model: &'a Model,
+        model_read: &'a IRModel<'a>,
+    ) -> CreateModelTxnCommitPL<'a> {
+        CreateModelTxnCommitPL {
+            space_name,
+            space_uuid,
+            model_name,
+            model,
+            model_read,
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+pub struct CreateModelTxnCommitPL<'a> {
+    space_name: &'a str,
+    space_uuid: Uuid,
+    model_name: &'a str,
+    model: &'a Model,
+    model_read: &'a IRModel<'a>,
+}
+
+pub struct CreateModelTxnRestorePL {
+    space_name: Box<str>,
+    space_uuid: Uuid,
+    model_name: Box<str>,
+    model: Model,
+}
+
+pub struct CreateModelTxnMD {
+    space_name_l: u64,
+    space_uuid: Uuid,
+    model_name_l: u64,
+    model_meta: <obj::ModelLayoutRef<'static> as PersistObject>::Metadata,
+}
+
+impl<'a> PersistObject for CreateModelTxn<'a> {
+    const METADATA_SIZE: usize =
+        sizeof!(u64, 2) + sizeof!(u128) + <obj::ModelLayoutRef<'a> as PersistObject>::METADATA_SIZE;
+    type InputType = CreateModelTxnCommitPL<'a>;
+    type OutputType = CreateModelTxnRestorePL;
+    type Metadata = CreateModelTxnMD;
+    fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
+        scanner.has_left((md.model_meta.p_key_len() + md.model_name_l) as usize)
+    }
+    fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.space_name.len().u64_bytes_le());
+        buf.extend(data.space_uuid.to_le_bytes());
+        buf.extend(data.model_name.len().u64_bytes_le());
+        <obj::ModelLayoutRef as PersistObject>::meta_enc(
+            buf,
+            obj::ModelLayoutRef::from((data.model, data.model_read)),
+        )
+    }
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+        let space_name_l = u64::from_le_bytes(scanner.next_chunk());
+        let space_uuid = Uuid::from_bytes(scanner.next_chunk());
+        let model_name_l = u64::from_le_bytes(scanner.next_chunk());
+        let model_meta = <obj::ModelLayoutRef as PersistObject>::meta_dec(scanner)?;
+        Ok(CreateModelTxnMD {
+            space_name_l,
+            space_uuid,
+            model_name_l,
+            model_meta,
+        })
+    }
+    fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.model_name.as_bytes());
+        <obj::ModelLayoutRef as PersistObject>::obj_enc(
+            buf,
+            obj::ModelLayoutRef::from((data.model, data.model_read)),
+        )
+    }
+    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+        let space_name =
+            inf::dec::utils::decode_string(s, md.space_name_l as usize)?.into_boxed_str();
+        let model_name =
+            inf::dec::utils::decode_string(s, md.model_name_l as usize)?.into_boxed_str();
+        let model = <obj::ModelLayoutRef as PersistObject>::obj_dec(s, md.model_meta)?;
+        Ok(CreateModelTxnRestorePL {
+            space_name,
+            space_uuid: md.space_uuid,
+            model_name,
+            model,
+        })
+    }
+}
+
+impl<'a> GNSEvent for CreateModelTxn<'a> {
+    const OPC: u16 = 3;
+    type CommitType = CreateModelTxnCommitPL<'a>;
+    type RestoreType = CreateModelTxnRestorePL;
+    fn update_global_state(
+        CreateModelTxnRestorePL {
+            space_name,
+            space_uuid,
+            model_name,
+            model,
+        }: Self::RestoreType,
+        gns: &crate::engine::core::GlobalNS,
+    ) -> crate::engine::txn::TransactionResult<()> {
+        let rgns = gns.spaces().read();
+        /*
+            NOTE(@ohsayan):
+            do note that this is a little interesting situation especially because we need to be able to handle
+            changes in the schema *and* be able to "sync" that (for consistency) with the model's primary index.
+
+            There is no evident way about how this is going to be handled, but the ideal way would be to keep
+            versioned index of schemas.
+        */
+        match rgns.st_get(&space_name) {
+            Some(space) if space.get_uuid() == space_uuid => {
+                if space._create_model(&model_name, model).is_ok() {
+                    Ok(())
+                } else {
+                    Err(TransactionError::OnRestoreDataConflictAlreadyExists)
+                }
+            }
+            Some(_) => return Err(TransactionError::OnRestoreDataConflictMismatch),
+            None => return Err(TransactionError::OnRestoreDataMissing),
+        }
+    }
+}
diff --git a/server/src/engine/txn/gns/space.rs b/server/src/engine/txn/gns/space.rs
index 6dd39694..3febe374 100644
--- a/server/src/engine/txn/gns/space.rs
+++ b/server/src/engine/txn/gns/space.rs
@@ -24,18 +24,18 @@
  *
 */
 
-use crate::engine::{idx::STIndex, txn::TransactionError};
-
 use {
     super::GNSEvent,
     crate::{
         engine::{
-            core::space::Space,
+            core::{space::Space, GlobalNS},
             data::{uuid::Uuid, DictGeneric},
+            idx::STIndex,
             storage::v1::{
-                inf::{map, obj, PersistObject},
-                SDSSError,
+                inf::{self, map, obj, PersistObject},
+                BufferedScanner, SDSSResult,
             },
+            txn::{TransactionError, TransactionResult},
         },
         util::EndianQW,
     },
@@ -46,8 +46,23 @@ use {
     create space
 */
 
+/// A transaction to run a `create space ...` operation
 pub struct CreateSpaceTxn<'a>(PhantomData<&'a ()>);
 
+impl<'a> CreateSpaceTxn<'a> {
+    pub const fn new_commit(
+        space_meta: &'a DictGeneric,
+        space_name: &'a str,
+        space: &'a Space,
+    ) -> CreateSpaceTxnCommitPL<'a> {
+        CreateSpaceTxnCommitPL {
+            space_meta,
+            space_name,
+            space,
+        }
+    }
+}
+
 #[derive(Clone, Copy)]
 pub struct CreateSpaceTxnCommitPL<'a> {
     pub(crate) space_meta: &'a DictGeneric,
@@ -71,10 +86,7 @@ impl<'a> PersistObject for CreateSpaceTxn<'a> {
     type InputType = CreateSpaceTxnCommitPL<'a>;
     type OutputType = CreateSpaceTxnRestorePL;
     type Metadata = CreateSpaceTxnMD;
-    fn pretest_can_dec_object(
-        scanner: &crate::engine::storage::v1::BufferedScanner,
-        md: &Self::Metadata,
-    ) -> bool {
+    fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
         scanner.has_left(md.space_name_l as usize)
     }
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
@@ -84,9 +96,7 @@ impl<'a> PersistObject for CreateSpaceTxn<'a> {
             obj::SpaceLayoutRef::from((data.space, data.space_meta)),
         );
     }
-    unsafe fn meta_dec(
-        scanner: &mut crate::engine::storage::v1::BufferedScanner,
-    ) -> crate::engine::storage::v1::SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
         let space_name_l = u64::from_le_bytes(scanner.next_chunk());
         let space_meta = <obj::SpaceLayoutRef as PersistObject>::meta_dec(scanner)?;
         Ok(CreateSpaceTxnMD {
@@ -98,14 +108,9 @@ impl<'a> PersistObject for CreateSpaceTxn<'a> {
         buf.extend(data.space_name.as_bytes());
         <obj::SpaceLayoutRef as PersistObject>::meta_enc(buf, (data.space, data.space_meta).into());
     }
-    unsafe fn obj_dec(
-        s: &mut crate::engine::storage::v1::BufferedScanner,
-        md: Self::Metadata,
-    ) -> crate::engine::storage::v1::SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
         let space_name =
-            String::from_utf8(s.next_chunk_variable(md.space_name_l as usize).to_owned())
-                .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)?
-                .into_boxed_str();
+            inf::dec::utils::decode_string(s, md.space_name_l as usize)?.into_boxed_str();
         let space = <obj::SpaceLayoutRef as PersistObject>::obj_dec(s, md.space_meta)?;
         Ok(CreateSpaceTxnRestorePL { space_name, space })
     }
@@ -134,18 +139,36 @@ impl<'a> GNSEvent for CreateSpaceTxn<'a> {
     for now dump the entire meta
 */
 
+/// A transaction to run `alter space ...`
 pub struct AlterSpaceTxn<'a>(PhantomData<&'a ()>);
+
+impl<'a> AlterSpaceTxn<'a> {
+    pub const fn new_commit(
+        space_uuid: Uuid,
+        space_name: &'a str,
+        space_meta: &'a DictGeneric,
+    ) -> AlterSpaceTxnCommitPL<'a> {
+        AlterSpaceTxnCommitPL {
+            space_uuid,
+            space_name,
+            space_meta,
+        }
+    }
+}
+
 pub struct AlterSpaceTxnMD {
     uuid: Uuid,
     space_name_l: u64,
     dict_len: u64,
 }
+
 #[derive(Clone, Copy)]
 pub struct AlterSpaceTxnCommitPL<'a> {
     space_uuid: Uuid,
     space_name: &'a str,
     space_meta: &'a DictGeneric,
 }
+
 pub struct AlterSpaceTxnRestorePL {
     space_name: Box<str>,
     space_meta: DictGeneric,
@@ -156,10 +179,7 @@ impl<'a> PersistObject for AlterSpaceTxn<'a> {
     type InputType = AlterSpaceTxnCommitPL<'a>;
     type OutputType = AlterSpaceTxnRestorePL;
     type Metadata = AlterSpaceTxnMD;
-    fn pretest_can_dec_object(
-        scanner: &crate::engine::storage::v1::BufferedScanner,
-        md: &Self::Metadata,
-    ) -> bool {
+    fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
         scanner.has_left(md.space_name_l as usize)
     }
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
@@ -167,9 +187,7 @@ impl<'a> PersistObject for AlterSpaceTxn<'a> {
         buf.extend(data.space_name.len().u64_bytes_le());
         buf.extend(data.space_meta.len().u64_bytes_le());
     }
-    unsafe fn meta_dec(
-        scanner: &mut crate::engine::storage::v1::BufferedScanner,
-    ) -> crate::engine::storage::v1::SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
         Ok(AlterSpaceTxnMD {
             uuid: Uuid::from_bytes(scanner.next_chunk()),
             space_name_l: u64::from_le_bytes(scanner.next_chunk()),
@@ -180,14 +198,9 @@ impl<'a> PersistObject for AlterSpaceTxn<'a> {
         buf.extend(data.space_name.as_bytes());
         <map::PersistMapImpl<map::GenericDictSpec> as PersistObject>::obj_enc(buf, data.space_meta);
     }
-    unsafe fn obj_dec(
-        s: &mut crate::engine::storage::v1::BufferedScanner,
-        md: Self::Metadata,
-    ) -> crate::engine::storage::v1::SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
         let space_name =
-            String::from_utf8(s.next_chunk_variable(md.space_name_l as usize).to_owned())
-                .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)?
-                .into_boxed_str();
+            inf::dec::utils::decode_string(s, md.space_name_l as usize)?.into_boxed_str();
         let space_meta = <map::PersistMapImpl<map::GenericDictSpec> as PersistObject>::obj_dec(
             s,
             map::MapIndexSizeMD(md.dict_len as usize),
@@ -199,11 +212,47 @@ impl<'a> PersistObject for AlterSpaceTxn<'a> {
     }
 }
 
+impl<'a> GNSEvent for AlterSpaceTxn<'a> {
+    const OPC: u16 = 1;
+
+    type CommitType = AlterSpaceTxnCommitPL<'a>;
+
+    type RestoreType = AlterSpaceTxnRestorePL;
+
+    fn update_global_state(
+        AlterSpaceTxnRestorePL {
+            space_name,
+            space_meta,
+        }: Self::RestoreType,
+        gns: &crate::engine::core::GlobalNS,
+    ) -> TransactionResult<()> {
+        let gns = gns.spaces().read();
+        match gns.st_get(&space_name) {
+            Some(space) => {
+                let mut wmeta = space.metadata().env().write();
+                space_meta
+                    .into_iter()
+                    .for_each(|(k, v)| wmeta.st_upsert(k, v));
+            }
+            None => return Err(TransactionError::OnRestoreDataMissing),
+        }
+        Ok(())
+    }
+}
+
 /*
     drop space
 */
 
-pub struct DropSpace<'a>(PhantomData<&'a ()>);
+/// A transaction to run `drop space ...`
+pub struct DropSpaceTxn<'a>(PhantomData<&'a ()>);
+
+impl<'a> DropSpaceTxn<'a> {
+    pub const fn new_commit(space_name: &'a str, uuid: Uuid) -> DropSpaceTxnCommitPL<'a> {
+        DropSpaceTxnCommitPL { space_name, uuid }
+    }
+}
+
 pub struct DropSpaceTxnMD {
     space_name_l: u64,
     uuid: Uuid,
@@ -213,29 +262,25 @@ pub struct DropSpaceTxnCommitPL<'a> {
     space_name: &'a str,
     uuid: Uuid,
 }
+
 pub struct DropSpaceTxnRestorePL {
     uuid: Uuid,
     space_name: Box<str>,
 }
 
-impl<'a> PersistObject for DropSpace<'a> {
+impl<'a> PersistObject for DropSpaceTxn<'a> {
     const METADATA_SIZE: usize = sizeof!(u128) + sizeof!(u64);
     type InputType = DropSpaceTxnCommitPL<'a>;
     type OutputType = DropSpaceTxnRestorePL;
     type Metadata = DropSpaceTxnMD;
-    fn pretest_can_dec_object(
-        scanner: &crate::engine::storage::v1::BufferedScanner,
-        md: &Self::Metadata,
-    ) -> bool {
+    fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
         scanner.has_left(md.space_name_l as usize)
     }
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
         buf.extend(data.space_name.len().u64_bytes_le());
         buf.extend(data.uuid.to_le_bytes());
     }
-    unsafe fn meta_dec(
-        scanner: &mut crate::engine::storage::v1::BufferedScanner,
-    ) -> crate::engine::storage::v1::SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
         Ok(DropSpaceTxnMD {
             space_name_l: u64::from_le_bytes(scanner.next_chunk()),
             uuid: Uuid::from_bytes(scanner.next_chunk()),
@@ -244,17 +289,37 @@ impl<'a> PersistObject for DropSpace<'a> {
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
         buf.extend(data.space_name.as_bytes());
     }
-    unsafe fn obj_dec(
-        s: &mut crate::engine::storage::v1::BufferedScanner,
-        md: Self::Metadata,
-    ) -> crate::engine::storage::v1::SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
         let space_name =
-            String::from_utf8(s.next_chunk_variable(md.space_name_l as usize).to_owned())
-                .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)?
-                .into_boxed_str();
+            inf::dec::utils::decode_string(s, md.space_name_l as usize)?.into_boxed_str();
         Ok(DropSpaceTxnRestorePL {
             uuid: md.uuid,
             space_name,
         })
     }
 }
+
+impl<'a> GNSEvent for DropSpaceTxn<'a> {
+    const OPC: u16 = 2;
+    type CommitType = DropSpaceTxnCommitPL<'a>;
+    type RestoreType = DropSpaceTxnRestorePL;
+    fn update_global_state(
+        DropSpaceTxnRestorePL { uuid, space_name }: Self::RestoreType,
+        gns: &GlobalNS,
+    ) -> TransactionResult<()> {
+        let mut wgns = gns.spaces().write();
+        match wgns.entry(space_name) {
+            std::collections::hash_map::Entry::Occupied(oe) => {
+                if oe.get().get_uuid() == uuid {
+                    oe.remove_entry();
+                    Ok(())
+                } else {
+                    return Err(TransactionError::OnRestoreDataConflictMismatch);
+                }
+            }
+            std::collections::hash_map::Entry::Vacant(_) => {
+                return Err(TransactionError::OnRestoreDataMissing)
+            }
+        }
+    }
+}
diff --git a/server/src/engine/txn/mod.rs b/server/src/engine/txn/mod.rs
index c7ba9657..8f1e5f99 100644
--- a/server/src/engine/txn/mod.rs
+++ b/server/src/engine/txn/mod.rs
@@ -36,6 +36,10 @@ pub enum TransactionError {
     /// While restoring a certain item, a non-resolvable conflict was encountered in the global state, because the item was
     /// already present (when it was expected to not be present)
     OnRestoreDataConflictAlreadyExists,
+    /// On restore, a certain item that was expected to be present was missing in the global state
+    OnRestoreDataMissing,
+    /// On restore, a certain item that was expected to match a certain value, has a different value
+    OnRestoreDataConflictMismatch,
 }
 
 direct_from! {

From 891252a89d55138d9c27e7edaa86705e3127136e Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 24 Aug 2023 06:32:46 +0000
Subject: [PATCH 230/310] Add remaining DDL txns

---
 server/src/engine/core/space.rs         |   2 +-
 server/src/engine/storage/v1/inf/map.rs |  65 +++
 server/src/engine/txn/gns/mod.rs        |  61 ++-
 server/src/engine/txn/gns/model.rs      | 535 ++++++++++++++++++++++--
 server/src/engine/txn/gns/space.rs      |  79 ++--
 5 files changed, 654 insertions(+), 88 deletions(-)

diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 86169d7c..d282d837 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -92,7 +92,7 @@ impl Space {
     pub fn get_uuid(&self) -> Uuid {
         self.uuid
     }
-    pub(super) fn models(&self) -> &RWLIdx<Box<str>, Model> {
+    pub fn models(&self) -> &RWLIdx<Box<str>, Model> {
         &self.mns
     }
     pub fn metadata(&self) -> &SpaceMeta {
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index 7fdc3b0a..40949bcb 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -412,3 +412,68 @@ impl PersistMapSpec for FieldMapSpec {
         unimplemented!()
     }
 }
+
+// TODO(@ohsayan): common trait for k/v associations, independent of underlying maptype
+pub struct FieldMapSpecST;
+impl PersistMapSpec for FieldMapSpecST {
+    type MapIter<'a> = std::collections::hash_map::Iter<'a, Box<str>, Field>;
+    type MapType = std::collections::HashMap<Box<str>, Field>;
+    type EntryMD = FieldMapEntryMD;
+    type Key = Box<str>;
+    type Value = Field;
+    const ENC_COUPLED: bool = false;
+    const DEC_COUPLED: bool = false;
+    fn _get_iter<'a>(m: &'a Self::MapType) -> Self::MapIter<'a> {
+        m.iter()
+    }
+    fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool {
+        scanner.has_left(sizeof!(u64, 3) + 1)
+    }
+    fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool {
+        scanner.has_left(md.field_id_l as usize) // TODO(@ohsayan): we can enforce way more here such as atleast one field etc
+    }
+    fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
+        buf.extend(key.len().u64_bytes_le());
+        buf.extend(0u64.to_le_bytes()); // TODO(@ohsayan): props
+        buf.extend(val.layers().len().u64_bytes_le());
+        buf.push(val.is_nullable() as u8);
+    }
+    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::EntryMD> {
+        Some(FieldMapEntryMD::new(
+            u64::from_le_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()),
+            u64::from_le_bytes(scanner.next_chunk()),
+            scanner.next_byte(),
+        ))
+    }
+    fn enc_key(buf: &mut VecU8, key: &Self::Key) {
+        buf.extend(key.as_bytes());
+    }
+    fn enc_val(buf: &mut VecU8, val: &Self::Value) {
+        for layer in val.layers() {
+            super::obj::LayerRef::default_full_enc(buf, super::obj::LayerRef(layer))
+        }
+    }
+    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Key> {
+        inf::dec::utils::decode_string(scanner, md.field_id_l as usize)
+            .map(|s| s.into_boxed_str())
+            .ok()
+    }
+    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value> {
+        super::obj::FieldRef::obj_dec(
+            scanner,
+            FieldMD::new(md.field_prop_c, md.field_layer_c, md.null),
+        )
+        .ok()
+    }
+    // unimplemented
+    fn enc_entry(_: &mut VecU8, _: &Self::Key, _: &Self::Value) {
+        unimplemented!()
+    }
+    unsafe fn dec_entry(
+        _: &mut BufferedScanner,
+        _: Self::EntryMD,
+    ) -> Option<(Self::Key, Self::Value)> {
+        unimplemented!()
+    }
+}
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index f0b705a3..b800295e 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -26,14 +26,18 @@
 
 use {
     super::{TransactionError, TransactionResult},
-    crate::engine::{
-        core::GlobalNS,
-        storage::v1::{
-            inf::{self, PersistObject},
-            BufferedScanner, JournalAdapter, JournalWriter,
+    crate::{
+        engine::{
+            core::GlobalNS,
+            data::uuid::Uuid,
+            storage::v1::{
+                inf::{self, PersistObject},
+                BufferedScanner, JournalAdapter, JournalWriter, SDSSResult,
+            },
         },
+        util::EndianQW,
     },
-    std::fs::File,
+    std::{fs::File, marker::PhantomData},
 };
 
 mod model;
@@ -121,3 +125,48 @@ where
     /// Update the global state from the restored event
     fn update_global_state(restore: Self::RestoreType, gns: &GlobalNS) -> TransactionResult<()>;
 }
+
+#[derive(Debug, Clone, Copy)]
+pub struct SpaceIDRef<'a> {
+    uuid: Uuid,
+    name: &'a str,
+}
+pub struct SpaceIDRes {
+    uuid: Uuid,
+    name: Box<str>,
+}
+struct SpaceID<'a>(PhantomData<SpaceIDRef<'a>>);
+pub struct SpaceIDMD {
+    uuid: Uuid,
+    space_name_l: u64,
+}
+
+impl<'a> PersistObject for SpaceID<'a> {
+    const METADATA_SIZE: usize = sizeof!(u128) + sizeof!(u64);
+    type InputType = SpaceIDRef<'a>;
+    type OutputType = SpaceIDRes;
+    type Metadata = SpaceIDMD;
+    fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
+        scanner.has_left(md.space_name_l as usize)
+    }
+    fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.uuid.to_le_bytes());
+        buf.extend(data.name.len().u64_bytes_le());
+    }
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+        Ok(SpaceIDMD {
+            uuid: Uuid::from_bytes(scanner.next_chunk()),
+            space_name_l: u64::from_le_bytes(scanner.next_chunk()),
+        })
+    }
+    fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.name.as_bytes());
+    }
+    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+        let str = inf::dec::utils::decode_string(s, md.space_name_l as usize)?;
+        Ok(SpaceIDRes {
+            uuid: md.uuid,
+            name: str.into_boxed_str(),
+        })
+    }
+}
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index 77c9d6a2..85792239 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -25,15 +25,20 @@
 */
 
 use {
-    super::GNSEvent,
+    super::{GNSEvent, TransactionResult},
     crate::{
         engine::{
-            core::model::{delta::IRModel, Model},
+            core::{
+                model::{delta::IRModel, Field, Model},
+                space::Space,
+                GlobalNS,
+            },
             data::uuid::Uuid,
-            idx::STIndex,
+            idx::{IndexST, IndexSTSeqCns, STIndex, STIndexSeq},
+            ql::lex::Ident,
             storage::v1::{
-                inf::{self, obj, PersistObject},
-                BufferedScanner, SDSSResult,
+                inf::{self, map, obj, PersistObject},
+                BufferedScanner, SDSSError, SDSSResult,
             },
             txn::TransactionError,
         },
@@ -42,6 +47,91 @@ use {
     std::marker::PhantomData,
 };
 
+pub struct ModelID<'a>(PhantomData<&'a ()>);
+#[derive(Debug, Clone, Copy)]
+pub struct ModelIDRef<'a> {
+    model_name: &'a str,
+    model_uuid: Uuid,
+    model_version: u64,
+}
+pub struct ModelIDRes {
+    model_name: Box<str>,
+    model_uuid: Uuid,
+    model_version: u64,
+}
+pub struct ModelIDMD {
+    model_name_l: u64,
+    model_version: u64,
+    model_uuid: Uuid,
+}
+
+impl<'a> PersistObject for ModelID<'a> {
+    const METADATA_SIZE: usize = sizeof!(u64, 2) + sizeof!(u128);
+    type InputType = ModelIDRef<'a>;
+    type OutputType = ModelIDRes;
+    type Metadata = ModelIDMD;
+    fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
+        scanner.has_left(md.model_name_l as usize)
+    }
+    fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.model_name.len().u64_bytes_le());
+        buf.extend(data.model_version.to_le_bytes());
+        buf.extend(data.model_uuid.to_le_bytes());
+    }
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+        Ok(ModelIDMD {
+            model_name_l: u64::from_le_bytes(scanner.next_chunk()),
+            model_version: u64::from_le_bytes(scanner.next_chunk()),
+            model_uuid: Uuid::from_bytes(scanner.next_chunk()),
+        })
+    }
+    fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        buf.extend(data.model_name.as_bytes());
+    }
+    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+        Ok(ModelIDRes {
+            model_name: inf::dec::utils::decode_string(s, md.model_name_l as usize)?
+                .into_boxed_str(),
+            model_uuid: md.model_uuid,
+            model_version: md.model_version,
+        })
+    }
+}
+
+fn with_space<T>(
+    gns: &GlobalNS,
+    space_id: &super::SpaceIDRes,
+    mut f: impl FnMut(&Space) -> TransactionResult<T>,
+) -> TransactionResult<T> {
+    let spaces = gns.spaces().read();
+    let Some(space) = spaces.st_get(&space_id.name) else {
+        return Err(TransactionError::OnRestoreDataMissing);
+    };
+    if space.get_uuid() != space_id.uuid {
+        return Err(TransactionError::OnRestoreDataConflictMismatch);
+    }
+    f(space)
+}
+
+fn with_model<T>(
+    gns: &GlobalNS,
+    space_id: &super::SpaceIDRes,
+    model_id: &ModelIDRes,
+    mut f: impl FnMut(&Model) -> TransactionResult<T>,
+) -> TransactionResult<T> {
+    with_space(gns, space_id, |space| {
+        let models = space.models().read();
+        let Some(model) = models.st_get(&model_id.model_name) else {
+            return Err(TransactionError::OnRestoreDataMissing);
+        };
+        if model.get_uuid() != model_id.model_uuid {
+            // this should have been handled by an earlier transaction
+            return Err(TransactionError::OnRestoreDataConflictMismatch);
+        }
+        f(model)
+    })
+}
+
 /*
     create model
 */
@@ -58,8 +148,10 @@ impl<'a> CreateModelTxn<'a> {
         model_read: &'a IRModel<'a>,
     ) -> CreateModelTxnCommitPL<'a> {
         CreateModelTxnCommitPL {
-            space_name,
-            space_uuid,
+            space_id: super::SpaceIDRef {
+                uuid: space_uuid,
+                name: space_name,
+            },
             model_name,
             model,
             model_read,
@@ -69,30 +161,28 @@ impl<'a> CreateModelTxn<'a> {
 
 #[derive(Clone, Copy)]
 pub struct CreateModelTxnCommitPL<'a> {
-    space_name: &'a str,
-    space_uuid: Uuid,
+    space_id: super::SpaceIDRef<'a>,
     model_name: &'a str,
     model: &'a Model,
     model_read: &'a IRModel<'a>,
 }
 
 pub struct CreateModelTxnRestorePL {
-    space_name: Box<str>,
-    space_uuid: Uuid,
+    space_id: super::SpaceIDRes,
     model_name: Box<str>,
     model: Model,
 }
 
 pub struct CreateModelTxnMD {
-    space_name_l: u64,
-    space_uuid: Uuid,
+    space_id_meta: super::SpaceIDMD,
     model_name_l: u64,
     model_meta: <obj::ModelLayoutRef<'static> as PersistObject>::Metadata,
 }
 
 impl<'a> PersistObject for CreateModelTxn<'a> {
-    const METADATA_SIZE: usize =
-        sizeof!(u64, 2) + sizeof!(u128) + <obj::ModelLayoutRef<'a> as PersistObject>::METADATA_SIZE;
+    const METADATA_SIZE: usize = <super::SpaceID as PersistObject>::METADATA_SIZE
+        + sizeof!(u64)
+        + <obj::ModelLayoutRef<'a> as PersistObject>::METADATA_SIZE;
     type InputType = CreateModelTxnCommitPL<'a>;
     type OutputType = CreateModelTxnRestorePL;
     type Metadata = CreateModelTxnMD;
@@ -100,42 +190,44 @@ impl<'a> PersistObject for CreateModelTxn<'a> {
         scanner.has_left((md.model_meta.p_key_len() + md.model_name_l) as usize)
     }
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        buf.extend(data.space_name.len().u64_bytes_le());
-        buf.extend(data.space_uuid.to_le_bytes());
+        // space ID
+        <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
+        // model name
         buf.extend(data.model_name.len().u64_bytes_le());
+        // model meta dump
         <obj::ModelLayoutRef as PersistObject>::meta_enc(
             buf,
             obj::ModelLayoutRef::from((data.model, data.model_read)),
         )
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
-        let space_name_l = u64::from_le_bytes(scanner.next_chunk());
-        let space_uuid = Uuid::from_bytes(scanner.next_chunk());
+        let space_id = <super::SpaceID as PersistObject>::meta_dec(scanner)?;
         let model_name_l = u64::from_le_bytes(scanner.next_chunk());
         let model_meta = <obj::ModelLayoutRef as PersistObject>::meta_dec(scanner)?;
         Ok(CreateModelTxnMD {
-            space_name_l,
-            space_uuid,
+            space_id_meta: space_id,
             model_name_l,
             model_meta,
         })
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        // space id dump
+        <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id);
+        // model name
         buf.extend(data.model_name.as_bytes());
+        // model dump
         <obj::ModelLayoutRef as PersistObject>::obj_enc(
             buf,
             obj::ModelLayoutRef::from((data.model, data.model_read)),
         )
     }
     unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
-        let space_name =
-            inf::dec::utils::decode_string(s, md.space_name_l as usize)?.into_boxed_str();
+        let space_id = <super::SpaceID as PersistObject>::obj_dec(s, md.space_id_meta)?;
         let model_name =
             inf::dec::utils::decode_string(s, md.model_name_l as usize)?.into_boxed_str();
         let model = <obj::ModelLayoutRef as PersistObject>::obj_dec(s, md.model_meta)?;
         Ok(CreateModelTxnRestorePL {
-            space_name,
-            space_uuid: md.space_uuid,
+            space_id,
             model_name,
             model,
         })
@@ -148,12 +240,11 @@ impl<'a> GNSEvent for CreateModelTxn<'a> {
     type RestoreType = CreateModelTxnRestorePL;
     fn update_global_state(
         CreateModelTxnRestorePL {
-            space_name,
-            space_uuid,
+            space_id,
             model_name,
             model,
         }: Self::RestoreType,
-        gns: &crate::engine::core::GlobalNS,
+        gns: &GlobalNS,
     ) -> crate::engine::txn::TransactionResult<()> {
         let rgns = gns.spaces().read();
         /*
@@ -164,8 +255,8 @@ impl<'a> GNSEvent for CreateModelTxn<'a> {
             There is no evident way about how this is going to be handled, but the ideal way would be to keep
             versioned index of schemas.
         */
-        match rgns.st_get(&space_name) {
-            Some(space) if space.get_uuid() == space_uuid => {
+        match rgns.st_get(&space_id.name) {
+            Some(space) if space.get_uuid() == space_id.uuid => {
                 if space._create_model(&model_name, model).is_ok() {
                     Ok(())
                 } else {
@@ -177,3 +268,387 @@ impl<'a> GNSEvent for CreateModelTxn<'a> {
         }
     }
 }
+
+/*
+    alter model add
+*/
+
+pub struct AlterModelAddTxn<'a>(PhantomData<&'a ()>);
+#[derive(Debug, Clone, Copy)]
+pub struct AlterModelAddTxnCommitPL<'a> {
+    space_id: super::SpaceIDRef<'a>,
+    model_id: ModelIDRef<'a>,
+    new_fields: &'a IndexSTSeqCns<Box<str>, Field>,
+}
+pub struct AlterModelAddTxnMD {
+    space_id_meta: super::SpaceIDMD,
+    model_id_meta: ModelIDMD,
+    new_field_c: u64,
+}
+pub struct AlterModelAddTxnRestorePL {
+    space_id: super::SpaceIDRes,
+    model_id: ModelIDRes,
+    new_fields: IndexSTSeqCns<Box<str>, Field>,
+}
+impl<'a> PersistObject for AlterModelAddTxn<'a> {
+    const METADATA_SIZE: usize = <super::SpaceID as PersistObject>::METADATA_SIZE
+        + <ModelID as PersistObject>::METADATA_SIZE
+        + sizeof!(u64);
+    type InputType = AlterModelAddTxnCommitPL<'a>;
+    type OutputType = AlterModelAddTxnRestorePL;
+    type Metadata = AlterModelAddTxnMD;
+    fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
+        scanner.has_left((md.space_id_meta.space_name_l + md.model_id_meta.model_name_l) as usize)
+    }
+    fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
+        <ModelID as PersistObject>::meta_enc(buf, data.model_id);
+        buf.extend(data.new_fields.st_len().u64_bytes_le());
+    }
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+        let space_id_meta = <super::SpaceID as PersistObject>::meta_dec(scanner)?;
+        let model_id_meta = <ModelID as PersistObject>::meta_dec(scanner)?;
+        let new_field_c = u64::from_le_bytes(scanner.next_chunk());
+        Ok(AlterModelAddTxnMD {
+            space_id_meta,
+            model_id_meta,
+            new_field_c,
+        })
+    }
+    fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id);
+        <ModelID as PersistObject>::obj_enc(buf, data.model_id);
+        <map::PersistMapImpl<map::FieldMapSpec> as PersistObject>::obj_enc(buf, data.new_fields);
+    }
+    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+        let space_id = <super::SpaceID as PersistObject>::obj_dec(s, md.space_id_meta)?;
+        let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_meta)?;
+        let new_fields = <map::PersistMapImpl<map::FieldMapSpec> as PersistObject>::obj_dec(
+            s,
+            map::MapIndexSizeMD(md.new_field_c as usize),
+        )?;
+        Ok(AlterModelAddTxnRestorePL {
+            space_id,
+            model_id,
+            new_fields,
+        })
+    }
+}
+
+impl<'a> GNSEvent for AlterModelAddTxn<'a> {
+    const OPC: u16 = 4;
+    type CommitType = AlterModelAddTxnCommitPL<'a>;
+    type RestoreType = AlterModelAddTxnRestorePL;
+    fn update_global_state(
+        AlterModelAddTxnRestorePL {
+            space_id,
+            model_id,
+            new_fields,
+        }: Self::RestoreType,
+        gns: &GlobalNS,
+    ) -> crate::engine::txn::TransactionResult<()> {
+        with_model(gns, &space_id, &model_id, |model| {
+            let mut wmodel = model.intent_write_model();
+            for (i, (field_name, field)) in new_fields.stseq_ord_kv().enumerate() {
+                if !wmodel
+                    .fields_mut()
+                    .st_insert(field_name.to_owned(), field.clone())
+                {
+                    // rollback; corrupted
+                    new_fields.stseq_ord_key().take(i).for_each(|field_id| {
+                        let _ = wmodel.fields_mut().st_delete(field_id);
+                    });
+                    return Err(TransactionError::OnRestoreDataConflictMismatch);
+                }
+            }
+            Ok(())
+        })
+    }
+}
+
+/*
+    alter model remove
+*/
+
+pub struct AlterModelRemoveTxn<'a>(PhantomData<&'a ()>);
+#[derive(Debug, Clone, Copy)]
+pub struct AlterModelRemoveTxnCommitPL<'a> {
+    space_id: super::SpaceIDRef<'a>,
+    model_id: ModelIDRef<'a>,
+    removed_fields: &'a [Ident<'a>],
+}
+pub struct AlterModelRemoveTxnMD {
+    space_id_meta: super::SpaceIDMD,
+    model_id_meta: ModelIDMD,
+    remove_field_c: u64,
+}
+pub struct AlterModelRemoveTxnRestorePL {
+    space_id: super::SpaceIDRes,
+    model_id: ModelIDRes,
+    removed_fields: Box<[Box<str>]>,
+}
+
+impl<'a> PersistObject for AlterModelRemoveTxn<'a> {
+    const METADATA_SIZE: usize = <super::SpaceID as PersistObject>::METADATA_SIZE
+        + <ModelID as PersistObject>::METADATA_SIZE
+        + sizeof!(u64);
+    type InputType = AlterModelRemoveTxnCommitPL<'a>;
+    type OutputType = AlterModelRemoveTxnRestorePL;
+    type Metadata = AlterModelRemoveTxnMD;
+    fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
+        scanner.has_left((md.space_id_meta.space_name_l + md.model_id_meta.model_name_l) as usize)
+    }
+    fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
+        <ModelID as PersistObject>::meta_enc(buf, data.model_id);
+        buf.extend(data.removed_fields.len().u64_bytes_le());
+    }
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+        let space_id_meta = <super::SpaceID as PersistObject>::meta_dec(scanner)?;
+        let model_id_meta = <ModelID as PersistObject>::meta_dec(scanner)?;
+        Ok(AlterModelRemoveTxnMD {
+            space_id_meta,
+            model_id_meta,
+            remove_field_c: u64::from_le_bytes(scanner.next_chunk()),
+        })
+    }
+    fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id);
+        <ModelID as PersistObject>::obj_enc(buf, data.model_id);
+        for field in data.removed_fields {
+            buf.extend(field.len().u64_bytes_le());
+            buf.extend(field.as_bytes());
+        }
+    }
+    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+        let space_id = <super::SpaceID as PersistObject>::obj_dec(s, md.space_id_meta)?;
+        let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_meta)?;
+        let mut removed_fields = Vec::with_capacity(md.remove_field_c as usize);
+        while !s.eof()
+            & (removed_fields.len() as u64 != md.remove_field_c)
+            & s.has_left(sizeof!(u64))
+        {
+            let len = u64::from_le_bytes(s.next_chunk()) as usize;
+            if !s.has_left(len) {
+                break;
+            }
+            removed_fields.push(inf::dec::utils::decode_string(s, len)?.into_boxed_str());
+        }
+        if removed_fields.len() as u64 != md.remove_field_c {
+            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+        }
+        Ok(AlterModelRemoveTxnRestorePL {
+            space_id,
+            model_id,
+            removed_fields: removed_fields.into_boxed_slice(),
+        })
+    }
+}
+
+impl<'a> GNSEvent for AlterModelRemoveTxn<'a> {
+    const OPC: u16 = 5;
+    type CommitType = AlterModelRemoveTxnCommitPL<'a>;
+    type RestoreType = AlterModelRemoveTxnRestorePL;
+    fn update_global_state(
+        AlterModelRemoveTxnRestorePL {
+            space_id,
+            model_id,
+            removed_fields,
+        }: Self::RestoreType,
+        gns: &GlobalNS,
+    ) -> crate::engine::txn::TransactionResult<()> {
+        with_model(gns, &space_id, &model_id, |model| {
+            let mut iwm = model.intent_write_model();
+            let mut removed_fields_rb = vec![];
+            for removed_field in removed_fields.iter() {
+                match iwm.fields_mut().st_delete_return(removed_field) {
+                    Some(field) => {
+                        removed_fields_rb.push((removed_field as &str, field));
+                    }
+                    None => {
+                        // rollback
+                        removed_fields_rb.into_iter().for_each(|(field_id, field)| {
+                            let _ = iwm.fields_mut().st_insert(field_id.into(), field);
+                        });
+                        return Err(TransactionError::OnRestoreDataConflictMismatch);
+                    }
+                }
+            }
+            Ok(())
+        })
+    }
+}
+
+/*
+    alter model update
+*/
+
+pub struct AlterModelUpdateTxn<'a>(PhantomData<&'a ()>);
+#[derive(Debug, Clone, Copy)]
+pub struct AlterModelUpdateTxnCommitPL<'a> {
+    space_id: super::SpaceIDRef<'a>,
+    model_id: ModelIDRef<'a>,
+    updated_fields: &'a IndexST<Box<str>, Field>,
+}
+pub struct AlterModelUpdateTxnMD {
+    space_id_md: super::SpaceIDMD,
+    model_id_md: ModelIDMD,
+    updated_field_c: u64,
+}
+pub struct AlterModelUpdateTxnRestorePL {
+    space_id: super::SpaceIDRes,
+    model_id: ModelIDRes,
+    updated_fields: IndexST<Box<str>, Field>,
+}
+
+impl<'a> PersistObject for AlterModelUpdateTxn<'a> {
+    const METADATA_SIZE: usize = <super::SpaceID as PersistObject>::METADATA_SIZE
+        + <ModelID as PersistObject>::METADATA_SIZE
+        + sizeof!(u64);
+    type InputType = AlterModelUpdateTxnCommitPL<'a>;
+    type OutputType = AlterModelUpdateTxnRestorePL;
+    type Metadata = AlterModelUpdateTxnMD;
+    fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
+        scanner
+            .has_left(md.space_id_md.space_name_l as usize + md.model_id_md.model_name_l as usize)
+    }
+    fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
+        <ModelID as PersistObject>::meta_enc(buf, data.model_id);
+        buf.extend(data.updated_fields.st_len().u64_bytes_le());
+    }
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+        let space_id_md = <super::SpaceID as PersistObject>::meta_dec(scanner)?;
+        let model_id_md = <ModelID as PersistObject>::meta_dec(scanner)?;
+        Ok(AlterModelUpdateTxnMD {
+            space_id_md,
+            model_id_md,
+            updated_field_c: u64::from_le_bytes(scanner.next_chunk()),
+        })
+    }
+    fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id);
+        <ModelID as PersistObject>::obj_enc(buf, data.model_id);
+        <map::PersistMapImpl<map::FieldMapSpecST> as PersistObject>::obj_enc(
+            buf,
+            data.updated_fields,
+        );
+    }
+    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+        let space_id = <super::SpaceID as PersistObject>::obj_dec(s, md.space_id_md)?;
+        let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_md)?;
+        let updated_fields = <map::PersistMapImpl<map::FieldMapSpecST> as PersistObject>::obj_dec(
+            s,
+            map::MapIndexSizeMD(md.updated_field_c as usize),
+        )?;
+        Ok(AlterModelUpdateTxnRestorePL {
+            space_id,
+            model_id,
+            updated_fields,
+        })
+    }
+}
+
+impl<'a> GNSEvent for AlterModelUpdateTxn<'a> {
+    const OPC: u16 = 6;
+    type CommitType = AlterModelUpdateTxnCommitPL<'a>;
+    type RestoreType = AlterModelUpdateTxnRestorePL;
+    fn update_global_state(
+        AlterModelUpdateTxnRestorePL {
+            space_id,
+            model_id,
+            updated_fields,
+        }: Self::RestoreType,
+        gns: &GlobalNS,
+    ) -> TransactionResult<()> {
+        with_model(gns, &space_id, &model_id, |model| {
+            let mut iwm = model.intent_write_model();
+            let mut fields_rb = vec![];
+            for (field_id, field) in updated_fields.iter() {
+                match iwm.fields_mut().st_update_return(field_id, field.clone()) {
+                    Some(f) => fields_rb.push((field_id as &str, f)),
+                    None => {
+                        // rollback
+                        fields_rb.into_iter().for_each(|(field_id, field)| {
+                            let _ = iwm.fields_mut().st_update(field_id, field);
+                        });
+                        return Err(TransactionError::OnRestoreDataConflictMismatch);
+                    }
+                }
+            }
+            Ok(())
+        })
+    }
+}
+
+/*
+    drop model
+*/
+
+pub struct DropModelTxn<'a>(PhantomData<&'a ()>);
+#[derive(Debug, Clone, Copy)]
+pub struct DropModelTxnCommitPL<'a> {
+    space_id: super::SpaceIDRef<'a>,
+    model_id: ModelIDRef<'a>,
+}
+pub struct DropModelTxnMD {
+    space_id_md: super::SpaceIDMD,
+    model_id_md: ModelIDMD,
+}
+pub struct DropModelTxnRestorePL {
+    space_id: super::SpaceIDRes,
+    model_id: ModelIDRes,
+}
+impl<'a> PersistObject for DropModelTxn<'a> {
+    const METADATA_SIZE: usize = <super::SpaceID as PersistObject>::METADATA_SIZE
+        + <ModelID as PersistObject>::METADATA_SIZE;
+    type InputType = DropModelTxnCommitPL<'a>;
+    type OutputType = DropModelTxnRestorePL;
+    type Metadata = DropModelTxnMD;
+    fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
+        scanner
+            .has_left(md.space_id_md.space_name_l as usize + md.model_id_md.model_name_l as usize)
+    }
+    fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
+        <ModelID as PersistObject>::meta_enc(buf, data.model_id);
+    }
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+        let space_id_md = <super::SpaceID as PersistObject>::meta_dec(scanner)?;
+        let model_id_md = <ModelID as PersistObject>::meta_dec(scanner)?;
+        Ok(DropModelTxnMD {
+            space_id_md,
+            model_id_md,
+        })
+    }
+    fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id);
+        <ModelID as PersistObject>::obj_enc(buf, data.model_id);
+    }
+    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+        let space_id = <super::SpaceID as PersistObject>::obj_dec(s, md.space_id_md)?;
+        let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_md)?;
+        Ok(DropModelTxnRestorePL { space_id, model_id })
+    }
+}
+
+impl<'a> GNSEvent for DropModelTxn<'a> {
+    const OPC: u16 = 7;
+    type CommitType = DropModelTxnCommitPL<'a>;
+    type RestoreType = DropModelTxnRestorePL;
+    fn update_global_state(
+        DropModelTxnRestorePL { space_id, model_id }: Self::RestoreType,
+        gns: &GlobalNS,
+    ) -> TransactionResult<()> {
+        with_space(gns, &space_id, |space| {
+            let mut wgns = space.models().write();
+            match wgns.st_delete_if(&model_id.model_name, |mdl| {
+                mdl.get_uuid() == model_id.model_uuid
+            }) {
+                Some(true) => Ok(()),
+                Some(false) => return Err(TransactionError::OnRestoreDataConflictMismatch),
+                None => Err(TransactionError::OnRestoreDataMissing),
+            }
+        })
+    }
+}
diff --git a/server/src/engine/txn/gns/space.rs b/server/src/engine/txn/gns/space.rs
index 3febe374..865dcf9b 100644
--- a/server/src/engine/txn/gns/space.rs
+++ b/server/src/engine/txn/gns/space.rs
@@ -144,33 +144,30 @@ pub struct AlterSpaceTxn<'a>(PhantomData<&'a ()>);
 
 impl<'a> AlterSpaceTxn<'a> {
     pub const fn new_commit(
-        space_uuid: Uuid,
-        space_name: &'a str,
+        uuid: Uuid,
+        name: &'a str,
         space_meta: &'a DictGeneric,
     ) -> AlterSpaceTxnCommitPL<'a> {
         AlterSpaceTxnCommitPL {
-            space_uuid,
-            space_name,
+            space_id: super::SpaceIDRef { uuid, name },
             space_meta,
         }
     }
 }
 
 pub struct AlterSpaceTxnMD {
-    uuid: Uuid,
-    space_name_l: u64,
+    space_id_meta: super::SpaceIDMD,
     dict_len: u64,
 }
 
 #[derive(Clone, Copy)]
 pub struct AlterSpaceTxnCommitPL<'a> {
-    space_uuid: Uuid,
-    space_name: &'a str,
+    space_id: super::SpaceIDRef<'a>,
     space_meta: &'a DictGeneric,
 }
 
 pub struct AlterSpaceTxnRestorePL {
-    space_name: Box<str>,
+    space_id: super::SpaceIDRes,
     space_meta: DictGeneric,
 }
 
@@ -180,33 +177,30 @@ impl<'a> PersistObject for AlterSpaceTxn<'a> {
     type OutputType = AlterSpaceTxnRestorePL;
     type Metadata = AlterSpaceTxnMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
-        scanner.has_left(md.space_name_l as usize)
+        scanner.has_left(md.space_id_meta.space_name_l as usize)
     }
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        buf.extend(data.space_uuid.to_le_bytes());
-        buf.extend(data.space_name.len().u64_bytes_le());
+        <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
         buf.extend(data.space_meta.len().u64_bytes_le());
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
         Ok(AlterSpaceTxnMD {
-            uuid: Uuid::from_bytes(scanner.next_chunk()),
-            space_name_l: u64::from_le_bytes(scanner.next_chunk()),
+            space_id_meta: <super::SpaceID as PersistObject>::meta_dec(scanner)?,
             dict_len: u64::from_le_bytes(scanner.next_chunk()),
         })
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        buf.extend(data.space_name.as_bytes());
+        <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id);
         <map::PersistMapImpl<map::GenericDictSpec> as PersistObject>::obj_enc(buf, data.space_meta);
     }
     unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
-        let space_name =
-            inf::dec::utils::decode_string(s, md.space_name_l as usize)?.into_boxed_str();
+        let space_id = <super::SpaceID as PersistObject>::obj_dec(s, md.space_id_meta)?;
         let space_meta = <map::PersistMapImpl<map::GenericDictSpec> as PersistObject>::obj_dec(
             s,
             map::MapIndexSizeMD(md.dict_len as usize),
         )?;
         Ok(AlterSpaceTxnRestorePL {
-            space_name,
+            space_id,
             space_meta,
         })
     }
@@ -221,13 +215,13 @@ impl<'a> GNSEvent for AlterSpaceTxn<'a> {
 
     fn update_global_state(
         AlterSpaceTxnRestorePL {
-            space_name,
+            space_id,
             space_meta,
         }: Self::RestoreType,
         gns: &crate::engine::core::GlobalNS,
     ) -> TransactionResult<()> {
         let gns = gns.spaces().read();
-        match gns.st_get(&space_name) {
+        match gns.st_get(&space_id.name) {
             Some(space) => {
                 let mut wmeta = space.metadata().env().write();
                 space_meta
@@ -248,67 +242,50 @@ impl<'a> GNSEvent for AlterSpaceTxn<'a> {
 pub struct DropSpaceTxn<'a>(PhantomData<&'a ()>);
 
 impl<'a> DropSpaceTxn<'a> {
-    pub const fn new_commit(space_name: &'a str, uuid: Uuid) -> DropSpaceTxnCommitPL<'a> {
-        DropSpaceTxnCommitPL { space_name, uuid }
+    pub const fn new_commit(name: &'a str, uuid: Uuid) -> DropSpaceTxnCommitPL<'a> {
+        DropSpaceTxnCommitPL {
+            space_id: super::SpaceIDRef { uuid, name },
+        }
     }
 }
 
-pub struct DropSpaceTxnMD {
-    space_name_l: u64,
-    uuid: Uuid,
-}
 #[derive(Clone, Copy)]
 pub struct DropSpaceTxnCommitPL<'a> {
-    space_name: &'a str,
-    uuid: Uuid,
-}
-
-pub struct DropSpaceTxnRestorePL {
-    uuid: Uuid,
-    space_name: Box<str>,
+    space_id: super::SpaceIDRef<'a>,
 }
 
 impl<'a> PersistObject for DropSpaceTxn<'a> {
     const METADATA_SIZE: usize = sizeof!(u128) + sizeof!(u64);
     type InputType = DropSpaceTxnCommitPL<'a>;
-    type OutputType = DropSpaceTxnRestorePL;
-    type Metadata = DropSpaceTxnMD;
+    type OutputType = super::SpaceIDRes;
+    type Metadata = super::SpaceIDMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
         scanner.has_left(md.space_name_l as usize)
     }
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        buf.extend(data.space_name.len().u64_bytes_le());
-        buf.extend(data.uuid.to_le_bytes());
+        <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
-        Ok(DropSpaceTxnMD {
-            space_name_l: u64::from_le_bytes(scanner.next_chunk()),
-            uuid: Uuid::from_bytes(scanner.next_chunk()),
-        })
+        <super::SpaceID as PersistObject>::meta_dec(scanner)
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        buf.extend(data.space_name.as_bytes());
+        <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id)
     }
     unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
-        let space_name =
-            inf::dec::utils::decode_string(s, md.space_name_l as usize)?.into_boxed_str();
-        Ok(DropSpaceTxnRestorePL {
-            uuid: md.uuid,
-            space_name,
-        })
+        <super::SpaceID as PersistObject>::obj_dec(s, md)
     }
 }
 
 impl<'a> GNSEvent for DropSpaceTxn<'a> {
     const OPC: u16 = 2;
     type CommitType = DropSpaceTxnCommitPL<'a>;
-    type RestoreType = DropSpaceTxnRestorePL;
+    type RestoreType = super::SpaceIDRes;
     fn update_global_state(
-        DropSpaceTxnRestorePL { uuid, space_name }: Self::RestoreType,
+        super::SpaceIDRes { uuid, name }: Self::RestoreType,
         gns: &GlobalNS,
     ) -> TransactionResult<()> {
         let mut wgns = gns.spaces().write();
-        match wgns.entry(space_name) {
+        match wgns.entry(name) {
             std::collections::hash_map::Entry::Occupied(oe) => {
                 if oe.get().get_uuid() == uuid {
                     oe.remove_entry();

From 13a39f1e1f0dc0b2f7d1ca157a32c0ebedf7b391 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 24 Aug 2023 13:07:29 +0000
Subject: [PATCH 231/310] Simplify enc/dec misc procedures

---
 server/src/engine/storage/v1/inf/map.rs |  16 ++--
 server/src/engine/storage/v1/inf/obj.rs |  17 ++--
 server/src/engine/storage/v1/rw.rs      |   5 +-
 server/src/engine/txn/gns/mod.rs        |   2 +-
 server/src/engine/txn/gns/model.rs      | 113 +++++++++---------------
 server/src/engine/txn/gns/space.rs      |   4 +-
 6 files changed, 58 insertions(+), 99 deletions(-)

diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index 40949bcb..e3047e3b 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -68,9 +68,7 @@ where
         buf.extend(data.st_len().u64_bytes_le());
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
-        Ok(MapIndexSizeMD(
-            u64::from_le_bytes(scanner.next_chunk()) as usize
-        ))
+        Ok(MapIndexSizeMD(scanner.next_u64_le() as usize))
     }
     fn obj_enc(buf: &mut VecU8, map: Self::InputType) {
         for (key, val) in M::_get_iter(map) {
@@ -375,9 +373,9 @@ impl PersistMapSpec for FieldMapSpec {
     }
     unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::EntryMD> {
         Some(FieldMapEntryMD::new(
-            u64::from_le_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()),
+            scanner.next_u64_le(),
+            scanner.next_u64_le(),
+            scanner.next_u64_le(),
             scanner.next_byte(),
         ))
     }
@@ -440,9 +438,9 @@ impl PersistMapSpec for FieldMapSpecST {
     }
     unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::EntryMD> {
         Some(FieldMapEntryMD::new(
-            u64::from_le_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()),
+            scanner.next_u64_le(),
+            scanner.next_u64_le(),
+            scanner.next_u64_le(),
             scanner.next_byte(),
         ))
     }
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 2942ff9a..0b1a6426 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -91,10 +91,7 @@ impl<'a> PersistObject for LayerRef<'a> {
         buf.extend(0u64.to_le_bytes());
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
-        Ok(LayerMD::new(
-            u64::from_le_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()),
-        ))
+        Ok(LayerMD::new(scanner.next_u64_le(), scanner.next_u64_le()))
     }
     fn obj_enc(_: &mut VecU8, _: Self::InputType) {}
     unsafe fn obj_dec(_: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
@@ -149,8 +146,8 @@ impl<'a> PersistObject for FieldRef<'a> {
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
         Ok(FieldMD::new(
-            u64::from_le_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()),
+            scanner.next_u64_le(),
+            scanner.next_u64_le(),
             scanner.next_byte(),
         ))
     }
@@ -238,9 +235,9 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
         Ok(ModelLayoutMD::new(
             Uuid::from_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()),
+            scanner.next_u64_le(),
+            scanner.next_u64_le(),
+            scanner.next_u64_le(),
         ))
     }
     fn obj_enc(buf: &mut VecU8, ModelLayoutRef(mdl, irm): Self::InputType) {
@@ -308,7 +305,7 @@ impl<'a> PersistObject for SpaceLayoutRef<'a> {
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
         Ok(SpaceLayoutMD::new(
             Uuid::from_bytes(scanner.next_chunk()),
-            u64::from_le_bytes(scanner.next_chunk()) as usize,
+            scanner.next_u64_le() as usize,
         ))
     }
     fn obj_enc(buf: &mut VecU8, SpaceLayoutRef(_, space_meta): Self::InputType) {
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 1d2c6f01..e8f8f856 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -229,10 +229,7 @@ impl<'a> BufferedScanner<'a> {
 
 impl<'a> BufferedScanner<'a> {
     pub unsafe fn next_u64_le(&mut self) -> u64 {
-        let mut b = [0u8; sizeof!(u64)];
-        ptr::copy_nonoverlapping(self._cursor(), b.as_mut_ptr(), sizeof!(u64));
-        self._incr(sizeof!(u64));
-        u64::from_le_bytes(b)
+        u64::from_le_bytes(self.next_chunk())
     }
     pub unsafe fn next_chunk<const N: usize>(&mut self) -> [u8; N] {
         let mut b = [0u8; N];
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index b800295e..d5eb8219 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -156,7 +156,7 @@ impl<'a> PersistObject for SpaceID<'a> {
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
         Ok(SpaceIDMD {
             uuid: Uuid::from_bytes(scanner.next_chunk()),
-            space_name_l: u64::from_le_bytes(scanner.next_chunk()),
+            space_name_l: scanner.next_u64_le(),
         })
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index 85792239..b295201e 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -50,46 +50,54 @@ use {
 pub struct ModelID<'a>(PhantomData<&'a ()>);
 #[derive(Debug, Clone, Copy)]
 pub struct ModelIDRef<'a> {
+    space_id: super::SpaceIDRef<'a>,
     model_name: &'a str,
     model_uuid: Uuid,
     model_version: u64,
 }
 pub struct ModelIDRes {
+    space_id: super::SpaceIDRes,
     model_name: Box<str>,
     model_uuid: Uuid,
     model_version: u64,
 }
 pub struct ModelIDMD {
+    space_id: super::SpaceIDMD,
     model_name_l: u64,
     model_version: u64,
     model_uuid: Uuid,
 }
 
 impl<'a> PersistObject for ModelID<'a> {
-    const METADATA_SIZE: usize = sizeof!(u64, 2) + sizeof!(u128);
+    const METADATA_SIZE: usize =
+        sizeof!(u64, 2) + sizeof!(u128) + <super::SpaceID as PersistObject>::METADATA_SIZE;
     type InputType = ModelIDRef<'a>;
     type OutputType = ModelIDRes;
     type Metadata = ModelIDMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
-        scanner.has_left(md.model_name_l as usize)
+        scanner.has_left(md.model_name_l as usize + md.space_id.space_name_l as usize)
     }
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
         buf.extend(data.model_name.len().u64_bytes_le());
         buf.extend(data.model_version.to_le_bytes());
         buf.extend(data.model_uuid.to_le_bytes());
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
         Ok(ModelIDMD {
-            model_name_l: u64::from_le_bytes(scanner.next_chunk()),
-            model_version: u64::from_le_bytes(scanner.next_chunk()),
+            space_id: <super::SpaceID as PersistObject>::meta_dec(scanner)?,
+            model_name_l: scanner.next_u64_le(),
+            model_version: scanner.next_u64_le(),
             model_uuid: Uuid::from_bytes(scanner.next_chunk()),
         })
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
+        <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id);
         buf.extend(data.model_name.as_bytes());
     }
     unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
         Ok(ModelIDRes {
+            space_id: <super::SpaceID as PersistObject>::obj_dec(s, md.space_id)?,
             model_name: inf::dec::utils::decode_string(s, md.model_name_l as usize)?
                 .into_boxed_str(),
             model_uuid: md.model_uuid,
@@ -202,7 +210,7 @@ impl<'a> PersistObject for CreateModelTxn<'a> {
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
         let space_id = <super::SpaceID as PersistObject>::meta_dec(scanner)?;
-        let model_name_l = u64::from_le_bytes(scanner.next_chunk());
+        let model_name_l = scanner.next_u64_le();
         let model_meta = <obj::ModelLayoutRef as PersistObject>::meta_dec(scanner)?;
         Ok(CreateModelTxnMD {
             space_id_meta: space_id,
@@ -276,59 +284,50 @@ impl<'a> GNSEvent for CreateModelTxn<'a> {
 pub struct AlterModelAddTxn<'a>(PhantomData<&'a ()>);
 #[derive(Debug, Clone, Copy)]
 pub struct AlterModelAddTxnCommitPL<'a> {
-    space_id: super::SpaceIDRef<'a>,
     model_id: ModelIDRef<'a>,
     new_fields: &'a IndexSTSeqCns<Box<str>, Field>,
 }
 pub struct AlterModelAddTxnMD {
-    space_id_meta: super::SpaceIDMD,
     model_id_meta: ModelIDMD,
     new_field_c: u64,
 }
 pub struct AlterModelAddTxnRestorePL {
-    space_id: super::SpaceIDRes,
     model_id: ModelIDRes,
     new_fields: IndexSTSeqCns<Box<str>, Field>,
 }
 impl<'a> PersistObject for AlterModelAddTxn<'a> {
-    const METADATA_SIZE: usize = <super::SpaceID as PersistObject>::METADATA_SIZE
-        + <ModelID as PersistObject>::METADATA_SIZE
-        + sizeof!(u64);
+    const METADATA_SIZE: usize = <ModelID as PersistObject>::METADATA_SIZE + sizeof!(u64);
     type InputType = AlterModelAddTxnCommitPL<'a>;
     type OutputType = AlterModelAddTxnRestorePL;
     type Metadata = AlterModelAddTxnMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
-        scanner.has_left((md.space_id_meta.space_name_l + md.model_id_meta.model_name_l) as usize)
+        scanner.has_left(
+            (md.model_id_meta.space_id.space_name_l + md.model_id_meta.model_name_l) as usize,
+        )
     }
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
         <ModelID as PersistObject>::meta_enc(buf, data.model_id);
         buf.extend(data.new_fields.st_len().u64_bytes_le());
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
-        let space_id_meta = <super::SpaceID as PersistObject>::meta_dec(scanner)?;
         let model_id_meta = <ModelID as PersistObject>::meta_dec(scanner)?;
-        let new_field_c = u64::from_le_bytes(scanner.next_chunk());
+        let new_field_c = scanner.next_u64_le();
         Ok(AlterModelAddTxnMD {
-            space_id_meta,
             model_id_meta,
             new_field_c,
         })
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id);
         <ModelID as PersistObject>::obj_enc(buf, data.model_id);
         <map::PersistMapImpl<map::FieldMapSpec> as PersistObject>::obj_enc(buf, data.new_fields);
     }
     unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
-        let space_id = <super::SpaceID as PersistObject>::obj_dec(s, md.space_id_meta)?;
         let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_meta)?;
         let new_fields = <map::PersistMapImpl<map::FieldMapSpec> as PersistObject>::obj_dec(
             s,
             map::MapIndexSizeMD(md.new_field_c as usize),
         )?;
         Ok(AlterModelAddTxnRestorePL {
-            space_id,
             model_id,
             new_fields,
         })
@@ -341,13 +340,12 @@ impl<'a> GNSEvent for AlterModelAddTxn<'a> {
     type RestoreType = AlterModelAddTxnRestorePL;
     fn update_global_state(
         AlterModelAddTxnRestorePL {
-            space_id,
             model_id,
             new_fields,
         }: Self::RestoreType,
         gns: &GlobalNS,
     ) -> crate::engine::txn::TransactionResult<()> {
-        with_model(gns, &space_id, &model_id, |model| {
+        with_model(gns, &model_id.space_id, &model_id, |model| {
             let mut wmodel = model.intent_write_model();
             for (i, (field_name, field)) in new_fields.stseq_ord_kv().enumerate() {
                 if !wmodel
@@ -373,47 +371,40 @@ impl<'a> GNSEvent for AlterModelAddTxn<'a> {
 pub struct AlterModelRemoveTxn<'a>(PhantomData<&'a ()>);
 #[derive(Debug, Clone, Copy)]
 pub struct AlterModelRemoveTxnCommitPL<'a> {
-    space_id: super::SpaceIDRef<'a>,
     model_id: ModelIDRef<'a>,
     removed_fields: &'a [Ident<'a>],
 }
 pub struct AlterModelRemoveTxnMD {
-    space_id_meta: super::SpaceIDMD,
     model_id_meta: ModelIDMD,
     remove_field_c: u64,
 }
 pub struct AlterModelRemoveTxnRestorePL {
-    space_id: super::SpaceIDRes,
     model_id: ModelIDRes,
     removed_fields: Box<[Box<str>]>,
 }
 
 impl<'a> PersistObject for AlterModelRemoveTxn<'a> {
-    const METADATA_SIZE: usize = <super::SpaceID as PersistObject>::METADATA_SIZE
-        + <ModelID as PersistObject>::METADATA_SIZE
-        + sizeof!(u64);
+    const METADATA_SIZE: usize = <ModelID as PersistObject>::METADATA_SIZE + sizeof!(u64);
     type InputType = AlterModelRemoveTxnCommitPL<'a>;
     type OutputType = AlterModelRemoveTxnRestorePL;
     type Metadata = AlterModelRemoveTxnMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
-        scanner.has_left((md.space_id_meta.space_name_l + md.model_id_meta.model_name_l) as usize)
+        scanner.has_left(
+            (md.model_id_meta.space_id.space_name_l + md.model_id_meta.model_name_l) as usize,
+        )
     }
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
         <ModelID as PersistObject>::meta_enc(buf, data.model_id);
         buf.extend(data.removed_fields.len().u64_bytes_le());
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
-        let space_id_meta = <super::SpaceID as PersistObject>::meta_dec(scanner)?;
         let model_id_meta = <ModelID as PersistObject>::meta_dec(scanner)?;
         Ok(AlterModelRemoveTxnMD {
-            space_id_meta,
             model_id_meta,
-            remove_field_c: u64::from_le_bytes(scanner.next_chunk()),
+            remove_field_c: scanner.next_u64_le(),
         })
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id);
         <ModelID as PersistObject>::obj_enc(buf, data.model_id);
         for field in data.removed_fields {
             buf.extend(field.len().u64_bytes_le());
@@ -421,14 +412,13 @@ impl<'a> PersistObject for AlterModelRemoveTxn<'a> {
         }
     }
     unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
-        let space_id = <super::SpaceID as PersistObject>::obj_dec(s, md.space_id_meta)?;
         let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_meta)?;
         let mut removed_fields = Vec::with_capacity(md.remove_field_c as usize);
         while !s.eof()
             & (removed_fields.len() as u64 != md.remove_field_c)
             & s.has_left(sizeof!(u64))
         {
-            let len = u64::from_le_bytes(s.next_chunk()) as usize;
+            let len = s.next_u64_le() as usize;
             if !s.has_left(len) {
                 break;
             }
@@ -438,7 +428,6 @@ impl<'a> PersistObject for AlterModelRemoveTxn<'a> {
             return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
         }
         Ok(AlterModelRemoveTxnRestorePL {
-            space_id,
             model_id,
             removed_fields: removed_fields.into_boxed_slice(),
         })
@@ -451,13 +440,12 @@ impl<'a> GNSEvent for AlterModelRemoveTxn<'a> {
     type RestoreType = AlterModelRemoveTxnRestorePL;
     fn update_global_state(
         AlterModelRemoveTxnRestorePL {
-            space_id,
             model_id,
             removed_fields,
         }: Self::RestoreType,
         gns: &GlobalNS,
     ) -> crate::engine::txn::TransactionResult<()> {
-        with_model(gns, &space_id, &model_id, |model| {
+        with_model(gns, &model_id.space_id, &model_id, |model| {
             let mut iwm = model.intent_write_model();
             let mut removed_fields_rb = vec![];
             for removed_field in removed_fields.iter() {
@@ -486,48 +474,40 @@ impl<'a> GNSEvent for AlterModelRemoveTxn<'a> {
 pub struct AlterModelUpdateTxn<'a>(PhantomData<&'a ()>);
 #[derive(Debug, Clone, Copy)]
 pub struct AlterModelUpdateTxnCommitPL<'a> {
-    space_id: super::SpaceIDRef<'a>,
     model_id: ModelIDRef<'a>,
     updated_fields: &'a IndexST<Box<str>, Field>,
 }
 pub struct AlterModelUpdateTxnMD {
-    space_id_md: super::SpaceIDMD,
     model_id_md: ModelIDMD,
     updated_field_c: u64,
 }
 pub struct AlterModelUpdateTxnRestorePL {
-    space_id: super::SpaceIDRes,
     model_id: ModelIDRes,
     updated_fields: IndexST<Box<str>, Field>,
 }
 
 impl<'a> PersistObject for AlterModelUpdateTxn<'a> {
-    const METADATA_SIZE: usize = <super::SpaceID as PersistObject>::METADATA_SIZE
-        + <ModelID as PersistObject>::METADATA_SIZE
-        + sizeof!(u64);
+    const METADATA_SIZE: usize = <ModelID as PersistObject>::METADATA_SIZE + sizeof!(u64);
     type InputType = AlterModelUpdateTxnCommitPL<'a>;
     type OutputType = AlterModelUpdateTxnRestorePL;
     type Metadata = AlterModelUpdateTxnMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
-        scanner
-            .has_left(md.space_id_md.space_name_l as usize + md.model_id_md.model_name_l as usize)
+        scanner.has_left(
+            md.model_id_md.space_id.space_name_l as usize + md.model_id_md.model_name_l as usize,
+        )
     }
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
         <ModelID as PersistObject>::meta_enc(buf, data.model_id);
         buf.extend(data.updated_fields.st_len().u64_bytes_le());
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
-        let space_id_md = <super::SpaceID as PersistObject>::meta_dec(scanner)?;
         let model_id_md = <ModelID as PersistObject>::meta_dec(scanner)?;
         Ok(AlterModelUpdateTxnMD {
-            space_id_md,
             model_id_md,
-            updated_field_c: u64::from_le_bytes(scanner.next_chunk()),
+            updated_field_c: scanner.next_u64_le(),
         })
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id);
         <ModelID as PersistObject>::obj_enc(buf, data.model_id);
         <map::PersistMapImpl<map::FieldMapSpecST> as PersistObject>::obj_enc(
             buf,
@@ -535,14 +515,12 @@ impl<'a> PersistObject for AlterModelUpdateTxn<'a> {
         );
     }
     unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
-        let space_id = <super::SpaceID as PersistObject>::obj_dec(s, md.space_id_md)?;
         let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_md)?;
         let updated_fields = <map::PersistMapImpl<map::FieldMapSpecST> as PersistObject>::obj_dec(
             s,
             map::MapIndexSizeMD(md.updated_field_c as usize),
         )?;
         Ok(AlterModelUpdateTxnRestorePL {
-            space_id,
             model_id,
             updated_fields,
         })
@@ -555,13 +533,12 @@ impl<'a> GNSEvent for AlterModelUpdateTxn<'a> {
     type RestoreType = AlterModelUpdateTxnRestorePL;
     fn update_global_state(
         AlterModelUpdateTxnRestorePL {
-            space_id,
             model_id,
             updated_fields,
         }: Self::RestoreType,
         gns: &GlobalNS,
     ) -> TransactionResult<()> {
-        with_model(gns, &space_id, &model_id, |model| {
+        with_model(gns, &model_id.space_id, &model_id, |model| {
             let mut iwm = model.intent_write_model();
             let mut fields_rb = vec![];
             for (field_id, field) in updated_fields.iter() {
@@ -588,47 +565,37 @@ impl<'a> GNSEvent for AlterModelUpdateTxn<'a> {
 pub struct DropModelTxn<'a>(PhantomData<&'a ()>);
 #[derive(Debug, Clone, Copy)]
 pub struct DropModelTxnCommitPL<'a> {
-    space_id: super::SpaceIDRef<'a>,
     model_id: ModelIDRef<'a>,
 }
 pub struct DropModelTxnMD {
-    space_id_md: super::SpaceIDMD,
     model_id_md: ModelIDMD,
 }
 pub struct DropModelTxnRestorePL {
-    space_id: super::SpaceIDRes,
     model_id: ModelIDRes,
 }
 impl<'a> PersistObject for DropModelTxn<'a> {
-    const METADATA_SIZE: usize = <super::SpaceID as PersistObject>::METADATA_SIZE
-        + <ModelID as PersistObject>::METADATA_SIZE;
+    const METADATA_SIZE: usize = <ModelID as PersistObject>::METADATA_SIZE;
     type InputType = DropModelTxnCommitPL<'a>;
     type OutputType = DropModelTxnRestorePL;
     type Metadata = DropModelTxnMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
-        scanner
-            .has_left(md.space_id_md.space_name_l as usize + md.model_id_md.model_name_l as usize)
+        scanner.has_left(
+            md.model_id_md.space_id.space_name_l as usize + md.model_id_md.model_name_l as usize,
+        )
     }
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
         <ModelID as PersistObject>::meta_enc(buf, data.model_id);
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
-        let space_id_md = <super::SpaceID as PersistObject>::meta_dec(scanner)?;
         let model_id_md = <ModelID as PersistObject>::meta_dec(scanner)?;
-        Ok(DropModelTxnMD {
-            space_id_md,
-            model_id_md,
-        })
+        Ok(DropModelTxnMD { model_id_md })
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
-        <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id);
         <ModelID as PersistObject>::obj_enc(buf, data.model_id);
     }
     unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
-        let space_id = <super::SpaceID as PersistObject>::obj_dec(s, md.space_id_md)?;
         let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_md)?;
-        Ok(DropModelTxnRestorePL { space_id, model_id })
+        Ok(DropModelTxnRestorePL { model_id })
     }
 }
 
@@ -637,10 +604,10 @@ impl<'a> GNSEvent for DropModelTxn<'a> {
     type CommitType = DropModelTxnCommitPL<'a>;
     type RestoreType = DropModelTxnRestorePL;
     fn update_global_state(
-        DropModelTxnRestorePL { space_id, model_id }: Self::RestoreType,
+        DropModelTxnRestorePL { model_id }: Self::RestoreType,
         gns: &GlobalNS,
     ) -> TransactionResult<()> {
-        with_space(gns, &space_id, |space| {
+        with_space(gns, &model_id.space_id, |space| {
             let mut wgns = space.models().write();
             match wgns.st_delete_if(&model_id.model_name, |mdl| {
                 mdl.get_uuid() == model_id.model_uuid
diff --git a/server/src/engine/txn/gns/space.rs b/server/src/engine/txn/gns/space.rs
index 865dcf9b..b4e1af57 100644
--- a/server/src/engine/txn/gns/space.rs
+++ b/server/src/engine/txn/gns/space.rs
@@ -97,7 +97,7 @@ impl<'a> PersistObject for CreateSpaceTxn<'a> {
         );
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
-        let space_name_l = u64::from_le_bytes(scanner.next_chunk());
+        let space_name_l = scanner.next_u64_le();
         let space_meta = <obj::SpaceLayoutRef as PersistObject>::meta_dec(scanner)?;
         Ok(CreateSpaceTxnMD {
             space_name_l,
@@ -186,7 +186,7 @@ impl<'a> PersistObject for AlterSpaceTxn<'a> {
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
         Ok(AlterSpaceTxnMD {
             space_id_meta: <super::SpaceID as PersistObject>::meta_dec(scanner)?,
-            dict_len: u64::from_le_bytes(scanner.next_chunk()),
+            dict_len: scanner.next_u64_le(),
         })
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {

From ea20611ebe25080046372b188f46943ed8b5434f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 24 Aug 2023 13:50:19 +0000
Subject: [PATCH 232/310] Cleanup gns txn impls

---
 server/src/engine/storage/v1/journal.rs |   4 +-
 server/src/engine/txn/gns/mod.rs        |  13 ++-
 server/src/engine/txn/gns/model.rs      | 106 +++++++++++++++---------
 server/src/engine/txn/gns/space.rs      |  83 +++++++------------
 server/src/engine/txn/gns/tests.rs      |  26 ++++++
 5 files changed, 137 insertions(+), 95 deletions(-)
 create mode 100644 server/src/engine/txn/gns/tests.rs

diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index a72bf512..d993f61e 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -408,9 +408,7 @@ impl<LF: RawFileIOInterface, TA: JournalAdapter> JournalWriter<LF, TA> {
         debug_assert!(TA::RECOVERY_PLUGIN);
         match self.append_event(event) {
             Ok(()) => Ok(()),
-            Err(_) => {
-                return self.appendrec_journal_reverse_entry();
-            }
+            Err(_) => compiler::cold_call(|| return self.appendrec_journal_reverse_entry()),
         }
     }
 }
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index d5eb8219..e72aa531 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -42,10 +42,15 @@ use {
 
 mod model;
 mod space;
+// test
+#[cfg(test)]
+mod tests;
 
 // re-exports
 pub use {
-    model::CreateModelTxn,
+    model::{
+        AlterModelAddTxn, AlterModelRemoveTxn, AlterModelUpdateTxn, CreateModelTxn, DropModelTxn,
+    },
     space::{AlterSpaceTxn, CreateSpaceTxn, DropSpaceTxn},
 };
 
@@ -58,7 +63,7 @@ pub struct GNSTransactionDriver {
 impl GNSTransactionDriver {
     /// Attempts to commit the given event into the journal, handling any possible recovery triggers and returning
     /// errors (if any)
-    pub fn try_commit<GE: GNSEvent>(&mut self, gns_event: GE::CommitType) -> TransactionResult<()> {
+    pub fn try_commit<GE: GNSEvent>(&mut self, gns_event: GE) -> TransactionResult<()> {
         let mut buf = vec![];
         buf.extend(GE::OPC.to_le_bytes());
         GE::encode_super_event(gns_event, &mut buf);
@@ -104,7 +109,7 @@ pub struct GNSSuperEvent(Box<[u8]>);
 /// Definition for an event in the GNS (DDL queries)
 pub trait GNSEvent
 where
-    Self: PersistObject<InputType = Self::CommitType, OutputType = Self::RestoreType> + Sized,
+    Self: PersistObject<InputType = Self, OutputType = Self::RestoreType> + Sized,
 {
     /// OPC for the event (unique)
     const OPC: u16;
@@ -113,7 +118,7 @@ where
     /// Expected type for a restore
     type RestoreType;
     /// Encodes the event into the given buffer
-    fn encode_super_event(commit: Self::CommitType, buf: &mut Vec<u8>) {
+    fn encode_super_event(commit: Self, buf: &mut Vec<u8>) {
         inf::enc::enc_full_into_buffer::<Self>(buf, commit)
     }
     /// Attempts to decode the event using the given scanner
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index b295201e..689018b2 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -144,22 +144,24 @@ fn with_model<T>(
     create model
 */
 
-/// Transaction for running a `create model ... (...) with {..}` query
-pub struct CreateModelTxn<'a>(PhantomData<&'a ()>);
+#[derive(Clone, Copy)]
+/// The commit payload for a `create model ... (...) with {...}` txn
+pub struct CreateModelTxn<'a> {
+    space_id: super::SpaceIDRef<'a>,
+    model_name: &'a str,
+    model: &'a Model,
+    model_read: &'a IRModel<'a>,
+}
 
 impl<'a> CreateModelTxn<'a> {
-    pub const fn new_commit(
-        space_name: &'a str,
-        space_uuid: Uuid,
+    pub const fn new(
+        space_id: super::SpaceIDRef<'a>,
         model_name: &'a str,
         model: &'a Model,
         model_read: &'a IRModel<'a>,
-    ) -> CreateModelTxnCommitPL<'a> {
-        CreateModelTxnCommitPL {
-            space_id: super::SpaceIDRef {
-                uuid: space_uuid,
-                name: space_name,
-            },
+    ) -> Self {
+        Self {
+            space_id,
             model_name,
             model,
             model_read,
@@ -167,14 +169,6 @@ impl<'a> CreateModelTxn<'a> {
     }
 }
 
-#[derive(Clone, Copy)]
-pub struct CreateModelTxnCommitPL<'a> {
-    space_id: super::SpaceIDRef<'a>,
-    model_name: &'a str,
-    model: &'a Model,
-    model_read: &'a IRModel<'a>,
-}
-
 pub struct CreateModelTxnRestorePL {
     space_id: super::SpaceIDRes,
     model_name: Box<str>,
@@ -191,7 +185,7 @@ impl<'a> PersistObject for CreateModelTxn<'a> {
     const METADATA_SIZE: usize = <super::SpaceID as PersistObject>::METADATA_SIZE
         + sizeof!(u64)
         + <obj::ModelLayoutRef<'a> as PersistObject>::METADATA_SIZE;
-    type InputType = CreateModelTxnCommitPL<'a>;
+    type InputType = CreateModelTxn<'a>;
     type OutputType = CreateModelTxnRestorePL;
     type Metadata = CreateModelTxnMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
@@ -244,7 +238,7 @@ impl<'a> PersistObject for CreateModelTxn<'a> {
 
 impl<'a> GNSEvent for CreateModelTxn<'a> {
     const OPC: u16 = 3;
-    type CommitType = CreateModelTxnCommitPL<'a>;
+    type CommitType = CreateModelTxn<'a>;
     type RestoreType = CreateModelTxnRestorePL;
     fn update_global_state(
         CreateModelTxnRestorePL {
@@ -281,12 +275,24 @@ impl<'a> GNSEvent for CreateModelTxn<'a> {
     alter model add
 */
 
-pub struct AlterModelAddTxn<'a>(PhantomData<&'a ()>);
 #[derive(Debug, Clone, Copy)]
-pub struct AlterModelAddTxnCommitPL<'a> {
+/// Transaction commit payload for an `alter model add ...` query
+pub struct AlterModelAddTxn<'a> {
     model_id: ModelIDRef<'a>,
     new_fields: &'a IndexSTSeqCns<Box<str>, Field>,
 }
+
+impl<'a> AlterModelAddTxn<'a> {
+    pub const fn new(
+        model_id: ModelIDRef<'a>,
+        new_fields: &'a IndexSTSeqCns<Box<str>, Field>,
+    ) -> Self {
+        Self {
+            model_id,
+            new_fields,
+        }
+    }
+}
 pub struct AlterModelAddTxnMD {
     model_id_meta: ModelIDMD,
     new_field_c: u64,
@@ -297,7 +303,7 @@ pub struct AlterModelAddTxnRestorePL {
 }
 impl<'a> PersistObject for AlterModelAddTxn<'a> {
     const METADATA_SIZE: usize = <ModelID as PersistObject>::METADATA_SIZE + sizeof!(u64);
-    type InputType = AlterModelAddTxnCommitPL<'a>;
+    type InputType = AlterModelAddTxn<'a>;
     type OutputType = AlterModelAddTxnRestorePL;
     type Metadata = AlterModelAddTxnMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
@@ -336,7 +342,7 @@ impl<'a> PersistObject for AlterModelAddTxn<'a> {
 
 impl<'a> GNSEvent for AlterModelAddTxn<'a> {
     const OPC: u16 = 4;
-    type CommitType = AlterModelAddTxnCommitPL<'a>;
+    type CommitType = AlterModelAddTxn<'a>;
     type RestoreType = AlterModelAddTxnRestorePL;
     fn update_global_state(
         AlterModelAddTxnRestorePL {
@@ -368,12 +374,20 @@ impl<'a> GNSEvent for AlterModelAddTxn<'a> {
     alter model remove
 */
 
-pub struct AlterModelRemoveTxn<'a>(PhantomData<&'a ()>);
 #[derive(Debug, Clone, Copy)]
-pub struct AlterModelRemoveTxnCommitPL<'a> {
+/// Transaction commit payload for an `alter model remove` transaction
+pub struct AlterModelRemoveTxn<'a> {
     model_id: ModelIDRef<'a>,
     removed_fields: &'a [Ident<'a>],
 }
+impl<'a> AlterModelRemoveTxn<'a> {
+    pub const fn new(model_id: ModelIDRef<'a>, removed_fields: &'a [Ident<'a>]) -> Self {
+        Self {
+            model_id,
+            removed_fields,
+        }
+    }
+}
 pub struct AlterModelRemoveTxnMD {
     model_id_meta: ModelIDMD,
     remove_field_c: u64,
@@ -385,7 +399,7 @@ pub struct AlterModelRemoveTxnRestorePL {
 
 impl<'a> PersistObject for AlterModelRemoveTxn<'a> {
     const METADATA_SIZE: usize = <ModelID as PersistObject>::METADATA_SIZE + sizeof!(u64);
-    type InputType = AlterModelRemoveTxnCommitPL<'a>;
+    type InputType = AlterModelRemoveTxn<'a>;
     type OutputType = AlterModelRemoveTxnRestorePL;
     type Metadata = AlterModelRemoveTxnMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
@@ -436,7 +450,7 @@ impl<'a> PersistObject for AlterModelRemoveTxn<'a> {
 
 impl<'a> GNSEvent for AlterModelRemoveTxn<'a> {
     const OPC: u16 = 5;
-    type CommitType = AlterModelRemoveTxnCommitPL<'a>;
+    type CommitType = AlterModelRemoveTxn<'a>;
     type RestoreType = AlterModelRemoveTxnRestorePL;
     fn update_global_state(
         AlterModelRemoveTxnRestorePL {
@@ -471,12 +485,24 @@ impl<'a> GNSEvent for AlterModelRemoveTxn<'a> {
     alter model update
 */
 
-pub struct AlterModelUpdateTxn<'a>(PhantomData<&'a ()>);
 #[derive(Debug, Clone, Copy)]
-pub struct AlterModelUpdateTxnCommitPL<'a> {
+/// Transaction commit payload for an `alter model update ...` query
+pub struct AlterModelUpdateTxn<'a> {
     model_id: ModelIDRef<'a>,
     updated_fields: &'a IndexST<Box<str>, Field>,
 }
+
+impl<'a> AlterModelUpdateTxn<'a> {
+    pub const fn new(
+        model_id: ModelIDRef<'a>,
+        updated_fields: &'a IndexST<Box<str>, Field>,
+    ) -> Self {
+        Self {
+            model_id,
+            updated_fields,
+        }
+    }
+}
 pub struct AlterModelUpdateTxnMD {
     model_id_md: ModelIDMD,
     updated_field_c: u64,
@@ -488,7 +514,7 @@ pub struct AlterModelUpdateTxnRestorePL {
 
 impl<'a> PersistObject for AlterModelUpdateTxn<'a> {
     const METADATA_SIZE: usize = <ModelID as PersistObject>::METADATA_SIZE + sizeof!(u64);
-    type InputType = AlterModelUpdateTxnCommitPL<'a>;
+    type InputType = AlterModelUpdateTxn<'a>;
     type OutputType = AlterModelUpdateTxnRestorePL;
     type Metadata = AlterModelUpdateTxnMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
@@ -529,7 +555,7 @@ impl<'a> PersistObject for AlterModelUpdateTxn<'a> {
 
 impl<'a> GNSEvent for AlterModelUpdateTxn<'a> {
     const OPC: u16 = 6;
-    type CommitType = AlterModelUpdateTxnCommitPL<'a>;
+    type CommitType = AlterModelUpdateTxn<'a>;
     type RestoreType = AlterModelUpdateTxnRestorePL;
     fn update_global_state(
         AlterModelUpdateTxnRestorePL {
@@ -562,11 +588,17 @@ impl<'a> GNSEvent for AlterModelUpdateTxn<'a> {
     drop model
 */
 
-pub struct DropModelTxn<'a>(PhantomData<&'a ()>);
 #[derive(Debug, Clone, Copy)]
-pub struct DropModelTxnCommitPL<'a> {
+/// Transaction commit payload for a `drop model ...` query
+pub struct DropModelTxn<'a> {
     model_id: ModelIDRef<'a>,
 }
+
+impl<'a> DropModelTxn<'a> {
+    pub const fn new(model_id: ModelIDRef<'a>) -> Self {
+        Self { model_id }
+    }
+}
 pub struct DropModelTxnMD {
     model_id_md: ModelIDMD,
 }
@@ -575,7 +607,7 @@ pub struct DropModelTxnRestorePL {
 }
 impl<'a> PersistObject for DropModelTxn<'a> {
     const METADATA_SIZE: usize = <ModelID as PersistObject>::METADATA_SIZE;
-    type InputType = DropModelTxnCommitPL<'a>;
+    type InputType = DropModelTxn<'a>;
     type OutputType = DropModelTxnRestorePL;
     type Metadata = DropModelTxnMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
@@ -601,7 +633,7 @@ impl<'a> PersistObject for DropModelTxn<'a> {
 
 impl<'a> GNSEvent for DropModelTxn<'a> {
     const OPC: u16 = 7;
-    type CommitType = DropModelTxnCommitPL<'a>;
+    type CommitType = DropModelTxn<'a>;
     type RestoreType = DropModelTxnRestorePL;
     fn update_global_state(
         DropModelTxnRestorePL { model_id }: Self::RestoreType,
diff --git a/server/src/engine/txn/gns/space.rs b/server/src/engine/txn/gns/space.rs
index b4e1af57..b99c7d31 100644
--- a/server/src/engine/txn/gns/space.rs
+++ b/server/src/engine/txn/gns/space.rs
@@ -29,7 +29,7 @@ use {
     crate::{
         engine::{
             core::{space::Space, GlobalNS},
-            data::{uuid::Uuid, DictGeneric},
+            data::DictGeneric,
             idx::STIndex,
             storage::v1::{
                 inf::{self, map, obj, PersistObject},
@@ -39,23 +39,23 @@ use {
         },
         util::EndianQW,
     },
-    std::marker::PhantomData,
 };
 
 /*
     create space
 */
 
-/// A transaction to run a `create space ...` operation
-pub struct CreateSpaceTxn<'a>(PhantomData<&'a ()>);
+#[derive(Clone, Copy)]
+/// Transaction commit payload for a `create space ...` query
+pub struct CreateSpaceTxn<'a> {
+    pub(crate) space_meta: &'a DictGeneric,
+    pub(crate) space_name: &'a str,
+    pub(crate) space: &'a Space,
+}
 
 impl<'a> CreateSpaceTxn<'a> {
-    pub const fn new_commit(
-        space_meta: &'a DictGeneric,
-        space_name: &'a str,
-        space: &'a Space,
-    ) -> CreateSpaceTxnCommitPL<'a> {
-        CreateSpaceTxnCommitPL {
+    pub const fn new(space_meta: &'a DictGeneric, space_name: &'a str, space: &'a Space) -> Self {
+        Self {
             space_meta,
             space_name,
             space,
@@ -63,13 +63,6 @@ impl<'a> CreateSpaceTxn<'a> {
     }
 }
 
-#[derive(Clone, Copy)]
-pub struct CreateSpaceTxnCommitPL<'a> {
-    pub(crate) space_meta: &'a DictGeneric,
-    pub(crate) space_name: &'a str,
-    pub(crate) space: &'a Space,
-}
-
 pub struct CreateSpaceTxnRestorePL {
     pub(crate) space_name: Box<str>,
     pub(crate) space: Space,
@@ -83,7 +76,7 @@ pub struct CreateSpaceTxnMD {
 impl<'a> PersistObject for CreateSpaceTxn<'a> {
     const METADATA_SIZE: usize =
         <obj::SpaceLayoutRef<'static> as PersistObject>::METADATA_SIZE + sizeof!(u64);
-    type InputType = CreateSpaceTxnCommitPL<'a>;
+    type InputType = CreateSpaceTxn<'a>;
     type OutputType = CreateSpaceTxnRestorePL;
     type Metadata = CreateSpaceTxnMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
@@ -118,7 +111,7 @@ impl<'a> PersistObject for CreateSpaceTxn<'a> {
 
 impl<'a> GNSEvent for CreateSpaceTxn<'a> {
     const OPC: u16 = 0;
-    type CommitType = CreateSpaceTxnCommitPL<'a>;
+    type CommitType = CreateSpaceTxn<'a>;
     type RestoreType = CreateSpaceTxnRestorePL;
     fn update_global_state(
         CreateSpaceTxnRestorePL { space_name, space }: CreateSpaceTxnRestorePL,
@@ -139,17 +132,17 @@ impl<'a> GNSEvent for CreateSpaceTxn<'a> {
     for now dump the entire meta
 */
 
-/// A transaction to run `alter space ...`
-pub struct AlterSpaceTxn<'a>(PhantomData<&'a ()>);
+#[derive(Clone, Copy)]
+/// Transaction payload for an `alter space ...` query
+pub struct AlterSpaceTxn<'a> {
+    space_id: super::SpaceIDRef<'a>,
+    space_meta: &'a DictGeneric,
+}
 
 impl<'a> AlterSpaceTxn<'a> {
-    pub const fn new_commit(
-        uuid: Uuid,
-        name: &'a str,
-        space_meta: &'a DictGeneric,
-    ) -> AlterSpaceTxnCommitPL<'a> {
-        AlterSpaceTxnCommitPL {
-            space_id: super::SpaceIDRef { uuid, name },
+    pub const fn new(space_id: super::SpaceIDRef<'a>, space_meta: &'a DictGeneric) -> Self {
+        Self {
+            space_id,
             space_meta,
         }
     }
@@ -160,12 +153,6 @@ pub struct AlterSpaceTxnMD {
     dict_len: u64,
 }
 
-#[derive(Clone, Copy)]
-pub struct AlterSpaceTxnCommitPL<'a> {
-    space_id: super::SpaceIDRef<'a>,
-    space_meta: &'a DictGeneric,
-}
-
 pub struct AlterSpaceTxnRestorePL {
     space_id: super::SpaceIDRes,
     space_meta: DictGeneric,
@@ -173,7 +160,7 @@ pub struct AlterSpaceTxnRestorePL {
 
 impl<'a> PersistObject for AlterSpaceTxn<'a> {
     const METADATA_SIZE: usize = sizeof!(u64, 2) + sizeof!(u128);
-    type InputType = AlterSpaceTxnCommitPL<'a>;
+    type InputType = AlterSpaceTxn<'a>;
     type OutputType = AlterSpaceTxnRestorePL;
     type Metadata = AlterSpaceTxnMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
@@ -208,9 +195,7 @@ impl<'a> PersistObject for AlterSpaceTxn<'a> {
 
 impl<'a> GNSEvent for AlterSpaceTxn<'a> {
     const OPC: u16 = 1;
-
-    type CommitType = AlterSpaceTxnCommitPL<'a>;
-
+    type CommitType = AlterSpaceTxn<'a>;
     type RestoreType = AlterSpaceTxnRestorePL;
 
     fn update_global_state(
@@ -238,25 +223,21 @@ impl<'a> GNSEvent for AlterSpaceTxn<'a> {
     drop space
 */
 
-/// A transaction to run `drop space ...`
-pub struct DropSpaceTxn<'a>(PhantomData<&'a ()>);
+#[derive(Clone, Copy)]
+/// Transaction commit payload for a `drop space ...` query
+pub struct DropSpaceTxn<'a> {
+    space_id: super::SpaceIDRef<'a>,
+}
 
 impl<'a> DropSpaceTxn<'a> {
-    pub const fn new_commit(name: &'a str, uuid: Uuid) -> DropSpaceTxnCommitPL<'a> {
-        DropSpaceTxnCommitPL {
-            space_id: super::SpaceIDRef { uuid, name },
-        }
+    pub const fn new(space_id: super::SpaceIDRef<'a>) -> Self {
+        Self { space_id }
     }
 }
 
-#[derive(Clone, Copy)]
-pub struct DropSpaceTxnCommitPL<'a> {
-    space_id: super::SpaceIDRef<'a>,
-}
-
 impl<'a> PersistObject for DropSpaceTxn<'a> {
     const METADATA_SIZE: usize = sizeof!(u128) + sizeof!(u64);
-    type InputType = DropSpaceTxnCommitPL<'a>;
+    type InputType = DropSpaceTxn<'a>;
     type OutputType = super::SpaceIDRes;
     type Metadata = super::SpaceIDMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
@@ -278,7 +259,7 @@ impl<'a> PersistObject for DropSpaceTxn<'a> {
 
 impl<'a> GNSEvent for DropSpaceTxn<'a> {
     const OPC: u16 = 2;
-    type CommitType = DropSpaceTxnCommitPL<'a>;
+    type CommitType = DropSpaceTxn<'a>;
     type RestoreType = super::SpaceIDRes;
     fn update_global_state(
         super::SpaceIDRes { uuid, name }: Self::RestoreType,
diff --git a/server/src/engine/txn/gns/tests.rs b/server/src/engine/txn/gns/tests.rs
new file mode 100644
index 00000000..7bc5d2f2
--- /dev/null
+++ b/server/src/engine/txn/gns/tests.rs
@@ -0,0 +1,26 @@
+/*
+ * Created on Thu Aug 24 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+

From 9133949bebaac159a478683591a858815f2f0b04 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 24 Aug 2023 17:47:29 +0000
Subject: [PATCH 233/310] Add IO tests for gns tx payloads

---
 server/src/engine/core/model/delta.rs   |   3 +
 server/src/engine/error.rs              |   9 +
 server/src/engine/storage/v1/inf/mod.rs |   8 +
 server/src/engine/txn/gns/mod.rs        |  19 +-
 server/src/engine/txn/gns/model.rs      |  80 ++++++--
 server/src/engine/txn/gns/space.rs      |  21 +-
 server/src/engine/txn/gns/tests.rs      |   1 +
 server/src/engine/txn/gns/tests/io.rs   | 250 ++++++++++++++++++++++++
 8 files changed, 360 insertions(+), 31 deletions(-)
 create mode 100644 server/src/engine/txn/gns/tests/io.rs

diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index a69af383..9ffa369f 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -197,6 +197,9 @@ impl DeltaVersion {
     fn step(&self) -> Self {
         Self(self.0 + 1)
     }
+    pub const fn value_u64(&self) -> u64 {
+        self.0
+    }
 }
 
 impl DeltaState {
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 54894643..a302fab8 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use super::txn::TransactionError;
+
 pub type LangResult<T> = Result<T, LangError>;
 pub type LexResult<T> = Result<T, LexError>;
 pub type DatabaseResult<T> = Result<T, DatabaseError>;
@@ -133,4 +135,11 @@ pub enum DatabaseError {
     /// field definition violation
     DmlConstraintViolationFieldTypedef,
     ServerError,
+    TransactionalError,
+}
+
+impl From<TransactionError> for DatabaseError {
+    fn from(_: TransactionError) -> Self {
+        Self::TransactionalError
+    }
 }
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index 03892bab..43486d23 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -236,6 +236,9 @@ pub mod enc {
     pub fn enc_full_into_buffer<Obj: PersistObject>(buf: &mut VecU8, obj: Obj::InputType) {
         Obj::default_full_enc(buf, obj)
     }
+    pub fn enc_full_self<Obj: PersistObject<InputType = Obj>>(obj: Obj) -> Vec<u8> {
+        enc_full::<Obj>(obj)
+    }
     // dict
     pub fn enc_dict_full<PM: PersistMapSpec>(dict: &PM::MapType) -> Vec<u8> {
         let mut v = vec![];
@@ -253,6 +256,7 @@ pub mod dec {
         super::{map, PersistMapSpec, PersistObject},
         crate::engine::storage::v1::{rw::BufferedScanner, SDSSResult},
     };
+    // obj
     pub fn dec_full<Obj: PersistObject>(data: &[u8]) -> SDSSResult<Obj::OutputType> {
         let mut scanner = BufferedScanner::new(data);
         dec_full_from_scanner::<Obj>(&mut scanner)
@@ -262,6 +266,10 @@ pub mod dec {
     ) -> SDSSResult<Obj::OutputType> {
         Obj::default_full_dec(scanner)
     }
+    pub fn dec_full_self<Obj: PersistObject<OutputType = Obj>>(data: &[u8]) -> SDSSResult<Obj> {
+        dec_full::<Obj>(data)
+    }
+    // dec
     pub fn dec_dict_full<PM: PersistMapSpec>(data: &[u8]) -> SDSSResult<PM::MapType> {
         let mut scanner = BufferedScanner::new(data);
         dec_dict_full_from_scanner::<PM>(&mut scanner)
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index e72aa531..449edfab 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -28,7 +28,7 @@ use {
     super::{TransactionError, TransactionResult},
     crate::{
         engine::{
-            core::GlobalNS,
+            core::{space::Space, GlobalNS},
             data::uuid::Uuid,
             storage::v1::{
                 inf::{self, PersistObject},
@@ -136,10 +136,27 @@ pub struct SpaceIDRef<'a> {
     uuid: Uuid,
     name: &'a str,
 }
+
+impl<'a> SpaceIDRef<'a> {
+    pub fn new(name: &'a str, space: &Space) -> Self {
+        Self {
+            uuid: space.get_uuid(),
+            name,
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
 pub struct SpaceIDRes {
     uuid: Uuid,
     name: Box<str>,
 }
+
+impl SpaceIDRes {
+    pub fn new(uuid: Uuid, name: Box<str>) -> Self {
+        Self { uuid, name }
+    }
+}
 struct SpaceID<'a>(PhantomData<SpaceIDRef<'a>>);
 pub struct SpaceIDMD {
     uuid: Uuid,
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index 689018b2..55c25060 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -55,12 +55,45 @@ pub struct ModelIDRef<'a> {
     model_uuid: Uuid,
     model_version: u64,
 }
+
+impl<'a> ModelIDRef<'a> {
+    pub fn new(
+        space_id: super::SpaceIDRef<'a>,
+        model_name: &'a str,
+        model_uuid: Uuid,
+        model_version: u64,
+    ) -> Self {
+        Self {
+            space_id,
+            model_name,
+            model_uuid,
+            model_version,
+        }
+    }
+}
+#[derive(Debug, PartialEq)]
 pub struct ModelIDRes {
     space_id: super::SpaceIDRes,
     model_name: Box<str>,
     model_uuid: Uuid,
     model_version: u64,
 }
+
+impl ModelIDRes {
+    pub fn new(
+        space_id: super::SpaceIDRes,
+        model_name: Box<str>,
+        model_uuid: Uuid,
+        model_version: u64,
+    ) -> Self {
+        Self {
+            space_id,
+            model_name,
+            model_uuid,
+            model_version,
+        }
+    }
+}
 pub struct ModelIDMD {
     space_id: super::SpaceIDMD,
     model_name_l: u64,
@@ -169,10 +202,12 @@ impl<'a> CreateModelTxn<'a> {
     }
 }
 
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub struct CreateModelTxnRestorePL {
-    space_id: super::SpaceIDRes,
-    model_name: Box<str>,
-    model: Model,
+    pub(super) space_id: super::SpaceIDRes,
+    pub(super) model_name: Box<str>,
+    pub(super) model: Model,
 }
 
 pub struct CreateModelTxnMD {
@@ -297,9 +332,11 @@ pub struct AlterModelAddTxnMD {
     model_id_meta: ModelIDMD,
     new_field_c: u64,
 }
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub struct AlterModelAddTxnRestorePL {
-    model_id: ModelIDRes,
-    new_fields: IndexSTSeqCns<Box<str>, Field>,
+    pub(super) model_id: ModelIDRes,
+    pub(super) new_fields: IndexSTSeqCns<Box<str>, Field>,
 }
 impl<'a> PersistObject for AlterModelAddTxn<'a> {
     const METADATA_SIZE: usize = <ModelID as PersistObject>::METADATA_SIZE + sizeof!(u64);
@@ -392,9 +429,10 @@ pub struct AlterModelRemoveTxnMD {
     model_id_meta: ModelIDMD,
     remove_field_c: u64,
 }
+#[derive(Debug, PartialEq)]
 pub struct AlterModelRemoveTxnRestorePL {
-    model_id: ModelIDRes,
-    removed_fields: Box<[Box<str>]>,
+    pub(super) model_id: ModelIDRes,
+    pub(super) removed_fields: Box<[Box<str>]>,
 }
 
 impl<'a> PersistObject for AlterModelRemoveTxn<'a> {
@@ -507,9 +545,10 @@ pub struct AlterModelUpdateTxnMD {
     model_id_md: ModelIDMD,
     updated_field_c: u64,
 }
+#[derive(Debug, PartialEq)]
 pub struct AlterModelUpdateTxnRestorePL {
-    model_id: ModelIDRes,
-    updated_fields: IndexST<Box<str>, Field>,
+    pub(super) model_id: ModelIDRes,
+    pub(super) updated_fields: IndexST<Box<str>, Field>,
 }
 
 impl<'a> PersistObject for AlterModelUpdateTxn<'a> {
@@ -602,13 +641,10 @@ impl<'a> DropModelTxn<'a> {
 pub struct DropModelTxnMD {
     model_id_md: ModelIDMD,
 }
-pub struct DropModelTxnRestorePL {
-    model_id: ModelIDRes,
-}
 impl<'a> PersistObject for DropModelTxn<'a> {
     const METADATA_SIZE: usize = <ModelID as PersistObject>::METADATA_SIZE;
     type InputType = DropModelTxn<'a>;
-    type OutputType = DropModelTxnRestorePL;
+    type OutputType = ModelIDRes;
     type Metadata = DropModelTxnMD;
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
         scanner.has_left(
@@ -626,24 +662,26 @@ impl<'a> PersistObject for DropModelTxn<'a> {
         <ModelID as PersistObject>::obj_enc(buf, data.model_id);
     }
     unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
-        let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_md)?;
-        Ok(DropModelTxnRestorePL { model_id })
+        <ModelID as PersistObject>::obj_dec(s, md.model_id_md)
     }
 }
 
 impl<'a> GNSEvent for DropModelTxn<'a> {
     const OPC: u16 = 7;
     type CommitType = DropModelTxn<'a>;
-    type RestoreType = DropModelTxnRestorePL;
+    type RestoreType = ModelIDRes;
     fn update_global_state(
-        DropModelTxnRestorePL { model_id }: Self::RestoreType,
+        ModelIDRes {
+            space_id,
+            model_name,
+            model_uuid,
+            model_version: _,
+        }: Self::RestoreType,
         gns: &GlobalNS,
     ) -> TransactionResult<()> {
-        with_space(gns, &model_id.space_id, |space| {
+        with_space(gns, &space_id, |space| {
             let mut wgns = space.models().write();
-            match wgns.st_delete_if(&model_id.model_name, |mdl| {
-                mdl.get_uuid() == model_id.model_uuid
-            }) {
+            match wgns.st_delete_if(&model_name, |mdl| mdl.get_uuid() == model_uuid) {
                 Some(true) => Ok(()),
                 Some(false) => return Err(TransactionError::OnRestoreDataConflictMismatch),
                 None => Err(TransactionError::OnRestoreDataMissing),
diff --git a/server/src/engine/txn/gns/space.rs b/server/src/engine/txn/gns/space.rs
index b99c7d31..5250a788 100644
--- a/server/src/engine/txn/gns/space.rs
+++ b/server/src/engine/txn/gns/space.rs
@@ -48,9 +48,9 @@ use {
 #[derive(Clone, Copy)]
 /// Transaction commit payload for a `create space ...` query
 pub struct CreateSpaceTxn<'a> {
-    pub(crate) space_meta: &'a DictGeneric,
-    pub(crate) space_name: &'a str,
-    pub(crate) space: &'a Space,
+    pub(super) space_meta: &'a DictGeneric,
+    pub(super) space_name: &'a str,
+    pub(super) space: &'a Space,
 }
 
 impl<'a> CreateSpaceTxn<'a> {
@@ -63,14 +63,16 @@ impl<'a> CreateSpaceTxn<'a> {
     }
 }
 
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub struct CreateSpaceTxnRestorePL {
-    pub(crate) space_name: Box<str>,
-    pub(crate) space: Space,
+    pub(super) space_name: Box<str>,
+    pub(super) space: Space,
 }
 
 pub struct CreateSpaceTxnMD {
-    pub(crate) space_name_l: u64,
-    pub(crate) space_meta: <obj::SpaceLayoutRef<'static> as PersistObject>::Metadata,
+    pub(super) space_name_l: u64,
+    pub(super) space_meta: <obj::SpaceLayoutRef<'static> as PersistObject>::Metadata,
 }
 
 impl<'a> PersistObject for CreateSpaceTxn<'a> {
@@ -153,9 +155,10 @@ pub struct AlterSpaceTxnMD {
     dict_len: u64,
 }
 
+#[derive(Debug, PartialEq)]
 pub struct AlterSpaceTxnRestorePL {
-    space_id: super::SpaceIDRes,
-    space_meta: DictGeneric,
+    pub(super) space_id: super::SpaceIDRes,
+    pub(super) space_meta: DictGeneric,
 }
 
 impl<'a> PersistObject for AlterSpaceTxn<'a> {
diff --git a/server/src/engine/txn/gns/tests.rs b/server/src/engine/txn/gns/tests.rs
index 7bc5d2f2..c9fa1028 100644
--- a/server/src/engine/txn/gns/tests.rs
+++ b/server/src/engine/txn/gns/tests.rs
@@ -24,3 +24,4 @@
  *
 */
 
+mod io;
diff --git a/server/src/engine/txn/gns/tests/io.rs b/server/src/engine/txn/gns/tests/io.rs
new file mode 100644
index 00000000..8a70c07d
--- /dev/null
+++ b/server/src/engine/txn/gns/tests/io.rs
@@ -0,0 +1,250 @@
+/*
+ * Created on Thu Aug 24 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::super::{
+        model::{self, ModelIDRef, ModelIDRes},
+        space, SpaceIDRef, SpaceIDRes,
+    },
+    crate::engine::{
+        core::{model::Model, space::Space},
+        storage::v1::inf::{dec, enc},
+    },
+};
+
+mod space_tests {
+    use super::{
+        dec, enc,
+        space::{
+            AlterSpaceTxn, AlterSpaceTxnRestorePL, CreateSpaceTxn, CreateSpaceTxnRestorePL,
+            DropSpaceTxn,
+        },
+        Space, SpaceIDRef,
+    };
+    #[test]
+    fn create() {
+        let orig_space = Space::empty();
+        let space_r = orig_space.metadata().env().read();
+        let txn = CreateSpaceTxn::new(&space_r, "myspace", &orig_space);
+        let encoded = enc::enc_full_self(txn);
+        let decoded = dec::dec_full::<CreateSpaceTxn>(&encoded).unwrap();
+        assert_eq!(
+            CreateSpaceTxnRestorePL {
+                space_name: "myspace".into(),
+                space: Space::empty_with_uuid(orig_space.get_uuid())
+            },
+            decoded
+        );
+    }
+    #[test]
+    fn alter() {
+        let space = Space::empty();
+        let space_r = space.metadata().env().read();
+        let txn = AlterSpaceTxn::new(SpaceIDRef::new("myspace", &space), &space_r);
+        let encoded = enc::enc_full_self(txn);
+        let decoded = dec::dec_full::<AlterSpaceTxn>(&encoded).unwrap();
+        assert_eq!(
+            AlterSpaceTxnRestorePL {
+                space_id: super::SpaceIDRes::new(space.get_uuid(), "myspace".into()),
+                space_meta: space_r.clone()
+            },
+            decoded
+        );
+    }
+    #[test]
+    fn drop() {
+        let space = Space::empty();
+        let txn = DropSpaceTxn::new(super::SpaceIDRef::new("myspace", &space));
+        let encoded = enc::enc_full_self(txn);
+        let decoded = dec::dec_full::<DropSpaceTxn>(&encoded).unwrap();
+        assert_eq!(
+            super::SpaceIDRes::new(space.get_uuid(), "myspace".into()),
+            decoded
+        );
+    }
+}
+
+mod model_tests {
+    use {
+        super::{
+            model::{
+                AlterModelAddTxn, AlterModelAddTxnRestorePL, AlterModelRemoveTxn,
+                AlterModelRemoveTxnRestorePL, AlterModelUpdateTxn, AlterModelUpdateTxnRestorePL,
+                CreateModelTxn, CreateModelTxnRestorePL, DropModelTxn,
+            },
+            Model, Space,
+        },
+        crate::engine::{
+            core::model::{Field, Layer},
+            data::{tag::TagSelector, uuid::Uuid},
+        },
+    };
+    fn default_space_model() -> (Space, Model) {
+        let space = Space::empty();
+        let model = Model::new_restore(
+            Uuid::new(),
+            "username".into(),
+            TagSelector::Str.into_full(),
+            into_dict!(
+                "password" => Field::new([Layer::bin()].into(), false),
+                "profile_pic" => Field::new([Layer::bin()].into(), true),
+            ),
+        );
+        (space, model)
+    }
+    #[test]
+    fn create() {
+        let (space, model) = default_space_model();
+        let irm = model.intent_read_model();
+        let txn = CreateModelTxn::new(
+            super::SpaceIDRef::new("myspace", &space),
+            "mymodel",
+            &model,
+            &irm,
+        );
+        let encoded = super::enc::enc_full_self(txn);
+        core::mem::drop(irm);
+        let decoded = super::dec::dec_full::<CreateModelTxn>(&encoded).unwrap();
+        assert_eq!(
+            CreateModelTxnRestorePL {
+                space_id: super::SpaceIDRes::new(space.get_uuid(), "myspace".into()),
+                model_name: "mymodel".into(),
+                model,
+            },
+            decoded
+        )
+    }
+    #[test]
+    fn alter_add() {
+        let (space, model) = default_space_model();
+        let new_fields = into_dict! {
+            "auth_2fa" => Field::new([Layer::bool()].into(), true),
+        };
+        let txn = AlterModelAddTxn::new(
+            super::ModelIDRef::new(
+                super::SpaceIDRef::new("myspace", &space),
+                "mymodel",
+                model.get_uuid(),
+                model.delta_state().current_version().value_u64(),
+            ),
+            &new_fields,
+        );
+        let encoded = super::enc::enc_full_self(txn);
+        let decoded = super::dec::dec_full::<AlterModelAddTxn>(&encoded).unwrap();
+        assert_eq!(
+            AlterModelAddTxnRestorePL {
+                model_id: super::ModelIDRes::new(
+                    super::SpaceIDRes::new(space.get_uuid(), "myspace".into()),
+                    "mymodel".into(),
+                    model.get_uuid(),
+                    model.delta_state().current_version().value_u64()
+                ),
+                new_fields
+            },
+            decoded
+        );
+    }
+    #[test]
+    fn alter_remove() {
+        let (space, model) = default_space_model();
+        let removed_fields = ["profile_pic".into()];
+        let txn = AlterModelRemoveTxn::new(
+            super::ModelIDRef::new(
+                super::SpaceIDRef::new("myspace", &space),
+                "mymodel",
+                model.get_uuid(),
+                model.delta_state().current_version().value_u64(),
+            ),
+            &removed_fields,
+        );
+        let encoded = super::enc::enc_full_self(txn);
+        let decoded = super::dec::dec_full::<AlterModelRemoveTxn>(&encoded).unwrap();
+        assert_eq!(
+            AlterModelRemoveTxnRestorePL {
+                model_id: super::ModelIDRes::new(
+                    super::SpaceIDRes::new(space.get_uuid(), "myspace".into()),
+                    "mymodel".into(),
+                    model.get_uuid(),
+                    model.delta_state().current_version().value_u64()
+                ),
+                removed_fields: ["profile_pic".into()].into()
+            },
+            decoded
+        );
+    }
+    #[test]
+    fn alter_update() {
+        let (space, model) = default_space_model();
+        let updated_fields = into_dict! {
+            // people of your social app will hate this, but hehe
+            "profile_pic" => Field::new([Layer::bin()].into(), false)
+        };
+        let txn = AlterModelUpdateTxn::new(
+            super::ModelIDRef::new(
+                super::SpaceIDRef::new("myspace", &space),
+                "mymodel".into(),
+                model.get_uuid(),
+                model.delta_state().current_version().value_u64(),
+            ),
+            &updated_fields,
+        );
+        let encoded = super::enc::enc_full_self(txn);
+        let decoded = super::dec::dec_full::<AlterModelUpdateTxn>(&encoded).unwrap();
+        assert_eq!(
+            AlterModelUpdateTxnRestorePL {
+                model_id: super::ModelIDRes::new(
+                    super::SpaceIDRes::new(space.get_uuid(), "myspace".into()),
+                    "mymodel".into(),
+                    model.get_uuid(),
+                    model.delta_state().current_version().value_u64()
+                ),
+                updated_fields
+            },
+            decoded
+        );
+    }
+    #[test]
+    fn drop() {
+        let (space, model) = default_space_model();
+        let txn = DropModelTxn::new(super::ModelIDRef::new(
+            super::SpaceIDRef::new("myspace", &space),
+            "mymodel",
+            model.get_uuid(),
+            model.delta_state().current_version().value_u64(),
+        ));
+        let encoded = super::enc::enc_full_self(txn);
+        let decoded = super::dec::dec_full::<DropModelTxn>(&encoded).unwrap();
+        assert_eq!(
+            super::ModelIDRes::new(
+                super::SpaceIDRes::new(space.get_uuid(), "myspace".into()),
+                "mymodel".into(),
+                model.get_uuid(),
+                model.delta_state().current_version().value_u64()
+            ),
+            decoded
+        );
+    }
+}

From 97a9471529ad8562474bd32cb739aca986d656c3 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 24 Aug 2023 19:38:18 +0000
Subject: [PATCH 234/310] Utilize VFS for all testing

---
 server/src/engine/storage/v1/mod.rs       |   2 +
 server/src/engine/storage/v1/test_util.rs | 220 ++++++++++++++++++++++
 server/src/engine/storage/v1/tests.rs     | 151 +--------------
 3 files changed, 227 insertions(+), 146 deletions(-)
 create mode 100644 server/src/engine/storage/v1/test_util.rs

diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index e8c51f5b..0031839c 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -34,6 +34,8 @@ pub mod inf;
 mod start_stop;
 // test
 #[cfg(test)]
+pub mod test_util;
+#[cfg(test)]
 mod tests;
 
 // re-exports
diff --git a/server/src/engine/storage/v1/test_util.rs b/server/src/engine/storage/v1/test_util.rs
new file mode 100644
index 00000000..a2df59fe
--- /dev/null
+++ b/server/src/engine/storage/v1/test_util.rs
@@ -0,0 +1,220 @@
+/*
+ * Created on Thu Aug 24 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{
+        header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
+        rw::{FileOpen, RawFileIOInterface, RawFileOpen, SDSSFileIO},
+        SDSSResult,
+    },
+    crate::engine::sync::cell::Lazy,
+    parking_lot::RwLock,
+    std::{
+        collections::hash_map::{Entry, HashMap},
+        io::{Error, ErrorKind},
+    },
+};
+
+static VFS: Lazy<RwLock<HashMap<Box<str>, VFile>>, fn() -> RwLock<HashMap<Box<str>, VFile>>> =
+    Lazy::new(|| RwLock::new(HashMap::new()));
+
+#[derive(Debug)]
+struct VFile {
+    read: bool,
+    write: bool,
+    data: Vec<u8>,
+    pos: usize,
+}
+
+impl VFile {
+    fn new(read: bool, write: bool, data: Vec<u8>, pos: usize) -> Self {
+        Self {
+            read,
+            write,
+            data,
+            pos,
+        }
+    }
+    fn current(&self) -> &[u8] {
+        &self.data[self.pos..]
+    }
+}
+
+#[derive(Debug)]
+pub struct VirtualFS(Box<str>);
+
+impl RawFileIOInterface for VirtualFS {
+    fn fopen_or_create_rw(file_path: &str) -> super::SDSSResult<RawFileOpen<Self>> {
+        match VFS.write().entry(file_path.into()) {
+            Entry::Occupied(mut oe) => {
+                oe.get_mut().read = true;
+                oe.get_mut().write = true;
+                oe.get_mut().pos = 0;
+                Ok(RawFileOpen::Existing(Self(file_path.into())))
+            }
+            Entry::Vacant(v) => {
+                v.insert(VFile::new(true, true, vec![], 0));
+                Ok(RawFileOpen::Created(Self(file_path.into())))
+            }
+        }
+    }
+    fn fread_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()> {
+        let mut vfs = VFS.write();
+        let file = vfs
+            .get_mut(&self.0)
+            .ok_or(Error::new(ErrorKind::NotFound, "File not found"))?;
+
+        if !file.read {
+            return Err(Error::new(ErrorKind::PermissionDenied, "Read permission denied").into());
+        }
+        let available_bytes = file.current().len();
+        if available_bytes < buf.len() {
+            return Err(Error::from(ErrorKind::UnexpectedEof).into());
+        }
+        buf.copy_from_slice(&file.data[file.pos..file.pos + buf.len()]);
+        file.pos += buf.len();
+        Ok(())
+    }
+    fn fwrite_all(&mut self, bytes: &[u8]) -> SDSSResult<()> {
+        let mut vfs = VFS.write();
+        let file = vfs
+            .get_mut(&self.0)
+            .ok_or(Error::new(ErrorKind::NotFound, "File not found"))?;
+
+        if !file.write {
+            return Err(Error::new(ErrorKind::PermissionDenied, "Write permission denied").into());
+        }
+
+        if file.pos + bytes.len() > file.data.len() {
+            file.data.resize(file.pos + bytes.len(), 0);
+        }
+        file.data[file.pos..file.pos + bytes.len()].copy_from_slice(bytes);
+        file.pos += bytes.len();
+
+        Ok(())
+    }
+    fn fsync_all(&mut self) -> super::SDSSResult<()> {
+        // pretty redundant for us
+        Ok(())
+    }
+    fn fseek_ahead(&mut self, by: u64) -> SDSSResult<()> {
+        let mut vfs = VFS.write();
+        let file = vfs
+            .get_mut(&self.0)
+            .ok_or(Error::new(ErrorKind::NotFound, "File not found"))?;
+
+        if by > file.data.len() as u64 {
+            return Err(Error::new(ErrorKind::InvalidInput, "Can't seek beyond file's end").into());
+        }
+
+        file.pos = by as usize;
+        Ok(())
+    }
+
+    fn flen(&self) -> SDSSResult<u64> {
+        let vfs = VFS.read();
+        let file = vfs
+            .get(&self.0)
+            .ok_or(Error::new(ErrorKind::NotFound, "File not found"))?;
+
+        Ok(file.data.len() as u64)
+    }
+
+    fn flen_set(&mut self, to: u64) -> SDSSResult<()> {
+        let mut vfs = VFS.write();
+        let file = vfs
+            .get_mut(&self.0)
+            .ok_or(Error::new(ErrorKind::NotFound, "File not found"))?;
+
+        if !file.write {
+            return Err(Error::new(ErrorKind::PermissionDenied, "Write permission denied").into());
+        }
+
+        if to as usize > file.data.len() {
+            file.data.resize(to as usize, 0);
+        } else {
+            file.data.truncate(to as usize);
+        }
+
+        if file.pos > file.data.len() {
+            file.pos = file.data.len();
+        }
+
+        Ok(())
+    }
+    fn fcursor(&mut self) -> SDSSResult<u64> {
+        let vfs = VFS.read();
+        let file = vfs
+            .get(&self.0)
+            .ok_or(Error::new(ErrorKind::NotFound, "File not found"))?;
+
+        Ok(file.pos as u64)
+    }
+}
+
+#[test]
+fn sdss_file() {
+    let f = SDSSFileIO::<VirtualFS>::open_or_create_perm_rw(
+        "this_is_a_test_file.db",
+        FileScope::Journal,
+        FileSpecifier::TestTransactionLog,
+        FileSpecifierVersion::__new(0),
+        0,
+        HostRunMode::Prod,
+        128,
+    )
+    .unwrap();
+
+    let FileOpen::Created(mut f) = f else {
+        panic!()
+    };
+
+    f.fsynced_write(b"hello, world\n").unwrap();
+    f.fsynced_write(b"hello, again\n").unwrap();
+
+    let f = SDSSFileIO::<VirtualFS>::open_or_create_perm_rw(
+        "this_is_a_test_file.db",
+        FileScope::Journal,
+        FileSpecifier::TestTransactionLog,
+        FileSpecifierVersion::__new(0),
+        0,
+        HostRunMode::Prod,
+        128,
+    )
+    .unwrap();
+
+    let FileOpen::Existing(mut f, _) = f else {
+        panic!()
+    };
+
+    let mut buf1 = [0u8; 13];
+    f.read_to_buffer(&mut buf1).unwrap();
+    let mut buf2 = [0u8; 13];
+    f.read_to_buffer(&mut buf2).unwrap();
+
+    assert_eq!(&buf1, b"hello, world\n");
+    assert_eq!(&buf2, b"hello, again\n");
+}
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index f86b8175..dbe92fc7 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -24,144 +24,7 @@
  *
 */
 
-use {
-    super::{
-        rw::{RawFileIOInterface, RawFileOpen},
-        SDSSError, SDSSResult,
-    },
-    crate::engine::sync::cell::Lazy,
-    parking_lot::RwLock,
-    std::{
-        collections::{hash_map::Entry, HashMap},
-        io::{ErrorKind, Read, Write},
-    },
-};
-
-static VFS: Lazy<
-    RwLock<HashMap<String, VirtualFile>>,
-    fn() -> RwLock<HashMap<String, VirtualFile>>,
-> = Lazy::new(|| RwLock::new(HashMap::new()));
-
-fn vfs<T>(fname: &str, mut func: impl FnMut(&mut VirtualFile) -> SDSSResult<T>) -> SDSSResult<T> {
-    let mut vfs = VFS.write();
-    let f = vfs
-        .get_mut(fname)
-        .ok_or(SDSSError::from(std::io::Error::from(ErrorKind::NotFound)))?;
-    func(f)
-}
-
-struct VirtualFile {
-    pos: u64,
-    read: bool,
-    write: bool,
-    data: Vec<u8>,
-}
-
-impl VirtualFile {
-    fn new(read: bool, write: bool, data: Vec<u8>) -> Self {
-        Self {
-            read,
-            write,
-            data,
-            pos: 0,
-        }
-    }
-    fn rw(data: Vec<u8>) -> Self {
-        Self::new(true, true, data)
-    }
-    fn w(data: Vec<u8>) -> Self {
-        Self::new(false, true, data)
-    }
-    fn r(data: Vec<u8>) -> Self {
-        Self::new(true, false, data)
-    }
-    fn seek_forward(&mut self, by: u64) {
-        self.pos += by;
-        assert!(self.pos <= self.data.len() as u64);
-    }
-    fn data(&self) -> &[u8] {
-        &self.data[self.pos as usize..]
-    }
-    fn data_mut(&mut self) -> &mut [u8] {
-        &mut self.data[self.pos as usize..]
-    }
-    fn close(&mut self) {
-        self.pos = 0;
-        self.read = false;
-        self.write = false;
-    }
-}
-
-struct VirtualFileInterface(Box<str>);
-
-impl Drop for VirtualFileInterface {
-    fn drop(&mut self) {
-        vfs(&self.0, |f| {
-            f.close();
-            Ok(())
-        })
-        .unwrap();
-    }
-}
-
-impl RawFileIOInterface for VirtualFileInterface {
-    fn fopen_or_create_rw(file_path: &str) -> SDSSResult<RawFileOpen<Self>> {
-        match VFS.write().entry(file_path.to_owned()) {
-            Entry::Occupied(mut oe) => {
-                let file_md = oe.get_mut();
-                file_md.read = true;
-                file_md.write = true;
-                Ok(RawFileOpen::Existing(Self(file_path.into())))
-            }
-            Entry::Vacant(ve) => {
-                ve.insert(VirtualFile::rw(vec![]));
-                Ok(RawFileOpen::Created(Self(file_path.into())))
-            }
-        }
-    }
-    fn fread_exact(&mut self, buf: &mut [u8]) -> super::SDSSResult<()> {
-        vfs(&self.0, |f| {
-            assert!(f.read);
-            f.data().read_exact(buf)?;
-            Ok(())
-        })
-    }
-    fn fwrite_all(&mut self, bytes: &[u8]) -> super::SDSSResult<()> {
-        vfs(&self.0, |f| {
-            assert!(f.write);
-            if f.data.len() < bytes.len() {
-                f.data.extend(bytes);
-            } else {
-                f.data_mut().write_all(bytes)?;
-            }
-            Ok(())
-        })
-    }
-    fn fsync_all(&mut self) -> super::SDSSResult<()> {
-        Ok(())
-    }
-    fn flen(&self) -> SDSSResult<u64> {
-        vfs(&self.0, |f| Ok(f.data.len() as _))
-    }
-    fn fseek_ahead(&mut self, by: u64) -> SDSSResult<()> {
-        vfs(&self.0, |f| {
-            f.seek_forward(by);
-            Ok(())
-        })
-    }
-    fn flen_set(&mut self, to: u64) -> SDSSResult<()> {
-        vfs(&self.0, |f| {
-            f.data.drain(f.data.len() - to as usize..);
-            Ok(())
-        })
-    }
-    fn fcursor(&mut self) -> SDSSResult<u64> {
-        vfs(&self.0, |f| Ok(f.pos))
-    }
-}
-
-type VirtualFS = VirtualFileInterface;
-type RealFS = std::fs::File;
+type VirtualFS = super::test_util::VirtualFS;
 
 mod rw {
     use crate::engine::storage::v1::{
@@ -213,8 +76,6 @@ mod tx {
         FileSpecifier, FileSpecifierVersion, HostRunMode,
     };
 
-    type FileInterface = super::RealFS;
-
     use {
         crate::{
             engine::storage::v1::{
@@ -242,7 +103,7 @@ mod tx {
         }
         fn txn_reset(
             &self,
-            txn_writer: &mut JournalWriter<FileInterface, DatabaseTxnAdapter>,
+            txn_writer: &mut JournalWriter<super::VirtualFS, DatabaseTxnAdapter>,
         ) -> SDSSResult<()> {
             self.reset();
             txn_writer.append_event(TxEvent::Reset)
@@ -254,7 +115,7 @@ mod tx {
             &self,
             pos: usize,
             val: u8,
-            txn_writer: &mut JournalWriter<FileInterface, DatabaseTxnAdapter>,
+            txn_writer: &mut JournalWriter<super::VirtualFS, DatabaseTxnAdapter>,
         ) -> SDSSResult<()> {
             self.set(pos, val);
             txn_writer.append_event(TxEvent::Set(pos, val))
@@ -323,8 +184,8 @@ mod tx {
     fn open_log(
         log_name: &str,
         db: &Database,
-    ) -> SDSSResult<JournalWriter<FileInterface, DatabaseTxnAdapter>> {
-        journal::open_journal::<DatabaseTxnAdapter, FileInterface>(
+    ) -> SDSSResult<JournalWriter<super::VirtualFS, DatabaseTxnAdapter>> {
+        journal::open_journal::<DatabaseTxnAdapter, super::VirtualFS>(
             log_name,
             FileSpecifier::TestTransactionLog,
             FileSpecifierVersion::__new(0),
@@ -355,7 +216,6 @@ mod tx {
             .append_journal_close_and_close()
             .unwrap();
         assert_eq!(original_data, empty_db2.copy_data());
-        std::fs::remove_file("testtxn.log").unwrap();
     }
     #[test]
     fn oneboot_mod_twoboot_mod_thirdboot_read() {
@@ -398,6 +258,5 @@ mod tx {
                 .collect::<Box<[u8]>>()
                 .as_ref()
         );
-        std::fs::remove_file("duatxn.db-tlog").unwrap();
     }
 }

From ea072f281c9705b439a4d84eba2a57ce1ef37860 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 25 Aug 2023 09:51:57 +0000
Subject: [PATCH 235/310] Add create space txn

---
 server/src/engine/core/space.rs               |  32 ++++-
 server/src/engine/storage/v1/journal.rs       |  31 ++++-
 server/src/engine/storage/v1/mod.rs           |   5 +-
 server/src/engine/storage/v1/rw.rs            |  33 ++++++
 server/src/engine/storage/v1/test_util.rs     |   8 +-
 server/src/engine/txn/gns/mod.rs              | 109 ++++++++++++++++--
 .../engine/txn/gns/{tests.rs => tests/mod.rs} |   0
 server/src/engine/txn/mod.rs                  |   8 +-
 8 files changed, 203 insertions(+), 23 deletions(-)
 rename server/src/engine/txn/gns/{tests.rs => tests/mod.rs} (100%)

diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index d282d837..ab686bc7 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -31,6 +31,7 @@ use {
         error::{DatabaseError, DatabaseResult},
         idx::{IndexST, STIndex},
         ql::ddl::{alt::AlterSpace, crt::CreateSpace, drop::DropSpace},
+        txn::gns as gnstxn,
     },
     parking_lot::RwLock,
 };
@@ -168,15 +169,34 @@ impl Space {
 }
 
 impl Space {
-    /// Execute a `create` stmt
-    pub fn exec_create(gns: &super::GlobalNS, space: CreateSpace) -> DatabaseResult<()> {
+    pub fn transactional_exec_create<TI: gnstxn::GNSTransactionDriverLLInterface>(
+        gns: &super::GlobalNS,
+        txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<TI>,
+        space: CreateSpace,
+    ) -> DatabaseResult<()> {
+        // process create
         let ProcedureCreate { space_name, space } = Self::process_create(space)?;
+        // acquire access
         let mut wl = gns.spaces().write();
-        if wl.st_insert(space_name, space) {
-            Ok(())
-        } else {
-            Err(DatabaseError::DdlSpaceAlreadyExists)
+        if wl.st_contains(&space_name) {
+            return Err(DatabaseError::DdlSpaceAlreadyExists);
         }
+        // commit txn
+        if TI::NONNULL {
+            // prepare and commit txn
+            let s_read = space.metadata().env().read();
+            txn_driver.try_commit(gnstxn::CreateSpaceTxn::new(&s_read, &space_name, &space))?;
+        }
+        // update global state
+        let _ = wl.st_insert(space_name, space);
+        Ok(())
+    }
+    /// Execute a `create` stmt
+    #[cfg(test)]
+    pub fn exec_create(gns: &super::GlobalNS, space: CreateSpace) -> DatabaseResult<()> {
+        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, move |driver| {
+            Self::transactional_exec_create(gns, driver, space)
+        })
     }
     /// Execute a `alter` stmt
     pub fn exec_alter(
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index d993f61e..eba132c3 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -57,10 +57,33 @@ use {
 const CRC: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
 const RECOVERY_BLOCK_AUTO_THRESHOLD: usize = 5;
 
-pub fn open_journal<
-    TA: JournalAdapter + core::fmt::Debug,
-    LF: RawFileIOInterface + core::fmt::Debug,
->(
+/// A journal to `/dev/null` (app. level impl)
+#[cfg(test)]
+pub fn null_journal<TA: JournalAdapter>(
+    log_file_name: &str,
+    log_kind: FileSpecifier,
+    log_kind_version: FileSpecifierVersion,
+    host_setting_version: u32,
+    host_run_mode: HostRunMode,
+    host_startup_counter: u64,
+    _: &TA::GlobalState,
+) -> JournalWriter<super::rw::NullZero, TA> {
+    let FileOpen::Created(journal) = SDSSFileIO::<super::rw::NullZero>::open_or_create_perm_rw(
+        log_file_name,
+        FileScope::Journal,
+        log_kind,
+        log_kind_version,
+        host_setting_version,
+        host_run_mode,
+        host_startup_counter,
+    )
+    .unwrap() else {
+        panic!()
+    };
+    JournalWriter::new(journal, 0, true).unwrap()
+}
+
+pub fn open_journal<TA: JournalAdapter, LF: RawFileIOInterface>(
     log_file_name: &str,
     log_kind: FileSpecifier,
     log_kind_version: FileSpecifierVersion,
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index 0031839c..b4113c78 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -41,8 +41,11 @@ mod tests;
 // re-exports
 pub use {
     journal::{open_journal, JournalAdapter, JournalWriter},
-    rw::BufferedScanner,
+    rw::{BufferedScanner, NullZero, RawFileIOInterface, SDSSFileIO},
 };
+pub mod header_meta {
+    pub use super::header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode};
+}
 
 use crate::util::os::SysIOError as IoError;
 
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index e8f8f856..969f8cbc 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -53,6 +53,8 @@ pub enum RawFileOpen<F> {
 }
 
 pub trait RawFileIOInterface: Sized {
+    /// Indicates that the interface is not a `/dev/null` (or related) implementation
+    const NOTNULL: bool = true;
     fn fopen_or_create_rw(file_path: &str) -> SDSSResult<RawFileOpen<Self>>;
     fn fread_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()>;
     fn fwrite_all(&mut self, bytes: &[u8]) -> SDSSResult<()>;
@@ -63,6 +65,37 @@ pub trait RawFileIOInterface: Sized {
     fn fcursor(&mut self) -> SDSSResult<u64>;
 }
 
+/// This is a kind of file like `/dev/null`. It exists in ... nothing!
+pub struct NullZero;
+
+impl RawFileIOInterface for NullZero {
+    const NOTNULL: bool = false;
+    fn fopen_or_create_rw(_: &str) -> SDSSResult<RawFileOpen<Self>> {
+        Ok(RawFileOpen::Created(Self))
+    }
+    fn fread_exact(&mut self, _: &mut [u8]) -> SDSSResult<()> {
+        Ok(())
+    }
+    fn fwrite_all(&mut self, _: &[u8]) -> SDSSResult<()> {
+        Ok(())
+    }
+    fn fsync_all(&mut self) -> SDSSResult<()> {
+        Ok(())
+    }
+    fn fseek_ahead(&mut self, _: u64) -> SDSSResult<()> {
+        Ok(())
+    }
+    fn flen(&self) -> SDSSResult<u64> {
+        Ok(0)
+    }
+    fn flen_set(&mut self, _: u64) -> SDSSResult<()> {
+        Ok(())
+    }
+    fn fcursor(&mut self) -> SDSSResult<u64> {
+        Ok(0)
+    }
+}
+
 impl RawFileIOInterface for File {
     fn fopen_or_create_rw(file_path: &str) -> SDSSResult<RawFileOpen<Self>> {
         let f = File::options()
diff --git a/server/src/engine/storage/v1/test_util.rs b/server/src/engine/storage/v1/test_util.rs
index a2df59fe..54452d20 100644
--- a/server/src/engine/storage/v1/test_util.rs
+++ b/server/src/engine/storage/v1/test_util.rs
@@ -24,10 +24,14 @@
  *
 */
 
+#[cfg(test)]
+use super::{
+    header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
+    rw::{FileOpen, SDSSFileIO},
+};
 use {
     super::{
-        header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
-        rw::{FileOpen, RawFileIOInterface, RawFileOpen, SDSSFileIO},
+        rw::{RawFileIOInterface, RawFileOpen},
         SDSSResult,
     },
     crate::engine::sync::cell::Lazy,
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index 449edfab..eca3b8f6 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -24,6 +24,8 @@
  *
 */
 
+#[cfg(test)]
+use crate::engine::storage::v1::test_util::VirtualFS;
 use {
     super::{TransactionError, TransactionResult},
     crate::{
@@ -31,8 +33,9 @@ use {
             core::{space::Space, GlobalNS},
             data::uuid::Uuid,
             storage::v1::{
+                self, header_meta,
                 inf::{self, PersistObject},
-                BufferedScanner, JournalAdapter, JournalWriter, SDSSResult,
+                BufferedScanner, JournalAdapter, JournalWriter, RawFileIOInterface, SDSSResult,
             },
         },
         util::EndianQW,
@@ -54,13 +57,70 @@ pub use {
     space::{AlterSpaceTxn, CreateSpaceTxn, DropSpaceTxn},
 };
 
+pub type GNSTransactionDriverNullZero =
+    GNSTransactionDriverAnyFS<crate::engine::storage::v1::NullZero>;
+pub type GNSTransactionDriver = GNSTransactionDriverAnyFS<File>;
+#[cfg(test)]
+pub type GNSTransactionDriverVFS = GNSTransactionDriverAnyFS<VirtualFS>;
+
+const CURRENT_LOG_VERSION: u32 = 0;
+
+pub trait GNSTransactionDriverLLInterface: RawFileIOInterface {
+    const NONNULL: bool = <Self as RawFileIOInterface>::NOTNULL;
+}
+impl<T: RawFileIOInterface> GNSTransactionDriverLLInterface for T {}
+
 #[derive(Debug)]
 /// The GNS transaction driver is used to handle DDL transactions
-pub struct GNSTransactionDriver {
-    journal: JournalWriter<File, GNSAdapter>,
+pub struct GNSTransactionDriverAnyFS<F = File> {
+    journal: JournalWriter<F, GNSAdapter>,
+}
+
+impl GNSTransactionDriverAnyFS<crate::engine::storage::v1::NullZero> {
+    pub fn nullzero(gns: &GlobalNS) -> Self {
+        let journal = v1::open_journal(
+            "gns.db-tlog",
+            header_meta::FileSpecifier::GNSTxnLog,
+            header_meta::FileSpecifierVersion::__new(CURRENT_LOG_VERSION),
+            0,
+            header_meta::HostRunMode::Dev,
+            0,
+            gns,
+        )
+        .unwrap();
+        Self { journal }
+    }
+    pub fn nullzero_create_exec<T>(gns: &GlobalNS, f: impl FnOnce(&mut Self) -> T) -> T {
+        let mut j = Self::nullzero(gns);
+        let r = f(&mut j);
+        j.close().unwrap();
+        r
+    }
 }
 
-impl GNSTransactionDriver {
+impl<F: GNSTransactionDriverLLInterface> GNSTransactionDriverAnyFS<F> {
+    pub fn close(self) -> TransactionResult<()> {
+        self.journal
+            .append_journal_close_and_close()
+            .map_err(|e| e.into())
+    }
+    pub fn open_or_reinit(
+        gns: &GlobalNS,
+        host_setting_version: u32,
+        host_run_mode: header_meta::HostRunMode,
+        host_startup_counter: u64,
+    ) -> TransactionResult<Self> {
+        let journal = v1::open_journal(
+            "gns.db-tlog",
+            header_meta::FileSpecifier::GNSTxnLog,
+            header_meta::FileSpecifierVersion::__new(CURRENT_LOG_VERSION),
+            host_setting_version,
+            host_run_mode,
+            host_startup_counter,
+            gns,
+        )?;
+        Ok(Self { journal })
+    }
     /// Attempts to commit the given event into the journal, handling any possible recovery triggers and returning
     /// errors (if any)
     pub fn try_commit<GE: GNSEvent>(&mut self, gns_event: GE) -> TransactionResult<()> {
@@ -89,8 +149,35 @@ impl JournalAdapter for GNSAdapter {
     fn encode(GNSSuperEvent(b): Self::JournalEvent) -> Box<[u8]> {
         b
     }
-    fn decode_and_update_state(_: &[u8], _: &Self::GlobalState) -> TransactionResult<()> {
-        todo!()
+    fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> TransactionResult<()> {
+        if payload.len() < 2 {
+            return Err(TransactionError::DecodedUnexpectedEof);
+        }
+        macro_rules! dispatch {
+            ($($item:ty),* $(,)?) => {
+                [$(<$item as GNSEvent>::decode_and_update_global_state),*, |_, _| Err(TransactionError::DecodeUnknownTxnOp)]
+            };
+        }
+        static DISPATCH: [fn(&mut BufferedScanner, &GlobalNS) -> TransactionResult<()>; 9] = dispatch!(
+            CreateSpaceTxn,
+            AlterSpaceTxn,
+            DropSpaceTxn,
+            CreateModelTxn,
+            AlterModelAddTxn,
+            AlterModelRemoveTxn,
+            AlterModelUpdateTxn,
+            DropModelTxn
+        );
+        let mut scanner = BufferedScanner::new(&payload);
+        let opc = unsafe {
+            // UNSAFE(@ohsayan):
+            u16::from_le_bytes(scanner.next_chunk())
+        };
+        match DISPATCH[core::cmp::min(opc as usize, DISPATCH.len())](&mut scanner, gs) {
+            Ok(()) if scanner.eof() => return Ok(()),
+            Ok(_) => Err(TransactionError::DecodeCorruptedPayloadMoreBytes),
+            Err(e) => Err(e),
+        }
     }
 }
 
@@ -121,10 +208,14 @@ where
     fn encode_super_event(commit: Self, buf: &mut Vec<u8>) {
         inf::enc::enc_full_into_buffer::<Self>(buf, commit)
     }
-    /// Attempts to decode the event using the given scanner
-    fn decode_from_super_event(
+    fn decode_and_update_global_state(
         scanner: &mut BufferedScanner,
-    ) -> TransactionResult<Self::RestoreType> {
+        gns: &GlobalNS,
+    ) -> TransactionResult<()> {
+        Self::update_global_state(Self::decode(scanner)?, gns)
+    }
+    /// Attempts to decode the event using the given scanner
+    fn decode(scanner: &mut BufferedScanner) -> TransactionResult<Self::RestoreType> {
         inf::dec::dec_full_from_scanner::<Self>(scanner).map_err(|e| e.into())
     }
     /// Update the global state from the restored event
diff --git a/server/src/engine/txn/gns/tests.rs b/server/src/engine/txn/gns/tests/mod.rs
similarity index 100%
rename from server/src/engine/txn/gns/tests.rs
rename to server/src/engine/txn/gns/tests/mod.rs
diff --git a/server/src/engine/txn/mod.rs b/server/src/engine/txn/mod.rs
index 8f1e5f99..3d9f9e31 100644
--- a/server/src/engine/txn/mod.rs
+++ b/server/src/engine/txn/mod.rs
@@ -32,7 +32,12 @@ pub type TransactionResult<T> = Result<T, TransactionError>;
 #[derive(Debug)]
 #[cfg_attr(test, derive(PartialEq))]
 pub enum TransactionError {
-    SDSSError(SDSSError),
+    /// corrupted txn payload. has more bytes than expected
+    DecodeCorruptedPayloadMoreBytes,
+    /// transaction payload is corrupted. has lesser bytes than expected
+    DecodedUnexpectedEof,
+    /// unknown transaction operation. usually indicates a corrupted payload
+    DecodeUnknownTxnOp,
     /// While restoring a certain item, a non-resolvable conflict was encountered in the global state, because the item was
     /// already present (when it was expected to not be present)
     OnRestoreDataConflictAlreadyExists,
@@ -40,6 +45,7 @@ pub enum TransactionError {
     OnRestoreDataMissing,
     /// On restore, a certain item that was expected to match a certain value, has a different value
     OnRestoreDataConflictMismatch,
+    SDSSError(SDSSError),
 }
 
 direct_from! {

From 39edfc64c93288d86376845e5a6183c8d3de420d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 25 Aug 2023 14:07:56 +0000
Subject: [PATCH 236/310] Make sure props are stored inside space.meta.env

Also fixed a bug in the create space txn payload
---
 server/src/engine/core/space.rs               | 44 +++++++++---
 .../src/engine/core/tests/ddl_space/alter.rs  |  9 ++-
 server/src/engine/data/dict.rs                | 70 +++++++++----------
 server/src/engine/storage/v1/inf/obj.rs       |  3 +-
 server/src/engine/storage/v1/inf/tests.rs     |  2 +-
 server/src/engine/txn/gns/space.rs            | 23 +++---
 server/src/engine/txn/gns/tests/io.rs         |  4 +-
 7 files changed, 91 insertions(+), 64 deletions(-)

diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index ab686bc7..1ab1e5a9 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -47,18 +47,37 @@ pub struct Space {
 #[derive(Debug, Default)]
 /// Space metadata
 pub struct SpaceMeta {
-    pub(super) env: RwLock<DictGeneric>,
+    pub(super) props: RwLock<DictGeneric>,
 }
 
 impl SpaceMeta {
     pub const KEY_ENV: &str = "env";
+    pub fn new_with_meta(props: DictGeneric) -> Self {
+        Self {
+            props: RwLock::new(props),
+        }
+    }
     pub fn with_env(env: DictGeneric) -> Self {
         Self {
-            env: RWLIdx::new(env),
+            props: RwLock::new(into_dict!("env" => DictEntryGeneric::Map(env))),
         }
     }
-    pub fn env(&self) -> &RwLock<DictGeneric> {
-        &self.env
+    pub fn dict(&self) -> &RwLock<DictGeneric> {
+        &self.props
+    }
+    pub fn get_env<'a>(rwl: &'a parking_lot::RwLockReadGuard<'a, DictGeneric>) -> &'a DictGeneric {
+        match rwl.get(Self::KEY_ENV).unwrap() {
+            DictEntryGeneric::Data(_) => unreachable!(),
+            DictEntryGeneric::Map(m) => m,
+        }
+    }
+    pub fn get_env_mut<'a>(
+        rwl: &'a mut parking_lot::RwLockWriteGuard<'a, DictGeneric>,
+    ) -> &'a mut DictGeneric {
+        match rwl.get_mut(Self::KEY_ENV).unwrap() {
+            DictEntryGeneric::Data(_) => unreachable!(),
+            DictEntryGeneric::Map(m) => m,
+        }
     }
 }
 
@@ -184,7 +203,7 @@ impl Space {
         // commit txn
         if TI::NONNULL {
             // prepare and commit txn
-            let s_read = space.metadata().env().read();
+            let s_read = space.metadata().dict().read();
             txn_driver.try_commit(gnstxn::CreateSpaceTxn::new(&s_read, &space_name, &space))?;
         }
         // update global state
@@ -207,15 +226,20 @@ impl Space {
         }: AlterSpace,
     ) -> DatabaseResult<()> {
         gns.with_space(&space_name, |space| {
-            let mut space_env = space.meta.env.write();
+            let mut space_props = space.meta.props.write();
+            let DictEntryGeneric::Map(space_env_mut) =
+                space_props.get_mut(SpaceMeta::KEY_ENV).unwrap()
+            else {
+                unreachable!()
+            };
             match updated_props.remove(SpaceMeta::KEY_ENV) {
                 Some(DictEntryGeneric::Map(env)) if updated_props.is_empty() => {
-                    if !dict::rmerge_metadata(&mut space_env, env) {
+                    if !dict::rmerge_metadata(space_env_mut, env) {
                         return Err(DatabaseError::DdlSpaceBadProperty);
                     }
                 }
                 Some(DictEntryGeneric::Data(l)) if updated_props.is_empty() & l.is_null() => {
-                    space_env.clear()
+                    space_env_mut.clear()
                 }
                 None => {}
                 _ => return Err(DatabaseError::DdlSpaceBadProperty),
@@ -244,8 +268,8 @@ impl Space {
 #[cfg(test)]
 impl PartialEq for SpaceMeta {
     fn eq(&self, other: &Self) -> bool {
-        let x = self.env.read();
-        let y = other.env.read();
+        let x = self.props.read();
+        let y = other.props.read();
         *x == *y
     }
 }
diff --git a/server/src/engine/core/tests/ddl_space/alter.rs b/server/src/engine/core/tests/ddl_space/alter.rs
index 65cb7e44..3cbcd4ab 100644
--- a/server/src/engine/core/tests/ddl_space/alter.rs
+++ b/server/src/engine/core/tests/ddl_space/alter.rs
@@ -61,8 +61,9 @@ fn alter_update_prop_env_var() {
         &gns,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
+            let rl = space.meta.dict().read();
             assert_eq!(
-                space.meta.env.read().get("MY_NEW_PROP").unwrap(),
+                SpaceMeta::get_env(&rl).get("MY_NEW_PROP").unwrap(),
                 &(Datacell::new_uint(100).into())
             )
         },
@@ -92,8 +93,9 @@ fn alter_remove_prop_env_var() {
         &gns,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
+            let rl = space.meta.dict().read();
             assert_eq!(
-                space.meta.env.read().get("MY_NEW_PROP").unwrap(),
+                SpaceMeta::get_env(&rl).get("MY_NEW_PROP").unwrap(),
                 &(Datacell::new_uint(100).into())
             )
         },
@@ -133,8 +135,9 @@ fn alter_remove_all_env() {
         &gns,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
+            let rl = space.meta.dict().read();
             assert_eq!(
-                space.meta.env.read().get("MY_NEW_PROP").unwrap(),
+                SpaceMeta::get_env(&rl).get("MY_NEW_PROP").unwrap(),
                 &(Datacell::new_uint(100).into())
             )
         },
diff --git a/server/src/engine/data/dict.rs b/server/src/engine/data/dict.rs
index 21aecfc0..035f75f4 100644
--- a/server/src/engine/data/dict.rs
+++ b/server/src/engine/data/dict.rs
@@ -52,14 +52,6 @@ pub enum DictEntryGeneric {
     patchsets
 */
 
-#[derive(Debug, PartialEq, Default)]
-struct DictGenericPatch(HashMap<Box<str>, Option<DictGenericPatchEntry>>);
-#[derive(Debug, PartialEq)]
-enum DictGenericPatchEntry {
-    Data(Datacell),
-    Map(DictGenericPatch),
-}
-
 /// Accepts a dict with possible null values, and removes those null values
 pub fn rflatten_metadata(mut new: DictGeneric) -> DictGeneric {
     _rflatten_metadata(&mut new);
@@ -79,25 +71,38 @@ fn _rflatten_metadata(new: &mut DictGeneric) {
 /// Recursively merge a [`DictGeneric`] into a [`DictGeneric`] with the use of an intermediary
 /// patchset to avoid inconsistent states
 pub fn rmerge_metadata(current: &mut DictGeneric, new: DictGeneric) -> bool {
-    let mut patch = DictGenericPatch::default();
-    let current_ref = current as &_;
-    let r = rmerge_metadata_prepare_patch(current_ref, new, &mut patch);
-    if r {
-        merge_data_with_patch(current, patch);
+    match rprepare_metadata_patch(current as &_, new) {
+        Some(patch) => {
+            rmerge_data_with_patch(current, patch);
+            true
+        }
+        None => false,
     }
-    r
 }
 
-fn merge_data_with_patch(current: &mut DictGeneric, patch: DictGenericPatch) {
-    for (key, patch) in patch.0 {
+pub fn rprepare_metadata_patch(current: &DictGeneric, new: DictGeneric) -> Option<DictGeneric> {
+    let mut patch = Default::default();
+    if rmerge_metadata_prepare_patch(current, new, &mut patch) {
+        Some(patch)
+    } else {
+        None
+    }
+}
+
+fn rmerge_data_with_patch(current: &mut DictGeneric, patch: DictGeneric) {
+    for (key, patch) in patch {
         match patch {
-            Some(DictGenericPatchEntry::Data(d)) => {
+            DictEntryGeneric::Data(d) if d.is_init() => {
                 current.st_upsert(key, DictEntryGeneric::Data(d));
             }
-            Some(DictGenericPatchEntry::Map(m)) => match current.get_mut(&key) {
+            DictEntryGeneric::Data(_) => {
+                // null
+                let _ = current.remove(&key);
+            }
+            DictEntryGeneric::Map(m) => match current.get_mut(&key) {
                 Some(current_recursive) => match current_recursive {
                     DictEntryGeneric::Map(current_m) => {
-                        merge_data_with_patch(current_m, m);
+                        rmerge_data_with_patch(current_m, m);
                     }
                     _ => {
                         // can never reach here since the patch is always correct
@@ -106,12 +111,9 @@ fn merge_data_with_patch(current: &mut DictGeneric, patch: DictGenericPatch) {
                 },
                 None => {
                     let mut new = DictGeneric::new();
-                    merge_data_with_patch(&mut new, m);
+                    rmerge_data_with_patch(&mut new, m);
                 }
             },
-            None => {
-                let _ = current.remove(&key);
-            }
         }
     }
 }
@@ -119,7 +121,7 @@ fn merge_data_with_patch(current: &mut DictGeneric, patch: DictGenericPatch) {
 fn rmerge_metadata_prepare_patch(
     current: &DictGeneric,
     new: DictGeneric,
-    patch: &mut DictGenericPatch,
+    patch: &mut DictGeneric,
 ) -> bool {
     let mut new = new.into_iter();
     let mut okay = true;
@@ -131,9 +133,7 @@ fn rmerge_metadata_prepare_patch(
                 if new_data.is_init() =>
             {
                 if this_data.kind() == new_data.kind() {
-                    patch
-                        .0
-                        .insert(key, Some(DictGenericPatchEntry::Data(new_data)));
+                    patch.insert(key, DictEntryGeneric::Data(new_data));
                 } else {
                     okay = false;
                 }
@@ -145,32 +145,28 @@ fn rmerge_metadata_prepare_patch(
                 Some(DictEntryGeneric::Map(this_recursive_data)),
                 DictEntryGeneric::Map(new_recursive_data),
             ) => {
-                let mut this_patch = DictGenericPatch::default();
+                let mut this_patch = DictGeneric::new();
                 let _okay = rmerge_metadata_prepare_patch(
                     this_recursive_data,
                     new_recursive_data,
                     &mut this_patch,
                 );
-                patch
-                    .0
-                    .insert(key, Some(DictGenericPatchEntry::Map(this_patch)));
+                patch.insert(key, DictEntryGeneric::Map(this_patch));
                 okay &= _okay;
             }
             // null -> non-null: flatten insert
             (None, DictEntryGeneric::Data(l)) if l.is_init() => {
-                let _ = patch.0.insert(key, Some(DictGenericPatchEntry::Data(l)));
+                let _ = patch.insert(key, DictEntryGeneric::Data(l));
             }
             (None, DictEntryGeneric::Map(m)) => {
-                let mut this_patch = DictGenericPatch::default();
+                let mut this_patch = DictGeneric::new();
                 okay &= rmerge_metadata_prepare_patch(&into_dict!(), m, &mut this_patch);
-                let _ = patch
-                    .0
-                    .insert(key, Some(DictGenericPatchEntry::Map(this_patch)));
+                let _ = patch.insert(key, DictEntryGeneric::Map(this_patch));
             }
             // non-null -> null: remove
             (Some(_), DictEntryGeneric::Data(l)) => {
                 debug_assert!(l.is_null());
-                patch.0.insert(key, None);
+                patch.insert(key, DictEntryGeneric::Data(Datacell::null()));
             }
             (None, DictEntryGeneric::Data(l)) => {
                 debug_assert!(l.is_null());
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 0b1a6426..8d992b1b 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -272,6 +272,7 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
 }
 
 pub struct SpaceLayout;
+#[derive(Debug)]
 pub struct SpaceLayoutMD {
     uuid: Uuid,
     prop_c: usize,
@@ -323,7 +324,7 @@ impl<'a> PersistObject for SpaceLayoutRef<'a> {
                 super::map::MapIndexSizeMD(md.prop_c),
             )?;
         Ok(Space::new_restore_empty(
-            SpaceMeta::with_env(space_meta),
+            SpaceMeta::new_with_meta(space_meta),
             md.uuid,
         ))
     }
diff --git a/server/src/engine/storage/v1/inf/tests.rs b/server/src/engine/storage/v1/inf/tests.rs
index 87752e18..9f0931b8 100644
--- a/server/src/engine/storage/v1/inf/tests.rs
+++ b/server/src/engine/storage/v1/inf/tests.rs
@@ -112,7 +112,7 @@ fn model() {
 fn space() {
     let uuid = Uuid::new();
     let space = Space::new_with_uuid(Default::default(), SpaceMeta::default(), uuid);
-    let space_meta_read = space.metadata().env().read();
+    let space_meta_read = space.metadata().dict().read();
     let enc = super::enc::enc_full::<obj::SpaceLayoutRef>(obj::SpaceLayoutRef::from((
         &space,
         &*space_meta_read,
diff --git a/server/src/engine/txn/gns/space.rs b/server/src/engine/txn/gns/space.rs
index 5250a788..066c2929 100644
--- a/server/src/engine/txn/gns/space.rs
+++ b/server/src/engine/txn/gns/space.rs
@@ -101,7 +101,7 @@ impl<'a> PersistObject for CreateSpaceTxn<'a> {
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
         buf.extend(data.space_name.as_bytes());
-        <obj::SpaceLayoutRef as PersistObject>::meta_enc(buf, (data.space, data.space_meta).into());
+        <obj::SpaceLayoutRef as PersistObject>::obj_enc(buf, (data.space, data.space_meta).into());
     }
     unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
         let space_name =
@@ -138,14 +138,14 @@ impl<'a> GNSEvent for CreateSpaceTxn<'a> {
 /// Transaction payload for an `alter space ...` query
 pub struct AlterSpaceTxn<'a> {
     space_id: super::SpaceIDRef<'a>,
-    space_meta: &'a DictGeneric,
+    updated_props: &'a DictGeneric,
 }
 
 impl<'a> AlterSpaceTxn<'a> {
-    pub const fn new(space_id: super::SpaceIDRef<'a>, space_meta: &'a DictGeneric) -> Self {
+    pub const fn new(space_id: super::SpaceIDRef<'a>, updated_props: &'a DictGeneric) -> Self {
         Self {
             space_id,
-            space_meta,
+            updated_props,
         }
     }
 }
@@ -171,7 +171,7 @@ impl<'a> PersistObject for AlterSpaceTxn<'a> {
     }
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
         <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
-        buf.extend(data.space_meta.len().u64_bytes_le());
+        buf.extend(data.updated_props.len().u64_bytes_le());
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
         Ok(AlterSpaceTxnMD {
@@ -181,7 +181,10 @@ impl<'a> PersistObject for AlterSpaceTxn<'a> {
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
         <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id);
-        <map::PersistMapImpl<map::GenericDictSpec> as PersistObject>::obj_enc(buf, data.space_meta);
+        <map::PersistMapImpl<map::GenericDictSpec> as PersistObject>::obj_enc(
+            buf,
+            data.updated_props,
+        );
     }
     unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
         let space_id = <super::SpaceID as PersistObject>::obj_dec(s, md.space_id_meta)?;
@@ -211,10 +214,10 @@ impl<'a> GNSEvent for AlterSpaceTxn<'a> {
         let gns = gns.spaces().read();
         match gns.st_get(&space_id.name) {
             Some(space) => {
-                let mut wmeta = space.metadata().env().write();
-                space_meta
-                    .into_iter()
-                    .for_each(|(k, v)| wmeta.st_upsert(k, v));
+                let mut wmeta = space.metadata().dict().write();
+                if !crate::engine::data::dict::rmerge_metadata(&mut wmeta, space_meta) {
+                    return Err(TransactionError::OnRestoreDataConflictMismatch);
+                }
             }
             None => return Err(TransactionError::OnRestoreDataMissing),
         }
diff --git a/server/src/engine/txn/gns/tests/io.rs b/server/src/engine/txn/gns/tests/io.rs
index 8a70c07d..ebe3ec91 100644
--- a/server/src/engine/txn/gns/tests/io.rs
+++ b/server/src/engine/txn/gns/tests/io.rs
@@ -47,7 +47,7 @@ mod space_tests {
     #[test]
     fn create() {
         let orig_space = Space::empty();
-        let space_r = orig_space.metadata().env().read();
+        let space_r = orig_space.metadata().dict().read();
         let txn = CreateSpaceTxn::new(&space_r, "myspace", &orig_space);
         let encoded = enc::enc_full_self(txn);
         let decoded = dec::dec_full::<CreateSpaceTxn>(&encoded).unwrap();
@@ -62,7 +62,7 @@ mod space_tests {
     #[test]
     fn alter() {
         let space = Space::empty();
-        let space_r = space.metadata().env().read();
+        let space_r = space.metadata().dict().read();
         let txn = AlterSpaceTxn::new(SpaceIDRef::new("myspace", &space), &space_r);
         let encoded = enc::enc_full_self(txn);
         let decoded = dec::dec_full::<AlterSpaceTxn>(&encoded).unwrap();

From 29d4137a2cb8c92fc8181250b1996de445102a74 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 25 Aug 2023 18:07:55 +0000
Subject: [PATCH 237/310] Add space DDL txns and full chain tests

Also ensure that the `env` key is not when merged
---
 server/src/engine/core/mod.rs                 |   1 +
 server/src/engine/core/space.rs               |  89 +++++++----
 server/src/engine/data/dict.rs                |   2 +-
 server/src/engine/txn/gns/mod.rs              |  18 ++-
 server/src/engine/txn/gns/tests/full_chain.rs | 145 ++++++++++++++++++
 server/src/engine/txn/gns/tests/mod.rs        |   1 +
 6 files changed, 226 insertions(+), 30 deletions(-)
 create mode 100644 server/src/engine/txn/gns/tests/full_chain.rs

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index b9fca120..50d6cdfe 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -50,6 +50,7 @@ type RWLIdx<K, V> = RwLock<IndexST<K, V>>;
 
 // FIXME(@ohsayan): Make sure we update what all structures we're making use of here
 
+#[cfg_attr(test, derive(Debug))]
 pub struct GlobalNS {
     index_space: RWLIdx<Box<str>, Space>,
 }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 1ab1e5a9..22f6febd 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -217,51 +217,84 @@ impl Space {
             Self::transactional_exec_create(gns, driver, space)
         })
     }
-    /// Execute a `alter` stmt
-    pub fn exec_alter(
+    pub fn transactional_exec_alter<TI: gnstxn::GNSTransactionDriverLLInterface>(
         gns: &super::GlobalNS,
+        txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<TI>,
         AlterSpace {
             space_name,
-            mut updated_props,
+            updated_props,
         }: AlterSpace,
     ) -> DatabaseResult<()> {
         gns.with_space(&space_name, |space| {
-            let mut space_props = space.meta.props.write();
-            let DictEntryGeneric::Map(space_env_mut) =
-                space_props.get_mut(SpaceMeta::KEY_ENV).unwrap()
-            else {
-                unreachable!()
-            };
-            match updated_props.remove(SpaceMeta::KEY_ENV) {
-                Some(DictEntryGeneric::Map(env)) if updated_props.is_empty() => {
-                    if !dict::rmerge_metadata(space_env_mut, env) {
-                        return Err(DatabaseError::DdlSpaceBadProperty);
-                    }
-                }
-                Some(DictEntryGeneric::Data(l)) if updated_props.is_empty() & l.is_null() => {
-                    space_env_mut.clear()
-                }
-                None => {}
+            match updated_props.get(SpaceMeta::KEY_ENV) {
+                Some(DictEntryGeneric::Map(_)) if updated_props.len() == 1 => {}
+                Some(DictEntryGeneric::Data(l)) if updated_props.len() == 1 && l.is_null() => {}
+                None if updated_props.is_empty() => return Ok(()),
                 _ => return Err(DatabaseError::DdlSpaceBadProperty),
             }
+            let mut space_props = space.meta.dict().write();
+            // create patch
+            let patch = match dict::rprepare_metadata_patch(&space_props, updated_props) {
+                Some(patch) => patch,
+                None => return Err(DatabaseError::DdlSpaceBadProperty),
+            };
+            if TI::NONNULL {
+                // prepare txn
+                let txn =
+                    gnstxn::AlterSpaceTxn::new(gnstxn::SpaceIDRef::new(&space_name, space), &patch);
+                // commit
+                txn_driver.try_commit(txn)?;
+            }
+            // merge
+            dict::rmerge_data_with_patch(&mut space_props, patch);
+            // the `env` key may have been popped, so put it back (setting `env: null` removes the env key and we don't want to waste time enforcing this in the
+            // merge algorithm)
+            let _ = space_props.st_insert(
+                SpaceMeta::KEY_ENV.into(),
+                DictEntryGeneric::Map(into_dict!()),
+            );
             Ok(())
         })
     }
-    pub fn exec_drop(
+    #[cfg(test)]
+    /// Execute a `alter` stmt
+    pub fn exec_alter(gns: &super::GlobalNS, alter: AlterSpace) -> DatabaseResult<()> {
+        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, move |driver| {
+            Self::transactional_exec_alter(gns, driver, alter)
+        })
+    }
+    pub fn transactional_exec_drop<TI: gnstxn::GNSTransactionDriverLLInterface>(
         gns: &super::GlobalNS,
+        txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<TI>,
         DropSpace { space, force: _ }: DropSpace,
     ) -> DatabaseResult<()> {
         // TODO(@ohsayan): force remove option
         // TODO(@ohsayan): should a drop space block the entire global table?
-        match gns
-            .spaces()
-            .write()
-            .st_delete_if(space.as_str(), |space| space.mns.read().len() == 0)
-        {
-            Some(true) => Ok(()),
-            Some(false) => Err(DatabaseError::DdlSpaceRemoveNonEmpty),
-            None => Err(DatabaseError::DdlSpaceNotFound),
+        let space_name = space;
+        let mut wgns = gns.spaces().write();
+        let space = match wgns.get(space_name.as_str()) {
+            Some(space) => space,
+            None => return Err(DatabaseError::DdlSpaceNotFound),
+        };
+        let space_w = space.mns.write();
+        if space_w.st_len() != 0 {
+            return Err(DatabaseError::DdlSpaceRemoveNonEmpty);
+        }
+        // we can remove this
+        if TI::NONNULL {
+            // prepare txn
+            let txn = gnstxn::DropSpaceTxn::new(gnstxn::SpaceIDRef::new(&space_name, space));
+            txn_driver.try_commit(txn)?;
         }
+        drop(space_w);
+        let _ = wgns.st_delete(space_name.as_str());
+        Ok(())
+    }
+    #[cfg(test)]
+    pub fn exec_drop(gns: &super::GlobalNS, drop_space: DropSpace) -> DatabaseResult<()> {
+        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, move |driver| {
+            Self::transactional_exec_drop(gns, driver, drop_space)
+        })
     }
 }
 
diff --git a/server/src/engine/data/dict.rs b/server/src/engine/data/dict.rs
index 035f75f4..6dc1894f 100644
--- a/server/src/engine/data/dict.rs
+++ b/server/src/engine/data/dict.rs
@@ -89,7 +89,7 @@ pub fn rprepare_metadata_patch(current: &DictGeneric, new: DictGeneric) -> Optio
     }
 }
 
-fn rmerge_data_with_patch(current: &mut DictGeneric, patch: DictGeneric) {
+pub fn rmerge_data_with_patch(current: &mut DictGeneric, patch: DictGeneric) {
     for (key, patch) in patch {
         match patch {
             DictEntryGeneric::Data(d) if d.is_init() => {
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index eca3b8f6..a604ee40 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -66,6 +66,7 @@ pub type GNSTransactionDriverVFS = GNSTransactionDriverAnyFS<VirtualFS>;
 const CURRENT_LOG_VERSION: u32 = 0;
 
 pub trait GNSTransactionDriverLLInterface: RawFileIOInterface {
+    /// If true, this is an actual txn driver with a non-null (not `/dev/null` like) journal
     const NONNULL: bool = <Self as RawFileIOInterface>::NOTNULL;
 }
 impl<T: RawFileIOInterface> GNSTransactionDriverLLInterface for T {}
@@ -110,8 +111,23 @@ impl<F: GNSTransactionDriverLLInterface> GNSTransactionDriverAnyFS<F> {
         host_run_mode: header_meta::HostRunMode,
         host_startup_counter: u64,
     ) -> TransactionResult<Self> {
-        let journal = v1::open_journal(
+        Self::open_or_reinit_with_name(
+            gns,
             "gns.db-tlog",
+            host_setting_version,
+            host_run_mode,
+            host_startup_counter,
+        )
+    }
+    pub fn open_or_reinit_with_name(
+        gns: &GlobalNS,
+        log_file_name: &str,
+        host_setting_version: u32,
+        host_run_mode: header_meta::HostRunMode,
+        host_startup_counter: u64,
+    ) -> TransactionResult<Self> {
+        let journal = v1::open_journal(
+            log_file_name,
             header_meta::FileSpecifier::GNSTxnLog,
             header_meta::FileSpecifierVersion::__new(CURRENT_LOG_VERSION),
             host_setting_version,
diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
new file mode 100644
index 00000000..03eb2669
--- /dev/null
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -0,0 +1,145 @@
+/*
+ * Created on Fri Aug 25 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    core::{
+        space::{Space, SpaceMeta},
+        GlobalNS,
+    },
+    data::{cell::Datacell, uuid::Uuid, DictEntryGeneric},
+    ql::{ast::parse_ast_node_full, ddl::crt::CreateSpace, tests::lex_insecure},
+    storage::v1::header_meta::HostRunMode,
+    txn::gns::GNSTransactionDriverVFS,
+};
+
+fn double_run(f: impl FnOnce() + Copy) {
+    f();
+    f();
+}
+
+fn with_variable<T>(var: T, f: impl FnOnce(T)) {
+    f(var);
+}
+
+fn init_txn_driver(gns: &GlobalNS, log_name: &str) -> GNSTransactionDriverVFS {
+    GNSTransactionDriverVFS::open_or_reinit_with_name(&gns, log_name, 0, HostRunMode::Prod, 0)
+        .unwrap()
+}
+
+fn init_space(
+    gns: &GlobalNS,
+    driver: &mut GNSTransactionDriverVFS,
+    space_name: &str,
+    env: &str,
+) -> Uuid {
+    let query = format!("create space {space_name} with {{ env: {env} }}");
+    let stmt = lex_insecure(query.as_bytes()).unwrap();
+    let stmt = parse_ast_node_full::<CreateSpace>(&stmt[2..]).unwrap();
+    let name = stmt.space_name;
+    Space::transactional_exec_create(&gns, driver, stmt).unwrap();
+    gns.spaces().read().get(name.as_str()).unwrap().get_uuid()
+}
+
+#[test]
+fn create_space() {
+    with_variable("create_space_test.gns.db-tlog", |log_name| {
+        let uuid;
+        // start 1
+        {
+            let gns = GlobalNS::empty();
+            let mut driver = init_txn_driver(&gns, log_name);
+            uuid = init_space(&gns, &mut driver, "myspace", "{ SAYAN_MAX: 65536 }"); // good lord that doesn't sound like a good variable
+            driver.close().unwrap();
+        }
+        double_run(|| {
+            let gns = GlobalNS::empty();
+            let driver = init_txn_driver(&gns, log_name);
+            assert_eq!(
+                gns.spaces().read().get("myspace").unwrap(),
+                &Space::new_restore_empty(
+                    SpaceMeta::with_env(
+                        into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint(65536)))
+                    ),
+                    uuid
+                )
+            );
+            driver.close().unwrap();
+        })
+    })
+}
+
+#[test]
+fn alter_space() {
+    with_variable("alter_space_test.gns.db-tlog", |log_name| {
+        let uuid;
+        {
+            let gns = GlobalNS::empty();
+            let mut driver = init_txn_driver(&gns, log_name);
+            uuid = init_space(&gns, &mut driver, "myspace", "{}");
+            let stmt =
+                lex_insecure("alter space myspace with { env: { SAYAN_MAX: 65536 } }".as_bytes())
+                    .unwrap();
+            let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
+            Space::transactional_exec_alter(&gns, &mut driver, stmt).unwrap();
+            driver.close().unwrap();
+        }
+        double_run(|| {
+            let gns = GlobalNS::empty();
+            let driver = init_txn_driver(&gns, log_name);
+            assert_eq!(
+                gns.spaces().read().get("myspace").unwrap(),
+                &Space::new_restore_empty(
+                    SpaceMeta::with_env(
+                        into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint(65536)))
+                    ),
+                    uuid
+                )
+            );
+            driver.close().unwrap();
+        })
+    })
+}
+
+#[test]
+fn drop_space() {
+    with_variable("drop_space_test.gns.db-tlog", |log_name| {
+        {
+            let gns = GlobalNS::empty();
+            let mut driver = init_txn_driver(&gns, log_name);
+            let _ = init_space(&gns, &mut driver, "myspace", "{}");
+            let stmt = lex_insecure("drop space myspace".as_bytes()).unwrap();
+            let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
+            Space::transactional_exec_drop(&gns, &mut driver, stmt).unwrap();
+            driver.close().unwrap();
+        }
+        double_run(|| {
+            let gns = GlobalNS::empty();
+            let driver = init_txn_driver(&gns, log_name);
+            assert_eq!(gns.spaces().read().get("myspace"), None);
+            driver.close().unwrap();
+        })
+    })
+}
diff --git a/server/src/engine/txn/gns/tests/mod.rs b/server/src/engine/txn/gns/tests/mod.rs
index c9fa1028..608315a3 100644
--- a/server/src/engine/txn/gns/tests/mod.rs
+++ b/server/src/engine/txn/gns/tests/mod.rs
@@ -24,4 +24,5 @@
  *
 */
 
+mod full_chain;
 mod io;

From 8d2097e526ba73db19b9948bb886c057b9166dff Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 26 Aug 2023 09:46:31 +0000
Subject: [PATCH 238/310] Add txn impls and tests for model DDL

---
 server/src/engine/core/model/alt.rs           | 122 ++++++++++++------
 server/src/engine/core/model/mod.rs           |  77 +++++++++--
 server/src/engine/core/space.rs               |  15 ++-
 server/src/engine/core/tests/ddl_model/mod.rs |   2 +-
 server/src/engine/core/tests/ddl_space/mod.rs |   4 +-
 server/src/engine/core/tests/dml/mod.rs       |   2 +-
 server/src/engine/txn/gns/mod.rs              |   1 +
 server/src/engine/txn/gns/model.rs            |  15 ++-
 server/src/engine/txn/gns/tests/full_chain.rs | 110 +++++++++++++++-
 9 files changed, 288 insertions(+), 60 deletions(-)

diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index 87868716..cdfb2431 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -28,7 +28,7 @@ use {
     super::{Field, IWModel, Layer, Model},
     crate::{
         engine::{
-            core::GlobalNS,
+            core::{util::EntityLocator, GlobalNS},
             data::{
                 tag::{DataTag, TagClass},
                 DictEntryGeneric,
@@ -43,6 +43,7 @@ use {
                 },
                 lex::Ident,
             },
+            txn::gns as gnstxn,
         },
         util,
     },
@@ -247,50 +248,89 @@ impl<'a> AlterPlan<'a> {
 }
 
 impl Model {
-    pub fn exec_alter(gns: &GlobalNS, alter: AlterModel) -> DatabaseResult<()> {
-        gns.with_model(alter.model, |model| {
-            // make intent
-            let iwm = model.intent_write_model();
-            // prepare plan
-            let plan = AlterPlan::fdeltas(model, &iwm, alter)?;
-            // we have a legal plan; acquire exclusive if we need it
-            if !plan.no_lock {
-                // TODO(@ohsayan): allow this later on, once we define the syntax
-                return Err(DatabaseError::NeedLock);
-            }
-            // fine, we're good
-            let mut iwm = iwm;
-            match plan.action {
-                AlterAction::Ignore => drop(iwm),
-                AlterAction::Add(new_fields) => {
-                    let mut guard = model.delta_state().wguard();
-                    // TODO(@ohsayan): this impacts lockdown duration; fix it
-                    new_fields
-                        .stseq_ord_kv()
-                        .map(|(x, y)| (x.clone(), y.clone()))
-                        .for_each(|(field_id, field)| {
+    pub fn transactional_exec_alter<GI: gnstxn::GNSTransactionDriverLLInterface>(
+        gns: &GlobalNS,
+        txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<GI>,
+        alter: AlterModel,
+    ) -> DatabaseResult<()> {
+        let (space_name, model_name) = EntityLocator::parse_entity(alter.model)?;
+        gns.with_space(space_name, |space| {
+            space.with_model(model_name, |model| {
+                // make intent
+                let iwm = model.intent_write_model();
+                // prepare plan
+                let plan = AlterPlan::fdeltas(model, &iwm, alter)?;
+                // we have a legal plan; acquire exclusive if we need it
+                if !plan.no_lock {
+                    // TODO(@ohsayan): allow this later on, once we define the syntax
+                    return Err(DatabaseError::NeedLock);
+                }
+                // fine, we're good
+                let mut iwm = iwm;
+                match plan.action {
+                    AlterAction::Ignore => drop(iwm),
+                    AlterAction::Add(new_fields) => {
+                        let mut guard = model.delta_state().wguard();
+                        // TODO(@ohsayan): this impacts lockdown duration; fix it
+                        if GI::NONNULL {
+                            // prepare txn
+                            let txn = gnstxn::AlterModelAddTxn::new(
+                                gnstxn::ModelIDRef::new_ref(space_name, space, model_name, model),
+                                &new_fields,
+                            );
+                            // commit txn
+                            txn_driver.try_commit(txn)?;
+                        }
+                        new_fields
+                            .stseq_ord_kv()
+                            .map(|(x, y)| (x.clone(), y.clone()))
+                            .for_each(|(field_id, field)| {
+                                model
+                                    .delta_state()
+                                    .append_unresolved_wl_field_add(&mut guard, &field_id);
+                                iwm.fields_mut().st_insert(field_id, field);
+                            });
+                    }
+                    AlterAction::Remove(removed) => {
+                        let mut guard = model.delta_state().wguard();
+                        if GI::NONNULL {
+                            // prepare txn
+                            let txn = gnstxn::AlterModelRemoveTxn::new(
+                                gnstxn::ModelIDRef::new_ref(space_name, space, model_name, model),
+                                &removed,
+                            );
+                            // commit txn
+                            txn_driver.try_commit(txn)?;
+                        }
+                        removed.iter().for_each(|field_id| {
                             model
                                 .delta_state()
-                                .append_unresolved_wl_field_add(&mut guard, &field_id);
-                            iwm.fields_mut().st_insert(field_id, field);
+                                .append_unresolved_wl_field_rem(&mut guard, field_id.as_str());
+                            iwm.fields_mut().st_delete(field_id.as_str());
                         });
+                    }
+                    AlterAction::Update(updated) => {
+                        if GI::NONNULL {
+                            // prepare txn
+                            let txn = gnstxn::AlterModelUpdateTxn::new(
+                                gnstxn::ModelIDRef::new_ref(space_name, space, model_name, model),
+                                &updated,
+                            );
+                            // commit txn
+                            txn_driver.try_commit(txn)?;
+                        }
+                        updated.into_iter().for_each(|(field_id, field)| {
+                            iwm.fields_mut().st_update(&field_id, field);
+                        });
+                    }
                 }
-                AlterAction::Remove(remove) => {
-                    let mut guard = model.delta_state().wguard();
-                    remove.iter().for_each(|field_id| {
-                        model
-                            .delta_state()
-                            .append_unresolved_wl_field_rem(&mut guard, field_id.as_str());
-                        iwm.fields_mut().st_delete(field_id.as_str());
-                    });
-                }
-                AlterAction::Update(u) => {
-                    u.into_iter().for_each(|(field_id, field)| {
-                        iwm.fields_mut().st_update(&field_id, field);
-                    });
-                }
-            }
-            Ok(())
+                Ok(())
+            })
+        })
+    }
+    pub fn exec_alter(gns: &GlobalNS, stmt: AlterModel) -> DatabaseResult<()> {
+        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, |driver| {
+            Self::transactional_exec_alter(gns, driver, stmt)
         })
     }
 }
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 727e800c..e712db06 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -47,6 +47,7 @@ use {
             drop::DropModel,
             syn::{FieldSpec, LayerSpec},
         },
+        txn::gns as gnstxn,
     },
     std::cell::UnsafeCell,
 };
@@ -202,20 +203,78 @@ impl Model {
 }
 
 impl Model {
-    pub fn exec_create(gns: &super::GlobalNS, stmt: CreateModel) -> DatabaseResult<()> {
+    pub fn transactional_exec_create<GI: gnstxn::GNSTransactionDriverLLInterface>(
+        gns: &super::GlobalNS,
+        txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<GI>,
+        stmt: CreateModel,
+    ) -> DatabaseResult<()> {
         let (space_name, model_name) = stmt.model_name.parse_entity()?;
         let model = Self::process_create(stmt)?;
-        gns.with_space(space_name, |space| space._create_model(model_name, model))
+        gns.with_space(space_name, |space| {
+            let mut w_space = space.models().write();
+            if w_space.st_contains(model_name) {
+                return Err(DatabaseError::DdlModelAlreadyExists);
+            }
+            if GI::NONNULL {
+                // prepare txn
+                let irm = model.intent_read_model();
+                let txn = gnstxn::CreateModelTxn::new(
+                    gnstxn::SpaceIDRef::new(space_name, space),
+                    model_name,
+                    &model,
+                    &irm,
+                );
+                // commit txn
+                txn_driver.try_commit(txn)?;
+            }
+            // update global state
+            let _ = w_space.st_insert(model_name.into(), model);
+            Ok(())
+        })
+    }
+    #[cfg(test)]
+    pub fn nontransactional_exec_create(
+        gns: &super::GlobalNS,
+        stmt: CreateModel,
+    ) -> DatabaseResult<()> {
+        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, |driver| {
+            Self::transactional_exec_create(gns, driver, stmt)
+        })
     }
-    pub fn exec_drop(gns: &super::GlobalNS, stmt: DropModel) -> DatabaseResult<()> {
-        let (space, model) = stmt.entity.parse_entity()?;
-        gns.with_space(space, |space| {
+    pub fn transactional_exec_drop<GI: gnstxn::GNSTransactionDriverLLInterface>(
+        gns: &super::GlobalNS,
+        txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<GI>,
+        stmt: DropModel,
+    ) -> DatabaseResult<()> {
+        let (space_name, model_name) = stmt.entity.parse_entity()?;
+        gns.with_space(space_name, |space| {
             let mut w_space = space.models().write();
-            match w_space.st_delete_if(model, |mdl| !mdl.is_empty_atomic()) {
-                Some(true) => Ok(()),
-                Some(false) => Err(DatabaseError::DdlModelViewNotEmpty),
-                None => Err(DatabaseError::DdlModelNotFound),
+            let Some(model) = w_space.get(model_name) else {
+                return Err(DatabaseError::DdlModelNotFound);
+            };
+            if GI::NONNULL {
+                // prepare txn
+                let txn = gnstxn::DropModelTxn::new(gnstxn::ModelIDRef::new(
+                    gnstxn::SpaceIDRef::new(space_name, space),
+                    model_name,
+                    model.get_uuid(),
+                    model.delta_state().current_version().value_u64(),
+                ));
+                // commit txn
+                txn_driver.try_commit(txn)?;
             }
+            // update global state
+            let _ = w_space.st_delete(model_name);
+            Ok(())
+        })
+    }
+    #[cfg(test)]
+    pub fn nontransactional_exec_drop(
+        gns: &super::GlobalNS,
+        stmt: DropModel,
+    ) -> DatabaseResult<()> {
+        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, |driver| {
+            Self::transactional_exec_drop(gns, driver, stmt)
         })
     }
 }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 22f6febd..a4094167 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -212,7 +212,10 @@ impl Space {
     }
     /// Execute a `create` stmt
     #[cfg(test)]
-    pub fn exec_create(gns: &super::GlobalNS, space: CreateSpace) -> DatabaseResult<()> {
+    pub fn nontransactional_exec_create(
+        gns: &super::GlobalNS,
+        space: CreateSpace,
+    ) -> DatabaseResult<()> {
         gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, move |driver| {
             Self::transactional_exec_create(gns, driver, space)
         })
@@ -258,7 +261,10 @@ impl Space {
     }
     #[cfg(test)]
     /// Execute a `alter` stmt
-    pub fn exec_alter(gns: &super::GlobalNS, alter: AlterSpace) -> DatabaseResult<()> {
+    pub fn nontransactional_exec_alter(
+        gns: &super::GlobalNS,
+        alter: AlterSpace,
+    ) -> DatabaseResult<()> {
         gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, move |driver| {
             Self::transactional_exec_alter(gns, driver, alter)
         })
@@ -291,7 +297,10 @@ impl Space {
         Ok(())
     }
     #[cfg(test)]
-    pub fn exec_drop(gns: &super::GlobalNS, drop_space: DropSpace) -> DatabaseResult<()> {
+    pub fn nontransactional_exec_drop(
+        gns: &super::GlobalNS,
+        drop_space: DropSpace,
+    ) -> DatabaseResult<()> {
         gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, move |driver| {
             Self::transactional_exec_drop(gns, driver, drop_space)
         })
diff --git a/server/src/engine/core/tests/ddl_model/mod.rs b/server/src/engine/core/tests/ddl_model/mod.rs
index 7ae3f75a..70b1b295 100644
--- a/server/src/engine/core/tests/ddl_model/mod.rs
+++ b/server/src/engine/core/tests/ddl_model/mod.rs
@@ -58,7 +58,7 @@ pub fn exec_create(
     if create_new_space {
         gns.test_new_empty_space(&create_model.model_name.into_full().unwrap().0);
     }
-    Model::exec_create(gns, create_model).map(|_| name)
+    Model::nontransactional_exec_create(gns, create_model).map(|_| name)
 }
 
 pub fn exec_create_new_space(gns: &GlobalNS, create_stmt: &str) -> DatabaseResult<()> {
diff --git a/server/src/engine/core/tests/ddl_space/mod.rs b/server/src/engine/core/tests/ddl_space/mod.rs
index b33babf1..0518c375 100644
--- a/server/src/engine/core/tests/ddl_space/mod.rs
+++ b/server/src/engine/core/tests/ddl_space/mod.rs
@@ -42,7 +42,7 @@ fn exec_create(gns: &GlobalNS, create: &str, verify: impl Fn(&Space)) -> Databas
     let ast_node =
         ast::parse_ast_node_full::<crate::engine::ql::ddl::crt::CreateSpace>(&tok[2..]).unwrap();
     let name = ast_node.space_name;
-    Space::exec_create(gns, ast_node)?;
+    Space::nontransactional_exec_create(gns, ast_node)?;
     gns.with_space(&name, |space| {
         verify(space);
         Ok(space.get_uuid())
@@ -54,7 +54,7 @@ fn exec_alter(gns: &GlobalNS, alter: &str, verify: impl Fn(&Space)) -> DatabaseR
     let ast_node =
         ast::parse_ast_node_full::<crate::engine::ql::ddl::alt::AlterSpace>(&tok[2..]).unwrap();
     let name = ast_node.space_name;
-    Space::exec_alter(gns, ast_node)?;
+    Space::nontransactional_exec_alter(gns, ast_node)?;
     gns.with_space(&name, |space| {
         verify(space);
         Ok(space.get_uuid())
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index dc710ed9..6a4b8d23 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -47,7 +47,7 @@ fn _exec_only_create_space_model(gns: &GlobalNS, model: &str) -> DatabaseResult<
     }
     let lex_create_model = lex_insecure(model.as_bytes()).unwrap();
     let stmt_create_model = parse_ast_node_full(&lex_create_model[2..]).unwrap();
-    Model::exec_create(gns, stmt_create_model)
+    Model::nontransactional_exec_create(gns, stmt_create_model)
 }
 
 fn _exec_only_insert<T>(
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index a604ee40..821a873a 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -53,6 +53,7 @@ mod tests;
 pub use {
     model::{
         AlterModelAddTxn, AlterModelRemoveTxn, AlterModelUpdateTxn, CreateModelTxn, DropModelTxn,
+        ModelIDRef,
     },
     space::{AlterSpaceTxn, CreateSpaceTxn, DropSpaceTxn},
 };
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index 55c25060..7ed0497e 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -57,6 +57,19 @@ pub struct ModelIDRef<'a> {
 }
 
 impl<'a> ModelIDRef<'a> {
+    pub fn new_ref(
+        space_name: &'a str,
+        space: &'a Space,
+        model_name: &'a str,
+        model: &'a Model,
+    ) -> ModelIDRef<'a> {
+        ModelIDRef::new(
+            super::SpaceIDRef::new(space_name, space),
+            model_name,
+            model.get_uuid(),
+            model.delta_state().current_version().value_u64(),
+        )
+    }
     pub fn new(
         space_id: super::SpaceIDRef<'a>,
         model_name: &'a str,
@@ -177,7 +190,7 @@ fn with_model<T>(
     create model
 */
 
-#[derive(Clone, Copy)]
+#[derive(Debug, Clone, Copy)]
 /// The commit payload for a `create model ... (...) with {...}` txn
 pub struct CreateModelTxn<'a> {
     space_id: super::SpaceIDRef<'a>,
diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
index 03eb2669..604c9023 100644
--- a/server/src/engine/txn/gns/tests/full_chain.rs
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -26,11 +26,17 @@
 
 use crate::engine::{
     core::{
+        model::{Field, Layer, Model},
         space::{Space, SpaceMeta},
         GlobalNS,
     },
-    data::{cell::Datacell, uuid::Uuid, DictEntryGeneric},
-    ql::{ast::parse_ast_node_full, ddl::crt::CreateSpace, tests::lex_insecure},
+    data::{cell::Datacell, tag::TagSelector, uuid::Uuid, DictEntryGeneric},
+    idx::STIndex,
+    ql::{
+        ast::parse_ast_node_full,
+        ddl::crt::{CreateModel, CreateSpace},
+        tests::lex_insecure,
+    },
     storage::v1::header_meta::HostRunMode,
     txn::gns::GNSTransactionDriverVFS,
 };
@@ -143,3 +149,103 @@ fn drop_space() {
         })
     })
 }
+
+fn init_model(
+    gns: &GlobalNS,
+    txn_driver: &mut GNSTransactionDriverVFS,
+    space_name: &str,
+    model_name: &str,
+    decl: &str,
+) -> Uuid {
+    let query = format!("create model {space_name}.{model_name} ({decl})");
+    let stmt = lex_insecure(query.as_bytes()).unwrap();
+    let stmt = parse_ast_node_full::<CreateModel>(&stmt[2..]).unwrap();
+    let model_name = stmt.model_name;
+    Model::transactional_exec_create(&gns, txn_driver, stmt).unwrap();
+    gns.with_model(model_name, |model| Ok(model.get_uuid()))
+        .unwrap()
+}
+
+fn init_default_model(gns: &GlobalNS, driver: &mut GNSTransactionDriverVFS) -> Uuid {
+    init_model(
+        gns,
+        driver,
+        "myspace",
+        "mymodel",
+        "username: string, password: binary",
+    )
+}
+
+#[test]
+fn create_model() {
+    with_variable("create_model_test.gns.db-tlog", |log_name| {
+        let _uuid_space;
+        let uuid_model;
+        {
+            let gns = GlobalNS::empty();
+            let mut driver = init_txn_driver(&gns, log_name);
+            _uuid_space = init_space(&gns, &mut driver, "myspace", "{}");
+            uuid_model = init_default_model(&gns, &mut driver);
+            driver.close().unwrap();
+        }
+        double_run(|| {
+            let gns = GlobalNS::empty();
+            let driver = init_txn_driver(&gns, log_name);
+            gns.with_model(("myspace", "mymodel"), |model| {
+                assert_eq!(
+                    model,
+                    &Model::new_restore(
+                        uuid_model,
+                        "username".into(),
+                        TagSelector::Str.into_full(),
+                        into_dict! {
+                            "username" => Field::new([Layer::str()].into(), false),
+                            "password" => Field::new([Layer::bin()].into(), false),
+                        }
+                    )
+                );
+                Ok(())
+            })
+            .unwrap();
+            driver.close().unwrap();
+        })
+    })
+}
+
+#[test]
+fn alter_model_add() {
+    with_variable("alter_model_add_test.gns.db-tlog", |log_name| {
+        {
+            let gns = GlobalNS::empty();
+            let mut driver = init_txn_driver(&gns, log_name);
+            init_space(&gns, &mut driver, "myspace", "{}");
+            init_default_model(&gns, &mut driver);
+            let stmt = lex_insecure(
+                b"alter model myspace.mymodel add profile_pic { type: binary, nullable: true }",
+            )
+            .unwrap();
+            let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
+            Model::transactional_exec_alter(&gns, &mut driver, stmt).unwrap();
+            driver.close().unwrap();
+        }
+        {
+            double_run(|| {
+                let gns = GlobalNS::empty();
+                let driver = init_txn_driver(&gns, log_name);
+                gns.with_model(("myspace", "mymodel"), |model| {
+                    assert_eq!(
+                        model
+                            .intent_read_model()
+                            .fields()
+                            .st_get("profile_pic")
+                            .unwrap(),
+                        &Field::new([Layer::bin()].into(), true)
+                    );
+                    Ok(())
+                })
+                .unwrap();
+                driver.close().unwrap();
+            })
+        }
+    })
+}

From f230bc79200ce8e3f1430c0502773f6c73a1b295 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 26 Aug 2023 13:48:38 +0000
Subject: [PATCH 239/310] Add all model DDL txn tests

---
 server/src/engine/txn/gns/tests/full_chain.rs | 150 +++++++++++++++---
 1 file changed, 127 insertions(+), 23 deletions(-)

diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
index 604c9023..4ea29bcf 100644
--- a/server/src/engine/txn/gns/tests/full_chain.rs
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -31,6 +31,7 @@ use crate::engine::{
         GlobalNS,
     },
     data::{cell::Datacell, tag::TagSelector, uuid::Uuid, DictEntryGeneric},
+    error::DatabaseError,
     idx::STIndex,
     ql::{
         ast::parse_ast_node_full,
@@ -41,9 +42,10 @@ use crate::engine::{
     txn::gns::GNSTransactionDriverVFS,
 };
 
-fn double_run(f: impl FnOnce() + Copy) {
-    f();
-    f();
+fn multirun(f: impl FnOnce() + Copy) {
+    for _ in 0..10 {
+        f()
+    }
 }
 
 fn with_variable<T>(var: T, f: impl FnOnce(T)) {
@@ -80,7 +82,7 @@ fn create_space() {
             uuid = init_space(&gns, &mut driver, "myspace", "{ SAYAN_MAX: 65536 }"); // good lord that doesn't sound like a good variable
             driver.close().unwrap();
         }
-        double_run(|| {
+        multirun(|| {
             let gns = GlobalNS::empty();
             let driver = init_txn_driver(&gns, log_name);
             assert_eq!(
@@ -112,7 +114,7 @@ fn alter_space() {
             Space::transactional_exec_alter(&gns, &mut driver, stmt).unwrap();
             driver.close().unwrap();
         }
-        double_run(|| {
+        multirun(|| {
             let gns = GlobalNS::empty();
             let driver = init_txn_driver(&gns, log_name);
             assert_eq!(
@@ -141,7 +143,7 @@ fn drop_space() {
             Space::transactional_exec_drop(&gns, &mut driver, stmt).unwrap();
             driver.close().unwrap();
         }
-        double_run(|| {
+        multirun(|| {
             let gns = GlobalNS::empty();
             let driver = init_txn_driver(&gns, log_name);
             assert_eq!(gns.spaces().read().get("myspace"), None);
@@ -188,7 +190,7 @@ fn create_model() {
             uuid_model = init_default_model(&gns, &mut driver);
             driver.close().unwrap();
         }
-        double_run(|| {
+        multirun(|| {
             let gns = GlobalNS::empty();
             let driver = init_txn_driver(&gns, log_name);
             gns.with_model(("myspace", "mymodel"), |model| {
@@ -228,24 +230,126 @@ fn alter_model_add() {
             Model::transactional_exec_alter(&gns, &mut driver, stmt).unwrap();
             driver.close().unwrap();
         }
+        multirun(|| {
+            let gns = GlobalNS::empty();
+            let driver = init_txn_driver(&gns, log_name);
+            gns.with_model(("myspace", "mymodel"), |model| {
+                assert_eq!(
+                    model
+                        .intent_read_model()
+                        .fields()
+                        .st_get("profile_pic")
+                        .unwrap(),
+                    &Field::new([Layer::bin()].into(), true)
+                );
+                Ok(())
+            })
+            .unwrap();
+            driver.close().unwrap();
+        })
+    })
+}
+
+#[test]
+fn alter_model_remove() {
+    with_variable("alter_model_remove_test.gns.db-tlog", |log_name| {
+        {
+            let gns = GlobalNS::empty();
+            let mut driver = init_txn_driver(&gns, log_name);
+            init_space(&gns, &mut driver, "myspace", "{}");
+            init_model(
+                &gns,
+                &mut driver,
+                "myspace",
+                "mymodel",
+                "username: string, password: binary, null profile_pic: binary, null has_2fa: bool, null has_secure_key: bool, is_dumb: bool",
+            );
+            let stmt = lex_insecure(
+                "alter model myspace.mymodel remove (has_secure_key, is_dumb)".as_bytes(),
+            )
+            .unwrap();
+            let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
+            Model::transactional_exec_alter(&gns, &mut driver, stmt).unwrap();
+            driver.close().unwrap()
+        }
+        multirun(|| {
+            let gns = GlobalNS::empty();
+            let driver = init_txn_driver(&gns, log_name);
+            gns.with_model(("myspace", "mymodel"), |model| {
+                let irm = model.intent_read_model();
+                assert!(irm.fields().st_get("has_secure_key").is_none());
+                assert!(irm.fields().st_get("is_dumb").is_none());
+                Ok(())
+            })
+            .unwrap();
+            driver.close().unwrap()
+        })
+    })
+}
+
+#[test]
+fn alter_model_update() {
+    with_variable("alter_model_update_test.gns.db-tlog", |log_name| {
         {
-            double_run(|| {
-                let gns = GlobalNS::empty();
-                let driver = init_txn_driver(&gns, log_name);
-                gns.with_model(("myspace", "mymodel"), |model| {
-                    assert_eq!(
-                        model
-                            .intent_read_model()
-                            .fields()
-                            .st_get("profile_pic")
-                            .unwrap(),
-                        &Field::new([Layer::bin()].into(), true)
-                    );
-                    Ok(())
-                })
-                .unwrap();
-                driver.close().unwrap();
+            let gns = GlobalNS::empty();
+            let mut driver = init_txn_driver(&gns, log_name);
+            init_space(&gns, &mut driver, "myspace", "{}");
+            init_model(
+                &gns,
+                &mut driver,
+                "myspace",
+                "mymodel",
+                "username: string, password: binary, profile_pic: binary",
+            );
+            let stmt =
+                lex_insecure(b"alter model myspace.mymodel update profile_pic { nullable: true }")
+                    .unwrap();
+            let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
+            Model::transactional_exec_alter(&gns, &mut driver, stmt).unwrap();
+            driver.close().unwrap();
+        }
+        multirun(|| {
+            let gns = GlobalNS::empty();
+            let driver = init_txn_driver(&gns, log_name);
+            gns.with_model(("myspace", "mymodel"), |model| {
+                assert_eq!(
+                    model
+                        .intent_read_model()
+                        .fields()
+                        .st_get("profile_pic")
+                        .unwrap(),
+                    &Field::new([Layer::bin()].into(), true)
+                );
+                Ok(())
             })
+            .unwrap();
+            driver.close().unwrap();
+        })
+    })
+}
+
+#[test]
+fn drop_model() {
+    with_variable("drop_model_test.gns.db-tlog", |log_name| {
+        {
+            let gns = GlobalNS::empty();
+            let mut driver = init_txn_driver(&gns, log_name);
+            init_space(&gns, &mut driver, "myspace", "{}");
+            init_default_model(&gns, &mut driver);
+            let stmt = lex_insecure(b"drop model myspace.mymodel").unwrap();
+            let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
+            Model::transactional_exec_drop(&gns, &mut driver, stmt).unwrap();
+            driver.close().unwrap();
         }
+        multirun(|| {
+            let gns = GlobalNS::empty();
+            let driver = init_txn_driver(&gns, log_name);
+            assert_eq!(
+                gns.with_model(("myspace", "mymodel"), |_| { Ok(()) })
+                    .unwrap_err(),
+                DatabaseError::DdlModelNotFound
+            );
+            driver.close().unwrap();
+        })
     })
 }

From e4848e645e3cd94e42bd45409ecd9794228283a5 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 2 Sep 2023 08:20:29 +0000
Subject: [PATCH 240/310] Sync row deltas to delta state

---
 server/src/engine/core/dml/del.rs             |  24 +-
 server/src/engine/core/dml/ins.rs             |  17 +-
 server/src/engine/core/dml/sel.rs             |   2 +-
 server/src/engine/core/dml/upd.rs             |  17 +-
 server/src/engine/core/index/key.rs           |   6 +
 server/src/engine/core/index/mod.rs           |  14 +-
 server/src/engine/core/index/row.rs           |  41 +--
 server/src/engine/core/mod.rs                 |   2 +-
 server/src/engine/core/model/alt.rs           |   8 +-
 server/src/engine/core/model/delta.rs         | 215 +++++++++++-----
 server/src/engine/core/model/mod.rs           |   4 +-
 server/src/engine/core/tests/ddl_model/alt.rs |   6 +-
 server/src/engine/core/tests/ddl_model/crt.rs |   6 +-
 server/src/engine/idx/mod.rs                  |   5 +
 server/src/engine/idx/mtchm/imp.rs            |  11 +-
 server/src/engine/idx/mtchm/patch.rs          |  34 +++
 server/src/engine/mem/ll.rs                   | 103 ++++++++
 server/src/engine/mem/mod.rs                  |   2 +
 server/src/engine/storage/v1/inf/map.rs       |  38 +--
 server/src/engine/storage/v1/inf/obj.rs       |  38 ++-
 server/src/engine/sync/mod.rs                 |   1 +
 server/src/engine/sync/queue.rs               | 236 ++++++++++++++++++
 server/src/engine/txn/data.rs                 |  81 ++++++
 server/src/engine/txn/gns/model.rs            |   2 +-
 server/src/engine/txn/gns/tests/io.rs         |  16 +-
 server/src/engine/txn/mod.rs                  |   1 +
 server/src/util/os.rs                         |   2 +
 server/src/util/os/free_memory.rs             |  90 +++++++
 28 files changed, 855 insertions(+), 167 deletions(-)
 create mode 100644 server/src/engine/mem/ll.rs
 create mode 100644 server/src/engine/sync/queue.rs
 create mode 100644 server/src/engine/txn/data.rs
 create mode 100644 server/src/util/os/free_memory.rs

diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs
index f85b00fd..0b1f6dfb 100644
--- a/server/src/engine/core/dml/del.rs
+++ b/server/src/engine/core/dml/del.rs
@@ -25,8 +25,9 @@
 */
 
 use crate::engine::{
-    core::GlobalNS,
+    core::{model::delta::DataDeltaKind, GlobalNS},
     error::{DatabaseError, DatabaseResult},
+    idx::MTIndex,
     ql::dml::del::DeleteStatement,
     sync,
 };
@@ -34,13 +35,24 @@ use crate::engine::{
 pub fn delete(gns: &GlobalNS, mut delete: DeleteStatement) -> DatabaseResult<()> {
     gns.with_model(delete.entity(), |model| {
         let g = sync::atm::cpin();
-        if model
+        let delta_state = model.delta_state();
+        // create new version
+        let new_version = delta_state.create_new_data_delta_version();
+        match model
             .primary_index()
-            .remove(model.resolve_where(delete.clauses_mut())?, &g)
+            .__raw_index()
+            .mt_delete_return_entry(&model.resolve_where(delete.clauses_mut())?, &g)
         {
-            Ok(())
-        } else {
-            Err(DatabaseError::DmlEntryNotFound)
+            Some(row) => {
+                delta_state.append_new_data_delta(
+                    DataDeltaKind::Delete,
+                    row.clone(),
+                    new_version,
+                    &g,
+                );
+                Ok(())
+            }
+            None => Err(DatabaseError::DmlEntryNotFound),
         }
     })
 }
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index bc3cd29e..ede87bf4 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -26,12 +26,12 @@
 
 use crate::engine::{
     core::{
-        index::{DcFieldIndex, PrimaryIndexKey},
-        model::{Fields, Model},
+        index::{DcFieldIndex, PrimaryIndexKey, Row},
+        model::{delta::DataDeltaKind, Fields, Model},
         GlobalNS,
     },
     error::{DatabaseError, DatabaseResult},
-    idx::{IndexBaseSpec, STIndex, STIndexSeq},
+    idx::{IndexBaseSpec, MTIndex, STIndex, STIndexSeq},
     ql::dml::ins::{InsertData, InsertStatement},
     sync::atm::cpin,
 };
@@ -41,10 +41,13 @@ pub fn insert(gns: &GlobalNS, insert: InsertStatement) -> DatabaseResult<()> {
         let irmwd = mdl.intent_write_new_data();
         let (pk, data) = prepare_insert(mdl, irmwd.fields(), insert.data())?;
         let g = cpin();
-        if mdl
-            .primary_index()
-            .insert(pk, data, mdl.delta_state().current_version(), &g)
-        {
+        let ds = mdl.delta_state();
+        // create new version
+        let cv = ds.create_new_data_delta_version();
+        let row = Row::new(pk, data, ds.schema_current_version(), cv);
+        if mdl.primary_index().__raw_index().mt_insert(row.clone(), &g) {
+            // append delta for new version
+            ds.append_new_data_delta(DataDeltaKind::Insert, row, cv, &g);
             Ok(())
         } else {
             Err(DatabaseError::DmlConstraintViolationDuplicate)
diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
index 869e9b6a..29d3881e 100644
--- a/server/src/engine/core/dml/sel.rs
+++ b/server/src/engine/core/dml/sel.rs
@@ -56,7 +56,7 @@ where
         };
         match mdl.primary_index().select(target_key.clone(), &g) {
             Some(row) => {
-                let r = row.resolve_deltas_and_freeze(mdl.delta_state());
+                let r = row.resolve_schema_deltas_and_freeze(mdl.delta_state());
                 if select.is_wildcard() {
                     for key in irm.fields().stseq_ord_key() {
                         read_field(key.as_ref(), r.fields())?;
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index 24237033..a53b5ec9 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -26,10 +26,11 @@
 
 #[cfg(test)]
 use std::cell::RefCell;
+
 use {
     crate::{
         engine::{
-            core::{query_meta::AssignmentOperator, GlobalNS},
+            core::{model::delta::DataDeltaKind, query_meta::AssignmentOperator, GlobalNS},
             data::{
                 cell::Datacell,
                 lit::LitIR,
@@ -234,19 +235,28 @@ pub fn collect_trace_path() -> Vec<&'static str> {
 pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()> {
     gns.with_model(update.entity(), |mdl| {
         let mut ret = Ok(());
+        // prepare row fetch
         let key = mdl.resolve_where(update.clauses_mut())?;
+        // freeze schema
         let irm = mdl.intent_read_model();
+        // fetch row
         let g = sync::atm::cpin();
         let Some(row) = mdl.primary_index().select(key, &g) else {
             return Err(DatabaseError::DmlEntryNotFound);
         };
+        // lock row
         let mut row_data_wl = row.d_data().write();
+        // create new version
+        let ds = mdl.delta_state();
+        let cv = ds.create_new_data_delta_version();
+        // process changes
         let mut rollback_now = false;
         let mut rollback_data = Vec::with_capacity(update.expressions().len());
         let mut assn_expressions = update.into_expressions().into_iter();
         /*
             FIXME(@ohsayan): where's my usual magic? I'll do it once we have the SE stabilized
         */
+        // apply changes
         while (assn_expressions.len() != 0) & (!rollback_now) {
             let AssignmentExpression {
                 lhs,
@@ -329,6 +339,11 @@ pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()>
                 .for_each(|(field_id, restored_data)| {
                     row_data_wl.fields_mut().st_update(field_id, restored_data);
                 });
+        } else {
+            // update revised tag
+            row_data_wl.set_txn_revised(cv);
+            // publish delta
+            ds.append_new_data_delta(DataDeltaKind::Update, row.clone(), cv, &g);
         }
         ret
     })
diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
index dea6c360..347a2c6a 100644
--- a/server/src/engine/core/index/key.rs
+++ b/server/src/engine/core/index/key.rs
@@ -50,6 +50,12 @@ pub struct PrimaryIndexKey {
     data: SpecialPaddedWord,
 }
 
+impl PrimaryIndexKey {
+    pub fn tag(&self) -> TagUnique {
+        self.tag
+    }
+}
+
 impl PrimaryIndexKey {
     pub unsafe fn read_uint(&self) -> u64 {
         self.data.load()
diff --git a/server/src/engine/core/index/mod.rs b/server/src/engine/core/index/mod.rs
index 9cd33a81..46d3a125 100644
--- a/server/src/engine/core/index/mod.rs
+++ b/server/src/engine/core/index/mod.rs
@@ -28,7 +28,6 @@ mod key;
 mod row;
 
 use crate::engine::{
-    core::model::DeltaVersion,
     data::lit::LitIR,
     idx::{IndexBaseSpec, IndexMTRaw, MTIndex},
     sync::atm::Guard,
@@ -50,16 +49,6 @@ impl PrimaryIndex {
             data: IndexMTRaw::idx_init(),
         }
     }
-    pub fn insert(
-        &self,
-        key: PrimaryIndexKey,
-        data: row::DcFieldIndex,
-        delta_version: DeltaVersion,
-        g: &Guard,
-    ) -> bool {
-        self.data
-            .mt_insert(Row::new(key, data, delta_version, delta_version), g)
-    }
     pub fn remove<'a>(&self, key: LitIR<'a>, g: &Guard) -> bool {
         self.data.mt_delete(&key, g)
     }
@@ -70,4 +59,7 @@ impl PrimaryIndex {
     ) -> Option<&'v Row> {
         self.data.mt_get_element(&key, g)
     }
+    pub fn __raw_index(&self) -> &IndexMTRaw<row::Row> {
+        &self.data
+    }
 }
diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
index 6cb7a907..5ad2d636 100644
--- a/server/src/engine/core/index/row.rs
+++ b/server/src/engine/core/index/row.rs
@@ -28,7 +28,7 @@ use {
     super::key::PrimaryIndexKey,
     crate::{
         engine::{
-            core::model::{DeltaKind, DeltaState, DeltaVersion},
+            core::model::{DeltaState, DeltaVersion, SchemaDeltaKind},
             data::cell::Datacell,
             idx::{meta::hash::HasherNativeFx, mtchm::meta::TreeElement, IndexST, STIndex},
             sync::smart::RawRC,
@@ -43,7 +43,7 @@ pub type DcFieldIndex = IndexST<Box<str>, Datacell, HasherNativeFx>;
 
 #[derive(Debug)]
 pub struct Row {
-    __txn_genesis: DeltaVersion,
+    __genesis_schema_version: DeltaVersion,
     __pk: ManuallyDrop<PrimaryIndexKey>,
     __rc: RawRC<RwLock<RowData>>,
 }
@@ -51,7 +51,8 @@ pub struct Row {
 #[derive(Debug, PartialEq)]
 pub struct RowData {
     fields: DcFieldIndex,
-    txn_revised: DeltaVersion,
+    txn_revised_data: DeltaVersion,
+    txn_revised_schema_version: DeltaVersion,
 }
 
 impl RowData {
@@ -61,6 +62,12 @@ impl RowData {
     pub fn fields_mut(&mut self) -> &mut DcFieldIndex {
         &mut self.fields
     }
+    pub fn set_txn_revised(&mut self, new: DeltaVersion) {
+        self.txn_revised_data = new;
+    }
+    pub fn get_txn_revised(&self) -> DeltaVersion {
+        self.txn_revised_data
+    }
 }
 
 impl TreeElement for Row {
@@ -90,17 +97,18 @@ impl Row {
     pub fn new(
         pk: PrimaryIndexKey,
         data: DcFieldIndex,
-        txn_genesis: DeltaVersion,
-        txn_revised: DeltaVersion,
+        schema_version: DeltaVersion,
+        txn_revised_data: DeltaVersion,
     ) -> Self {
         Self {
-            __txn_genesis: txn_genesis,
+            __genesis_schema_version: schema_version,
             __pk: ManuallyDrop::new(pk),
             __rc: unsafe {
                 // UNSAFE(@ohsayan): we free this up later
                 RawRC::new(RwLock::new(RowData {
                     fields: data,
-                    txn_revised,
+                    txn_revised_schema_version: schema_version,
+                    txn_revised_data,
                 }))
             },
         }
@@ -131,31 +139,32 @@ impl Row {
 }
 
 impl Row {
-    pub fn resolve_deltas_and_freeze<'g>(
+    pub fn resolve_schema_deltas_and_freeze<'g>(
         &'g self,
         delta_state: &DeltaState,
     ) -> RwLockReadGuard<'g, RowData> {
         let rwl_ug = self.d_data().upgradable_read();
-        let current_version = delta_state.current_version();
-        if compiler::likely(current_version <= rwl_ug.txn_revised) {
+        let current_version = delta_state.schema_current_version();
+        if compiler::likely(current_version <= rwl_ug.txn_revised_schema_version) {
             return RwLockUpgradableReadGuard::downgrade(rwl_ug);
         }
         // we have deltas to apply
         let mut wl = RwLockUpgradableReadGuard::upgrade(rwl_ug);
-        let delta_read = delta_state.rguard();
-        let mut max_delta = wl.txn_revised;
-        for (delta_id, delta) in delta_read.resolve_iter_since(wl.txn_revised) {
+        let delta_read = delta_state.schema_delta_read();
+        let mut max_delta = wl.txn_revised_schema_version;
+        for (delta_id, delta) in delta_read.resolve_iter_since(wl.txn_revised_schema_version) {
             match delta.kind() {
-                DeltaKind::FieldAdd(f) => {
+                SchemaDeltaKind::FieldAdd(f) => {
                     wl.fields.st_insert(f.clone(), Datacell::null());
                 }
-                DeltaKind::FieldRem(f) => {
+                SchemaDeltaKind::FieldRem(f) => {
                     wl.fields.st_delete(f);
                 }
             }
             max_delta = *delta_id;
         }
-        wl.txn_revised = max_delta;
+        // we've revised upto the most most recent delta version (that we saw at this point)
+        wl.txn_revised_schema_version = max_delta;
         return RwLockWriteGuard::downgrade(wl);
     }
 }
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 50d6cdfe..34a0427f 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -25,7 +25,7 @@
 */
 
 mod dml;
-mod index;
+pub(in crate::engine) mod index;
 pub(in crate::engine) mod model;
 pub(in crate::engine) mod query_meta;
 pub mod space;
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index cdfb2431..3f0a5d2e 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -270,7 +270,7 @@ impl Model {
                 match plan.action {
                     AlterAction::Ignore => drop(iwm),
                     AlterAction::Add(new_fields) => {
-                        let mut guard = model.delta_state().wguard();
+                        let mut guard = model.delta_state().schema_delta_write();
                         // TODO(@ohsayan): this impacts lockdown duration; fix it
                         if GI::NONNULL {
                             // prepare txn
@@ -287,12 +287,12 @@ impl Model {
                             .for_each(|(field_id, field)| {
                                 model
                                     .delta_state()
-                                    .append_unresolved_wl_field_add(&mut guard, &field_id);
+                                    .schema_append_unresolved_wl_field_add(&mut guard, &field_id);
                                 iwm.fields_mut().st_insert(field_id, field);
                             });
                     }
                     AlterAction::Remove(removed) => {
-                        let mut guard = model.delta_state().wguard();
+                        let mut guard = model.delta_state().schema_delta_write();
                         if GI::NONNULL {
                             // prepare txn
                             let txn = gnstxn::AlterModelRemoveTxn::new(
@@ -305,7 +305,7 @@ impl Model {
                         removed.iter().for_each(|field_id| {
                             model
                                 .delta_state()
-                                .append_unresolved_wl_field_rem(&mut guard, field_id.as_str());
+                                .schema_append_unresolved_wl_field_rem(&mut guard, field_id.as_str());
                             iwm.fields_mut().st_delete(field_id.as_str());
                         });
                     }
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index 9ffa369f..970094cb 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -26,6 +26,7 @@
 
 use {
     super::{Fields, Model},
+    crate::engine::{core::index::Row, sync::atm::Guard, sync::queue::Queue},
     parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard},
     std::{
         collections::btree_map::{BTreeMap, Range},
@@ -39,6 +40,8 @@ use {
 
 // FIXME(@ohsayan): This an inefficient repr of the matrix; replace it with my other design
 #[derive(Debug)]
+/// A sync matrix enables different queries to have different access permissions on the data model, and the data in the
+/// index
 pub struct ISyncMatrix {
     // virtual privileges
     /// read/write model
@@ -55,6 +58,7 @@ impl PartialEq for ISyncMatrix {
 }
 
 #[derive(Debug)]
+/// Read model, write new data
 pub struct IRModelSMData<'a> {
     rmodel: RwLockReadGuard<'a, ()>,
     mdata: RwLockReadGuard<'a, ()>,
@@ -80,6 +84,7 @@ impl<'a> IRModelSMData<'a> {
 }
 
 #[derive(Debug)]
+/// Read model
 pub struct IRModel<'a> {
     rmodel: RwLockReadGuard<'a, ()>,
     fields: &'a Fields,
@@ -101,6 +106,7 @@ impl<'a> IRModel<'a> {
 }
 
 #[derive(Debug)]
+/// Write model
 pub struct IWModel<'a> {
     wmodel: RwLockWriteGuard<'a, ()>,
     fields: &'a mut Fields,
@@ -139,48 +145,99 @@ impl ISyncMatrix {
 */
 
 #[derive(Debug)]
+/// A delta state for the model
 pub struct DeltaState {
-    current_version: AtomicU64,
-    deltas: RwLock<BTreeMap<DeltaVersion, DeltaPart>>,
+    // schema
+    schema_current_version: AtomicU64,
+    schema_deltas: RwLock<BTreeMap<DeltaVersion, SchemaDeltaPart>>,
+    // data
+    data_current_version: AtomicU64,
+    data_deltas: Queue<DataDelta>,
 }
 
-#[derive(Debug)]
-pub struct DeltaPart {
-    kind: DeltaKind,
+impl DeltaState {
+    /// A new, fully resolved delta state with version counters set to 0
+    pub fn new_resolved() -> Self {
+        Self {
+            schema_current_version: AtomicU64::new(0),
+            schema_deltas: RwLock::new(BTreeMap::new()),
+            data_current_version: AtomicU64::new(0),
+            data_deltas: Queue::new(),
+        }
+    }
 }
 
-impl DeltaPart {
-    pub fn kind(&self) -> &DeltaKind {
-        &self.kind
+// data
+impl DeltaState {
+    pub fn append_new_data_delta(
+        &self,
+        kind: DataDeltaKind,
+        row: Row,
+        version: DeltaVersion,
+        g: &Guard,
+    ) {
+        self.data_deltas
+            .blocking_enqueue(DataDelta::new(version.0, row, kind), g);
+    }
+    pub fn create_new_data_delta_version(&self) -> DeltaVersion {
+        DeltaVersion(self.__data_delta_step())
     }
 }
 
-#[derive(Debug)]
-pub enum DeltaKind {
-    FieldAdd(Box<str>),
-    FieldRem(Box<str>),
+impl DeltaState {
+    pub fn __data_delta_step(&self) -> u64 {
+        self.data_current_version.fetch_add(1, Ordering::AcqRel)
+    }
 }
 
-impl DeltaPart {
-    fn new(kind: DeltaKind) -> Self {
-        Self { kind }
+// schema
+impl DeltaState {
+    pub fn schema_delta_write<'a>(&'a self) -> SchemaDeltaIndexWGuard<'a> {
+        SchemaDeltaIndexWGuard(self.schema_deltas.write())
     }
-    fn field_add(field_name: &str) -> Self {
-        Self::new(DeltaKind::FieldAdd(field_name.to_owned().into_boxed_str()))
+    pub fn schema_delta_read<'a>(&'a self) -> SchemaDeltaIndexRGuard<'a> {
+        SchemaDeltaIndexRGuard(self.schema_deltas.read())
     }
-    fn field_rem(field_name: &str) -> Self {
-        Self::new(DeltaKind::FieldRem(field_name.to_owned().into_boxed_str()))
+    pub fn schema_current_version(&self) -> DeltaVersion {
+        self.__schema_delta_current()
+    }
+    pub fn schema_append_unresolved_wl_field_add(
+        &self,
+        guard: &mut SchemaDeltaIndexWGuard,
+        field_name: &str,
+    ) {
+        self.__schema_append_unresolved_delta(&mut guard.0, SchemaDeltaPart::field_add(field_name));
+    }
+    pub fn schema_append_unresolved_wl_field_rem(
+        &self,
+        guard: &mut SchemaDeltaIndexWGuard,
+        field_name: &str,
+    ) {
+        self.__schema_append_unresolved_delta(&mut guard.0, SchemaDeltaPart::field_rem(field_name));
+    }
+    pub fn schema_append_unresolved_field_add(&self, field_name: &str) {
+        self.schema_append_unresolved_wl_field_add(&mut self.schema_delta_write(), field_name);
+    }
+    pub fn schema_append_unresolved_field_rem(&self, field_name: &str) {
+        self.schema_append_unresolved_wl_field_rem(&mut self.schema_delta_write(), field_name);
     }
 }
 
-pub struct DeltaIndexWGuard<'a>(RwLockWriteGuard<'a, BTreeMap<DeltaVersion, DeltaPart>>);
-pub struct DeltaIndexRGuard<'a>(RwLockReadGuard<'a, BTreeMap<DeltaVersion, DeltaPart>>);
-impl<'a> DeltaIndexRGuard<'a> {
-    pub fn resolve_iter_since(
+impl DeltaState {
+    fn __schema_delta_step(&self) -> DeltaVersion {
+        DeltaVersion(self.schema_current_version.fetch_add(1, Ordering::AcqRel))
+    }
+    fn __schema_delta_current(&self) -> DeltaVersion {
+        DeltaVersion(self.schema_current_version.load(Ordering::Acquire))
+    }
+    fn __schema_append_unresolved_delta(
         &self,
-        current_version: DeltaVersion,
-    ) -> Range<DeltaVersion, DeltaPart> {
-        self.0.range(current_version.step()..)
+        w: &mut BTreeMap<DeltaVersion, SchemaDeltaPart>,
+        part: SchemaDeltaPart,
+    ) -> DeltaVersion {
+        let v = self.__schema_delta_step();
+        w.insert(v, part);
+        v
     }
 }
 
@@ -202,50 +259,80 @@ impl DeltaVersion {
     }
 }
 
-impl DeltaState {
-    pub fn new_resolved() -> Self {
-        Self {
-            current_version: AtomicU64::new(0),
-            deltas: RwLock::new(BTreeMap::new()),
-        }
-    }
-    pub fn wguard<'a>(&'a self) -> DeltaIndexWGuard<'a> {
-        DeltaIndexWGuard(self.deltas.write())
-    }
-    pub fn rguard<'a>(&'a self) -> DeltaIndexRGuard<'a> {
-        DeltaIndexRGuard(self.deltas.read())
-    }
-    pub fn current_version(&self) -> DeltaVersion {
-        self.__delta_current()
-    }
-    pub fn append_unresolved_wl_field_add(&self, guard: &mut DeltaIndexWGuard, field_name: &str) {
-        self.__append_unresolved_delta(&mut guard.0, DeltaPart::field_add(field_name));
+/*
+    schema delta
+*/
+
+#[derive(Debug)]
+pub struct SchemaDeltaPart {
+    kind: SchemaDeltaKind,
+}
+
+impl SchemaDeltaPart {
+    pub fn kind(&self) -> &SchemaDeltaKind {
+        &self.kind
     }
-    pub fn append_unresolved_wl_field_rem(&self, guard: &mut DeltaIndexWGuard, field_name: &str) {
-        self.__append_unresolved_delta(&mut guard.0, DeltaPart::field_rem(field_name));
+}
+
+#[derive(Debug)]
+pub enum SchemaDeltaKind {
+    FieldAdd(Box<str>),
+    FieldRem(Box<str>),
+}
+
+impl SchemaDeltaPart {
+    fn new(kind: SchemaDeltaKind) -> Self {
+        Self { kind }
     }
-    pub fn append_unresolved_field_add(&self, field_name: &str) {
-        self.append_unresolved_wl_field_add(&mut self.wguard(), field_name);
+    fn field_add(field_name: &str) -> Self {
+        Self::new(SchemaDeltaKind::FieldAdd(
+            field_name.to_owned().into_boxed_str(),
+        ))
     }
-    pub fn append_unresolved_field_rem(&self, field_name: &str) {
-        self.append_unresolved_wl_field_rem(&mut self.wguard(), field_name);
+    fn field_rem(field_name: &str) -> Self {
+        Self::new(SchemaDeltaKind::FieldRem(
+            field_name.to_owned().into_boxed_str(),
+        ))
     }
 }
 
-impl DeltaState {
-    fn __delta_step(&self) -> DeltaVersion {
-        DeltaVersion(self.current_version.fetch_add(1, Ordering::AcqRel))
-    }
-    fn __delta_current(&self) -> DeltaVersion {
-        DeltaVersion(self.current_version.load(Ordering::Acquire))
-    }
-    fn __append_unresolved_delta(
+pub struct SchemaDeltaIndexWGuard<'a>(
+    RwLockWriteGuard<'a, BTreeMap<DeltaVersion, SchemaDeltaPart>>,
+);
+pub struct SchemaDeltaIndexRGuard<'a>(RwLockReadGuard<'a, BTreeMap<DeltaVersion, SchemaDeltaPart>>);
+impl<'a> SchemaDeltaIndexRGuard<'a> {
+    pub fn resolve_iter_since(
         &self,
-        w: &mut BTreeMap<DeltaVersion, DeltaPart>,
-        part: DeltaPart,
-    ) -> DeltaVersion {
-        let v = self.__delta_step();
-        w.insert(v, part);
-        v
+        current_version: DeltaVersion,
+    ) -> Range<DeltaVersion, SchemaDeltaPart> {
+        self.0.range(current_version.step()..)
+    }
+}
+
+/*
+    data delta
+*/
+
+#[derive(Debug)]
+pub struct DataDelta {
+    version: u64,
+    row: Row,
+    change: DataDeltaKind,
+}
+
+impl DataDelta {
+    pub const fn new(version: u64, row: Row, change: DataDeltaKind) -> Self {
+        Self {
+            version,
+            row,
+            change,
+        }
     }
 }
+
+#[derive(Debug)]
+pub enum DataDeltaKind {
+    Insert,
+    Update,
+    Delete,
+}
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index e712db06..82e4ddf3 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -52,7 +52,7 @@ use {
     std::cell::UnsafeCell,
 };
 
-pub(in crate::engine::core) use self::delta::{DeltaKind, DeltaState, DeltaVersion};
+pub(in crate::engine::core) use self::delta::{SchemaDeltaKind, DeltaState, DeltaVersion};
 pub(in crate::engine) type Fields = IndexSTSeqCns<Box<str>, Field>;
 
 #[derive(Debug)]
@@ -258,7 +258,7 @@ impl Model {
                     gnstxn::SpaceIDRef::new(space_name, space),
                     model_name,
                     model.get_uuid(),
-                    model.delta_state().current_version().value_u64(),
+                    model.delta_state().schema_current_version().value_u64(),
                 ));
                 // commit txn
                 txn_driver.try_commit(txn)?;
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index 53cc1656..abcf5d12 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -383,7 +383,7 @@ mod exec {
                     ]
                 );
                 assert_eq!(
-                    model.delta_state().current_version(),
+                    model.delta_state().schema_current_version(),
                     DeltaVersion::test_new(2)
                 );
             },
@@ -410,7 +410,7 @@ mod exec {
                     [("username".into(), Field::new([Layer::str()].into(), false))]
                 );
                 assert_eq!(
-                    mdl.delta_state().current_version(),
+                    mdl.delta_state().schema_current_version(),
                     DeltaVersion::test_new(4)
                 );
             }
@@ -428,7 +428,7 @@ mod exec {
                 let schema = model.intent_read_model();
                 assert!(schema.fields().st_get("password").unwrap().is_nullable());
                 assert_eq!(
-                    model.delta_state().current_version(),
+                    model.delta_state().schema_current_version(),
                     DeltaVersion::genesis()
                 );
             },
diff --git a/server/src/engine/core/tests/ddl_model/crt.rs b/server/src/engine/core/tests/ddl_model/crt.rs
index 4f6a2276..cc87b474 100644
--- a/server/src/engine/core/tests/ddl_model/crt.rs
+++ b/server/src/engine/core/tests/ddl_model/crt.rs
@@ -53,7 +53,7 @@ mod validation {
             ]
         );
         assert_eq!(
-            model.delta_state().current_version(),
+            model.delta_state().schema_current_version(),
             DeltaVersion::genesis()
         );
     }
@@ -77,7 +77,7 @@ mod validation {
             ]
         );
         assert_eq!(
-            model.delta_state().current_version(),
+            model.delta_state().schema_current_version(),
             DeltaVersion::genesis()
         );
     }
@@ -174,7 +174,7 @@ mod exec {
                 ]
             );
             assert_eq!(
-                model.delta_state().current_version(),
+                model.delta_state().schema_current_version(),
                 DeltaVersion::genesis()
             );
         });
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index abcb7719..835fca50 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -207,6 +207,11 @@ pub trait MTIndex<E, K, V>: IndexBaseSpec {
         Q: ?Sized + Comparable<K>,
         't: 'v,
         'g: 't + 'v;
+    fn mt_delete_return_entry<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v E>
+    where
+        Q: ?Sized + Comparable<K>,
+        't: 'v,
+        'g: 't + 'v;
 }
 
 /// An unordered STIndex
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index ca71697d..a253a81e 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -30,7 +30,7 @@ use {
     super::{
         iter::{IterKV, IterKey, IterVal},
         meta::{Config, TreeElement},
-        patch::{VanillaInsert, VanillaUpdate, VanillaUpdateRet, VanillaUpsert},
+        patch::{DeleteRetEntry, VanillaInsert, VanillaUpdate, VanillaUpdateRet, VanillaUpsert},
         RawTree,
     },
     crate::engine::{
@@ -171,4 +171,13 @@ impl<E: TreeElement, C: Config> MTIndex<E, E::Key, E::Value> for Raw<E, C> {
     {
         self.remove_return(key, g)
     }
+
+    fn mt_delete_return_entry<'t, 'g, 'v, Q>(&'t self, key: &Q, g: &'g Guard) -> Option<&'v E>
+    where
+        Q: ?Sized + Comparable<E::Key>,
+        't: 'v,
+        'g: 't + 'v,
+    {
+        self._remove(DeleteRetEntry::new(key), g)
+    }
 }
diff --git a/server/src/engine/idx/mtchm/patch.rs b/server/src/engine/idx/mtchm/patch.rs
index 849ddc09..23c6d3ee 100644
--- a/server/src/engine/idx/mtchm/patch.rs
+++ b/server/src/engine/idx/mtchm/patch.rs
@@ -211,6 +211,40 @@ impl<'d, T: TreeElement, U: Comparable<T::Key> + ?Sized> PatchDelete<T> for Dele
     }
 }
 
+pub struct DeleteRetEntry<'a, T: TreeElement, U: ?Sized> {
+    target: &'a U,
+    _m: PhantomData<T>,
+}
+
+impl<'a, T: TreeElement, U: ?Sized> DeleteRetEntry<'a, T, U> {
+    pub fn new(target: &'a U) -> Self {
+        Self {
+            target,
+            _m: PhantomData,
+        }
+    }
+}
+
+impl<'dr, T: TreeElement, U: Comparable<T::Key> + ?Sized> PatchDelete<T>
+    for DeleteRetEntry<'dr, T, U>
+{
+    type Ret<'a> = Option<&'a T>;
+
+    type Target = U;
+
+    fn target(&self) -> &Self::Target {
+        self.target
+    }
+
+    fn ex<'a>(v: &'a T) -> Self::Ret<'a> {
+        Some(v)
+    }
+
+    fn nx<'a>() -> Self::Ret<'a> {
+        None
+    }
+}
+
 pub struct DeleteRet<'a, T: TreeElement, U: ?Sized> {
     target: &'a U,
     _m: PhantomData<T>,
diff --git a/server/src/engine/mem/ll.rs b/server/src/engine/mem/ll.rs
new file mode 100644
index 00000000..b8ed9866
--- /dev/null
+++ b/server/src/engine/mem/ll.rs
@@ -0,0 +1,103 @@
+/*
+ * Created on Fri Sep 01 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use core::ops::{Deref, DerefMut};
+
+#[derive(Debug, Clone, Copy, Default, Hash, PartialEq, Eq, PartialOrd, Ord)]
+#[cfg_attr(target_arch = "s390x", repr(align(256)))]
+#[cfg_attr(
+    any(
+        target_arch = "aarch64",
+        target_arch = "powerpc64",
+        target_arch = "x86_64",
+    ),
+    repr(align(128))
+)]
+#[cfg_attr(
+    any(
+        target_arch = "arm",
+        target_arch = "hexagon",
+        target_arch = "mips",
+        target_arch = "mips64",
+        target_arch = "riscv32",
+        target_arch = "riscv64",
+        target_arch = "sparc"
+    ),
+    repr(align(32))
+)]
+#[cfg_attr(
+    not(any(
+        target_arch = "aarch64",
+        target_arch = "arm",
+        target_arch = "hexagon",
+        target_arch = "m68k",
+        target_arch = "mips",
+        target_arch = "mips64",
+        target_arch = "powerpc64",
+        target_arch = "riscv32",
+        target_arch = "riscv64",
+        target_arch = "s390x",
+        target_arch = "sparc",
+        target_arch = "x86_64",
+    )),
+    repr(align(64))
+)]
+#[cfg_attr(target_arch = "m68k", repr(align(16)))]
+/**
+    cache line padding (to avoid unintended cache line invalidation)
+    - 256-bit (on a side note, good lord):
+        -> s390x: https://community.ibm.com/community/user/ibmz-and-linuxone/viewdocument/microprocessor-optimization-primer
+    - 128-bit:
+        -> aarch64: ARM64's big.LITTLE (it's a funny situation because there's a silly situation where one set of cores have one cache line
+        size while the other ones have a different size; see this excellent article: https://www.mono-project.com/news/2016/09/12/arm64-icache/)
+        -> powerpc64: https://reviews.llvm.org/D33656
+        -> x86_64: Intel's Sandy Bridge+ (https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf)
+    - 64-bit: default for all non-specific targets
+    - 32-bit: arm, hexagon, mips, mips64, riscv64, and sparc have 32-byte cache line size
+    - 16-bit: m68k (not very useful for us, but yeah)
+*/
+pub struct CachePadded<T> {
+    data: T,
+}
+
+impl<T> CachePadded<T> {
+    pub const fn new(data: T) -> Self {
+        Self { data }
+    }
+}
+
+impl<T> Deref for CachePadded<T> {
+    type Target = T;
+    fn deref(&self) -> &Self::Target {
+        &self.data
+    }
+}
+
+impl<T> DerefMut for CachePadded<T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.data
+    }
+}
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index 8c1a2b1f..68a429a3 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -25,6 +25,7 @@
 */
 
 mod astr;
+mod ll;
 mod stackop;
 mod uarray;
 mod vinline;
@@ -35,6 +36,7 @@ mod tests;
 // re-exports
 pub use {
     astr::AStr,
+    ll::CachePadded,
     stackop::ByteStack,
     uarray::UArray,
     vinline::VInline,
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index e3047e3b..7cd872a4 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -25,14 +25,17 @@
 */
 
 use {
-    super::{obj::FieldMD, PersistDictEntryDscr, PersistMapSpec, PersistObject, VecU8},
+    super::{
+        obj::{self, FieldMD},
+        PersistDictEntryDscr, PersistMapSpec, PersistObject, VecU8,
+    },
     crate::{
         engine::{
             core::model::Field,
             data::{
                 cell::Datacell,
                 dict::DictEntryGeneric,
-                tag::{CUTag, DataTag, TagClass, TagUnique},
+                tag::{CUTag, TagUnique},
                 DictGeneric,
             },
             idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
@@ -191,36 +194,9 @@ impl PersistMapSpec for GenericDictSpec {
                 <PersistMapImpl<Self> as PersistObject>::default_full_enc(buf, map);
             }
             DictEntryGeneric::Data(dc) => {
-                buf.push(
-                    PersistDictEntryDscr::translate_from_class(dc.tag().tag_class()).value_u8()
-                        * (!dc.is_null() as u8),
-                );
+                obj::encode_datacell_tag(buf, dc);
                 buf.extend(key.as_bytes());
-                fn encode_element(buf: &mut VecU8, dc: &Datacell) {
-                    unsafe {
-                        use TagClass::*;
-                        match dc.tag().tag_class() {
-                            Bool if dc.is_init() => buf.push(dc.read_bool() as u8),
-                            Bool => {}
-                            UnsignedInt | SignedInt | Float => {
-                                buf.extend(dc.read_uint().to_le_bytes())
-                            }
-                            Str | Bin => {
-                                let slc = dc.read_bin();
-                                buf.extend(slc.len().u64_bytes_le());
-                                buf.extend(slc);
-                            }
-                            List => {
-                                let lst = dc.read_list().read();
-                                buf.extend(lst.len().u64_bytes_le());
-                                for item in lst.iter() {
-                                    encode_element(buf, item);
-                                }
-                            }
-                        }
-                    }
-                }
-                encode_element(buf, dc);
+                obj::encode_element(buf, dc);
             }
         }
     }
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 8d992b1b..f1c11152 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -27,7 +27,7 @@
 use crate::engine::{core::model::delta::IRModel, data::DictGeneric};
 
 use {
-    super::{PersistObject, VecU8},
+    super::{PersistDictEntryDscr, PersistObject, VecU8},
     crate::{
         engine::{
             core::{
@@ -35,6 +35,7 @@ use {
                 space::{Space, SpaceMeta},
             },
             data::{
+                cell::Datacell,
                 tag::{DataTag, TagClass, TagSelector},
                 uuid::Uuid,
             },
@@ -45,12 +46,35 @@ use {
     },
 };
 
-/*
-    Full 8B tag block. Notes:
-    1. 7B at this moment is currently unused but there's a lot of additional flags that we might want to store here
-    2. If we end up deciding that this is indeed a waste of space, version this out and get rid of the 7B (or whatever we determine
-    to be the correct size.)
-*/
+pub fn encode_element(buf: &mut VecU8, dc: &Datacell) {
+    unsafe {
+        use TagClass::*;
+        match dc.tag().tag_class() {
+            Bool if dc.is_init() => buf.push(dc.read_bool() as u8),
+            Bool => {}
+            UnsignedInt | SignedInt | Float => buf.extend(dc.read_uint().to_le_bytes()),
+            Str | Bin => {
+                let slc = dc.read_bin();
+                buf.extend(slc.len().u64_bytes_le());
+                buf.extend(slc);
+            }
+            List => {
+                let lst = dc.read_list().read();
+                buf.extend(lst.len().u64_bytes_le());
+                for item in lst.iter() {
+                    encode_element(buf, item);
+                }
+            }
+        }
+    }
+}
+
+pub fn encode_datacell_tag(buf: &mut VecU8, dc: &Datacell) {
+    buf.push(
+        PersistDictEntryDscr::translate_from_class(dc.tag().tag_class()).value_u8()
+            * (!dc.is_null() as u8),
+    )
+}
 
 /*
     layer
diff --git a/server/src/engine/sync/mod.rs b/server/src/engine/sync/mod.rs
index fc18e702..47fbd521 100644
--- a/server/src/engine/sync/mod.rs
+++ b/server/src/engine/sync/mod.rs
@@ -26,6 +26,7 @@
 
 pub(super) mod atm;
 pub(super) mod cell;
+pub(super) mod queue;
 pub(super) mod smart;
 
 use std::{cell::Cell, hint::spin_loop, thread};
diff --git a/server/src/engine/sync/queue.rs b/server/src/engine/sync/queue.rs
new file mode 100644
index 00000000..640c3367
--- /dev/null
+++ b/server/src/engine/sync/queue.rs
@@ -0,0 +1,236 @@
+/*
+ * Created on Wed Aug 30 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::atm::Atomic,
+    crate::engine::mem::CachePadded,
+    crossbeam_epoch::{pin, unprotected, Guard, Owned, Shared},
+    std::{mem::MaybeUninit, sync::atomic::Ordering},
+};
+
+#[derive(Debug)]
+struct QNode<T> {
+    data: MaybeUninit<T>,
+    next: Atomic<Self>,
+}
+
+impl<T> QNode<T> {
+    fn new(data: MaybeUninit<T>, next: Atomic<Self>) -> Self {
+        Self { data, next }
+    }
+    fn null() -> Self {
+        Self::new(MaybeUninit::uninit(), Atomic::null())
+    }
+    fn new_data(val: T) -> Self {
+        Self::new(MaybeUninit::new(val), Atomic::null())
+    }
+}
+
+#[derive(Debug)]
+pub struct Queue<T> {
+    head: CachePadded<Atomic<QNode<T>>>,
+    tail: CachePadded<Atomic<QNode<T>>>,
+}
+
+impl<T> Queue<T> {
+    pub fn new() -> Self {
+        let slf = Self {
+            head: CachePadded::new(Atomic::null()),
+            tail: CachePadded::new(Atomic::null()),
+        };
+        let g = unsafe { unprotected() };
+        let sentinel = Owned::new(QNode::null()).into_shared(&g);
+        slf.head.store(sentinel, Ordering::Relaxed);
+        slf.tail.store(sentinel, Ordering::Relaxed);
+        slf
+    }
+    pub fn blocking_enqueue_autopin(&self, new: T) {
+        let g = pin();
+        self.blocking_enqueue(new, &g);
+    }
+    pub fn blocking_enqueue(&self, new: T, g: &Guard) {
+        let newptr = Owned::new(QNode::new_data(new)).into_shared(g);
+        loop {
+            // get current tail
+            let tailptr = self.tail.load(Ordering::Acquire, g);
+            let tail = unsafe { tailptr.deref() };
+            let tail_nextptr = tail.next.load(Ordering::Acquire, g);
+            if tail_nextptr.is_null() {
+                // tail points to null which means this should ideally by the last LL node
+                if tail
+                    .next
+                    .compare_exchange(
+                        Shared::null(),
+                        newptr,
+                        Ordering::Release,
+                        Ordering::Relaxed,
+                        g,
+                    )
+                    .is_ok()
+                {
+                    /*
+                        CAS'd in but tail is *probably* lagging behind. This CAS might fail but we don't care since we're allowed to have a lagging tail
+                    */
+                    let _ = self.tail.compare_exchange(
+                        tailptr,
+                        newptr,
+                        Ordering::Release,
+                        Ordering::Relaxed,
+                        g,
+                    );
+                    break;
+                }
+            } else {
+                // tail is lagging behind; attempt to help update it
+                let _ = self.tail.compare_exchange(
+                    tailptr,
+                    tail_nextptr,
+                    Ordering::Release,
+                    Ordering::Relaxed,
+                    g,
+                );
+            }
+        }
+    }
+    pub fn blocking_try_dequeue_autopin(&self) -> Option<T> {
+        let g = pin();
+        self.blocking_try_dequeue(&g)
+    }
+    pub fn blocking_try_dequeue(&self, g: &Guard) -> Option<T> {
+        loop {
+            // get current head
+            let headptr = self.head.load(Ordering::Acquire, g);
+            let head = unsafe { headptr.deref() };
+            let head_nextptr = head.next.load(Ordering::Acquire, g);
+            if head_nextptr.is_null() {
+                // this is the sentinel; queue is empty
+                return None;
+            }
+            // we observe at this point in time that there is atleast one element in the list
+            // let us swing that into sentinel position
+            if self
+                .head
+                .compare_exchange(
+                    headptr,
+                    head_nextptr,
+                    Ordering::Release,
+                    Ordering::Relaxed,
+                    g,
+                )
+                .is_ok()
+            {
+                // good so we were able to update the head
+                let tailptr = self.tail.load(Ordering::Acquire, g);
+                // but wait, was this the last node? in that case, we need to update the tail before we destroy it.
+                // this is fine though, as nothing will go boom right now since the tail is allowed to lag by one
+                if headptr == tailptr {
+                    // right so this was the last node uh oh
+                    let _ = self.tail.compare_exchange(
+                        tailptr,
+                        head_nextptr,
+                        Ordering::Release,
+                        Ordering::Relaxed,
+                        g,
+                    );
+                }
+                // now we're in a position to happily destroy this
+                unsafe { g.defer_destroy(headptr) }
+                // read out the ptr
+                return Some(unsafe { head_nextptr.deref().data.as_ptr().read() });
+            }
+        }
+    }
+}
+
+impl<T> Drop for Queue<T> {
+    fn drop(&mut self) {
+        let g = unsafe { unprotected() };
+        while self.blocking_try_dequeue(g).is_some() {}
+        // dealloc sentinel
+        unsafe {
+            self.head.load(Ordering::Relaxed, g).into_owned();
+        }
+    }
+}
+
+#[cfg(test)]
+type StringQueue = Queue<String>;
+
+#[test]
+fn empty() {
+    let q = StringQueue::new();
+    drop(q);
+}
+
+#[test]
+fn empty_deq() {
+    let g = pin();
+    let q = StringQueue::new();
+    assert_eq!(q.blocking_try_dequeue(&g), None);
+}
+
+#[test]
+fn empty_enq() {
+    let g = pin();
+    let q = StringQueue::new();
+    q.blocking_enqueue("hello".into(), &g);
+}
+
+#[test]
+fn multi_eq_dq() {
+    const ITEMS_L: usize = 100;
+    use std::{sync::Arc, thread};
+    let q = Arc::new(StringQueue::new());
+    let producer_q = q.clone();
+    let consumer_q = q.clone();
+    let producer = thread::spawn(move || {
+        let mut sent = vec![];
+        let g = pin();
+        for i in 0..ITEMS_L {
+            let item = format!("time-{i}");
+            // send a message and then sleep for two seconds
+            producer_q.blocking_enqueue(item.clone(), &g);
+            sent.push(item);
+        }
+        sent
+    });
+    let consumer = thread::spawn(move || {
+        let g = pin();
+        let mut received = vec![];
+        loop {
+            if received.len() == ITEMS_L {
+                break;
+            }
+            if let Some(item) = consumer_q.blocking_try_dequeue(&g) {
+                received.push(item);
+            }
+        }
+        received
+    });
+    let sent = producer.join().unwrap();
+    let received = consumer.join().unwrap();
+    assert_eq!(sent, received);
+}
diff --git a/server/src/engine/txn/data.rs b/server/src/engine/txn/data.rs
new file mode 100644
index 00000000..a9e57fa8
--- /dev/null
+++ b/server/src/engine/txn/data.rs
@@ -0,0 +1,81 @@
+/*
+ * Created on Mon Aug 28 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::{
+    engine::{
+        core::{index::PrimaryIndexKey, GlobalNS},
+        data::cell::Datacell,
+        storage::v1::inf::obj,
+    },
+    util::{os, EndianQW},
+};
+
+type Buf = Vec<u8>;
+
+static mut CAP_PER_LL: usize = 0;
+static mut FREEMEM: u64 = 0;
+
+/// Set the free memory and cap for deltas so that we don't bust through memory
+///
+/// ## Safety
+/// - All models must have been loaded
+/// - This must be called **before** the arbiter spawns threads for connections
+pub unsafe fn set_limits(gns: &GlobalNS) {
+    let model_cnt: usize = gns
+        .spaces()
+        .read()
+        .values()
+        .map(|space| space.models().read().len())
+        .sum();
+    let available_mem = os::free_memory_in_bytes();
+    FREEMEM = available_mem;
+    CAP_PER_LL = ((available_mem as usize / core::cmp::max(1, model_cnt)) as f64 * 0.01) as usize;
+}
+
+/*
+    misc. methods
+*/
+
+fn encode_primary_key(buf: &mut Buf, pk: &PrimaryIndexKey) {
+    buf.push(pk.tag().d());
+    static EXEC: [unsafe fn(&mut Buf, &PrimaryIndexKey); 2] = [
+        |buf, pk| unsafe { buf.extend(pk.read_uint().to_le_bytes()) },
+        |buf, pk| unsafe {
+            let bin = pk.read_bin();
+            buf.extend(bin.len().u64_bytes_le());
+            buf.extend(bin);
+        },
+    ];
+    unsafe {
+        // UNSAFE(@ohsayan): tag map
+        assert!((pk.tag().d() / 2) < 2);
+        EXEC[(pk.tag().d() / 2) as usize](buf, pk);
+    }
+}
+
+fn encode_dc(buf: &mut Buf, dc: &Datacell) {
+    obj::encode_element(buf, dc)
+}
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index 7ed0497e..7bd03090 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -67,7 +67,7 @@ impl<'a> ModelIDRef<'a> {
             super::SpaceIDRef::new(space_name, space),
             model_name,
             model.get_uuid(),
-            model.delta_state().current_version().value_u64(),
+            model.delta_state().schema_current_version().value_u64(),
         )
     }
     pub fn new(
diff --git a/server/src/engine/txn/gns/tests/io.rs b/server/src/engine/txn/gns/tests/io.rs
index ebe3ec91..acffafd8 100644
--- a/server/src/engine/txn/gns/tests/io.rs
+++ b/server/src/engine/txn/gns/tests/io.rs
@@ -148,7 +148,7 @@ mod model_tests {
                 super::SpaceIDRef::new("myspace", &space),
                 "mymodel",
                 model.get_uuid(),
-                model.delta_state().current_version().value_u64(),
+                model.delta_state().schema_current_version().value_u64(),
             ),
             &new_fields,
         );
@@ -160,7 +160,7 @@ mod model_tests {
                     super::SpaceIDRes::new(space.get_uuid(), "myspace".into()),
                     "mymodel".into(),
                     model.get_uuid(),
-                    model.delta_state().current_version().value_u64()
+                    model.delta_state().schema_current_version().value_u64()
                 ),
                 new_fields
             },
@@ -176,7 +176,7 @@ mod model_tests {
                 super::SpaceIDRef::new("myspace", &space),
                 "mymodel",
                 model.get_uuid(),
-                model.delta_state().current_version().value_u64(),
+                model.delta_state().schema_current_version().value_u64(),
             ),
             &removed_fields,
         );
@@ -188,7 +188,7 @@ mod model_tests {
                     super::SpaceIDRes::new(space.get_uuid(), "myspace".into()),
                     "mymodel".into(),
                     model.get_uuid(),
-                    model.delta_state().current_version().value_u64()
+                    model.delta_state().schema_current_version().value_u64()
                 ),
                 removed_fields: ["profile_pic".into()].into()
             },
@@ -207,7 +207,7 @@ mod model_tests {
                 super::SpaceIDRef::new("myspace", &space),
                 "mymodel".into(),
                 model.get_uuid(),
-                model.delta_state().current_version().value_u64(),
+                model.delta_state().schema_current_version().value_u64(),
             ),
             &updated_fields,
         );
@@ -219,7 +219,7 @@ mod model_tests {
                     super::SpaceIDRes::new(space.get_uuid(), "myspace".into()),
                     "mymodel".into(),
                     model.get_uuid(),
-                    model.delta_state().current_version().value_u64()
+                    model.delta_state().schema_current_version().value_u64()
                 ),
                 updated_fields
             },
@@ -233,7 +233,7 @@ mod model_tests {
             super::SpaceIDRef::new("myspace", &space),
             "mymodel",
             model.get_uuid(),
-            model.delta_state().current_version().value_u64(),
+            model.delta_state().schema_current_version().value_u64(),
         ));
         let encoded = super::enc::enc_full_self(txn);
         let decoded = super::dec::dec_full::<DropModelTxn>(&encoded).unwrap();
@@ -242,7 +242,7 @@ mod model_tests {
                 super::SpaceIDRes::new(space.get_uuid(), "myspace".into()),
                 "mymodel".into(),
                 model.get_uuid(),
-                model.delta_state().current_version().value_u64()
+                model.delta_state().schema_current_version().value_u64()
             ),
             decoded
         );
diff --git a/server/src/engine/txn/mod.rs b/server/src/engine/txn/mod.rs
index 3d9f9e31..ae4a6cb0 100644
--- a/server/src/engine/txn/mod.rs
+++ b/server/src/engine/txn/mod.rs
@@ -24,6 +24,7 @@
  *
 */
 
+pub mod data;
 pub mod gns;
 
 use super::storage::v1::SDSSError;
diff --git a/server/src/util/os.rs b/server/src/util/os.rs
index a043da62..bfab9801 100644
--- a/server/src/util/os.rs
+++ b/server/src/util/os.rs
@@ -28,7 +28,9 @@
 pub use unix::*;
 #[cfg(windows)]
 pub use windows::*;
+mod free_memory;
 
+pub use free_memory::free_memory_in_bytes;
 use {
     crate::IoResult,
     std::{
diff --git a/server/src/util/os/free_memory.rs b/server/src/util/os/free_memory.rs
new file mode 100644
index 00000000..2671b307
--- /dev/null
+++ b/server/src/util/os/free_memory.rs
@@ -0,0 +1,90 @@
+/*
+ * Created on Sat Sep 02 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+#[cfg(target_os = "windows")]
+extern crate winapi;
+
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+extern crate libc;
+
+pub fn free_memory_in_bytes() -> u64 {
+    #[cfg(target_os = "windows")]
+    {
+        use winapi::um::sysinfoapi::{GlobalMemoryStatusEx, MEMORYSTATUSEX};
+
+        let mut statex: MEMORYSTATUSEX = unsafe { std::mem::zeroed() };
+        statex.dwLength = std::mem::size_of::<MEMORYSTATUSEX>() as u32;
+
+        unsafe {
+            GlobalMemoryStatusEx(&mut statex);
+        }
+
+        // Return free physical memory
+        return statex.ullAvailPhys;
+    }
+
+    #[cfg(target_os = "linux")]
+    {
+        use libc::sysinfo;
+        let mut info: libc::sysinfo = unsafe { core::mem::zeroed() };
+
+        unsafe {
+            if sysinfo(&mut info) == 0 {
+                // Return free memory
+                return (info.freeram as u64) * (info.mem_unit as u64);
+            }
+        }
+
+        return 0;
+    }
+
+    #[cfg(target_os = "macos")]
+    {
+        use std::mem;
+        unsafe {
+            let page_size = libc::sysconf(libc::_SC_PAGESIZE);
+            let mut count: u32 = libc::HOST_VM_INFO64_COUNT as _;
+            let mut stat: libc::vm_statistics64 = mem::zeroed();
+            libc::host_statistics64(
+                libc::mach_host_self(),
+                libc::HOST_VM_INFO64,
+                &mut stat as *mut libc::vm_statistics64 as *mut _,
+                &mut count,
+            );
+
+            // see this: https://opensource.apple.com/source/xnu/xnu-4570.31.3/osfmk/mach/vm_statistics.h.auto.html
+            return (stat.free_count as u64)
+                .saturating_add(stat.inactive_count as _)
+                .saturating_add(stat.compressor_page_count as u64)
+                .saturating_mul(page_size as _);
+        }
+    }
+
+    #[cfg(not(any(target_os = "windows", target_os = "linux", target_os = "macos")))]
+    {
+        return 0;
+    }
+}

From ee9ccd5a30e3e67a0f80c0fea06403ae1cef497e Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 2 Sep 2023 17:57:13 +0000
Subject: [PATCH 241/310] Also store schema version in delta

---
 server/src/engine/core/dml/del.rs     |  2 ++
 server/src/engine/core/dml/ins.rs     |  8 +++++++-
 server/src/engine/core/dml/upd.rs     |  8 +++++++-
 server/src/engine/core/model/delta.rs | 18 ++++++++++++-----
 server/src/engine/core/space.rs       |  2 +-
 server/src/engine/txn/data.rs         | 28 ++++++++++++++++++++++-----
 6 files changed, 53 insertions(+), 13 deletions(-)

diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs
index 0b1f6dfb..4deab3a5 100644
--- a/server/src/engine/core/dml/del.rs
+++ b/server/src/engine/core/dml/del.rs
@@ -35,6 +35,7 @@ use crate::engine::{
 pub fn delete(gns: &GlobalNS, mut delete: DeleteStatement) -> DatabaseResult<()> {
     gns.with_model(delete.entity(), |model| {
         let g = sync::atm::cpin();
+        let schema_version = model.delta_state().schema_current_version();
         let delta_state = model.delta_state();
         // create new version
         let new_version = delta_state.create_new_data_delta_version();
@@ -47,6 +48,7 @@ pub fn delete(gns: &GlobalNS, mut delete: DeleteStatement) -> DatabaseResult<()>
                 delta_state.append_new_data_delta(
                     DataDeltaKind::Delete,
                     row.clone(),
+                    schema_version,
                     new_version,
                     &g,
                 );
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index ede87bf4..75d37bd1 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -47,7 +47,13 @@ pub fn insert(gns: &GlobalNS, insert: InsertStatement) -> DatabaseResult<()> {
         let row = Row::new(pk, data, ds.schema_current_version(), cv);
         if mdl.primary_index().__raw_index().mt_insert(row.clone(), &g) {
             // append delta for new version
-            ds.append_new_data_delta(DataDeltaKind::Insert, row, cv, &g);
+            ds.append_new_data_delta(
+                DataDeltaKind::Insert,
+                row,
+                ds.schema_current_version(),
+                cv,
+                &g,
+            );
             Ok(())
         } else {
             Err(DatabaseError::DmlConstraintViolationDuplicate)
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index a53b5ec9..3077d4b3 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -343,7 +343,13 @@ pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()>
             // update revised tag
             row_data_wl.set_txn_revised(cv);
             // publish delta
-            ds.append_new_data_delta(DataDeltaKind::Update, row.clone(), cv, &g);
+            ds.append_new_data_delta(
+                DataDeltaKind::Update,
+                row.clone(),
+                ds.schema_current_version(),
+                cv,
+                &g,
+            );
         }
         ret
     })
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index 970094cb..4265a4c9 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -173,11 +173,12 @@ impl DeltaState {
         &self,
         kind: DataDeltaKind,
         row: Row,
-        version: DeltaVersion,
+        schema_version: DeltaVersion,
+        data_version: DeltaVersion,
         g: &Guard,
     ) {
         self.data_deltas
-            .blocking_enqueue(DataDelta::new(version.0, row, kind), g);
+            .blocking_enqueue(DataDelta::new(schema_version, data_version, row, kind), g);
     }
     pub fn create_new_data_delta_version(&self) -> DeltaVersion {
         DeltaVersion(self.__data_delta_step())
@@ -315,15 +316,22 @@ impl<'a> SchemaDeltaIndexRGuard<'a> {
 
 #[derive(Debug)]
 pub struct DataDelta {
-    version: u64,
+    schema_version: DeltaVersion,
+    data_version: DeltaVersion,
     row: Row,
     change: DataDeltaKind,
 }
 
 impl DataDelta {
-    pub const fn new(version: u64, row: Row, change: DataDeltaKind) -> Self {
+    pub const fn new(
+        schema_version: DeltaVersion,
+        data_version: DeltaVersion,
+        row: Row,
+        change: DataDeltaKind,
+    ) -> Self {
         Self {
-            version,
+            schema_version,
+            data_version,
             row,
             change,
         }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index a4094167..f6da7c3e 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -51,7 +51,7 @@ pub struct SpaceMeta {
 }
 
 impl SpaceMeta {
-    pub const KEY_ENV: &str = "env";
+    pub const KEY_ENV: &'static str = "env";
     pub fn new_with_meta(props: DictGeneric) -> Self {
         Self {
             props: RwLock::new(props),
diff --git a/server/src/engine/txn/data.rs b/server/src/engine/txn/data.rs
index a9e57fa8..fcc9cd48 100644
--- a/server/src/engine/txn/data.rs
+++ b/server/src/engine/txn/data.rs
@@ -26,7 +26,7 @@
 
 use crate::{
     engine::{
-        core::{index::PrimaryIndexKey, GlobalNS},
+        core::{index::PrimaryIndexKey, model::delta::DataDelta, GlobalNS},
         data::cell::Datacell,
         storage::v1::inf::obj,
     },
@@ -35,8 +35,16 @@ use crate::{
 
 type Buf = Vec<u8>;
 
-static mut CAP_PER_LL: usize = 0;
-static mut FREEMEM: u64 = 0;
+/*
+    memory adjustments
+*/
+
+/// free memory in bytes
+static mut FREEMEM_BYTES: u64 = 0;
+/// capacity in bytes, per linked list
+static mut CAP_PER_LL_BYTES: u64 = 0;
+/// maximum number of nodes in linked list
+static mut MAX_NODES_IN_LL_CNT: usize = 0;
 
 /// Set the free memory and cap for deltas so that we don't bust through memory
 ///
@@ -51,8 +59,18 @@ pub unsafe fn set_limits(gns: &GlobalNS) {
         .map(|space| space.models().read().len())
         .sum();
     let available_mem = os::free_memory_in_bytes();
-    FREEMEM = available_mem;
-    CAP_PER_LL = ((available_mem as usize / core::cmp::max(1, model_cnt)) as f64 * 0.01) as usize;
+    FREEMEM_BYTES = available_mem;
+    CAP_PER_LL_BYTES =
+        ((available_mem / core::cmp::max(1, model_cnt) as u64) as f64 * 0.002) as u64;
+    MAX_NODES_IN_LL_CNT = CAP_PER_LL_BYTES as usize / (sizeof!(DataDelta) + sizeof!(u64));
+}
+
+/// Returns the maximum number of nodes that can be stored inside a delta queue for a model
+///
+/// Currently hardcoded to 0.2% of free memory after all datasets have been loaded
+pub unsafe fn get_max_delta_queue_size() -> usize {
+    // TODO(@ohsayan): dynamically approximate this limit
+    MAX_NODES_IN_LL_CNT
 }
 
 /*

From 20c937451f3b2993b75a3b14b596f0de5ee4438f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 3 Sep 2023 18:20:38 +0000
Subject: [PATCH 242/310] Implement batched persistence system

---
 server/src/engine/core/dml/del.rs             |   2 +-
 server/src/engine/core/dml/ins.rs             |   8 +-
 server/src/engine/core/dml/upd.rs             |   8 +-
 server/src/engine/core/index/key.rs           |  45 ++
 server/src/engine/core/index/row.rs           |  13 +-
 server/src/engine/core/mod.rs                 |   4 +-
 server/src/engine/core/model/delta.rs         |  44 +-
 server/src/engine/core/model/mod.rs           |   2 +-
 server/src/engine/data/tag.rs                 |  26 +
 server/src/engine/storage/checksum.rs         |  58 +++
 server/src/engine/storage/mod.rs              |   3 +
 .../src/engine/storage/v1/batch_jrnl/mod.rs   |  45 ++
 .../engine/storage/v1/batch_jrnl/persist.rs   | 290 ++++++++++++
 .../engine/storage/v1/batch_jrnl/restore.rs   | 445 ++++++++++++++++++
 .../src/engine/storage/v1/header_impl/mod.rs  |  10 +-
 server/src/engine/storage/v1/inf/map.rs       |  32 +-
 server/src/engine/storage/v1/inf/mod.rs       |  12 +-
 server/src/engine/storage/v1/inf/obj.rs       |   4 +-
 server/src/engine/storage/v1/journal.rs       |  52 +-
 server/src/engine/storage/v1/mod.rs           |  11 +
 server/src/engine/storage/v1/rw.rs            | 137 +++++-
 server/src/engine/storage/v1/test_util.rs     |   9 +-
 server/src/engine/storage/v1/tests.rs         | 237 +---------
 server/src/engine/storage/v1/tests/batch.rs   | 210 +++++++++
 server/src/engine/storage/v1/tests/rw.rs      |  68 +++
 server/src/engine/storage/v1/tests/tx.rs      | 210 +++++++++
 server/src/engine/txn/data.rs                 |  33 +-
 server/src/engine/txn/mod.rs                  |   1 +
 server/src/util/os.rs                         |   6 +
 29 files changed, 1684 insertions(+), 341 deletions(-)
 create mode 100644 server/src/engine/storage/checksum.rs
 create mode 100644 server/src/engine/storage/v1/batch_jrnl/mod.rs
 create mode 100644 server/src/engine/storage/v1/batch_jrnl/persist.rs
 create mode 100644 server/src/engine/storage/v1/batch_jrnl/restore.rs
 create mode 100644 server/src/engine/storage/v1/tests/batch.rs
 create mode 100644 server/src/engine/storage/v1/tests/rw.rs
 create mode 100644 server/src/engine/storage/v1/tests/tx.rs

diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs
index 4deab3a5..88d0ba5f 100644
--- a/server/src/engine/core/dml/del.rs
+++ b/server/src/engine/core/dml/del.rs
@@ -45,7 +45,7 @@ pub fn delete(gns: &GlobalNS, mut delete: DeleteStatement) -> DatabaseResult<()>
             .mt_delete_return_entry(&model.resolve_where(delete.clauses_mut())?, &g)
         {
             Some(row) => {
-                delta_state.append_new_data_delta(
+                delta_state.append_new_data_delta_with(
                     DataDeltaKind::Delete,
                     row.clone(),
                     schema_version,
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index 75d37bd1..f93fa556 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -43,15 +43,15 @@ pub fn insert(gns: &GlobalNS, insert: InsertStatement) -> DatabaseResult<()> {
         let g = cpin();
         let ds = mdl.delta_state();
         // create new version
-        let cv = ds.create_new_data_delta_version();
-        let row = Row::new(pk, data, ds.schema_current_version(), cv);
+        let new_version = ds.create_new_data_delta_version();
+        let row = Row::new(pk, data, ds.schema_current_version(), new_version);
         if mdl.primary_index().__raw_index().mt_insert(row.clone(), &g) {
             // append delta for new version
-            ds.append_new_data_delta(
+            ds.append_new_data_delta_with(
                 DataDeltaKind::Insert,
                 row,
                 ds.schema_current_version(),
-                cv,
+                new_version,
                 &g,
             );
             Ok(())
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index 3077d4b3..45b62532 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -248,7 +248,7 @@ pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()>
         let mut row_data_wl = row.d_data().write();
         // create new version
         let ds = mdl.delta_state();
-        let cv = ds.create_new_data_delta_version();
+        let new_version = ds.create_new_data_delta_version();
         // process changes
         let mut rollback_now = false;
         let mut rollback_data = Vec::with_capacity(update.expressions().len());
@@ -341,13 +341,13 @@ pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()>
                 });
         } else {
             // update revised tag
-            row_data_wl.set_txn_revised(cv);
+            row_data_wl.set_txn_revised(new_version);
             // publish delta
-            ds.append_new_data_delta(
+            ds.append_new_data_delta_with(
                 DataDeltaKind::Update,
                 row.clone(),
                 ds.schema_current_version(),
-                cv,
+                new_version,
                 &g,
             );
         }
diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
index 347a2c6a..470168c9 100644
--- a/server/src/engine/core/index/key.rs
+++ b/server/src/engine/core/index/key.rs
@@ -24,6 +24,7 @@
  *
 */
 
+use crate::engine::mem::ZERO_BLOCK;
 #[cfg(test)]
 use crate::{engine::data::spec::Dataspec1D, util::test_utils};
 use {
@@ -50,6 +51,29 @@ pub struct PrimaryIndexKey {
     data: SpecialPaddedWord,
 }
 
+impl Clone for PrimaryIndexKey {
+    fn clone(&self) -> Self {
+        match self.tag {
+            TagUnique::SignedInt | TagUnique::UnsignedInt => {
+                let (qw, nw) = self.data.dwordqn_load_qw_nw();
+                unsafe {
+                    let slice = slice::from_raw_parts(nw as *const u8, qw as _);
+                    let mut data = ManuallyDrop::new(slice.to_owned().into_boxed_slice());
+                    Self {
+                        tag: self.tag,
+                        data: SpecialPaddedWord::new(qw, data.as_mut_ptr() as usize),
+                    }
+                }
+            }
+            TagUnique::Bin | TagUnique::Str => Self {
+                tag: self.tag,
+                data: unsafe { core::mem::transmute_copy(&self.data) },
+            },
+            _ => unreachable!(),
+        }
+    }
+}
+
 impl PrimaryIndexKey {
     pub fn tag(&self) -> TagUnique {
         self.tag
@@ -127,6 +151,27 @@ impl PrimaryIndexKey {
             },
         }
     }
+    /// Create a new quadword based primary key
+    pub unsafe fn new_from_qw(tag: TagUnique, qw: u64) -> Self {
+        debug_assert!(tag == TagUnique::SignedInt || tag == TagUnique::UnsignedInt);
+        Self {
+            tag,
+            data: unsafe {
+                // UNSAFE(@ohsayan): manually choosing block
+                SpecialPaddedWord::new(qw, ZERO_BLOCK.as_ptr() as usize)
+            },
+        }
+    }
+    pub unsafe fn new_from_dual(tag: TagUnique, qw: u64, ptr: usize) -> Self {
+        debug_assert!(tag == TagUnique::Str || tag == TagUnique::Bin);
+        Self {
+            tag,
+            data: unsafe {
+                // UNSAFE(@ohsayan): manually choosing qw and nw
+                SpecialPaddedWord::new(qw, ptr)
+            },
+        }
+    }
     pub unsafe fn raw_clone(&self) -> Self {
         Self {
             tag: self.tag,
diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
index 5ad2d636..f47861a8 100644
--- a/server/src/engine/core/index/row.rs
+++ b/server/src/engine/core/index/row.rs
@@ -139,11 +139,16 @@ impl Row {
 }
 
 impl Row {
-    pub fn resolve_schema_deltas_and_freeze<'g>(
+    /// Only apply deltas if a certain condition is met
+    pub fn resolve_schema_deltas_and_freeze_if<'g>(
         &'g self,
         delta_state: &DeltaState,
+        iff: impl Fn(&RowData) -> bool,
     ) -> RwLockReadGuard<'g, RowData> {
         let rwl_ug = self.d_data().upgradable_read();
+        if !iff(&rwl_ug) {
+            return RwLockUpgradableReadGuard::downgrade(rwl_ug);
+        }
         let current_version = delta_state.schema_current_version();
         if compiler::likely(current_version <= rwl_ug.txn_revised_schema_version) {
             return RwLockUpgradableReadGuard::downgrade(rwl_ug);
@@ -167,6 +172,12 @@ impl Row {
         wl.txn_revised_schema_version = max_delta;
         return RwLockWriteGuard::downgrade(wl);
     }
+    pub fn resolve_schema_deltas_and_freeze<'g>(
+        &'g self,
+        delta_state: &DeltaState,
+    ) -> RwLockReadGuard<'g, RowData> {
+        self.resolve_schema_deltas_and_freeze_if(delta_state, |_| true)
+    }
 }
 
 impl Clone for Row {
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 34a0427f..340df67b 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -24,7 +24,7 @@
  *
 */
 
-mod dml;
+pub(in crate::engine) mod dml;
 pub(in crate::engine) mod index;
 pub(in crate::engine) mod model;
 pub(in crate::engine) mod query_meta;
@@ -32,7 +32,7 @@ pub mod space;
 mod util;
 // test
 #[cfg(test)]
-mod tests;
+pub(super) mod tests;
 // imports
 use {
     self::{model::Model, util::EntityLocator},
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index 4265a4c9..d3c6720b 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -30,7 +30,7 @@ use {
     parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard},
     std::{
         collections::btree_map::{BTreeMap, Range},
-        sync::atomic::{AtomicU64, Ordering},
+        sync::atomic::{AtomicU64, AtomicUsize, Ordering},
     },
 };
 
@@ -153,6 +153,7 @@ pub struct DeltaState {
     // data
     data_current_version: AtomicU64,
     data_deltas: Queue<DataDelta>,
+    data_deltas_size: AtomicUsize,
 }
 
 impl DeltaState {
@@ -163,13 +164,14 @@ impl DeltaState {
             schema_deltas: RwLock::new(BTreeMap::new()),
             data_current_version: AtomicU64::new(0),
             data_deltas: Queue::new(),
+            data_deltas_size: AtomicUsize::new(0),
         }
     }
 }
 
 // data
 impl DeltaState {
-    pub fn append_new_data_delta(
+    pub fn append_new_data_delta_with(
         &self,
         kind: DataDeltaKind,
         row: Row,
@@ -177,18 +179,27 @@ impl DeltaState {
         data_version: DeltaVersion,
         g: &Guard,
     ) {
-        self.data_deltas
-            .blocking_enqueue(DataDelta::new(schema_version, data_version, row, kind), g);
+        self.append_new_data_delta(DataDelta::new(schema_version, data_version, row, kind), g);
+    }
+    pub fn append_new_data_delta(&self, delta: DataDelta, g: &Guard) {
+        self.data_deltas.blocking_enqueue(delta, g);
+        self.data_deltas_size.fetch_add(1, Ordering::Release);
     }
     pub fn create_new_data_delta_version(&self) -> DeltaVersion {
         DeltaVersion(self.__data_delta_step())
     }
+    pub fn get_data_delta_size(&self) -> usize {
+        self.data_deltas_size.load(Ordering::Acquire)
+    }
 }
 
 impl DeltaState {
-    pub fn __data_delta_step(&self) -> u64 {
+    fn __data_delta_step(&self) -> u64 {
         self.data_current_version.fetch_add(1, Ordering::AcqRel)
     }
+    pub fn __data_delta_queue(&self) -> &Queue<DataDelta> {
+        &self.data_deltas
+    }
 }
 
 // schema
@@ -314,7 +325,7 @@ impl<'a> SchemaDeltaIndexRGuard<'a> {
     data delta
 */
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct DataDelta {
     schema_version: DeltaVersion,
     data_version: DeltaVersion,
@@ -336,11 +347,24 @@ impl DataDelta {
             change,
         }
     }
+    pub fn schema_version(&self) -> DeltaVersion {
+        self.schema_version
+    }
+    pub fn data_version(&self) -> DeltaVersion {
+        self.data_version
+    }
+    pub fn row(&self) -> &Row {
+        &self.row
+    }
+    pub fn change(&self) -> DataDeltaKind {
+        self.change
+    }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone, Copy, sky_macros::EnumMethods, PartialEq)]
+#[repr(u8)]
 pub enum DataDeltaKind {
-    Insert,
-    Update,
-    Delete,
+    Delete = 0,
+    Insert = 1,
+    Update = 2,
 }
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 82e4ddf3..13f7137e 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -52,7 +52,7 @@ use {
     std::cell::UnsafeCell,
 };
 
-pub(in crate::engine::core) use self::delta::{SchemaDeltaKind, DeltaState, DeltaVersion};
+pub(in crate::engine::core) use self::delta::{DeltaState, DeltaVersion, SchemaDeltaKind};
 pub(in crate::engine) type Fields = IndexSTSeqCns<Box<str>, Field>;
 
 #[derive(Debug)]
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
index 53c4067e..00bee7c8 100644
--- a/server/src/engine/data/tag.rs
+++ b/server/src/engine/data/tag.rs
@@ -41,6 +41,26 @@ impl TagClass {
     pub const fn max() -> usize {
         Self::List.d() as _
     }
+    pub const fn try_from_raw(v: u8) -> Option<Self> {
+        if v > Self::List.d() {
+            return None;
+        }
+        Some(unsafe { Self::from_raw(v) })
+    }
+    pub const unsafe fn from_raw(v: u8) -> Self {
+        core::mem::transmute(v)
+    }
+    pub const fn tag_unique(&self) -> TagUnique {
+        [
+            TagUnique::Illegal,
+            TagUnique::UnsignedInt,
+            TagUnique::SignedInt,
+            TagUnique::Illegal,
+            TagUnique::Bin,
+            TagUnique::Str,
+            TagUnique::Illegal,
+        ][self.d() as usize]
+    }
 }
 
 #[repr(u8)]
@@ -124,6 +144,12 @@ impl TagUnique {
     pub const fn is_unique(&self) -> bool {
         self.d() != Self::Illegal.d()
     }
+    pub const fn try_from_raw(raw: u8) -> Option<Self> {
+        if raw > 3 {
+            return None;
+        }
+        Some(unsafe { core::mem::transmute(raw) })
+    }
 }
 
 macro_rules! d {
diff --git a/server/src/engine/storage/checksum.rs b/server/src/engine/storage/checksum.rs
new file mode 100644
index 00000000..aa32c3d4
--- /dev/null
+++ b/server/src/engine/storage/checksum.rs
@@ -0,0 +1,58 @@
+/*
+ * Created on Sun Sep 03 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crc::{Crc, Digest, CRC_64_XZ};
+
+/*
+    NOTE(@ohsayan): we're currently using crc's impl. but the reason I decided to make a wrapper is because I have a
+    different impl in mind
+*/
+
+const CRC64: Crc<u64> = Crc::<u64>::new(&CRC_64_XZ);
+
+pub struct SCrc {
+    digest: Digest<'static, u64>,
+}
+
+impl SCrc {
+    pub const fn new() -> Self {
+        Self {
+            digest: CRC64.digest(),
+        }
+    }
+    pub fn recompute_with_new_byte(&mut self, b: u8) {
+        self.digest.update(&[b])
+    }
+    pub fn recompute_with_new_block<const N: usize>(&mut self, b: [u8; N]) {
+        self.digest.update(&b);
+    }
+    pub fn recompute_with_new_var_block(&mut self, b: &[u8]) {
+        self.digest.update(b)
+    }
+    pub fn finish(self) -> u64 {
+        self.digest.finalize()
+    }
+}
diff --git a/server/src/engine/storage/mod.rs b/server/src/engine/storage/mod.rs
index c656c894..6f0d2a15 100644
--- a/server/src/engine/storage/mod.rs
+++ b/server/src/engine/storage/mod.rs
@@ -26,7 +26,10 @@
 
 //! Implementations of the Skytable Disk Storage Subsystem (SDSS)
 
+mod checksum;
 mod header;
 mod versions;
 // impls
 pub mod v1;
+
+pub use checksum::SCrc;
diff --git a/server/src/engine/storage/v1/batch_jrnl/mod.rs b/server/src/engine/storage/v1/batch_jrnl/mod.rs
new file mode 100644
index 00000000..68b32d4b
--- /dev/null
+++ b/server/src/engine/storage/v1/batch_jrnl/mod.rs
@@ -0,0 +1,45 @@
+/*
+ * Created on Sun Sep 03 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod persist;
+mod restore;
+
+/// the data batch file was reopened
+const MARKER_BATCH_REOPEN: u8 = 0xFB;
+/// the data batch file was closed
+const MARKER_BATCH_CLOSED: u8 = 0xFC;
+/// end of batch marker
+const MARKER_END_OF_BATCH: u8 = 0xFD;
+/// "real" batch event marker
+const MARKER_ACTUAL_BATCH_EVENT: u8 = 0xFE;
+/// recovery batch event marker
+const MARKER_RECOVERY_EVENT: u8 = 0xFF;
+/// recovery threshold
+const RECOVERY_THRESHOLD: usize = 10;
+
+#[cfg(test)]
+pub(super) use restore::{DecodedBatchEvent, DecodedBatchEventKind, NormalBatch};
+pub use {persist::DataBatchPersistDriver, restore::DataBatchRestoreDriver};
diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs
new file mode 100644
index 00000000..1dcf824e
--- /dev/null
+++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs
@@ -0,0 +1,290 @@
+/*
+ * Created on Tue Sep 05 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{
+        MARKER_ACTUAL_BATCH_EVENT, MARKER_BATCH_CLOSED, MARKER_BATCH_REOPEN, MARKER_END_OF_BATCH,
+        MARKER_RECOVERY_EVENT,
+    },
+    crate::{
+        engine::{
+            core::{
+                index::{PrimaryIndexKey, RowData},
+                model::{
+                    delta::{DataDelta, DataDeltaKind, DeltaVersion, IRModel},
+                    Model,
+                },
+            },
+            data::{
+                cell::Datacell,
+                tag::{DataTag, TagClass, TagUnique},
+            },
+            idx::STIndexSeq,
+            storage::v1::{
+                inf::PersistTypeDscr,
+                rw::{RawFileIOInterface, SDSSFileIO, SDSSFileTrackedWriter},
+                SDSSError, SDSSResult,
+            },
+        },
+        util::EndianQW,
+    },
+    crossbeam_epoch::pin,
+};
+
+pub struct DataBatchPersistDriver<F> {
+    f: SDSSFileTrackedWriter<F>,
+}
+
+impl<F: RawFileIOInterface> DataBatchPersistDriver<F> {
+    pub fn new(mut file: SDSSFileIO<F>, is_new: bool) -> SDSSResult<Self> {
+        if !is_new {
+            file.fsynced_write(&[MARKER_BATCH_REOPEN])?;
+        }
+        Ok(Self {
+            f: SDSSFileTrackedWriter::new(file),
+        })
+    }
+    pub fn close(mut self) -> SDSSResult<()> {
+        if self
+            .f
+            .inner_file()
+            .fsynced_write(&[MARKER_BATCH_CLOSED])
+            .is_ok()
+        {
+            return Ok(());
+        } else {
+            return Err(SDSSError::DataBatchCloseError);
+        }
+    }
+    pub fn write_new_batch(&mut self, model: &Model, observed_len: usize) -> SDSSResult<()> {
+        // pin model
+        let irm = model.intent_read_model();
+        let schema_version = model.delta_state().schema_current_version();
+        let data_q = model.delta_state().__data_delta_queue();
+        let g = pin();
+        // init restore list
+        let mut restore_list = Vec::new();
+        // prepare computations
+        let mut i = 0;
+        let mut inconsistent_reads = 0;
+        let mut exec = || -> SDSSResult<()> {
+            // write batch start
+            self.write_batch_start(observed_len, schema_version)?;
+            while i < observed_len {
+                let delta = data_q.blocking_try_dequeue(&g).unwrap();
+                restore_list.push(delta.clone()); // TODO(@ohsayan): avoid this
+                match delta.change() {
+                    DataDeltaKind::Delete => {
+                        self.write_batch_item_common_row_data(&delta)?;
+                        self.encode_pk_only(delta.row().d_key())?;
+                    }
+                    DataDeltaKind::Insert | DataDeltaKind::Update => {
+                        // resolve deltas (this is yet another opportunity for us to reclaim memory from deleted items)
+                        let row_data = delta
+                            .row()
+                            .resolve_schema_deltas_and_freeze_if(&model.delta_state(), |row| {
+                                row.get_txn_revised() <= delta.data_version()
+                            });
+                        if row_data.get_txn_revised() > delta.data_version() {
+                            // we made an inconsistent (stale) read; someone updated the state after our snapshot
+                            inconsistent_reads += 1;
+                            i += 1;
+                            continue;
+                        }
+                        self.write_batch_item_common_row_data(&delta)?;
+                        // encode data
+                        self.encode_pk_only(delta.row().d_key())?;
+                        self.encode_row_data(model, &irm, &row_data)?;
+                    }
+                }
+                // fsync now; we're good to go
+                self.f.fsync_all()?;
+                i += 1;
+            }
+            return self.append_batch_summary(observed_len, inconsistent_reads);
+        };
+        match exec() {
+            Ok(()) => Ok(()),
+            Err(_) => {
+                // republish changes since we failed to commit
+                restore_list
+                    .into_iter()
+                    .for_each(|delta| model.delta_state().append_new_data_delta(delta, &g));
+                // now attempt to fix the file
+                return self.attempt_fix_data_batchfile();
+            }
+        }
+    }
+    /// Write the batch start block:
+    /// - Batch start magic
+    /// - Expected commit
+    /// - Schema version
+    fn write_batch_start(
+        &mut self,
+        observed_len: usize,
+        schema_version: DeltaVersion,
+    ) -> Result<(), SDSSError> {
+        self.f.unfsynced_write(&[MARKER_ACTUAL_BATCH_EVENT])?;
+        let observed_len_bytes = observed_len.u64_bytes_le();
+        self.f.unfsynced_write(&observed_len_bytes)?;
+        self.f
+            .unfsynced_write(&schema_version.value_u64().to_le_bytes())?;
+        Ok(())
+    }
+    /// Append a summary of this batch
+    fn append_batch_summary(
+        &mut self,
+        observed_len: usize,
+        inconsistent_reads: usize,
+    ) -> Result<(), SDSSError> {
+        // [0xFD][actual_commit][checksum]
+        self.f.unfsynced_write(&[MARKER_END_OF_BATCH])?;
+        let actual_commit = (observed_len - inconsistent_reads).u64_bytes_le();
+        self.f.unfsynced_write(&actual_commit)?;
+        let cs = self.f.reset_and_finish_checksum().to_le_bytes();
+        self.f.inner_file().fsynced_write(&cs)?;
+        Ok(())
+    }
+    /// Attempt to fix the batch journal
+    // TODO(@ohsayan): declare an "international system disaster" when this happens
+    fn attempt_fix_data_batchfile(&mut self) -> SDSSResult<()> {
+        /*
+            attempt to append 0xFF to the part of the file where a corruption likely occurred, marking
+            it recoverable
+        */
+        let f = self.f.inner_file();
+        if f.fsynced_write(&[MARKER_RECOVERY_EVENT]).is_ok() {
+            return Ok(());
+        }
+        Err(SDSSError::DataBatchRecoveryFailStageOne)
+    }
+}
+
+impl<F: RawFileIOInterface> DataBatchPersistDriver<F> {
+    /// encode the primary key only. this means NO TAG is encoded.
+    fn encode_pk_only(&mut self, pk: &PrimaryIndexKey) -> SDSSResult<()> {
+        let buf = &mut self.f;
+        match pk.tag() {
+            TagUnique::UnsignedInt | TagUnique::SignedInt => {
+                let data = unsafe {
+                    // UNSAFE(@ohsayan): +tagck
+                    pk.read_uint()
+                }
+                .to_le_bytes();
+                buf.unfsynced_write(&data)?;
+            }
+            TagUnique::Str | TagUnique::Bin => {
+                let slice = unsafe {
+                    // UNSAFE(@ohsayan): +tagck
+                    pk.read_bin()
+                };
+                let slice_l = slice.len().u64_bytes_le();
+                buf.unfsynced_write(&slice_l)?;
+                buf.unfsynced_write(slice)?;
+            }
+            TagUnique::Illegal => unsafe {
+                // UNSAFE(@ohsayan): a pk can't be constructed with illegal
+                impossible!()
+            },
+        }
+        Ok(())
+    }
+    /// Encode a single cell
+    fn encode_cell(&mut self, value: &Datacell) -> SDSSResult<()> {
+        let ref mut buf = self.f;
+        buf.unfsynced_write(&[
+            PersistTypeDscr::translate_from_class(value.tag().tag_class()).value_u8(),
+        ])?;
+        match value.tag().tag_class() {
+            TagClass::Bool if value.is_null() => {}
+            TagClass::Bool => {
+                let bool = unsafe {
+                    // UNSAFE(@ohsayan): +tagck
+                    value.read_bool()
+                } as u8;
+                buf.unfsynced_write(&[bool])?;
+            }
+            TagClass::SignedInt | TagClass::UnsignedInt | TagClass::Float => {
+                let chunk = unsafe {
+                    // UNSAFE(@ohsayan): +tagck
+                    value.read_uint()
+                }
+                .to_le_bytes();
+                buf.unfsynced_write(&chunk)?;
+            }
+            TagClass::Str | TagClass::Bin => {
+                let slice = unsafe {
+                    // UNSAFE(@ohsayan): +tagck
+                    value.read_bin()
+                };
+                let slice_l = slice.len().u64_bytes_le();
+                buf.unfsynced_write(&slice_l)?;
+                buf.unfsynced_write(slice)?;
+            }
+            TagClass::List => {
+                let list = unsafe {
+                    // UNSAFE(@ohsayan): +tagck
+                    value.read_list()
+                }
+                .read();
+                let list_l = list.len().u64_bytes_le();
+                buf.unfsynced_write(&list_l)?;
+                for item in list.iter() {
+                    self.encode_cell(item)?;
+                }
+            }
+        }
+        Ok(())
+    }
+    /// Encode row data
+    fn encode_row_data(
+        &mut self,
+        mdl: &Model,
+        irm: &IRModel,
+        row_data: &RowData,
+    ) -> SDSSResult<()> {
+        // nasty hack; we need to avoid the pk
+        self.f
+            .unfsynced_write(&(row_data.fields().len()).to_le_bytes())?;
+        for field_name in irm.fields().stseq_ord_key() {
+            match row_data.fields().get(field_name) {
+                Some(cell) => {
+                    self.encode_cell(cell)?;
+                }
+                None if field_name.as_ref() == mdl.p_key() => {}
+                None => self.f.unfsynced_write(&[0])?,
+            }
+        }
+        Ok(())
+    }
+    fn write_batch_item_common_row_data(&mut self, delta: &DataDelta) -> Result<(), SDSSError> {
+        let p1_dc_pk_ty = [delta.change().value_u8(), delta.row().d_key().tag().d()];
+        self.f.unfsynced_write(&p1_dc_pk_ty)?;
+        let txn_id = delta.data_version().value_u64().to_le_bytes();
+        self.f.unfsynced_write(&txn_id)?;
+        Ok(())
+    }
+}
diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs
new file mode 100644
index 00000000..7e54fe6d
--- /dev/null
+++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs
@@ -0,0 +1,445 @@
+/*
+ * Created on Tue Sep 05 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use super::{MARKER_BATCH_CLOSED, MARKER_BATCH_REOPEN};
+
+use {
+    super::{
+        MARKER_ACTUAL_BATCH_EVENT, MARKER_END_OF_BATCH, MARKER_RECOVERY_EVENT, RECOVERY_THRESHOLD,
+    },
+    crate::{
+        engine::{
+            core::{index::PrimaryIndexKey, model::Model},
+            data::{
+                cell::Datacell,
+                tag::{CUTag, TagClass, TagUnique},
+            },
+            storage::v1::{
+                inf::PersistTypeDscr,
+                rw::{RawFileIOInterface, SDSSFileIO, SDSSFileTrackedReader},
+                SDSSError, SDSSResult,
+            },
+        },
+        util::copy_slice_to_array,
+    },
+    std::mem::ManuallyDrop,
+};
+
+#[derive(Debug, PartialEq)]
+pub(in crate::engine::storage::v1) struct DecodedBatchEvent {
+    txn_id: u64,
+    pk: PrimaryIndexKey,
+    kind: DecodedBatchEventKind,
+}
+
+impl DecodedBatchEvent {
+    pub(in crate::engine::storage::v1) const fn new(
+        txn_id: u64,
+        pk: PrimaryIndexKey,
+        kind: DecodedBatchEventKind,
+    ) -> Self {
+        Self { txn_id, pk, kind }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub(in crate::engine::storage::v1) enum DecodedBatchEventKind {
+    Delete,
+    Insert(Vec<Datacell>),
+    Update(Vec<Datacell>),
+}
+
+#[derive(Debug, PartialEq)]
+pub(in crate::engine::storage::v1) struct NormalBatch {
+    events: Vec<DecodedBatchEvent>,
+    schema_version: u64,
+}
+
+impl NormalBatch {
+    pub(in crate::engine::storage::v1) fn new(
+        events: Vec<DecodedBatchEvent>,
+        schema_version: u64,
+    ) -> Self {
+        Self {
+            events,
+            schema_version,
+        }
+    }
+}
+
+enum Batch {
+    RecoveredFromerror,
+    Normal(NormalBatch),
+    FinishedEarly(NormalBatch),
+    BatchClosed,
+}
+
+pub struct DataBatchRestoreDriver<F> {
+    f: SDSSFileTrackedReader<F>,
+}
+
+impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
+    pub fn new(f: SDSSFileIO<F>) -> SDSSResult<Self> {
+        Ok(Self {
+            f: SDSSFileTrackedReader::new(f)?,
+        })
+    }
+    pub fn into_file(self) -> SDSSFileIO<F> {
+        self.f.into_inner_file()
+    }
+    pub(in crate::engine::storage::v1) fn read_data_batch_into_model(
+        &mut self,
+        model: &Model,
+    ) -> SDSSResult<()> {
+        self.read_all_batches_and_for_each(|batch| {
+            // apply the batch
+            Self::apply_batch(model, batch)
+        })
+    }
+    pub fn read_all_batches(&mut self) -> SDSSResult<Vec<NormalBatch>> {
+        let mut all_batches = vec![];
+        self.read_all_batches_and_for_each(|batch| {
+            all_batches.push(batch);
+            Ok(())
+        })?;
+        Ok(all_batches)
+    }
+}
+
+impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
+    fn read_all_batches_and_for_each(
+        &mut self,
+        mut f: impl FnMut(NormalBatch) -> SDSSResult<()>,
+    ) -> SDSSResult<()> {
+        // begin
+        let mut closed = false;
+        while !self.f.is_eof() && !closed {
+            // try to decode this batch
+            let Ok(batch) = self.read_batch() else {
+                self.attempt_recover_data_batch()?;
+                continue;
+            };
+            // see what happened when decoding it
+            let finished_early = matches!(batch, Batch::FinishedEarly { .. });
+            let batch = match batch {
+                Batch::RecoveredFromerror => {
+                    // there was an error, but it was safely "handled" because of a recovery byte mark
+                    continue;
+                }
+                Batch::FinishedEarly(batch) | Batch::Normal(batch) => batch,
+                Batch::BatchClosed => {
+                    // the batch was closed; this means that we probably are done with this round; but was it re-opened?
+                    closed = self.handle_reopen_is_actual_close()?;
+                    continue;
+                }
+            };
+            // now we need to read the batch summary
+            let Ok(actual_commit) = self.read_batch_summary(finished_early) else {
+                self.attempt_recover_data_batch()?;
+                continue;
+            };
+            // check if we have the expected batch size
+            if batch.events.len() as u64 != actual_commit {
+                // corrupted
+                self.attempt_recover_data_batch()?;
+                continue;
+            }
+            f(batch)?;
+            // apply the batch
+        }
+        if closed {
+            if self.f.is_eof() {
+                // that was the last batch
+                return Ok(());
+            }
+        }
+        // nope, this is a corrupted file
+        Err(SDSSError::DataBatchRestoreCorruptedBatchFile)
+    }
+    fn handle_reopen_is_actual_close(&mut self) -> SDSSResult<bool> {
+        if self.f.is_eof() {
+            // yup, it was closed
+            Ok(true)
+        } else {
+            // maybe not
+            if self.f.read_byte()? == MARKER_BATCH_REOPEN {
+                // driver was closed, but reopened
+                Ok(false)
+            } else {
+                // that's just a nice bug
+                Err(SDSSError::DataBatchRestoreCorruptedBatchFile)
+            }
+        }
+    }
+}
+
+impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
+    fn apply_batch(_: &Model, _: NormalBatch) -> SDSSResult<()> {
+        todo!()
+    }
+}
+
+impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
+    fn read_batch_summary(&mut self, finished_early: bool) -> SDSSResult<u64> {
+        if !finished_early {
+            // we must read the batch termination signature
+            let b = self.f.read_byte()?;
+            if b != MARKER_END_OF_BATCH {
+                return Err(SDSSError::DataBatchRestoreCorruptedBatch);
+            }
+        }
+        // read actual commit
+        let mut actual_commit = [0; sizeof!(u64)];
+        self.f.read_into_buffer(&mut actual_commit)?;
+        // find actual checksum
+        let actual_checksum = self.f.__reset_checksum();
+        // find hardcoded checksum
+        let mut hardcoded_checksum = [0; sizeof!(u64)];
+        self.f
+            .inner_file()
+            .read_to_buffer(&mut hardcoded_checksum)?;
+        // move file cursor ahead
+        self.f.__cursor_ahead_by(sizeof!(u64));
+        if actual_checksum == u64::from_le_bytes(hardcoded_checksum) {
+            Ok(u64::from_le_bytes(actual_commit))
+        } else {
+            Err(SDSSError::DataBatchRestoreCorruptedBatch)
+        }
+    }
+    fn read_batch(&mut self) -> SDSSResult<Batch> {
+        let mut this_batch = vec![];
+        // check batch type
+        let batch_type = self.f.read_byte()?;
+        match batch_type {
+            MARKER_ACTUAL_BATCH_EVENT => {}
+            MARKER_RECOVERY_EVENT => {
+                // while attempting to write this batch, some sort of an error occurred but we got a nice recovery byte
+                // so proceed that way
+                return Ok(Batch::RecoveredFromerror);
+            }
+            MARKER_BATCH_CLOSED => {
+                // this isn't a batch; it has been closed
+                return Ok(Batch::BatchClosed);
+            }
+            _ => {
+                // this is the only singular byte that is expected to be intact. If this isn't intact either, I'm sorry
+                return Err(SDSSError::DataBatchRestoreCorruptedBatch);
+            }
+        }
+        // we're expecting a "good batch"
+        let mut batch_size_schema_version = [0; sizeof!(u64, 2)];
+        self.f.read_into_buffer(&mut batch_size_schema_version)?;
+        // we have the batch length
+        let batch_size = u64::from_le_bytes(copy_slice_to_array(&batch_size_schema_version[..8]));
+        let schema_version =
+            u64::from_le_bytes(copy_slice_to_array(&batch_size_schema_version[8..]));
+        let mut processed_in_this_batch = 0;
+        while (processed_in_this_batch != batch_size) & !self.f.is_eof() {
+            // decode common row data
+            let change_type = self.f.read_byte()?;
+            // now decode event
+            match change_type {
+                MARKER_END_OF_BATCH => {
+                    // the file tells us that we've reached the end of this batch; hmmm
+                    return Ok(Batch::FinishedEarly(NormalBatch::new(
+                        this_batch,
+                        schema_version,
+                    )));
+                }
+                normal_event => {
+                    let (pk_type, txnid) = self.read_normal_event_metadata()?;
+                    match normal_event {
+                        0 => {
+                            // delete
+                            let pk = self.decode_primary_key(pk_type)?;
+                            this_batch.push(DecodedBatchEvent::new(
+                                txnid,
+                                pk,
+                                DecodedBatchEventKind::Delete,
+                            ));
+                            processed_in_this_batch += 1;
+                        }
+                        1 | 2 => {
+                            // insert or update
+                            // get pk
+                            let pk = self.decode_primary_key(pk_type)?;
+                            // get column count
+                            let mut column_count = [0; sizeof!(u64)];
+                            self.f.read_into_buffer(&mut column_count)?;
+                            let mut column_count = u64::from_le_bytes(column_count);
+                            // prepare row
+                            let mut row = vec![];
+                            while column_count != 0 && !self.f.is_eof() {
+                                row.push(self.decode_cell()?);
+                                column_count -= 1;
+                            }
+                            if column_count != 0 {
+                                return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+                            }
+                            if change_type == 1 {
+                                this_batch.push(DecodedBatchEvent::new(
+                                    txnid,
+                                    pk,
+                                    DecodedBatchEventKind::Insert(row),
+                                ));
+                            } else {
+                                this_batch.push(DecodedBatchEvent::new(
+                                    txnid,
+                                    pk,
+                                    DecodedBatchEventKind::Update(row),
+                                ));
+                            }
+                            processed_in_this_batch += 1;
+                        }
+                        _ => {
+                            return Err(SDSSError::DataBatchRestoreCorruptedBatch);
+                        }
+                    }
+                }
+            }
+        }
+        Ok(Batch::Normal(NormalBatch::new(this_batch, schema_version)))
+    }
+    fn read_normal_event_metadata(&mut self) -> Result<(u8, u64), SDSSError> {
+        let pk_type = self.f.read_byte()?;
+        let mut txnid = [0; sizeof!(u64)];
+        self.f.read_into_buffer(&mut txnid)?;
+        let txnid = u64::from_le_bytes(txnid);
+        Ok((pk_type, txnid))
+    }
+    fn attempt_recover_data_batch(&mut self) -> SDSSResult<()> {
+        let mut max_threshold = RECOVERY_THRESHOLD;
+        while max_threshold != 0 && self.f.has_left(1) {
+            if let Ok(MARKER_RECOVERY_EVENT) = self.f.inner_file().read_byte() {
+                return Ok(());
+            }
+            max_threshold -= 1;
+        }
+        Err(SDSSError::DataBatchRestoreCorruptedBatch)
+    }
+}
+
+impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
+    fn decode_primary_key(&mut self, pk_type: u8) -> SDSSResult<PrimaryIndexKey> {
+        let Some(pk_type) = TagUnique::try_from_raw(pk_type) else {
+            return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+        };
+        Ok(match pk_type {
+            TagUnique::SignedInt | TagUnique::UnsignedInt => {
+                let mut chunk = [0; sizeof!(u64)];
+                self.f.read_into_buffer(&mut chunk)?;
+                unsafe {
+                    // UNSAFE(@ohsayan): +tagck
+                    PrimaryIndexKey::new_from_qw(pk_type, u64::from_le_bytes(chunk))
+                }
+            }
+            TagUnique::Str | TagUnique::Bin => {
+                let mut len = [0; sizeof!(u64)];
+                self.f.read_into_buffer(&mut len)?;
+                let mut data = vec![0; u64::from_le_bytes(len) as usize];
+                self.f.read_into_buffer(&mut data)?;
+                if pk_type == TagUnique::Str {
+                    if core::str::from_utf8(&data).is_err() {
+                        return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+                    }
+                }
+                unsafe {
+                    // UNSAFE(@ohsayan): +tagck +verityck
+                    let mut md = ManuallyDrop::new(data);
+                    PrimaryIndexKey::new_from_dual(
+                        pk_type,
+                        u64::from_le_bytes(len),
+                        md.as_mut_ptr() as usize,
+                    )
+                }
+            }
+            _ => unsafe {
+                // UNSAFE(@ohsayan): TagUnique::try_from_raw rejects an construction with Invalid as the dscr
+                impossible!()
+            },
+        })
+    }
+    fn decode_cell(&mut self) -> SDSSResult<Datacell> {
+        let cell_type_sig = self.f.read_byte()?;
+        let Some(cell_type) = PersistTypeDscr::try_from_raw(cell_type_sig) else {
+            return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+        };
+        Ok(match cell_type {
+            PersistTypeDscr::Null => Datacell::null(),
+            PersistTypeDscr::Bool => {
+                let bool = self.f.read_byte()?;
+                if bool > 1 {
+                    return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+                }
+                Datacell::new_bool(bool == 1)
+            }
+            PersistTypeDscr::UnsignedInt | PersistTypeDscr::SignedInt | PersistTypeDscr::Float => {
+                let mut block = [0; sizeof!(u64)];
+                self.f.read_into_buffer(&mut block)?;
+                unsafe {
+                    // UNSAFE(@ohsayan): choosing the correct type and tag
+                    let tc = TagClass::from_raw(cell_type.value_u8() - 1);
+                    Datacell::new_qw(u64::from_le_bytes(block), CUTag::new(tc, tc.tag_unique()))
+                }
+            }
+            PersistTypeDscr::Str | PersistTypeDscr::Bin => {
+                let mut len_block = [0; sizeof!(u64)];
+                self.f.read_into_buffer(&mut len_block)?;
+                let len = u64::from_le_bytes(len_block) as usize;
+                let mut data = vec![0; len];
+                self.f.read_into_buffer(&mut data)?;
+                unsafe {
+                    // UNSAFE(@ohsayan): +tagck
+                    if cell_type == PersistTypeDscr::Str {
+                        if core::str::from_utf8(&data).is_err() {
+                            return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+                        }
+                        Datacell::new_str(String::from_utf8_unchecked(data).into_boxed_str())
+                    } else {
+                        Datacell::new_bin(data.into_boxed_slice())
+                    }
+                }
+            }
+            PersistTypeDscr::List => {
+                let mut len_block = [0; sizeof!(u64)];
+                self.f.read_into_buffer(&mut len_block)?;
+                let len = u64::from_le_bytes(len_block);
+                let mut list = Vec::new();
+                while !self.f.is_eof() && list.len() as u64 != len {
+                    list.push(self.decode_cell()?);
+                }
+                if len != list.len() as u64 {
+                    return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+                }
+                Datacell::new_list(list)
+            }
+            PersistTypeDscr::Dict => {
+                // we don't support dicts just yet
+                return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+            }
+        })
+    }
+}
diff --git a/server/src/engine/storage/v1/header_impl/mod.rs b/server/src/engine/storage/v1/header_impl/mod.rs
index abec2a20..b8e1ae1f 100644
--- a/server/src/engine/storage/v1/header_impl/mod.rs
+++ b/server/src/engine/storage/v1/header_impl/mod.rs
@@ -72,14 +72,14 @@ mod dr;
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
 pub enum FileScope {
     Journal = 0,
-    TransactionLogCompacted = 1,
+    DataBatch = 1,
 }
 
 impl FileScope {
     pub const fn try_new(id: u64) -> Option<Self> {
         Some(match id {
             0 => Self::Journal,
-            1 => Self::TransactionLogCompacted,
+            1 => Self::DataBatch,
             _ => return None,
         })
     }
@@ -95,16 +95,18 @@ impl FileScope {
 #[repr(u8)]
 pub enum FileSpecifier {
     GNSTxnLog = 0,
+    TableDataBatch = 1,
     #[cfg(test)]
-    TestTransactionLog = 1,
+    TestTransactionLog = 0xFF,
 }
 
 impl FileSpecifier {
     pub const fn try_new(v: u32) -> Option<Self> {
         Some(match v {
             0 => Self::GNSTxnLog,
+            1 => Self::TableDataBatch,
             #[cfg(test)]
-            1 => Self::TestTransactionLog,
+            0xFF => Self::TestTransactionLog,
             _ => return None,
         })
     }
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index 7cd872a4..660a4569 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -27,7 +27,7 @@
 use {
     super::{
         obj::{self, FieldMD},
-        PersistDictEntryDscr, PersistMapSpec, PersistObject, VecU8,
+        PersistTypeDscr, PersistMapSpec, PersistObject, VecU8,
     },
     crate::{
         engine::{
@@ -178,7 +178,7 @@ impl PersistMapSpec for GenericDictSpec {
     fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool {
         static EXPECT_ATLEAST: [u8; 4] = [0, 1, 8, 8]; // PAD to align
         let lbound_rem = md.klen + EXPECT_ATLEAST[cmp::min(md.dscr, 3) as usize] as usize;
-        scanner.has_left(lbound_rem) & (md.dscr <= PersistDictEntryDscr::Dict.value_u8())
+        scanner.has_left(lbound_rem) & (md.dscr <= PersistTypeDscr::Dict.value_u8())
     }
     fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) {
         buf.extend(key.len().u64_bytes_le());
@@ -189,7 +189,7 @@ impl PersistMapSpec for GenericDictSpec {
     fn enc_entry(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
         match val {
             DictEntryGeneric::Map(map) => {
-                buf.push(PersistDictEntryDscr::Dict.value_u8());
+                buf.push(PersistTypeDscr::Dict.value_u8());
                 buf.extend(key.as_bytes());
                 <PersistMapImpl<Self> as PersistObject>::default_full_enc(buf, map);
             }
@@ -208,17 +208,17 @@ impl PersistMapSpec for GenericDictSpec {
     unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value> {
         unsafe fn decode_element(
             scanner: &mut BufferedScanner,
-            dscr: PersistDictEntryDscr,
+            dscr: PersistTypeDscr,
             dg_top_element: bool,
         ) -> Option<DictEntryGeneric> {
             let r = match dscr {
-                PersistDictEntryDscr::Null => DictEntryGeneric::Data(Datacell::null()),
-                PersistDictEntryDscr::Bool => {
+                PersistTypeDscr::Null => DictEntryGeneric::Data(Datacell::null()),
+                PersistTypeDscr::Bool => {
                     DictEntryGeneric::Data(Datacell::new_bool(scanner.next_byte() == 1))
                 }
-                PersistDictEntryDscr::UnsignedInt
-                | PersistDictEntryDscr::SignedInt
-                | PersistDictEntryDscr::Float => DictEntryGeneric::Data(Datacell::new_qw(
+                PersistTypeDscr::UnsignedInt
+                | PersistTypeDscr::SignedInt
+                | PersistTypeDscr::Float => DictEntryGeneric::Data(Datacell::new_qw(
                     scanner.next_u64_le(),
                     CUTag::new(
                         dscr.into_class(),
@@ -230,13 +230,13 @@ impl PersistMapSpec for GenericDictSpec {
                         ][(dscr.value_u8() - 2) as usize],
                     ),
                 )),
-                PersistDictEntryDscr::Str | PersistDictEntryDscr::Bin => {
+                PersistTypeDscr::Str | PersistTypeDscr::Bin => {
                     let slc_len = scanner.next_u64_le() as usize;
                     if !scanner.has_left(slc_len) {
                         return None;
                     }
                     let slc = scanner.next_chunk_variable(slc_len);
-                    DictEntryGeneric::Data(if dscr == PersistDictEntryDscr::Str {
+                    DictEntryGeneric::Data(if dscr == PersistTypeDscr::Str {
                         if core::str::from_utf8(slc).is_err() {
                             return None;
                         }
@@ -247,18 +247,18 @@ impl PersistMapSpec for GenericDictSpec {
                         Datacell::new_bin(slc.to_owned().into_boxed_slice())
                     })
                 }
-                PersistDictEntryDscr::List => {
+                PersistTypeDscr::List => {
                     let list_len = scanner.next_u64_le() as usize;
                     let mut v = Vec::with_capacity(list_len);
                     while (!scanner.eof()) & (v.len() < list_len) {
                         let dscr = scanner.next_byte();
-                        if dscr > PersistDictEntryDscr::Dict.value_u8() {
+                        if dscr > PersistTypeDscr::Dict.value_u8() {
                             return None;
                         }
                         v.push(
                             match decode_element(
                                 scanner,
-                                PersistDictEntryDscr::from_raw(dscr),
+                                PersistTypeDscr::from_raw(dscr),
                                 false,
                             ) {
                                 Some(DictEntryGeneric::Data(l)) => l,
@@ -273,7 +273,7 @@ impl PersistMapSpec for GenericDictSpec {
                         return None;
                     }
                 }
-                PersistDictEntryDscr::Dict => {
+                PersistTypeDscr::Dict => {
                     if dg_top_element {
                         DictEntryGeneric::Map(
                             <PersistMapImpl<GenericDictSpec> as PersistObject>::default_full_dec(
@@ -288,7 +288,7 @@ impl PersistMapSpec for GenericDictSpec {
             };
             Some(r)
         }
-        decode_element(scanner, PersistDictEntryDscr::from_raw(md.dscr), true)
+        decode_element(scanner, PersistTypeDscr::from_raw(md.dscr), true)
     }
     // not implemented
     fn enc_key(_: &mut VecU8, _: &Self::Key) {
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index 43486d23..3a0769ea 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -51,7 +51,7 @@ type VecU8 = Vec<u8>;
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, sky_macros::EnumMethods)]
 #[repr(u8)]
 /// Disambiguation for data
-pub enum PersistDictEntryDscr {
+pub enum PersistTypeDscr {
     Null = 0,
     Bool = 1,
     UnsignedInt = 2,
@@ -63,11 +63,19 @@ pub enum PersistDictEntryDscr {
     Dict = 8,
 }
 
-impl PersistDictEntryDscr {
+impl PersistTypeDscr {
+    pub(super) const MAX: Self = Self::Dict;
     /// translates the tag class definition into the dscr definition
     pub const fn translate_from_class(class: TagClass) -> Self {
         unsafe { Self::from_raw(class.d() + 1) }
     }
+    pub const fn try_from_raw(v: u8) -> Option<Self> {
+        if v > Self::MAX.value_u8() {
+            None
+        } else {
+            unsafe { Some(Self::from_raw(v)) }
+        }
+    }
     pub const unsafe fn from_raw(v: u8) -> Self {
         core::mem::transmute(v)
     }
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index f1c11152..18cdd5bf 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -27,7 +27,7 @@
 use crate::engine::{core::model::delta::IRModel, data::DictGeneric};
 
 use {
-    super::{PersistDictEntryDscr, PersistObject, VecU8},
+    super::{PersistTypeDscr, PersistObject, VecU8},
     crate::{
         engine::{
             core::{
@@ -71,7 +71,7 @@ pub fn encode_element(buf: &mut VecU8, dc: &Datacell) {
 
 pub fn encode_datacell_tag(buf: &mut VecU8, dc: &Datacell) {
     buf.push(
-        PersistDictEntryDscr::translate_from_class(dc.tag().tag_class()).value_u8()
+        PersistTypeDscr::translate_from_class(dc.tag().tag_class()).value_u8()
             * (!dc.is_null() as u8),
     )
 }
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index eba132c3..84ea217c 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -68,16 +68,18 @@ pub fn null_journal<TA: JournalAdapter>(
     host_startup_counter: u64,
     _: &TA::GlobalState,
 ) -> JournalWriter<super::rw::NullZero, TA> {
-    let FileOpen::Created(journal) = SDSSFileIO::<super::rw::NullZero>::open_or_create_perm_rw(
-        log_file_name,
-        FileScope::Journal,
-        log_kind,
-        log_kind_version,
-        host_setting_version,
-        host_run_mode,
-        host_startup_counter,
-    )
-    .unwrap() else {
+    let FileOpen::Created(journal) =
+        SDSSFileIO::<super::rw::NullZero>::open_or_create_perm_rw::<false>(
+            log_file_name,
+            FileScope::Journal,
+            log_kind,
+            log_kind_version,
+            host_setting_version,
+            host_run_mode,
+            host_startup_counter,
+        )
+        .unwrap()
+    else {
         panic!()
     };
     JournalWriter::new(journal, 0, true).unwrap()
@@ -92,15 +94,25 @@ pub fn open_journal<TA: JournalAdapter, LF: RawFileIOInterface>(
     host_startup_counter: u64,
     gs: &TA::GlobalState,
 ) -> SDSSResult<JournalWriter<LF, TA>> {
-    let f = SDSSFileIO::<LF>::open_or_create_perm_rw(
-        log_file_name,
-        FileScope::Journal,
-        log_kind,
-        log_kind_version,
-        host_setting_version,
-        host_run_mode,
-        host_startup_counter,
-    )?;
+    macro_rules! open_file {
+        ($modify:literal) => {
+            SDSSFileIO::<LF>::open_or_create_perm_rw::<$modify>(
+                log_file_name,
+                FileScope::Journal,
+                log_kind,
+                log_kind_version,
+                host_setting_version,
+                host_run_mode,
+                host_startup_counter,
+            )
+        };
+    }
+    // HACK(@ohsayan): until generic const exprs are stabilized, we're in a state of hell
+    let f = if TA::DENY_NONAPPEND {
+        open_file!(false)
+    } else {
+        open_file!(true)
+    }?;
     let file = match f {
         FileOpen::Created(f) => return JournalWriter::new(f, 0, true),
         FileOpen::Existing(file, _) => file,
@@ -111,6 +123,8 @@ pub fn open_journal<TA: JournalAdapter, LF: RawFileIOInterface>(
 
 /// The journal adapter
 pub trait JournalAdapter {
+    /// deny any SDSS file level operations that require non-append mode writes (for example, updating the SDSS header's modify count)
+    const DENY_NONAPPEND: bool = true;
     /// enable/disable automated recovery algorithms
     const RECOVERY_PLUGIN: bool;
     /// The journal event
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index b4113c78..0c111ef0 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -27,6 +27,7 @@
 // raw
 mod header_impl;
 // impls
+mod batch_jrnl;
 mod journal;
 mod rw;
 // hl
@@ -110,6 +111,16 @@ pub enum SDSSError {
     InternalDecodeStructureCorruptedPayload,
     /// the data for an internal structure was decoded but is logically invalid
     InternalDecodeStructureIllegalData,
+    /// when attempting to flush a data batch, the batch journal crashed and a recovery event was triggered. But even then,
+    /// the data batch journal could not be fixed
+    DataBatchRecoveryFailStageOne,
+    /// when attempting to restore a data batch from disk, the batch journal crashed and had a corruption, but it is irrecoverable
+    DataBatchRestoreCorruptedBatch,
+    /// when attempting to restore a data batch from disk, the driver encountered a corrupted entry
+    DataBatchRestoreCorruptedEntry,
+    /// we failed to close the data batch
+    DataBatchCloseError,
+    DataBatchRestoreCorruptedBatchFile,
 }
 
 impl SDSSError {
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 969f8cbc..a21b2b5e 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -31,7 +31,10 @@ use {
         },
         SDSSResult,
     },
-    crate::engine::storage::v1::SDSSError,
+    crate::{
+        engine::storage::{v1::SDSSError, SCrc},
+        util::os::SysIOError,
+    },
     std::{
         fs::File,
         io::{Read, Seek, SeekFrom, Write},
@@ -46,6 +49,21 @@ pub enum FileOpen<F> {
     Existing(F, SDSSHeader),
 }
 
+impl<F> FileOpen<F> {
+    pub fn into_existing(self) -> Option<(F, SDSSHeader)> {
+        match self {
+            Self::Existing(f, h) => Some((f, h)),
+            Self::Created(_) => None,
+        }
+    }
+    pub fn into_created(self) -> Option<F> {
+        match self {
+            Self::Created(f) => Some(f),
+            Self::Existing(_, _) => None,
+        }
+    }
+}
+
 #[derive(Debug)]
 pub enum RawFileOpen<F> {
     Created(F),
@@ -138,6 +156,103 @@ impl RawFileIOInterface for File {
     }
 }
 
+pub struct SDSSFileTrackedWriter<F> {
+    f: SDSSFileIO<F>,
+    cs: SCrc,
+}
+
+impl<F: RawFileIOInterface> SDSSFileTrackedWriter<F> {
+    pub fn new(f: SDSSFileIO<F>) -> Self {
+        Self { f, cs: SCrc::new() }
+    }
+    pub fn unfsynced_write(&mut self, block: &[u8]) -> SDSSResult<()> {
+        match self.f.unfsynced_write(block) {
+            Ok(()) => {
+                self.cs.recompute_with_new_var_block(block);
+                Ok(())
+            }
+            e => e,
+        }
+    }
+    pub fn fsync_all(&mut self) -> SDSSResult<()> {
+        self.f.fsync_all()
+    }
+    pub fn reset_and_finish_checksum(&mut self) -> u64 {
+        let mut scrc = SCrc::new();
+        core::mem::swap(&mut self.cs, &mut scrc);
+        scrc.finish()
+    }
+    pub fn inner_file(&mut self) -> &mut SDSSFileIO<F> {
+        &mut self.f
+    }
+}
+
+/// [`SDSSFileLenTracked`] simply maintains application level length and checksum tracking to avoid frequent syscalls because we
+/// do not expect (even though it's very possible) users to randomly modify file lengths while we're reading them
+pub struct SDSSFileTrackedReader<F> {
+    f: SDSSFileIO<F>,
+    len: u64,
+    pos: u64,
+    cs: SCrc,
+}
+
+impl<F: RawFileIOInterface> SDSSFileTrackedReader<F> {
+    /// Important: this will only look at the data post the current cursor!
+    pub fn new(mut f: SDSSFileIO<F>) -> SDSSResult<Self> {
+        let len = f.file_length()?;
+        let pos = f.retrieve_cursor()?;
+        Ok(Self {
+            f,
+            len,
+            pos,
+            cs: SCrc::new(),
+        })
+    }
+    pub fn remaining(&self) -> u64 {
+        self.len - self.pos
+    }
+    pub fn is_eof(&self) -> bool {
+        self.len == self.pos
+    }
+    pub fn has_left(&self, v: u64) -> bool {
+        self.remaining() >= v
+    }
+    pub fn read_into_buffer(&mut self, buf: &mut [u8]) -> SDSSResult<()> {
+        if self.remaining() >= buf.len() as u64 {
+            match self.f.read_to_buffer(buf) {
+                Ok(()) => {
+                    self.pos += buf.len() as u64;
+                    self.cs.recompute_with_new_var_block(buf);
+                    Ok(())
+                }
+                Err(e) => return Err(e),
+            }
+        } else {
+            Err(SDSSError::IoError(SysIOError::from(
+                std::io::ErrorKind::InvalidInput,
+            )))
+        }
+    }
+    pub fn read_byte(&mut self) -> SDSSResult<u8> {
+        let mut buf = [0u8; 1];
+        self.read_into_buffer(&mut buf).map(|_| buf[0])
+    }
+    pub fn __reset_checksum(&mut self) -> u64 {
+        let mut crc = SCrc::new();
+        core::mem::swap(&mut crc, &mut self.cs);
+        crc.finish()
+    }
+    pub fn inner_file(&mut self) -> &mut SDSSFileIO<F> {
+        &mut self.f
+    }
+    pub fn into_inner_file(self) -> SDSSFileIO<F> {
+        self.f
+    }
+    pub fn __cursor_ahead_by(&mut self, sizeof: usize) {
+        self.pos += sizeof as u64;
+    }
+}
+
 #[derive(Debug)]
 pub struct SDSSFileIO<F> {
     f: F,
@@ -145,7 +260,7 @@ pub struct SDSSFileIO<F> {
 
 impl<F: RawFileIOInterface> SDSSFileIO<F> {
     /// **IMPORTANT: File position: end-of-header-section**
-    pub fn open_or_create_perm_rw(
+    pub fn open_or_create_perm_rw<const REWRITE_MODIFY_COUNTER: bool>(
         file_path: &str,
         file_scope: FileScope,
         file_specifier: FileSpecifier,
@@ -180,13 +295,15 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
                     .ok_or(SDSSError::HeaderDecodeCorruptedHeader)?;
                 // now validate the header
                 header.verify(file_scope, file_specifier, file_specifier_version)?;
-                // since we updated this file, let us update the header
-                let mut new_header = header.clone();
-                new_header.dr_rs_mut().bump_modify_count();
                 let mut f = Self::_new(f);
-                f.seek_from_start(0)?;
-                f.fsynced_write(new_header.encoded().array().as_ref())?;
-                f.seek_from_start(SDSSHeaderRaw::header_size() as _)?;
+                if REWRITE_MODIFY_COUNTER {
+                    // since we updated this file, let us update the header
+                    let mut new_header = header.clone();
+                    new_header.dr_rs_mut().bump_modify_count();
+                    f.seek_from_start(0)?;
+                    f.fsynced_write(new_header.encoded().array().as_ref())?;
+                    f.seek_from_start(SDSSHeaderRaw::header_size() as _)?;
+                }
                 Ok(FileOpen::Existing(f, header))
             }
         }
@@ -223,6 +340,10 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
     pub fn retrieve_cursor(&mut self) -> SDSSResult<u64> {
         self.f.fcursor()
     }
+    pub fn read_byte(&mut self) -> SDSSResult<u8> {
+        let mut r = [0; 1];
+        self.read_to_buffer(&mut r).map(|_| r[0])
+    }
 }
 
 pub struct BufferedScanner<'a> {
diff --git a/server/src/engine/storage/v1/test_util.rs b/server/src/engine/storage/v1/test_util.rs
index 54452d20..263d00c9 100644
--- a/server/src/engine/storage/v1/test_util.rs
+++ b/server/src/engine/storage/v1/test_util.rs
@@ -69,6 +69,11 @@ impl VFile {
 
 #[derive(Debug)]
 pub struct VirtualFS(Box<str>);
+impl VirtualFS {
+    pub fn get_file_data(f: &str) -> Option<Vec<u8>> {
+        VFS.read().get(f).map(|f| f.data.clone())
+    }
+}
 
 impl RawFileIOInterface for VirtualFS {
     fn fopen_or_create_rw(file_path: &str) -> super::SDSSResult<RawFileOpen<Self>> {
@@ -181,7 +186,7 @@ impl RawFileIOInterface for VirtualFS {
 
 #[test]
 fn sdss_file() {
-    let f = SDSSFileIO::<VirtualFS>::open_or_create_perm_rw(
+    let f = SDSSFileIO::<VirtualFS>::open_or_create_perm_rw::<false>(
         "this_is_a_test_file.db",
         FileScope::Journal,
         FileSpecifier::TestTransactionLog,
@@ -199,7 +204,7 @@ fn sdss_file() {
     f.fsynced_write(b"hello, world\n").unwrap();
     f.fsynced_write(b"hello, again\n").unwrap();
 
-    let f = SDSSFileIO::<VirtualFS>::open_or_create_perm_rw(
+    let f = SDSSFileIO::<VirtualFS>::open_or_create_perm_rw::<false>(
         "this_is_a_test_file.db",
         FileScope::Journal,
         FileSpecifier::TestTransactionLog,
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index dbe92fc7..8439377c 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -26,237 +26,6 @@
 
 type VirtualFS = super::test_util::VirtualFS;
 
-mod rw {
-    use crate::engine::storage::v1::{
-        header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
-        rw::{FileOpen, SDSSFileIO},
-    };
-
-    #[test]
-    fn create_delete() {
-        let f = SDSSFileIO::<super::VirtualFS>::open_or_create_perm_rw(
-            "hello_world.db-tlog",
-            FileScope::TransactionLogCompacted,
-            FileSpecifier::GNSTxnLog,
-            FileSpecifierVersion::__new(0),
-            0,
-            HostRunMode::Prod,
-            0,
-        )
-        .unwrap();
-        match f {
-            FileOpen::Existing(_, _) => panic!(),
-            FileOpen::Created(_) => {}
-        };
-        let open = SDSSFileIO::<super::VirtualFS>::open_or_create_perm_rw(
-            "hello_world.db-tlog",
-            FileScope::TransactionLogCompacted,
-            FileSpecifier::GNSTxnLog,
-            FileSpecifierVersion::__new(0),
-            0,
-            HostRunMode::Prod,
-            0,
-        )
-        .unwrap();
-        let h = match open {
-            FileOpen::Existing(_, header) => header,
-            _ => panic!(),
-        };
-        assert_eq!(h.gr_mdr().file_scope(), FileScope::TransactionLogCompacted);
-        assert_eq!(h.gr_mdr().file_spec(), FileSpecifier::GNSTxnLog);
-        assert_eq!(h.gr_mdr().file_spec_id(), FileSpecifierVersion::__new(0));
-        assert_eq!(h.gr_hr().run_mode(), HostRunMode::Prod);
-        assert_eq!(h.gr_hr().setting_version(), 0);
-        assert_eq!(h.gr_hr().startup_counter(), 0);
-    }
-}
-
-mod tx {
-    use crate::engine::storage::v1::header_impl::{
-        FileSpecifier, FileSpecifierVersion, HostRunMode,
-    };
-
-    use {
-        crate::{
-            engine::storage::v1::{
-                journal::{self, JournalAdapter, JournalWriter},
-                SDSSError, SDSSResult,
-            },
-            util,
-        },
-        std::cell::RefCell,
-    };
-    pub struct Database {
-        data: RefCell<[u8; 10]>,
-    }
-    impl Database {
-        fn copy_data(&self) -> [u8; 10] {
-            *self.data.borrow()
-        }
-        fn new() -> Self {
-            Self {
-                data: RefCell::new([0; 10]),
-            }
-        }
-        fn reset(&self) {
-            *self.data.borrow_mut() = [0; 10];
-        }
-        fn txn_reset(
-            &self,
-            txn_writer: &mut JournalWriter<super::VirtualFS, DatabaseTxnAdapter>,
-        ) -> SDSSResult<()> {
-            self.reset();
-            txn_writer.append_event(TxEvent::Reset)
-        }
-        fn set(&self, pos: usize, val: u8) {
-            self.data.borrow_mut()[pos] = val;
-        }
-        fn txn_set(
-            &self,
-            pos: usize,
-            val: u8,
-            txn_writer: &mut JournalWriter<super::VirtualFS, DatabaseTxnAdapter>,
-        ) -> SDSSResult<()> {
-            self.set(pos, val);
-            txn_writer.append_event(TxEvent::Set(pos, val))
-        }
-    }
-    pub enum TxEvent {
-        Reset,
-        Set(usize, u8),
-    }
-    #[derive(Debug)]
-    pub enum TxError {
-        SDSS(SDSSError),
-    }
-    direct_from! {
-        TxError => {
-            SDSSError as SDSS
-        }
-    }
-    #[derive(Debug)]
-    pub struct DatabaseTxnAdapter;
-    impl JournalAdapter for DatabaseTxnAdapter {
-        const RECOVERY_PLUGIN: bool = false;
-        type Error = TxError;
-        type JournalEvent = TxEvent;
-        type GlobalState = Database;
-
-        fn encode(event: Self::JournalEvent) -> Box<[u8]> {
-            /*
-                [1B: opcode][8B:Index][1B: New value]
-            */
-            let opcode = match event {
-                TxEvent::Reset => 0u8,
-                TxEvent::Set(_, _) => 1u8,
-            };
-            let index = match event {
-                TxEvent::Reset => 0u64,
-                TxEvent::Set(index, _) => index as u64,
-            };
-            let new_value = match event {
-                TxEvent::Reset => 0,
-                TxEvent::Set(_, val) => val,
-            };
-            let mut ret = Vec::with_capacity(10);
-            ret.push(opcode);
-            ret.extend(index.to_le_bytes());
-            ret.push(new_value);
-            ret.into_boxed_slice()
-        }
-
-        fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> Result<(), TxError> {
-            if payload.len() != 10 {
-                return Err(SDSSError::CorruptedFile("testtxn.log").into());
-            }
-            let opcode = payload[0];
-            let index = u64::from_le_bytes(util::copy_slice_to_array(&payload[1..9]));
-            let new_value = payload[9];
-            match opcode {
-                0 if index == 0 && new_value == 0 => gs.reset(),
-                1 if index < 10 && index < isize::MAX as u64 => gs.set(index as usize, new_value),
-                _ => return Err(SDSSError::JournalLogEntryCorrupted.into()),
-            }
-            Ok(())
-        }
-    }
-
-    fn open_log(
-        log_name: &str,
-        db: &Database,
-    ) -> SDSSResult<JournalWriter<super::VirtualFS, DatabaseTxnAdapter>> {
-        journal::open_journal::<DatabaseTxnAdapter, super::VirtualFS>(
-            log_name,
-            FileSpecifier::TestTransactionLog,
-            FileSpecifierVersion::__new(0),
-            0,
-            HostRunMode::Prod,
-            1,
-            &db,
-        )
-    }
-
-    #[test]
-    fn first_boot_second_readonly() {
-        // create log
-        let db1 = Database::new();
-        let x = || -> SDSSResult<()> {
-            let mut log = open_log("testtxn.log", &db1)?;
-            db1.txn_set(0, 20, &mut log)?;
-            db1.txn_set(9, 21, &mut log)?;
-            log.append_journal_close_and_close()
-        };
-        x().unwrap();
-        // backup original data
-        let original_data = db1.copy_data();
-        // restore log
-        let empty_db2 = Database::new();
-        open_log("testtxn.log", &empty_db2)
-            .unwrap()
-            .append_journal_close_and_close()
-            .unwrap();
-        assert_eq!(original_data, empty_db2.copy_data());
-    }
-    #[test]
-    fn oneboot_mod_twoboot_mod_thirdboot_read() {
-        // first boot: set all to 1
-        let db1 = Database::new();
-        let x = || -> SDSSResult<()> {
-            let mut log = open_log("duatxn.db-tlog", &db1)?;
-            for i in 0..10 {
-                db1.txn_set(i, 1, &mut log)?;
-            }
-            log.append_journal_close_and_close()
-        };
-        x().unwrap();
-        let bkp_db1 = db1.copy_data();
-        drop(db1);
-        // second boot
-        let db2 = Database::new();
-        let x = || -> SDSSResult<()> {
-            let mut log = open_log("duatxn.db-tlog", &db2)?;
-            assert_eq!(bkp_db1, db2.copy_data());
-            for i in 0..10 {
-                let current_val = db2.data.borrow()[i];
-                db2.txn_set(i, current_val + i as u8, &mut log)?;
-            }
-            log.append_journal_close_and_close()
-        };
-        x().unwrap();
-        let bkp_db2 = db2.copy_data();
-        drop(db2);
-        // third boot
-        let db3 = Database::new();
-        let log = open_log("duatxn.db-tlog", &db3).unwrap();
-        log.append_journal_close_and_close().unwrap();
-        assert_eq!(bkp_db2, db3.copy_data());
-        assert_eq!(
-            db3.copy_data(),
-            (1..=10)
-                .into_iter()
-                .map(u8::from)
-                .collect::<Box<[u8]>>()
-                .as_ref()
-        );
-    }
-}
+mod batch;
+mod rw;
+mod tx;
diff --git a/server/src/engine/storage/v1/tests/batch.rs b/server/src/engine/storage/v1/tests/batch.rs
new file mode 100644
index 00000000..7db00240
--- /dev/null
+++ b/server/src/engine/storage/v1/tests/batch.rs
@@ -0,0 +1,210 @@
+/*
+ * Created on Wed Sep 06 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::engine::{
+        core::{
+            index::{DcFieldIndex, PrimaryIndexKey, Row},
+            model::{
+                delta::{DataDelta, DataDeltaKind, DeltaVersion},
+                Field, Layer, Model,
+            },
+        },
+        data::{cell::Datacell, tag::TagSelector, uuid::Uuid},
+        storage::v1::{
+            batch_jrnl::{
+                DataBatchPersistDriver, DataBatchRestoreDriver, DecodedBatchEvent,
+                DecodedBatchEventKind, NormalBatch,
+            },
+            header_meta::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
+            rw::{FileOpen, SDSSFileIO},
+            test_util::VirtualFS,
+        },
+    },
+    crossbeam_epoch::pin,
+};
+
+fn pkey(v: impl Into<Datacell>) -> PrimaryIndexKey {
+    PrimaryIndexKey::try_from_dc(v.into()).unwrap()
+}
+
+fn open_file(fpath: &str) -> FileOpen<SDSSFileIO<VirtualFS>> {
+    SDSSFileIO::open_or_create_perm_rw::<false>(
+        fpath,
+        FileScope::DataBatch,
+        FileSpecifier::TableDataBatch,
+        FileSpecifierVersion::__new(0),
+        0,
+        HostRunMode::Dev,
+        1,
+    )
+    .unwrap()
+}
+
+fn open_batch_data(fpath: &str, mdl: &Model) -> DataBatchPersistDriver<VirtualFS> {
+    match open_file(fpath) {
+        FileOpen::Created(f) => DataBatchPersistDriver::new(f, true),
+        FileOpen::Existing(f, _) => {
+            let mut dbr = DataBatchRestoreDriver::new(f).unwrap();
+            dbr.read_data_batch_into_model(mdl).unwrap();
+            DataBatchPersistDriver::new(dbr.into_file(), false)
+        }
+    }
+    .unwrap()
+}
+
+fn new_delta(
+    schema: u64,
+    txnid: u64,
+    pk: Datacell,
+    data: DcFieldIndex,
+    change: DataDeltaKind,
+) -> DataDelta {
+    new_delta_with_row(
+        schema,
+        txnid,
+        Row::new(
+            pkey(pk),
+            data,
+            DeltaVersion::test_new(schema),
+            DeltaVersion::test_new(txnid),
+        ),
+        change,
+    )
+}
+
+fn new_delta_with_row(schema: u64, txnid: u64, row: Row, change: DataDeltaKind) -> DataDelta {
+    DataDelta::new(
+        DeltaVersion::test_new(schema),
+        DeltaVersion::test_new(txnid),
+        row,
+        change,
+    )
+}
+
+#[test]
+fn deltas_only_insert() {
+    // prepare model definition
+    let uuid = Uuid::new();
+    let mdl = Model::new_restore(
+        uuid,
+        "catname".into(),
+        TagSelector::Str.into_full(),
+        into_dict!(
+            "catname" => Field::new([Layer::str()].into(), false),
+            "is_good" => Field::new([Layer::bool()].into(), false),
+            "magical" => Field::new([Layer::bool()].into(), false),
+        ),
+    );
+    let row = Row::new(
+        pkey("Schrödinger's cat"),
+        into_dict!("is_good" => Datacell::new_bool(true), "magical" => Datacell::new_bool(false)),
+        DeltaVersion::test_new(0),
+        DeltaVersion::test_new(2),
+    );
+    {
+        // update the row
+        let mut wl = row.d_data().write();
+        wl.set_txn_revised(DeltaVersion::test_new(3));
+        *wl.fields_mut().get_mut("magical").unwrap() = Datacell::new_bool(true);
+    }
+    // prepare deltas
+    let deltas = [
+        // insert catname: Schrödinger's cat, is_good: true
+        new_delta_with_row(0, 0, row.clone(), DataDeltaKind::Insert),
+        // insert catname: good cat, is_good: true, magical: false
+        new_delta(
+            0,
+            1,
+            Datacell::new_str("good cat".into()),
+            into_dict!("is_good" => Datacell::new_bool(true), "magical" => Datacell::new_bool(false)),
+            DataDeltaKind::Insert,
+        ),
+        // insert catname: bad cat, is_good: false, magical: false
+        new_delta(
+            0,
+            2,
+            Datacell::new_str("bad cat".into()),
+            into_dict!("is_good" => Datacell::new_bool(false), "magical" => Datacell::new_bool(false)),
+            DataDeltaKind::Insert,
+        ),
+        // update catname: Schrödinger's cat, is_good: true, magical: true
+        new_delta_with_row(0, 3, row.clone(), DataDeltaKind::Update),
+    ];
+    // delta queue
+    let g = pin();
+    for delta in deltas.clone() {
+        mdl.delta_state().append_new_data_delta(delta, &g);
+    }
+    let file = open_file("deltas_only_insert.db-btlog")
+        .into_created()
+        .unwrap();
+    {
+        let mut persist_driver = DataBatchPersistDriver::new(file, true).unwrap();
+        persist_driver.write_new_batch(&mdl, deltas.len()).unwrap();
+        persist_driver.close().unwrap();
+    }
+    let mut restore_driver = DataBatchRestoreDriver::new(
+        open_file("deltas_only_insert.db-btlog")
+            .into_existing()
+            .unwrap()
+            .0,
+    )
+    .unwrap();
+    let batch = restore_driver.read_all_batches().unwrap();
+    assert_eq!(
+        batch,
+        vec![NormalBatch::new(
+            vec![
+                DecodedBatchEvent::new(
+                    1,
+                    pkey("good cat"),
+                    DecodedBatchEventKind::Insert(vec![
+                        Datacell::new_bool(true),
+                        Datacell::new_bool(false)
+                    ])
+                ),
+                DecodedBatchEvent::new(
+                    2,
+                    pkey("bad cat"),
+                    DecodedBatchEventKind::Insert(vec![
+                        Datacell::new_bool(false),
+                        Datacell::new_bool(false)
+                    ])
+                ),
+                DecodedBatchEvent::new(
+                    3,
+                    pkey("Schrödinger's cat"),
+                    DecodedBatchEventKind::Update(vec![
+                        Datacell::new_bool(true),
+                        Datacell::new_bool(true)
+                    ])
+                )
+            ],
+            0
+        )]
+    )
+}
diff --git a/server/src/engine/storage/v1/tests/rw.rs b/server/src/engine/storage/v1/tests/rw.rs
new file mode 100644
index 00000000..23e296f3
--- /dev/null
+++ b/server/src/engine/storage/v1/tests/rw.rs
@@ -0,0 +1,68 @@
+/*
+ * Created on Tue Sep 05 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::storage::v1::{
+    header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
+    rw::{FileOpen, SDSSFileIO},
+};
+
+#[test]
+fn create_delete() {
+    let f = SDSSFileIO::<super::VirtualFS>::open_or_create_perm_rw::<false>(
+        "hello_world.db-tlog",
+        FileScope::Journal,
+        FileSpecifier::GNSTxnLog,
+        FileSpecifierVersion::__new(0),
+        0,
+        HostRunMode::Prod,
+        0,
+    )
+    .unwrap();
+    match f {
+        FileOpen::Existing(_, _) => panic!(),
+        FileOpen::Created(_) => {}
+    };
+    let open = SDSSFileIO::<super::VirtualFS>::open_or_create_perm_rw::<false>(
+        "hello_world.db-tlog",
+        FileScope::Journal,
+        FileSpecifier::GNSTxnLog,
+        FileSpecifierVersion::__new(0),
+        0,
+        HostRunMode::Prod,
+        0,
+    )
+    .unwrap();
+    let h = match open {
+        FileOpen::Existing(_, header) => header,
+        _ => panic!(),
+    };
+    assert_eq!(h.gr_mdr().file_scope(), FileScope::Journal);
+    assert_eq!(h.gr_mdr().file_spec(), FileSpecifier::GNSTxnLog);
+    assert_eq!(h.gr_mdr().file_spec_id(), FileSpecifierVersion::__new(0));
+    assert_eq!(h.gr_hr().run_mode(), HostRunMode::Prod);
+    assert_eq!(h.gr_hr().setting_version(), 0);
+    assert_eq!(h.gr_hr().startup_counter(), 0);
+}
diff --git a/server/src/engine/storage/v1/tests/tx.rs b/server/src/engine/storage/v1/tests/tx.rs
new file mode 100644
index 00000000..01c24403
--- /dev/null
+++ b/server/src/engine/storage/v1/tests/tx.rs
@@ -0,0 +1,210 @@
+/*
+ * Created on Tue Sep 05 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::{
+        engine::storage::v1::{
+            header_impl::{FileSpecifier, FileSpecifierVersion, HostRunMode},
+            journal::{self, JournalAdapter, JournalWriter},
+            SDSSError, SDSSResult,
+        },
+        util,
+    },
+    std::cell::RefCell,
+};
+pub struct Database {
+    data: RefCell<[u8; 10]>,
+}
+impl Database {
+    fn copy_data(&self) -> [u8; 10] {
+        *self.data.borrow()
+    }
+    fn new() -> Self {
+        Self {
+            data: RefCell::new([0; 10]),
+        }
+    }
+    fn reset(&self) {
+        *self.data.borrow_mut() = [0; 10];
+    }
+    fn txn_reset(
+        &self,
+        txn_writer: &mut JournalWriter<super::VirtualFS, DatabaseTxnAdapter>,
+    ) -> SDSSResult<()> {
+        self.reset();
+        txn_writer.append_event(TxEvent::Reset)
+    }
+    fn set(&self, pos: usize, val: u8) {
+        self.data.borrow_mut()[pos] = val;
+    }
+    fn txn_set(
+        &self,
+        pos: usize,
+        val: u8,
+        txn_writer: &mut JournalWriter<super::VirtualFS, DatabaseTxnAdapter>,
+    ) -> SDSSResult<()> {
+        self.set(pos, val);
+        txn_writer.append_event(TxEvent::Set(pos, val))
+    }
+}
+pub enum TxEvent {
+    Reset,
+    Set(usize, u8),
+}
+#[derive(Debug)]
+pub enum TxError {
+    SDSS(SDSSError),
+}
+direct_from! {
+    TxError => {
+        SDSSError as SDSS
+    }
+}
+#[derive(Debug)]
+pub struct DatabaseTxnAdapter;
+impl JournalAdapter for DatabaseTxnAdapter {
+    const RECOVERY_PLUGIN: bool = false;
+    type Error = TxError;
+    type JournalEvent = TxEvent;
+    type GlobalState = Database;
+
+    fn encode(event: Self::JournalEvent) -> Box<[u8]> {
+        /*
+            [1B: opcode][8B:Index][1B: New value]
+        */
+        let opcode = match event {
+            TxEvent::Reset => 0u8,
+            TxEvent::Set(_, _) => 1u8,
+        };
+        let index = match event {
+            TxEvent::Reset => 0u64,
+            TxEvent::Set(index, _) => index as u64,
+        };
+        let new_value = match event {
+            TxEvent::Reset => 0,
+            TxEvent::Set(_, val) => val,
+        };
+        let mut ret = Vec::with_capacity(10);
+        ret.push(opcode);
+        ret.extend(index.to_le_bytes());
+        ret.push(new_value);
+        ret.into_boxed_slice()
+    }
+
+    fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> Result<(), TxError> {
+        if payload.len() != 10 {
+            return Err(SDSSError::CorruptedFile("testtxn.log").into());
+        }
+        let opcode = payload[0];
+        let index = u64::from_le_bytes(util::copy_slice_to_array(&payload[1..9]));
+        let new_value = payload[9];
+        match opcode {
+            0 if index == 0 && new_value == 0 => gs.reset(),
+            1 if index < 10 && index < isize::MAX as u64 => gs.set(index as usize, new_value),
+            _ => return Err(SDSSError::JournalLogEntryCorrupted.into()),
+        }
+        Ok(())
+    }
+}
+
+fn open_log(
+    log_name: &str,
+    db: &Database,
+) -> SDSSResult<JournalWriter<super::VirtualFS, DatabaseTxnAdapter>> {
+    journal::open_journal::<DatabaseTxnAdapter, super::VirtualFS>(
+        log_name,
+        FileSpecifier::TestTransactionLog,
+        FileSpecifierVersion::__new(0),
+        0,
+        HostRunMode::Prod,
+        1,
+        &db,
+    )
+}
+
+#[test]
+fn first_boot_second_readonly() {
+    // create log
+    let db1 = Database::new();
+    let x = || -> SDSSResult<()> {
+        let mut log = open_log("testtxn.log", &db1)?;
+        db1.txn_set(0, 20, &mut log)?;
+        db1.txn_set(9, 21, &mut log)?;
+        log.append_journal_close_and_close()
+    };
+    x().unwrap();
+    // backup original data
+    let original_data = db1.copy_data();
+    // restore log
+    let empty_db2 = Database::new();
+    open_log("testtxn.log", &empty_db2)
+        .unwrap()
+        .append_journal_close_and_close()
+        .unwrap();
+    assert_eq!(original_data, empty_db2.copy_data());
+}
+#[test]
+fn oneboot_mod_twoboot_mod_thirdboot_read() {
+    // first boot: set all to 1
+    let db1 = Database::new();
+    let x = || -> SDSSResult<()> {
+        let mut log = open_log("duatxn.db-tlog", &db1)?;
+        for i in 0..10 {
+            db1.txn_set(i, 1, &mut log)?;
+        }
+        log.append_journal_close_and_close()
+    };
+    x().unwrap();
+    let bkp_db1 = db1.copy_data();
+    drop(db1);
+    // second boot
+    let db2 = Database::new();
+    let x = || -> SDSSResult<()> {
+        let mut log = open_log("duatxn.db-tlog", &db2)?;
+        assert_eq!(bkp_db1, db2.copy_data());
+        for i in 0..10 {
+            let current_val = db2.data.borrow()[i];
+            db2.txn_set(i, current_val + i as u8, &mut log)?;
+        }
+        log.append_journal_close_and_close()
+    };
+    x().unwrap();
+    let bkp_db2 = db2.copy_data();
+    drop(db2);
+    // third boot
+    let db3 = Database::new();
+    let log = open_log("duatxn.db-tlog", &db3).unwrap();
+    log.append_journal_close_and_close().unwrap();
+    assert_eq!(bkp_db2, db3.copy_data());
+    assert_eq!(
+        db3.copy_data(),
+        (1..=10)
+            .into_iter()
+            .map(u8::from)
+            .collect::<Box<[u8]>>()
+            .as_ref()
+    );
+}
diff --git a/server/src/engine/txn/data.rs b/server/src/engine/txn/data.rs
index fcc9cd48..38c70437 100644
--- a/server/src/engine/txn/data.rs
+++ b/server/src/engine/txn/data.rs
@@ -25,12 +25,8 @@
 */
 
 use crate::{
-    engine::{
-        core::{index::PrimaryIndexKey, model::delta::DataDelta, GlobalNS},
-        data::cell::Datacell,
-        storage::v1::inf::obj,
-    },
-    util::{os, EndianQW},
+    engine::core::{model::delta::DataDelta, GlobalNS},
+    util::os,
 };
 
 type Buf = Vec<u8>;
@@ -72,28 +68,3 @@ pub unsafe fn get_max_delta_queue_size() -> usize {
     // TODO(@ohsayan): dynamically approximate this limit
     MAX_NODES_IN_LL_CNT
 }
-
-/*
-    misc. methods
-*/
-
-fn encode_primary_key(buf: &mut Buf, pk: &PrimaryIndexKey) {
-    buf.push(pk.tag().d());
-    static EXEC: [unsafe fn(&mut Buf, &PrimaryIndexKey); 2] = [
-        |buf, pk| unsafe { buf.extend(pk.read_uint().to_le_bytes()) },
-        |buf, pk| unsafe {
-            let bin = pk.read_bin();
-            buf.extend(bin.len().u64_bytes_le());
-            buf.extend(bin);
-        },
-    ];
-    unsafe {
-        // UNSAFE(@ohsayan): tag map
-        assert!((pk.tag().d() / 2) < 2);
-        EXEC[(pk.tag().d() / 2) as usize](buf, pk);
-    }
-}
-
-fn encode_dc(buf: &mut Buf, dc: &Datacell) {
-    obj::encode_element(buf, dc)
-}
diff --git a/server/src/engine/txn/mod.rs b/server/src/engine/txn/mod.rs
index ae4a6cb0..fc4863e7 100644
--- a/server/src/engine/txn/mod.rs
+++ b/server/src/engine/txn/mod.rs
@@ -47,6 +47,7 @@ pub enum TransactionError {
     /// On restore, a certain item that was expected to match a certain value, has a different value
     OnRestoreDataConflictMismatch,
     SDSSError(SDSSError),
+    OutOfMemory,
 }
 
 direct_from! {
diff --git a/server/src/util/os.rs b/server/src/util/os.rs
index bfab9801..ffb5234c 100644
--- a/server/src/util/os.rs
+++ b/server/src/util/os.rs
@@ -52,6 +52,12 @@ impl From<std::io::Error> for SysIOError {
     }
 }
 
+impl From<std::io::ErrorKind> for SysIOError {
+    fn from(e: std::io::ErrorKind) -> Self {
+        Self(e.into())
+    }
+}
+
 #[cfg(test)]
 impl PartialEq for SysIOError {
     fn eq(&self, other: &Self) -> bool {

From 7dff706115f33874d4710abad4452a00e6f18cfa Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 7 Sep 2023 04:21:13 +0000
Subject: [PATCH 243/310] Reduce batch metadata size

---
 server/src/engine/core/model/delta.rs         |  10 +-
 .../engine/storage/v1/batch_jrnl/persist.rs   |  26 ++--
 .../engine/storage/v1/batch_jrnl/restore.rs   | 147 ++++++++++--------
 server/src/engine/storage/v1/rw.rs            |  13 ++
 4 files changed, 121 insertions(+), 75 deletions(-)

diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index d3c6720b..245d9542 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -197,8 +197,14 @@ impl DeltaState {
     fn __data_delta_step(&self) -> u64 {
         self.data_current_version.fetch_add(1, Ordering::AcqRel)
     }
-    pub fn __data_delta_queue(&self) -> &Queue<DataDelta> {
-        &self.data_deltas
+    pub fn __data_delta_dequeue(&self, g: &Guard) -> Option<DataDelta> {
+        match self.data_deltas.blocking_try_dequeue(g) {
+            Some(d) => {
+                self.data_deltas_size.fetch_sub(1, Ordering::Release);
+                Some(d)
+            }
+            None => None,
+        }
     }
 }
 
diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs
index 1dcf824e..b9e99ec0 100644
--- a/server/src/engine/storage/v1/batch_jrnl/persist.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs
@@ -83,7 +83,6 @@ impl<F: RawFileIOInterface> DataBatchPersistDriver<F> {
         // pin model
         let irm = model.intent_read_model();
         let schema_version = model.delta_state().schema_current_version();
-        let data_q = model.delta_state().__data_delta_queue();
         let g = pin();
         // init restore list
         let mut restore_list = Vec::new();
@@ -92,9 +91,14 @@ impl<F: RawFileIOInterface> DataBatchPersistDriver<F> {
         let mut inconsistent_reads = 0;
         let mut exec = || -> SDSSResult<()> {
             // write batch start
-            self.write_batch_start(observed_len, schema_version)?;
+            self.write_batch_start(
+                observed_len,
+                schema_version,
+                model.p_tag().tag_unique(),
+                irm.fields().len() - 1,
+            )?;
             while i < observed_len {
-                let delta = data_q.blocking_try_dequeue(&g).unwrap();
+                let delta = model.delta_state().__data_delta_dequeue(&g).unwrap();
                 restore_list.push(delta.clone()); // TODO(@ohsayan): avoid this
                 match delta.change() {
                     DataDeltaKind::Delete => {
@@ -140,18 +144,24 @@ impl<F: RawFileIOInterface> DataBatchPersistDriver<F> {
     }
     /// Write the batch start block:
     /// - Batch start magic
+    /// - Primary key type
     /// - Expected commit
     /// - Schema version
+    /// - Column count
     fn write_batch_start(
         &mut self,
         observed_len: usize,
         schema_version: DeltaVersion,
+        pk_tag: TagUnique,
+        col_cnt: usize,
     ) -> Result<(), SDSSError> {
-        self.f.unfsynced_write(&[MARKER_ACTUAL_BATCH_EVENT])?;
+        self.f
+            .unfsynced_write(&[MARKER_ACTUAL_BATCH_EVENT, pk_tag.d()])?;
         let observed_len_bytes = observed_len.u64_bytes_le();
         self.f.unfsynced_write(&observed_len_bytes)?;
         self.f
             .unfsynced_write(&schema_version.value_u64().to_le_bytes())?;
+        self.f.unfsynced_write(&col_cnt.u64_bytes_le())?;
         Ok(())
     }
     /// Append a summary of this batch
@@ -266,9 +276,6 @@ impl<F: RawFileIOInterface> DataBatchPersistDriver<F> {
         irm: &IRModel,
         row_data: &RowData,
     ) -> SDSSResult<()> {
-        // nasty hack; we need to avoid the pk
-        self.f
-            .unfsynced_write(&(row_data.fields().len()).to_le_bytes())?;
         for field_name in irm.fields().stseq_ord_key() {
             match row_data.fields().get(field_name) {
                 Some(cell) => {
@@ -280,9 +287,10 @@ impl<F: RawFileIOInterface> DataBatchPersistDriver<F> {
         }
         Ok(())
     }
+    /// Write the change type and txnid
     fn write_batch_item_common_row_data(&mut self, delta: &DataDelta) -> Result<(), SDSSError> {
-        let p1_dc_pk_ty = [delta.change().value_u8(), delta.row().d_key().tag().d()];
-        self.f.unfsynced_write(&p1_dc_pk_ty)?;
+        let change_type = [delta.change().value_u8()];
+        self.f.unfsynced_write(&change_type)?;
         let txn_id = delta.data_version().value_u64().to_le_bytes();
         self.f.unfsynced_write(&txn_id)?;
         Ok(())
diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs
index 7e54fe6d..213c7a1c 100644
--- a/server/src/engine/storage/v1/batch_jrnl/restore.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs
@@ -30,20 +30,17 @@ use {
     super::{
         MARKER_ACTUAL_BATCH_EVENT, MARKER_END_OF_BATCH, MARKER_RECOVERY_EVENT, RECOVERY_THRESHOLD,
     },
-    crate::{
-        engine::{
-            core::{index::PrimaryIndexKey, model::Model},
-            data::{
-                cell::Datacell,
-                tag::{CUTag, TagClass, TagUnique},
-            },
-            storage::v1::{
-                inf::PersistTypeDscr,
-                rw::{RawFileIOInterface, SDSSFileIO, SDSSFileTrackedReader},
-                SDSSError, SDSSResult,
-            },
+    crate::engine::{
+        core::{index::PrimaryIndexKey, model::Model},
+        data::{
+            cell::Datacell,
+            tag::{CUTag, TagClass, TagUnique},
+        },
+        storage::v1::{
+            inf::PersistTypeDscr,
+            rw::{RawFileIOInterface, SDSSFileIO, SDSSFileTrackedReader},
+            SDSSError, SDSSResult,
         },
-        util::copy_slice_to_array,
     },
     std::mem::ManuallyDrop,
 };
@@ -119,7 +116,9 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
             Self::apply_batch(model, batch)
         })
     }
-    pub fn read_all_batches(&mut self) -> SDSSResult<Vec<NormalBatch>> {
+    pub(in crate::engine::storage::v1) fn read_all_batches(
+        &mut self,
+    ) -> SDSSResult<Vec<NormalBatch>> {
         let mut all_batches = vec![];
         self.read_all_batches_and_for_each(|batch| {
             all_batches.push(batch);
@@ -212,8 +211,7 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
             }
         }
         // read actual commit
-        let mut actual_commit = [0; sizeof!(u64)];
-        self.f.read_into_buffer(&mut actual_commit)?;
+        let actual_commit = self.f.read_u64_le()?;
         // find actual checksum
         let actual_checksum = self.f.__reset_checksum();
         // find hardcoded checksum
@@ -224,7 +222,7 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
         // move file cursor ahead
         self.f.__cursor_ahead_by(sizeof!(u64));
         if actual_checksum == u64::from_le_bytes(hardcoded_checksum) {
-            Ok(u64::from_le_bytes(actual_commit))
+            Ok(actual_commit)
         } else {
             Err(SDSSError::DataBatchRestoreCorruptedBatch)
         }
@@ -249,15 +247,11 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
                 return Err(SDSSError::DataBatchRestoreCorruptedBatch);
             }
         }
-        // we're expecting a "good batch"
-        let mut batch_size_schema_version = [0; sizeof!(u64, 2)];
-        self.f.read_into_buffer(&mut batch_size_schema_version)?;
-        // we have the batch length
-        let batch_size = u64::from_le_bytes(copy_slice_to_array(&batch_size_schema_version[..8]));
-        let schema_version =
-            u64::from_le_bytes(copy_slice_to_array(&batch_size_schema_version[8..]));
+        // decode batch start block
+        let batch_start_block = self.read_start_batch_block()?;
+
         let mut processed_in_this_batch = 0;
-        while (processed_in_this_batch != batch_size) & !self.f.is_eof() {
+        while (processed_in_this_batch != batch_start_block.expected_commit()) & !self.f.is_eof() {
             // decode common row data
             let change_type = self.f.read_byte()?;
             // now decode event
@@ -266,15 +260,15 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
                     // the file tells us that we've reached the end of this batch; hmmm
                     return Ok(Batch::FinishedEarly(NormalBatch::new(
                         this_batch,
-                        schema_version,
+                        batch_start_block.schema_version(),
                     )));
                 }
                 normal_event => {
-                    let (pk_type, txnid) = self.read_normal_event_metadata()?;
+                    let txnid = self.f.read_u64_le()?;
                     match normal_event {
                         0 => {
                             // delete
-                            let pk = self.decode_primary_key(pk_type)?;
+                            let pk = self.decode_primary_key(batch_start_block.pk_tag())?;
                             this_batch.push(DecodedBatchEvent::new(
                                 txnid,
                                 pk,
@@ -285,18 +279,15 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
                         1 | 2 => {
                             // insert or update
                             // get pk
-                            let pk = self.decode_primary_key(pk_type)?;
-                            // get column count
-                            let mut column_count = [0; sizeof!(u64)];
-                            self.f.read_into_buffer(&mut column_count)?;
-                            let mut column_count = u64::from_le_bytes(column_count);
+                            let pk = self.decode_primary_key(batch_start_block.pk_tag())?;
                             // prepare row
                             let mut row = vec![];
-                            while column_count != 0 && !self.f.is_eof() {
+                            let mut this_col_cnt = batch_start_block.column_cnt();
+                            while this_col_cnt != 0 && !self.f.is_eof() {
                                 row.push(self.decode_cell()?);
-                                column_count -= 1;
+                                this_col_cnt -= 1;
                             }
-                            if column_count != 0 {
+                            if this_col_cnt != 0 {
                                 return Err(SDSSError::DataBatchRestoreCorruptedEntry);
                             }
                             if change_type == 1 {
@@ -321,14 +312,10 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
                 }
             }
         }
-        Ok(Batch::Normal(NormalBatch::new(this_batch, schema_version)))
-    }
-    fn read_normal_event_metadata(&mut self) -> Result<(u8, u64), SDSSError> {
-        let pk_type = self.f.read_byte()?;
-        let mut txnid = [0; sizeof!(u64)];
-        self.f.read_into_buffer(&mut txnid)?;
-        let txnid = u64::from_le_bytes(txnid);
-        Ok((pk_type, txnid))
+        Ok(Batch::Normal(NormalBatch::new(
+            this_batch,
+            batch_start_block.schema_version(),
+        )))
     }
     fn attempt_recover_data_batch(&mut self) -> SDSSResult<()> {
         let mut max_threshold = RECOVERY_THRESHOLD;
@@ -340,6 +327,49 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
         }
         Err(SDSSError::DataBatchRestoreCorruptedBatch)
     }
+    fn read_start_batch_block(&mut self) -> SDSSResult<BatchStartBlock> {
+        let pk_tag = self.f.read_byte()?;
+        let expected_commit = self.f.read_u64_le()?;
+        let schema_version = self.f.read_u64_le()?;
+        let column_cnt = self.f.read_u64_le()?;
+        Ok(BatchStartBlock::new(
+            pk_tag,
+            expected_commit,
+            schema_version,
+            column_cnt,
+        ))
+    }
+}
+
+#[derive(Debug, PartialEq)]
+struct BatchStartBlock {
+    pk_tag: u8,
+    expected_commit: u64,
+    schema_version: u64,
+    column_cnt: u64,
+}
+
+impl BatchStartBlock {
+    const fn new(pk_tag: u8, expected_commit: u64, schema_version: u64, column_cnt: u64) -> Self {
+        Self {
+            pk_tag,
+            expected_commit,
+            schema_version,
+            column_cnt,
+        }
+    }
+    fn pk_tag(&self) -> u8 {
+        self.pk_tag
+    }
+    fn expected_commit(&self) -> u64 {
+        self.expected_commit
+    }
+    fn schema_version(&self) -> u64 {
+        self.schema_version
+    }
+    fn column_cnt(&self) -> u64 {
+        self.column_cnt
+    }
 }
 
 impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
@@ -349,17 +379,15 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
         };
         Ok(match pk_type {
             TagUnique::SignedInt | TagUnique::UnsignedInt => {
-                let mut chunk = [0; sizeof!(u64)];
-                self.f.read_into_buffer(&mut chunk)?;
+                let qw = self.f.read_u64_le()?;
                 unsafe {
                     // UNSAFE(@ohsayan): +tagck
-                    PrimaryIndexKey::new_from_qw(pk_type, u64::from_le_bytes(chunk))
+                    PrimaryIndexKey::new_from_qw(pk_type, qw)
                 }
             }
             TagUnique::Str | TagUnique::Bin => {
-                let mut len = [0; sizeof!(u64)];
-                self.f.read_into_buffer(&mut len)?;
-                let mut data = vec![0; u64::from_le_bytes(len) as usize];
+                let len = self.f.read_u64_le()?;
+                let mut data = vec![0; len as usize];
                 self.f.read_into_buffer(&mut data)?;
                 if pk_type == TagUnique::Str {
                     if core::str::from_utf8(&data).is_err() {
@@ -369,11 +397,7 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
                 unsafe {
                     // UNSAFE(@ohsayan): +tagck +verityck
                     let mut md = ManuallyDrop::new(data);
-                    PrimaryIndexKey::new_from_dual(
-                        pk_type,
-                        u64::from_le_bytes(len),
-                        md.as_mut_ptr() as usize,
-                    )
+                    PrimaryIndexKey::new_from_dual(pk_type, len, md.as_mut_ptr() as usize)
                 }
             }
             _ => unsafe {
@@ -397,18 +421,15 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
                 Datacell::new_bool(bool == 1)
             }
             PersistTypeDscr::UnsignedInt | PersistTypeDscr::SignedInt | PersistTypeDscr::Float => {
-                let mut block = [0; sizeof!(u64)];
-                self.f.read_into_buffer(&mut block)?;
+                let qw = self.f.read_u64_le()?;
                 unsafe {
                     // UNSAFE(@ohsayan): choosing the correct type and tag
                     let tc = TagClass::from_raw(cell_type.value_u8() - 1);
-                    Datacell::new_qw(u64::from_le_bytes(block), CUTag::new(tc, tc.tag_unique()))
+                    Datacell::new_qw(qw, CUTag::new(tc, tc.tag_unique()))
                 }
             }
             PersistTypeDscr::Str | PersistTypeDscr::Bin => {
-                let mut len_block = [0; sizeof!(u64)];
-                self.f.read_into_buffer(&mut len_block)?;
-                let len = u64::from_le_bytes(len_block) as usize;
+                let len = self.f.read_u64_le()? as usize;
                 let mut data = vec![0; len];
                 self.f.read_into_buffer(&mut data)?;
                 unsafe {
@@ -424,9 +445,7 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
                 }
             }
             PersistTypeDscr::List => {
-                let mut len_block = [0; sizeof!(u64)];
-                self.f.read_into_buffer(&mut len_block)?;
-                let len = u64::from_le_bytes(len_block);
+                let len = self.f.read_u64_le()?;
                 let mut list = Vec::new();
                 while !self.f.is_eof() && list.len() as u64 != len {
                     list.push(self.decode_cell()?);
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index a21b2b5e..8c6f87dd 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -251,6 +251,19 @@ impl<F: RawFileIOInterface> SDSSFileTrackedReader<F> {
     pub fn __cursor_ahead_by(&mut self, sizeof: usize) {
         self.pos += sizeof as u64;
     }
+    pub fn read_block<const N: usize>(&mut self) -> SDSSResult<[u8; N]> {
+        if !self.has_left(N as _) {
+            return Err(SDSSError::IoError(SysIOError::from(
+                std::io::ErrorKind::InvalidInput,
+            )));
+        }
+        let mut buf = [0; N];
+        self.read_into_buffer(&mut buf)?;
+        Ok(buf)
+    }
+    pub fn read_u64_le(&mut self) -> SDSSResult<u64> {
+        Ok(u64::from_le_bytes(self.read_block()?))
+    }
 }
 
 #[derive(Debug)]

From f4d2adc4e88c669617eca55602b820f8fafcbf93 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 7 Sep 2023 04:44:33 +0000
Subject: [PATCH 244/310] Add tests for skewed and unskewed deltas

---
 server/src/engine/storage/v1/tests/batch.rs | 121 ++++++++++++++++----
 1 file changed, 96 insertions(+), 25 deletions(-)

diff --git a/server/src/engine/storage/v1/tests/batch.rs b/server/src/engine/storage/v1/tests/batch.rs
index 7db00240..6682fe55 100644
--- a/server/src/engine/storage/v1/tests/batch.rs
+++ b/server/src/engine/storage/v1/tests/batch.rs
@@ -79,7 +79,7 @@ fn open_batch_data(fpath: &str, mdl: &Model) -> DataBatchPersistDriver<VirtualFS
 fn new_delta(
     schema: u64,
     txnid: u64,
-    pk: Datacell,
+    pk: impl Into<Datacell>,
     data: DcFieldIndex,
     change: DataDeltaKind,
 ) -> DataDelta {
@@ -105,8 +105,99 @@ fn new_delta_with_row(schema: u64, txnid: u64, row: Row, change: DataDeltaKind)
     )
 }
 
+fn flush_deltas_and_re_read<const N: usize>(
+    mdl: &Model,
+    dt: [DataDelta; N],
+    fname: &str,
+) -> Vec<NormalBatch> {
+    // delta queue
+    let g = pin();
+    for delta in dt {
+        mdl.delta_state().append_new_data_delta(delta, &g);
+    }
+    let file = open_file(fname).into_created().unwrap();
+    {
+        let mut persist_driver = DataBatchPersistDriver::new(file, true).unwrap();
+        persist_driver.write_new_batch(&mdl, N).unwrap();
+        persist_driver.close().unwrap();
+    }
+    let mut restore_driver =
+        DataBatchRestoreDriver::new(open_file(fname).into_existing().unwrap().0).unwrap();
+    let batch = restore_driver.read_all_batches().unwrap();
+    batch
+}
+
 #[test]
-fn deltas_only_insert() {
+fn unskewed_delta() {
+    let uuid = Uuid::new();
+    let mdl = Model::new_restore(
+        uuid,
+        "username".into(),
+        TagSelector::Str.into_full(),
+        into_dict!(
+            "username" => Field::new([Layer::str()].into(), false),
+            "password" => Field::new([Layer::bin()].into(), false)
+        ),
+    );
+    let deltas = [
+        new_delta(
+            0,
+            0,
+            "sayan",
+            into_dict!("password" => Datacell::new_bin("37ae4b773a9fc7a20164eb16".as_bytes().into())),
+            DataDeltaKind::Insert,
+        ),
+        new_delta(
+            0,
+            1,
+            "badguy",
+            into_dict!("password" => Datacell::new_bin("5fe3cbdc470b667cb1ba288a".as_bytes().into())),
+            DataDeltaKind::Insert,
+        ),
+        new_delta(
+            0,
+            2,
+            "doggo",
+            into_dict!("password" => Datacell::new_bin("c80403f9d0ae4d5d0e829dd0".as_bytes().into())),
+            DataDeltaKind::Insert,
+        ),
+        new_delta(0, 3, "badguy", into_dict!(), DataDeltaKind::Delete),
+    ];
+    let batches = flush_deltas_and_re_read(&mdl, deltas, "unskewed_delta.db-btlog");
+    assert_eq!(
+        batches,
+        vec![NormalBatch::new(
+            vec![
+                DecodedBatchEvent::new(
+                    0,
+                    pkey("sayan"),
+                    DecodedBatchEventKind::Insert(vec![Datacell::new_bin(
+                        b"37ae4b773a9fc7a20164eb16".to_vec().into_boxed_slice()
+                    )])
+                ),
+                DecodedBatchEvent::new(
+                    1,
+                    pkey("badguy"),
+                    DecodedBatchEventKind::Insert(vec![Datacell::new_bin(
+                        b"5fe3cbdc470b667cb1ba288a".to_vec().into_boxed_slice()
+                    )])
+                ),
+                DecodedBatchEvent::new(
+                    2,
+                    pkey("doggo"),
+                    DecodedBatchEventKind::Insert(vec![Datacell::new_bin(
+                        b"c80403f9d0ae4d5d0e829dd0".to_vec().into_boxed_slice()
+                    )])
+                ),
+                DecodedBatchEvent::new(3, pkey("badguy"), DecodedBatchEventKind::Delete)
+            ],
+            0
+        )]
+    )
+}
+
+#[test]
+fn skewed_delta() {
     // prepare model definition
     let uuid = Uuid::new();
     let mdl = Model::new_restore(
@@ -139,7 +230,7 @@ fn deltas_only_insert() {
         new_delta(
             0,
             1,
-            Datacell::new_str("good cat".into()),
+            "good cat",
             into_dict!("is_good" => Datacell::new_bool(true), "magical" => Datacell::new_bool(false)),
             DataDeltaKind::Insert,
         ),
@@ -147,34 +238,14 @@ fn deltas_only_insert() {
         new_delta(
             0,
             2,
-            Datacell::new_str("bad cat".into()),
+            "bad cat",
             into_dict!("is_good" => Datacell::new_bool(false), "magical" => Datacell::new_bool(false)),
             DataDeltaKind::Insert,
         ),
         // update catname: Schrödinger's cat, is_good: true, magical: true
         new_delta_with_row(0, 3, row.clone(), DataDeltaKind::Update),
     ];
-    // delta queue
-    let g = pin();
-    for delta in deltas.clone() {
-        mdl.delta_state().append_new_data_delta(delta, &g);
-    }
-    let file = open_file("deltas_only_insert.db-btlog")
-        .into_created()
-        .unwrap();
-    {
-        let mut persist_driver = DataBatchPersistDriver::new(file, true).unwrap();
-        persist_driver.write_new_batch(&mdl, deltas.len()).unwrap();
-        persist_driver.close().unwrap();
-    }
-    let mut restore_driver = DataBatchRestoreDriver::new(
-        open_file("deltas_only_insert.db-btlog")
-            .into_existing()
-            .unwrap()
-            .0,
-    )
-    .unwrap();
-    let batch = restore_driver.read_all_batches().unwrap();
+    let batch = flush_deltas_and_re_read(&mdl, deltas, "skewed_delta.db-btlog");
     assert_eq!(
         batch,
         vec![NormalBatch::new(

From cbac4e67394bcc028811ffc5a5713169bf34629e Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 7 Sep 2023 07:12:02 +0000
Subject: [PATCH 245/310] Add tests for skewed and shuffled deltas

---
 server/src/engine/core/index/row.rs           |  25 +++
 server/src/engine/core/model/delta.rs         |   3 +-
 server/src/engine/core/tests/ddl_model/alt.rs |   4 +-
 .../engine/storage/v1/batch_jrnl/restore.rs   | 114 +++++++++++-
 server/src/engine/storage/v1/tests/batch.rs   | 168 +++++++++++++++---
 server/src/engine/txn/gns/model.rs            |  14 ++
 server/src/util/test_utils.rs                 |   6 +
 7 files changed, 295 insertions(+), 39 deletions(-)

diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
index f47861a8..5729bfaf 100644
--- a/server/src/engine/core/index/row.rs
+++ b/server/src/engine/core/index/row.rs
@@ -53,6 +53,8 @@ pub struct RowData {
     fields: DcFieldIndex,
     txn_revised_data: DeltaVersion,
     txn_revised_schema_version: DeltaVersion,
+    // pretty useless from an operational POV; only used during restore
+    restore_txn_id: DeltaVersion,
 }
 
 impl RowData {
@@ -68,6 +70,12 @@ impl RowData {
     pub fn get_txn_revised(&self) -> DeltaVersion {
         self.txn_revised_data
     }
+    pub fn set_restored_txn_revised(&mut self, new: DeltaVersion) {
+        self.restore_txn_id = new;
+    }
+    pub fn get_restored_txn_revised(&self) -> DeltaVersion {
+        self.restore_txn_id
+    }
 }
 
 impl TreeElement for Row {
@@ -99,6 +107,21 @@ impl Row {
         data: DcFieldIndex,
         schema_version: DeltaVersion,
         txn_revised_data: DeltaVersion,
+    ) -> Self {
+        Self::new_restored(
+            pk,
+            data,
+            schema_version,
+            txn_revised_data,
+            DeltaVersion::__new(0),
+        )
+    }
+    pub fn new_restored(
+        pk: PrimaryIndexKey,
+        data: DcFieldIndex,
+        schema_version: DeltaVersion,
+        txn_revised_data: DeltaVersion,
+        restore_txn_id: DeltaVersion,
     ) -> Self {
         Self {
             __genesis_schema_version: schema_version,
@@ -109,6 +132,8 @@ impl Row {
                     fields: data,
                     txn_revised_schema_version: schema_version,
                     txn_revised_data,
+                    // pretty useless here
+                    restore_txn_id,
                 }))
             },
         }
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index 245d9542..78b532c1 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -265,8 +265,7 @@ impl DeltaVersion {
     pub const fn genesis() -> Self {
         Self(0)
     }
-    #[cfg(test)]
-    pub fn test_new(v: u64) -> Self {
+    pub const  fn __new(v: u64) -> Self {
         Self(v)
     }
     fn step(&self) -> Self {
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index abcf5d12..bcbb16a4 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -384,7 +384,7 @@ mod exec {
                 );
                 assert_eq!(
                     model.delta_state().schema_current_version(),
-                    DeltaVersion::test_new(2)
+                    DeltaVersion::__new(2)
                 );
             },
         )
@@ -411,7 +411,7 @@ mod exec {
                 );
                 assert_eq!(
                     mdl.delta_state().schema_current_version(),
-                    DeltaVersion::test_new(4)
+                    DeltaVersion::__new(4)
                 );
             }
         ).unwrap();
diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs
index 213c7a1c..89d452a7 100644
--- a/server/src/engine/storage/v1/batch_jrnl/restore.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs
@@ -24,30 +24,37 @@
  *
 */
 
-use super::{MARKER_BATCH_CLOSED, MARKER_BATCH_REOPEN};
-
 use {
     super::{
-        MARKER_ACTUAL_BATCH_EVENT, MARKER_END_OF_BATCH, MARKER_RECOVERY_EVENT, RECOVERY_THRESHOLD,
+        MARKER_ACTUAL_BATCH_EVENT, MARKER_BATCH_CLOSED, MARKER_BATCH_REOPEN, MARKER_END_OF_BATCH,
+        MARKER_RECOVERY_EVENT, RECOVERY_THRESHOLD,
     },
     crate::engine::{
-        core::{index::PrimaryIndexKey, model::Model},
+        core::{
+            index::{DcFieldIndex, PrimaryIndexKey, Row},
+            model::{delta::DeltaVersion, Model},
+        },
         data::{
             cell::Datacell,
             tag::{CUTag, TagClass, TagUnique},
         },
+        idx::{MTIndex, STIndex, STIndexSeq},
         storage::v1::{
             inf::PersistTypeDscr,
             rw::{RawFileIOInterface, SDSSFileIO, SDSSFileTrackedReader},
             SDSSError, SDSSResult,
         },
     },
-    std::mem::ManuallyDrop,
+    crossbeam_epoch::pin,
+    std::{
+        collections::{hash_map::Entry as HMEntry, HashMap},
+        mem::ManuallyDrop,
+    },
 };
 
 #[derive(Debug, PartialEq)]
 pub(in crate::engine::storage::v1) struct DecodedBatchEvent {
-    txn_id: u64,
+    txn_id: DeltaVersion,
     pk: PrimaryIndexKey,
     kind: DecodedBatchEventKind,
 }
@@ -58,7 +65,11 @@ impl DecodedBatchEvent {
         pk: PrimaryIndexKey,
         kind: DecodedBatchEventKind,
     ) -> Self {
-        Self { txn_id, pk, kind }
+        Self {
+            txn_id: DeltaVersion::__new(txn_id),
+            pk,
+            kind,
+        }
     }
 }
 
@@ -196,8 +207,93 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
 }
 
 impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
-    fn apply_batch(_: &Model, _: NormalBatch) -> SDSSResult<()> {
-        todo!()
+    fn apply_batch(
+        m: &Model,
+        NormalBatch {
+            events,
+            schema_version,
+        }: NormalBatch,
+    ) -> SDSSResult<()> {
+        // NOTE(@ohsayan): current complexity is O(n) which is good enough (in the future I might revise this to a fancier impl)
+        // pin model
+        let irm = m.intent_read_model();
+        let g = pin();
+        let mut pending_delete = HashMap::new();
+        let p_index = m.primary_index().__raw_index();
+        // scan rows
+        for DecodedBatchEvent { txn_id, pk, kind } in events {
+            match kind {
+                DecodedBatchEventKind::Insert(new_row) | DecodedBatchEventKind::Update(new_row) => {
+                    // this is more like a "newrow"
+                    match p_index.mt_get_element(&pk, &g) {
+                        Some(row) if row.d_data().read().get_restored_txn_revised() > txn_id => {
+                            // skewed
+                            // resolve deltas if any
+                            let _ = row.resolve_schema_deltas_and_freeze(m.delta_state());
+                            continue;
+                        }
+                        Some(_) | None => {
+                            // new row (logically)
+                            let _ = p_index.mt_delete(&pk, &g);
+                            let mut data = DcFieldIndex::default();
+                            for (field_name, new_data) in irm
+                                .fields()
+                                .stseq_ord_key()
+                                .filter(|key| key.as_ref() != m.p_key())
+                                .zip(new_row)
+                            {
+                                data.st_insert(field_name.clone(), new_data);
+                            }
+                            let row = Row::new_restored(
+                                pk,
+                                data,
+                                DeltaVersion::__new(schema_version),
+                                DeltaVersion::__new(0),
+                                txn_id,
+                            );
+                            // resolve any deltas
+                            let _ = row.resolve_schema_deltas_and_freeze(m.delta_state());
+                            // put it back in (lol); blame @ohsayan for this joke
+                            p_index.mt_insert(row, &g);
+                        }
+                    }
+                }
+                DecodedBatchEventKind::Delete => {
+                    match pending_delete.entry(pk) {
+                        HMEntry::Occupied(mut existing_delete) => {
+                            if *existing_delete.get() > txn_id {
+                                // the existing delete "happened after" our delete, so it takes precedence
+                                continue;
+                            }
+                            // the existing delete happened before our delete, so our delete takes precedence
+                            // we have a newer delete for the same key
+                            *existing_delete.get_mut() = txn_id;
+                        }
+                        HMEntry::Vacant(new) => {
+                            // we never deleted this
+                            new.insert(txn_id);
+                        }
+                    }
+                }
+            }
+        }
+        for (pk, txn_id) in pending_delete {
+            match p_index.mt_get(&pk, &g) {
+                Some(row) => {
+                    if row.read().get_restored_txn_revised() > txn_id {
+                        // our delete "happened before" this row was inserted
+                        continue;
+                    }
+                    // yup, go ahead and chuck it
+                    let _ = p_index.mt_delete(&pk, &g);
+                }
+                None => {
+                    // since we never delete rows until here, this is quite impossible
+                    unreachable!()
+                }
+            }
+        }
+        Ok(())
     }
 }
 
diff --git a/server/src/engine/storage/v1/tests/batch.rs b/server/src/engine/storage/v1/tests/batch.rs
index 6682fe55..cd12b639 100644
--- a/server/src/engine/storage/v1/tests/batch.rs
+++ b/server/src/engine/storage/v1/tests/batch.rs
@@ -25,24 +25,28 @@
 */
 
 use {
-    crate::engine::{
-        core::{
-            index::{DcFieldIndex, PrimaryIndexKey, Row},
-            model::{
-                delta::{DataDelta, DataDeltaKind, DeltaVersion},
-                Field, Layer, Model,
+    crate::{
+        engine::{
+            core::{
+                index::{DcFieldIndex, PrimaryIndexKey, Row},
+                model::{
+                    delta::{DataDelta, DataDeltaKind, DeltaVersion},
+                    Field, Layer, Model,
+                },
             },
-        },
-        data::{cell::Datacell, tag::TagSelector, uuid::Uuid},
-        storage::v1::{
-            batch_jrnl::{
-                DataBatchPersistDriver, DataBatchRestoreDriver, DecodedBatchEvent,
-                DecodedBatchEventKind, NormalBatch,
+            data::{cell::Datacell, tag::TagSelector, uuid::Uuid},
+            idx::MTIndex,
+            storage::v1::{
+                batch_jrnl::{
+                    DataBatchPersistDriver, DataBatchRestoreDriver, DecodedBatchEvent,
+                    DecodedBatchEventKind, NormalBatch,
+                },
+                header_meta::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
+                rw::{FileOpen, SDSSFileIO},
+                test_util::VirtualFS,
             },
-            header_meta::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
-            rw::{FileOpen, SDSSFileIO},
-            test_util::VirtualFS,
         },
+        util::test_utils,
     },
     crossbeam_epoch::pin,
 };
@@ -89,8 +93,8 @@ fn new_delta(
         Row::new(
             pkey(pk),
             data,
-            DeltaVersion::test_new(schema),
-            DeltaVersion::test_new(txnid),
+            DeltaVersion::__new(schema),
+            DeltaVersion::__new(txnid),
         ),
         change,
     )
@@ -98,8 +102,8 @@ fn new_delta(
 
 fn new_delta_with_row(schema: u64, txnid: u64, row: Row, change: DataDeltaKind) -> DataDelta {
     DataDelta::new(
-        DeltaVersion::test_new(schema),
-        DeltaVersion::test_new(txnid),
+        DeltaVersion::__new(schema),
+        DeltaVersion::__new(txnid),
         row,
         change,
     )
@@ -110,6 +114,16 @@ fn flush_deltas_and_re_read<const N: usize>(
     dt: [DataDelta; N],
     fname: &str,
 ) -> Vec<NormalBatch> {
+    let mut restore_driver = flush_batches_and_return_restore_driver(dt, mdl, fname);
+    let batch = restore_driver.read_all_batches().unwrap();
+    batch
+}
+
+fn flush_batches_and_return_restore_driver<const N: usize>(
+    dt: [DataDelta; N],
+    mdl: &Model,
+    fname: &str,
+) -> DataBatchRestoreDriver<VirtualFS> {
     // delta queue
     let g = pin();
     for delta in dt {
@@ -121,10 +135,25 @@ fn flush_deltas_and_re_read<const N: usize>(
         persist_driver.write_new_batch(&mdl, N).unwrap();
         persist_driver.close().unwrap();
     }
-    let mut restore_driver =
-        DataBatchRestoreDriver::new(open_file(fname).into_existing().unwrap().0).unwrap();
-    let batch = restore_driver.read_all_batches().unwrap();
-    batch
+    DataBatchRestoreDriver::new(open_file(fname).into_existing().unwrap().0).unwrap()
+}
+
+#[test]
+fn empty_multi_open_reopen() {
+    let uuid = Uuid::new();
+    let mdl = Model::new_restore(
+        uuid,
+        "username".into(),
+        TagSelector::Str.into_full(),
+        into_dict!(
+            "username" => Field::new([Layer::str()].into(), false),
+            "password" => Field::new([Layer::bin()].into(), false)
+        ),
+    );
+    for _ in 0..100 {
+        let writer = open_batch_data("empty_multi_open_reopen.db-btlog", &mdl);
+        writer.close().unwrap();
+    }
 }
 
 #[test]
@@ -213,13 +242,13 @@ fn skewed_delta() {
     let row = Row::new(
         pkey("Schrödinger's cat"),
         into_dict!("is_good" => Datacell::new_bool(true), "magical" => Datacell::new_bool(false)),
-        DeltaVersion::test_new(0),
-        DeltaVersion::test_new(2),
+        DeltaVersion::__new(0),
+        DeltaVersion::__new(2),
     );
     {
         // update the row
         let mut wl = row.d_data().write();
-        wl.set_txn_revised(DeltaVersion::test_new(3));
+        wl.set_txn_revised(DeltaVersion::__new(3));
         *wl.fields_mut().get_mut("magical").unwrap() = Datacell::new_bool(true);
     }
     // prepare deltas
@@ -279,3 +308,90 @@ fn skewed_delta() {
         )]
     )
 }
+
+#[test]
+fn skewed_shuffled_persist_restore() {
+    let uuid = Uuid::new();
+    let model = Model::new_restore(
+        uuid,
+        "username".into(),
+        TagSelector::Str.into_full(),
+        into_dict!("username" => Field::new([Layer::str()].into(), false), "password" => Field::new([Layer::str()].into(), false)),
+    );
+    let mongobongo = Row::new(
+        pkey("mongobongo"),
+        into_dict!("password" => "dumbo"),
+        DeltaVersion::__new(0),
+        DeltaVersion::__new(4),
+    );
+    let rds = Row::new(
+        pkey("rds"),
+        into_dict!("password" => "snail"),
+        DeltaVersion::__new(0),
+        DeltaVersion::__new(5),
+    );
+    let deltas = [
+        new_delta(
+            0,
+            0,
+            "sayan",
+            into_dict!("password" => "pwd123456"),
+            DataDeltaKind::Insert,
+        ),
+        new_delta(
+            0,
+            1,
+            "joseph",
+            into_dict!("password" => "pwd234567"),
+            DataDeltaKind::Insert,
+        ),
+        new_delta(
+            0,
+            2,
+            "haley",
+            into_dict!("password" => "pwd345678"),
+            DataDeltaKind::Insert,
+        ),
+        new_delta(
+            0,
+            3,
+            "charlotte",
+            into_dict!("password" => "pwd456789"),
+            DataDeltaKind::Insert,
+        ),
+        new_delta_with_row(0, 4, mongobongo.clone(), DataDeltaKind::Insert),
+        new_delta_with_row(0, 5, rds.clone(), DataDeltaKind::Insert),
+        new_delta_with_row(0, 6, mongobongo.clone(), DataDeltaKind::Delete),
+        new_delta_with_row(0, 7, rds.clone(), DataDeltaKind::Delete),
+    ];
+    for i in 0..deltas.len() {
+        // prepare pretest
+        let fname = format!("skewed_shuffled_persist_restore_round{i}.db-btlog");
+        let mut deltas = deltas.clone();
+        let mut randomizer = test_utils::randomizer();
+        test_utils::shuffle_slice(&mut deltas, &mut randomizer);
+        // restore
+        let mut restore_driver = flush_batches_and_return_restore_driver(deltas, &model, &fname);
+        restore_driver.read_data_batch_into_model(&model).unwrap();
+    }
+    let g = pin();
+    for delta in &deltas[..4] {
+        let row = model
+            .primary_index()
+            .__raw_index()
+            .mt_get(delta.row().d_key(), &g)
+            .unwrap();
+        let row_data = row.read();
+        assert_eq!(row_data.fields().len(), 1);
+        assert_eq!(
+            row_data.fields().get("password").unwrap(),
+            delta
+                .row()
+                .d_data()
+                .read()
+                .fields()
+                .get("password")
+                .unwrap()
+        );
+    }
+}
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index 7bd03090..0fe37564 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -415,6 +415,13 @@ impl<'a> GNSEvent for AlterModelAddTxn<'a> {
                     return Err(TransactionError::OnRestoreDataConflictMismatch);
                 }
             }
+            // TODO(@ohsayan): avoid double iteration
+            // publish deltas
+            for field_name in new_fields.stseq_ord_key() {
+                model
+                    .delta_state()
+                    .schema_append_unresolved_field_add(field_name);
+            }
             Ok(())
         })
     }
@@ -527,6 +534,13 @@ impl<'a> GNSEvent for AlterModelRemoveTxn<'a> {
                     }
                 }
             }
+            // TODO(@ohsayan): avoid double iteration
+            // publish deltas
+            for field_name in removed_fields.iter() {
+                model
+                    .delta_state()
+                    .schema_append_unresolved_field_rem(field_name);
+            }
             Ok(())
         })
     }
diff --git a/server/src/util/test_utils.rs b/server/src/util/test_utils.rs
index 01d80524..59beb02f 100644
--- a/server/src/util/test_utils.rs
+++ b/server/src/util/test_utils.rs
@@ -26,6 +26,8 @@
 
 use std::io::Read;
 
+use rand::seq::SliceRandom;
+
 use {
     rand::{
         distributions::{uniform::SampleUniform, Alphanumeric},
@@ -38,6 +40,10 @@ use {
     },
 };
 
+pub fn shuffle_slice<T>(slice: &mut [T], rng: &mut impl Rng) {
+    slice.shuffle(rng)
+}
+
 pub fn wait_for_key(msg: &str) {
     use std::io::{self, Write};
     print!("{msg}");

From e309e35b63ae840fceaaddd789e3db4cb7a3baea Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 8 Sep 2023 19:12:33 +0000
Subject: [PATCH 246/310] Make the FS definition more generic

---
 .../engine/storage/v1/batch_jrnl/persist.rs   |  12 +-
 .../engine/storage/v1/batch_jrnl/restore.rs   |  14 +-
 server/src/engine/storage/v1/journal.rs       |  46 +-
 server/src/engine/storage/v1/memfs.rs         | 502 ++++++++++++++++++
 server/src/engine/storage/v1/mod.rs           |   6 +-
 server/src/engine/storage/v1/rw.rs            | 255 ++++++---
 server/src/engine/storage/v1/test_util.rs     | 229 --------
 server/src/engine/storage/v1/tests.rs         |   2 +-
 server/src/engine/storage/v1/tests/batch.rs   |   2 +-
 server/src/engine/storage/v1/tests/rw.rs      |  30 +-
 server/src/engine/txn/gns/mod.rs              |  18 +-
 11 files changed, 736 insertions(+), 380 deletions(-)
 create mode 100644 server/src/engine/storage/v1/memfs.rs
 delete mode 100644 server/src/engine/storage/v1/test_util.rs

diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs
index b9e99ec0..56499f92 100644
--- a/server/src/engine/storage/v1/batch_jrnl/persist.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs
@@ -45,7 +45,7 @@ use {
             idx::STIndexSeq,
             storage::v1::{
                 inf::PersistTypeDscr,
-                rw::{RawFileIOInterface, SDSSFileIO, SDSSFileTrackedWriter},
+                rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedWriter},
                 SDSSError, SDSSResult,
             },
         },
@@ -54,12 +54,12 @@ use {
     crossbeam_epoch::pin,
 };
 
-pub struct DataBatchPersistDriver<F> {
-    f: SDSSFileTrackedWriter<F>,
+pub struct DataBatchPersistDriver<Fs: RawFSInterface> {
+    f: SDSSFileTrackedWriter<Fs>,
 }
 
-impl<F: RawFileIOInterface> DataBatchPersistDriver<F> {
-    pub fn new(mut file: SDSSFileIO<F>, is_new: bool) -> SDSSResult<Self> {
+impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
+    pub fn new(mut file: SDSSFileIO<Fs>, is_new: bool) -> SDSSResult<Self> {
         if !is_new {
             file.fsynced_write(&[MARKER_BATCH_REOPEN])?;
         }
@@ -193,7 +193,7 @@ impl<F: RawFileIOInterface> DataBatchPersistDriver<F> {
     }
 }
 
-impl<F: RawFileIOInterface> DataBatchPersistDriver<F> {
+impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
     /// encode the primary key only. this means NO TAG is encoded.
     fn encode_pk_only(&mut self, pk: &PrimaryIndexKey) -> SDSSResult<()> {
         let buf = &mut self.f;
diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs
index 89d452a7..f7bf0dc1 100644
--- a/server/src/engine/storage/v1/batch_jrnl/restore.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs
@@ -41,7 +41,7 @@ use {
         idx::{MTIndex, STIndex, STIndexSeq},
         storage::v1::{
             inf::PersistTypeDscr,
-            rw::{RawFileIOInterface, SDSSFileIO, SDSSFileTrackedReader},
+            rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedReader},
             SDSSError, SDSSResult,
         },
     },
@@ -105,11 +105,11 @@ enum Batch {
     BatchClosed,
 }
 
-pub struct DataBatchRestoreDriver<F> {
+pub struct DataBatchRestoreDriver<F: RawFSInterface> {
     f: SDSSFileTrackedReader<F>,
 }
 
-impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
+impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
     pub fn new(f: SDSSFileIO<F>) -> SDSSResult<Self> {
         Ok(Self {
             f: SDSSFileTrackedReader::new(f)?,
@@ -139,7 +139,7 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
     }
 }
 
-impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
+impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
     fn read_all_batches_and_for_each(
         &mut self,
         mut f: impl FnMut(NormalBatch) -> SDSSResult<()>,
@@ -206,7 +206,7 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
     }
 }
 
-impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
+impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
     fn apply_batch(
         m: &Model,
         NormalBatch {
@@ -297,7 +297,7 @@ impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
     }
 }
 
-impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
+impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
     fn read_batch_summary(&mut self, finished_early: bool) -> SDSSResult<u64> {
         if !finished_early {
             // we must read the batch termination signature
@@ -468,7 +468,7 @@ impl BatchStartBlock {
     }
 }
 
-impl<F: RawFileIOInterface> DataBatchRestoreDriver<F> {
+impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
     fn decode_primary_key(&mut self, pk_type: u8) -> SDSSResult<PrimaryIndexKey> {
         let Some(pk_type) = TagUnique::try_from_raw(pk_type) else {
             return Err(SDSSError::DataBatchRestoreCorruptedEntry);
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index 84ea217c..0a39daff 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -41,10 +41,12 @@
   - FIXME(@ohsayan): we will probably (naively) need to dynamically reposition the cursor in case the metadata is corrupted as well
 */
 
+use super::rw::RawFSInterface;
+
 use {
     super::{
         header_impl::{FileSpecifierVersion, HostRunMode, SDSSHeaderRaw},
-        rw::{FileOpen, RawFileIOInterface, SDSSFileIO},
+        rw::{FileOpen, SDSSFileIO},
         SDSSError, SDSSResult,
     },
     crate::{
@@ -67,9 +69,9 @@ pub fn null_journal<TA: JournalAdapter>(
     host_run_mode: HostRunMode,
     host_startup_counter: u64,
     _: &TA::GlobalState,
-) -> JournalWriter<super::rw::NullZero, TA> {
+) -> JournalWriter<super::memfs::NullFS, TA> {
     let FileOpen::Created(journal) =
-        SDSSFileIO::<super::rw::NullZero>::open_or_create_perm_rw::<false>(
+        SDSSFileIO::<super::memfs::NullFS>::open_or_create_perm_rw::<false>(
             log_file_name,
             FileScope::Journal,
             log_kind,
@@ -85,7 +87,7 @@ pub fn null_journal<TA: JournalAdapter>(
     JournalWriter::new(journal, 0, true).unwrap()
 }
 
-pub fn open_journal<TA: JournalAdapter, LF: RawFileIOInterface>(
+pub fn open_journal<TA: JournalAdapter, Fs: RawFSInterface>(
     log_file_name: &str,
     log_kind: FileSpecifier,
     log_kind_version: FileSpecifierVersion,
@@ -93,10 +95,10 @@ pub fn open_journal<TA: JournalAdapter, LF: RawFileIOInterface>(
     host_run_mode: HostRunMode,
     host_startup_counter: u64,
     gs: &TA::GlobalState,
-) -> SDSSResult<JournalWriter<LF, TA>> {
+) -> SDSSResult<JournalWriter<Fs, TA>> {
     macro_rules! open_file {
         ($modify:literal) => {
-            SDSSFileIO::<LF>::open_or_create_perm_rw::<$modify>(
+            SDSSFileIO::<Fs>::open_or_create_perm_rw::<$modify>(
                 log_file_name,
                 FileScope::Journal,
                 log_kind,
@@ -117,7 +119,7 @@ pub fn open_journal<TA: JournalAdapter, LF: RawFileIOInterface>(
         FileOpen::Created(f) => return JournalWriter::new(f, 0, true),
         FileOpen::Existing(file, _) => file,
     };
-    let (file, last_txn) = JournalReader::<TA, LF>::scroll(file, gs)?;
+    let (file, last_txn) = JournalReader::<TA, Fs>::scroll(file, gs)?;
     JournalWriter::new(file, last_txn, false)
 }
 
@@ -233,9 +235,8 @@ impl JournalEntryMetadata {
     }
 }
 
-#[derive(Debug)]
-pub struct JournalReader<TA, LF> {
-    log_file: SDSSFileIO<LF>,
+pub struct JournalReader<TA, Fs: RawFSInterface> {
+    log_file: SDSSFileIO<Fs>,
     log_size: u64,
     evid: u64,
     closed: bool,
@@ -243,8 +244,8 @@ pub struct JournalReader<TA, LF> {
     _m: PhantomData<TA>,
 }
 
-impl<TA: JournalAdapter, LF: RawFileIOInterface> JournalReader<TA, LF> {
-    pub fn new(log_file: SDSSFileIO<LF>) -> SDSSResult<Self> {
+impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
+    pub fn new(log_file: SDSSFileIO<Fs>) -> SDSSResult<Self> {
         let log_size = log_file.file_length()? - SDSSHeaderRaw::header_size() as u64;
         Ok(Self {
             log_file,
@@ -361,7 +362,7 @@ impl<TA: JournalAdapter, LF: RawFileIOInterface> JournalReader<TA, LF> {
         Err(SDSSError::JournalCorrupted)
     }
     /// Read and apply all events in the given log file to the global state, returning the (open file, last event ID)
-    pub fn scroll(file: SDSSFileIO<LF>, gs: &TA::GlobalState) -> SDSSResult<(SDSSFileIO<LF>, u64)> {
+    pub fn scroll(file: SDSSFileIO<Fs>, gs: &TA::GlobalState) -> SDSSResult<(SDSSFileIO<Fs>, u64)> {
         let mut slf = Self::new(file)?;
         while !slf.end_of_file() {
             slf.rapply_next_event(gs)?;
@@ -374,7 +375,7 @@ impl<TA: JournalAdapter, LF: RawFileIOInterface> JournalReader<TA, LF> {
     }
 }
 
-impl<TA, LF> JournalReader<TA, LF> {
+impl<TA, Fs: RawFSInterface> JournalReader<TA, Fs> {
     fn _incr_evid(&mut self) {
         self.evid += 1;
     }
@@ -389,7 +390,7 @@ impl<TA, LF> JournalReader<TA, LF> {
     }
 }
 
-impl<TA, LF: RawFileIOInterface> JournalReader<TA, LF> {
+impl<TA, Fs: RawFSInterface> JournalReader<TA, Fs> {
     fn logfile_read_into_buffer(&mut self, buf: &mut [u8]) -> SDSSResult<()> {
         if !self.has_remaining_bytes(buf.len() as _) {
             // do this right here to avoid another syscall
@@ -401,18 +402,17 @@ impl<TA, LF: RawFileIOInterface> JournalReader<TA, LF> {
     }
 }
 
-#[derive(Debug)]
-pub struct JournalWriter<LF, TA> {
+pub struct JournalWriter<Fs: RawFSInterface, TA> {
     /// the txn log file
-    log_file: SDSSFileIO<LF>,
+    log_file: SDSSFileIO<Fs>,
     /// the id of the **next** journal
     id: u64,
     _m: PhantomData<TA>,
     closed: bool,
 }
 
-impl<LF: RawFileIOInterface, TA: JournalAdapter> JournalWriter<LF, TA> {
-    pub fn new(mut log_file: SDSSFileIO<LF>, last_txn_id: u64, new: bool) -> SDSSResult<Self> {
+impl<Fs: RawFSInterface, TA: JournalAdapter> JournalWriter<Fs, TA> {
+    pub fn new(mut log_file: SDSSFileIO<Fs>, last_txn_id: u64, new: bool) -> SDSSResult<Self> {
         let log_size = log_file.file_length()?;
         log_file.seek_from_start(log_size)?; // avoid jumbling with headers
         let mut slf = Self {
@@ -450,7 +450,7 @@ impl<LF: RawFileIOInterface, TA: JournalAdapter> JournalWriter<LF, TA> {
     }
 }
 
-impl<LF: RawFileIOInterface, TA> JournalWriter<LF, TA> {
+impl<Fs: RawFSInterface, TA> JournalWriter<Fs, TA> {
     pub fn appendrec_journal_reverse_entry(&mut self) -> SDSSResult<()> {
         let mut threshold = Threshold::<RECOVERY_BLOCK_AUTO_THRESHOLD>::new();
         let mut entry =
@@ -480,7 +480,7 @@ impl<LF: RawFileIOInterface, TA> JournalWriter<LF, TA> {
     }
 }
 
-impl<LF, TA> JournalWriter<LF, TA> {
+impl<Fs: RawFSInterface, TA> JournalWriter<Fs, TA> {
     fn _incr_id(&mut self) -> u64 {
         let current = self.id;
         self.id += 1;
@@ -488,7 +488,7 @@ impl<LF, TA> JournalWriter<LF, TA> {
     }
 }
 
-impl<LF, TA> Drop for JournalWriter<LF, TA> {
+impl<Fs: RawFSInterface, TA> Drop for JournalWriter<Fs, TA> {
     fn drop(&mut self) {
         assert!(self.closed, "log not closed");
     }
diff --git a/server/src/engine/storage/v1/memfs.rs b/server/src/engine/storage/v1/memfs.rs
new file mode 100644
index 00000000..7c81f548
--- /dev/null
+++ b/server/src/engine/storage/v1/memfs.rs
@@ -0,0 +1,502 @@
+/*
+ * Created on Fri Sep 08 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::engine::{
+        storage::v1::{
+            rw::{
+                RawFSInterface, RawFileInterface, RawFileInterfaceExt, RawFileInterfaceRead,
+                RawFileInterfaceWrite, RawFileInterfaceWriteExt, RawFileOpen,
+            },
+            SDSSResult,
+        },
+        sync::cell::Lazy,
+    },
+    parking_lot::RwLock,
+    std::{
+        collections::{hash_map::Entry, HashMap},
+        io::{Error, ErrorKind},
+    },
+};
+
+static VFS: Lazy<RwLock<HashMap<Box<str>, VNode>>, fn() -> RwLock<HashMap<Box<str>, VNode>>> =
+    Lazy::new(|| Default::default());
+
+/*
+    vnode
+    ---
+    either a vfile or a vdir
+*/
+
+#[derive(Debug)]
+enum VNode {
+    Dir(HashMap<Box<str>, Self>),
+    File(VFile),
+}
+
+impl VNode {
+    const fn is_file(&self) -> bool {
+        matches!(self, Self::File(_))
+    }
+    const fn is_dir(&self) -> bool {
+        matches!(self, Self::Dir(_))
+    }
+    fn as_dir_mut(&mut self) -> Option<&mut HashMap<Box<str>, Self>> {
+        match self {
+            Self::Dir(d) => Some(d),
+            Self::File(_) => None,
+        }
+    }
+}
+
+/*
+    vfs impl:
+    - nested directory structure
+    - make parents
+    - make child
+*/
+
+#[derive(Debug)]
+pub struct VirtualFS;
+
+impl RawFSInterface for VirtualFS {
+    type File = VFileDescriptor;
+    fn fs_create_dir(fpath: &str) -> super::SDSSResult<()> {
+        // get vfs
+        let mut vfs = VFS.write();
+        // get root dir
+        let path = fpath.split("/").collect::<Vec<&str>>();
+        // get target
+        let target = *path.last().unwrap();
+        let mut current = &mut *vfs;
+        // process components
+        let component_len = path.len() - 1;
+        let mut path = path.into_iter().take(component_len);
+        while let Some(component) = path.next() {
+            match current.get_mut(component) {
+                Some(VNode::Dir(d)) => {
+                    current = d;
+                }
+                Some(VNode::File(_)) => {
+                    return Err(Error::new(ErrorKind::InvalidInput, "found file in path").into())
+                }
+                None => {
+                    return Err(
+                        Error::new(ErrorKind::NotFound, "could not find directory in path").into(),
+                    )
+                }
+            }
+        }
+        match current.entry(target.into()) {
+            Entry::Occupied(_) => return Err(Error::from(ErrorKind::AlreadyExists).into()),
+            Entry::Vacant(ve) => {
+                ve.insert(VNode::Dir(into_dict!()));
+                Ok(())
+            }
+        }
+    }
+    fn fs_create_dir_all(fpath: &str) -> super::SDSSResult<()> {
+        let mut vfs = VFS.write();
+        fn create_ahead(
+            mut ahead: &[&str],
+            current: &mut HashMap<Box<str>, VNode>,
+        ) -> SDSSResult<()> {
+            if ahead.is_empty() {
+                return Ok(());
+            }
+            let this = ahead[0];
+            ahead = &ahead[1..];
+            match current.get_mut(this) {
+                Some(VNode::Dir(d)) => {
+                    if ahead.is_empty() {
+                        // hmm, this was the list dir that was to be created, but it already exists
+                        return Err(Error::from(ErrorKind::AlreadyExists).into());
+                    }
+                    return create_ahead(ahead, d);
+                }
+                Some(VNode::File(_)) => {
+                    return Err(Error::new(ErrorKind::InvalidInput, "found file in path").into())
+                }
+                None => {
+                    let _ = current.insert(this.into(), VNode::Dir(into_dict!()));
+                    let dir = current.get_mut(this).unwrap().as_dir_mut().unwrap();
+                    return create_ahead(ahead, dir);
+                }
+            }
+        }
+        let pieces: Vec<&str> = fpath.split("/").collect();
+        create_ahead(&pieces, &mut *vfs)
+    }
+    fn fs_delete_dir(fpath: &str) -> super::SDSSResult<()> {
+        delete_dir(fpath, false)
+    }
+    fn fs_delete_dir_all(fpath: &str) -> super::SDSSResult<()> {
+        delete_dir(fpath, true)
+    }
+    fn fs_fopen_or_create_rw(fpath: &str) -> super::SDSSResult<super::rw::RawFileOpen<Self::File>> {
+        let mut vfs = VFS.write();
+        // components
+        let components = fpath.split("/").collect::<Vec<&str>>();
+        let file = components.last().unwrap().to_owned().into();
+        let target_dir = find_target_dir_mut(components, &mut vfs)?;
+        match target_dir.entry(file) {
+            Entry::Occupied(mut oe) => match oe.get_mut() {
+                VNode::File(f) => {
+                    f.read = true;
+                    f.write = true;
+                    Ok(RawFileOpen::Existing(VFileDescriptor(fpath.into())))
+                }
+                VNode::Dir(_) => {
+                    return Err(
+                        Error::new(ErrorKind::InvalidInput, "found directory, not a file").into(),
+                    )
+                }
+            },
+            Entry::Vacant(v) => {
+                v.insert(VNode::File(VFile::new(true, true, vec![], 0)));
+                Ok(RawFileOpen::Created(VFileDescriptor(fpath.into())))
+            }
+        }
+    }
+}
+
+fn find_target_dir_mut<'a>(
+    components: Vec<&str>,
+    mut current: &'a mut HashMap<Box<str>, VNode>,
+) -> Result<&'a mut HashMap<Box<str>, VNode>, super::SDSSError> {
+    let path_len = components.len() - 1;
+    for component in components.into_iter().take(path_len) {
+        match current.get_mut(component) {
+            Some(VNode::Dir(d)) => current = d,
+            Some(VNode::File(_)) => {
+                return Err(Error::new(ErrorKind::InvalidInput, "found file in path").into())
+            }
+            None => {
+                return Err(
+                    Error::new(ErrorKind::NotFound, "could not find directory in path").into(),
+                )
+            }
+        }
+    }
+    Ok(current)
+}
+
+fn find_target_dir<'a>(
+    components: Vec<&str>,
+    mut current: &'a HashMap<Box<str>, VNode>,
+) -> Result<&'a HashMap<Box<str>, VNode>, super::SDSSError> {
+    let path_len = components.len() - 1;
+    for component in components.into_iter().take(path_len) {
+        match current.get(component) {
+            Some(VNode::Dir(d)) => current = d,
+            Some(VNode::File(_)) => {
+                return Err(Error::new(ErrorKind::InvalidInput, "found file in path").into())
+            }
+            None => {
+                return Err(
+                    Error::new(ErrorKind::NotFound, "could not find directory in path").into(),
+                )
+            }
+        }
+    }
+    Ok(current)
+}
+
+fn delete_dir(fpath: &str, allow_if_non_empty: bool) -> Result<(), super::SDSSError> {
+    let mut vfs = VFS.write();
+    let mut current = &mut *vfs;
+    // process components
+    let components = fpath.split("/").collect::<Vec<&str>>();
+    let components_len = components.len() - 1;
+    let target = *components.last().unwrap();
+    for component in components.into_iter().take(components_len) {
+        match current.get_mut(component) {
+            Some(VNode::Dir(dir)) => {
+                current = dir;
+            }
+            Some(VNode::File(_)) => {
+                return Err(Error::new(ErrorKind::InvalidInput, "found file in path").into())
+            }
+            None => {
+                return Err(
+                    Error::new(ErrorKind::NotFound, "could not find directory in path").into(),
+                )
+            }
+        }
+    }
+    match current.entry(target.into()) {
+        Entry::Occupied(dir) => match dir.get() {
+            VNode::Dir(d) => {
+                if allow_if_non_empty || d.is_empty() {
+                    dir.remove();
+                    return Ok(());
+                }
+                return Err(Error::new(ErrorKind::InvalidInput, "directory is not empty").into());
+            }
+            VNode::File(_) => {
+                return Err(Error::new(ErrorKind::InvalidInput, "found file in path").into())
+            }
+        },
+        Entry::Vacant(_) => {
+            return Err(Error::new(ErrorKind::NotFound, "could not find directory in path").into())
+        }
+    }
+}
+
+/*
+    vfile impl
+    ---
+    - all r/w operations
+    - all seek operations
+    - dummy sync operations
+*/
+
+#[derive(Debug)]
+pub struct VFile {
+    read: bool,
+    write: bool,
+    data: Vec<u8>,
+    pos: usize,
+}
+
+impl VFile {
+    fn new(read: bool, write: bool, data: Vec<u8>, pos: usize) -> Self {
+        Self {
+            read,
+            write,
+            data,
+            pos,
+        }
+    }
+    fn current(&self) -> &[u8] {
+        &self.data[self.pos..]
+    }
+}
+
+pub struct VFileDescriptor(Box<str>);
+impl Drop for VFileDescriptor {
+    fn drop(&mut self) {
+        let _ = with_file_mut(&self.0, |f| {
+            f.read = false;
+            f.write = false;
+            f.pos = 0;
+            Ok(())
+        });
+    }
+}
+
+fn with_file_mut<T>(fpath: &str, mut f: impl FnMut(&mut VFile) -> SDSSResult<T>) -> SDSSResult<T> {
+    let mut vfs = VFS.write();
+    let components = fpath.split("/").collect::<Vec<&str>>();
+    let file = *components.last().unwrap();
+    let target_dir = find_target_dir_mut(components, &mut vfs)?;
+    match target_dir.get_mut(file) {
+        Some(VNode::File(file)) => f(file),
+        Some(VNode::Dir(_)) => {
+            return Err(Error::new(ErrorKind::InvalidInput, "found directory, not a file").into())
+        }
+        None => return Err(Error::from(ErrorKind::NotFound).into()),
+    }
+}
+
+fn with_file<T>(fpath: &str, mut f: impl FnMut(&VFile) -> SDSSResult<T>) -> SDSSResult<T> {
+    let vfs = VFS.read();
+    let components = fpath.split("/").collect::<Vec<&str>>();
+    let file = *components.last().unwrap();
+    let target_dir = find_target_dir(components, &vfs)?;
+    match target_dir.get(file) {
+        Some(VNode::File(file)) => f(file),
+        Some(VNode::Dir(_)) => {
+            return Err(Error::new(ErrorKind::InvalidInput, "found directory, not a file").into())
+        }
+        None => return Err(Error::from(ErrorKind::NotFound).into()),
+    }
+}
+
+impl RawFileInterface for VFileDescriptor {
+    type Reader = Self;
+    type Writer = Self;
+    fn into_buffered_reader(self) -> super::SDSSResult<Self::Reader> {
+        Ok(self)
+    }
+    fn into_buffered_writer(self) -> super::SDSSResult<Self::Writer> {
+        Ok(self)
+    }
+}
+
+impl RawFileInterfaceRead for VFileDescriptor {
+    fn fr_read_exact(&mut self, buf: &mut [u8]) -> super::SDSSResult<()> {
+        with_file_mut(&self.0, |file| {
+            if !file.read {
+                return Err(
+                    Error::new(ErrorKind::PermissionDenied, "Read permission denied").into(),
+                );
+            }
+            let available_bytes = file.current().len();
+            if available_bytes < buf.len() {
+                return Err(Error::from(ErrorKind::UnexpectedEof).into());
+            }
+            buf.copy_from_slice(&file.data[file.pos..file.pos + buf.len()]);
+            file.pos += buf.len();
+            Ok(())
+        })
+    }
+}
+
+impl RawFileInterfaceWrite for VFileDescriptor {
+    fn fw_write_all(&mut self, bytes: &[u8]) -> super::SDSSResult<()> {
+        with_file_mut(&self.0, |file| {
+            if !file.write {
+                return Err(
+                    Error::new(ErrorKind::PermissionDenied, "Write permission denied").into(),
+                );
+            }
+            if file.pos + bytes.len() > file.data.len() {
+                file.data.resize(file.pos + bytes.len(), 0);
+            }
+            file.data[file.pos..file.pos + bytes.len()].copy_from_slice(bytes);
+            file.pos += bytes.len();
+            Ok(())
+        })
+    }
+}
+
+impl RawFileInterfaceWriteExt for VFileDescriptor {
+    fn fw_fsync_all(&mut self) -> super::SDSSResult<()> {
+        with_file(&self.0, |_| Ok(()))
+    }
+    fn fw_truncate_to(&mut self, to: u64) -> super::SDSSResult<()> {
+        with_file_mut(&self.0, |file| {
+            if !file.write {
+                return Err(
+                    Error::new(ErrorKind::PermissionDenied, "Write permission denied").into(),
+                );
+            }
+            if to as usize > file.data.len() {
+                file.data.resize(to as usize, 0);
+            } else {
+                file.data.truncate(to as usize);
+            }
+            if file.pos > file.data.len() {
+                file.pos = file.data.len();
+            }
+            Ok(())
+        })
+    }
+}
+
+impl RawFileInterfaceExt for VFileDescriptor {
+    fn fext_file_length(&self) -> super::SDSSResult<u64> {
+        with_file(&self.0, |f| Ok(f.data.len() as u64))
+    }
+    fn fext_cursor(&mut self) -> super::SDSSResult<u64> {
+        with_file(&self.0, |f| Ok(f.pos as u64))
+    }
+    fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> super::SDSSResult<()> {
+        with_file_mut(&self.0, |file| {
+            if by > file.data.len() as u64 {
+                return Err(
+                    Error::new(ErrorKind::InvalidInput, "Can't seek beyond file's end").into(),
+                );
+            }
+            file.pos = by as usize;
+            Ok(())
+        })
+    }
+}
+
+/*
+    nullfs
+    ---
+    - equivalent of `/dev/null`
+    - all calls are no-ops
+    - infallible
+*/
+
+/// An infallible `/dev/null` implementation. Whatever you run on this, it will always be a no-op since nothing
+/// is actually happening
+#[derive(Debug)]
+pub struct NullFS;
+pub struct NullFile;
+impl RawFSInterface for NullFS {
+    type File = NullFile;
+    fn fs_create_dir(_: &str) -> SDSSResult<()> {
+        Ok(())
+    }
+    fn fs_create_dir_all(_: &str) -> SDSSResult<()> {
+        Ok(())
+    }
+    fn fs_delete_dir(_: &str) -> SDSSResult<()> {
+        Ok(())
+    }
+    fn fs_delete_dir_all(_: &str) -> SDSSResult<()> {
+        Ok(())
+    }
+    fn fs_fopen_or_create_rw(_: &str) -> SDSSResult<RawFileOpen<Self::File>> {
+        Ok(RawFileOpen::Created(NullFile))
+    }
+}
+impl RawFileInterfaceRead for NullFile {
+    fn fr_read_exact(&mut self, _: &mut [u8]) -> SDSSResult<()> {
+        Ok(())
+    }
+}
+impl RawFileInterfaceWrite for NullFile {
+    fn fw_write_all(&mut self, _: &[u8]) -> SDSSResult<()> {
+        Ok(())
+    }
+}
+impl RawFileInterfaceWriteExt for NullFile {
+    fn fw_fsync_all(&mut self) -> SDSSResult<()> {
+        Ok(())
+    }
+    fn fw_truncate_to(&mut self, _: u64) -> SDSSResult<()> {
+        Ok(())
+    }
+}
+impl RawFileInterfaceExt for NullFile {
+    fn fext_file_length(&self) -> SDSSResult<u64> {
+        Ok(0)
+    }
+
+    fn fext_cursor(&mut self) -> SDSSResult<u64> {
+        Ok(0)
+    }
+
+    fn fext_seek_ahead_from_start_by(&mut self, _: u64) -> SDSSResult<()> {
+        Ok(())
+    }
+}
+impl RawFileInterface for NullFile {
+    type Reader = Self;
+    type Writer = Self;
+    fn into_buffered_reader(self) -> SDSSResult<Self::Reader> {
+        Ok(self)
+    }
+    fn into_buffered_writer(self) -> SDSSResult<Self::Writer> {
+        Ok(self)
+    }
+}
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index 0c111ef0..ca269318 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -34,15 +34,15 @@ mod rw;
 pub mod inf;
 mod start_stop;
 // test
-#[cfg(test)]
-pub mod test_util;
+pub mod memfs;
 #[cfg(test)]
 mod tests;
 
 // re-exports
 pub use {
     journal::{open_journal, JournalAdapter, JournalWriter},
-    rw::{BufferedScanner, NullZero, RawFileIOInterface, SDSSFileIO},
+    memfs::NullFS,
+    rw::{BufferedScanner, LocalFS, RawFSInterface, SDSSFileIO},
 };
 pub mod header_meta {
     pub use super::header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode};
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 8c6f87dd..6319ba60 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use std::marker::PhantomData;
+
 use {
     super::{
         header_impl::{
@@ -36,8 +38,8 @@ use {
         util::os::SysIOError,
     },
     std::{
-        fs::File,
-        io::{Read, Seek, SeekFrom, Write},
+        fs::{self, File},
+        io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write},
         ptr, slice,
     },
 };
@@ -70,57 +72,94 @@ pub enum RawFileOpen<F> {
     Existing(F),
 }
 
-pub trait RawFileIOInterface: Sized {
-    /// Indicates that the interface is not a `/dev/null` (or related) implementation
-    const NOTNULL: bool = true;
-    fn fopen_or_create_rw(file_path: &str) -> SDSSResult<RawFileOpen<Self>>;
-    fn fread_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()>;
-    fn fwrite_all(&mut self, bytes: &[u8]) -> SDSSResult<()>;
-    fn fsync_all(&mut self) -> SDSSResult<()>;
-    fn fseek_ahead(&mut self, by: u64) -> SDSSResult<()>;
-    fn flen(&self) -> SDSSResult<u64>;
-    fn flen_set(&mut self, to: u64) -> SDSSResult<()>;
-    fn fcursor(&mut self) -> SDSSResult<u64>;
+pub trait RawFSInterface {
+    const NOT_NULL: bool = true;
+    type File: RawFileInterface;
+    fn fs_create_dir(fpath: &str) -> SDSSResult<()>;
+    fn fs_create_dir_all(fpath: &str) -> SDSSResult<()>;
+    fn fs_delete_dir(fpath: &str) -> SDSSResult<()>;
+    fn fs_delete_dir_all(fpath: &str) -> SDSSResult<()>;
+    fn fs_fopen_or_create_rw(fpath: &str) -> SDSSResult<RawFileOpen<Self::File>>;
 }
 
-/// This is a kind of file like `/dev/null`. It exists in ... nothing!
-pub struct NullZero;
+/// A file (well, probably) that can be used for RW operations along with advanced write and extended operations (such as seeking)
+pub trait RawFileInterface
+where
+    Self: RawFileInterfaceRead
+        + RawFileInterfaceWrite
+        + RawFileInterfaceWriteExt
+        + RawFileInterfaceExt,
+{
+    type Reader: RawFileInterfaceRead + RawFileInterfaceExt;
+    type Writer: RawFileInterfaceWrite + RawFileInterfaceExt;
+    fn into_buffered_reader(self) -> SDSSResult<Self::Reader>;
+    fn into_buffered_writer(self) -> SDSSResult<Self::Writer>;
+}
 
-impl RawFileIOInterface for NullZero {
-    const NOTNULL: bool = false;
-    fn fopen_or_create_rw(_: &str) -> SDSSResult<RawFileOpen<Self>> {
-        Ok(RawFileOpen::Created(Self))
-    }
-    fn fread_exact(&mut self, _: &mut [u8]) -> SDSSResult<()> {
-        Ok(())
-    }
-    fn fwrite_all(&mut self, _: &[u8]) -> SDSSResult<()> {
-        Ok(())
+/// A file interface that supports read operations
+pub trait RawFileInterfaceRead {
+    fn fr_read_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()>;
+}
+
+impl<R: Read> RawFileInterfaceRead for R {
+    fn fr_read_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()> {
+        self.read_exact(buf).map_err(From::from)
     }
-    fn fsync_all(&mut self) -> SDSSResult<()> {
-        Ok(())
+}
+
+/// A file interface that supports write operations
+pub trait RawFileInterfaceWrite {
+    fn fw_write_all(&mut self, buf: &[u8]) -> SDSSResult<()>;
+}
+
+impl<W: Write> RawFileInterfaceWrite for W {
+    fn fw_write_all(&mut self, buf: &[u8]) -> SDSSResult<()> {
+        self.write_all(buf).map_err(From::from)
     }
-    fn fseek_ahead(&mut self, _: u64) -> SDSSResult<()> {
-        Ok(())
+}
+
+/// A file interface that supports advanced write operations
+pub trait RawFileInterfaceWriteExt {
+    fn fw_fsync_all(&mut self) -> SDSSResult<()>;
+    fn fw_truncate_to(&mut self, to: u64) -> SDSSResult<()>;
+}
+
+/// A file interface that supports advanced file operations
+pub trait RawFileInterfaceExt {
+    fn fext_file_length(&self) -> SDSSResult<u64>;
+    fn fext_cursor(&mut self) -> SDSSResult<u64>;
+    fn fext_seek_ahead_from_start_by(&mut self, ahead_by: u64) -> SDSSResult<()>;
+}
+
+fn cvt<T>(v: std::io::Result<T>) -> SDSSResult<T> {
+    let r = v?;
+    Ok(r)
+}
+
+/// The actual local host file system (as an abstraction [`fs`])
+#[derive(Debug)]
+pub struct LocalFS;
+
+impl RawFSInterface for LocalFS {
+    type File = File;
+    fn fs_create_dir(fpath: &str) -> SDSSResult<()> {
+        cvt(fs::create_dir(fpath))
     }
-    fn flen(&self) -> SDSSResult<u64> {
-        Ok(0)
+    fn fs_create_dir_all(fpath: &str) -> SDSSResult<()> {
+        cvt(fs::create_dir_all(fpath))
     }
-    fn flen_set(&mut self, _: u64) -> SDSSResult<()> {
-        Ok(())
+    fn fs_delete_dir(fpath: &str) -> SDSSResult<()> {
+        cvt(fs::remove_dir(fpath))
     }
-    fn fcursor(&mut self) -> SDSSResult<u64> {
-        Ok(0)
+    fn fs_delete_dir_all(fpath: &str) -> SDSSResult<()> {
+        cvt(fs::remove_dir_all(fpath))
     }
-}
-
-impl RawFileIOInterface for File {
-    fn fopen_or_create_rw(file_path: &str) -> SDSSResult<RawFileOpen<Self>> {
+    fn fs_fopen_or_create_rw(fpath: &str) -> SDSSResult<RawFileOpen<Self::File>> {
         let f = File::options()
             .create(true)
             .read(true)
             .write(true)
-            .open(file_path)?;
+            .open(fpath)?;
         let md = f.metadata()?;
         if md.len() == 0 {
             Ok(RawFileOpen::Created(f))
@@ -128,41 +167,79 @@ impl RawFileIOInterface for File {
             Ok(RawFileOpen::Existing(f))
         }
     }
-    fn fread_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()> {
-        self.read_exact(buf)?;
-        Ok(())
+}
+
+impl RawFileInterface for File {
+    type Reader = BufReader<Self>;
+    type Writer = BufWriter<Self>;
+    fn into_buffered_reader(self) -> SDSSResult<Self::Reader> {
+        Ok(BufReader::new(self))
     }
-    fn fwrite_all(&mut self, bytes: &[u8]) -> SDSSResult<()> {
-        self.write_all(bytes)?;
-        Ok(())
+    fn into_buffered_writer(self) -> SDSSResult<Self::Writer> {
+        Ok(BufWriter::new(self))
     }
-    fn fsync_all(&mut self) -> SDSSResult<()> {
-        self.sync_all()?;
-        Ok(())
+}
+
+impl RawFileInterfaceWriteExt for File {
+    fn fw_fsync_all(&mut self) -> SDSSResult<()> {
+        cvt(self.sync_all())
     }
-    fn flen(&self) -> SDSSResult<u64> {
-        Ok(self.metadata()?.len())
+    fn fw_truncate_to(&mut self, to: u64) -> SDSSResult<()> {
+        cvt(self.set_len(to))
     }
-    fn fseek_ahead(&mut self, by: u64) -> SDSSResult<()> {
-        self.seek(SeekFrom::Start(by))?;
-        Ok(())
+}
+
+trait LocalFSFile {
+    fn file_mut(&mut self) -> &mut File;
+    fn file(&self) -> &File;
+}
+
+impl LocalFSFile for File {
+    fn file_mut(&mut self) -> &mut File {
+        self
     }
-    fn flen_set(&mut self, to: u64) -> SDSSResult<()> {
-        self.set_len(to)?;
-        Ok(())
+    fn file(&self) -> &File {
+        self
     }
-    fn fcursor(&mut self) -> SDSSResult<u64> {
-        self.stream_position().map_err(From::from)
+}
+
+impl LocalFSFile for BufReader<File> {
+    fn file_mut(&mut self) -> &mut File {
+        self.get_mut()
+    }
+    fn file(&self) -> &File {
+        self.get_ref()
     }
 }
 
-pub struct SDSSFileTrackedWriter<F> {
-    f: SDSSFileIO<F>,
+impl LocalFSFile for BufWriter<File> {
+    fn file_mut(&mut self) -> &mut File {
+        self.get_mut()
+    }
+    fn file(&self) -> &File {
+        self.get_ref()
+    }
+}
+
+impl<F: LocalFSFile> RawFileInterfaceExt for F {
+    fn fext_file_length(&self) -> SDSSResult<u64> {
+        Ok(self.file().metadata()?.len())
+    }
+    fn fext_cursor(&mut self) -> SDSSResult<u64> {
+        cvt(self.file_mut().stream_position())
+    }
+    fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> SDSSResult<()> {
+        cvt(self.file_mut().seek(SeekFrom::Start(by)).map(|_| ()))
+    }
+}
+
+pub struct SDSSFileTrackedWriter<Fs: RawFSInterface> {
+    f: SDSSFileIO<Fs>,
     cs: SCrc,
 }
 
-impl<F: RawFileIOInterface> SDSSFileTrackedWriter<F> {
-    pub fn new(f: SDSSFileIO<F>) -> Self {
+impl<Fs: RawFSInterface> SDSSFileTrackedWriter<Fs> {
+    pub fn new(f: SDSSFileIO<Fs>) -> Self {
         Self { f, cs: SCrc::new() }
     }
     pub fn unfsynced_write(&mut self, block: &[u8]) -> SDSSResult<()> {
@@ -182,23 +259,23 @@ impl<F: RawFileIOInterface> SDSSFileTrackedWriter<F> {
         core::mem::swap(&mut self.cs, &mut scrc);
         scrc.finish()
     }
-    pub fn inner_file(&mut self) -> &mut SDSSFileIO<F> {
+    pub fn inner_file(&mut self) -> &mut SDSSFileIO<Fs> {
         &mut self.f
     }
 }
 
 /// [`SDSSFileLenTracked`] simply maintains application level length and checksum tracking to avoid frequent syscalls because we
 /// do not expect (even though it's very possible) users to randomly modify file lengths while we're reading them
-pub struct SDSSFileTrackedReader<F> {
-    f: SDSSFileIO<F>,
+pub struct SDSSFileTrackedReader<Fs: RawFSInterface> {
+    f: SDSSFileIO<Fs>,
     len: u64,
     pos: u64,
     cs: SCrc,
 }
 
-impl<F: RawFileIOInterface> SDSSFileTrackedReader<F> {
+impl<Fs: RawFSInterface> SDSSFileTrackedReader<Fs> {
     /// Important: this will only look at the data post the current cursor!
-    pub fn new(mut f: SDSSFileIO<F>) -> SDSSResult<Self> {
+    pub fn new(mut f: SDSSFileIO<Fs>) -> SDSSResult<Self> {
         let len = f.file_length()?;
         let pos = f.retrieve_cursor()?;
         Ok(Self {
@@ -242,10 +319,10 @@ impl<F: RawFileIOInterface> SDSSFileTrackedReader<F> {
         core::mem::swap(&mut crc, &mut self.cs);
         crc.finish()
     }
-    pub fn inner_file(&mut self) -> &mut SDSSFileIO<F> {
+    pub fn inner_file(&mut self) -> &mut SDSSFileIO<Fs> {
         &mut self.f
     }
-    pub fn into_inner_file(self) -> SDSSFileIO<F> {
+    pub fn into_inner_file(self) -> SDSSFileIO<Fs> {
         self.f
     }
     pub fn __cursor_ahead_by(&mut self, sizeof: usize) {
@@ -267,11 +344,12 @@ impl<F: RawFileIOInterface> SDSSFileTrackedReader<F> {
 }
 
 #[derive(Debug)]
-pub struct SDSSFileIO<F> {
-    f: F,
+pub struct SDSSFileIO<Fs: RawFSInterface> {
+    f: Fs::File,
+    _fs: PhantomData<Fs>,
 }
 
-impl<F: RawFileIOInterface> SDSSFileIO<F> {
+impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
     /// **IMPORTANT: File position: end-of-header-section**
     pub fn open_or_create_perm_rw<const REWRITE_MODIFY_COUNTER: bool>(
         file_path: &str,
@@ -282,7 +360,7 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
         host_run_mode: HostRunMode,
         host_startup_counter: u64,
     ) -> SDSSResult<FileOpen<Self>> {
-        let f = F::fopen_or_create_rw(file_path)?;
+        let f = Fs::fs_fopen_or_create_rw(file_path)?;
         match f {
             RawFileOpen::Created(f) => {
                 // since this file was just created, we need to append the header
@@ -303,7 +381,7 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
             RawFileOpen::Existing(mut f) => {
                 // this is an existing file. decoded the header
                 let mut header_raw = [0u8; SDSSHeaderRaw::header_size()];
-                f.fread_exact(&mut header_raw)?;
+                f.fr_read_exact(&mut header_raw)?;
                 let header = SDSSHeaderRaw::decode_noverify(header_raw)
                     .ok_or(SDSSError::HeaderDecodeCorruptedHeader)?;
                 // now validate the header
@@ -323,35 +401,38 @@ impl<F: RawFileIOInterface> SDSSFileIO<F> {
     }
 }
 
-impl<F: RawFileIOInterface> SDSSFileIO<F> {
-    fn _new(f: F) -> Self {
-        Self { f }
+impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
+    fn _new(f: Fs::File) -> Self {
+        Self {
+            f,
+            _fs: PhantomData,
+        }
     }
     pub fn unfsynced_write(&mut self, data: &[u8]) -> SDSSResult<()> {
-        self.f.fwrite_all(data)
+        self.f.fw_write_all(data)
     }
     pub fn fsync_all(&mut self) -> SDSSResult<()> {
-        self.f.fsync_all()?;
+        self.f.fw_fsync_all()?;
         Ok(())
     }
     pub fn fsynced_write(&mut self, data: &[u8]) -> SDSSResult<()> {
-        self.f.fwrite_all(data)?;
-        self.f.fsync_all()
+        self.f.fw_write_all(data)?;
+        self.f.fw_fsync_all()
     }
     pub fn read_to_buffer(&mut self, buffer: &mut [u8]) -> SDSSResult<()> {
-        self.f.fread_exact(buffer)
+        self.f.fr_read_exact(buffer)
     }
     pub fn file_length(&self) -> SDSSResult<u64> {
-        self.f.flen()
+        self.f.fext_file_length()
     }
     pub fn seek_from_start(&mut self, by: u64) -> SDSSResult<()> {
-        self.f.fseek_ahead(by)
+        self.f.fext_seek_ahead_from_start_by(by)
     }
     pub fn trim_file_to(&mut self, to: u64) -> SDSSResult<()> {
-        self.f.flen_set(to)
+        self.f.fw_truncate_to(to)
     }
     pub fn retrieve_cursor(&mut self) -> SDSSResult<u64> {
-        self.f.fcursor()
+        self.f.fext_cursor()
     }
     pub fn read_byte(&mut self) -> SDSSResult<u8> {
         let mut r = [0; 1];
diff --git a/server/src/engine/storage/v1/test_util.rs b/server/src/engine/storage/v1/test_util.rs
deleted file mode 100644
index 263d00c9..00000000
--- a/server/src/engine/storage/v1/test_util.rs
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Created on Thu Aug 24 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#[cfg(test)]
-use super::{
-    header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
-    rw::{FileOpen, SDSSFileIO},
-};
-use {
-    super::{
-        rw::{RawFileIOInterface, RawFileOpen},
-        SDSSResult,
-    },
-    crate::engine::sync::cell::Lazy,
-    parking_lot::RwLock,
-    std::{
-        collections::hash_map::{Entry, HashMap},
-        io::{Error, ErrorKind},
-    },
-};
-
-static VFS: Lazy<RwLock<HashMap<Box<str>, VFile>>, fn() -> RwLock<HashMap<Box<str>, VFile>>> =
-    Lazy::new(|| RwLock::new(HashMap::new()));
-
-#[derive(Debug)]
-struct VFile {
-    read: bool,
-    write: bool,
-    data: Vec<u8>,
-    pos: usize,
-}
-
-impl VFile {
-    fn new(read: bool, write: bool, data: Vec<u8>, pos: usize) -> Self {
-        Self {
-            read,
-            write,
-            data,
-            pos,
-        }
-    }
-    fn current(&self) -> &[u8] {
-        &self.data[self.pos..]
-    }
-}
-
-#[derive(Debug)]
-pub struct VirtualFS(Box<str>);
-impl VirtualFS {
-    pub fn get_file_data(f: &str) -> Option<Vec<u8>> {
-        VFS.read().get(f).map(|f| f.data.clone())
-    }
-}
-
-impl RawFileIOInterface for VirtualFS {
-    fn fopen_or_create_rw(file_path: &str) -> super::SDSSResult<RawFileOpen<Self>> {
-        match VFS.write().entry(file_path.into()) {
-            Entry::Occupied(mut oe) => {
-                oe.get_mut().read = true;
-                oe.get_mut().write = true;
-                oe.get_mut().pos = 0;
-                Ok(RawFileOpen::Existing(Self(file_path.into())))
-            }
-            Entry::Vacant(v) => {
-                v.insert(VFile::new(true, true, vec![], 0));
-                Ok(RawFileOpen::Created(Self(file_path.into())))
-            }
-        }
-    }
-    fn fread_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()> {
-        let mut vfs = VFS.write();
-        let file = vfs
-            .get_mut(&self.0)
-            .ok_or(Error::new(ErrorKind::NotFound, "File not found"))?;
-
-        if !file.read {
-            return Err(Error::new(ErrorKind::PermissionDenied, "Read permission denied").into());
-        }
-        let available_bytes = file.current().len();
-        if available_bytes < buf.len() {
-            return Err(Error::from(ErrorKind::UnexpectedEof).into());
-        }
-        buf.copy_from_slice(&file.data[file.pos..file.pos + buf.len()]);
-        file.pos += buf.len();
-        Ok(())
-    }
-    fn fwrite_all(&mut self, bytes: &[u8]) -> SDSSResult<()> {
-        let mut vfs = VFS.write();
-        let file = vfs
-            .get_mut(&self.0)
-            .ok_or(Error::new(ErrorKind::NotFound, "File not found"))?;
-
-        if !file.write {
-            return Err(Error::new(ErrorKind::PermissionDenied, "Write permission denied").into());
-        }
-
-        if file.pos + bytes.len() > file.data.len() {
-            file.data.resize(file.pos + bytes.len(), 0);
-        }
-        file.data[file.pos..file.pos + bytes.len()].copy_from_slice(bytes);
-        file.pos += bytes.len();
-
-        Ok(())
-    }
-    fn fsync_all(&mut self) -> super::SDSSResult<()> {
-        // pretty redundant for us
-        Ok(())
-    }
-    fn fseek_ahead(&mut self, by: u64) -> SDSSResult<()> {
-        let mut vfs = VFS.write();
-        let file = vfs
-            .get_mut(&self.0)
-            .ok_or(Error::new(ErrorKind::NotFound, "File not found"))?;
-
-        if by > file.data.len() as u64 {
-            return Err(Error::new(ErrorKind::InvalidInput, "Can't seek beyond file's end").into());
-        }
-
-        file.pos = by as usize;
-        Ok(())
-    }
-
-    fn flen(&self) -> SDSSResult<u64> {
-        let vfs = VFS.read();
-        let file = vfs
-            .get(&self.0)
-            .ok_or(Error::new(ErrorKind::NotFound, "File not found"))?;
-
-        Ok(file.data.len() as u64)
-    }
-
-    fn flen_set(&mut self, to: u64) -> SDSSResult<()> {
-        let mut vfs = VFS.write();
-        let file = vfs
-            .get_mut(&self.0)
-            .ok_or(Error::new(ErrorKind::NotFound, "File not found"))?;
-
-        if !file.write {
-            return Err(Error::new(ErrorKind::PermissionDenied, "Write permission denied").into());
-        }
-
-        if to as usize > file.data.len() {
-            file.data.resize(to as usize, 0);
-        } else {
-            file.data.truncate(to as usize);
-        }
-
-        if file.pos > file.data.len() {
-            file.pos = file.data.len();
-        }
-
-        Ok(())
-    }
-    fn fcursor(&mut self) -> SDSSResult<u64> {
-        let vfs = VFS.read();
-        let file = vfs
-            .get(&self.0)
-            .ok_or(Error::new(ErrorKind::NotFound, "File not found"))?;
-
-        Ok(file.pos as u64)
-    }
-}
-
-#[test]
-fn sdss_file() {
-    let f = SDSSFileIO::<VirtualFS>::open_or_create_perm_rw::<false>(
-        "this_is_a_test_file.db",
-        FileScope::Journal,
-        FileSpecifier::TestTransactionLog,
-        FileSpecifierVersion::__new(0),
-        0,
-        HostRunMode::Prod,
-        128,
-    )
-    .unwrap();
-
-    let FileOpen::Created(mut f) = f else {
-        panic!()
-    };
-
-    f.fsynced_write(b"hello, world\n").unwrap();
-    f.fsynced_write(b"hello, again\n").unwrap();
-
-    let f = SDSSFileIO::<VirtualFS>::open_or_create_perm_rw::<false>(
-        "this_is_a_test_file.db",
-        FileScope::Journal,
-        FileSpecifier::TestTransactionLog,
-        FileSpecifierVersion::__new(0),
-        0,
-        HostRunMode::Prod,
-        128,
-    )
-    .unwrap();
-
-    let FileOpen::Existing(mut f, _) = f else {
-        panic!()
-    };
-
-    let mut buf1 = [0u8; 13];
-    f.read_to_buffer(&mut buf1).unwrap();
-    let mut buf2 = [0u8; 13];
-    f.read_to_buffer(&mut buf2).unwrap();
-
-    assert_eq!(&buf1, b"hello, world\n");
-    assert_eq!(&buf2, b"hello, again\n");
-}
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index 8439377c..4c72da2d 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -24,7 +24,7 @@
  *
 */
 
-type VirtualFS = super::test_util::VirtualFS;
+type VirtualFS = super::memfs::VirtualFS;
 
 mod batch;
 mod rw;
diff --git a/server/src/engine/storage/v1/tests/batch.rs b/server/src/engine/storage/v1/tests/batch.rs
index cd12b639..5f63a1ef 100644
--- a/server/src/engine/storage/v1/tests/batch.rs
+++ b/server/src/engine/storage/v1/tests/batch.rs
@@ -42,8 +42,8 @@ use {
                     DecodedBatchEventKind, NormalBatch,
                 },
                 header_meta::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
+                memfs::VirtualFS,
                 rw::{FileOpen, SDSSFileIO},
-                test_util::VirtualFS,
             },
         },
         util::test_utils,
diff --git a/server/src/engine/storage/v1/tests/rw.rs b/server/src/engine/storage/v1/tests/rw.rs
index 23e296f3..ff70d03a 100644
--- a/server/src/engine/storage/v1/tests/rw.rs
+++ b/server/src/engine/storage/v1/tests/rw.rs
@@ -31,20 +31,22 @@ use crate::engine::storage::v1::{
 
 #[test]
 fn create_delete() {
-    let f = SDSSFileIO::<super::VirtualFS>::open_or_create_perm_rw::<false>(
-        "hello_world.db-tlog",
-        FileScope::Journal,
-        FileSpecifier::GNSTxnLog,
-        FileSpecifierVersion::__new(0),
-        0,
-        HostRunMode::Prod,
-        0,
-    )
-    .unwrap();
-    match f {
-        FileOpen::Existing(_, _) => panic!(),
-        FileOpen::Created(_) => {}
-    };
+    {
+        let f = SDSSFileIO::<super::VirtualFS>::open_or_create_perm_rw::<false>(
+            "hello_world.db-tlog",
+            FileScope::Journal,
+            FileSpecifier::GNSTxnLog,
+            FileSpecifierVersion::__new(0),
+            0,
+            HostRunMode::Prod,
+            0,
+        )
+        .unwrap();
+        match f {
+            FileOpen::Existing(_, _) => panic!(),
+            FileOpen::Created(_) => {}
+        };
+    }
     let open = SDSSFileIO::<super::VirtualFS>::open_or_create_perm_rw::<false>(
         "hello_world.db-tlog",
         FileScope::Journal,
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index 821a873a..a290f53e 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -25,7 +25,7 @@
 */
 
 #[cfg(test)]
-use crate::engine::storage::v1::test_util::VirtualFS;
+use crate::engine::storage::v1::memfs::VirtualFS;
 use {
     super::{TransactionError, TransactionResult},
     crate::{
@@ -35,7 +35,8 @@ use {
             storage::v1::{
                 self, header_meta,
                 inf::{self, PersistObject},
-                BufferedScanner, JournalAdapter, JournalWriter, RawFileIOInterface, SDSSResult,
+                BufferedScanner, JournalAdapter, JournalWriter, LocalFS, RawFSInterface,
+                SDSSResult,
             },
         },
         util::EndianQW,
@@ -59,26 +60,25 @@ pub use {
 };
 
 pub type GNSTransactionDriverNullZero =
-    GNSTransactionDriverAnyFS<crate::engine::storage::v1::NullZero>;
+    GNSTransactionDriverAnyFS<crate::engine::storage::v1::NullFS>;
 pub type GNSTransactionDriver = GNSTransactionDriverAnyFS<File>;
 #[cfg(test)]
 pub type GNSTransactionDriverVFS = GNSTransactionDriverAnyFS<VirtualFS>;
 
 const CURRENT_LOG_VERSION: u32 = 0;
 
-pub trait GNSTransactionDriverLLInterface: RawFileIOInterface {
+pub trait GNSTransactionDriverLLInterface: RawFSInterface {
     /// If true, this is an actual txn driver with a non-null (not `/dev/null` like) journal
-    const NONNULL: bool = <Self as RawFileIOInterface>::NOTNULL;
+    const NONNULL: bool = <Self as RawFSInterface>::NOT_NULL;
 }
-impl<T: RawFileIOInterface> GNSTransactionDriverLLInterface for T {}
+impl<T: RawFSInterface> GNSTransactionDriverLLInterface for T {}
 
-#[derive(Debug)]
 /// The GNS transaction driver is used to handle DDL transactions
-pub struct GNSTransactionDriverAnyFS<F = File> {
+pub struct GNSTransactionDriverAnyFS<F: RawFSInterface = LocalFS> {
     journal: JournalWriter<F, GNSAdapter>,
 }
 
-impl GNSTransactionDriverAnyFS<crate::engine::storage::v1::NullZero> {
+impl GNSTransactionDriverAnyFS<crate::engine::storage::v1::NullFS> {
     pub fn nullzero(gns: &GlobalNS) -> Self {
         let journal = v1::open_journal(
             "gns.db-tlog",

From 5cafc612319ec65d43a7953f6b795d14f83e783c Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 10 Sep 2023 10:19:57 +0000
Subject: [PATCH 247/310] Implement services

---
 server/src/engine/core/model/delta.rs         |  16 +-
 server/src/engine/data/uuid.rs                |   8 +-
 server/src/engine/error.rs                    |  12 +-
 server/src/engine/fractal/config.rs           |  47 +++
 server/src/engine/fractal/drivers.rs          |  88 +++++
 server/src/engine/fractal/mgr.rs              | 303 ++++++++++++++++++
 server/src/engine/fractal/mod.rs              | 204 ++++++++++++
 server/src/engine/fractal/util.rs             |  78 +++++
 server/src/engine/mod.rs                      |   1 +
 server/src/engine/ql/ast/mod.rs               |   3 +-
 server/src/engine/ql/dml/ins.rs               |   3 +-
 server/src/engine/ql/lex/mod.rs               |   8 +-
 .../src/engine/storage/v1/batch_jrnl/mod.rs   |  42 +++
 server/src/engine/storage/v1/inf/map.rs       |  11 +-
 server/src/engine/storage/v1/loader.rs        | 100 ++++++
 server/src/engine/storage/v1/memfs.rs         |  54 ++--
 server/src/engine/storage/v1/mod.rs           |  24 +-
 server/src/engine/txn/data.rs                 |  70 ----
 server/src/engine/txn/gns/mod.rs              |  19 +-
 server/src/engine/txn/mod.rs                  |   1 -
 20 files changed, 954 insertions(+), 138 deletions(-)
 create mode 100644 server/src/engine/fractal/config.rs
 create mode 100644 server/src/engine/fractal/drivers.rs
 create mode 100644 server/src/engine/fractal/mgr.rs
 create mode 100644 server/src/engine/fractal/mod.rs
 create mode 100644 server/src/engine/fractal/util.rs
 create mode 100644 server/src/engine/storage/v1/loader.rs
 delete mode 100644 server/src/engine/txn/data.rs

diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index 78b532c1..5365f1c4 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -26,7 +26,9 @@
 
 use {
     super::{Fields, Model},
-    crate::engine::{core::index::Row, sync::atm::Guard, sync::queue::Queue},
+    crate::engine::{
+        core::index::Row, fractal::FractalToken, sync::atm::Guard, sync::queue::Queue,
+    },
     parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard},
     std::{
         collections::btree_map::{BTreeMap, Range},
@@ -259,13 +261,23 @@ impl DeltaState {
     }
 }
 
+// fractal
+impl DeltaState {
+    pub fn __fractal_take_from_data_delta(&self, cnt: usize, _token: FractalToken) {
+        let _ = self.data_deltas_size.fetch_sub(cnt, Ordering::Release);
+    }
+    pub fn __fractal_take_full_from_data_delta(&self, _token: FractalToken) -> usize {
+        self.data_deltas_size.swap(0, Ordering::AcqRel)
+    }
+}
+
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
 pub struct DeltaVersion(u64);
 impl DeltaVersion {
     pub const fn genesis() -> Self {
         Self(0)
     }
-    pub const  fn __new(v: u64) -> Self {
+    pub const fn __new(v: u64) -> Self {
         Self(v)
     }
     fn step(&self) -> Self {
diff --git a/server/src/engine/data/uuid.rs b/server/src/engine/data/uuid.rs
index 154d51fc..819c536c 100644
--- a/server/src/engine/data/uuid.rs
+++ b/server/src/engine/data/uuid.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use core::fmt;
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
 pub struct Uuid {
     data: uuid::Uuid,
@@ -48,8 +50,8 @@ impl Uuid {
     }
 }
 
-impl ToString for Uuid {
-    fn to_string(&self) -> String {
-        self.data.to_string()
+impl fmt::Display for Uuid {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.data.fmt(f)
     }
 }
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index a302fab8..542b8785 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -24,7 +24,7 @@
  *
 */
 
-use super::txn::TransactionError;
+use super::{storage::v1::SDSSError, txn::TransactionError};
 
 pub type LangResult<T> = Result<T, LangError>;
 pub type LexResult<T> = Result<T, LexError>;
@@ -81,8 +81,9 @@ pub enum LangError {
     StmtUnknownDrop,
 }
 
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+#[derive(Debug)]
 #[repr(u8)]
+#[cfg_attr(test, derive(PartialEq))]
 /// Executor errors
 pub enum DatabaseError {
     // sys
@@ -136,6 +137,13 @@ pub enum DatabaseError {
     DmlConstraintViolationFieldTypedef,
     ServerError,
     TransactionalError,
+    StorageSubsystemErr(SDSSError),
+}
+
+impl From<SDSSError> for DatabaseError {
+    fn from(e: SDSSError) -> Self {
+        Self::StorageSubsystemErr(e)
+    }
 }
 
 impl From<TransactionError> for DatabaseError {
diff --git a/server/src/engine/fractal/config.rs b/server/src/engine/fractal/config.rs
new file mode 100644
index 00000000..5b898ec0
--- /dev/null
+++ b/server/src/engine/fractal/config.rs
@@ -0,0 +1,47 @@
+/*
+ * Created on Sun Sep 10 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::storage::v1::header_meta::HostRunMode;
+
+pub struct ServerConfig {
+    host_settings_version: u32,
+    host_run_mode: HostRunMode,
+    host_startup_counter: u64,
+}
+
+impl ServerConfig {
+    pub fn new(
+        host_settings_version: u32,
+        host_run_mode: HostRunMode,
+        host_startup_counter: u64,
+    ) -> Self {
+        Self {
+            host_settings_version,
+            host_run_mode,
+            host_startup_counter,
+        }
+    }
+}
diff --git a/server/src/engine/fractal/drivers.rs b/server/src/engine/fractal/drivers.rs
new file mode 100644
index 00000000..8bd42c9d
--- /dev/null
+++ b/server/src/engine/fractal/drivers.rs
@@ -0,0 +1,88 @@
+/*
+ * Created on Sun Sep 10 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::util,
+    crate::engine::{
+        storage::v1::{data_batch::DataBatchPersistDriver, LocalFS},
+        txn::gns::GNSTransactionDriverAnyFS,
+    },
+    parking_lot::Mutex,
+    std::sync::Arc,
+};
+
+/// GNS driver
+pub(super) struct FractalGNSDriver {
+    status: util::Status,
+    txn_driver: Mutex<GNSTransactionDriverAnyFS<LocalFS>>,
+}
+
+impl FractalGNSDriver {
+    pub(super) fn new(txn_driver: GNSTransactionDriverAnyFS<LocalFS>) -> Self {
+        Self {
+            status: util::Status::new_okay(),
+            txn_driver: Mutex::new(txn_driver),
+        }
+    }
+}
+
+/// Model driver
+pub struct FractalModelDriver {
+    hooks: Arc<FractalModelHooks>,
+    batch_driver: Mutex<DataBatchPersistDriver<LocalFS>>,
+}
+
+impl FractalModelDriver {
+    /// Initialize a model driver with default settings
+    pub fn init(batch_driver: DataBatchPersistDriver<LocalFS>) -> Self {
+        Self {
+            hooks: Arc::new(FractalModelHooks::new()),
+            batch_driver: Mutex::new(batch_driver),
+        }
+    }
+    /// Returns a reference to the batch persist driver
+    pub fn batch_driver(&self) -> &Mutex<DataBatchPersistDriver<LocalFS>> {
+        &self.batch_driver
+    }
+}
+
+/// Model hooks
+#[derive(Debug)]
+pub struct FractalModelHooks {
+    status: util::Status,
+}
+
+impl FractalModelHooks {
+    #[cfg(test)]
+    pub fn test() -> Self {
+        Self::new()
+    }
+    fn new() -> Self {
+        Self {
+            status: util::Status::new_okay(),
+        }
+    }
+}
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
new file mode 100644
index 00000000..51dc80ac
--- /dev/null
+++ b/server/src/engine/fractal/mgr.rs
@@ -0,0 +1,303 @@
+/*
+ * Created on Sat Sep 09 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::ModelUniqueID,
+    crate::{
+        engine::core::model::{delta::DataDelta, Model},
+        util::os,
+    },
+    std::path::PathBuf,
+    tokio::{
+        fs,
+        sync::mpsc::{UnboundedReceiver, UnboundedSender},
+        task::JoinHandle,
+    },
+};
+
+/// A task for the [`FractalMgr`] to perform
+pub struct Task<T> {
+    threshold: usize,
+    task: T,
+}
+
+impl<T> Task<T> {
+    const THRESHOLD: usize = 10;
+    /// Create a new task with the default threshold
+    pub fn new(task: T) -> Self {
+        Self::with_threshold(task, Self::THRESHOLD)
+    }
+    /// Create a task with the given threshold
+    fn with_threshold(task: T, threshold: usize) -> Self {
+        Self { threshold, task }
+    }
+}
+
+/// A general task
+pub enum GenericTask {
+    /// Delete a single file
+    DeleteFile(PathBuf),
+    /// Delete a directory (and all its children)
+    DeleteDirAll(PathBuf),
+}
+
+/// A critical task
+pub enum CriticalTask {
+    /// Write a new data batch
+    WriteBatch(ModelUniqueID, usize),
+}
+
+/// The task manager
+pub(super) struct FractalMgr {
+    hp_dispatcher: UnboundedSender<Task<CriticalTask>>,
+    general_dispatcher: UnboundedSender<Task<GenericTask>>,
+    runtime_stats: FractalRTStat,
+}
+
+pub(super) struct FractalRTStat {
+    mem_free_bytes: u64,
+    per_mdl_delta_max_size: usize,
+}
+
+impl FractalRTStat {
+    fn init(model_cnt: usize) -> Self {
+        let mem_free_bytes = os::free_memory_in_bytes();
+        let allowed_delta_limit = mem_free_bytes as f64 * 0.02;
+        let per_model_limit = allowed_delta_limit / model_cnt.max(1) as f64;
+        Self {
+            mem_free_bytes,
+            per_mdl_delta_max_size: per_model_limit as usize / sizeof!(DataDelta),
+        }
+    }
+    pub(super) fn mem_free_bytes(&self) -> u64 {
+        self.mem_free_bytes
+    }
+    pub(super) fn per_mdl_delta_max_size(&self) -> usize {
+        self.per_mdl_delta_max_size
+    }
+}
+
+impl FractalMgr {
+    pub(super) fn new(
+        hp_dispatcher: UnboundedSender<Task<CriticalTask>>,
+        general_dispatcher: UnboundedSender<Task<GenericTask>>,
+        model_count: usize,
+    ) -> Self {
+        Self {
+            hp_dispatcher,
+            general_dispatcher,
+            runtime_stats: FractalRTStat::init(model_count),
+        }
+    }
+    pub fn get_rt_stat(&self) -> &FractalRTStat {
+        &self.runtime_stats
+    }
+    /// Add a high priority task to the queue
+    ///
+    /// ## Panics
+    ///
+    /// This will panic if the high priority executor has crashed or exited
+    pub fn post_high_priority(&self, task: Task<CriticalTask>) {
+        self.hp_dispatcher.send(task).unwrap()
+    }
+    /// Add a low priority task to the queue
+    ///
+    /// ## Panics
+    ///
+    /// This will panic if the low priority executor has crashed or exited
+    pub fn post_low_priority(&self, task: Task<GenericTask>) {
+        self.general_dispatcher.send(task).unwrap()
+    }
+}
+
+/// Handles to all the services that fractal needs. These are spawned on the default runtime
+pub struct FractalServiceHandles {
+    pub hp_handle: JoinHandle<()>,
+    pub lp_handle: JoinHandle<()>,
+}
+
+impl FractalMgr {
+    /// Start all background services, and return their handles
+    pub(super) fn start_all(
+        global: super::Global,
+        lp_receiver: UnboundedReceiver<Task<GenericTask>>,
+        hp_receiver: UnboundedReceiver<Task<CriticalTask>>,
+    ) -> FractalServiceHandles {
+        let fractal_mgr = global.get_state().fractal_mgr();
+        let hp_handle = tokio::spawn(async move {
+            FractalMgr::hp_executor_svc(fractal_mgr, global, hp_receiver).await
+        });
+        let lp_handle = tokio::spawn(async move {
+            FractalMgr::general_executor_svc(fractal_mgr, global, lp_receiver).await
+        });
+        FractalServiceHandles {
+            hp_handle,
+            lp_handle,
+        }
+    }
+}
+
+// services
+impl FractalMgr {
+    const GENERAL_EXECUTOR_WINDOW: u64 = 5 * 60;
+    /// The high priority executor service runs in the background to take care of high priority tasks and take any
+    /// appropriate action. It will exclusively own the high priority queue since it is the only broker that is
+    /// allowed to perform HP tasks
+    pub async fn hp_executor_svc(
+        &'static self,
+        global: super::Global,
+        mut receiver: UnboundedReceiver<Task<CriticalTask>>,
+    ) {
+        loop {
+            let Some(Task { threshold, task }) = receiver.recv().await else {
+                return; // all handles closed; nothing left to do
+            };
+            // TODO(@ohsayan): check threshold and update hooks
+            match task {
+                CriticalTask::WriteBatch(model_id, observed_size) => {
+                    let mdl_drivers = global.get_state().get_mdl_drivers().read();
+                    let Some(mdl_driver) = mdl_drivers.get(&model_id) else {
+                        // because we maximize throughput, the model driver may have been already removed but this task
+                        // was way behind in the queue
+                        continue;
+                    };
+                    let res = global.namespace().with_model(
+                        (model_id.space(), model_id.model()),
+                        |model| {
+                            if model.get_uuid() != model_id.uuid() {
+                                // once again, throughput maximization will lead to, in extremely rare cases, this
+                                // branch returning. but it is okay
+                                return Ok(());
+                            }
+                            // mark that we're taking these deltas
+                            model.delta_state().__fractal_take_from_data_delta(
+                                observed_size,
+                                super::FractalToken::new(),
+                            );
+                            Self::try_write_model_data_batch(model, observed_size, mdl_driver)
+                        },
+                    );
+                    match res {
+                        Ok(()) => {}
+                        Err(_) => {
+                            log::error!(
+                                "Error writing data batch for model {}. Retrying...",
+                                model_id.uuid()
+                            );
+                            // enqueue again for retrying
+                            self.hp_dispatcher
+                                .send(Task::with_threshold(
+                                    CriticalTask::WriteBatch(model_id, observed_size),
+                                    threshold - 1,
+                                ))
+                                .unwrap();
+                        }
+                    }
+                }
+            }
+        }
+    }
+    /// The general priority task or simply the general queue takes of care of low priority and other standard priority
+    /// tasks (such as those running on a schedule). A low priority task can be promoted to a high priority task, and the
+    /// discretion of the GP executor. Similarly, the executor owns the general purpose task queue since it is the sole broker
+    /// for such tasks
+    pub async fn general_executor_svc(
+        &'static self,
+        global: super::Global,
+        mut lpq: UnboundedReceiver<Task<GenericTask>>,
+    ) {
+        loop {
+            tokio::select! {
+                _ = tokio::time::sleep(std::time::Duration::from_secs(Self::GENERAL_EXECUTOR_WINDOW)) => {
+                    let mdl_drivers = global.get_state().get_mdl_drivers().read();
+                    for (model_id, driver) in mdl_drivers.iter() {
+                        let mut observed_len = 0;
+                        let res = global.namespace().with_model((model_id.space(), model_id.model()), |model| {
+                            if model.get_uuid() != model_id.uuid() {
+                                // once again, throughput maximization will lead to, in extremely rare cases, this
+                                // branch returning. but it is okay
+                                return Ok(());
+                            }
+                            // mark that we're taking these deltas
+                            observed_len = model.delta_state().__fractal_take_full_from_data_delta(super::FractalToken::new());
+                            Self::try_write_model_data_batch(model, observed_len, driver)
+                        });
+                        match res {
+                            Ok(()) => {}
+                            Err(_) => {
+                                // this failure is *not* good, so we want to promote this to a critical task
+                                self.hp_dispatcher.send(Task::new(CriticalTask::WriteBatch(model_id.clone(), observed_len))).unwrap()
+                            }
+                        }
+                    }
+                }
+                task = lpq.recv() => {
+                    let Some(Task { threshold, task }) = task else {
+                        return;
+                    };
+                    // TODO(@ohsayan): threshold
+                    match task {
+                        GenericTask::DeleteFile(f) => {
+                            if let Err(_) = fs::remove_file(&f).await {
+                                self.general_dispatcher.send(
+                                    Task::with_threshold(GenericTask::DeleteFile(f), threshold - 1)
+                                ).unwrap();
+                            }
+                        }
+                        GenericTask::DeleteDirAll(dir) => {
+                            if let Err(_) = fs::remove_dir_all(&dir).await {
+                                self.general_dispatcher.send(
+                                    Task::with_threshold(GenericTask::DeleteDirAll(dir), threshold - 1)
+                                ).unwrap();
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+// util
+impl FractalMgr {
+    /// Attempt to write a model data batch with the observed size.
+    ///
+    /// The zero check is essential
+    fn try_write_model_data_batch(
+        model: &Model,
+        observed_size: usize,
+        mdl_driver: &super::FractalModelDriver,
+    ) -> Result<(), crate::engine::error::DatabaseError> {
+        if observed_size == 0 {
+            // no changes, all good
+            return Ok(());
+        }
+        // try flushing the batch
+        let mut batch_driver = mdl_driver.batch_driver().lock();
+        batch_driver.write_new_batch(model, observed_size)?;
+        Ok(())
+    }
+}
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
new file mode 100644
index 00000000..5d6bb0a6
--- /dev/null
+++ b/server/src/engine/fractal/mod.rs
@@ -0,0 +1,204 @@
+/*
+ * Created on Sat Sep 09 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{
+        core::GlobalNS, data::uuid::Uuid, storage::v1::LocalFS, txn::gns::GNSTransactionDriverAnyFS,
+    },
+    parking_lot::RwLock,
+    std::{collections::HashMap, mem::MaybeUninit},
+    tokio::sync::mpsc::unbounded_channel,
+};
+
+mod config;
+mod drivers;
+mod mgr;
+mod util;
+pub use {
+    config::ServerConfig,
+    drivers::FractalModelDriver,
+    mgr::{CriticalTask, GenericTask, Task},
+    util::FractalToken,
+};
+
+pub type ModelDrivers = HashMap<ModelUniqueID, drivers::FractalModelDriver>;
+
+static mut GLOBAL: MaybeUninit<GlobalState> = MaybeUninit::uninit();
+
+/*
+    global state init
+*/
+
+/// Returned by [`enable_and_start_all`]. This contains a [`Global`] handle that can be used to easily access global
+/// data
+pub struct GlobalStateStart {
+    pub global: Global,
+    pub mgr_handles: mgr::FractalServiceHandles,
+}
+
+/// Enable all drivers and start all engines
+///
+/// ## Safety
+///
+/// Must be called iff this is the only thread calling it
+pub unsafe fn enable_and_start_all(
+    gns: GlobalNS,
+    config: config::ServerConfig,
+    gns_driver: GNSTransactionDriverAnyFS<LocalFS>,
+    model_drivers: ModelDrivers,
+) -> GlobalStateStart {
+    let model_cnt_on_boot = model_drivers.len();
+    let gns_driver = drivers::FractalGNSDriver::new(gns_driver);
+    let mdl_driver = RwLock::new(model_drivers);
+    let (hp_sender, hp_recv) = unbounded_channel();
+    let (lp_sender, lp_recv) = unbounded_channel();
+    let global_state = GlobalState::new(
+        gns,
+        gns_driver,
+        mdl_driver,
+        mgr::FractalMgr::new(hp_sender, lp_sender, model_cnt_on_boot),
+        config,
+    );
+    GLOBAL = MaybeUninit::new(global_state);
+    let token = Global::new();
+    GlobalStateStart {
+        global: token,
+        mgr_handles: mgr::FractalMgr::start_all(token, lp_recv, hp_recv),
+    }
+}
+
+/*
+    global access
+*/
+
+#[derive(Debug, Clone, Copy)]
+/// A handle to the global state
+pub struct Global(());
+
+impl Global {
+    fn new() -> Self {
+        Self(())
+    }
+    pub(self) fn get_state(&self) -> &'static GlobalState {
+        unsafe { GLOBAL.assume_init_ref() }
+    }
+    /// Returns a handle to the [`GlobalNS`]
+    pub fn namespace(&self) -> &'static GlobalNS {
+        &unsafe { GLOBAL.assume_init_ref() }.gns
+    }
+    /// Post an urgent task
+    pub fn post_high_priority_task(&self, task: Task<CriticalTask>) {
+        self.get_state().fractal_mgr().post_high_priority(task)
+    }
+    /// Post a task with normal priority
+    ///
+    /// NB: It is not guaranteed that the task will remain as a low priority task because the scheduler can choose
+    /// to promote the task to a high priority task, if it deems necessary.
+    pub fn post_standard_priority_task(&self, task: Task<GenericTask>) {
+        self.get_state().fractal_mgr().post_low_priority(task)
+    }
+    /// Returns the maximum size a model's delta size can hit before it should immediately issue a batch write request
+    /// to avoid memory pressure
+    pub fn get_max_delta_size(&self) -> usize {
+        self.get_state()
+            .fractal_mgr()
+            .get_rt_stat()
+            .per_mdl_delta_max_size()
+    }
+}
+
+/*
+    global state
+*/
+
+/// The global state
+struct GlobalState {
+    gns: GlobalNS,
+    gns_driver: drivers::FractalGNSDriver,
+    mdl_driver: RwLock<ModelDrivers>,
+    task_mgr: mgr::FractalMgr,
+    config: config::ServerConfig,
+}
+
+impl GlobalState {
+    fn new(
+        gns: GlobalNS,
+        gns_driver: drivers::FractalGNSDriver,
+        mdl_driver: RwLock<ModelDrivers>,
+        task_mgr: mgr::FractalMgr,
+        config: config::ServerConfig,
+    ) -> Self {
+        Self {
+            gns,
+            gns_driver,
+            mdl_driver,
+            task_mgr,
+            config,
+        }
+    }
+    pub(self) fn get_mdl_drivers(&self) -> &RwLock<ModelDrivers> {
+        &self.mdl_driver
+    }
+    pub(self) fn fractal_mgr(&self) -> &mgr::FractalMgr {
+        &self.task_mgr
+    }
+}
+
+// these impls are completely fine
+unsafe impl Send for GlobalState {}
+unsafe impl Sync for GlobalState {}
+
+/// An unique signature that identifies a model, and only that model (guaranteed by the OS's random source)
+// NB(@ohsayan): if there are collisions, which I absolutely do not expect any instances of, pool in the space's UUID
+#[derive(Debug, PartialEq, Eq, Hash, Clone)]
+pub struct ModelUniqueID {
+    space: Box<str>,
+    model: Box<str>,
+    uuid: Uuid,
+}
+
+impl ModelUniqueID {
+    /// Create a new unique model ID
+    pub fn new(space: &str, model: &str, uuid: Uuid) -> Self {
+        Self {
+            space: space.into(),
+            model: model.into(),
+            uuid,
+        }
+    }
+    /// Returns the space name
+    pub fn space(&self) -> &str {
+        self.space.as_ref()
+    }
+    /// Returns the model name
+    pub fn model(&self) -> &str {
+        self.model.as_ref()
+    }
+    /// Returns the uuid
+    pub fn uuid(&self) -> Uuid {
+        self.uuid
+    }
+}
diff --git a/server/src/engine/fractal/util.rs b/server/src/engine/fractal/util.rs
new file mode 100644
index 00000000..c3bb986b
--- /dev/null
+++ b/server/src/engine/fractal/util.rs
@@ -0,0 +1,78 @@
+/*
+ * Created on Sat Sep 09 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use std::sync::atomic::{AtomicBool, Ordering};
+
+#[derive(Debug)]
+pub struct Status {
+    okay: AtomicBool,
+}
+
+impl Status {
+    pub const fn new_okay() -> Self {
+        Self::new(true)
+    }
+    pub const fn new_iffy() -> Self {
+        Self::new(false)
+    }
+    const fn new(v: bool) -> Self {
+        Self {
+            okay: AtomicBool::new(v),
+        }
+    }
+}
+
+impl Status {
+    pub fn is_iffy(&self) -> bool {
+        !self._get()
+    }
+    pub fn is_healthy(&self) -> bool {
+        self._get()
+    }
+    fn _get(&self) -> bool {
+        self.okay.load(Ordering::Acquire)
+    }
+}
+
+impl Status {
+    pub(super) fn set_okay(&self) {
+        self._set(true)
+    }
+    pub(super) fn set_iffy(&self) {
+        self._set(false)
+    }
+    fn _set(&self, v: bool) {
+        self.okay.store(v, Ordering::Release)
+    }
+}
+
+/// A special token for fractal calls
+pub struct FractalToken(());
+impl FractalToken {
+    pub(super) fn new() -> Self {
+        Self(())
+    }
+}
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 7bc029f4..4d43a88f 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -31,6 +31,7 @@ mod macros;
 mod core;
 mod data;
 mod error;
+mod fractal;
 mod idx;
 mod mem;
 mod ql;
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 9d27d045..0a306411 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -41,12 +41,11 @@ use {
         },
         util::{compiler, MaybeInit},
     },
-    core::cmp,
 };
 
 #[inline(always)]
 pub fn minidx<T>(src: &[T], index: usize) -> usize {
-    cmp::min(src.len() - 1, index)
+    (src.len() - 1).min(index)
 }
 
 #[derive(Debug, PartialEq)]
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index 608ccbed..fabf4b47 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -36,7 +36,6 @@ use {
         },
         util::{compiler, MaybeInit},
     },
-    core::cmp,
     std::{
         collections::HashMap,
         time::{Duration, SystemTime, UNIX_EPOCH},
@@ -114,7 +113,7 @@ fn hashp(key: &[u8]) -> u32 {
 fn ldfunc(func: Ident<'_>) -> Option<ProducerFn> {
     let func = func.as_bytes();
     let ph = hashp(func) as usize;
-    let min = cmp::min(ph, PRODUCER_F.len() - 1);
+    let min = ph.min(PRODUCER_F.len() - 1);
     let data = PRODUCER_F[min];
     if data.0 == func {
         Some(data.1)
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index b376631f..839d795e 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -38,7 +38,7 @@ use {
         },
         util::compiler,
     },
-    core::{cmp, fmt, ops::BitOr, slice, str},
+    core::{fmt, ops::BitOr, slice, str},
 };
 
 pub use self::raw::{Ident, Keyword, Symbol, Token};
@@ -484,7 +484,7 @@ impl<'a> SafeQueryData<'a> {
         while src.len() >= 3 && okay {
             let tc = src[0];
             okay &= tc <= nonpadded_offset;
-            let mx = cmp::min(ecc_offset, tc as usize);
+            let mx = ecc_offset.min(tc as usize);
             let mut i_ = 1;
             okay &= LITIR_TF[mx](&src[1..], &mut i_, &mut data);
             src = &src[i_..];
@@ -508,7 +508,7 @@ impl<'b> SafeQueryData<'b> {
         let src = &src[i..];
         // find payload
         *flag &= src.len() >= payload_len;
-        let mx_extract = cmp::min(payload_len, src.len());
+        let mx_extract = payload_len.min(src.len());
         // incr cursor
         i += mx_extract;
         *cnt += i;
@@ -534,7 +534,7 @@ impl<'b> SafeQueryData<'b> {
     #[inline(always)]
     pub(super) fn bool<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
         // `true\n` or `false\n`
-        let mx = cmp::min(6, src.len());
+        let mx = 6.min(src.len());
         let slice = &src[..mx];
         let v_true = slice.starts_with(b"true\n");
         let v_false = slice.starts_with(b"false\n");
diff --git a/server/src/engine/storage/v1/batch_jrnl/mod.rs b/server/src/engine/storage/v1/batch_jrnl/mod.rs
index 68b32d4b..0632e27e 100644
--- a/server/src/engine/storage/v1/batch_jrnl/mod.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/mod.rs
@@ -43,3 +43,45 @@ const RECOVERY_THRESHOLD: usize = 10;
 #[cfg(test)]
 pub(super) use restore::{DecodedBatchEvent, DecodedBatchEventKind, NormalBatch};
 pub use {persist::DataBatchPersistDriver, restore::DataBatchRestoreDriver};
+
+use {
+    super::{
+        header_meta,
+        rw::{FileOpen, SDSSFileIO},
+        RawFSInterface, SDSSResult,
+    },
+    crate::engine::core::model::Model,
+};
+
+const LOG_SPECIFIER_VERSION: header_meta::FileSpecifierVersion =
+    header_meta::FileSpecifierVersion::__new(0);
+
+pub fn open_or_reinit<Fs: RawFSInterface>(
+    name: &str,
+    model: &Model,
+    host_setting_version: u32,
+    host_run_mode: header_meta::HostRunMode,
+    host_startup_counter: u64,
+) -> SDSSResult<DataBatchPersistDriver<Fs>> {
+    let f = SDSSFileIO::<Fs>::open_or_create_perm_rw::<false>(
+        name,
+        header_meta::FileScope::Journal,
+        header_meta::FileSpecifier::TableDataBatch,
+        LOG_SPECIFIER_VERSION,
+        host_setting_version,
+        host_run_mode,
+        host_startup_counter,
+    )?;
+    match f {
+        FileOpen::Created(new_file) => Ok(DataBatchPersistDriver::new(new_file, true)?),
+        FileOpen::Existing(existing, _) => {
+            // restore
+            let mut restore_driver = DataBatchRestoreDriver::new(existing)?;
+            restore_driver.read_data_batch_into_model(model)?;
+            Ok(DataBatchPersistDriver::new(
+                restore_driver.into_file(),
+                false,
+            )?)
+        }
+    }
+}
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index 660a4569..17e9485a 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -27,7 +27,7 @@
 use {
     super::{
         obj::{self, FieldMD},
-        PersistTypeDscr, PersistMapSpec, PersistObject, VecU8,
+        PersistMapSpec, PersistObject, PersistTypeDscr, VecU8,
     },
     crate::{
         engine::{
@@ -44,7 +44,6 @@ use {
         util::{copy_slice_to_array as memcpy, EndianQW},
     },
     core::marker::PhantomData,
-    std::cmp,
 };
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord)]
@@ -177,7 +176,7 @@ impl PersistMapSpec for GenericDictSpec {
     }
     fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool {
         static EXPECT_ATLEAST: [u8; 4] = [0, 1, 8, 8]; // PAD to align
-        let lbound_rem = md.klen + EXPECT_ATLEAST[cmp::min(md.dscr, 3) as usize] as usize;
+        let lbound_rem = md.klen + EXPECT_ATLEAST[md.dscr.min(3) as usize] as usize;
         scanner.has_left(lbound_rem) & (md.dscr <= PersistTypeDscr::Dict.value_u8())
     }
     fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) {
@@ -256,11 +255,7 @@ impl PersistMapSpec for GenericDictSpec {
                             return None;
                         }
                         v.push(
-                            match decode_element(
-                                scanner,
-                                PersistTypeDscr::from_raw(dscr),
-                                false,
-                            ) {
+                            match decode_element(scanner, PersistTypeDscr::from_raw(dscr), false) {
                                 Some(DictEntryGeneric::Data(l)) => l,
                                 None => return None,
                                 _ => unreachable!("found top-level dict item in datacell"),
diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs
new file mode 100644
index 00000000..2c3876e5
--- /dev/null
+++ b/server/src/engine/storage/v1/loader.rs
@@ -0,0 +1,100 @@
+/*
+ * Created on Sun Sep 10 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    core::GlobalNS,
+    data::uuid::Uuid,
+    fractal::{FractalModelDriver, ModelDrivers, ModelUniqueID},
+    storage::v1::{batch_jrnl, header_meta::HostRunMode, LocalFS, SDSSErrorContext, SDSSResult},
+    txn::gns::GNSTransactionDriverAnyFS,
+};
+
+const GNS_FILE_PATH: &str = "gns.db-tlog";
+
+pub struct SEInitState {
+    pub txn_driver: GNSTransactionDriverAnyFS<super::LocalFS>,
+    pub model_drivers: ModelDrivers,
+}
+
+impl SEInitState {
+    pub fn new(
+        txn_driver: GNSTransactionDriverAnyFS<super::LocalFS>,
+        model_drivers: ModelDrivers,
+    ) -> Self {
+        Self {
+            txn_driver,
+            model_drivers,
+        }
+    }
+    pub fn try_init(
+        host_setting_version: u32,
+        host_run_mode: HostRunMode,
+        host_startup_counter: u64,
+    ) -> SDSSResult<Self> {
+        let gns = GlobalNS::empty();
+        let gns_txn_driver = GNSTransactionDriverAnyFS::<LocalFS>::open_or_reinit_with_name(
+            &gns,
+            GNS_FILE_PATH,
+            host_setting_version,
+            host_run_mode,
+            host_startup_counter,
+        )?;
+        let mut model_drivers = ModelDrivers::new();
+        for (space_name, space) in gns.spaces().read().iter() {
+            let space_uuid = space.get_uuid();
+            for (model_name, model) in space.models().read().iter() {
+                let path = Self::model_path(space_name, space_uuid, model_name, model.get_uuid());
+                let persist_driver = match batch_jrnl::open_or_reinit(
+                    &path,
+                    model,
+                    host_setting_version,
+                    host_run_mode,
+                    host_startup_counter,
+                ) {
+                    Ok(j) => j,
+                    Err(e) => {
+                        return Err(e.with_extra(format!(
+                            "failed to restore model data from journal in `{path}`"
+                        )))
+                    }
+                };
+                let _ = model_drivers.insert(
+                    ModelUniqueID::new(space_name, model_name, model.get_uuid()),
+                    FractalModelDriver::init(persist_driver),
+                );
+            }
+        }
+        Ok(SEInitState::new(gns_txn_driver, model_drivers))
+    }
+    fn model_path(
+        space_name: &str,
+        space_uuid: Uuid,
+        model_name: &str,
+        model_uuid: Uuid,
+    ) -> String {
+        format!("data/{space_name}-{space_uuid}/{model_name}-{model_uuid}/data.db-btlog")
+    }
+}
diff --git a/server/src/engine/storage/v1/memfs.rs b/server/src/engine/storage/v1/memfs.rs
index 7c81f548..d6480519 100644
--- a/server/src/engine/storage/v1/memfs.rs
+++ b/server/src/engine/storage/v1/memfs.rs
@@ -45,6 +45,8 @@ use {
 static VFS: Lazy<RwLock<HashMap<Box<str>, VNode>>, fn() -> RwLock<HashMap<Box<str>, VNode>>> =
     Lazy::new(|| Default::default());
 
+type ComponentIter<'a> = std::iter::Take<std::vec::IntoIter<&'a str>>;
+
 /*
     vnode
     ---
@@ -79,6 +81,17 @@ impl VNode {
     - make child
 */
 
+fn split_parts(fpath: &str) -> Vec<&str> {
+    fpath.split("/").collect()
+}
+
+fn split_target_and_components(fpath: &str) -> (&str, ComponentIter) {
+    let parts = split_parts(fpath);
+    let target = parts.last().unwrap();
+    let component_len = parts.len() - 1;
+    (target, parts.into_iter().take(component_len))
+}
+
 #[derive(Debug)]
 pub struct VirtualFS;
 
@@ -88,14 +101,10 @@ impl RawFSInterface for VirtualFS {
         // get vfs
         let mut vfs = VFS.write();
         // get root dir
-        let path = fpath.split("/").collect::<Vec<&str>>();
-        // get target
-        let target = *path.last().unwrap();
         let mut current = &mut *vfs;
         // process components
-        let component_len = path.len() - 1;
-        let mut path = path.into_iter().take(component_len);
-        while let Some(component) = path.next() {
+        let (target, mut components) = split_target_and_components(fpath);
+        while let Some(component) = components.next() {
             match current.get_mut(component) {
                 Some(VNode::Dir(d)) => {
                     current = d;
@@ -147,7 +156,7 @@ impl RawFSInterface for VirtualFS {
                 }
             }
         }
-        let pieces: Vec<&str> = fpath.split("/").collect();
+        let pieces = split_parts(fpath);
         create_ahead(&pieces, &mut *vfs)
     }
     fn fs_delete_dir(fpath: &str) -> super::SDSSResult<()> {
@@ -159,10 +168,9 @@ impl RawFSInterface for VirtualFS {
     fn fs_fopen_or_create_rw(fpath: &str) -> super::SDSSResult<super::rw::RawFileOpen<Self::File>> {
         let mut vfs = VFS.write();
         // components
-        let components = fpath.split("/").collect::<Vec<&str>>();
-        let file = components.last().unwrap().to_owned().into();
+        let (target_file, components) = split_target_and_components(fpath);
         let target_dir = find_target_dir_mut(components, &mut vfs)?;
-        match target_dir.entry(file) {
+        match target_dir.entry(target_file.into()) {
             Entry::Occupied(mut oe) => match oe.get_mut() {
                 VNode::File(f) => {
                     f.read = true;
@@ -184,11 +192,10 @@ impl RawFSInterface for VirtualFS {
 }
 
 fn find_target_dir_mut<'a>(
-    components: Vec<&str>,
+    components: ComponentIter,
     mut current: &'a mut HashMap<Box<str>, VNode>,
 ) -> Result<&'a mut HashMap<Box<str>, VNode>, super::SDSSError> {
-    let path_len = components.len() - 1;
-    for component in components.into_iter().take(path_len) {
+    for component in components {
         match current.get_mut(component) {
             Some(VNode::Dir(d)) => current = d,
             Some(VNode::File(_)) => {
@@ -205,11 +212,10 @@ fn find_target_dir_mut<'a>(
 }
 
 fn find_target_dir<'a>(
-    components: Vec<&str>,
+    components: ComponentIter,
     mut current: &'a HashMap<Box<str>, VNode>,
 ) -> Result<&'a HashMap<Box<str>, VNode>, super::SDSSError> {
-    let path_len = components.len() - 1;
-    for component in components.into_iter().take(path_len) {
+    for component in components {
         match current.get(component) {
             Some(VNode::Dir(d)) => current = d,
             Some(VNode::File(_)) => {
@@ -229,10 +235,8 @@ fn delete_dir(fpath: &str, allow_if_non_empty: bool) -> Result<(), super::SDSSEr
     let mut vfs = VFS.write();
     let mut current = &mut *vfs;
     // process components
-    let components = fpath.split("/").collect::<Vec<&str>>();
-    let components_len = components.len() - 1;
-    let target = *components.last().unwrap();
-    for component in components.into_iter().take(components_len) {
+    let (target, components) = split_target_and_components(fpath);
+    for component in components {
         match current.get_mut(component) {
             Some(VNode::Dir(dir)) => {
                 current = dir;
@@ -310,10 +314,9 @@ impl Drop for VFileDescriptor {
 
 fn with_file_mut<T>(fpath: &str, mut f: impl FnMut(&mut VFile) -> SDSSResult<T>) -> SDSSResult<T> {
     let mut vfs = VFS.write();
-    let components = fpath.split("/").collect::<Vec<&str>>();
-    let file = *components.last().unwrap();
+    let (target_file, components) = split_target_and_components(fpath);
     let target_dir = find_target_dir_mut(components, &mut vfs)?;
-    match target_dir.get_mut(file) {
+    match target_dir.get_mut(target_file) {
         Some(VNode::File(file)) => f(file),
         Some(VNode::Dir(_)) => {
             return Err(Error::new(ErrorKind::InvalidInput, "found directory, not a file").into())
@@ -324,10 +327,9 @@ fn with_file_mut<T>(fpath: &str, mut f: impl FnMut(&mut VFile) -> SDSSResult<T>)
 
 fn with_file<T>(fpath: &str, mut f: impl FnMut(&VFile) -> SDSSResult<T>) -> SDSSResult<T> {
     let vfs = VFS.read();
-    let components = fpath.split("/").collect::<Vec<&str>>();
-    let file = *components.last().unwrap();
+    let (target_file, components) = split_target_and_components(fpath);
     let target_dir = find_target_dir(components, &vfs)?;
-    match target_dir.get(file) {
+    match target_dir.get(target_file) {
         Some(VNode::File(file)) => f(file),
         Some(VNode::Dir(_)) => {
             return Err(Error::new(ErrorKind::InvalidInput, "found directory, not a file").into())
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index ca269318..ca060411 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -29,6 +29,7 @@ mod header_impl;
 // impls
 mod batch_jrnl;
 mod journal;
+mod loader;
 mod rw;
 // hl
 pub mod inf;
@@ -44,11 +45,14 @@ pub use {
     memfs::NullFS,
     rw::{BufferedScanner, LocalFS, RawFSInterface, SDSSFileIO},
 };
+pub mod data_batch {
+    pub use super::batch_jrnl::{DataBatchPersistDriver, DataBatchRestoreDriver};
+}
 pub mod header_meta {
     pub use super::header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode};
 }
 
-use crate::util::os::SysIOError as IoError;
+use crate::{engine::txn::TransactionError, util::os::SysIOError as IoError};
 
 pub type SDSSResult<T> = Result<T, SDSSError>;
 
@@ -71,6 +75,14 @@ impl SDSSErrorContext for std::io::Error {
     }
 }
 
+impl SDSSErrorContext for SDSSError {
+    type ExtraData = String;
+
+    fn with_extra(self, extra: Self::ExtraData) -> SDSSError {
+        SDSSError::Extra(Box::new(self), extra)
+    }
+}
+
 #[derive(Debug)]
 #[cfg_attr(test, derive(PartialEq))]
 pub enum SDSSError {
@@ -121,6 +133,16 @@ pub enum SDSSError {
     /// we failed to close the data batch
     DataBatchCloseError,
     DataBatchRestoreCorruptedBatchFile,
+    JournalRestoreTxnError,
+    /// An error with more context
+    // TODO(@ohsayan): avoid the box; we'll clean this up soon
+    Extra(Box<Self>, String),
+}
+
+impl From<TransactionError> for SDSSError {
+    fn from(_: TransactionError) -> Self {
+        Self::JournalRestoreTxnError
+    }
 }
 
 impl SDSSError {
diff --git a/server/src/engine/txn/data.rs b/server/src/engine/txn/data.rs
deleted file mode 100644
index 38c70437..00000000
--- a/server/src/engine/txn/data.rs
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Created on Mon Aug 28 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{
-    engine::core::{model::delta::DataDelta, GlobalNS},
-    util::os,
-};
-
-type Buf = Vec<u8>;
-
-/*
-    memory adjustments
-*/
-
-/// free memory in bytes
-static mut FREEMEM_BYTES: u64 = 0;
-/// capacity in bytes, per linked list
-static mut CAP_PER_LL_BYTES: u64 = 0;
-/// maximum number of nodes in linked list
-static mut MAX_NODES_IN_LL_CNT: usize = 0;
-
-/// Set the free memory and cap for deltas so that we don't bust through memory
-///
-/// ## Safety
-/// - All models must have been loaded
-/// - This must be called **before** the arbiter spawns threads for connections
-pub unsafe fn set_limits(gns: &GlobalNS) {
-    let model_cnt: usize = gns
-        .spaces()
-        .read()
-        .values()
-        .map(|space| space.models().read().len())
-        .sum();
-    let available_mem = os::free_memory_in_bytes();
-    FREEMEM_BYTES = available_mem;
-    CAP_PER_LL_BYTES =
-        ((available_mem / core::cmp::max(1, model_cnt) as u64) as f64 * 0.002) as u64;
-    MAX_NODES_IN_LL_CNT = CAP_PER_LL_BYTES as usize / (sizeof!(DataDelta) + sizeof!(u64));
-}
-
-/// Returns the maximum number of nodes that can be stored inside a delta queue for a model
-///
-/// Currently hardcoded to 0.2% of free memory after all datasets have been loaded
-pub unsafe fn get_max_delta_queue_size() -> usize {
-    // TODO(@ohsayan): dynamically approximate this limit
-    MAX_NODES_IN_LL_CNT
-}
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index a290f53e..73938417 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -41,7 +41,7 @@ use {
         },
         util::EndianQW,
     },
-    std::{fs::File, marker::PhantomData},
+    std::marker::PhantomData,
 };
 
 mod model;
@@ -61,7 +61,6 @@ pub use {
 
 pub type GNSTransactionDriverNullZero =
     GNSTransactionDriverAnyFS<crate::engine::storage::v1::NullFS>;
-pub type GNSTransactionDriver = GNSTransactionDriverAnyFS<File>;
 #[cfg(test)]
 pub type GNSTransactionDriverVFS = GNSTransactionDriverAnyFS<VirtualFS>;
 
@@ -106,20 +105,6 @@ impl<F: GNSTransactionDriverLLInterface> GNSTransactionDriverAnyFS<F> {
             .append_journal_close_and_close()
             .map_err(|e| e.into())
     }
-    pub fn open_or_reinit(
-        gns: &GlobalNS,
-        host_setting_version: u32,
-        host_run_mode: header_meta::HostRunMode,
-        host_startup_counter: u64,
-    ) -> TransactionResult<Self> {
-        Self::open_or_reinit_with_name(
-            gns,
-            "gns.db-tlog",
-            host_setting_version,
-            host_run_mode,
-            host_startup_counter,
-        )
-    }
     pub fn open_or_reinit_with_name(
         gns: &GlobalNS,
         log_file_name: &str,
@@ -190,7 +175,7 @@ impl JournalAdapter for GNSAdapter {
             // UNSAFE(@ohsayan):
             u16::from_le_bytes(scanner.next_chunk())
         };
-        match DISPATCH[core::cmp::min(opc as usize, DISPATCH.len())](&mut scanner, gs) {
+        match DISPATCH[(opc as usize).min(DISPATCH.len())](&mut scanner, gs) {
             Ok(()) if scanner.eof() => return Ok(()),
             Ok(_) => Err(TransactionError::DecodeCorruptedPayloadMoreBytes),
             Err(e) => Err(e),
diff --git a/server/src/engine/txn/mod.rs b/server/src/engine/txn/mod.rs
index fc4863e7..733839dc 100644
--- a/server/src/engine/txn/mod.rs
+++ b/server/src/engine/txn/mod.rs
@@ -24,7 +24,6 @@
  *
 */
 
-pub mod data;
 pub mod gns;
 
 use super::storage::v1::SDSSError;

From 0d076dd1b30548acf80c231ecb36d767ffae7cee Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 13 Sep 2023 07:20:34 +0000
Subject: [PATCH 248/310] Simplify global namespace handling

---
 server/src/engine/core/dml/del.rs             |   7 +-
 server/src/engine/core/dml/ins.rs             |   6 +-
 server/src/engine/core/dml/sel.rs             |   7 +-
 server/src/engine/core/dml/upd.rs             |   7 +-
 server/src/engine/core/model/alt.rs           |  20 +-
 server/src/engine/core/model/mod.rs           |  21 +-
 server/src/engine/core/space.rs               |  40 +--
 server/src/engine/core/tests/ddl_model/alt.rs |  34 ++-
 server/src/engine/core/tests/ddl_model/crt.rs |   8 +-
 server/src/engine/core/tests/ddl_model/mod.rs |  38 ++-
 .../src/engine/core/tests/ddl_space/alter.rs  |  32 +--
 .../src/engine/core/tests/ddl_space/create.rs |  22 +-
 server/src/engine/core/tests/ddl_space/mod.rs |  21 +-
 server/src/engine/core/tests/dml/delete.rs    |  10 +-
 server/src/engine/core/tests/dml/insert.rs    |  16 +-
 server/src/engine/core/tests/dml/mod.rs       |  91 +++---
 server/src/engine/core/tests/dml/select.rs    |  22 +-
 server/src/engine/core/tests/dml/update.rs    |  37 +--
 server/src/engine/fractal/mgr.rs              |   4 +-
 server/src/engine/fractal/mod.rs              |  37 ++-
 server/src/engine/fractal/test_utils.rs       |  68 +++++
 server/src/engine/txn/gns/tests/full_chain.rs | 271 ++++++++++--------
 22 files changed, 492 insertions(+), 327 deletions(-)
 create mode 100644 server/src/engine/fractal/test_utils.rs

diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs
index 88d0ba5f..667cb39d 100644
--- a/server/src/engine/core/dml/del.rs
+++ b/server/src/engine/core/dml/del.rs
@@ -25,15 +25,16 @@
 */
 
 use crate::engine::{
-    core::{model::delta::DataDeltaKind, GlobalNS},
+    core::model::delta::DataDeltaKind,
     error::{DatabaseError, DatabaseResult},
+    fractal::GlobalInstanceLike,
     idx::MTIndex,
     ql::dml::del::DeleteStatement,
     sync,
 };
 
-pub fn delete(gns: &GlobalNS, mut delete: DeleteStatement) -> DatabaseResult<()> {
-    gns.with_model(delete.entity(), |model| {
+pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) -> DatabaseResult<()> {
+    global.namespace().with_model(delete.entity(), |model| {
         let g = sync::atm::cpin();
         let schema_version = model.delta_state().schema_current_version();
         let delta_state = model.delta_state();
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index f93fa556..0ea02d72 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -28,16 +28,16 @@ use crate::engine::{
     core::{
         index::{DcFieldIndex, PrimaryIndexKey, Row},
         model::{delta::DataDeltaKind, Fields, Model},
-        GlobalNS,
     },
     error::{DatabaseError, DatabaseResult},
+    fractal::GlobalInstanceLike,
     idx::{IndexBaseSpec, MTIndex, STIndex, STIndexSeq},
     ql::dml::ins::{InsertData, InsertStatement},
     sync::atm::cpin,
 };
 
-pub fn insert(gns: &GlobalNS, insert: InsertStatement) -> DatabaseResult<()> {
-    gns.with_model(insert.entity(), |mdl| {
+pub fn insert(gns: &impl GlobalInstanceLike, insert: InsertStatement) -> DatabaseResult<()> {
+    gns.namespace().with_model(insert.entity(), |mdl| {
         let irmwd = mdl.intent_write_new_data();
         let (pk, data) = prepare_insert(mdl, irmwd.fields(), insert.data())?;
         let g = cpin();
diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
index 29d3881e..12a7e894 100644
--- a/server/src/engine/core/dml/sel.rs
+++ b/server/src/engine/core/dml/sel.rs
@@ -25,23 +25,24 @@
 */
 
 use crate::engine::{
-    core::{index::DcFieldIndex, GlobalNS},
+    core::index::DcFieldIndex,
     data::cell::{Datacell, VirtualDatacell},
     error::{DatabaseError, DatabaseResult},
+    fractal::GlobalInstanceLike,
     idx::{STIndex, STIndexSeq},
     ql::dml::sel::SelectStatement,
     sync,
 };
 
 pub fn select_custom<F>(
-    gns: &GlobalNS,
+    global: &impl GlobalInstanceLike,
     mut select: SelectStatement,
     mut cellfn: F,
 ) -> DatabaseResult<()>
 where
     F: FnMut(&Datacell),
 {
-    gns.with_model(select.entity(), |mdl| {
+    global.namespace().with_model(select.entity(), |mdl| {
         let irm = mdl.intent_read_model();
         let target_key = mdl.resolve_where(select.clauses_mut())?;
         let pkdc = VirtualDatacell::new(target_key.clone());
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index 45b62532..9a3cb285 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -30,7 +30,7 @@ use std::cell::RefCell;
 use {
     crate::{
         engine::{
-            core::{model::delta::DataDeltaKind, query_meta::AssignmentOperator, GlobalNS},
+            core::{model::delta::DataDeltaKind, query_meta::AssignmentOperator},
             data::{
                 cell::Datacell,
                 lit::LitIR,
@@ -38,6 +38,7 @@ use {
                 tag::{DataTag, TagClass},
             },
             error::{DatabaseError, DatabaseResult},
+            fractal::GlobalInstanceLike,
             idx::STIndex,
             ql::dml::upd::{AssignmentExpression, UpdateStatement},
             sync,
@@ -232,8 +233,8 @@ pub fn collect_trace_path() -> Vec<&'static str> {
     ROUTE_TRACE.with(|v| v.borrow().iter().cloned().collect())
 }
 
-pub fn update(gns: &GlobalNS, mut update: UpdateStatement) -> DatabaseResult<()> {
-    gns.with_model(update.entity(), |mdl| {
+pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) -> DatabaseResult<()> {
+    global.namespace().with_model(update.entity(), |mdl| {
         let mut ret = Ok(());
         // prepare row fetch
         let key = mdl.resolve_where(update.clauses_mut())?;
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index 3f0a5d2e..a4273e3f 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -28,12 +28,13 @@ use {
     super::{Field, IWModel, Layer, Model},
     crate::{
         engine::{
-            core::{util::EntityLocator, GlobalNS},
+            core::util::EntityLocator,
             data::{
                 tag::{DataTag, TagClass},
                 DictEntryGeneric,
             },
             error::{DatabaseError, DatabaseResult},
+            fractal::GlobalInstanceLike,
             idx::{IndexST, IndexSTSeqCns, STIndex, STIndexSeq},
             ql::{
                 ast::Entity,
@@ -249,12 +250,12 @@ impl<'a> AlterPlan<'a> {
 
 impl Model {
     pub fn transactional_exec_alter<GI: gnstxn::GNSTransactionDriverLLInterface>(
-        gns: &GlobalNS,
+        global: &impl GlobalInstanceLike,
         txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<GI>,
         alter: AlterModel,
     ) -> DatabaseResult<()> {
         let (space_name, model_name) = EntityLocator::parse_entity(alter.model)?;
-        gns.with_space(space_name, |space| {
+        global.namespace().with_space(space_name, |space| {
             space.with_model(model_name, |model| {
                 // make intent
                 let iwm = model.intent_write_model();
@@ -303,9 +304,10 @@ impl Model {
                             txn_driver.try_commit(txn)?;
                         }
                         removed.iter().for_each(|field_id| {
-                            model
-                                .delta_state()
-                                .schema_append_unresolved_wl_field_rem(&mut guard, field_id.as_str());
+                            model.delta_state().schema_append_unresolved_wl_field_rem(
+                                &mut guard,
+                                field_id.as_str(),
+                            );
                             iwm.fields_mut().st_delete(field_id.as_str());
                         });
                     }
@@ -328,9 +330,9 @@ impl Model {
             })
         })
     }
-    pub fn exec_alter(gns: &GlobalNS, stmt: AlterModel) -> DatabaseResult<()> {
-        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, |driver| {
-            Self::transactional_exec_alter(gns, driver, stmt)
+    pub fn exec_alter(global: &impl GlobalInstanceLike, stmt: AlterModel) -> DatabaseResult<()> {
+        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(global.namespace(), |driver| {
+            Self::transactional_exec_alter(global, driver, stmt)
         })
     }
 }
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 13f7137e..fb037092 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -40,6 +40,7 @@ use {
             uuid::Uuid,
         },
         error::{DatabaseError, DatabaseResult},
+        fractal::GlobalInstanceLike,
         idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
         mem::VInline,
         ql::ddl::{
@@ -204,13 +205,13 @@ impl Model {
 
 impl Model {
     pub fn transactional_exec_create<GI: gnstxn::GNSTransactionDriverLLInterface>(
-        gns: &super::GlobalNS,
+        global: &impl GlobalInstanceLike,
         txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<GI>,
         stmt: CreateModel,
     ) -> DatabaseResult<()> {
         let (space_name, model_name) = stmt.model_name.parse_entity()?;
         let model = Self::process_create(stmt)?;
-        gns.with_space(space_name, |space| {
+        global.namespace().with_space(space_name, |space| {
             let mut w_space = space.models().write();
             if w_space.st_contains(model_name) {
                 return Err(DatabaseError::DdlModelAlreadyExists);
@@ -234,20 +235,20 @@ impl Model {
     }
     #[cfg(test)]
     pub fn nontransactional_exec_create(
-        gns: &super::GlobalNS,
+        global: &impl GlobalInstanceLike,
         stmt: CreateModel,
     ) -> DatabaseResult<()> {
-        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, |driver| {
-            Self::transactional_exec_create(gns, driver, stmt)
+        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(global.namespace(), |driver| {
+            Self::transactional_exec_create(global, driver, stmt)
         })
     }
     pub fn transactional_exec_drop<GI: gnstxn::GNSTransactionDriverLLInterface>(
-        gns: &super::GlobalNS,
+        global: &impl GlobalInstanceLike,
         txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<GI>,
         stmt: DropModel,
     ) -> DatabaseResult<()> {
         let (space_name, model_name) = stmt.entity.parse_entity()?;
-        gns.with_space(space_name, |space| {
+        global.namespace().with_space(space_name, |space| {
             let mut w_space = space.models().write();
             let Some(model) = w_space.get(model_name) else {
                 return Err(DatabaseError::DdlModelNotFound);
@@ -270,11 +271,11 @@ impl Model {
     }
     #[cfg(test)]
     pub fn nontransactional_exec_drop(
-        gns: &super::GlobalNS,
+        global: &impl GlobalInstanceLike,
         stmt: DropModel,
     ) -> DatabaseResult<()> {
-        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, |driver| {
-            Self::transactional_exec_drop(gns, driver, stmt)
+        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(global.namespace(), |driver| {
+            Self::transactional_exec_drop(global, driver, stmt)
         })
     }
 }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index f6da7c3e..6a948dfc 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -29,6 +29,7 @@ use {
         core::{model::Model, RWLIdx},
         data::{dict, uuid::Uuid, DictEntryGeneric, DictGeneric},
         error::{DatabaseError, DatabaseResult},
+        fractal::GlobalInstanceLike,
         idx::{IndexST, STIndex},
         ql::ddl::{alt::AlterSpace, crt::CreateSpace, drop::DropSpace},
         txn::gns as gnstxn,
@@ -189,14 +190,14 @@ impl Space {
 
 impl Space {
     pub fn transactional_exec_create<TI: gnstxn::GNSTransactionDriverLLInterface>(
-        gns: &super::GlobalNS,
+        global: &impl GlobalInstanceLike,
         txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<TI>,
         space: CreateSpace,
     ) -> DatabaseResult<()> {
         // process create
         let ProcedureCreate { space_name, space } = Self::process_create(space)?;
         // acquire access
-        let mut wl = gns.spaces().write();
+        let mut wl = global.namespace().spaces().write();
         if wl.st_contains(&space_name) {
             return Err(DatabaseError::DdlSpaceAlreadyExists);
         }
@@ -213,22 +214,23 @@ impl Space {
     /// Execute a `create` stmt
     #[cfg(test)]
     pub fn nontransactional_exec_create(
-        gns: &super::GlobalNS,
+        global: &impl GlobalInstanceLike,
         space: CreateSpace,
     ) -> DatabaseResult<()> {
-        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, move |driver| {
-            Self::transactional_exec_create(gns, driver, space)
-        })
+        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(
+            global.namespace(),
+            move |driver| Self::transactional_exec_create(global, driver, space),
+        )
     }
     pub fn transactional_exec_alter<TI: gnstxn::GNSTransactionDriverLLInterface>(
-        gns: &super::GlobalNS,
+        global: &impl GlobalInstanceLike,
         txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<TI>,
         AlterSpace {
             space_name,
             updated_props,
         }: AlterSpace,
     ) -> DatabaseResult<()> {
-        gns.with_space(&space_name, |space| {
+        global.namespace().with_space(&space_name, |space| {
             match updated_props.get(SpaceMeta::KEY_ENV) {
                 Some(DictEntryGeneric::Map(_)) if updated_props.len() == 1 => {}
                 Some(DictEntryGeneric::Data(l)) if updated_props.len() == 1 && l.is_null() => {}
@@ -262,22 +264,23 @@ impl Space {
     #[cfg(test)]
     /// Execute a `alter` stmt
     pub fn nontransactional_exec_alter(
-        gns: &super::GlobalNS,
+        global: &impl GlobalInstanceLike,
         alter: AlterSpace,
     ) -> DatabaseResult<()> {
-        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, move |driver| {
-            Self::transactional_exec_alter(gns, driver, alter)
-        })
+        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(
+            global.namespace(),
+            move |driver| Self::transactional_exec_alter(global, driver, alter),
+        )
     }
     pub fn transactional_exec_drop<TI: gnstxn::GNSTransactionDriverLLInterface>(
-        gns: &super::GlobalNS,
+        global: &impl GlobalInstanceLike,
         txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<TI>,
         DropSpace { space, force: _ }: DropSpace,
     ) -> DatabaseResult<()> {
         // TODO(@ohsayan): force remove option
         // TODO(@ohsayan): should a drop space block the entire global table?
         let space_name = space;
-        let mut wgns = gns.spaces().write();
+        let mut wgns = global.namespace().spaces().write();
         let space = match wgns.get(space_name.as_str()) {
             Some(space) => space,
             None => return Err(DatabaseError::DdlSpaceNotFound),
@@ -298,12 +301,13 @@ impl Space {
     }
     #[cfg(test)]
     pub fn nontransactional_exec_drop(
-        gns: &super::GlobalNS,
+        global: &impl GlobalInstanceLike,
         drop_space: DropSpace,
     ) -> DatabaseResult<()> {
-        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(gns, move |driver| {
-            Self::transactional_exec_drop(gns, driver, drop_space)
-        })
+        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(
+            global.namespace(),
+            move |driver| Self::transactional_exec_drop(global, driver, drop_space),
+        )
     }
 }
 
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index bcbb16a4..ca017e03 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -28,9 +28,9 @@ use crate::engine::{
     core::{
         model::{alt::AlterPlan, Model},
         tests::ddl_model::{create, exec_create},
-        GlobalNS,
     },
     error::DatabaseResult,
+    fractal::GlobalInstanceLike,
     idx::STIndex,
     ql::{ast::parse_ast_node_full, ddl::alt::AlterModel, tests::lex_insecure},
 };
@@ -47,15 +47,15 @@ fn plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) {
     with_plan(model, plan, f).unwrap()
 }
 fn exec_plan(
-    gns: &GlobalNS,
+    global: &impl GlobalInstanceLike,
     new_space: bool,
     model: &str,
     plan: &str,
     f: impl Fn(&Model),
 ) -> DatabaseResult<()> {
-    let mdl_name = exec_create(gns, model, new_space)?;
+    let mdl_name = exec_create(global, model, new_space)?;
     let prev_uuid = {
-        let gns = gns.spaces().read();
+        let gns = global.namespace().spaces().read();
         let space = gns.get("myspace").unwrap();
         let space_read = space.models().read();
         space_read.get(mdl_name.as_str()).unwrap().get_uuid()
@@ -63,8 +63,8 @@ fn exec_plan(
     let tok = lex_insecure(plan.as_bytes()).unwrap();
     let alter = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
     let (_space, model_name) = alter.model.into_full().unwrap();
-    Model::exec_alter(gns, alter)?;
-    let gns_read = gns.spaces().read();
+    Model::exec_alter(global, alter)?;
+    let gns_read = global.namespace().spaces().read();
     let space = gns_read.st_get("myspace").unwrap();
     let model = space.models().read();
     let model = model.st_get(model_name.as_str()).unwrap();
@@ -352,18 +352,16 @@ mod plan {
 
 mod exec {
     use crate::engine::{
-        core::{
-            model::{DeltaVersion, Field, Layer},
-            GlobalNS,
-        },
+        core::model::{DeltaVersion, Field, Layer},
         error::DatabaseError,
+        fractal::test_utils::TestGlobal,
         idx::{STIndex, STIndexSeq},
     };
     #[test]
     fn simple_add() {
-        let gns = GlobalNS::empty();
+        let global = TestGlobal::empty();
         super::exec_plan(
-            &gns,
+            &global,
             true,
             "create model myspace.mymodel(username: string, col1: uint64)",
             "alter model myspace.mymodel add (col2 { type: uint32, nullable: true }, col3 { type: uint16, nullable: true })",
@@ -392,9 +390,9 @@ mod exec {
     }
     #[test]
     fn simple_remove() {
-        let gns = GlobalNS::empty();
+        let global = TestGlobal::empty();
         super::exec_plan(
-            &gns,
+            &global,
             true,
             "create model myspace.mymodel(username: string, col1: uint64, col2: uint32, col3: uint16, col4: uint8)",
             "alter model myspace.mymodel remove (col1, col2, col3, col4)",
@@ -418,9 +416,9 @@ mod exec {
     }
     #[test]
     fn simple_update() {
-        let gns = GlobalNS::empty();
+        let global = TestGlobal::empty();
         super::exec_plan(
-            &gns,
+            &global,
             true,
             "create model myspace.mymodel(username: string, password: binary)",
             "alter model myspace.mymodel update password { nullable: true }",
@@ -437,10 +435,10 @@ mod exec {
     }
     #[test]
     fn failing_alter_nullable_switch_need_lock() {
-        let gns = GlobalNS::empty();
+        let global = TestGlobal::empty();
         assert_eq!(
             super::exec_plan(
-                &gns,
+                &global,
                 true,
                 "create model myspace.mymodel(username: string, null gh_handle: string)",
                 "alter model myspace.mymodel update gh_handle { nullable: false }",
diff --git a/server/src/engine/core/tests/ddl_model/crt.rs b/server/src/engine/core/tests/ddl_model/crt.rs
index cc87b474..4b247002 100644
--- a/server/src/engine/core/tests/ddl_model/crt.rs
+++ b/server/src/engine/core/tests/ddl_model/crt.rs
@@ -135,9 +135,9 @@ mod exec {
         core::{
             model::{DeltaVersion, Field, Layer},
             tests::ddl_model::{exec_create_new_space, with_model},
-            GlobalNS,
         },
         data::tag::{DataTag, FullTag},
+        fractal::test_utils::TestGlobal,
         idx::STIndexSeq,
     };
 
@@ -145,13 +145,13 @@ mod exec {
 
     #[test]
     fn simple() {
-        let gns = GlobalNS::empty();
+        let global = TestGlobal::empty();
         exec_create_new_space(
-            &gns,
+            &global,
             "create model myspace.mymodel(username: string, password: binary)",
         )
         .unwrap();
-        with_model(&gns, SPACE, "mymodel", |model| {
+        with_model(&global, SPACE, "mymodel", |model| {
             let models: Vec<(String, Field)> = model
                 .intent_read_model()
                 .fields()
diff --git a/server/src/engine/core/tests/ddl_model/mod.rs b/server/src/engine/core/tests/ddl_model/mod.rs
index 70b1b295..61af1fc4 100644
--- a/server/src/engine/core/tests/ddl_model/mod.rs
+++ b/server/src/engine/core/tests/ddl_model/mod.rs
@@ -29,8 +29,9 @@ mod crt;
 mod layer;
 
 use crate::engine::{
-    core::{model::Model, space::Space, GlobalNS},
+    core::{model::Model, space::Space},
     error::DatabaseResult,
+    fractal::GlobalInstanceLike,
     idx::STIndex,
     ql::{
         ast::{parse_ast_node_full, Entity},
@@ -46,7 +47,7 @@ fn create(s: &str) -> DatabaseResult<Model> {
 }
 
 pub fn exec_create(
-    gns: &GlobalNS,
+    global: &impl GlobalInstanceLike,
     create_stmt: &str,
     create_new_space: bool,
 ) -> DatabaseResult<String> {
@@ -56,27 +57,40 @@ pub fn exec_create(
         Entity::Single(tbl) | Entity::Full(_, tbl) => tbl.to_string(),
     };
     if create_new_space {
-        gns.test_new_empty_space(&create_model.model_name.into_full().unwrap().0);
+        global
+            .namespace()
+            .test_new_empty_space(&create_model.model_name.into_full().unwrap().0);
     }
-    Model::nontransactional_exec_create(gns, create_model).map(|_| name)
+    Model::nontransactional_exec_create(global, create_model).map(|_| name)
 }
 
-pub fn exec_create_new_space(gns: &GlobalNS, create_stmt: &str) -> DatabaseResult<()> {
-    exec_create(gns, create_stmt, true).map(|_| ())
+pub fn exec_create_new_space(
+    global: &impl GlobalInstanceLike,
+    create_stmt: &str,
+) -> DatabaseResult<()> {
+    exec_create(global, create_stmt, true).map(|_| ())
 }
 
-pub fn exec_create_no_create(gns: &GlobalNS, create_stmt: &str) -> DatabaseResult<()> {
-    exec_create(gns, create_stmt, false).map(|_| ())
+pub fn exec_create_no_create(
+    global: &impl GlobalInstanceLike,
+    create_stmt: &str,
+) -> DatabaseResult<()> {
+    exec_create(global, create_stmt, false).map(|_| ())
 }
 
-fn with_space(gns: &GlobalNS, space_name: &str, f: impl Fn(&Space)) {
-    let rl = gns.spaces().read();
+fn with_space(global: &impl GlobalInstanceLike, space_name: &str, f: impl Fn(&Space)) {
+    let rl = global.namespace().spaces().read();
     let space = rl.st_get(space_name).unwrap();
     f(space);
 }
 
-fn with_model(gns: &GlobalNS, space_id: &str, model_name: &str, f: impl Fn(&Model)) {
-    with_space(gns, space_id, |space| {
+fn with_model(
+    global: &impl GlobalInstanceLike,
+    space_id: &str,
+    model_name: &str,
+    f: impl Fn(&Model),
+) {
+    with_space(global, space_id, |space| {
         let space_rl = space.models().read();
         let model = space_rl.st_get(model_name).unwrap();
         f(model)
diff --git a/server/src/engine/core/tests/ddl_space/alter.rs b/server/src/engine/core/tests/ddl_space/alter.rs
index 3cbcd4ab..ecfd7314 100644
--- a/server/src/engine/core/tests/ddl_space/alter.rs
+++ b/server/src/engine/core/tests/ddl_space/alter.rs
@@ -25,19 +25,17 @@
 */
 
 use crate::engine::{
-    core::{
-        space::{Space, SpaceMeta},
-        GlobalNS,
-    },
+    core::space::{Space, SpaceMeta},
     data::cell::Datacell,
     error::DatabaseError,
+    fractal::test_utils::TestGlobal,
 };
 
 #[test]
 fn alter_add_prop_env_var() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     super::exec_create_alter(
-        &gns,
+        &global,
         "create space myspace",
         "alter space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
@@ -56,9 +54,9 @@ fn alter_add_prop_env_var() {
 
 #[test]
 fn alter_update_prop_env_var() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     let uuid = super::exec_create(
-        &gns,
+        &global,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
             let rl = space.meta.dict().read();
@@ -70,7 +68,7 @@ fn alter_update_prop_env_var() {
     )
     .unwrap();
     super::exec_alter(
-        &gns,
+        &global,
         "alter space myspace with { env: { MY_NEW_PROP: 200 } }",
         |space| {
             assert_eq!(
@@ -88,9 +86,9 @@ fn alter_update_prop_env_var() {
 
 #[test]
 fn alter_remove_prop_env_var() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     let uuid = super::exec_create(
-        &gns,
+        &global,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
             let rl = space.meta.dict().read();
@@ -102,7 +100,7 @@ fn alter_remove_prop_env_var() {
     )
     .unwrap();
     super::exec_alter(
-        &gns,
+        &global,
         "alter space myspace with { env: { MY_NEW_PROP: null } }",
         |space| {
             assert_eq!(
@@ -116,10 +114,10 @@ fn alter_remove_prop_env_var() {
 
 #[test]
 fn alter_nx() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_alter(
-            &gns,
+            &global,
             "alter space myspace with { env: { MY_NEW_PROP: 100 } }",
             |_| {},
         )
@@ -130,9 +128,9 @@ fn alter_nx() {
 
 #[test]
 fn alter_remove_all_env() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     let uuid = super::exec_create(
-        &gns,
+        &global,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
             let rl = space.meta.dict().read();
@@ -143,7 +141,7 @@ fn alter_remove_all_env() {
         },
     )
     .unwrap();
-    super::exec_alter(&gns, "alter space myspace with { env: null }", |space| {
+    super::exec_alter(&global, "alter space myspace with { env: null }", |space| {
         assert_eq!(space, &Space::empty_with_uuid(uuid))
     })
     .unwrap();
diff --git a/server/src/engine/core/tests/ddl_space/create.rs b/server/src/engine/core/tests/ddl_space/create.rs
index e66f4926..bcf9cf2b 100644
--- a/server/src/engine/core/tests/ddl_space/create.rs
+++ b/server/src/engine/core/tests/ddl_space/create.rs
@@ -25,18 +25,16 @@
 */
 
 use crate::engine::{
-    core::{
-        space::{Space, SpaceMeta},
-        GlobalNS,
-    },
+    core::space::{Space, SpaceMeta},
     data::cell::Datacell,
     error::DatabaseError,
+    fractal::test_utils::TestGlobal,
 };
 
 #[test]
 fn exec_create_space_simple() {
-    let gns = GlobalNS::empty();
-    super::exec_create(&gns, "create space myspace", |spc| {
+    let global = TestGlobal::empty();
+    super::exec_create(&global, "create space myspace", |spc| {
         assert!(spc.models().read().is_empty())
     })
     .unwrap();
@@ -44,9 +42,9 @@ fn exec_create_space_simple() {
 
 #[test]
 fn exec_create_space_with_env() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     super::exec_create(
-        &gns,
+        &global,
         r#"
         create space myspace with {
             env: {
@@ -72,19 +70,19 @@ fn exec_create_space_with_env() {
 
 #[test]
 fn exec_create_space_with_bad_env_type() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
-        super::exec_create(&gns, "create space myspace with { env: 100 }", |_| {}).unwrap_err(),
+        super::exec_create(&global, "create space myspace with { env: 100 }", |_| {}).unwrap_err(),
         DatabaseError::DdlSpaceBadProperty
     );
 }
 
 #[test]
 fn exec_create_space_with_random_property() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_create(
-            &gns,
+            &global,
             "create space myspace with { i_am_blue_da_ba_dee: 100 }",
             |_| {}
         )
diff --git a/server/src/engine/core/tests/ddl_space/mod.rs b/server/src/engine/core/tests/ddl_space/mod.rs
index 0518c375..7e05f0a8 100644
--- a/server/src/engine/core/tests/ddl_space/mod.rs
+++ b/server/src/engine/core/tests/ddl_space/mod.rs
@@ -28,41 +28,50 @@ mod alter;
 mod create;
 
 use crate::engine::{
-    core::{space::Space, GlobalNS},
+    core::space::Space,
     data::uuid::Uuid,
     error::DatabaseResult,
+    fractal::GlobalInstanceLike,
     ql::{
         ast::{self},
         tests::lex_insecure as lex,
     },
 };
 
-fn exec_create(gns: &GlobalNS, create: &str, verify: impl Fn(&Space)) -> DatabaseResult<Uuid> {
+fn exec_create(
+    gns: &impl GlobalInstanceLike,
+    create: &str,
+    verify: impl Fn(&Space),
+) -> DatabaseResult<Uuid> {
     let tok = lex(create.as_bytes()).unwrap();
     let ast_node =
         ast::parse_ast_node_full::<crate::engine::ql::ddl::crt::CreateSpace>(&tok[2..]).unwrap();
     let name = ast_node.space_name;
     Space::nontransactional_exec_create(gns, ast_node)?;
-    gns.with_space(&name, |space| {
+    gns.namespace().with_space(&name, |space| {
         verify(space);
         Ok(space.get_uuid())
     })
 }
 
-fn exec_alter(gns: &GlobalNS, alter: &str, verify: impl Fn(&Space)) -> DatabaseResult<Uuid> {
+fn exec_alter(
+    gns: &impl GlobalInstanceLike,
+    alter: &str,
+    verify: impl Fn(&Space),
+) -> DatabaseResult<Uuid> {
     let tok = lex(alter.as_bytes()).unwrap();
     let ast_node =
         ast::parse_ast_node_full::<crate::engine::ql::ddl::alt::AlterSpace>(&tok[2..]).unwrap();
     let name = ast_node.space_name;
     Space::nontransactional_exec_alter(gns, ast_node)?;
-    gns.with_space(&name, |space| {
+    gns.namespace().with_space(&name, |space| {
         verify(space);
         Ok(space.get_uuid())
     })
 }
 
 fn exec_create_alter(
-    gns: &GlobalNS,
+    gns: &impl GlobalInstanceLike,
     crt: &str,
     alt: &str,
     verify_post_alt: impl Fn(&Space),
diff --git a/server/src/engine/core/tests/dml/delete.rs b/server/src/engine/core/tests/dml/delete.rs
index 1bb4fcc0..92183ab2 100644
--- a/server/src/engine/core/tests/dml/delete.rs
+++ b/server/src/engine/core/tests/dml/delete.rs
@@ -24,13 +24,13 @@
  *
 */
 
-use crate::engine::{core::GlobalNS, error::DatabaseError};
+use crate::engine::{error::DatabaseError, fractal::test_utils::TestGlobal};
 
 #[test]
 fn simple_delete() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     super::exec_delete(
-        &gns,
+        &global,
         "create model myspace.mymodel(username: string, password: string)",
         Some("insert into myspace.mymodel('sayan', 'pass123')"),
         "delete from myspace.mymodel where username = 'sayan'",
@@ -41,10 +41,10 @@ fn simple_delete() {
 
 #[test]
 fn delete_nonexisting() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_delete(
-            &gns,
+            &global,
             "create model myspace.mymodel(username: string, password: string)",
             None,
             "delete from myspace.mymodel where username = 'sayan'",
diff --git a/server/src/engine/core/tests/dml/insert.rs b/server/src/engine/core/tests/dml/insert.rs
index 10d77ff5..de36b22a 100644
--- a/server/src/engine/core/tests/dml/insert.rs
+++ b/server/src/engine/core/tests/dml/insert.rs
@@ -24,16 +24,16 @@
  *
 */
 
-use crate::engine::{core::GlobalNS, data::cell::Datacell, error::DatabaseError};
+use crate::engine::{data::cell::Datacell, error::DatabaseError, fractal::test_utils::TestGlobal};
 
 #[derive(sky_macros::Wrapper, Debug)]
 struct Tuple(Vec<(Box<str>, Datacell)>);
 
 #[test]
 fn insert_simple() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     super::exec_insert(
-        &gns,
+        &global,
         "create model myspace.mymodel(username: string, password: string)",
         "insert into myspace.mymodel('sayan', 'pass123')",
         "sayan",
@@ -46,9 +46,9 @@ fn insert_simple() {
 
 #[test]
 fn insert_with_null() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     super::exec_insert(
-        &gns,
+        &global,
         "create model myspace.mymodel(username: string, null useless_password: string, null useless_email: string, null useless_random_column: uint64)",
         "insert into myspace.mymodel('sayan', null, null, null)",
         "sayan",
@@ -69,9 +69,9 @@ fn insert_with_null() {
 
 #[test]
 fn insert_duplicate() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     super::exec_insert(
-        &gns,
+        &global,
         "create model myspace.mymodel(username: string, password: string)",
         "insert into myspace.mymodel('sayan', 'pass123')",
         "sayan",
@@ -81,7 +81,7 @@ fn insert_duplicate() {
     )
     .unwrap();
     assert_eq!(
-        super::exec_insert_only(&gns, "insert into myspace.mymodel('sayan', 'pass123')")
+        super::exec_insert_only(&global, "insert into myspace.mymodel('sayan', 'pass123')")
             .unwrap_err(),
         DatabaseError::DmlConstraintViolationDuplicate
     );
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index 6a4b8d23..8f14363a 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -30,9 +30,10 @@ mod select;
 mod update;
 
 use crate::engine::{
-    core::{dml, index::Row, model::Model, GlobalNS},
+    core::{dml, index::Row, model::Model},
     data::{cell::Datacell, lit::LitIR},
     error::DatabaseResult,
+    fractal::GlobalInstanceLike,
     ql::{
         ast::{parse_ast_node_full, Entity},
         dml::{del::DeleteStatement, ins::InsertStatement},
@@ -41,36 +42,39 @@ use crate::engine::{
     sync,
 };
 
-fn _exec_only_create_space_model(gns: &GlobalNS, model: &str) -> DatabaseResult<()> {
-    if !gns.spaces().read().contains_key("myspace") {
-        gns.test_new_empty_space("myspace");
+fn _exec_only_create_space_model(
+    global: &impl GlobalInstanceLike,
+    model: &str,
+) -> DatabaseResult<()> {
+    if !global.namespace().spaces().read().contains_key("myspace") {
+        global.namespace().test_new_empty_space("myspace");
     }
     let lex_create_model = lex_insecure(model.as_bytes()).unwrap();
     let stmt_create_model = parse_ast_node_full(&lex_create_model[2..]).unwrap();
-    Model::nontransactional_exec_create(gns, stmt_create_model)
+    Model::nontransactional_exec_create(global, stmt_create_model)
 }
 
 fn _exec_only_insert<T>(
-    gns: &GlobalNS,
+    global: &impl GlobalInstanceLike,
     insert: &str,
     and_then: impl Fn(Entity) -> T,
 ) -> DatabaseResult<T> {
     let lex_insert = lex_insecure(insert.as_bytes()).unwrap();
     let stmt_insert = parse_ast_node_full::<InsertStatement>(&lex_insert[1..]).unwrap();
     let entity = stmt_insert.entity();
-    dml::insert(gns, stmt_insert)?;
+    dml::insert(global, stmt_insert)?;
     let r = and_then(entity);
     Ok(r)
 }
 
 fn _exec_only_read_key_and_then<T>(
-    gns: &GlobalNS,
+    global: &impl GlobalInstanceLike,
     entity: Entity,
     key_name: &str,
     and_then: impl Fn(Row) -> T,
 ) -> DatabaseResult<T> {
     let guard = sync::atm::cpin();
-    gns.with_model(entity, |mdl| {
+    global.namespace().with_model(entity, |mdl| {
         let _irm = mdl.intent_read_model();
         let row = mdl
             .primary_index()
@@ -82,13 +86,17 @@ fn _exec_only_read_key_and_then<T>(
     })
 }
 
-fn _exec_delete_only(gns: &GlobalNS, delete: &str, key: &str) -> DatabaseResult<()> {
+fn _exec_delete_only(
+    global: &impl GlobalInstanceLike,
+    delete: &str,
+    key: &str,
+) -> DatabaseResult<()> {
     let lex_del = lex_insecure(delete.as_bytes()).unwrap();
     let delete = parse_ast_node_full::<DeleteStatement>(&lex_del[1..]).unwrap();
     let entity = delete.entity();
-    dml::delete(gns, delete)?;
+    dml::delete(global, delete)?;
     assert_eq!(
-        gns.with_model(entity, |model| {
+        global.namespace().with_model(entity, |model| {
             let _ = model.intent_read_model();
             let g = sync::atm::cpin();
             Ok(model.primary_index().select(key.into(), &g).is_none())
@@ -98,75 +106,84 @@ fn _exec_delete_only(gns: &GlobalNS, delete: &str, key: &str) -> DatabaseResult<
     Ok(())
 }
 
-fn _exec_only_select(gns: &GlobalNS, select: &str) -> DatabaseResult<Vec<Datacell>> {
+fn _exec_only_select(
+    global: &impl GlobalInstanceLike,
+    select: &str,
+) -> DatabaseResult<Vec<Datacell>> {
     let lex_sel = lex_insecure(select.as_bytes()).unwrap();
     let select = parse_ast_node_full(&lex_sel[1..]).unwrap();
     let mut r = Vec::new();
-    dml::select_custom(gns, select, |cell| r.push(cell.clone()))?;
+    dml::select_custom(global, select, |cell| r.push(cell.clone()))?;
     Ok(r)
 }
 
-fn _exec_only_update(gns: &GlobalNS, update: &str) -> DatabaseResult<()> {
+fn _exec_only_update(global: &impl GlobalInstanceLike, update: &str) -> DatabaseResult<()> {
     let lex_upd = lex_insecure(update.as_bytes()).unwrap();
     let update = parse_ast_node_full(&lex_upd[1..]).unwrap();
-    dml::update(gns, update)
+    dml::update(global, update)
 }
 
 pub(self) fn exec_insert<T: Default>(
-    gns: &GlobalNS,
+    global: &impl GlobalInstanceLike,
     model: &str,
     insert: &str,
     key_name: &str,
     f: impl Fn(Row) -> T,
 ) -> DatabaseResult<T> {
-    _exec_only_create_space_model(gns, model)?;
-    _exec_only_insert(gns, insert, |entity| {
-        _exec_only_read_key_and_then(gns, entity, key_name, |row| f(row))
+    _exec_only_create_space_model(global, model)?;
+    _exec_only_insert(global, insert, |entity| {
+        _exec_only_read_key_and_then(global, entity, key_name, |row| f(row))
     })?
 }
 
-pub(self) fn exec_insert_only(gns: &GlobalNS, insert: &str) -> DatabaseResult<()> {
-    _exec_only_insert(gns, insert, |_| {})
+pub(self) fn exec_insert_only(
+    global: &impl GlobalInstanceLike,
+    insert: &str,
+) -> DatabaseResult<()> {
+    _exec_only_insert(global, insert, |_| {})
 }
 
 pub(self) fn exec_delete(
-    gns: &GlobalNS,
+    global: &impl GlobalInstanceLike,
     model: &str,
     insert: Option<&str>,
     delete: &str,
     key: &str,
 ) -> DatabaseResult<()> {
-    _exec_only_create_space_model(gns, model)?;
+    _exec_only_create_space_model(global, model)?;
     if let Some(insert) = insert {
-        _exec_only_insert(gns, insert, |_| {})?;
+        _exec_only_insert(global, insert, |_| {})?;
     }
-    _exec_delete_only(gns, delete, key)
+    _exec_delete_only(global, delete, key)
 }
 
 pub(self) fn exec_select(
-    gns: &GlobalNS,
+    global: &impl GlobalInstanceLike,
     model: &str,
     insert: &str,
     select: &str,
 ) -> DatabaseResult<Vec<Datacell>> {
-    _exec_only_create_space_model(gns, model)?;
-    _exec_only_insert(gns, insert, |_| {})?;
-    _exec_only_select(gns, select)
+    _exec_only_create_space_model(global, model)?;
+    _exec_only_insert(global, insert, |_| {})?;
+    _exec_only_select(global, select)
 }
 
-pub(self) fn exec_select_only(gns: &GlobalNS, select: &str) -> DatabaseResult<Vec<Datacell>> {
-    _exec_only_select(gns, select)
+pub(self) fn exec_select_only(
+    global: &impl GlobalInstanceLike,
+    select: &str,
+) -> DatabaseResult<Vec<Datacell>> {
+    _exec_only_select(global, select)
 }
 
 pub(self) fn exec_update(
-    gns: &GlobalNS,
+    global: &impl GlobalInstanceLike,
     model: &str,
     insert: &str,
     update: &str,
     select: &str,
 ) -> DatabaseResult<Vec<Datacell>> {
-    _exec_only_create_space_model(gns, model)?;
-    _exec_only_insert(gns, insert, |_| {})?;
-    _exec_only_update(gns, update)?;
-    _exec_only_select(gns, select)
+    _exec_only_create_space_model(global, model)?;
+    _exec_only_insert(global, insert, |_| {})?;
+    _exec_only_update(global, update)?;
+    _exec_only_select(global, select)
 }
diff --git a/server/src/engine/core/tests/dml/select.rs b/server/src/engine/core/tests/dml/select.rs
index e7501d07..8c5f079c 100644
--- a/server/src/engine/core/tests/dml/select.rs
+++ b/server/src/engine/core/tests/dml/select.rs
@@ -24,14 +24,14 @@
  *
 */
 
-use crate::engine::{core::GlobalNS, data::cell::Datacell, error::DatabaseError};
+use crate::engine::{data::cell::Datacell, error::DatabaseError, fractal::test_utils::TestGlobal};
 
 #[test]
 fn simple_select_wildcard() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_select(
-            &gns,
+            &global,
             "create model myspace.mymodel(username: string, password: string)",
             "insert into myspace.mymodel('sayan', 'pass123')",
             "select * from myspace.mymodel where username = 'sayan'",
@@ -43,10 +43,10 @@ fn simple_select_wildcard() {
 
 #[test]
 fn simple_select_specified_same_order() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_select(
-            &gns,
+            &global,
             "create model myspace.mymodel(username: string, password: string)",
             "insert into myspace.mymodel('sayan', 'pass123')",
             "select username, password from myspace.mymodel where username = 'sayan'",
@@ -58,10 +58,10 @@ fn simple_select_specified_same_order() {
 
 #[test]
 fn simple_select_specified_reversed_order() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_select(
-            &gns,
+            &global,
             "create model myspace.mymodel(username: string, password: string)",
             "insert into myspace.mymodel('sayan', 'pass123')",
             "select password, username from myspace.mymodel where username = 'sayan'",
@@ -73,10 +73,10 @@ fn simple_select_specified_reversed_order() {
 
 #[test]
 fn select_null() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_select(
-            &gns,
+            &global,
             "create model myspace.mymodel(username: string, null password: string)",
             "insert into myspace.mymodel('sayan', null)",
             "select username, password from myspace.mymodel where username = 'sayan'",
@@ -88,10 +88,10 @@ fn select_null() {
 
 #[test]
 fn select_nonexisting() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_select(
-            &gns,
+            &global,
             "create model myspace.mymodel(username: string, null password: string)",
             "insert into myspace.mymodel('sayan', null)",
             "select username, password from myspace.mymodel where username = 'notsayan'",
diff --git a/server/src/engine/core/tests/dml/update.rs b/server/src/engine/core/tests/dml/update.rs
index 5f74d26d..f49ed310 100644
--- a/server/src/engine/core/tests/dml/update.rs
+++ b/server/src/engine/core/tests/dml/update.rs
@@ -25,17 +25,15 @@
 */
 
 use crate::engine::{
-    core::{dml, GlobalNS},
-    data::cell::Datacell,
-    error::DatabaseError,
+    core::dml, data::cell::Datacell, error::DatabaseError, fractal::test_utils::TestGlobal,
 };
 
 #[test]
 fn simple() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_update(
-            &gns,
+            &global,
             "create model myspace.mymodel(username: string, email: string, followers: uint64, following: uint64)",
             "insert into myspace.mymodel('sayan', 'sayan@example.com', 0, 100)",
             "update myspace.mymodel set followers += 200000, following -= 15, email = 'sn@example.com' where username = 'sayan'",
@@ -51,10 +49,10 @@ fn simple() {
 
 #[test]
 fn with_null() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_update(
-            &gns,
+            &global,
             "create model myspace.mymodel(username: string, password: string, null email: string)",
             "insert into myspace.mymodel('sayan', 'pass123', null)",
             "update myspace.mymodel set email = 'sayan@example.com' where username = 'sayan'",
@@ -68,10 +66,10 @@ fn with_null() {
 
 #[test]
 fn with_list() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_update(
-            &gns,
+            &global,
             "create model myspace.mymodel(link: string, click_ids: list { type: string })",
             "insert into myspace.mymodel('example.com', [])",
             "update myspace.mymodel set click_ids += 'ios_client_uuid' where link = 'example.com'",
@@ -88,10 +86,10 @@ fn with_list() {
 
 #[test]
 fn fail_operation_on_null() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_update(
-            &gns,
+            &global,
             "create model myspace.mymodel(username: string, password: string, null email: string)",
             "insert into myspace.mymodel('sayan', 'pass123', null)",
             "update myspace.mymodel set email += '.com' where username = 'sayan'",
@@ -108,10 +106,10 @@ fn fail_operation_on_null() {
 
 #[test]
 fn fail_unknown_fields() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_update(
-            &gns,
+            &global,
             "create model myspace.mymodel(username: string, password: string, null email: string)",
             "insert into myspace.mymodel('sayan', 'pass123', null)",
             "update myspace.mymodel set email2 = 'sayan@example.com', password += '4' where username = 'sayan'",
@@ -123,18 +121,21 @@ fn fail_unknown_fields() {
     assert_eq!(dml::update_flow_trace(), ["fieldnotfound", "rollback"]);
     // verify integrity
     assert_eq!(
-        super::exec_select_only(&gns, "select * from myspace.mymodel where username='sayan'")
-            .unwrap(),
+        super::exec_select_only(
+            &global,
+            "select * from myspace.mymodel where username='sayan'"
+        )
+        .unwrap(),
         intovec!["sayan", "pass123", Datacell::null()]
     );
 }
 
 #[test]
 fn fail_typedef_violation() {
-    let gns = GlobalNS::empty();
+    let global = TestGlobal::empty();
     assert_eq!(
         super::exec_update(
-            &gns,
+            &global,
             "create model myspace.mymodel(username: string, password: string, rank: uint8)",
             "insert into myspace.mymodel('sayan', 'pass123', 1)",
             "update myspace.mymodel set password = 'pass1234', rank = 'one' where username = 'sayan'",
@@ -150,7 +151,7 @@ fn fail_typedef_violation() {
     // verify integrity
     assert_eq!(
         super::exec_select_only(
-            &gns,
+            &global,
             "select * from myspace.mymodel where username = 'sayan'"
         )
         .unwrap(),
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 51dc80ac..93cdf85d 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -184,7 +184,7 @@ impl FractalMgr {
                         // was way behind in the queue
                         continue;
                     };
-                    let res = global.namespace().with_model(
+                    let res = global._namespace().with_model(
                         (model_id.space(), model_id.model()),
                         |model| {
                             if model.get_uuid() != model_id.uuid() {
@@ -235,7 +235,7 @@ impl FractalMgr {
                     let mdl_drivers = global.get_state().get_mdl_drivers().read();
                     for (model_id, driver) in mdl_drivers.iter() {
                         let mut observed_len = 0;
-                        let res = global.namespace().with_model((model_id.space(), model_id.model()), |model| {
+                        let res = global._namespace().with_model((model_id.space(), model_id.model()), |model| {
                             if model.get_uuid() != model_id.uuid() {
                                 // once again, throughput maximization will lead to, in extremely rare cases, this
                                 // branch returning. but it is okay
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index 5d6bb0a6..cc875963 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -36,6 +36,8 @@ use {
 mod config;
 mod drivers;
 mod mgr;
+#[cfg(test)]
+pub mod test_utils;
 mod util;
 pub use {
     config::ServerConfig,
@@ -94,35 +96,58 @@ pub unsafe fn enable_and_start_all(
     global access
 */
 
+/// Something that represents the global state
+pub trait GlobalInstanceLike {
+    fn namespace(&self) -> &GlobalNS;
+    fn post_high_priority_task(&self, task: Task<CriticalTask>);
+    fn post_standard_priority_task(&self, task: Task<GenericTask>);
+    fn get_max_delta_size(&self) -> usize;
+}
+
+impl GlobalInstanceLike for Global {
+    fn namespace(&self) -> &GlobalNS {
+        self._namespace()
+    }
+    fn post_high_priority_task(&self, task: Task<CriticalTask>) {
+        self._post_high_priority_task(task)
+    }
+    fn post_standard_priority_task(&self, task: Task<GenericTask>) {
+        self._post_standard_priority_task(task)
+    }
+    fn get_max_delta_size(&self) -> usize {
+        self._get_max_delta_size()
+    }
+}
+
 #[derive(Debug, Clone, Copy)]
 /// A handle to the global state
 pub struct Global(());
 
 impl Global {
-    fn new() -> Self {
+    unsafe fn new() -> Self {
         Self(())
     }
-    pub(self) fn get_state(&self) -> &'static GlobalState {
+    fn get_state(&self) -> &'static GlobalState {
         unsafe { GLOBAL.assume_init_ref() }
     }
     /// Returns a handle to the [`GlobalNS`]
-    pub fn namespace(&self) -> &'static GlobalNS {
+    fn _namespace(&self) -> &'static GlobalNS {
         &unsafe { GLOBAL.assume_init_ref() }.gns
     }
     /// Post an urgent task
-    pub fn post_high_priority_task(&self, task: Task<CriticalTask>) {
+    fn _post_high_priority_task(&self, task: Task<CriticalTask>) {
         self.get_state().fractal_mgr().post_high_priority(task)
     }
     /// Post a task with normal priority
     ///
     /// NB: It is not guaranteed that the task will remain as a low priority task because the scheduler can choose
     /// to promote the task to a high priority task, if it deems necessary.
-    pub fn post_standard_priority_task(&self, task: Task<GenericTask>) {
+    fn _post_standard_priority_task(&self, task: Task<GenericTask>) {
         self.get_state().fractal_mgr().post_low_priority(task)
     }
     /// Returns the maximum size a model's delta size can hit before it should immediately issue a batch write request
     /// to avoid memory pressure
-    pub fn get_max_delta_size(&self) -> usize {
+    fn _get_max_delta_size(&self) -> usize {
         self.get_state()
             .fractal_mgr()
             .get_rt_stat()
diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
new file mode 100644
index 00000000..39e2e99d
--- /dev/null
+++ b/server/src/engine/fractal/test_utils.rs
@@ -0,0 +1,68 @@
+/*
+ * Created on Wed Sep 13 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{CriticalTask, GenericTask, GlobalInstanceLike, Task},
+    crate::engine::core::GlobalNS,
+    parking_lot::RwLock,
+};
+
+/// A `test` mode global implementation
+pub struct TestGlobal {
+    gns: GlobalNS,
+    hp_queue: RwLock<Vec<Task<CriticalTask>>>,
+    lp_queue: RwLock<Vec<Task<GenericTask>>>,
+    max_delta_size: usize,
+}
+
+impl TestGlobal {
+    pub fn empty() -> Self {
+        Self::with_max_delta_size(0)
+    }
+    pub fn with_max_delta_size(max_delta_size: usize) -> Self {
+        Self {
+            gns: GlobalNS::empty(),
+            hp_queue: RwLock::default(),
+            lp_queue: RwLock::default(),
+            max_delta_size,
+        }
+    }
+}
+
+impl GlobalInstanceLike for TestGlobal {
+    fn namespace(&self) -> &GlobalNS {
+        &self.gns
+    }
+    fn post_high_priority_task(&self, task: Task<CriticalTask>) {
+        self.hp_queue.write().push(task)
+    }
+    fn post_standard_priority_task(&self, task: Task<GenericTask>) {
+        self.lp_queue.write().push(task)
+    }
+    fn get_max_delta_size(&self) -> usize {
+        100
+    }
+}
diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
index 4ea29bcf..71d12ead 100644
--- a/server/src/engine/txn/gns/tests/full_chain.rs
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -28,10 +28,10 @@ use crate::engine::{
     core::{
         model::{Field, Layer, Model},
         space::{Space, SpaceMeta},
-        GlobalNS,
     },
     data::{cell::Datacell, tag::TagSelector, uuid::Uuid, DictEntryGeneric},
     error::DatabaseError,
+    fractal::{test_utils::TestGlobal, GlobalInstanceLike},
     idx::STIndex,
     ql::{
         ast::parse_ast_node_full,
@@ -52,13 +52,19 @@ fn with_variable<T>(var: T, f: impl FnOnce(T)) {
     f(var);
 }
 
-fn init_txn_driver(gns: &GlobalNS, log_name: &str) -> GNSTransactionDriverVFS {
-    GNSTransactionDriverVFS::open_or_reinit_with_name(&gns, log_name, 0, HostRunMode::Prod, 0)
-        .unwrap()
+fn init_txn_driver(global: &impl GlobalInstanceLike, log_name: &str) -> GNSTransactionDriverVFS {
+    GNSTransactionDriverVFS::open_or_reinit_with_name(
+        global.namespace(),
+        log_name,
+        0,
+        HostRunMode::Prod,
+        0,
+    )
+    .unwrap()
 }
 
 fn init_space(
-    gns: &GlobalNS,
+    global: &impl GlobalInstanceLike,
     driver: &mut GNSTransactionDriverVFS,
     space_name: &str,
     env: &str,
@@ -67,26 +73,32 @@ fn init_space(
     let stmt = lex_insecure(query.as_bytes()).unwrap();
     let stmt = parse_ast_node_full::<CreateSpace>(&stmt[2..]).unwrap();
     let name = stmt.space_name;
-    Space::transactional_exec_create(&gns, driver, stmt).unwrap();
-    gns.spaces().read().get(name.as_str()).unwrap().get_uuid()
+    Space::transactional_exec_create(global, driver, stmt).unwrap();
+    global
+        .namespace()
+        .spaces()
+        .read()
+        .get(name.as_str())
+        .unwrap()
+        .get_uuid()
 }
 
 #[test]
 fn create_space() {
-    with_variable("create_space_test.gns.db-tlog", |log_name| {
+    with_variable("create_space_test.global.db-tlog", |log_name| {
         let uuid;
         // start 1
         {
-            let gns = GlobalNS::empty();
-            let mut driver = init_txn_driver(&gns, log_name);
-            uuid = init_space(&gns, &mut driver, "myspace", "{ SAYAN_MAX: 65536 }"); // good lord that doesn't sound like a good variable
+            let global = TestGlobal::empty();
+            let mut driver = init_txn_driver(&global, log_name);
+            uuid = init_space(&global, &mut driver, "myspace", "{ SAYAN_MAX: 65536 }"); // good lord that doesn't sound like a good variable
             driver.close().unwrap();
         }
         multirun(|| {
-            let gns = GlobalNS::empty();
-            let driver = init_txn_driver(&gns, log_name);
+            let global = TestGlobal::empty();
+            let driver = init_txn_driver(&global, log_name);
             assert_eq!(
-                gns.spaces().read().get("myspace").unwrap(),
+                global.namespace().spaces().read().get("myspace").unwrap(),
                 &Space::new_restore_empty(
                     SpaceMeta::with_env(
                         into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint(65536)))
@@ -101,24 +113,24 @@ fn create_space() {
 
 #[test]
 fn alter_space() {
-    with_variable("alter_space_test.gns.db-tlog", |log_name| {
+    with_variable("alter_space_test.global.db-tlog", |log_name| {
         let uuid;
         {
-            let gns = GlobalNS::empty();
-            let mut driver = init_txn_driver(&gns, log_name);
-            uuid = init_space(&gns, &mut driver, "myspace", "{}");
+            let global = TestGlobal::empty();
+            let mut driver = init_txn_driver(&global, log_name);
+            uuid = init_space(&global, &mut driver, "myspace", "{}");
             let stmt =
                 lex_insecure("alter space myspace with { env: { SAYAN_MAX: 65536 } }".as_bytes())
                     .unwrap();
             let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
-            Space::transactional_exec_alter(&gns, &mut driver, stmt).unwrap();
+            Space::transactional_exec_alter(&global, &mut driver, stmt).unwrap();
             driver.close().unwrap();
         }
         multirun(|| {
-            let gns = GlobalNS::empty();
-            let driver = init_txn_driver(&gns, log_name);
+            let global = TestGlobal::empty();
+            let driver = init_txn_driver(&global, log_name);
             assert_eq!(
-                gns.spaces().read().get("myspace").unwrap(),
+                global.namespace().spaces().read().get("myspace").unwrap(),
                 &Space::new_restore_empty(
                     SpaceMeta::with_env(
                         into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint(65536)))
@@ -133,27 +145,27 @@ fn alter_space() {
 
 #[test]
 fn drop_space() {
-    with_variable("drop_space_test.gns.db-tlog", |log_name| {
+    with_variable("drop_space_test.global.db-tlog", |log_name| {
         {
-            let gns = GlobalNS::empty();
-            let mut driver = init_txn_driver(&gns, log_name);
-            let _ = init_space(&gns, &mut driver, "myspace", "{}");
+            let global = TestGlobal::empty();
+            let mut driver = init_txn_driver(&global, log_name);
+            let _ = init_space(&global, &mut driver, "myspace", "{}");
             let stmt = lex_insecure("drop space myspace".as_bytes()).unwrap();
             let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
-            Space::transactional_exec_drop(&gns, &mut driver, stmt).unwrap();
+            Space::transactional_exec_drop(&global, &mut driver, stmt).unwrap();
             driver.close().unwrap();
         }
         multirun(|| {
-            let gns = GlobalNS::empty();
-            let driver = init_txn_driver(&gns, log_name);
-            assert_eq!(gns.spaces().read().get("myspace"), None);
+            let global = TestGlobal::empty();
+            let driver = init_txn_driver(&global, log_name);
+            assert_eq!(global.namespace().spaces().read().get("myspace"), None);
             driver.close().unwrap();
         })
     })
 }
 
 fn init_model(
-    gns: &GlobalNS,
+    global: &impl GlobalInstanceLike,
     txn_driver: &mut GNSTransactionDriverVFS,
     space_name: &str,
     model_name: &str,
@@ -163,14 +175,19 @@ fn init_model(
     let stmt = lex_insecure(query.as_bytes()).unwrap();
     let stmt = parse_ast_node_full::<CreateModel>(&stmt[2..]).unwrap();
     let model_name = stmt.model_name;
-    Model::transactional_exec_create(&gns, txn_driver, stmt).unwrap();
-    gns.with_model(model_name, |model| Ok(model.get_uuid()))
+    Model::transactional_exec_create(global, txn_driver, stmt).unwrap();
+    global
+        .namespace()
+        .with_model(model_name, |model| Ok(model.get_uuid()))
         .unwrap()
 }
 
-fn init_default_model(gns: &GlobalNS, driver: &mut GNSTransactionDriverVFS) -> Uuid {
+fn init_default_model(
+    global: &impl GlobalInstanceLike,
+    driver: &mut GNSTransactionDriverVFS,
+) -> Uuid {
     init_model(
-        gns,
+        global,
         driver,
         "myspace",
         "mymodel",
@@ -180,35 +197,37 @@ fn init_default_model(gns: &GlobalNS, driver: &mut GNSTransactionDriverVFS) -> U
 
 #[test]
 fn create_model() {
-    with_variable("create_model_test.gns.db-tlog", |log_name| {
+    with_variable("create_model_test.global.db-tlog", |log_name| {
         let _uuid_space;
         let uuid_model;
         {
-            let gns = GlobalNS::empty();
-            let mut driver = init_txn_driver(&gns, log_name);
-            _uuid_space = init_space(&gns, &mut driver, "myspace", "{}");
-            uuid_model = init_default_model(&gns, &mut driver);
+            let global = TestGlobal::empty();
+            let mut driver = init_txn_driver(&global, log_name);
+            _uuid_space = init_space(&global, &mut driver, "myspace", "{}");
+            uuid_model = init_default_model(&global, &mut driver);
             driver.close().unwrap();
         }
         multirun(|| {
-            let gns = GlobalNS::empty();
-            let driver = init_txn_driver(&gns, log_name);
-            gns.with_model(("myspace", "mymodel"), |model| {
-                assert_eq!(
-                    model,
-                    &Model::new_restore(
-                        uuid_model,
-                        "username".into(),
-                        TagSelector::Str.into_full(),
-                        into_dict! {
-                            "username" => Field::new([Layer::str()].into(), false),
-                            "password" => Field::new([Layer::bin()].into(), false),
-                        }
-                    )
-                );
-                Ok(())
-            })
-            .unwrap();
+            let global = TestGlobal::empty();
+            let driver = init_txn_driver(&global, log_name);
+            global
+                .namespace()
+                .with_model(("myspace", "mymodel"), |model| {
+                    assert_eq!(
+                        model,
+                        &Model::new_restore(
+                            uuid_model,
+                            "username".into(),
+                            TagSelector::Str.into_full(),
+                            into_dict! {
+                                "username" => Field::new([Layer::str()].into(), false),
+                                "password" => Field::new([Layer::bin()].into(), false),
+                            }
+                        )
+                    );
+                    Ok(())
+                })
+                .unwrap();
             driver.close().unwrap();
         })
     })
@@ -216,35 +235,37 @@ fn create_model() {
 
 #[test]
 fn alter_model_add() {
-    with_variable("alter_model_add_test.gns.db-tlog", |log_name| {
+    with_variable("alter_model_add_test.global.db-tlog", |log_name| {
         {
-            let gns = GlobalNS::empty();
-            let mut driver = init_txn_driver(&gns, log_name);
-            init_space(&gns, &mut driver, "myspace", "{}");
-            init_default_model(&gns, &mut driver);
+            let global = TestGlobal::empty();
+            let mut driver = init_txn_driver(&global, log_name);
+            init_space(&global, &mut driver, "myspace", "{}");
+            init_default_model(&global, &mut driver);
             let stmt = lex_insecure(
                 b"alter model myspace.mymodel add profile_pic { type: binary, nullable: true }",
             )
             .unwrap();
             let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
-            Model::transactional_exec_alter(&gns, &mut driver, stmt).unwrap();
+            Model::transactional_exec_alter(&global, &mut driver, stmt).unwrap();
             driver.close().unwrap();
         }
         multirun(|| {
-            let gns = GlobalNS::empty();
-            let driver = init_txn_driver(&gns, log_name);
-            gns.with_model(("myspace", "mymodel"), |model| {
-                assert_eq!(
-                    model
-                        .intent_read_model()
-                        .fields()
-                        .st_get("profile_pic")
-                        .unwrap(),
-                    &Field::new([Layer::bin()].into(), true)
-                );
-                Ok(())
-            })
-            .unwrap();
+            let global = TestGlobal::empty();
+            let driver = init_txn_driver(&global, log_name);
+            global
+                .namespace()
+                .with_model(("myspace", "mymodel"), |model| {
+                    assert_eq!(
+                        model
+                            .intent_read_model()
+                            .fields()
+                            .st_get("profile_pic")
+                            .unwrap(),
+                        &Field::new([Layer::bin()].into(), true)
+                    );
+                    Ok(())
+                })
+                .unwrap();
             driver.close().unwrap();
         })
     })
@@ -252,13 +273,13 @@ fn alter_model_add() {
 
 #[test]
 fn alter_model_remove() {
-    with_variable("alter_model_remove_test.gns.db-tlog", |log_name| {
+    with_variable("alter_model_remove_test.global.db-tlog", |log_name| {
         {
-            let gns = GlobalNS::empty();
-            let mut driver = init_txn_driver(&gns, log_name);
-            init_space(&gns, &mut driver, "myspace", "{}");
+            let global = TestGlobal::empty();
+            let mut driver = init_txn_driver(&global, log_name);
+            init_space(&global, &mut driver, "myspace", "{}");
             init_model(
-                &gns,
+                &global,
                 &mut driver,
                 "myspace",
                 "mymodel",
@@ -269,19 +290,21 @@ fn alter_model_remove() {
             )
             .unwrap();
             let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
-            Model::transactional_exec_alter(&gns, &mut driver, stmt).unwrap();
+            Model::transactional_exec_alter(&global, &mut driver, stmt).unwrap();
             driver.close().unwrap()
         }
         multirun(|| {
-            let gns = GlobalNS::empty();
-            let driver = init_txn_driver(&gns, log_name);
-            gns.with_model(("myspace", "mymodel"), |model| {
-                let irm = model.intent_read_model();
-                assert!(irm.fields().st_get("has_secure_key").is_none());
-                assert!(irm.fields().st_get("is_dumb").is_none());
-                Ok(())
-            })
-            .unwrap();
+            let global = TestGlobal::empty();
+            let driver = init_txn_driver(&global, log_name);
+            global
+                .namespace()
+                .with_model(("myspace", "mymodel"), |model| {
+                    let irm = model.intent_read_model();
+                    assert!(irm.fields().st_get("has_secure_key").is_none());
+                    assert!(irm.fields().st_get("is_dumb").is_none());
+                    Ok(())
+                })
+                .unwrap();
             driver.close().unwrap()
         })
     })
@@ -289,13 +312,13 @@ fn alter_model_remove() {
 
 #[test]
 fn alter_model_update() {
-    with_variable("alter_model_update_test.gns.db-tlog", |log_name| {
+    with_variable("alter_model_update_test.global.db-tlog", |log_name| {
         {
-            let gns = GlobalNS::empty();
-            let mut driver = init_txn_driver(&gns, log_name);
-            init_space(&gns, &mut driver, "myspace", "{}");
+            let global = TestGlobal::empty();
+            let mut driver = init_txn_driver(&global, log_name);
+            init_space(&global, &mut driver, "myspace", "{}");
             init_model(
-                &gns,
+                &global,
                 &mut driver,
                 "myspace",
                 "mymodel",
@@ -305,24 +328,26 @@ fn alter_model_update() {
                 lex_insecure(b"alter model myspace.mymodel update profile_pic { nullable: true }")
                     .unwrap();
             let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
-            Model::transactional_exec_alter(&gns, &mut driver, stmt).unwrap();
+            Model::transactional_exec_alter(&global, &mut driver, stmt).unwrap();
             driver.close().unwrap();
         }
         multirun(|| {
-            let gns = GlobalNS::empty();
-            let driver = init_txn_driver(&gns, log_name);
-            gns.with_model(("myspace", "mymodel"), |model| {
-                assert_eq!(
-                    model
-                        .intent_read_model()
-                        .fields()
-                        .st_get("profile_pic")
-                        .unwrap(),
-                    &Field::new([Layer::bin()].into(), true)
-                );
-                Ok(())
-            })
-            .unwrap();
+            let global = TestGlobal::empty();
+            let driver = init_txn_driver(&global, log_name);
+            global
+                .namespace()
+                .with_model(("myspace", "mymodel"), |model| {
+                    assert_eq!(
+                        model
+                            .intent_read_model()
+                            .fields()
+                            .st_get("profile_pic")
+                            .unwrap(),
+                        &Field::new([Layer::bin()].into(), true)
+                    );
+                    Ok(())
+                })
+                .unwrap();
             driver.close().unwrap();
         })
     })
@@ -330,22 +355,24 @@ fn alter_model_update() {
 
 #[test]
 fn drop_model() {
-    with_variable("drop_model_test.gns.db-tlog", |log_name| {
+    with_variable("drop_model_test.global.db-tlog", |log_name| {
         {
-            let gns = GlobalNS::empty();
-            let mut driver = init_txn_driver(&gns, log_name);
-            init_space(&gns, &mut driver, "myspace", "{}");
-            init_default_model(&gns, &mut driver);
+            let global = TestGlobal::empty();
+            let mut driver = init_txn_driver(&global, log_name);
+            init_space(&global, &mut driver, "myspace", "{}");
+            init_default_model(&global, &mut driver);
             let stmt = lex_insecure(b"drop model myspace.mymodel").unwrap();
             let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
-            Model::transactional_exec_drop(&gns, &mut driver, stmt).unwrap();
+            Model::transactional_exec_drop(&global, &mut driver, stmt).unwrap();
             driver.close().unwrap();
         }
         multirun(|| {
-            let gns = GlobalNS::empty();
-            let driver = init_txn_driver(&gns, log_name);
+            let global = TestGlobal::empty();
+            let driver = init_txn_driver(&global, log_name);
             assert_eq!(
-                gns.with_model(("myspace", "mymodel"), |_| { Ok(()) })
+                global
+                    .namespace()
+                    .with_model(("myspace", "mymodel"), |_| { Ok(()) })
                     .unwrap_err(),
                 DatabaseError::DdlModelNotFound
             );

From cf055f418dbc7362b20f0cad9c93e54ed48c9216 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 13 Sep 2023 08:18:18 +0000
Subject: [PATCH 249/310] Request delta updates

---
 server/src/engine/core/dml/del.rs     |  4 ++--
 server/src/engine/core/dml/ins.rs     |  5 ++--
 server/src/engine/core/dml/upd.rs     |  4 ++--
 server/src/engine/core/index/mod.rs   |  3 +++
 server/src/engine/core/mod.rs         | 23 +++++++++++++++++++
 server/src/engine/core/model/delta.rs | 33 +++++++++++++++++++++++++--
 server/src/engine/fractal/mod.rs      | 13 +++++++++++
 server/src/engine/idx/mod.rs          |  2 ++
 server/src/engine/idx/mtchm/imp.rs    |  3 +++
 9 files changed, 82 insertions(+), 8 deletions(-)

diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs
index 667cb39d..41ab2b64 100644
--- a/server/src/engine/core/dml/del.rs
+++ b/server/src/engine/core/dml/del.rs
@@ -25,7 +25,7 @@
 */
 
 use crate::engine::{
-    core::model::delta::DataDeltaKind,
+    core::{self, model::delta::DataDeltaKind},
     error::{DatabaseError, DatabaseResult},
     fractal::GlobalInstanceLike,
     idx::MTIndex,
@@ -34,7 +34,7 @@ use crate::engine::{
 };
 
 pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) -> DatabaseResult<()> {
-    global.namespace().with_model(delete.entity(), |model| {
+    core::with_model_for_data_update(global, delete.entity(), |model| {
         let g = sync::atm::cpin();
         let schema_version = model.delta_state().schema_current_version();
         let delta_state = model.delta_state();
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index 0ea02d72..d1e10610 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -26,6 +26,7 @@
 
 use crate::engine::{
     core::{
+        self,
         index::{DcFieldIndex, PrimaryIndexKey, Row},
         model::{delta::DataDeltaKind, Fields, Model},
     },
@@ -36,8 +37,8 @@ use crate::engine::{
     sync::atm::cpin,
 };
 
-pub fn insert(gns: &impl GlobalInstanceLike, insert: InsertStatement) -> DatabaseResult<()> {
-    gns.namespace().with_model(insert.entity(), |mdl| {
+pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> DatabaseResult<()> {
+    core::with_model_for_data_update(global, insert.entity(), |mdl| {
         let irmwd = mdl.intent_write_new_data();
         let (pk, data) = prepare_insert(mdl, irmwd.fields(), insert.data())?;
         let g = cpin();
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index 9a3cb285..71908643 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -30,7 +30,7 @@ use std::cell::RefCell;
 use {
     crate::{
         engine::{
-            core::{model::delta::DataDeltaKind, query_meta::AssignmentOperator},
+            core::{self, model::delta::DataDeltaKind, query_meta::AssignmentOperator},
             data::{
                 cell::Datacell,
                 lit::LitIR,
@@ -234,7 +234,7 @@ pub fn collect_trace_path() -> Vec<&'static str> {
 }
 
 pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) -> DatabaseResult<()> {
-    global.namespace().with_model(update.entity(), |mdl| {
+    core::with_model_for_data_update(global, update.entity(), |mdl| {
         let mut ret = Ok(());
         // prepare row fetch
         let key = mdl.resolve_where(update.clauses_mut())?;
diff --git a/server/src/engine/core/index/mod.rs b/server/src/engine/core/index/mod.rs
index 46d3a125..4fafe5f3 100644
--- a/server/src/engine/core/index/mod.rs
+++ b/server/src/engine/core/index/mod.rs
@@ -62,4 +62,7 @@ impl PrimaryIndex {
     pub fn __raw_index(&self) -> &IndexMTRaw<row::Row> {
         &self.data
     }
+    pub fn count(&self) -> usize {
+        self.data.mt_len()
+    }
 }
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 340df67b..05e7ca70 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -39,6 +39,7 @@ use {
     crate::engine::{
         core::space::Space,
         error::{DatabaseError, DatabaseResult},
+        fractal::GlobalInstanceLike,
         idx::{IndexST, STIndex},
     },
     parking_lot::RwLock,
@@ -55,6 +56,28 @@ pub struct GlobalNS {
     index_space: RWLIdx<Box<str>, Space>,
 }
 
+pub(self) fn with_model_for_data_update<'a, T, E, F>(
+    global: &impl GlobalInstanceLike,
+    entity: E,
+    f: F,
+) -> DatabaseResult<T>
+where
+    F: FnOnce(&Model) -> DatabaseResult<T>,
+    E: 'a + EntityLocator<'a>,
+{
+    let (space_name, model_name) = entity.parse_entity()?;
+    global
+        .namespace()
+        .with_model((space_name, model_name), |mdl| {
+            let r = f(mdl);
+            // see if this task local delta is full
+            if r.is_ok() {
+                model::DeltaState::guard_delta_overflow(global, space_name, model_name, mdl);
+            }
+            r
+        })
+}
+
 impl GlobalNS {
     pub fn spaces(&self) -> &RWLIdx<Box<str>, Space> {
         &self.index_space
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index 5365f1c4..dd5b13b3 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -26,8 +26,14 @@
 
 use {
     super::{Fields, Model},
-    crate::engine::{
-        core::index::Row, fractal::FractalToken, sync::atm::Guard, sync::queue::Queue,
+    crate::{
+        engine::{
+            core::index::Row,
+            fractal::{FractalToken, GlobalInstanceLike},
+            sync::atm::Guard,
+            sync::queue::Queue,
+        },
+        util::compiler,
     },
     parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard},
     std::{
@@ -171,6 +177,29 @@ impl DeltaState {
     }
 }
 
+// data direct
+impl DeltaState {
+    pub(in crate::engine::core) fn guard_delta_overflow(
+        global: &impl GlobalInstanceLike,
+        space_name: &str,
+        model_name: &str,
+        model: &Model,
+    ) {
+        let current_deltas_size = model.delta_state().data_deltas_size.load(Ordering::Acquire);
+        let max_len = global
+            .get_max_delta_size()
+            .min((model.primary_index().count() as f64 * 0.05) as usize);
+        if compiler::unlikely(current_deltas_size >= max_len) {
+            global.request_batch_resolve(
+                space_name,
+                model_name,
+                model.get_uuid(),
+                current_deltas_size,
+            );
+        }
+    }
+}
+
 // data
 impl DeltaState {
     pub fn append_new_data_delta_with(
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index cc875963..4358eea5 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -102,6 +102,19 @@ pub trait GlobalInstanceLike {
     fn post_high_priority_task(&self, task: Task<CriticalTask>);
     fn post_standard_priority_task(&self, task: Task<GenericTask>);
     fn get_max_delta_size(&self) -> usize;
+    // default impls
+    fn request_batch_resolve(
+        &self,
+        space_name: &str,
+        model_name: &str,
+        model_uuid: Uuid,
+        observed_len: usize,
+    ) {
+        self.post_high_priority_task(Task::new(CriticalTask::WriteBatch(
+            ModelUniqueID::new(space_name, model_name, model_uuid),
+            observed_len,
+        )))
+    }
 }
 
 impl GlobalInstanceLike for Global {
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 835fca50..449a5318 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -149,6 +149,8 @@ pub trait MTIndex<E, K, V>: IndexBaseSpec {
         't: 'v,
         V: 'v,
         Self: 't;
+    /// Returns the length of the index
+    fn mt_len(&self) -> usize;
     /// Attempts to compact the backing storage
     fn mt_compact(&self) {}
     /// Clears all the entries in the MTIndex
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index a253a81e..94d1b2d1 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -87,6 +87,9 @@ impl<E: TreeElement, C: Config> MTIndex<E, E::Key, E::Value> for Raw<E, C> {
         E::Value: 'v,
         Self: 't;
 
+    fn mt_len(&self) -> usize {
+        self.len()
+    }
     fn mt_clear(&self, g: &Guard) {
         self.nontransactional_clear(g)
     }

From 1abe4a72179afa6f860d94a3cb264d26afb2121f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 13 Sep 2023 15:33:24 +0000
Subject: [PATCH 250/310] Simplify gns txn driver handling

---
 server/src/engine/core/model/alt.rs           |  22 ++-
 server/src/engine/core/model/mod.rs           |  35 ++---
 server/src/engine/core/space.rs               |  62 ++-------
 server/src/engine/core/tests/ddl_model/alt.rs |  10 +-
 server/src/engine/core/tests/ddl_model/crt.rs |   2 +-
 server/src/engine/core/tests/ddl_model/mod.rs |   2 +-
 .../src/engine/core/tests/ddl_space/alter.rs  |  10 +-
 .../src/engine/core/tests/ddl_space/create.rs |   8 +-
 server/src/engine/core/tests/ddl_space/mod.rs |   4 +-
 server/src/engine/core/tests/dml/delete.rs    |   4 +-
 server/src/engine/core/tests/dml/insert.rs    |   6 +-
 server/src/engine/core/tests/dml/mod.rs       |   2 +-
 server/src/engine/core/tests/dml/select.rs    |  10 +-
 server/src/engine/core/tests/dml/update.rs    |  12 +-
 server/src/engine/fractal/drivers.rs          |   3 +
 server/src/engine/fractal/mod.rs              |  32 +++--
 server/src/engine/fractal/test_utils.rs       |  79 +++++++++--
 server/src/engine/idx/mtchm/imp.rs            |   2 +-
 server/src/engine/idx/mtchm/mod.rs            |   2 +-
 server/src/engine/storage/v1/journal.rs       |   5 +-
 server/src/engine/txn/gns/mod.rs              |  38 +----
 server/src/engine/txn/gns/tests/full_chain.rs | 131 +++++-------------
 22 files changed, 207 insertions(+), 274 deletions(-)

diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index a4273e3f..659c126a 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -249,9 +249,8 @@ impl<'a> AlterPlan<'a> {
 }
 
 impl Model {
-    pub fn transactional_exec_alter<GI: gnstxn::GNSTransactionDriverLLInterface>(
-        global: &impl GlobalInstanceLike,
-        txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<GI>,
+    pub fn transactional_exec_alter<G: GlobalInstanceLike>(
+        global: &G,
         alter: AlterModel,
     ) -> DatabaseResult<()> {
         let (space_name, model_name) = EntityLocator::parse_entity(alter.model)?;
@@ -273,14 +272,14 @@ impl Model {
                     AlterAction::Add(new_fields) => {
                         let mut guard = model.delta_state().schema_delta_write();
                         // TODO(@ohsayan): this impacts lockdown duration; fix it
-                        if GI::NONNULL {
+                        if G::FS_IS_NON_NULL {
                             // prepare txn
                             let txn = gnstxn::AlterModelAddTxn::new(
                                 gnstxn::ModelIDRef::new_ref(space_name, space, model_name, model),
                                 &new_fields,
                             );
                             // commit txn
-                            txn_driver.try_commit(txn)?;
+                            global.namespace_txn_driver().lock().try_commit(txn)?;
                         }
                         new_fields
                             .stseq_ord_kv()
@@ -294,14 +293,14 @@ impl Model {
                     }
                     AlterAction::Remove(removed) => {
                         let mut guard = model.delta_state().schema_delta_write();
-                        if GI::NONNULL {
+                        if G::FS_IS_NON_NULL {
                             // prepare txn
                             let txn = gnstxn::AlterModelRemoveTxn::new(
                                 gnstxn::ModelIDRef::new_ref(space_name, space, model_name, model),
                                 &removed,
                             );
                             // commit txn
-                            txn_driver.try_commit(txn)?;
+                            global.namespace_txn_driver().lock().try_commit(txn)?;
                         }
                         removed.iter().for_each(|field_id| {
                             model.delta_state().schema_append_unresolved_wl_field_rem(
@@ -312,14 +311,14 @@ impl Model {
                         });
                     }
                     AlterAction::Update(updated) => {
-                        if GI::NONNULL {
+                        if G::FS_IS_NON_NULL {
                             // prepare txn
                             let txn = gnstxn::AlterModelUpdateTxn::new(
                                 gnstxn::ModelIDRef::new_ref(space_name, space, model_name, model),
                                 &updated,
                             );
                             // commit txn
-                            txn_driver.try_commit(txn)?;
+                            global.namespace_txn_driver().lock().try_commit(txn)?;
                         }
                         updated.into_iter().for_each(|(field_id, field)| {
                             iwm.fields_mut().st_update(&field_id, field);
@@ -330,9 +329,4 @@ impl Model {
             })
         })
     }
-    pub fn exec_alter(global: &impl GlobalInstanceLike, stmt: AlterModel) -> DatabaseResult<()> {
-        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(global.namespace(), |driver| {
-            Self::transactional_exec_alter(global, driver, stmt)
-        })
-    }
 }
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index fb037092..3926f8fe 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -204,9 +204,8 @@ impl Model {
 }
 
 impl Model {
-    pub fn transactional_exec_create<GI: gnstxn::GNSTransactionDriverLLInterface>(
-        global: &impl GlobalInstanceLike,
-        txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<GI>,
+    pub fn transactional_exec_create<G: GlobalInstanceLike>(
+        global: &G,
         stmt: CreateModel,
     ) -> DatabaseResult<()> {
         let (space_name, model_name) = stmt.model_name.parse_entity()?;
@@ -216,9 +215,10 @@ impl Model {
             if w_space.st_contains(model_name) {
                 return Err(DatabaseError::DdlModelAlreadyExists);
             }
-            if GI::NONNULL {
+            if G::FS_IS_NON_NULL {
                 // prepare txn
                 let irm = model.intent_read_model();
+                let mut txn_driver = global.namespace_txn_driver().lock();
                 let txn = gnstxn::CreateModelTxn::new(
                     gnstxn::SpaceIDRef::new(space_name, space),
                     model_name,
@@ -233,18 +233,8 @@ impl Model {
             Ok(())
         })
     }
-    #[cfg(test)]
-    pub fn nontransactional_exec_create(
-        global: &impl GlobalInstanceLike,
-        stmt: CreateModel,
-    ) -> DatabaseResult<()> {
-        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(global.namespace(), |driver| {
-            Self::transactional_exec_create(global, driver, stmt)
-        })
-    }
-    pub fn transactional_exec_drop<GI: gnstxn::GNSTransactionDriverLLInterface>(
-        global: &impl GlobalInstanceLike,
-        txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<GI>,
+    pub fn transactional_exec_drop<G: GlobalInstanceLike>(
+        global: &G,
         stmt: DropModel,
     ) -> DatabaseResult<()> {
         let (space_name, model_name) = stmt.entity.parse_entity()?;
@@ -253,7 +243,7 @@ impl Model {
             let Some(model) = w_space.get(model_name) else {
                 return Err(DatabaseError::DdlModelNotFound);
             };
-            if GI::NONNULL {
+            if G::FS_IS_NON_NULL {
                 // prepare txn
                 let txn = gnstxn::DropModelTxn::new(gnstxn::ModelIDRef::new(
                     gnstxn::SpaceIDRef::new(space_name, space),
@@ -262,22 +252,13 @@ impl Model {
                     model.delta_state().schema_current_version().value_u64(),
                 ));
                 // commit txn
-                txn_driver.try_commit(txn)?;
+                global.namespace_txn_driver().lock().try_commit(txn)?;
             }
             // update global state
             let _ = w_space.st_delete(model_name);
             Ok(())
         })
     }
-    #[cfg(test)]
-    pub fn nontransactional_exec_drop(
-        global: &impl GlobalInstanceLike,
-        stmt: DropModel,
-    ) -> DatabaseResult<()> {
-        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(global.namespace(), |driver| {
-            Self::transactional_exec_drop(global, driver, stmt)
-        })
-    }
 }
 
 /*
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 6a948dfc..74d2d068 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -189,9 +189,8 @@ impl Space {
 }
 
 impl Space {
-    pub fn transactional_exec_create<TI: gnstxn::GNSTransactionDriverLLInterface>(
-        global: &impl GlobalInstanceLike,
-        txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<TI>,
+    pub fn transactional_exec_create<G: GlobalInstanceLike>(
+        global: &G,
         space: CreateSpace,
     ) -> DatabaseResult<()> {
         // process create
@@ -202,29 +201,20 @@ impl Space {
             return Err(DatabaseError::DdlSpaceAlreadyExists);
         }
         // commit txn
-        if TI::NONNULL {
+        if G::FS_IS_NON_NULL {
             // prepare and commit txn
             let s_read = space.metadata().dict().read();
-            txn_driver.try_commit(gnstxn::CreateSpaceTxn::new(&s_read, &space_name, &space))?;
+            global
+                .namespace_txn_driver()
+                .lock()
+                .try_commit(gnstxn::CreateSpaceTxn::new(&s_read, &space_name, &space))?;
         }
         // update global state
         let _ = wl.st_insert(space_name, space);
         Ok(())
     }
-    /// Execute a `create` stmt
-    #[cfg(test)]
-    pub fn nontransactional_exec_create(
-        global: &impl GlobalInstanceLike,
-        space: CreateSpace,
-    ) -> DatabaseResult<()> {
-        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(
-            global.namespace(),
-            move |driver| Self::transactional_exec_create(global, driver, space),
-        )
-    }
-    pub fn transactional_exec_alter<TI: gnstxn::GNSTransactionDriverLLInterface>(
-        global: &impl GlobalInstanceLike,
-        txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<TI>,
+    pub fn transactional_exec_alter<G: GlobalInstanceLike>(
+        global: &G,
         AlterSpace {
             space_name,
             updated_props,
@@ -243,12 +233,12 @@ impl Space {
                 Some(patch) => patch,
                 None => return Err(DatabaseError::DdlSpaceBadProperty),
             };
-            if TI::NONNULL {
+            if G::FS_IS_NON_NULL {
                 // prepare txn
                 let txn =
                     gnstxn::AlterSpaceTxn::new(gnstxn::SpaceIDRef::new(&space_name, space), &patch);
                 // commit
-                txn_driver.try_commit(txn)?;
+                global.namespace_txn_driver().lock().try_commit(txn)?;
             }
             // merge
             dict::rmerge_data_with_patch(&mut space_props, patch);
@@ -261,20 +251,8 @@ impl Space {
             Ok(())
         })
     }
-    #[cfg(test)]
-    /// Execute a `alter` stmt
-    pub fn nontransactional_exec_alter(
-        global: &impl GlobalInstanceLike,
-        alter: AlterSpace,
-    ) -> DatabaseResult<()> {
-        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(
-            global.namespace(),
-            move |driver| Self::transactional_exec_alter(global, driver, alter),
-        )
-    }
-    pub fn transactional_exec_drop<TI: gnstxn::GNSTransactionDriverLLInterface>(
-        global: &impl GlobalInstanceLike,
-        txn_driver: &mut gnstxn::GNSTransactionDriverAnyFS<TI>,
+    pub fn transactional_exec_drop<G: GlobalInstanceLike>(
+        global: &G,
         DropSpace { space, force: _ }: DropSpace,
     ) -> DatabaseResult<()> {
         // TODO(@ohsayan): force remove option
@@ -290,25 +268,15 @@ impl Space {
             return Err(DatabaseError::DdlSpaceRemoveNonEmpty);
         }
         // we can remove this
-        if TI::NONNULL {
+        if G::FS_IS_NON_NULL {
             // prepare txn
             let txn = gnstxn::DropSpaceTxn::new(gnstxn::SpaceIDRef::new(&space_name, space));
-            txn_driver.try_commit(txn)?;
+            global.namespace_txn_driver().lock().try_commit(txn)?;
         }
         drop(space_w);
         let _ = wgns.st_delete(space_name.as_str());
         Ok(())
     }
-    #[cfg(test)]
-    pub fn nontransactional_exec_drop(
-        global: &impl GlobalInstanceLike,
-        drop_space: DropSpace,
-    ) -> DatabaseResult<()> {
-        gnstxn::GNSTransactionDriverNullZero::nullzero_create_exec(
-            global.namespace(),
-            move |driver| Self::transactional_exec_drop(global, driver, drop_space),
-        )
-    }
 }
 
 #[cfg(test)]
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index ca017e03..c86cf586 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -63,7 +63,7 @@ fn exec_plan(
     let tok = lex_insecure(plan.as_bytes()).unwrap();
     let alter = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
     let (_space, model_name) = alter.model.into_full().unwrap();
-    Model::exec_alter(global, alter)?;
+    Model::transactional_exec_alter(global, alter)?;
     let gns_read = global.namespace().spaces().read();
     let space = gns_read.st_get("myspace").unwrap();
     let model = space.models().read();
@@ -359,7 +359,7 @@ mod exec {
     };
     #[test]
     fn simple_add() {
-        let global = TestGlobal::empty();
+        let global = TestGlobal::new_with_tmp_nullfs_driver();
         super::exec_plan(
             &global,
             true,
@@ -390,7 +390,7 @@ mod exec {
     }
     #[test]
     fn simple_remove() {
-        let global = TestGlobal::empty();
+        let global = TestGlobal::new_with_tmp_nullfs_driver();
         super::exec_plan(
             &global,
             true,
@@ -416,7 +416,7 @@ mod exec {
     }
     #[test]
     fn simple_update() {
-        let global = TestGlobal::empty();
+        let global = TestGlobal::new_with_tmp_nullfs_driver();
         super::exec_plan(
             &global,
             true,
@@ -435,7 +435,7 @@ mod exec {
     }
     #[test]
     fn failing_alter_nullable_switch_need_lock() {
-        let global = TestGlobal::empty();
+        let global = TestGlobal::new_with_tmp_nullfs_driver();
         assert_eq!(
             super::exec_plan(
                 &global,
diff --git a/server/src/engine/core/tests/ddl_model/crt.rs b/server/src/engine/core/tests/ddl_model/crt.rs
index 4b247002..1011d8a2 100644
--- a/server/src/engine/core/tests/ddl_model/crt.rs
+++ b/server/src/engine/core/tests/ddl_model/crt.rs
@@ -145,7 +145,7 @@ mod exec {
 
     #[test]
     fn simple() {
-        let global = TestGlobal::empty();
+        let global = TestGlobal::new_with_tmp_nullfs_driver();
         exec_create_new_space(
             &global,
             "create model myspace.mymodel(username: string, password: binary)",
diff --git a/server/src/engine/core/tests/ddl_model/mod.rs b/server/src/engine/core/tests/ddl_model/mod.rs
index 61af1fc4..c2a52baf 100644
--- a/server/src/engine/core/tests/ddl_model/mod.rs
+++ b/server/src/engine/core/tests/ddl_model/mod.rs
@@ -61,7 +61,7 @@ pub fn exec_create(
             .namespace()
             .test_new_empty_space(&create_model.model_name.into_full().unwrap().0);
     }
-    Model::nontransactional_exec_create(global, create_model).map(|_| name)
+    Model::transactional_exec_create(global, create_model).map(|_| name)
 }
 
 pub fn exec_create_new_space(
diff --git a/server/src/engine/core/tests/ddl_space/alter.rs b/server/src/engine/core/tests/ddl_space/alter.rs
index ecfd7314..aa772b25 100644
--- a/server/src/engine/core/tests/ddl_space/alter.rs
+++ b/server/src/engine/core/tests/ddl_space/alter.rs
@@ -33,7 +33,7 @@ use crate::engine::{
 
 #[test]
 fn alter_add_prop_env_var() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     super::exec_create_alter(
         &global,
         "create space myspace",
@@ -54,7 +54,7 @@ fn alter_add_prop_env_var() {
 
 #[test]
 fn alter_update_prop_env_var() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     let uuid = super::exec_create(
         &global,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
@@ -86,7 +86,7 @@ fn alter_update_prop_env_var() {
 
 #[test]
 fn alter_remove_prop_env_var() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     let uuid = super::exec_create(
         &global,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
@@ -114,7 +114,7 @@ fn alter_remove_prop_env_var() {
 
 #[test]
 fn alter_nx() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_alter(
             &global,
@@ -128,7 +128,7 @@ fn alter_nx() {
 
 #[test]
 fn alter_remove_all_env() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     let uuid = super::exec_create(
         &global,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
diff --git a/server/src/engine/core/tests/ddl_space/create.rs b/server/src/engine/core/tests/ddl_space/create.rs
index bcf9cf2b..574121fa 100644
--- a/server/src/engine/core/tests/ddl_space/create.rs
+++ b/server/src/engine/core/tests/ddl_space/create.rs
@@ -33,7 +33,7 @@ use crate::engine::{
 
 #[test]
 fn exec_create_space_simple() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     super::exec_create(&global, "create space myspace", |spc| {
         assert!(spc.models().read().is_empty())
     })
@@ -42,7 +42,7 @@ fn exec_create_space_simple() {
 
 #[test]
 fn exec_create_space_with_env() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     super::exec_create(
         &global,
         r#"
@@ -70,7 +70,7 @@ fn exec_create_space_with_env() {
 
 #[test]
 fn exec_create_space_with_bad_env_type() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_create(&global, "create space myspace with { env: 100 }", |_| {}).unwrap_err(),
         DatabaseError::DdlSpaceBadProperty
@@ -79,7 +79,7 @@ fn exec_create_space_with_bad_env_type() {
 
 #[test]
 fn exec_create_space_with_random_property() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_create(
             &global,
diff --git a/server/src/engine/core/tests/ddl_space/mod.rs b/server/src/engine/core/tests/ddl_space/mod.rs
index 7e05f0a8..c3942564 100644
--- a/server/src/engine/core/tests/ddl_space/mod.rs
+++ b/server/src/engine/core/tests/ddl_space/mod.rs
@@ -47,7 +47,7 @@ fn exec_create(
     let ast_node =
         ast::parse_ast_node_full::<crate::engine::ql::ddl::crt::CreateSpace>(&tok[2..]).unwrap();
     let name = ast_node.space_name;
-    Space::nontransactional_exec_create(gns, ast_node)?;
+    Space::transactional_exec_create(gns, ast_node)?;
     gns.namespace().with_space(&name, |space| {
         verify(space);
         Ok(space.get_uuid())
@@ -63,7 +63,7 @@ fn exec_alter(
     let ast_node =
         ast::parse_ast_node_full::<crate::engine::ql::ddl::alt::AlterSpace>(&tok[2..]).unwrap();
     let name = ast_node.space_name;
-    Space::nontransactional_exec_alter(gns, ast_node)?;
+    Space::transactional_exec_alter(gns, ast_node)?;
     gns.namespace().with_space(&name, |space| {
         verify(space);
         Ok(space.get_uuid())
diff --git a/server/src/engine/core/tests/dml/delete.rs b/server/src/engine/core/tests/dml/delete.rs
index 92183ab2..e0b0ee39 100644
--- a/server/src/engine/core/tests/dml/delete.rs
+++ b/server/src/engine/core/tests/dml/delete.rs
@@ -28,7 +28,7 @@ use crate::engine::{error::DatabaseError, fractal::test_utils::TestGlobal};
 
 #[test]
 fn simple_delete() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     super::exec_delete(
         &global,
         "create model myspace.mymodel(username: string, password: string)",
@@ -41,7 +41,7 @@ fn simple_delete() {
 
 #[test]
 fn delete_nonexisting() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_delete(
             &global,
diff --git a/server/src/engine/core/tests/dml/insert.rs b/server/src/engine/core/tests/dml/insert.rs
index de36b22a..fb3a07a5 100644
--- a/server/src/engine/core/tests/dml/insert.rs
+++ b/server/src/engine/core/tests/dml/insert.rs
@@ -31,7 +31,7 @@ struct Tuple(Vec<(Box<str>, Datacell)>);
 
 #[test]
 fn insert_simple() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     super::exec_insert(
         &global,
         "create model myspace.mymodel(username: string, password: string)",
@@ -46,7 +46,7 @@ fn insert_simple() {
 
 #[test]
 fn insert_with_null() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     super::exec_insert(
         &global,
         "create model myspace.mymodel(username: string, null useless_password: string, null useless_email: string, null useless_random_column: uint64)",
@@ -69,7 +69,7 @@ fn insert_with_null() {
 
 #[test]
 fn insert_duplicate() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     super::exec_insert(
         &global,
         "create model myspace.mymodel(username: string, password: string)",
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index 8f14363a..c1d489a9 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -51,7 +51,7 @@ fn _exec_only_create_space_model(
     }
     let lex_create_model = lex_insecure(model.as_bytes()).unwrap();
     let stmt_create_model = parse_ast_node_full(&lex_create_model[2..]).unwrap();
-    Model::nontransactional_exec_create(global, stmt_create_model)
+    Model::transactional_exec_create(global, stmt_create_model)
 }
 
 fn _exec_only_insert<T>(
diff --git a/server/src/engine/core/tests/dml/select.rs b/server/src/engine/core/tests/dml/select.rs
index 8c5f079c..600c4dd8 100644
--- a/server/src/engine/core/tests/dml/select.rs
+++ b/server/src/engine/core/tests/dml/select.rs
@@ -28,7 +28,7 @@ use crate::engine::{data::cell::Datacell, error::DatabaseError, fractal::test_ut
 
 #[test]
 fn simple_select_wildcard() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_select(
             &global,
@@ -43,7 +43,7 @@ fn simple_select_wildcard() {
 
 #[test]
 fn simple_select_specified_same_order() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_select(
             &global,
@@ -58,7 +58,7 @@ fn simple_select_specified_same_order() {
 
 #[test]
 fn simple_select_specified_reversed_order() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_select(
             &global,
@@ -73,7 +73,7 @@ fn simple_select_specified_reversed_order() {
 
 #[test]
 fn select_null() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_select(
             &global,
@@ -88,7 +88,7 @@ fn select_null() {
 
 #[test]
 fn select_nonexisting() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_select(
             &global,
diff --git a/server/src/engine/core/tests/dml/update.rs b/server/src/engine/core/tests/dml/update.rs
index f49ed310..b1338b17 100644
--- a/server/src/engine/core/tests/dml/update.rs
+++ b/server/src/engine/core/tests/dml/update.rs
@@ -30,7 +30,7 @@ use crate::engine::{
 
 #[test]
 fn simple() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_update(
             &global,
@@ -49,7 +49,7 @@ fn simple() {
 
 #[test]
 fn with_null() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_update(
             &global,
@@ -66,7 +66,7 @@ fn with_null() {
 
 #[test]
 fn with_list() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_update(
             &global,
@@ -86,7 +86,7 @@ fn with_list() {
 
 #[test]
 fn fail_operation_on_null() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_update(
             &global,
@@ -106,7 +106,7 @@ fn fail_operation_on_null() {
 
 #[test]
 fn fail_unknown_fields() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_update(
             &global,
@@ -132,7 +132,7 @@ fn fail_unknown_fields() {
 
 #[test]
 fn fail_typedef_violation() {
-    let global = TestGlobal::empty();
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_update(
             &global,
diff --git a/server/src/engine/fractal/drivers.rs b/server/src/engine/fractal/drivers.rs
index 8bd42c9d..c003c8c2 100644
--- a/server/src/engine/fractal/drivers.rs
+++ b/server/src/engine/fractal/drivers.rs
@@ -47,6 +47,9 @@ impl FractalGNSDriver {
             txn_driver: Mutex::new(txn_driver),
         }
     }
+    pub fn txn_driver(&self) -> &Mutex<GNSTransactionDriverAnyFS<LocalFS>> {
+        &self.txn_driver
+    }
 }
 
 /// Model driver
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index 4358eea5..26f5d1db 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -26,9 +26,12 @@
 
 use {
     super::{
-        core::GlobalNS, data::uuid::Uuid, storage::v1::LocalFS, txn::gns::GNSTransactionDriverAnyFS,
+        core::GlobalNS,
+        data::uuid::Uuid,
+        storage::v1::{LocalFS, RawFSInterface},
+        txn::gns::GNSTransactionDriverAnyFS,
     },
-    parking_lot::RwLock,
+    parking_lot::{Mutex, RwLock},
     std::{collections::HashMap, mem::MaybeUninit},
     tokio::sync::mpsc::unbounded_channel,
 };
@@ -98,10 +101,16 @@ pub unsafe fn enable_and_start_all(
 
 /// Something that represents the global state
 pub trait GlobalInstanceLike {
-    fn namespace(&self) -> &GlobalNS;
-    fn post_high_priority_task(&self, task: Task<CriticalTask>);
-    fn post_standard_priority_task(&self, task: Task<GenericTask>);
+    type FileSystem: RawFSInterface;
+    const FS_IS_NON_NULL: bool = Self::FileSystem::NOT_NULL;
+    // stat
     fn get_max_delta_size(&self) -> usize;
+    // global namespace
+    fn namespace(&self) -> &GlobalNS;
+    fn namespace_txn_driver(&self) -> &Mutex<GNSTransactionDriverAnyFS<Self::FileSystem>>;
+    // taskmgr
+    fn taskmgr_post_high_priority(&self, task: Task<CriticalTask>);
+    fn taskmgr_post_standard_priority(&self, task: Task<GenericTask>);
     // default impls
     fn request_batch_resolve(
         &self,
@@ -110,7 +119,7 @@ pub trait GlobalInstanceLike {
         model_uuid: Uuid,
         observed_len: usize,
     ) {
-        self.post_high_priority_task(Task::new(CriticalTask::WriteBatch(
+        self.taskmgr_post_high_priority(Task::new(CriticalTask::WriteBatch(
             ModelUniqueID::new(space_name, model_name, model_uuid),
             observed_len,
         )))
@@ -118,15 +127,22 @@ pub trait GlobalInstanceLike {
 }
 
 impl GlobalInstanceLike for Global {
+    type FileSystem = LocalFS;
+    // ns
     fn namespace(&self) -> &GlobalNS {
         self._namespace()
     }
-    fn post_high_priority_task(&self, task: Task<CriticalTask>) {
+    fn namespace_txn_driver(&self) -> &Mutex<GNSTransactionDriverAnyFS<Self::FileSystem>> {
+        self.get_state().gns_driver.txn_driver()
+    }
+    // taskmgr
+    fn taskmgr_post_high_priority(&self, task: Task<CriticalTask>) {
         self._post_high_priority_task(task)
     }
-    fn post_standard_priority_task(&self, task: Task<GenericTask>) {
+    fn taskmgr_post_standard_priority(&self, task: Task<GenericTask>) {
         self._post_standard_priority_task(task)
     }
+    // stat
     fn get_max_delta_size(&self) -> usize {
         self._get_max_delta_size()
     }
diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
index 39e2e99d..9789754a 100644
--- a/server/src/engine/fractal/test_utils.rs
+++ b/server/src/engine/fractal/test_utils.rs
@@ -26,43 +26,98 @@
 
 use {
     super::{CriticalTask, GenericTask, GlobalInstanceLike, Task},
-    crate::engine::core::GlobalNS,
-    parking_lot::RwLock,
+    crate::engine::{
+        core::GlobalNS,
+        storage::v1::{
+            header_meta::HostRunMode,
+            memfs::{NullFS, VirtualFS},
+            RawFSInterface,
+        },
+        txn::gns::GNSTransactionDriverAnyFS,
+    },
+    parking_lot::{Mutex, RwLock},
 };
 
 /// A `test` mode global implementation
-pub struct TestGlobal {
+pub struct TestGlobal<Fs: RawFSInterface = VirtualFS> {
     gns: GlobalNS,
     hp_queue: RwLock<Vec<Task<CriticalTask>>>,
     lp_queue: RwLock<Vec<Task<GenericTask>>>,
     max_delta_size: usize,
+    txn_driver: Mutex<GNSTransactionDriverAnyFS<Fs>>,
 }
 
-impl TestGlobal {
-    pub fn empty() -> Self {
-        Self::with_max_delta_size(0)
-    }
-    pub fn with_max_delta_size(max_delta_size: usize) -> Self {
+impl<Fs: RawFSInterface> TestGlobal<Fs> {
+    fn new(
+        gns: GlobalNS,
+        max_delta_size: usize,
+        txn_driver: GNSTransactionDriverAnyFS<Fs>,
+    ) -> Self {
         Self {
-            gns: GlobalNS::empty(),
+            gns,
             hp_queue: RwLock::default(),
             lp_queue: RwLock::default(),
             max_delta_size,
+            txn_driver: Mutex::new(txn_driver),
         }
     }
 }
 
-impl GlobalInstanceLike for TestGlobal {
+impl<Fs: RawFSInterface> TestGlobal<Fs> {
+    pub fn new_with_driver_id(log_name: &str) -> Self {
+        let gns = GlobalNS::empty();
+        let driver = GNSTransactionDriverAnyFS::open_or_reinit_with_name(
+            &gns,
+            log_name,
+            0,
+            HostRunMode::Prod,
+            0,
+        )
+        .unwrap();
+        Self::new(gns, 0, driver)
+    }
+}
+
+impl TestGlobal<VirtualFS> {
+    pub fn new_with_vfs_driver(log_name: &str) -> Self {
+        Self::new_with_driver_id(log_name)
+    }
+}
+
+impl TestGlobal<NullFS> {
+    pub fn new_with_nullfs_driver(log_name: &str) -> Self {
+        Self::new_with_driver_id(log_name)
+    }
+    pub fn new_with_tmp_nullfs_driver() -> Self {
+        Self::new_with_nullfs_driver("")
+    }
+}
+
+impl<Fs: RawFSInterface> GlobalInstanceLike for TestGlobal<Fs> {
+    type FileSystem = Fs;
     fn namespace(&self) -> &GlobalNS {
         &self.gns
     }
-    fn post_high_priority_task(&self, task: Task<CriticalTask>) {
+    fn namespace_txn_driver(&self) -> &Mutex<GNSTransactionDriverAnyFS<Self::FileSystem>> {
+        &self.txn_driver
+    }
+    fn taskmgr_post_high_priority(&self, task: Task<CriticalTask>) {
         self.hp_queue.write().push(task)
     }
-    fn post_standard_priority_task(&self, task: Task<GenericTask>) {
+    fn taskmgr_post_standard_priority(&self, task: Task<GenericTask>) {
         self.lp_queue.write().push(task)
     }
     fn get_max_delta_size(&self) -> usize {
         100
     }
 }
+
+impl<Fs: RawFSInterface> Drop for TestGlobal<Fs> {
+    fn drop(&mut self) {
+        let mut txn_driver = self.txn_driver.lock();
+        txn_driver
+            .__journal_mut()
+            .__append_journal_close_and_close()
+            .unwrap();
+    }
+}
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index 94d1b2d1..7972d2dd 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -91,7 +91,7 @@ impl<E: TreeElement, C: Config> MTIndex<E, E::Key, E::Value> for Raw<E, C> {
         self.len()
     }
     fn mt_clear(&self, g: &Guard) {
-        self.nontransactional_clear(g)
+        self.transactional_clear(g)
     }
 
     fn mt_insert(&self, e: E, g: &Guard) -> bool
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index 0c49b6c3..0a75cbe1 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -236,7 +236,7 @@ impl<T: TreeElement, C: Config> RawTree<T, C> {
 }
 
 impl<T: TreeElement, C: Config> RawTree<T, C> {
-    fn nontransactional_clear(&self, g: &Guard) {
+    fn transactional_clear(&self, g: &Guard) {
         self.iter_key(g).for_each(|k| {
             let _ = self.remove(k, g);
         });
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index 0a39daff..7336b932 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -470,7 +470,7 @@ impl<Fs: RawFSInterface, TA> JournalWriter<Fs, TA> {
             &JournalEntryMetadata::new(id, EventSourceMarker::DRIVER_REOPENED, 0, 0).encoded(),
         )
     }
-    pub fn append_journal_close_and_close(mut self) -> SDSSResult<()> {
+    pub fn __append_journal_close_and_close(&mut self) -> SDSSResult<()> {
         self.closed = true;
         let id = self._incr_id() as u128;
         self.log_file.fsynced_write(
@@ -478,6 +478,9 @@ impl<Fs: RawFSInterface, TA> JournalWriter<Fs, TA> {
         )?;
         Ok(())
     }
+    pub fn append_journal_close_and_close(mut self) -> SDSSResult<()> {
+        self.__append_journal_close_and_close()
+    }
 }
 
 impl<Fs: RawFSInterface, TA> JournalWriter<Fs, TA> {
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index 73938417..e7b7e5ad 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -66,44 +66,14 @@ pub type GNSTransactionDriverVFS = GNSTransactionDriverAnyFS<VirtualFS>;
 
 const CURRENT_LOG_VERSION: u32 = 0;
 
-pub trait GNSTransactionDriverLLInterface: RawFSInterface {
-    /// If true, this is an actual txn driver with a non-null (not `/dev/null` like) journal
-    const NONNULL: bool = <Self as RawFSInterface>::NOT_NULL;
-}
-impl<T: RawFSInterface> GNSTransactionDriverLLInterface for T {}
-
 /// The GNS transaction driver is used to handle DDL transactions
 pub struct GNSTransactionDriverAnyFS<F: RawFSInterface = LocalFS> {
     journal: JournalWriter<F, GNSAdapter>,
 }
 
-impl GNSTransactionDriverAnyFS<crate::engine::storage::v1::NullFS> {
-    pub fn nullzero(gns: &GlobalNS) -> Self {
-        let journal = v1::open_journal(
-            "gns.db-tlog",
-            header_meta::FileSpecifier::GNSTxnLog,
-            header_meta::FileSpecifierVersion::__new(CURRENT_LOG_VERSION),
-            0,
-            header_meta::HostRunMode::Dev,
-            0,
-            gns,
-        )
-        .unwrap();
-        Self { journal }
-    }
-    pub fn nullzero_create_exec<T>(gns: &GlobalNS, f: impl FnOnce(&mut Self) -> T) -> T {
-        let mut j = Self::nullzero(gns);
-        let r = f(&mut j);
-        j.close().unwrap();
-        r
-    }
-}
-
-impl<F: GNSTransactionDriverLLInterface> GNSTransactionDriverAnyFS<F> {
-    pub fn close(self) -> TransactionResult<()> {
-        self.journal
-            .append_journal_close_and_close()
-            .map_err(|e| e.into())
+impl<Fs: RawFSInterface> GNSTransactionDriverAnyFS<Fs> {
+    pub fn __journal_mut(&mut self) -> &mut JournalWriter<Fs, GNSAdapter> {
+        &mut self.journal
     }
     pub fn open_or_reinit_with_name(
         gns: &GlobalNS,
@@ -141,7 +111,7 @@ impl<F: GNSTransactionDriverLLInterface> GNSTransactionDriverAnyFS<F> {
 
 /// the journal adapter for DDL queries on the GNS
 #[derive(Debug)]
-struct GNSAdapter;
+pub struct GNSAdapter;
 
 impl JournalAdapter for GNSAdapter {
     const RECOVERY_PLUGIN: bool = true;
diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
index 71d12ead..1199d4d4 100644
--- a/server/src/engine/txn/gns/tests/full_chain.rs
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -38,8 +38,6 @@ use crate::engine::{
         ddl::crt::{CreateModel, CreateSpace},
         tests::lex_insecure,
     },
-    storage::v1::header_meta::HostRunMode,
-    txn::gns::GNSTransactionDriverVFS,
 };
 
 fn multirun(f: impl FnOnce() + Copy) {
@@ -52,28 +50,12 @@ fn with_variable<T>(var: T, f: impl FnOnce(T)) {
     f(var);
 }
 
-fn init_txn_driver(global: &impl GlobalInstanceLike, log_name: &str) -> GNSTransactionDriverVFS {
-    GNSTransactionDriverVFS::open_or_reinit_with_name(
-        global.namespace(),
-        log_name,
-        0,
-        HostRunMode::Prod,
-        0,
-    )
-    .unwrap()
-}
-
-fn init_space(
-    global: &impl GlobalInstanceLike,
-    driver: &mut GNSTransactionDriverVFS,
-    space_name: &str,
-    env: &str,
-) -> Uuid {
+fn init_space(global: &impl GlobalInstanceLike, space_name: &str, env: &str) -> Uuid {
     let query = format!("create space {space_name} with {{ env: {env} }}");
     let stmt = lex_insecure(query.as_bytes()).unwrap();
     let stmt = parse_ast_node_full::<CreateSpace>(&stmt[2..]).unwrap();
     let name = stmt.space_name;
-    Space::transactional_exec_create(global, driver, stmt).unwrap();
+    Space::transactional_exec_create(global, stmt).unwrap();
     global
         .namespace()
         .spaces()
@@ -89,14 +71,11 @@ fn create_space() {
         let uuid;
         // start 1
         {
-            let global = TestGlobal::empty();
-            let mut driver = init_txn_driver(&global, log_name);
-            uuid = init_space(&global, &mut driver, "myspace", "{ SAYAN_MAX: 65536 }"); // good lord that doesn't sound like a good variable
-            driver.close().unwrap();
+            let global = TestGlobal::new_with_vfs_driver(log_name);
+            uuid = init_space(&global, "myspace", "{ SAYAN_MAX: 65536 }"); // good lord that doesn't sound like a good variable
         }
         multirun(|| {
-            let global = TestGlobal::empty();
-            let driver = init_txn_driver(&global, log_name);
+            let global = TestGlobal::new_with_vfs_driver(log_name);
             assert_eq!(
                 global.namespace().spaces().read().get("myspace").unwrap(),
                 &Space::new_restore_empty(
@@ -106,7 +85,6 @@ fn create_space() {
                     uuid
                 )
             );
-            driver.close().unwrap();
         })
     })
 }
@@ -116,19 +94,16 @@ fn alter_space() {
     with_variable("alter_space_test.global.db-tlog", |log_name| {
         let uuid;
         {
-            let global = TestGlobal::empty();
-            let mut driver = init_txn_driver(&global, log_name);
-            uuid = init_space(&global, &mut driver, "myspace", "{}");
+            let global = TestGlobal::new_with_vfs_driver(log_name);
+            uuid = init_space(&global, "myspace", "{}");
             let stmt =
                 lex_insecure("alter space myspace with { env: { SAYAN_MAX: 65536 } }".as_bytes())
                     .unwrap();
             let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
-            Space::transactional_exec_alter(&global, &mut driver, stmt).unwrap();
-            driver.close().unwrap();
+            Space::transactional_exec_alter(&global, stmt).unwrap();
         }
         multirun(|| {
-            let global = TestGlobal::empty();
-            let driver = init_txn_driver(&global, log_name);
+            let global = TestGlobal::new_with_vfs_driver(log_name);
             assert_eq!(
                 global.namespace().spaces().read().get("myspace").unwrap(),
                 &Space::new_restore_empty(
@@ -138,7 +113,6 @@ fn alter_space() {
                     uuid
                 )
             );
-            driver.close().unwrap();
         })
     })
 }
@@ -147,26 +121,21 @@ fn alter_space() {
 fn drop_space() {
     with_variable("drop_space_test.global.db-tlog", |log_name| {
         {
-            let global = TestGlobal::empty();
-            let mut driver = init_txn_driver(&global, log_name);
-            let _ = init_space(&global, &mut driver, "myspace", "{}");
+            let global = TestGlobal::new_with_vfs_driver(log_name);
+            let _ = init_space(&global, "myspace", "{}");
             let stmt = lex_insecure("drop space myspace".as_bytes()).unwrap();
             let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
-            Space::transactional_exec_drop(&global, &mut driver, stmt).unwrap();
-            driver.close().unwrap();
+            Space::transactional_exec_drop(&global, stmt).unwrap();
         }
         multirun(|| {
-            let global = TestGlobal::empty();
-            let driver = init_txn_driver(&global, log_name);
+            let global = TestGlobal::new_with_vfs_driver(log_name);
             assert_eq!(global.namespace().spaces().read().get("myspace"), None);
-            driver.close().unwrap();
         })
     })
 }
 
 fn init_model(
     global: &impl GlobalInstanceLike,
-    txn_driver: &mut GNSTransactionDriverVFS,
     space_name: &str,
     model_name: &str,
     decl: &str,
@@ -175,20 +144,16 @@ fn init_model(
     let stmt = lex_insecure(query.as_bytes()).unwrap();
     let stmt = parse_ast_node_full::<CreateModel>(&stmt[2..]).unwrap();
     let model_name = stmt.model_name;
-    Model::transactional_exec_create(global, txn_driver, stmt).unwrap();
+    Model::transactional_exec_create(global, stmt).unwrap();
     global
         .namespace()
         .with_model(model_name, |model| Ok(model.get_uuid()))
         .unwrap()
 }
 
-fn init_default_model(
-    global: &impl GlobalInstanceLike,
-    driver: &mut GNSTransactionDriverVFS,
-) -> Uuid {
+fn init_default_model(global: &impl GlobalInstanceLike) -> Uuid {
     init_model(
         global,
-        driver,
         "myspace",
         "mymodel",
         "username: string, password: binary",
@@ -201,15 +166,12 @@ fn create_model() {
         let _uuid_space;
         let uuid_model;
         {
-            let global = TestGlobal::empty();
-            let mut driver = init_txn_driver(&global, log_name);
-            _uuid_space = init_space(&global, &mut driver, "myspace", "{}");
-            uuid_model = init_default_model(&global, &mut driver);
-            driver.close().unwrap();
+            let global = TestGlobal::new_with_vfs_driver(log_name);
+            _uuid_space = init_space(&global, "myspace", "{}");
+            uuid_model = init_default_model(&global);
         }
         multirun(|| {
-            let global = TestGlobal::empty();
-            let driver = init_txn_driver(&global, log_name);
+            let global = TestGlobal::new_with_vfs_driver(log_name);
             global
                 .namespace()
                 .with_model(("myspace", "mymodel"), |model| {
@@ -228,7 +190,6 @@ fn create_model() {
                     Ok(())
                 })
                 .unwrap();
-            driver.close().unwrap();
         })
     })
 }
@@ -237,21 +198,18 @@ fn create_model() {
 fn alter_model_add() {
     with_variable("alter_model_add_test.global.db-tlog", |log_name| {
         {
-            let global = TestGlobal::empty();
-            let mut driver = init_txn_driver(&global, log_name);
-            init_space(&global, &mut driver, "myspace", "{}");
-            init_default_model(&global, &mut driver);
+            let global = TestGlobal::new_with_vfs_driver(log_name);
+            init_space(&global, "myspace", "{}");
+            init_default_model(&global);
             let stmt = lex_insecure(
                 b"alter model myspace.mymodel add profile_pic { type: binary, nullable: true }",
             )
             .unwrap();
             let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
-            Model::transactional_exec_alter(&global, &mut driver, stmt).unwrap();
-            driver.close().unwrap();
+            Model::transactional_exec_alter(&global, stmt).unwrap();
         }
         multirun(|| {
-            let global = TestGlobal::empty();
-            let driver = init_txn_driver(&global, log_name);
+            let global = TestGlobal::new_with_vfs_driver(log_name);
             global
                 .namespace()
                 .with_model(("myspace", "mymodel"), |model| {
@@ -266,7 +224,6 @@ fn alter_model_add() {
                     Ok(())
                 })
                 .unwrap();
-            driver.close().unwrap();
         })
     })
 }
@@ -275,12 +232,10 @@ fn alter_model_add() {
 fn alter_model_remove() {
     with_variable("alter_model_remove_test.global.db-tlog", |log_name| {
         {
-            let global = TestGlobal::empty();
-            let mut driver = init_txn_driver(&global, log_name);
-            init_space(&global, &mut driver, "myspace", "{}");
+            let global = TestGlobal::new_with_vfs_driver(log_name);
+            init_space(&global, "myspace", "{}");
             init_model(
                 &global,
-                &mut driver,
                 "myspace",
                 "mymodel",
                 "username: string, password: binary, null profile_pic: binary, null has_2fa: bool, null has_secure_key: bool, is_dumb: bool",
@@ -290,12 +245,10 @@ fn alter_model_remove() {
             )
             .unwrap();
             let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
-            Model::transactional_exec_alter(&global, &mut driver, stmt).unwrap();
-            driver.close().unwrap()
+            Model::transactional_exec_alter(&global, stmt).unwrap();
         }
         multirun(|| {
-            let global = TestGlobal::empty();
-            let driver = init_txn_driver(&global, log_name);
+            let global = TestGlobal::new_with_vfs_driver(log_name);
             global
                 .namespace()
                 .with_model(("myspace", "mymodel"), |model| {
@@ -305,7 +258,6 @@ fn alter_model_remove() {
                     Ok(())
                 })
                 .unwrap();
-            driver.close().unwrap()
         })
     })
 }
@@ -314,12 +266,10 @@ fn alter_model_remove() {
 fn alter_model_update() {
     with_variable("alter_model_update_test.global.db-tlog", |log_name| {
         {
-            let global = TestGlobal::empty();
-            let mut driver = init_txn_driver(&global, log_name);
-            init_space(&global, &mut driver, "myspace", "{}");
+            let global = TestGlobal::new_with_vfs_driver(log_name);
+            init_space(&global, "myspace", "{}");
             init_model(
                 &global,
-                &mut driver,
                 "myspace",
                 "mymodel",
                 "username: string, password: binary, profile_pic: binary",
@@ -328,12 +278,10 @@ fn alter_model_update() {
                 lex_insecure(b"alter model myspace.mymodel update profile_pic { nullable: true }")
                     .unwrap();
             let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
-            Model::transactional_exec_alter(&global, &mut driver, stmt).unwrap();
-            driver.close().unwrap();
+            Model::transactional_exec_alter(&global, stmt).unwrap();
         }
         multirun(|| {
-            let global = TestGlobal::empty();
-            let driver = init_txn_driver(&global, log_name);
+            let global = TestGlobal::new_with_vfs_driver(log_name);
             global
                 .namespace()
                 .with_model(("myspace", "mymodel"), |model| {
@@ -348,7 +296,6 @@ fn alter_model_update() {
                     Ok(())
                 })
                 .unwrap();
-            driver.close().unwrap();
         })
     })
 }
@@ -357,18 +304,15 @@ fn alter_model_update() {
 fn drop_model() {
     with_variable("drop_model_test.global.db-tlog", |log_name| {
         {
-            let global = TestGlobal::empty();
-            let mut driver = init_txn_driver(&global, log_name);
-            init_space(&global, &mut driver, "myspace", "{}");
-            init_default_model(&global, &mut driver);
+            let global = TestGlobal::new_with_vfs_driver(log_name);
+            init_space(&global, "myspace", "{}");
+            init_default_model(&global);
             let stmt = lex_insecure(b"drop model myspace.mymodel").unwrap();
             let stmt = parse_ast_node_full(&stmt[2..]).unwrap();
-            Model::transactional_exec_drop(&global, &mut driver, stmt).unwrap();
-            driver.close().unwrap();
+            Model::transactional_exec_drop(&global, stmt).unwrap();
         }
         multirun(|| {
-            let global = TestGlobal::empty();
-            let driver = init_txn_driver(&global, log_name);
+            let global = TestGlobal::new_with_vfs_driver(log_name);
             assert_eq!(
                 global
                     .namespace()
@@ -376,7 +320,6 @@ fn drop_model() {
                     .unwrap_err(),
                 DatabaseError::DdlModelNotFound
             );
-            driver.close().unwrap();
         })
     })
 }

From 9ceaa54abdc4a022f46271cb27769ac8c93ce046 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 14 Sep 2023 16:01:54 +0000
Subject: [PATCH 251/310] Simplify error propagation

---
 server/src/engine/core/dml/del.rs             |   6 +-
 server/src/engine/core/dml/ins.rs             |  10 +-
 server/src/engine/core/dml/mod.rs             |   6 +-
 server/src/engine/core/dml/sel.rs             |   8 +-
 server/src/engine/core/dml/upd.rs             |  14 +-
 server/src/engine/core/mod.rs                 |  16 +-
 server/src/engine/core/model/alt.rs           |  32 +--
 server/src/engine/core/model/mod.rs           |  22 +-
 server/src/engine/core/space.rs               |  33 +--
 server/src/engine/core/tests/ddl_model/alt.rs |  26 +--
 server/src/engine/core/tests/ddl_model/crt.rs |  12 +-
 .../src/engine/core/tests/ddl_model/layer.rs  |  12 +-
 server/src/engine/core/tests/ddl_model/mod.rs |  10 +-
 .../src/engine/core/tests/ddl_space/alter.rs  |   4 +-
 .../src/engine/core/tests/ddl_space/create.rs |   6 +-
 server/src/engine/core/tests/ddl_space/mod.rs |   8 +-
 server/src/engine/core/tests/dml/delete.rs    |   4 +-
 server/src/engine/core/tests/dml/insert.rs    |   4 +-
 server/src/engine/core/tests/dml/mod.rs       |  39 ++--
 server/src/engine/core/tests/dml/select.rs    |   4 +-
 server/src/engine/core/tests/dml/update.rs    |   8 +-
 server/src/engine/core/util.rs                |  10 +-
 server/src/engine/error.rs                    | 190 +++++++-----------
 server/src/engine/fractal/mgr.rs              |   2 +-
 server/src/engine/macros.rs                   |   3 +
 server/src/engine/ql/ast/mod.rs               |  22 +-
 server/src/engine/ql/ast/traits.rs            |  24 +--
 server/src/engine/ql/ddl/alt.rs               |  34 ++--
 server/src/engine/ql/ddl/crt.rs               |  20 +-
 server/src/engine/ql/ddl/drop.rs              |  22 +-
 server/src/engine/ql/ddl/ins.rs               |  12 +-
 server/src/engine/ql/ddl/syn.rs               |  46 +++--
 server/src/engine/ql/dml/del.rs               |  12 +-
 server/src/engine/ql/dml/ins.rs               |  20 +-
 server/src/engine/ql/dml/mod.rs               |   8 +-
 server/src/engine/ql/dml/sel.rs               |  14 +-
 server/src/engine/ql/dml/upd.rs               |  18 +-
 server/src/engine/ql/lex/mod.rs               |  24 +--
 server/src/engine/ql/lex/raw.rs               |   8 +-
 server/src/engine/ql/tests.rs                 |   6 +-
 server/src/engine/ql/tests/lexer_tests.rs     |  17 +-
 server/src/engine/txn/gns/tests/full_chain.rs |   4 +-
 42 files changed, 367 insertions(+), 433 deletions(-)

diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs
index 41ab2b64..8256b9f3 100644
--- a/server/src/engine/core/dml/del.rs
+++ b/server/src/engine/core/dml/del.rs
@@ -26,14 +26,14 @@
 
 use crate::engine::{
     core::{self, model::delta::DataDeltaKind},
-    error::{DatabaseError, DatabaseResult},
+    error::{Error, QueryResult},
     fractal::GlobalInstanceLike,
     idx::MTIndex,
     ql::dml::del::DeleteStatement,
     sync,
 };
 
-pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) -> DatabaseResult<()> {
+pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) -> QueryResult<()> {
     core::with_model_for_data_update(global, delete.entity(), |model| {
         let g = sync::atm::cpin();
         let schema_version = model.delta_state().schema_current_version();
@@ -55,7 +55,7 @@ pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) ->
                 );
                 Ok(())
             }
-            None => Err(DatabaseError::DmlEntryNotFound),
+            None => Err(Error::QPDmlRowNotFound),
         }
     })
 }
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index d1e10610..d64efef6 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -30,14 +30,14 @@ use crate::engine::{
         index::{DcFieldIndex, PrimaryIndexKey, Row},
         model::{delta::DataDeltaKind, Fields, Model},
     },
-    error::{DatabaseError, DatabaseResult},
+    error::{Error, QueryResult},
     fractal::GlobalInstanceLike,
     idx::{IndexBaseSpec, MTIndex, STIndex, STIndexSeq},
     ql::dml::ins::{InsertData, InsertStatement},
     sync::atm::cpin,
 };
 
-pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> DatabaseResult<()> {
+pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> QueryResult<()> {
     core::with_model_for_data_update(global, insert.entity(), |mdl| {
         let irmwd = mdl.intent_write_new_data();
         let (pk, data) = prepare_insert(mdl, irmwd.fields(), insert.data())?;
@@ -57,7 +57,7 @@ pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> Data
             );
             Ok(())
         } else {
-            Err(DatabaseError::DmlConstraintViolationDuplicate)
+            Err(Error::QPDmlDuplicate)
         }
     })
 }
@@ -67,7 +67,7 @@ fn prepare_insert(
     model: &Model,
     fields: &Fields,
     insert: InsertData,
-) -> DatabaseResult<(PrimaryIndexKey, DcFieldIndex)> {
+) -> QueryResult<(PrimaryIndexKey, DcFieldIndex)> {
     let mut okay = fields.len() == insert.column_count();
     let mut prepared_data = DcFieldIndex::idx_init_cap(fields.len());
     match insert {
@@ -113,6 +113,6 @@ fn prepare_insert(
         };
         Ok((primary_key, prepared_data))
     } else {
-        Err(DatabaseError::DmlDataValidationError)
+        Err(Error::QPDmlValidationError)
     }
 }
diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index 65f0f384..bb65edbf 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -33,7 +33,7 @@ use crate::{
     engine::{
         core::model::Model,
         data::{lit::LitIR, spec::DataspecMeta1D, tag::DataTag},
-        error::{DatabaseError, DatabaseResult},
+        error::{Error, QueryResult},
         ql::dml::WhereClause,
     },
     util::compiler,
@@ -47,7 +47,7 @@ impl Model {
     pub(self) fn resolve_where<'a>(
         &self,
         where_clause: &mut WhereClause<'a>,
-    ) -> DatabaseResult<LitIR<'a>> {
+    ) -> QueryResult<LitIR<'a>> {
         match where_clause.clauses_mut().remove(self.p_key().as_bytes()) {
             Some(clause)
                 if clause.filter_hint_none()
@@ -55,7 +55,7 @@ impl Model {
             {
                 Ok(clause.rhs())
             }
-            _ => compiler::cold_rerr(DatabaseError::DmlWhereClauseUnindexedExpr),
+            _ => compiler::cold_rerr(Error::QPDmlWhereHasUnindexedColumn),
         }
     }
 }
diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
index 12a7e894..a7af4fdc 100644
--- a/server/src/engine/core/dml/sel.rs
+++ b/server/src/engine/core/dml/sel.rs
@@ -27,7 +27,7 @@
 use crate::engine::{
     core::index::DcFieldIndex,
     data::cell::{Datacell, VirtualDatacell},
-    error::{DatabaseError, DatabaseResult},
+    error::{Error, QueryResult},
     fractal::GlobalInstanceLike,
     idx::{STIndex, STIndexSeq},
     ql::dml::sel::SelectStatement,
@@ -38,7 +38,7 @@ pub fn select_custom<F>(
     global: &impl GlobalInstanceLike,
     mut select: SelectStatement,
     mut cellfn: F,
-) -> DatabaseResult<()>
+) -> QueryResult<()>
 where
     F: FnMut(&Datacell),
 {
@@ -51,7 +51,7 @@ where
             match fields.st_get(key) {
                 Some(dc) => cellfn(dc),
                 None if key == mdl.p_key() => cellfn(&pkdc),
-                None => return Err(DatabaseError::FieldNotFound),
+                None => return Err(Error::QPUnknownField),
             }
             Ok(())
         };
@@ -68,7 +68,7 @@ where
                     }
                 }
             }
-            None => return Err(DatabaseError::DmlEntryNotFound),
+            None => return Err(Error::QPDmlRowNotFound),
         }
         Ok(())
     })
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index 71908643..cecd074c 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -37,7 +37,7 @@ use {
                 spec::{Dataspec1D, DataspecMeta1D},
                 tag::{DataTag, TagClass},
             },
-            error::{DatabaseError, DatabaseResult},
+            error::{Error, QueryResult},
             fractal::GlobalInstanceLike,
             idx::STIndex,
             ql::dml::upd::{AssignmentExpression, UpdateStatement},
@@ -233,7 +233,7 @@ pub fn collect_trace_path() -> Vec<&'static str> {
     ROUTE_TRACE.with(|v| v.borrow().iter().cloned().collect())
 }
 
-pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) -> DatabaseResult<()> {
+pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) -> QueryResult<()> {
     core::with_model_for_data_update(global, update.entity(), |mdl| {
         let mut ret = Ok(());
         // prepare row fetch
@@ -243,7 +243,7 @@ pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) ->
         // fetch row
         let g = sync::atm::cpin();
         let Some(row) = mdl.primary_index().select(key, &g) else {
-            return Err(DatabaseError::DmlEntryNotFound);
+            return Err(Error::QPDmlRowNotFound);
         };
         // lock row
         let mut row_data_wl = row.d_data().write();
@@ -280,7 +280,7 @@ pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) ->
                 _ => {
                     input_trace("fieldnotfound");
                     rollback_now = true;
-                    ret = Err(DatabaseError::FieldNotFound);
+                    ret = Err(Error::QPUnknownField);
                     break;
                 }
             }
@@ -314,20 +314,20 @@ pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) ->
                                 list.push(rhs.into());
                             } else {
                                 rollback_now = true;
-                                ret = Err(DatabaseError::ServerError);
+                                ret = Err(Error::SysOutOfMemory);
                                 break;
                             }
                         }
                     } else {
                         input_trace("list;badtag");
                         rollback_now = true;
-                        ret = Err(DatabaseError::DmlConstraintViolationFieldTypedef);
+                        ret = Err(Error::QPDmlValidationError);
                         break;
                     }
                 }
                 _ => {
                     input_trace("unknown_reason;exitmainloop");
-                    ret = Err(DatabaseError::DmlConstraintViolationFieldTypedef);
+                    ret = Err(Error::QPDmlValidationError);
                     rollback_now = true;
                     break;
                 }
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 05e7ca70..ddcb599e 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -38,7 +38,7 @@ use {
     self::{model::Model, util::EntityLocator},
     crate::engine::{
         core::space::Space,
-        error::{DatabaseError, DatabaseResult},
+        error::{Error, QueryResult},
         fractal::GlobalInstanceLike,
         idx::{IndexST, STIndex},
     },
@@ -60,9 +60,9 @@ pub(self) fn with_model_for_data_update<'a, T, E, F>(
     global: &impl GlobalInstanceLike,
     entity: E,
     f: F,
-) -> DatabaseResult<T>
+) -> QueryResult<T>
 where
-    F: FnOnce(&Model) -> DatabaseResult<T>,
+    F: FnOnce(&Model) -> QueryResult<T>,
     E: 'a + EntityLocator<'a>,
 {
     let (space_name, model_name) = entity.parse_entity()?;
@@ -96,17 +96,17 @@ impl GlobalNS {
     pub fn with_space<T>(
         &self,
         space: &str,
-        f: impl FnOnce(&Space) -> DatabaseResult<T>,
-    ) -> DatabaseResult<T> {
+        f: impl FnOnce(&Space) -> QueryResult<T>,
+    ) -> QueryResult<T> {
         let sread = self.index_space.read();
         let Some(space) = sread.st_get(space) else {
-            return Err(DatabaseError::DdlSpaceNotFound);
+            return Err(Error::QPObjectNotFound);
         };
         f(space)
     }
-    pub fn with_model<'a, T, E, F>(&self, entity: E, f: F) -> DatabaseResult<T>
+    pub fn with_model<'a, T, E, F>(&self, entity: E, f: F) -> QueryResult<T>
     where
-        F: FnOnce(&Model) -> DatabaseResult<T>,
+        F: FnOnce(&Model) -> QueryResult<T>,
         E: 'a + EntityLocator<'a>,
     {
         entity
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index 659c126a..d98f77cb 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -33,7 +33,7 @@ use {
                 tag::{DataTag, TagClass},
                 DictEntryGeneric,
             },
-            error::{DatabaseError, DatabaseResult},
+            error::{Error, QueryResult},
             fractal::GlobalInstanceLike,
             idx::{IndexST, IndexSTSeqCns, STIndex, STIndexSeq},
             ql::{
@@ -81,10 +81,10 @@ fn no_field(mr: &IWModel, new: &str) -> bool {
     !mr.fields().st_contains(new)
 }
 
-fn check_nullable(props: &mut HashMap<Box<str>, DictEntryGeneric>) -> DatabaseResult<bool> {
+fn check_nullable(props: &mut HashMap<Box<str>, DictEntryGeneric>) -> QueryResult<bool> {
     match props.remove("nullable") {
         Some(DictEntryGeneric::Data(b)) if b.kind() == TagClass::Bool => Ok(b.bool()),
-        Some(_) => Err(DatabaseError::DdlModelAlterBadProperty),
+        Some(_) => Err(Error::QPDdlInvalidProperties),
         None => Ok(false),
     }
 }
@@ -94,14 +94,14 @@ impl<'a> AlterPlan<'a> {
         mv: &Model,
         wm: &IWModel,
         AlterModel { model, kind }: AlterModel<'a>,
-    ) -> DatabaseResult<AlterPlan<'a>> {
+    ) -> QueryResult<AlterPlan<'a>> {
         let mut no_lock = true;
         let mut okay = true;
         let action = match kind {
             AlterKind::Remove(r) => {
                 let mut x = HashSet::new();
                 if !r.iter().all(|id| x.insert(id.as_str())) {
-                    return Err(DatabaseError::DdlModelAlterBad);
+                    return Err(Error::QPDdlModelAlterIllegal);
                 }
                 let mut not_found = false;
                 if r.iter().all(|id| {
@@ -112,9 +112,9 @@ impl<'a> AlterPlan<'a> {
                 }) {
                     can_ignore!(AlterAction::Remove(r))
                 } else if not_found {
-                    return Err(DatabaseError::FieldNotFound);
+                    return Err(Error::QPUnknownField);
                 } else {
-                    return Err(DatabaseError::DdlModelAlterProtectedField);
+                    return Err(Error::QPDdlModelAlterIllegal);
                 }
             }
             AlterKind::Add(new_fields) => {
@@ -148,7 +148,7 @@ impl<'a> AlterPlan<'a> {
                     mv.guard_pk(&field_name)?;
                     // get the current field
                     let Some(current_field) = wm.fields().st_get(field_name.as_str()) else {
-                        return Err(DatabaseError::FieldNotFound);
+                        return Err(Error::QPUnknownField);
                     };
                     // check props
                     let is_nullable = check_nullable(&mut props)?;
@@ -174,7 +174,7 @@ impl<'a> AlterPlan<'a> {
                 no_lock,
             })
         } else {
-            Err(DatabaseError::DdlModelAlterBad)
+            Err(Error::QPDdlModelAlterIllegal)
         }
     }
     fn ldeltas(
@@ -183,7 +183,7 @@ impl<'a> AlterPlan<'a> {
         nullable: bool,
         super_nlck: &mut bool,
         super_okay: &mut bool,
-    ) -> DatabaseResult<(bool, Field)> {
+    ) -> QueryResult<(bool, Field)> {
         #[inline(always)]
         fn classeq(current: &Layer, new: &Layer, class: TagClass) -> bool {
             // KIDDOS, LEARN SOME RELATIONS BEFORE WRITING CODE
@@ -197,7 +197,7 @@ impl<'a> AlterPlan<'a> {
         }
         if layers.len() > current.layers().len() {
             // simply a dumb tomato; ELIMINATE THESE DUMB TOMATOES
-            return Err(DatabaseError::DdlModelAlterBad);
+            return Err(Error::QPDdlModelAlterIllegal);
         }
         let mut no_lock = !(current.is_nullable() & !nullable);
         let mut deltasize = (current.is_nullable() ^ nullable) as usize;
@@ -216,7 +216,7 @@ impl<'a> AlterPlan<'a> {
             // actually parse the new layer
             okay &= props.is_empty();
             let Some(new_parsed_layer) = Layer::get_layer(&ty) else {
-                return Err(DatabaseError::DdlModelAlterBadTypedef);
+                return Err(Error::QPDdlInvalidTypeDefinition);
             };
             match (
                 current_layer.tag.tag_selector(),
@@ -233,7 +233,7 @@ impl<'a> AlterPlan<'a> {
                 }
                 _ => {
                     // can't cast this directly
-                    return Err(DatabaseError::DdlModelAlterBadTypedef);
+                    return Err(Error::QPDdlInvalidTypeDefinition);
                 }
             }
             *new_layer = new_parsed_layer;
@@ -243,7 +243,7 @@ impl<'a> AlterPlan<'a> {
         if okay {
             Ok((deltasize != 0, new_field))
         } else {
-            Err(DatabaseError::DdlModelAlterBad)
+            Err(Error::QPDdlModelAlterIllegal)
         }
     }
 }
@@ -252,7 +252,7 @@ impl Model {
     pub fn transactional_exec_alter<G: GlobalInstanceLike>(
         global: &G,
         alter: AlterModel,
-    ) -> DatabaseResult<()> {
+    ) -> QueryResult<()> {
         let (space_name, model_name) = EntityLocator::parse_entity(alter.model)?;
         global.namespace().with_space(space_name, |space| {
             space.with_model(model_name, |model| {
@@ -263,7 +263,7 @@ impl Model {
                 // we have a legal plan; acquire exclusive if we need it
                 if !plan.no_lock {
                     // TODO(@ohsayan): allow this later on, once we define the syntax
-                    return Err(DatabaseError::NeedLock);
+                    return Err(Error::QPNeedLock);
                 }
                 // fine, we're good
                 let mut iwm = iwm;
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 3926f8fe..e1ed2dd0 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -39,7 +39,7 @@ use {
             tag::{DataTag, FullTag, TagClass, TagSelector},
             uuid::Uuid,
         },
-        error::{DatabaseError, DatabaseResult},
+        error::{Error, QueryResult},
         fractal::GlobalInstanceLike,
         idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
         mem::VInline,
@@ -132,9 +132,9 @@ impl Model {
     fn not_pk(&self, new: &str) -> bool {
         !self.is_pk(new)
     }
-    fn guard_pk(&self, new: &str) -> DatabaseResult<()> {
+    fn guard_pk(&self, new: &str) -> QueryResult<()> {
         if self.is_pk(new) {
-            Err(DatabaseError::DdlModelAlterProtectedField)
+            Err(Error::QPDdlModelAlterIllegal)
         } else {
             Ok(())
         }
@@ -169,7 +169,7 @@ impl Model {
             fields,
             props,
         }: CreateModel,
-    ) -> DatabaseResult<Self> {
+    ) -> QueryResult<Self> {
         let mut okay = props.is_empty() & !fields.is_empty();
         // validate fields
         let mut field_spec = fields.into_iter();
@@ -199,7 +199,7 @@ impl Model {
                 return Ok(Self::new_restore(Uuid::new(), last_pk.into(), tag, fields));
             }
         }
-        Err(DatabaseError::DdlModelBadDefinition)
+        Err(Error::QPDdlModelBadDefinition)
     }
 }
 
@@ -207,13 +207,13 @@ impl Model {
     pub fn transactional_exec_create<G: GlobalInstanceLike>(
         global: &G,
         stmt: CreateModel,
-    ) -> DatabaseResult<()> {
+    ) -> QueryResult<()> {
         let (space_name, model_name) = stmt.model_name.parse_entity()?;
         let model = Self::process_create(stmt)?;
         global.namespace().with_space(space_name, |space| {
             let mut w_space = space.models().write();
             if w_space.st_contains(model_name) {
-                return Err(DatabaseError::DdlModelAlreadyExists);
+                return Err(Error::QPDdlObjectAlreadyExists);
             }
             if G::FS_IS_NON_NULL {
                 // prepare txn
@@ -236,12 +236,12 @@ impl Model {
     pub fn transactional_exec_drop<G: GlobalInstanceLike>(
         global: &G,
         stmt: DropModel,
-    ) -> DatabaseResult<()> {
+    ) -> QueryResult<()> {
         let (space_name, model_name) = stmt.entity.parse_entity()?;
         global.namespace().with_space(space_name, |space| {
             let mut w_space = space.models().write();
             let Some(model) = w_space.get(model_name) else {
-                return Err(DatabaseError::DdlModelNotFound);
+                return Err(Error::QPObjectNotFound);
             };
             if G::FS_IS_NON_NULL {
                 // prepare txn
@@ -310,7 +310,7 @@ impl Field {
     pub fn layers(&self) -> &[Layer] {
         &self.layers
     }
-    pub fn parse_layers(spec: Vec<LayerSpec>, nullable: bool) -> DatabaseResult<Self> {
+    pub fn parse_layers(spec: Vec<LayerSpec>, nullable: bool) -> QueryResult<Self> {
         let mut layers = spec.into_iter().rev();
         let mut okay = true;
         let mut fin = false;
@@ -333,7 +333,7 @@ impl Field {
                 nullable,
             })
         } else {
-            Err(DatabaseError::DdlModelInvalidTypeDefinition)
+            Err(Error::QPDdlInvalidTypeDefinition)
         }
     }
     #[inline(always)]
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 74d2d068..ec07bf43 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -28,7 +28,7 @@ use {
     crate::engine::{
         core::{model::Model, RWLIdx},
         data::{dict, uuid::Uuid, DictEntryGeneric, DictGeneric},
-        error::{DatabaseError, DatabaseResult},
+        error::{Error, QueryResult},
         fractal::GlobalInstanceLike,
         idx::{IndexST, STIndex},
         ql::ddl::{alt::AlterSpace, crt::CreateSpace, drop::DropSpace},
@@ -99,7 +99,7 @@ impl ProcedureCreate {
 }
 
 impl Space {
-    pub fn _create_model(&self, name: &str, model: Model) -> DatabaseResult<()> {
+    pub fn _create_model(&self, name: &str, model: Model) -> QueryResult<()> {
         if self
             .mns
             .write()
@@ -107,7 +107,7 @@ impl Space {
         {
             Ok(())
         } else {
-            Err(DatabaseError::DdlModelAlreadyExists)
+            Err(Error::QPDdlObjectAlreadyExists)
         }
     }
     pub fn get_uuid(&self) -> Uuid {
@@ -122,11 +122,11 @@ impl Space {
     pub fn with_model<T>(
         &self,
         model: &str,
-        f: impl FnOnce(&Model) -> DatabaseResult<T>,
-    ) -> DatabaseResult<T> {
+        f: impl FnOnce(&Model) -> QueryResult<T>,
+    ) -> QueryResult<T> {
         let mread = self.mns.read();
         let Some(model) = mread.st_get(model) else {
-            return Err(DatabaseError::DdlModelNotFound);
+            return Err(Error::QPObjectNotFound);
         };
         f(model)
     }
@@ -164,7 +164,7 @@ impl Space {
             space_name,
             mut props,
         }: CreateSpace,
-    ) -> DatabaseResult<ProcedureCreate> {
+    ) -> QueryResult<ProcedureCreate> {
         let space_name = space_name.to_string().into_boxed_str();
         // check env
         let env = match props.remove(SpaceMeta::KEY_ENV) {
@@ -172,7 +172,8 @@ impl Space {
             Some(DictEntryGeneric::Data(l)) if l.is_null() => IndexST::default(),
             None if props.is_empty() => IndexST::default(),
             _ => {
-                return Err(DatabaseError::DdlSpaceBadProperty);
+                // unknown properties
+                return Err(Error::QPDdlInvalidProperties);
             }
         };
         Ok(ProcedureCreate {
@@ -192,13 +193,13 @@ impl Space {
     pub fn transactional_exec_create<G: GlobalInstanceLike>(
         global: &G,
         space: CreateSpace,
-    ) -> DatabaseResult<()> {
+    ) -> QueryResult<()> {
         // process create
         let ProcedureCreate { space_name, space } = Self::process_create(space)?;
         // acquire access
         let mut wl = global.namespace().spaces().write();
         if wl.st_contains(&space_name) {
-            return Err(DatabaseError::DdlSpaceAlreadyExists);
+            return Err(Error::QPDdlObjectAlreadyExists);
         }
         // commit txn
         if G::FS_IS_NON_NULL {
@@ -219,19 +220,19 @@ impl Space {
             space_name,
             updated_props,
         }: AlterSpace,
-    ) -> DatabaseResult<()> {
+    ) -> QueryResult<()> {
         global.namespace().with_space(&space_name, |space| {
             match updated_props.get(SpaceMeta::KEY_ENV) {
                 Some(DictEntryGeneric::Map(_)) if updated_props.len() == 1 => {}
                 Some(DictEntryGeneric::Data(l)) if updated_props.len() == 1 && l.is_null() => {}
                 None if updated_props.is_empty() => return Ok(()),
-                _ => return Err(DatabaseError::DdlSpaceBadProperty),
+                _ => return Err(Error::QPDdlInvalidProperties),
             }
             let mut space_props = space.meta.dict().write();
             // create patch
             let patch = match dict::rprepare_metadata_patch(&space_props, updated_props) {
                 Some(patch) => patch,
-                None => return Err(DatabaseError::DdlSpaceBadProperty),
+                None => return Err(Error::QPDdlInvalidProperties),
             };
             if G::FS_IS_NON_NULL {
                 // prepare txn
@@ -254,18 +255,18 @@ impl Space {
     pub fn transactional_exec_drop<G: GlobalInstanceLike>(
         global: &G,
         DropSpace { space, force: _ }: DropSpace,
-    ) -> DatabaseResult<()> {
+    ) -> QueryResult<()> {
         // TODO(@ohsayan): force remove option
         // TODO(@ohsayan): should a drop space block the entire global table?
         let space_name = space;
         let mut wgns = global.namespace().spaces().write();
         let space = match wgns.get(space_name.as_str()) {
             Some(space) => space,
-            None => return Err(DatabaseError::DdlSpaceNotFound),
+            None => return Err(Error::QPObjectNotFound),
         };
         let space_w = space.mns.write();
         if space_w.st_len() != 0 {
-            return Err(DatabaseError::DdlSpaceRemoveNonEmpty);
+            return Err(Error::QPDdlNotEmpty);
         }
         // we can remove this
         if G::FS_IS_NON_NULL {
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index c86cf586..70526fc9 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -29,13 +29,13 @@ use crate::engine::{
         model::{alt::AlterPlan, Model},
         tests::ddl_model::{create, exec_create},
     },
-    error::DatabaseResult,
+    error::QueryResult,
     fractal::GlobalInstanceLike,
     idx::STIndex,
     ql::{ast::parse_ast_node_full, ddl::alt::AlterModel, tests::lex_insecure},
 };
 
-fn with_plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) -> DatabaseResult<()> {
+fn with_plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) -> QueryResult<()> {
     let model = create(model)?;
     let tok = lex_insecure(plan.as_bytes()).unwrap();
     let alter = parse_ast_node_full(&tok[2..]).unwrap();
@@ -52,7 +52,7 @@ fn exec_plan(
     model: &str,
     plan: &str,
     f: impl Fn(&Model),
-) -> DatabaseResult<()> {
+) -> QueryResult<()> {
     let mdl_name = exec_create(global, model, new_space)?;
     let prev_uuid = {
         let gns = global.namespace().spaces().read();
@@ -77,7 +77,7 @@ mod plan {
     use crate::{
         engine::{
             core::model::{self, alt::AlterAction, Field, Layer},
-            error::DatabaseError,
+            error::Error,
         },
         vecfuse,
     };
@@ -164,7 +164,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            DatabaseError::FieldNotFound
+            Error::QPUnknownField
         );
     }
     #[test]
@@ -176,7 +176,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            DatabaseError::DdlModelAlterProtectedField
+            Error::QPDdlModelAlterIllegal
         );
     }
     #[test]
@@ -188,7 +188,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            DatabaseError::DdlModelAlterBad
+            Error::QPDdlModelAlterIllegal
         );
     }
     #[test]
@@ -200,7 +200,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            DatabaseError::DdlModelAlterBad
+            Error::QPDdlModelAlterIllegal
         );
     }
     #[test]
@@ -212,7 +212,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            DatabaseError::DdlModelAlterProtectedField
+            Error::QPDdlModelAlterIllegal
         );
     }
     #[test]
@@ -224,7 +224,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            DatabaseError::FieldNotFound
+            Error::QPUnknownField
         );
     }
     fn bad_type_cast(orig_ty: &str, new_ty: &str) {
@@ -235,7 +235,7 @@ mod plan {
             super::with_plan(&create, &alter, |_| {}).expect_err(&format!(
                 "found no error in transformation: {orig_ty} -> {new_ty}"
             )),
-            DatabaseError::DdlModelAlterBadTypedef,
+            Error::QPDdlInvalidTypeDefinition,
             "failed to match error in transformation: {orig_ty} -> {new_ty}",
         )
     }
@@ -353,7 +353,7 @@ mod plan {
 mod exec {
     use crate::engine::{
         core::model::{DeltaVersion, Field, Layer},
-        error::DatabaseError,
+        error::Error,
         fractal::test_utils::TestGlobal,
         idx::{STIndex, STIndexSeq},
     };
@@ -445,7 +445,7 @@ mod exec {
                 |_| {},
             )
             .unwrap_err(),
-            DatabaseError::NeedLock
+            Error::QPNeedLock
         );
     }
 }
diff --git a/server/src/engine/core/tests/ddl_model/crt.rs b/server/src/engine/core/tests/ddl_model/crt.rs
index 1011d8a2..a42c5904 100644
--- a/server/src/engine/core/tests/ddl_model/crt.rs
+++ b/server/src/engine/core/tests/ddl_model/crt.rs
@@ -30,7 +30,7 @@ mod validation {
         crate::engine::{
             core::model::{DeltaVersion, Field, Layer},
             data::tag::{DataTag, FullTag},
-            error::DatabaseError,
+            error::Error,
             idx::STIndexSeq,
         },
     };
@@ -89,7 +89,7 @@ mod validation {
                 "create model mymodel(primary username: string, primary contract_location: binary)"
             )
             .unwrap_err(),
-            DatabaseError::DdlModelBadDefinition
+            Error::QPDdlModelBadDefinition
         );
     }
 
@@ -97,7 +97,7 @@ mod validation {
     fn duplicate_fields() {
         assert_eq!(
             create("create model mymodel(primary username: string, username: binary)").unwrap_err(),
-            DatabaseError::DdlModelBadDefinition
+            Error::QPDdlModelBadDefinition
         );
     }
 
@@ -105,7 +105,7 @@ mod validation {
     fn illegal_props() {
         assert_eq!(
         create("create model mymodel(primary username: string, password: binary) with { lol_prop: false }").unwrap_err(),
-        DatabaseError::DdlModelBadDefinition
+        Error::QPDdlModelBadDefinition
     );
     }
 
@@ -116,12 +116,12 @@ mod validation {
             "create model mymodel(primary username_bytes: list { type: uint8 }, password: binary)"
         )
         .unwrap_err(),
-        DatabaseError::DdlModelBadDefinition
+        Error::QPDdlModelBadDefinition
     );
         assert_eq!(
             create("create model mymodel(primary username: float32, password: binary)")
                 .unwrap_err(),
-            DatabaseError::DdlModelBadDefinition
+            Error::QPDdlModelBadDefinition
         );
     }
 }
diff --git a/server/src/engine/core/tests/ddl_model/layer.rs b/server/src/engine/core/tests/ddl_model/layer.rs
index f4025af2..e5241483 100644
--- a/server/src/engine/core/tests/ddl_model/layer.rs
+++ b/server/src/engine/core/tests/ddl_model/layer.rs
@@ -26,23 +26,23 @@
 
 use crate::engine::{
     core::model::Field,
-    error::DatabaseResult,
+    error::QueryResult,
     ql::{ast::parse_ast_node_multiple_full, tests::lex_insecure},
 };
 
-fn layerview_nullable(layer_def: &str, nullable: bool) -> DatabaseResult<Field> {
+fn layerview_nullable(layer_def: &str, nullable: bool) -> QueryResult<Field> {
     let tok = lex_insecure(layer_def.as_bytes()).unwrap();
     let spec = parse_ast_node_multiple_full(&tok).unwrap();
     Field::parse_layers(spec, nullable)
 }
-fn layerview(layer_def: &str) -> DatabaseResult<Field> {
+fn layerview(layer_def: &str) -> QueryResult<Field> {
     layerview_nullable(layer_def, false)
 }
 
 mod layer_spec_validation {
     use {
         super::layerview,
-        crate::engine::{core::model::Layer, error::DatabaseError},
+        crate::engine::{core::model::Layer, error::Error},
     };
 
     #[test]
@@ -64,7 +64,7 @@ mod layer_spec_validation {
     fn invalid_list() {
         assert_eq!(
             layerview("list").unwrap_err(),
-            DatabaseError::DdlModelInvalidTypeDefinition
+            Error::QPDdlInvalidTypeDefinition
         );
     }
 
@@ -72,7 +72,7 @@ mod layer_spec_validation {
     fn invalid_flat() {
         assert_eq!(
             layerview("string { type: string }").unwrap_err(),
-            DatabaseError::DdlModelInvalidTypeDefinition
+            Error::QPDdlInvalidTypeDefinition
         );
     }
 }
diff --git a/server/src/engine/core/tests/ddl_model/mod.rs b/server/src/engine/core/tests/ddl_model/mod.rs
index c2a52baf..a40710d7 100644
--- a/server/src/engine/core/tests/ddl_model/mod.rs
+++ b/server/src/engine/core/tests/ddl_model/mod.rs
@@ -30,7 +30,7 @@ mod layer;
 
 use crate::engine::{
     core::{model::Model, space::Space},
-    error::DatabaseResult,
+    error::QueryResult,
     fractal::GlobalInstanceLike,
     idx::STIndex,
     ql::{
@@ -40,7 +40,7 @@ use crate::engine::{
     },
 };
 
-fn create(s: &str) -> DatabaseResult<Model> {
+fn create(s: &str) -> QueryResult<Model> {
     let tok = lex_insecure(s.as_bytes()).unwrap();
     let create_model = parse_ast_node_full(&tok[2..]).unwrap();
     Model::process_create(create_model)
@@ -50,7 +50,7 @@ pub fn exec_create(
     global: &impl GlobalInstanceLike,
     create_stmt: &str,
     create_new_space: bool,
-) -> DatabaseResult<String> {
+) -> QueryResult<String> {
     let tok = lex_insecure(create_stmt.as_bytes()).unwrap();
     let create_model = parse_ast_node_full::<CreateModel>(&tok[2..]).unwrap();
     let name = match create_model.model_name {
@@ -67,14 +67,14 @@ pub fn exec_create(
 pub fn exec_create_new_space(
     global: &impl GlobalInstanceLike,
     create_stmt: &str,
-) -> DatabaseResult<()> {
+) -> QueryResult<()> {
     exec_create(global, create_stmt, true).map(|_| ())
 }
 
 pub fn exec_create_no_create(
     global: &impl GlobalInstanceLike,
     create_stmt: &str,
-) -> DatabaseResult<()> {
+) -> QueryResult<()> {
     exec_create(global, create_stmt, false).map(|_| ())
 }
 
diff --git a/server/src/engine/core/tests/ddl_space/alter.rs b/server/src/engine/core/tests/ddl_space/alter.rs
index aa772b25..a37c9255 100644
--- a/server/src/engine/core/tests/ddl_space/alter.rs
+++ b/server/src/engine/core/tests/ddl_space/alter.rs
@@ -27,7 +27,7 @@
 use crate::engine::{
     core::space::{Space, SpaceMeta},
     data::cell::Datacell,
-    error::DatabaseError,
+    error::Error,
     fractal::test_utils::TestGlobal,
 };
 
@@ -122,7 +122,7 @@ fn alter_nx() {
             |_| {},
         )
         .unwrap_err(),
-        DatabaseError::DdlSpaceNotFound
+        Error::QPObjectNotFound
     );
 }
 
diff --git a/server/src/engine/core/tests/ddl_space/create.rs b/server/src/engine/core/tests/ddl_space/create.rs
index 574121fa..669f9051 100644
--- a/server/src/engine/core/tests/ddl_space/create.rs
+++ b/server/src/engine/core/tests/ddl_space/create.rs
@@ -27,7 +27,7 @@
 use crate::engine::{
     core::space::{Space, SpaceMeta},
     data::cell::Datacell,
-    error::DatabaseError,
+    error::Error,
     fractal::test_utils::TestGlobal,
 };
 
@@ -73,7 +73,7 @@ fn exec_create_space_with_bad_env_type() {
     let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_create(&global, "create space myspace with { env: 100 }", |_| {}).unwrap_err(),
-        DatabaseError::DdlSpaceBadProperty
+        Error::QPDdlInvalidProperties
     );
 }
 
@@ -87,6 +87,6 @@ fn exec_create_space_with_random_property() {
             |_| {}
         )
         .unwrap_err(),
-        DatabaseError::DdlSpaceBadProperty
+        Error::QPDdlInvalidProperties
     );
 }
diff --git a/server/src/engine/core/tests/ddl_space/mod.rs b/server/src/engine/core/tests/ddl_space/mod.rs
index c3942564..fb0c6799 100644
--- a/server/src/engine/core/tests/ddl_space/mod.rs
+++ b/server/src/engine/core/tests/ddl_space/mod.rs
@@ -30,7 +30,7 @@ mod create;
 use crate::engine::{
     core::space::Space,
     data::uuid::Uuid,
-    error::DatabaseResult,
+    error::QueryResult,
     fractal::GlobalInstanceLike,
     ql::{
         ast::{self},
@@ -42,7 +42,7 @@ fn exec_create(
     gns: &impl GlobalInstanceLike,
     create: &str,
     verify: impl Fn(&Space),
-) -> DatabaseResult<Uuid> {
+) -> QueryResult<Uuid> {
     let tok = lex(create.as_bytes()).unwrap();
     let ast_node =
         ast::parse_ast_node_full::<crate::engine::ql::ddl::crt::CreateSpace>(&tok[2..]).unwrap();
@@ -58,7 +58,7 @@ fn exec_alter(
     gns: &impl GlobalInstanceLike,
     alter: &str,
     verify: impl Fn(&Space),
-) -> DatabaseResult<Uuid> {
+) -> QueryResult<Uuid> {
     let tok = lex(alter.as_bytes()).unwrap();
     let ast_node =
         ast::parse_ast_node_full::<crate::engine::ql::ddl::alt::AlterSpace>(&tok[2..]).unwrap();
@@ -75,7 +75,7 @@ fn exec_create_alter(
     crt: &str,
     alt: &str,
     verify_post_alt: impl Fn(&Space),
-) -> DatabaseResult<Uuid> {
+) -> QueryResult<Uuid> {
     let uuid_crt = exec_create(gns, crt, |_| {})?;
     let uuid_alt = exec_alter(gns, alt, verify_post_alt)?;
     assert_eq!(uuid_crt, uuid_alt);
diff --git a/server/src/engine/core/tests/dml/delete.rs b/server/src/engine/core/tests/dml/delete.rs
index e0b0ee39..8e7d375a 100644
--- a/server/src/engine/core/tests/dml/delete.rs
+++ b/server/src/engine/core/tests/dml/delete.rs
@@ -24,7 +24,7 @@
  *
 */
 
-use crate::engine::{error::DatabaseError, fractal::test_utils::TestGlobal};
+use crate::engine::{error::Error, fractal::test_utils::TestGlobal};
 
 #[test]
 fn simple_delete() {
@@ -51,6 +51,6 @@ fn delete_nonexisting() {
             "sayan",
         )
         .unwrap_err(),
-        DatabaseError::DmlEntryNotFound
+        Error::QPDmlRowNotFound
     );
 }
diff --git a/server/src/engine/core/tests/dml/insert.rs b/server/src/engine/core/tests/dml/insert.rs
index fb3a07a5..09d47806 100644
--- a/server/src/engine/core/tests/dml/insert.rs
+++ b/server/src/engine/core/tests/dml/insert.rs
@@ -24,7 +24,7 @@
  *
 */
 
-use crate::engine::{data::cell::Datacell, error::DatabaseError, fractal::test_utils::TestGlobal};
+use crate::engine::{data::cell::Datacell, error::Error, fractal::test_utils::TestGlobal};
 
 #[derive(sky_macros::Wrapper, Debug)]
 struct Tuple(Vec<(Box<str>, Datacell)>);
@@ -83,6 +83,6 @@ fn insert_duplicate() {
     assert_eq!(
         super::exec_insert_only(&global, "insert into myspace.mymodel('sayan', 'pass123')")
             .unwrap_err(),
-        DatabaseError::DmlConstraintViolationDuplicate
+        Error::QPDmlDuplicate
     );
 }
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index c1d489a9..b2bcdc0e 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -32,7 +32,7 @@ mod update;
 use crate::engine::{
     core::{dml, index::Row, model::Model},
     data::{cell::Datacell, lit::LitIR},
-    error::DatabaseResult,
+    error::QueryResult,
     fractal::GlobalInstanceLike,
     ql::{
         ast::{parse_ast_node_full, Entity},
@@ -42,10 +42,7 @@ use crate::engine::{
     sync,
 };
 
-fn _exec_only_create_space_model(
-    global: &impl GlobalInstanceLike,
-    model: &str,
-) -> DatabaseResult<()> {
+fn _exec_only_create_space_model(global: &impl GlobalInstanceLike, model: &str) -> QueryResult<()> {
     if !global.namespace().spaces().read().contains_key("myspace") {
         global.namespace().test_new_empty_space("myspace");
     }
@@ -58,7 +55,7 @@ fn _exec_only_insert<T>(
     global: &impl GlobalInstanceLike,
     insert: &str,
     and_then: impl Fn(Entity) -> T,
-) -> DatabaseResult<T> {
+) -> QueryResult<T> {
     let lex_insert = lex_insecure(insert.as_bytes()).unwrap();
     let stmt_insert = parse_ast_node_full::<InsertStatement>(&lex_insert[1..]).unwrap();
     let entity = stmt_insert.entity();
@@ -72,7 +69,7 @@ fn _exec_only_read_key_and_then<T>(
     entity: Entity,
     key_name: &str,
     and_then: impl Fn(Row) -> T,
-) -> DatabaseResult<T> {
+) -> QueryResult<T> {
     let guard = sync::atm::cpin();
     global.namespace().with_model(entity, |mdl| {
         let _irm = mdl.intent_read_model();
@@ -86,11 +83,7 @@ fn _exec_only_read_key_and_then<T>(
     })
 }
 
-fn _exec_delete_only(
-    global: &impl GlobalInstanceLike,
-    delete: &str,
-    key: &str,
-) -> DatabaseResult<()> {
+fn _exec_delete_only(global: &impl GlobalInstanceLike, delete: &str, key: &str) -> QueryResult<()> {
     let lex_del = lex_insecure(delete.as_bytes()).unwrap();
     let delete = parse_ast_node_full::<DeleteStatement>(&lex_del[1..]).unwrap();
     let entity = delete.entity();
@@ -106,10 +99,7 @@ fn _exec_delete_only(
     Ok(())
 }
 
-fn _exec_only_select(
-    global: &impl GlobalInstanceLike,
-    select: &str,
-) -> DatabaseResult<Vec<Datacell>> {
+fn _exec_only_select(global: &impl GlobalInstanceLike, select: &str) -> QueryResult<Vec<Datacell>> {
     let lex_sel = lex_insecure(select.as_bytes()).unwrap();
     let select = parse_ast_node_full(&lex_sel[1..]).unwrap();
     let mut r = Vec::new();
@@ -117,7 +107,7 @@ fn _exec_only_select(
     Ok(r)
 }
 
-fn _exec_only_update(global: &impl GlobalInstanceLike, update: &str) -> DatabaseResult<()> {
+fn _exec_only_update(global: &impl GlobalInstanceLike, update: &str) -> QueryResult<()> {
     let lex_upd = lex_insecure(update.as_bytes()).unwrap();
     let update = parse_ast_node_full(&lex_upd[1..]).unwrap();
     dml::update(global, update)
@@ -129,17 +119,14 @@ pub(self) fn exec_insert<T: Default>(
     insert: &str,
     key_name: &str,
     f: impl Fn(Row) -> T,
-) -> DatabaseResult<T> {
+) -> QueryResult<T> {
     _exec_only_create_space_model(global, model)?;
     _exec_only_insert(global, insert, |entity| {
         _exec_only_read_key_and_then(global, entity, key_name, |row| f(row))
     })?
 }
 
-pub(self) fn exec_insert_only(
-    global: &impl GlobalInstanceLike,
-    insert: &str,
-) -> DatabaseResult<()> {
+pub(self) fn exec_insert_only(global: &impl GlobalInstanceLike, insert: &str) -> QueryResult<()> {
     _exec_only_insert(global, insert, |_| {})
 }
 
@@ -149,7 +136,7 @@ pub(self) fn exec_delete(
     insert: Option<&str>,
     delete: &str,
     key: &str,
-) -> DatabaseResult<()> {
+) -> QueryResult<()> {
     _exec_only_create_space_model(global, model)?;
     if let Some(insert) = insert {
         _exec_only_insert(global, insert, |_| {})?;
@@ -162,7 +149,7 @@ pub(self) fn exec_select(
     model: &str,
     insert: &str,
     select: &str,
-) -> DatabaseResult<Vec<Datacell>> {
+) -> QueryResult<Vec<Datacell>> {
     _exec_only_create_space_model(global, model)?;
     _exec_only_insert(global, insert, |_| {})?;
     _exec_only_select(global, select)
@@ -171,7 +158,7 @@ pub(self) fn exec_select(
 pub(self) fn exec_select_only(
     global: &impl GlobalInstanceLike,
     select: &str,
-) -> DatabaseResult<Vec<Datacell>> {
+) -> QueryResult<Vec<Datacell>> {
     _exec_only_select(global, select)
 }
 
@@ -181,7 +168,7 @@ pub(self) fn exec_update(
     insert: &str,
     update: &str,
     select: &str,
-) -> DatabaseResult<Vec<Datacell>> {
+) -> QueryResult<Vec<Datacell>> {
     _exec_only_create_space_model(global, model)?;
     _exec_only_insert(global, insert, |_| {})?;
     _exec_only_update(global, update)?;
diff --git a/server/src/engine/core/tests/dml/select.rs b/server/src/engine/core/tests/dml/select.rs
index 600c4dd8..206b3adc 100644
--- a/server/src/engine/core/tests/dml/select.rs
+++ b/server/src/engine/core/tests/dml/select.rs
@@ -24,7 +24,7 @@
  *
 */
 
-use crate::engine::{data::cell::Datacell, error::DatabaseError, fractal::test_utils::TestGlobal};
+use crate::engine::{data::cell::Datacell, error::Error, fractal::test_utils::TestGlobal};
 
 #[test]
 fn simple_select_wildcard() {
@@ -97,6 +97,6 @@ fn select_nonexisting() {
             "select username, password from myspace.mymodel where username = 'notsayan'",
         )
         .unwrap_err(),
-        DatabaseError::DmlEntryNotFound
+        Error::QPDmlRowNotFound
     );
 }
diff --git a/server/src/engine/core/tests/dml/update.rs b/server/src/engine/core/tests/dml/update.rs
index b1338b17..f32b4ee0 100644
--- a/server/src/engine/core/tests/dml/update.rs
+++ b/server/src/engine/core/tests/dml/update.rs
@@ -25,7 +25,7 @@
 */
 
 use crate::engine::{
-    core::dml, data::cell::Datacell, error::DatabaseError, fractal::test_utils::TestGlobal,
+    core::dml, data::cell::Datacell, error::Error, fractal::test_utils::TestGlobal,
 };
 
 #[test]
@@ -96,7 +96,7 @@ fn fail_operation_on_null() {
             "select * from myspace.mymodel where username='sayan'"
         )
         .unwrap_err(),
-        DatabaseError::DmlConstraintViolationFieldTypedef
+        Error::QPDmlValidationError
     );
     assert_eq!(
         dml::update_flow_trace(),
@@ -116,7 +116,7 @@ fn fail_unknown_fields() {
             "select * from myspace.mymodel where username='sayan'"
         )
         .unwrap_err(),
-        DatabaseError::FieldNotFound
+        Error::QPUnknownField
     );
     assert_eq!(dml::update_flow_trace(), ["fieldnotfound", "rollback"]);
     // verify integrity
@@ -142,7 +142,7 @@ fn fail_typedef_violation() {
             "select * from myspace.mymodel where username = 'sayan'"
         )
         .unwrap_err(),
-        DatabaseError::DmlConstraintViolationFieldTypedef
+        Error::QPDmlValidationError
     );
     assert_eq!(
         dml::update_flow_trace(),
diff --git a/server/src/engine/core/util.rs b/server/src/engine/core/util.rs
index 6d83edfc..4f74fb8c 100644
--- a/server/src/engine/core/util.rs
+++ b/server/src/engine/core/util.rs
@@ -25,27 +25,27 @@
 */
 
 use crate::engine::{
-    error::{DatabaseError, DatabaseResult},
+    error::{Error, QueryResult},
     ql::ast::Entity,
 };
 
 pub trait EntityLocator<'a> {
-    fn parse_entity(self) -> DatabaseResult<(&'a str, &'a str)>
+    fn parse_entity(self) -> QueryResult<(&'a str, &'a str)>
     where
         Self: 'a;
 }
 
 impl<'a> EntityLocator<'a> for (&'a str, &'a str) {
-    fn parse_entity(self) -> DatabaseResult<(&'a str, &'a str)> {
+    fn parse_entity(self) -> QueryResult<(&'a str, &'a str)> {
         Ok(self)
     }
 }
 
 impl<'a> EntityLocator<'a> for Entity<'a> {
-    fn parse_entity(self) -> DatabaseResult<(&'a str, &'a str)>
+    fn parse_entity(self) -> QueryResult<(&'a str, &'a str)>
     where
         Self: 'a,
     {
-        self.into_full_str().ok_or(DatabaseError::ExpectedEntity)
+        self.into_full_str().ok_or(Error::QPExpectedEntity)
     }
 }
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 542b8785..e4fd079d 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -25,129 +25,79 @@
 */
 
 use super::{storage::v1::SDSSError, txn::TransactionError};
+pub type QueryResult<T> = Result<T, Error>;
 
-pub type LangResult<T> = Result<T, LangError>;
-pub type LexResult<T> = Result<T, LexError>;
-pub type DatabaseResult<T> = Result<T, DatabaseError>;
-
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
-#[repr(u8)]
-/// Lex phase errors
-pub enum LexError {
-    // insecure lex
-    /// Invalid signed numeric literal
-    InvalidSignedNumericLit,
-    /// Invalid unsigned literal
-    InvalidUnsignedLiteral,
-    /// Invaid binary literal
-    InvalidBinaryLiteral,
-    /// Invalid string literal
-    InvalidStringLiteral,
-    // secure lex
-    /// Dataframe params are invalid
-    BadPframe,
-    // generic
-    /// Unrecognized byte in stream   
-    UnexpectedByte,
-}
-
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
-#[repr(u8)]
-/// AST errors
-pub enum LangError {
-    // generic
-    /// Unexpected end of syntax
-    UnexpectedEOS,
-    /// Last resort error kind when error specificity is hard to trace
-    BadSyntax,
-    /// Expected a token that defines a statement, found something else
-    ExpectedStatement,
-    // ast nodes: usually parents at heigher hights
-    /// Expected an entity, but found invalid tokens
-    ExpectedEntity,
-    // ast nodes: usually children wrt height
-    /// Bad syn tymeta element
-    SynBadTyMeta,
-    /// Bad syn map element
-    SynBadMap,
-    /// Bad expr: relational
-    ExprBadRel,
-    // ast nodes: usually the root
-    /// Unknown `create` statement
-    StmtUnknownCreate,
-    /// Unknown `alter` statement
-    StmtUnknownAlter,
-    /// unknown `drop` statement
-    StmtUnknownDrop,
-}
-
-#[derive(Debug)]
-#[repr(u8)]
-#[cfg_attr(test, derive(PartialEq))]
-/// Executor errors
-pub enum DatabaseError {
-    // sys
-    SysBadItemID,
-    // query generic
-    /// this needs an explicit lock
-    NeedLock,
-    /// expected a full entity, but found a single implicit entity
-    ExpectedEntity,
-    // ddl
-    /// unknown property or bad type for property
-    DdlSpaceBadProperty,
-    /// the space already exists
-    DdlSpaceAlreadyExists,
-    /// the space doesn't exist
-    DdlSpaceNotFound,
-    /// the space that we attempted to remove is non-empty
-    DdlSpaceRemoveNonEmpty,
-    /// bad definition for some typedef in a model
-    DdlModelInvalidTypeDefinition,
-    /// bad model definition; most likely an illegal primary key
-    DdlModelBadDefinition,
-    /// the model already exists
-    DdlModelAlreadyExists,
-    /// an alter attempted to remove a protected field (usually the primary key)
-    DdlModelAlterProtectedField,
-    /// an alter model attempted to modify an invalid property/a property with an illegal value
-    DdlModelAlterBadProperty,
-    /// the alter model statement is "wrong"
-    DdlModelAlterBad,
-    /// an alter attempted to update an nx field
-    FieldNotFound,
-    /// bad type definition to alter
-    DdlModelAlterBadTypedef,
-    /// didn't find the model
-    DdlModelNotFound,
-    /// attempted a remove, but the model view is nonempty
-    DdlModelViewNotEmpty,
-    // dml
-    /// Duplicate
-    DmlConstraintViolationDuplicate,
-    /// data validation error
-    DmlDataValidationError,
-    /// The expression in a where clause is not indexed (in the way the expression expects it to be)
-    DmlWhereClauseUnindexedExpr,
-    /// The entry was not found
-    DmlEntryNotFound,
-    /// illegal data
-    DmlIllegalData,
-    /// field definition violation
-    DmlConstraintViolationFieldTypedef,
-    ServerError,
+/// an enumeration of 'flat' errors that the server actually responds to the client with, since we do not want to send specific information
+/// about anything (as that will be a security hole). The variants correspond with their actual response codes
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum Error {
+    /// I/O error
+    SysIOError,
+    /// out of memory
+    SysOutOfMemory,
+    /// unknown server error
+    SysUnknownError,
+    /// invalid protocol packet
+    NetProtocolIllegalPacket,
+    /// something like an integer that randomly has a character to attached to it like `1234q`
+    LexInvalidLiteral,
+    /// something like an invalid 'string" or a safe string with a bad length etc
+    LexInvalidEscapedLiteral,
+    /// unexpected byte
+    LexUnexpectedByte,
+    /// expected a longer statement
+    QLUnexpectedEndOfStatement,
+    /// incorrect syntax for "something"
+    QLInvalidSyntax,
+    /// expected a statement keyword found something else
+    QLExpectedStatement,
+    /// invalid collection definition definition
+    QLInvalidCollectionSyntax,
+    /// invalid type definition syntax
+    QLInvalidTypeDefinitionSyntax,
+    /// invalid relational expression
+    QLIllegalRelExp,
+    /// expected a full entity definition
+    QPExpectedEntity,
+    /// expected a statement, found something else
+    QPExpectedStatement,
+    /// unknown statement
+    QPUnknownStatement,
+    /// this query needs a lock for execution, but that wasn't explicitly allowed anywhere
+    QPNeedLock,
+    /// the object to be used as the "query container" is missing (for example, insert when the model was missing)
+    QPObjectNotFound,
+    /// an unknown field was attempted to be accessed/modified/...
+    QPUnknownField,
+    /// invalid property for an object
+    QPDdlInvalidProperties,
+    /// create space/model, but the object already exists
+    QPDdlObjectAlreadyExists,
+    /// an object that was attempted to be removed is non-empty, and for this object, removals require it to be empty
+    QPDdlNotEmpty,
+    /// invalid type definition
+    QPDdlInvalidTypeDefinition,
+    /// bad model definition
+    QPDdlModelBadDefinition,
+    /// illegal alter model query
+    QPDdlModelAlterIllegal,
+    /// violated the uniqueness property
+    QPDmlDuplicate,
+    /// the data could not be validated for being accepted into a field/function/etc.
+    QPDmlValidationError,
+    /// the where expression has an unindexed column essentially implying that we can't run this query because of perf concerns
+    QPDmlWhereHasUnindexedColumn,
+    /// the row matching the given match expression was not found
+    QPDmlRowNotFound,
+    /// transactional error
     TransactionalError,
-    StorageSubsystemErr(SDSSError),
-}
-
-impl From<SDSSError> for DatabaseError {
-    fn from(e: SDSSError) -> Self {
-        Self::StorageSubsystemErr(e)
-    }
+    /// storage subsystem error
+    StorageSubsystemError,
 }
 
-impl From<TransactionError> for DatabaseError {
-    fn from(_: TransactionError) -> Self {
-        Self::TransactionalError
+direct_from! {
+    Error[_] => {
+        SDSSError as StorageSubsystemError,
+        TransactionError as TransactionalError,
     }
 }
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 93cdf85d..32614e5e 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -290,7 +290,7 @@ impl FractalMgr {
         model: &Model,
         observed_size: usize,
         mdl_driver: &super::FractalModelDriver,
-    ) -> Result<(), crate::engine::error::DatabaseError> {
+    ) -> crate::engine::error::QueryResult<()> {
         if observed_size == 0 {
             // no changes, all good
             return Ok(());
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index bb9ce76d..91058d0d 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -60,6 +60,9 @@ macro_rules! direct_from {
     ($for:ty => {$($other:ty as $me:ident),*$(,)?}) => {
         $(impl ::core::convert::From<$other> for $for {fn from(v: $other) -> Self {Self::$me(v.into())}})*
     };
+    ($for:ty[_] => {$($other:ty as $me:ident),*$(,)?}) => {
+        $(impl ::core::convert::From<$other> for $for {fn from(_: $other) -> Self {Self::$me}})*
+    };
 }
 
 #[allow(unused_macros)]
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 0a306411..1ec298db 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -37,7 +37,7 @@ use {
     crate::{
         engine::{
             data::{cell::Datacell, lit::LitIR},
-            error::{LangError, LangResult},
+            error::{Error, QueryResult},
         },
         util::{compiler, MaybeInit},
     },
@@ -443,7 +443,7 @@ impl<'a> Entity<'a> {
     #[inline(always)]
     /// Attempt to parse an entity using the given token stream. It also accepts a counter
     /// argument to forward the cursor
-    pub fn parse_from_tokens_len_checked(tok: &'a [Token], c: &mut usize) -> LangResult<Self> {
+    pub fn parse_from_tokens_len_checked(tok: &'a [Token], c: &mut usize) -> QueryResult<Self> {
         let is_current = Self::signature_matches_single_len_checked(tok);
         let is_full = Self::signature_matches_full_len_checked(tok);
         let r = match () {
@@ -457,14 +457,14 @@ impl<'a> Entity<'a> {
                 *c += 1;
                 Self::parse_uck_tokens_single(tok)
             },
-            _ => return Err(LangError::ExpectedEntity),
+            _ => return Err(Error::QPExpectedEntity),
         };
         Ok(r)
     }
     #[inline(always)]
     pub fn parse_from_state_rounded_result<Qd: QueryData<'a>>(
         state: &mut State<'a, Qd>,
-    ) -> LangResult<Self> {
+    ) -> QueryResult<Self> {
         let mut e = MaybeInit::uninit();
         Self::parse_from_state_rounded(state, &mut e);
         if compiler::likely(state.okay()) {
@@ -473,7 +473,7 @@ impl<'a> Entity<'a> {
                 Ok(e.assume_init())
             }
         } else {
-            Err(LangError::ExpectedEntity)
+            Err(Error::QPExpectedEntity)
         }
     }
     #[inline(always)]
@@ -560,14 +560,14 @@ pub enum Statement<'a> {
 }
 
 #[cfg(test)]
-pub fn compile_test<'a>(tok: &'a [Token<'a>]) -> LangResult<Statement<'a>> {
+pub fn compile_test<'a>(tok: &'a [Token<'a>]) -> QueryResult<Statement<'a>> {
     self::compile(tok, InplaceData::new())
 }
 
 #[inline(always)]
-pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> LangResult<Statement<'a>> {
+pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResult<Statement<'a>> {
     if compiler::unlikely(tok.len() < 2) {
-        return Err(LangError::UnexpectedEOS);
+        return Err(Error::QLUnexpectedEndOfStatement);
     }
     let mut state = State::new(tok, d);
     match state.fw_read() {
@@ -576,12 +576,12 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> LangResult
         Token![create] => match state.fw_read() {
             Token![model] => ASTNode::from_state(&mut state).map(Statement::CreateModel),
             Token![space] => ASTNode::from_state(&mut state).map(Statement::CreateSpace),
-            _ => compiler::cold_rerr(LangError::StmtUnknownCreate),
+            _ => compiler::cold_rerr(Error::QPUnknownStatement),
         },
         Token![alter] => match state.fw_read() {
             Token![model] => ASTNode::from_state(&mut state).map(Statement::AlterModel),
             Token![space] => ASTNode::from_state(&mut state).map(Statement::AlterSpace),
-            _ => compiler::cold_rerr(LangError::StmtUnknownAlter),
+            _ => compiler::cold_rerr(Error::QPUnknownStatement),
         },
         Token![drop] if state.remaining() >= 2 => ddl::drop::parse_drop(&mut state),
         Token::Ident(id) if id.eq_ignore_ascii_case("inspect") => {
@@ -592,6 +592,6 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> LangResult
         Token![select] => ASTNode::from_state(&mut state).map(Statement::Select),
         Token![update] => ASTNode::from_state(&mut state).map(Statement::Update),
         Token![delete] => ASTNode::from_state(&mut state).map(Statement::Delete),
-        _ => compiler::cold_rerr(LangError::ExpectedStatement),
+        _ => compiler::cold_rerr(Error::QPUnknownStatement),
     }
 }
diff --git a/server/src/engine/ql/ast/traits.rs b/server/src/engine/ql/ast/traits.rs
index b3ba812c..a8750a2c 100644
--- a/server/src/engine/ql/ast/traits.rs
+++ b/server/src/engine/ql/ast/traits.rs
@@ -27,7 +27,7 @@
 #[cfg(test)]
 use crate::engine::ql::{ast::InplaceData, lex::Token};
 use crate::engine::{
-    error::{LangError, LangResult},
+    error::{Error, QueryResult},
     ql::ast::{QueryData, State},
 };
 
@@ -40,38 +40,38 @@ pub trait ASTNode<'a>: Sized {
     /// - If the implementor uses a cow style parse, then set [`ASTNode::VERIFY`] to
     /// true
     /// - Try to propagate errors via [`State`] if possible
-    fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self>;
-    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self>;
+    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         let r = <Self as ASTNode>::_from_state(state);
         if Self::VERIFY {
             return if state.okay() {
                 r
             } else {
-                Err(LangError::BadSyntax)
+                Err(Error::QLInvalidSyntax)
             };
         }
         r
     }
     #[cfg(test)]
     /// Parse multiple nodes of this AST node type. Intended for the test suite.
-    fn _multiple_from_state<Qd: QueryData<'a>>(_: &mut State<'a, Qd>) -> LangResult<Vec<Self>> {
+    fn _multiple_from_state<Qd: QueryData<'a>>(_: &mut State<'a, Qd>) -> QueryResult<Vec<Self>> {
         unimplemented!()
     }
     #[cfg(test)]
-    fn multiple_from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Vec<Self>> {
+    fn multiple_from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Vec<Self>> {
         let r = <Self as ASTNode>::_multiple_from_state(state);
         if Self::VERIFY {
             return if state.okay() {
                 r
             } else {
-                Err(LangError::BadSyntax)
+                Err(Error::QLInvalidSyntax)
             };
         }
         r
     }
     #[cfg(test)]
     /// Parse this AST node utilizing the full token-stream. Intended for the test suite.
-    fn from_insecure_tokens_full(tok: &'a [Token<'a>]) -> LangResult<Self> {
+    fn from_insecure_tokens_full(tok: &'a [Token<'a>]) -> QueryResult<Self> {
         let mut state = State::new(tok, InplaceData::new());
         let r = <Self as ASTNode>::from_state(&mut state)?;
         assert!(state.exhausted());
@@ -80,24 +80,24 @@ pub trait ASTNode<'a>: Sized {
     #[cfg(test)]
     /// Parse multiple nodes of this AST node type, utilizing the full token stream.
     /// Intended for the test suite.
-    fn multiple_from_insecure_tokens_full(tok: &'a [Token<'a>]) -> LangResult<Vec<Self>> {
+    fn multiple_from_insecure_tokens_full(tok: &'a [Token<'a>]) -> QueryResult<Vec<Self>> {
         let mut state = State::new(tok, InplaceData::new());
         let r = Self::multiple_from_state(&mut state);
         if state.exhausted() && state.okay() {
             r
         } else {
-            Err(LangError::BadSyntax)
+            Err(Error::QLInvalidSyntax)
         }
     }
 }
 
 #[cfg(test)]
-pub fn parse_ast_node_full<'a, N: ASTNode<'a>>(tok: &'a [Token<'a>]) -> LangResult<N> {
+pub fn parse_ast_node_full<'a, N: ASTNode<'a>>(tok: &'a [Token<'a>]) -> QueryResult<N> {
     N::from_insecure_tokens_full(tok)
 }
 #[cfg(test)]
 pub fn parse_ast_node_multiple_full<'a, N: ASTNode<'a>>(
     tok: &'a [Token<'a>],
-) -> LangResult<Vec<N>> {
+) -> QueryResult<Vec<N>> {
     N::multiple_from_insecure_tokens_full(tok)
 }
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 1d2d571c..14ddea42 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -29,7 +29,7 @@ use {
     crate::{
         engine::{
             data::DictGeneric,
-            error::{LangError, LangResult},
+            error::{Error, QueryResult},
             ql::{
                 ast::{Entity, QueryData, State},
                 lex::{Ident, Token},
@@ -55,9 +55,9 @@ impl<'a> AlterSpace<'a> {
     }
     #[inline(always)]
     /// Parse alter space from tokens
-    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         if compiler::unlikely(state.remaining() <= 3) {
-            return compiler::cold_rerr(LangError::UnexpectedEOS);
+            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
         }
         let space_name = state.fw_read();
         state.poison_if_not(space_name.is_ident());
@@ -67,7 +67,7 @@ impl<'a> AlterSpace<'a> {
         state.cursor_ahead(); // ignore errors
 
         if compiler::unlikely(!state.okay()) {
-            return Err(LangError::BadSyntax);
+            return Err(Error::QLInvalidSyntax);
         }
 
         let space_name = unsafe {
@@ -82,7 +82,7 @@ impl<'a> AlterSpace<'a> {
                 updated_props: d,
             })
         } else {
-            Err(LangError::SynBadMap)
+            Err(Error::QLInvalidCollectionSyntax)
         }
     }
 }
@@ -111,10 +111,10 @@ pub enum AlterKind<'a> {
 impl<'a> AlterModel<'a> {
     #[inline(always)]
     /// Parse an [`AlterKind`] from the given token stream
-    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         // alter model mymodel remove x
         if state.remaining() <= 2 || !state.cursor_has_ident_rounded() {
-            return compiler::cold_rerr(LangError::BadSyntax);
+            return compiler::cold_rerr(Error::QLInvalidSyntax);
             // FIXME(@ohsayan): bad because no specificity
         }
         let model_name = Entity::parse_from_state_rounded_result(state)?;
@@ -122,7 +122,7 @@ impl<'a> AlterModel<'a> {
             Token![add] => AlterKind::alter_add(state),
             Token![remove] => AlterKind::alter_remove(state),
             Token![update] => AlterKind::alter_update(state),
-            _ => Err(LangError::ExpectedStatement),
+            _ => Err(Error::QPExpectedStatement),
         };
         kind.map(|kind| AlterModel::new(model_name, kind))
     }
@@ -131,24 +131,24 @@ impl<'a> AlterModel<'a> {
 impl<'a> AlterKind<'a> {
     #[inline(always)]
     /// Parse the expression for `alter model <> add (..)`
-    fn alter_add<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    fn alter_add<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         ExpandedField::parse_multiple(state).map(Self::Add)
     }
     #[inline(always)]
     /// Parse the expression for `alter model <> add (..)`
-    fn alter_update<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    fn alter_update<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         ExpandedField::parse_multiple(state).map(Self::Update)
     }
     #[inline(always)]
     /// Parse the expression for `alter model <> remove (..)`
-    fn alter_remove<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    fn alter_remove<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         const DEFAULT_REMOVE_COL_CNT: usize = 4;
         /*
             WARNING: No trailing commas allowed
             <remove> ::= <ident> | <openparen> (<ident> <comma>)*<closeparen>
         */
         if compiler::unlikely(state.exhausted()) {
-            return compiler::cold_rerr(LangError::UnexpectedEOS);
+            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
         }
 
         let r = match state.fw_read() {
@@ -177,10 +177,10 @@ impl<'a> AlterKind<'a> {
                 if state.okay() {
                     cols.into_boxed_slice()
                 } else {
-                    return Err(LangError::BadSyntax);
+                    return Err(Error::QLInvalidSyntax);
                 }
             }
-            _ => return Err(LangError::BadSyntax),
+            _ => return Err(Error::QLInvalidSyntax),
         };
         Ok(Self::Remove(r))
     }
@@ -190,17 +190,17 @@ mod impls {
     use {
         super::{AlterModel, AlterSpace},
         crate::engine::{
-            error::LangResult,
+            error::QueryResult,
             ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for AlterModel<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
     impl<'a> ASTNode<'a> for AlterSpace<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index ede26fa1..39f1c9ac 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -29,7 +29,7 @@ use {
     crate::{
         engine::{
             data::DictGeneric,
-            error::{LangError, LangResult},
+            error::{Error, QueryResult},
             ql::{
                 ast::{Entity, QueryData, State},
                 lex::Ident,
@@ -51,10 +51,10 @@ pub struct CreateSpace<'a> {
 impl<'a> CreateSpace<'a> {
     #[inline(always)]
     /// Parse space data from the given tokens
-    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         // smallest declaration: `create space myspace` -> >= 1 token
         if compiler::unlikely(state.remaining() < 1) {
-            return compiler::cold_rerr(LangError::UnexpectedEOS);
+            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
         }
         let space_name = state.fw_read();
         state.poison_if_not(space_name.is_ident());
@@ -76,7 +76,7 @@ impl<'a> CreateSpace<'a> {
                 props: d,
             })
         } else {
-            Err(LangError::BadSyntax)
+            Err(Error::QLInvalidSyntax)
         }
     }
 }
@@ -108,9 +108,9 @@ impl<'a> CreateModel<'a> {
         }
     }
 
-    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         if compiler::unlikely(state.remaining() < 10) {
-            return compiler::cold_rerr(LangError::UnexpectedEOS);
+            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
         }
         // model name; ignore errors
         let mut model_uninit = MaybeInit::uninit();
@@ -147,7 +147,7 @@ impl<'a> CreateModel<'a> {
                 props,
             })
         } else {
-            Err(LangError::BadSyntax)
+            Err(Error::QLInvalidSyntax)
         }
     }
 }
@@ -156,17 +156,17 @@ mod impls {
     use {
         super::{CreateModel, CreateSpace},
         crate::engine::{
-            error::LangResult,
+            error::QueryResult,
             ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for CreateSpace<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
     impl<'a> ASTNode<'a> for CreateModel<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index bba4ee95..aedb11a9 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -25,7 +25,7 @@
 */
 
 use crate::engine::{
-    error::{LangError, LangResult},
+    error::{Error, QueryResult},
     ql::{
         ast::{Entity, QueryData, State, Statement},
         lex::{Ident, Token},
@@ -45,7 +45,7 @@ impl<'a> DropSpace<'a> {
     pub const fn new(space: Ident<'a>, force: bool) -> Self {
         Self { space, force }
     }
-    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<DropSpace<'a>> {
+    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<DropSpace<'a>> {
         if state.cursor_is_ident() {
             let ident = state.fw_read();
             // should we force drop?
@@ -62,7 +62,7 @@ impl<'a> DropSpace<'a> {
                 ));
             }
         }
-        Err(LangError::BadSyntax)
+        Err(Error::QLInvalidSyntax)
     }
 }
 
@@ -77,14 +77,14 @@ impl<'a> DropModel<'a> {
     pub fn new(entity: Entity<'a>, force: bool) -> Self {
         Self { entity, force }
     }
-    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         let e = Entity::parse_from_state_rounded_result(state)?;
         let force = state.cursor_rounded_eq(Token::Ident(Ident::from("force")));
         state.cursor_ahead_if(force);
         if state.exhausted() {
             return Ok(DropModel::new(e, force));
         } else {
-            Err(LangError::BadSyntax)
+            Err(Error::QLInvalidSyntax)
         }
     }
 }
@@ -93,11 +93,11 @@ impl<'a> DropModel<'a> {
 /// ## Panic
 ///
 /// If token stream length is < 2
-pub fn parse_drop<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Statement<'a>> {
+pub fn parse_drop<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Statement<'a>> {
     match state.fw_read() {
         Token![model] => DropModel::parse(state).map(Statement::DropModel),
         Token![space] => return DropSpace::parse(state).map(Statement::DropSpace),
-        _ => Err(LangError::StmtUnknownDrop),
+        _ => Err(Error::QPUnknownStatement),
     }
 }
 
@@ -106,24 +106,24 @@ mod impls {
     use {
         super::{DropModel, DropSpace},
         crate::engine::{
-            error::LangResult,
+            error::QueryResult,
             ql::ast::{traits::ASTNode, QueryData, State, Statement},
         },
     };
     impl<'a> ASTNode<'a> for DropModel<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
     impl<'a> ASTNode<'a> for DropSpace<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
     #[derive(sky_macros::Wrapper, Debug)]
     pub struct DropStatementAST<'a>(Statement<'a>);
     impl<'a> ASTNode<'a> for DropStatementAST<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             super::parse_drop(state).map(Self)
         }
     }
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
index 757c36d8..db5f5842 100644
--- a/server/src/engine/ql/ddl/ins.rs
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -26,7 +26,7 @@
 
 use crate::{
     engine::{
-        error::{LangError, LangResult},
+        error::{Error, QueryResult},
         ql::{
             ast::{Entity, QueryData, State, Statement},
             lex::Token,
@@ -37,7 +37,7 @@ use crate::{
 
 pub fn parse_inspect<'a, Qd: QueryData<'a>>(
     state: &mut State<'a, Qd>,
-) -> LangResult<Statement<'a>> {
+) -> QueryResult<Statement<'a>> {
     /*
         inpsect model <entity>
         inspect space <entity>
@@ -47,7 +47,7 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
     */
 
     if compiler::unlikely(state.remaining() < 1) {
-        return compiler::cold_rerr(LangError::UnexpectedEOS);
+        return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
     }
 
     match state.fw_read() {
@@ -65,7 +65,7 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
         }
         _ => {
             state.cursor_back();
-            Err(LangError::ExpectedStatement)
+            Err(Error::QPExpectedStatement)
         }
     }
 }
@@ -73,13 +73,13 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
 pub use impls::InspectStatementAST;
 mod impls {
     use crate::engine::{
-        error::LangResult,
+        error::QueryResult,
         ql::ast::{traits::ASTNode, QueryData, State, Statement},
     };
     #[derive(sky_macros::Wrapper, Debug)]
     pub struct InspectStatementAST<'a>(Statement<'a>);
     impl<'a> ASTNode<'a> for InspectStatementAST<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             super::parse_inspect(state).map(Self)
         }
     }
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index feff7dab..da376e1d 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -50,7 +50,7 @@ use crate::{
             cell::Datacell,
             dict::{DictEntryGeneric, DictGeneric},
         },
-        error::{LangError, LangResult},
+        error::{Error, QueryResult},
         ql::{
             ast::{QueryData, State},
             lex::{Ident, Token},
@@ -356,10 +356,10 @@ impl<'a> FieldSpec<'a> {
             primary,
         }
     }
-    pub fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    pub fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         if compiler::unlikely(state.remaining() < 2) {
             // smallest field: `ident: type`
-            return Err(LangError::UnexpectedEOS);
+            return Err(Error::QLUnexpectedEndOfStatement);
         }
         // check if primary or null
         let is_primary = state.cursor_eq(Token![primary]);
@@ -371,7 +371,7 @@ impl<'a> FieldSpec<'a> {
         // field name
         let field_name = match (state.fw_read(), state.fw_read()) {
             (Token::Ident(id), Token![:]) => id,
-            _ => return Err(LangError::BadSyntax),
+            _ => return Err(Error::QLInvalidSyntax),
         };
         // layers
         let mut layers = Vec::new();
@@ -384,7 +384,7 @@ impl<'a> FieldSpec<'a> {
                 primary: is_primary,
             })
         } else {
-            Err(LangError::SynBadTyMeta)
+            Err(Error::QLInvalidTypeDefinitionSyntax)
         }
     }
 }
@@ -407,10 +407,10 @@ impl<'a> ExpandedField<'a> {
     }
     #[inline(always)]
     /// Parse a field declared using the field syntax
-    pub(super) fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    pub(super) fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         if compiler::unlikely(state.remaining() < 6) {
             // smallest: fieldname { type: ident }
-            return Err(LangError::UnexpectedEOS);
+            return Err(Error::QLUnexpectedEndOfStatement);
         }
         let field_name = state.fw_read();
         state.poison_if_not(field_name.is_ident());
@@ -423,7 +423,7 @@ impl<'a> ExpandedField<'a> {
             // this has layers. fold them; but don't forget the colon
             if compiler::unlikely(state.exhausted()) {
                 // we need more tokens
-                return Err(LangError::UnexpectedEOS);
+                return Err(Error::QLUnexpectedEndOfStatement);
             }
             state.poison_if_not(state.cursor_eq(Token![:]));
             state.cursor_ahead();
@@ -450,12 +450,14 @@ impl<'a> ExpandedField<'a> {
                 layers,
             })
         } else {
-            Err(LangError::BadSyntax)
+            Err(Error::QLInvalidSyntax)
         }
     }
     #[inline(always)]
     /// Parse multiple fields declared using the field syntax. Flag setting allows or disallows reset syntax
-    pub fn parse_multiple<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Box<[Self]>> {
+    pub fn parse_multiple<Qd: QueryData<'a>>(
+        state: &mut State<'a, Qd>,
+    ) -> QueryResult<Box<[Self]>> {
         const DEFAULT_ADD_COL_CNT: usize = 4;
         /*
             WARNING: No trailing commas allowed
@@ -466,7 +468,7 @@ impl<'a> ExpandedField<'a> {
             alter model add myfield { type string }
         */
         if compiler::unlikely(state.remaining() < 5) {
-            return compiler::cold_rerr(LangError::UnexpectedEOS);
+            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
         }
         match state.read() {
             Token::Ident(_) => {
@@ -498,10 +500,10 @@ impl<'a> ExpandedField<'a> {
                 if state.okay() {
                     Ok(cols.into_boxed_slice())
                 } else {
-                    Err(LangError::BadSyntax)
+                    Err(Error::QLInvalidSyntax)
                 }
             }
-            _ => Err(LangError::ExpectedStatement),
+            _ => Err(Error::QPExpectedStatement),
         }
     }
 }
@@ -516,24 +518,24 @@ mod impls {
             FieldSpec, LayerSpec,
         },
         crate::engine::{
-            error::LangResult,
+            error::QueryResult,
             ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for ExpandedField<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             Self::parse(state)
         }
         fn _multiple_from_state<Qd: QueryData<'a>>(
             state: &mut State<'a, Qd>,
-        ) -> LangResult<Vec<Self>> {
+        ) -> QueryResult<Vec<Self>> {
             Self::parse_multiple(state).map(Vec::from)
         }
     }
     impl<'a> ASTNode<'a> for LayerSpec<'a> {
         // important: upstream must verify this
         const VERIFY: bool = true;
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             let mut layers = Vec::new();
             rfold_layers(state, &mut layers);
             assert!(layers.len() == 1);
@@ -541,7 +543,7 @@ mod impls {
         }
         fn _multiple_from_state<Qd: QueryData<'a>>(
             state: &mut State<'a, Qd>,
-        ) -> LangResult<Vec<Self>> {
+        ) -> QueryResult<Vec<Self>> {
             let mut l = Vec::new();
             rfold_layers(state, &mut l);
             Ok(l)
@@ -552,7 +554,7 @@ mod impls {
     impl<'a> ASTNode<'a> for DictBasic {
         // important: upstream must verify this
         const VERIFY: bool = true;
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             let mut dict = DictGeneric::new();
             rfold_dict(DictFoldState::OB, state, &mut dict);
             Ok(Self(dict))
@@ -563,7 +565,7 @@ mod impls {
     impl<'a> ASTNode<'a> for DictTypeMetaSplit {
         // important: upstream must verify this
         const VERIFY: bool = true;
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             let mut dict = DictGeneric::new();
             rfold_tymeta(DictFoldState::CB_OR_IDENT, state, &mut dict);
             Ok(Self(dict))
@@ -574,14 +576,14 @@ mod impls {
     impl<'a> ASTNode<'a> for DictTypeMeta {
         // important: upstream must verify this
         const VERIFY: bool = true;
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             let mut dict = DictGeneric::new();
             rfold_tymeta(DictFoldState::OB, state, &mut dict);
             Ok(Self(dict))
         }
     }
     impl<'a> ASTNode<'a> for FieldSpec<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
diff --git a/server/src/engine/ql/dml/del.rs b/server/src/engine/ql/dml/del.rs
index e68d21ca..3d3a283b 100644
--- a/server/src/engine/ql/dml/del.rs
+++ b/server/src/engine/ql/dml/del.rs
@@ -30,7 +30,7 @@ use {
     super::WhereClause,
     crate::{
         engine::{
-            error::{LangError, LangResult},
+            error::{Error, QueryResult},
             ql::ast::{Entity, QueryData, State},
         },
         util::{compiler, MaybeInit},
@@ -73,7 +73,7 @@ impl<'a> DeleteStatement<'a> {
         Self::new(entity, WhereClause::new(wc))
     }
     #[inline(always)]
-    pub fn parse_delete<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    pub fn parse_delete<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         /*
             TODO(@ohsayan): Volcano
             smallest tt:
@@ -81,7 +81,7 @@ impl<'a> DeleteStatement<'a> {
                    ^1   ^2    ^3    ^4  ^5
         */
         if compiler::unlikely(state.remaining() < 5) {
-            return compiler::cold_rerr(LangError::UnexpectedEOS);
+            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
         }
         // from + entity
         state.poison_if_not(state.cursor_eq(Token![from]));
@@ -101,7 +101,7 @@ impl<'a> DeleteStatement<'a> {
                 wc,
             })
         } else {
-            compiler::cold_rerr(LangError::BadSyntax)
+            compiler::cold_rerr(Error::QLInvalidSyntax)
         }
     }
 }
@@ -110,12 +110,12 @@ mod impls {
     use {
         super::DeleteStatement,
         crate::engine::{
-            error::LangResult,
+            error::QueryResult,
             ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for DeleteStatement<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             Self::parse_delete(state)
         }
     }
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index fabf4b47..e3d9b117 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -28,7 +28,7 @@ use {
     crate::{
         engine::{
             data::cell::Datacell,
-            error::{LangError, LangResult},
+            error::{Error, QueryResult},
             ql::{
                 ast::{Entity, QueryData, State},
                 lex::{Ident, Token},
@@ -350,14 +350,14 @@ impl<'a> InsertStatement<'a> {
 }
 
 impl<'a> InsertStatement<'a> {
-    pub fn parse_insert<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    pub fn parse_insert<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         /*
             smallest:
             insert into model (primarykey)
                    ^1    ^2   ^3      ^4 ^5
         */
         if compiler::unlikely(state.remaining() < 5) {
-            return compiler::cold_rerr(LangError::UnexpectedEOS);
+            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
         }
         state.poison_if_not(state.cursor_eq(Token![into]));
         state.cursor_ahead(); // ignore errors
@@ -392,7 +392,7 @@ impl<'a> InsertStatement<'a> {
                 data,
             })
         } else {
-            compiler::cold_rerr(LangError::BadSyntax)
+            compiler::cold_rerr(Error::QLInvalidSyntax)
         }
     }
 }
@@ -405,12 +405,12 @@ mod impls {
     use {
         super::InsertStatement,
         crate::engine::{
-            error::LangResult,
+            error::QueryResult,
             ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for InsertStatement<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             Self::parse_insert(state)
         }
     }
@@ -421,7 +421,7 @@ mod impls {
                 parse_data_map_syntax, parse_data_tuple_syntax, parse_list, Datacell, HashMap,
             },
             crate::engine::{
-                error::LangResult,
+                error::QueryResult,
                 ql::ast::{traits::ASTNode, QueryData, State},
             },
         };
@@ -430,7 +430,7 @@ mod impls {
         impl<'a> ASTNode<'a> for List {
             // important: upstream must verify this
             const VERIFY: bool = true;
-            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
                 let mut l = Vec::new();
                 parse_list(state, &mut l);
                 Ok(List(l))
@@ -441,7 +441,7 @@ mod impls {
         impl<'a> ASTNode<'a> for DataTuple {
             // important: upstream must verify this
             const VERIFY: bool = true;
-            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
                 let r = parse_data_tuple_syntax(state);
                 Ok(Self(r))
             }
@@ -451,7 +451,7 @@ mod impls {
         impl<'a> ASTNode<'a> for DataMap {
             // important: upstream must verify this
             const VERIFY: bool = true;
-            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
                 let r = parse_data_map_syntax(state);
                 Ok(Self(
                     r.into_iter()
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index f105ad85..11645848 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -168,21 +168,21 @@ mod impls {
     use {
         super::{RelationalExpr, WhereClause},
         crate::engine::{
-            error::{LangError, LangResult},
+            error::{Error, QueryResult},
             ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for WhereClause<'a> {
         // important: upstream must verify this
         const VERIFY: bool = true;
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             let wh = Self::parse_where(state);
             Ok(wh)
         }
     }
     impl<'a> ASTNode<'a> for RelationalExpr<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
-            Self::try_parse(state).ok_or(LangError::ExprBadRel)
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+            Self::try_parse(state).ok_or(Error::QLIllegalRelExp)
         }
     }
 }
diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs
index c4f8b219..e387daab 100644
--- a/server/src/engine/ql/dml/sel.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -28,7 +28,7 @@ use {
     super::{WhereClause, WhereClauseCollection},
     crate::{
         engine::{
-            error::{LangError, LangResult},
+            error::{Error, QueryResult},
             ql::{
                 ast::{Entity, QueryData, State},
                 lex::{Ident, Token},
@@ -96,7 +96,7 @@ impl<'a> SelectStatement<'a> {
 }
 
 impl<'a> SelectStatement<'a> {
-    pub fn parse_select<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    pub fn parse_select<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         /*
             Smallest query:
             select * from model
@@ -104,7 +104,7 @@ impl<'a> SelectStatement<'a> {
                    1 2    3
         */
         if compiler::unlikely(state.remaining() < 3) {
-            return compiler::cold_rerr(LangError::UnexpectedEOS);
+            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
         }
         let mut select_fields = Vec::new();
         let is_wildcard = state.cursor_eq(Token![*]);
@@ -123,7 +123,7 @@ impl<'a> SelectStatement<'a> {
         state.poison_if_not(is_wildcard | !select_fields.is_empty());
         // we should have from + model
         if compiler::unlikely(state.remaining() < 2 || !state.okay()) {
-            return compiler::cold_rerr(LangError::BadSyntax);
+            return compiler::cold_rerr(Error::QLInvalidSyntax);
         }
         state.poison_if_not(state.cursor_eq(Token![from]));
         state.cursor_ahead(); // ignore errors
@@ -146,7 +146,7 @@ impl<'a> SelectStatement<'a> {
                 clause: WhereClause::new(clauses),
             })
         } else {
-            compiler::cold_rerr(LangError::BadSyntax)
+            compiler::cold_rerr(Error::QLInvalidSyntax)
         }
     }
 }
@@ -155,12 +155,12 @@ mod impls {
     use {
         super::SelectStatement,
         crate::engine::{
-            error::LangResult,
+            error::QueryResult,
             ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for SelectStatement<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             Self::parse_select(state)
         }
     }
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index 6a395bc9..f91ae41a 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -32,7 +32,7 @@ use {
         engine::{
             core::query_meta::AssignmentOperator,
             data::lit::LitIR,
-            error::{LangError, LangResult},
+            error::{Error, QueryResult},
             ql::{
                 ast::{Entity, QueryData, State},
                 lex::Ident,
@@ -172,7 +172,7 @@ impl<'a> UpdateStatement<'a> {
         }
     }
     #[inline(always)]
-    pub fn parse_update<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+    pub fn parse_update<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         /*
             TODO(@ohsayan): Allow volcanoes
             smallest tt:
@@ -180,7 +180,7 @@ impl<'a> UpdateStatement<'a> {
                    ^1    ^2  ^3 ^4 ^5^6    ^7^8^9
         */
         if compiler::unlikely(state.remaining() < 9) {
-            return compiler::cold_rerr(LangError::UnexpectedEOS);
+            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
         }
         // parse entity
         let mut entity = MaybeInit::uninit();
@@ -218,7 +218,7 @@ impl<'a> UpdateStatement<'a> {
                 wc: WhereClause::new(clauses),
             })
         } else {
-            compiler::cold_rerr(LangError::BadSyntax)
+            compiler::cold_rerr(Error::QLInvalidSyntax)
         }
     }
 }
@@ -227,22 +227,22 @@ mod impls {
     use {
         super::UpdateStatement,
         crate::engine::{
-            error::LangResult,
+            error::QueryResult,
             ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for UpdateStatement<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
             Self::parse_update(state)
         }
     }
     #[cfg(test)]
     mod test {
-        use super::{super::AssignmentExpression, ASTNode, LangResult, QueryData, State};
+        use super::{super::AssignmentExpression, ASTNode, QueryResult, QueryData, State};
         impl<'a> ASTNode<'a> for AssignmentExpression<'a> {
             // important: upstream must verify this
             const VERIFY: bool = true;
-            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> LangResult<Self> {
+            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
                 let mut expr = Vec::new();
                 AssignmentExpression::parse_and_append_expression(state, &mut expr);
                 state.poison_if_not(expr.len() == 1);
@@ -250,7 +250,7 @@ mod impls {
             }
             fn _multiple_from_state<Qd: QueryData<'a>>(
                 state: &mut State<'a, Qd>,
-            ) -> LangResult<Vec<Self>> {
+            ) -> QueryResult<Vec<Self>> {
                 let mut expr = Vec::new();
                 AssignmentExpression::parse_and_append_expression(state, &mut expr);
                 Ok(expr)
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index 839d795e..6c5ca70c 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -34,7 +34,7 @@ use {
                 lit::{Lit, LitIR},
                 spec::Dataspec1D,
             },
-            error::{LexError, LexResult},
+            error::{Error, QueryResult},
         },
         util::compiler,
     },
@@ -62,7 +62,7 @@ impl<'a> InsecureLexer<'a> {
         }
     }
     #[inline(always)]
-    pub fn lex(src: Slice<'a>) -> LexResult<Vec<Token<'a>>> {
+    pub fn lex(src: Slice<'a>) -> QueryResult<Vec<Token<'a>>> {
         let mut slf = Self::new(src);
         slf._lex();
         let RawLexer {
@@ -133,7 +133,7 @@ impl<'a> InsecureLexer<'a> {
                     slf.push_token(Lit::SignedInt(num));
                 }
                 _ => {
-                    compiler::cold_call(|| slf.set_error(LexError::InvalidSignedNumericLit));
+                    compiler::cold_call(|| slf.set_error(Error::LexInvalidLiteral));
                 }
             }
         } else {
@@ -174,7 +174,7 @@ impl<'a> InsecureLexer<'a> {
             Ok(num) if compiler::likely(wseof) => {
                 slf.tokens.push(Token::Lit(Lit::UnsignedInt(num)))
             }
-            _ => slf.set_error(LexError::InvalidUnsignedLiteral),
+            _ => slf.set_error(Error::LexInvalidLiteral),
         }
     }
 
@@ -224,7 +224,7 @@ impl<'a> InsecureLexer<'a> {
                 slf.incr_cursor_by(size);
             }
         } else {
-            slf.set_error(LexError::InvalidBinaryLiteral);
+            slf.set_error(Error::LexInvalidLiteral);
         }
     }
     #[inline(always)]
@@ -275,7 +275,7 @@ impl<'a> InsecureLexer<'a> {
             let terminated = slf.peek_eq_and_forward(quote_style);
             match String::from_utf8(buf) {
                 Ok(st) if terminated => slf.tokens.push(Token::Lit(st.into_boxed_str().into())),
-                _ => slf.set_error(LexError::InvalidStringLiteral),
+                _ => slf.set_error(Error::LexInvalidLiteral),
             }
         }
     }
@@ -295,11 +295,11 @@ impl<'a> SafeLexer<'a> {
         }
     }
     #[inline(always)]
-    pub fn lex(src: Slice<'a>) -> LexResult<Vec<Token>> {
+    pub fn lex(src: Slice<'a>) -> QueryResult<Vec<Token>> {
         Self::new(src)._lex()
     }
     #[inline(always)]
-    fn _lex(self) -> LexResult<Vec<Token<'a>>> {
+    fn _lex(self) -> QueryResult<Vec<Token<'a>>> {
         let Self { base: mut l } = self;
         while l.not_exhausted() && l.no_error() {
             let b = unsafe {
@@ -453,11 +453,11 @@ impl<'a> SafeQueryData<'a> {
         Self { p, t }
     }
     #[inline(always)]
-    pub fn parse_data(pf: Slice<'a>, pf_sz: usize) -> LexResult<Box<[LitIR<'a>]>> {
+    pub fn parse_data(pf: Slice<'a>, pf_sz: usize) -> QueryResult<Box<[LitIR<'a>]>> {
         Self::p_revloop(pf, pf_sz)
     }
     #[inline(always)]
-    pub fn parse(qf: Slice<'a>, pf: Slice<'a>, pf_sz: usize) -> LexResult<Self> {
+    pub fn parse(qf: Slice<'a>, pf: Slice<'a>, pf_sz: usize) -> QueryResult<Self> {
         let q = SafeLexer::lex(qf);
         let p = Self::p_revloop(pf, pf_sz);
         match (q, p) {
@@ -467,7 +467,7 @@ impl<'a> SafeQueryData<'a> {
         }
     }
     #[inline]
-    pub(super) fn p_revloop(mut src: Slice<'a>, size: usize) -> LexResult<Box<[LitIR<'a>]>> {
+    pub(super) fn p_revloop(mut src: Slice<'a>, size: usize) -> QueryResult<Box<[LitIR<'a>]>> {
         static LITIR_TF: [for<'a> fn(Slice<'a>, &mut usize, &mut Vec<LitIR<'a>>) -> bool; 7] = [
             SafeQueryData::uint,  // tc: 0
             SafeQueryData::sint,  // tc: 1
@@ -493,7 +493,7 @@ impl<'a> SafeQueryData<'a> {
         if compiler::likely(okay) {
             Ok(data.into_boxed_slice())
         } else {
-            Err(LexError::BadPframe)
+            Err(Error::LexInvalidEscapedLiteral)
         }
     }
 }
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index a40b511e..74061272 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -28,7 +28,7 @@ use {
     super::Slice,
     crate::engine::{
         data::{lit::Lit, spec::Dataspec1D},
-        error::LexError,
+        error::Error,
     },
     core::{borrow::Borrow, fmt, ops::Deref, slice, str},
 };
@@ -373,7 +373,7 @@ pub struct RawLexer<'a> {
     c: *const u8,
     e: *const u8,
     pub(super) tokens: Vec<Token<'a>>,
-    pub(super) last_error: Option<LexError>,
+    pub(super) last_error: Option<Error>,
 }
 
 // ctor
@@ -491,7 +491,7 @@ impl<'a> RawLexer<'a> {
         while self.peek_is_and_forward(|b| b == b' ' || b == b'\t' || b == b'\n') {}
     }
     #[inline(always)]
-    pub(super) fn set_error(&mut self, e: LexError) {
+    pub(super) fn set_error(&mut self, e: Error) {
         self.last_error = Some(e);
     }
     #[inline(always)]
@@ -532,7 +532,7 @@ impl<'a> RawLexer<'a> {
     pub(super) fn scan_byte(&mut self, byte: u8) {
         match symof(byte) {
             Some(tok) => self.push_token(tok),
-            None => return self.set_error(LexError::UnexpectedByte),
+            None => return self.set_error(Error::LexUnexpectedByte),
         }
         unsafe {
             // UNSAFE(@ohsayan): we are sent a byte, so fw cursor
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index b5ecd462..259d97da 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -27,7 +27,7 @@
 use {
     super::lex::{InsecureLexer, SafeLexer, Symbol, Token},
     crate::{
-        engine::{data::cell::Datacell, error::LexResult},
+        engine::{data::cell::Datacell, error::QueryResult},
         util::test_utils,
     },
     rand::{self, Rng},
@@ -41,12 +41,12 @@ mod structure_syn;
 
 #[inline(always)]
 /// Uses the [`InsecureLexer`] to lex the given input
-pub fn lex_insecure(src: &[u8]) -> LexResult<Vec<Token<'_>>> {
+pub fn lex_insecure(src: &[u8]) -> QueryResult<Vec<Token<'_>>> {
     InsecureLexer::lex(src)
 }
 #[inline(always)]
 /// Uses the [`SafeLexer`] to lex the given input
-pub fn lex_secure(src: &[u8]) -> LexResult<Vec<Token>> {
+pub fn lex_secure(src: &[u8]) -> QueryResult<Vec<Token>> {
     SafeLexer::lex(src)
 }
 
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index 274cfa56..b680a080 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -31,7 +31,7 @@ use {
     },
     crate::engine::{
         data::{lit::Lit, spec::Dataspec1D},
-        error::LexError,
+        error::Error,
     },
 };
 
@@ -143,23 +143,14 @@ fn lex_string_escape_bs() {
 #[test]
 fn lex_string_bad_escape() {
     let wth = br#" '\a should be an alert on windows apparently' "#;
-    assert_eq!(
-        lex_insecure(wth).unwrap_err(),
-        LexError::InvalidStringLiteral
-    );
+    assert_eq!(lex_insecure(wth).unwrap_err(), Error::LexInvalidLiteral);
 }
 #[test]
 fn lex_string_unclosed() {
     let wth = br#" 'omg where did the end go "#;
-    assert_eq!(
-        lex_insecure(wth).unwrap_err(),
-        LexError::InvalidStringLiteral
-    );
+    assert_eq!(lex_insecure(wth).unwrap_err(), Error::LexInvalidLiteral);
     let wth = br#" 'see, we escaped the end\' "#;
-    assert_eq!(
-        lex_insecure(wth).unwrap_err(),
-        LexError::InvalidStringLiteral
-    );
+    assert_eq!(lex_insecure(wth).unwrap_err(), Error::LexInvalidLiteral);
 }
 #[test]
 fn lex_unsafe_literal_mini() {
diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
index 1199d4d4..175d6311 100644
--- a/server/src/engine/txn/gns/tests/full_chain.rs
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -30,7 +30,7 @@ use crate::engine::{
         space::{Space, SpaceMeta},
     },
     data::{cell::Datacell, tag::TagSelector, uuid::Uuid, DictEntryGeneric},
-    error::DatabaseError,
+    error::Error,
     fractal::{test_utils::TestGlobal, GlobalInstanceLike},
     idx::STIndex,
     ql::{
@@ -318,7 +318,7 @@ fn drop_model() {
                     .namespace()
                     .with_model(("myspace", "mymodel"), |_| { Ok(()) })
                     .unwrap_err(),
-                DatabaseError::DdlModelNotFound
+                Error::QPObjectNotFound
             );
         })
     })

From 9e9a7b9c9a37fbf04ea6994f0fc8a696deb9b4e8 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 17 Sep 2023 13:14:57 +0000
Subject: [PATCH 252/310] Clean up enum methods

---
 server/src/engine/core/dml/upd.rs             |  4 +-
 server/src/engine/core/model/mod.rs           | 14 +--
 server/src/engine/core/query_meta.rs          | 22 +----
 server/src/engine/data/tag.rs                 | 29 ++-----
 server/src/engine/mem/buf.rs                  | 85 +++++++++++++++++++
 server/src/engine/mem/mod.rs                  |  2 +
 .../engine/storage/v1/batch_jrnl/persist.rs   |  2 +-
 server/src/engine/storage/v1/inf/map.rs       |  3 +-
 server/src/engine/storage/v1/inf/mod.rs       | 15 ++--
 server/src/engine/storage/v1/inf/obj.rs       | 19 ++---
 server/src/engine/storage/v1/mod.rs           |  2 +-
 server/src/engine/storage/v1/rw.rs            | 58 -------------
 server/src/engine/txn/gns/mod.rs              |  4 +-
 server/src/engine/txn/gns/model.rs            |  3 +-
 server/src/engine/txn/gns/space.rs            |  3 +-
 sky-macros/src/lib.rs                         | 29 ++++++-
 16 files changed, 159 insertions(+), 135 deletions(-)
 create mode 100644 server/src/engine/mem/buf.rs

diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index cecd074c..3586168f 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -167,7 +167,7 @@ unsafe fn dc_op_str_add(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
 }
 
 static OPERATOR: [unsafe fn(&Datacell, LitIR) -> (bool, Datacell); {
-    TagClass::max() * (AssignmentOperator::max() + 1)
+    TagClass::MAX as usize * AssignmentOperator::VARIANTS
 }] = [
     // bool
     dc_op_bool_ass,
@@ -212,7 +212,7 @@ static OPERATOR: [unsafe fn(&Datacell, LitIR) -> (bool, Datacell); {
 
 #[inline(always)]
 const fn opc(opr: TagClass, ope: AssignmentOperator) -> usize {
-    (AssignmentOperator::count() * opr.word()) + ope.word()
+    (AssignmentOperator::VARIANTS * opr.value_word()) + ope.value_word()
 }
 
 #[cfg(test)]
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index e1ed2dd0..405e0641 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -345,7 +345,7 @@ impl Field {
             // illegal states: (1) bad null (2) tags don't match
             7
         } else {
-            dc.kind().word()
+            dc.kind().value_word()
         }
     }
     pub fn validate_data_fpath(&self, data: &Datacell) -> bool {
@@ -368,7 +368,7 @@ impl Field {
             (TagClass::List, TagClass::List) if !layers.is_empty() => {
                 let mut okay = unsafe {
                     // UNSAFE(@ohsayan): we've verified this
-                    LVERIFY[TagClass::List.word()](layer, data)
+                    LVERIFY[TagClass::List.value_word()](layer, data)
                 };
                 let list = unsafe {
                     // UNSAFE(@ohsayan): we verified tags
@@ -385,7 +385,7 @@ impl Field {
             (tag_a, tag_b) if tag_a == tag_b => {
                 unsafe {
                     // UNSAFE(@ohsayan): same tags; not-null for now so no extra handling required here
-                    LVERIFY[tag_a.word()](layer, data)
+                    LVERIFY[tag_a.value_word()](layer, data)
                 }
             }
             _ => false,
@@ -452,7 +452,7 @@ impl Layer {
     }
     #[inline(always)]
     fn compute_index(&self, dc: &Datacell) -> usize {
-        self.tag.tag_class().word() * (dc.is_null() as usize)
+        self.tag.tag_class().value_word() * (dc.is_null() as usize)
     }
     const fn new(tag: FullTag) -> Self {
         Self { tag }
@@ -523,7 +523,7 @@ unsafe fn lverify_bool(_: Layer, _: &Datacell) -> bool {
 unsafe fn lverify_uint(l: Layer, d: &Datacell) -> bool {
     layertrace("uint");
     const MX: [u64; 4] = [u8::MAX as _, u16::MAX as _, u32::MAX as _, u64::MAX];
-    d.read_uint() <= MX[l.tag.tag_selector().word() - 1]
+    d.read_uint() <= MX[l.tag.tag_selector().value_word() - 1]
 }
 unsafe fn lverify_sint(l: Layer, d: &Datacell) -> bool {
     layertrace("sint");
@@ -533,13 +533,13 @@ unsafe fn lverify_sint(l: Layer, d: &Datacell) -> bool {
         (i32::MIN as _, i32::MAX as _),
         (i64::MIN, i64::MAX),
     ];
-    let (mn, mx) = MN_MX[l.tag.tag_selector().word() - 5];
+    let (mn, mx) = MN_MX[l.tag.tag_selector().value_word() - 5];
     (d.read_sint() >= mn) & (d.read_sint() <= mx)
 }
 unsafe fn lverify_float(l: Layer, d: &Datacell) -> bool {
     layertrace("float");
     const MN_MX: [(f64, f64); 2] = [(f32::MIN as _, f32::MAX as _), (f64::MIN, f64::MAX)];
-    let (mn, mx) = MN_MX[l.tag.tag_selector().word() - 9];
+    let (mn, mx) = MN_MX[l.tag.tag_selector().value_word() - 9];
     (d.read_float() >= mn) & (d.read_float() <= mx)
 }
 unsafe fn lverify_bin(_: Layer, _: &Datacell) -> bool {
diff --git a/server/src/engine/core/query_meta.rs b/server/src/engine/core/query_meta.rs
index 7c429cba..c85b1fe7 100644
--- a/server/src/engine/core/query_meta.rs
+++ b/server/src/engine/core/query_meta.rs
@@ -24,9 +24,7 @@
  *
 */
 
-use std::mem;
-
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, sky_macros::EnumMethods)]
 #[repr(u8)]
 pub enum AssignmentOperator {
     Assign = 0,
@@ -35,21 +33,3 @@ pub enum AssignmentOperator {
     MulAssign = 3,
     DivAssign = 4,
 }
-
-impl AssignmentOperator {
-    pub const fn disc(&self) -> u8 {
-        unsafe {
-            // UNSAFE(@ohsayan): just go back to school already; dscr
-            mem::transmute(*self)
-        }
-    }
-    pub const fn max() -> usize {
-        Self::DivAssign.disc() as _
-    }
-    pub const fn word(&self) -> usize {
-        self.disc() as _
-    }
-    pub const fn count() -> usize {
-        5
-    }
-}
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
index 00bee7c8..d8694b5f 100644
--- a/server/src/engine/data/tag.rs
+++ b/server/src/engine/data/tag.rs
@@ -25,7 +25,7 @@
 */
 
 #[repr(u8)]
-#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, sky_macros::EnumMethods)]
 pub enum TagClass {
     Bool = 0,
     UnsignedInt = 1,
@@ -37,12 +37,8 @@ pub enum TagClass {
 }
 
 impl TagClass {
-    /// ☢WARNING☢: Don't forget offset
-    pub const fn max() -> usize {
-        Self::List.d() as _
-    }
     pub const fn try_from_raw(v: u8) -> Option<Self> {
-        if v > Self::List.d() {
+        if v > Self::MAX {
             return None;
         }
         Some(unsafe { Self::from_raw(v) })
@@ -59,12 +55,12 @@ impl TagClass {
             TagUnique::Bin,
             TagUnique::Str,
             TagUnique::Illegal,
-        ][self.d() as usize]
+        ][self.value_word()]
     }
 }
 
 #[repr(u8)]
-#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, sky_macros::EnumMethods)]
 pub enum TagSelector {
     Bool = 0,
     UInt8 = 1,
@@ -83,9 +79,6 @@ pub enum TagSelector {
 }
 
 impl TagSelector {
-    pub const fn max_dscr() -> u8 {
-        Self::List.d()
-    }
     pub const fn into_full(self) -> FullTag {
         FullTag::new(self.tag_class(), self, self.tag_unique())
     }
@@ -108,7 +101,7 @@ impl TagSelector {
             TagUnique::Bin,
             TagUnique::Str,
             TagUnique::Illegal,
-        ][self.d() as usize]
+        ][self.value_word()]
     }
     pub const fn tag_class(&self) -> TagClass {
         [
@@ -126,12 +119,12 @@ impl TagSelector {
             TagClass::Bin,
             TagClass::Str,
             TagClass::List,
-        ][self.d() as usize]
+        ][self.value_word()]
     }
 }
 
 #[repr(u8)]
-#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, sky_macros::EnumMethods)]
 pub enum TagUnique {
     UnsignedInt = 0,
     SignedInt = 1,
@@ -142,7 +135,7 @@ pub enum TagUnique {
 
 impl TagUnique {
     pub const fn is_unique(&self) -> bool {
-        self.d() != Self::Illegal.d()
+        self.value_u8() != Self::Illegal.value_u8()
     }
     pub const fn try_from_raw(raw: u8) -> Option<Self> {
         if raw > 3 {
@@ -152,12 +145,6 @@ impl TagUnique {
     }
 }
 
-macro_rules! d {
-    ($($ty:ty),*) => {$(impl $ty { pub const fn d(&self) -> u8 {unsafe{::core::mem::transmute(*self)}} pub const fn word(&self) -> usize {Self::d(self) as usize} } )*}
-}
-
-d!(TagClass, TagSelector, TagUnique);
-
 pub trait DataTag {
     const BOOL: Self;
     const UINT: Self;
diff --git a/server/src/engine/mem/buf.rs b/server/src/engine/mem/buf.rs
new file mode 100644
index 00000000..0909c4a4
--- /dev/null
+++ b/server/src/engine/mem/buf.rs
@@ -0,0 +1,85 @@
+/*
+ * Created on Fri Sep 15 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use core::{ptr, slice};
+
+#[derive(Debug)]
+pub struct BufferedScanner<'a> {
+    d: &'a [u8],
+    i: usize,
+}
+
+impl<'a> BufferedScanner<'a> {
+    pub const fn new(d: &'a [u8]) -> Self {
+        Self { d, i: 0 }
+    }
+    pub const fn remaining(&self) -> usize {
+        self.d.len() - self.i
+    }
+    pub const fn consumed(&self) -> usize {
+        self.i
+    }
+    pub const fn cursor(&self) -> usize {
+        self.i
+    }
+    pub(crate) fn has_left(&self, sizeof: usize) -> bool {
+        self.remaining() >= sizeof
+    }
+    unsafe fn _cursor(&self) -> *const u8 {
+        self.d.as_ptr().add(self.i)
+    }
+    pub fn eof(&self) -> bool {
+        self.remaining() == 0
+    }
+    unsafe fn _incr(&mut self, by: usize) {
+        self.i += by;
+    }
+    pub fn current(&self) -> &[u8] {
+        &self.d[self.i..]
+    }
+}
+
+impl<'a> BufferedScanner<'a> {
+    pub unsafe fn next_u64_le(&mut self) -> u64 {
+        u64::from_le_bytes(self.next_chunk())
+    }
+    pub unsafe fn next_chunk<const N: usize>(&mut self) -> [u8; N] {
+        let mut b = [0u8; N];
+        ptr::copy_nonoverlapping(self._cursor(), b.as_mut_ptr(), N);
+        self._incr(N);
+        b
+    }
+    pub unsafe fn next_chunk_variable(&mut self, size: usize) -> &[u8] {
+        let r = slice::from_raw_parts(self._cursor(), size);
+        self._incr(size);
+        r
+    }
+    pub unsafe fn next_byte(&mut self) -> u8 {
+        let r = *self._cursor();
+        self._incr(1);
+        r
+    }
+}
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index 68a429a3..5b150612 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -25,6 +25,7 @@
 */
 
 mod astr;
+mod buf;
 mod ll;
 mod stackop;
 mod uarray;
@@ -36,6 +37,7 @@ mod tests;
 // re-exports
 pub use {
     astr::AStr,
+    buf::BufferedScanner,
     ll::CachePadded,
     stackop::ByteStack,
     uarray::UArray,
diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs
index 56499f92..f47ee07f 100644
--- a/server/src/engine/storage/v1/batch_jrnl/persist.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs
@@ -156,7 +156,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         col_cnt: usize,
     ) -> Result<(), SDSSError> {
         self.f
-            .unfsynced_write(&[MARKER_ACTUAL_BATCH_EVENT, pk_tag.d()])?;
+            .unfsynced_write(&[MARKER_ACTUAL_BATCH_EVENT, pk_tag.value_u8()])?;
         let observed_len_bytes = observed_len.u64_bytes_le();
         self.f.unfsynced_write(&observed_len_bytes)?;
         self.f
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index 17e9485a..380d5ca2 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -39,7 +39,8 @@ use {
                 DictGeneric,
             },
             idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
-            storage::v1::{inf, rw::BufferedScanner, SDSSError, SDSSResult},
+            mem::BufferedScanner,
+            storage::v1::{inf, SDSSError, SDSSResult},
         },
         util::{copy_slice_to_array as memcpy, EndianQW},
     },
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index 3a0769ea..afa17003 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -41,7 +41,8 @@ use {
             tag::{DataTag, TagClass},
         },
         idx::{AsKey, AsValue},
-        storage::v1::{rw::BufferedScanner, SDSSError, SDSSResult},
+        mem::BufferedScanner,
+        storage::v1::{SDSSError, SDSSResult},
     },
     std::mem,
 };
@@ -64,13 +65,12 @@ pub enum PersistTypeDscr {
 }
 
 impl PersistTypeDscr {
-    pub(super) const MAX: Self = Self::Dict;
     /// translates the tag class definition into the dscr definition
     pub const fn translate_from_class(class: TagClass) -> Self {
-        unsafe { Self::from_raw(class.d() + 1) }
+        unsafe { Self::from_raw(class.value_u8() + 1) }
     }
     pub const fn try_from_raw(v: u8) -> Option<Self> {
-        if v > Self::MAX.value_u8() {
+        if v > Self::MAX {
             None
         } else {
             unsafe { Some(Self::from_raw(v)) }
@@ -262,7 +262,7 @@ pub mod enc {
 pub mod dec {
     use {
         super::{map, PersistMapSpec, PersistObject},
-        crate::engine::storage::v1::{rw::BufferedScanner, SDSSResult},
+        crate::engine::{mem::BufferedScanner, storage::v1::SDSSResult},
     };
     // obj
     pub fn dec_full<Obj: PersistObject>(data: &[u8]) -> SDSSResult<Obj::OutputType> {
@@ -288,7 +288,10 @@ pub mod dec {
         <map::PersistMapImpl<PM> as PersistObject>::default_full_dec(scanner)
     }
     pub mod utils {
-        use crate::engine::storage::v1::{BufferedScanner, SDSSError, SDSSResult};
+        use crate::engine::{
+            mem::BufferedScanner,
+            storage::v1::{SDSSError, SDSSResult},
+        };
         pub unsafe fn decode_string(s: &mut BufferedScanner, len: usize) -> SDSSResult<String> {
             String::from_utf8(s.next_chunk_variable(len).to_owned())
                 .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 18cdd5bf..01afe7fd 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -24,23 +24,22 @@
  *
 */
 
-use crate::engine::{core::model::delta::IRModel, data::DictGeneric};
-
 use {
-    super::{PersistTypeDscr, PersistObject, VecU8},
+    super::{PersistObject, PersistTypeDscr, VecU8},
     crate::{
         engine::{
             core::{
-                model::{Field, Layer, Model},
+                model::{delta::IRModel, Field, Layer, Model},
                 space::{Space, SpaceMeta},
             },
             data::{
                 cell::Datacell,
                 tag::{DataTag, TagClass, TagSelector},
                 uuid::Uuid,
+                DictGeneric,
             },
-            mem::VInline,
-            storage::v1::{inf, rw::BufferedScanner, SDSSError, SDSSResult},
+            mem::{BufferedScanner, VInline},
+            storage::v1::{inf, SDSSError, SDSSResult},
         },
         util::EndianQW,
     },
@@ -111,7 +110,7 @@ impl<'a> PersistObject for LayerRef<'a> {
         true
     }
     fn meta_enc(buf: &mut VecU8, LayerRef(layer): Self::InputType) {
-        buf.extend(layer.tag().tag_selector().d().u64_bytes_le());
+        buf.extend(layer.tag().tag_selector().value_qword().to_le_bytes());
         buf.extend(0u64.to_le_bytes());
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
@@ -119,7 +118,7 @@ impl<'a> PersistObject for LayerRef<'a> {
     }
     fn obj_enc(_: &mut VecU8, _: Self::InputType) {}
     unsafe fn obj_dec(_: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
-        if (md.type_selector > TagSelector::List.d() as u64) | (md.prop_set_arity != 0) {
+        if (md.type_selector > TagSelector::List.value_qword()) | (md.prop_set_arity != 0) {
             return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
         }
         Ok(Layer::new_empty_props(
@@ -253,7 +252,7 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
     fn meta_enc(buf: &mut VecU8, ModelLayoutRef(v, irm): Self::InputType) {
         buf.extend(v.get_uuid().to_le_bytes());
         buf.extend(v.p_key().len().u64_bytes_le());
-        buf.extend(v.p_tag().tag_selector().d().u64_bytes_le());
+        buf.extend(v.p_tag().tag_selector().value_qword().to_le_bytes());
         buf.extend(irm.fields().len().u64_bytes_le());
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
@@ -281,7 +280,7 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
                 scanner,
                 super::map::MapIndexSizeMD(md.field_c as usize),
             )?;
-        let ptag = if md.p_key_tag > TagSelector::max_dscr() as u64 {
+        let ptag = if md.p_key_tag > TagSelector::MAX as u64 {
             return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
         } else {
             TagSelector::from_raw(md.p_key_tag as u8)
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index ca060411..555b8239 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -43,7 +43,7 @@ mod tests;
 pub use {
     journal::{open_journal, JournalAdapter, JournalWriter},
     memfs::NullFS,
-    rw::{BufferedScanner, LocalFS, RawFSInterface, SDSSFileIO},
+    rw::{LocalFS, RawFSInterface, SDSSFileIO},
 };
 pub mod data_batch {
     pub use super::batch_jrnl::{DataBatchPersistDriver, DataBatchRestoreDriver};
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 6319ba60..09406b5f 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -40,7 +40,6 @@ use {
     std::{
         fs::{self, File},
         io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write},
-        ptr, slice,
     },
 };
 
@@ -439,60 +438,3 @@ impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
         self.read_to_buffer(&mut r).map(|_| r[0])
     }
 }
-
-pub struct BufferedScanner<'a> {
-    d: &'a [u8],
-    i: usize,
-}
-
-impl<'a> BufferedScanner<'a> {
-    pub const fn new(d: &'a [u8]) -> Self {
-        Self { d, i: 0 }
-    }
-    pub const fn remaining(&self) -> usize {
-        self.d.len() - self.i
-    }
-    pub const fn consumed(&self) -> usize {
-        self.i
-    }
-    pub const fn cursor(&self) -> usize {
-        self.i
-    }
-    pub(crate) fn has_left(&self, sizeof: usize) -> bool {
-        self.remaining() >= sizeof
-    }
-    unsafe fn _cursor(&self) -> *const u8 {
-        self.d.as_ptr().add(self.i)
-    }
-    pub fn eof(&self) -> bool {
-        self.remaining() == 0
-    }
-    unsafe fn _incr(&mut self, by: usize) {
-        self.i += by;
-    }
-    pub fn current(&self) -> &[u8] {
-        &self.d[self.i..]
-    }
-}
-
-impl<'a> BufferedScanner<'a> {
-    pub unsafe fn next_u64_le(&mut self) -> u64 {
-        u64::from_le_bytes(self.next_chunk())
-    }
-    pub unsafe fn next_chunk<const N: usize>(&mut self) -> [u8; N] {
-        let mut b = [0u8; N];
-        ptr::copy_nonoverlapping(self._cursor(), b.as_mut_ptr(), N);
-        self._incr(N);
-        b
-    }
-    pub unsafe fn next_chunk_variable(&mut self, size: usize) -> &[u8] {
-        let r = slice::from_raw_parts(self._cursor(), size);
-        self._incr(size);
-        r
-    }
-    pub unsafe fn next_byte(&mut self) -> u8 {
-        let r = *self._cursor();
-        self._incr(1);
-        r
-    }
-}
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index e7b7e5ad..85905315 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -32,11 +32,11 @@ use {
         engine::{
             core::{space::Space, GlobalNS},
             data::uuid::Uuid,
+            mem::BufferedScanner,
             storage::v1::{
                 self, header_meta,
                 inf::{self, PersistObject},
-                BufferedScanner, JournalAdapter, JournalWriter, LocalFS, RawFSInterface,
-                SDSSResult,
+                JournalAdapter, JournalWriter, LocalFS, RawFSInterface, SDSSResult,
             },
         },
         util::EndianQW,
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index 0fe37564..a31b28ac 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -35,10 +35,11 @@ use {
             },
             data::uuid::Uuid,
             idx::{IndexST, IndexSTSeqCns, STIndex, STIndexSeq},
+            mem::BufferedScanner,
             ql::lex::Ident,
             storage::v1::{
                 inf::{self, map, obj, PersistObject},
-                BufferedScanner, SDSSError, SDSSResult,
+                SDSSError, SDSSResult,
             },
             txn::TransactionError,
         },
diff --git a/server/src/engine/txn/gns/space.rs b/server/src/engine/txn/gns/space.rs
index 066c2929..04bfe874 100644
--- a/server/src/engine/txn/gns/space.rs
+++ b/server/src/engine/txn/gns/space.rs
@@ -31,9 +31,10 @@ use {
             core::{space::Space, GlobalNS},
             data::DictGeneric,
             idx::STIndex,
+            mem::BufferedScanner,
             storage::v1::{
                 inf::{self, map, obj, PersistObject},
-                BufferedScanner, SDSSResult,
+                SDSSResult,
             },
             txn::{TransactionError, TransactionResult},
         },
diff --git a/sky-macros/src/lib.rs b/sky-macros/src/lib.rs
index 0b69f98d..ad2356fe 100644
--- a/sky-macros/src/lib.rs
+++ b/sky-macros/src/lib.rs
@@ -243,14 +243,17 @@ pub fn derive_value_methods(input: TokenStream) -> TokenStream {
         }
     }
     let repr_type = repr_type.expect("Must have repr(u8) or repr(u16) etc.");
+    let mut dscr_expressions = vec![];
     // Ensure all variants have explicit discriminants
     if let Data::Enum(data) = &ast.data {
         for variant in &data.variants {
             match &variant.fields {
                 Fields::Unit => {
-                    if variant.discriminant.as_ref().is_none() {
-                        panic!("All enum variants must have explicit discriminants");
-                    }
+                    let (_, dscr_expr) = variant
+                        .discriminant
+                        .as_ref()
+                        .expect("All enum variants must have explicit discriminants");
+                    dscr_expressions.push(dscr_expr.clone());
                 }
                 _ => panic!("All enum variants must be unit variants"),
             }
@@ -259,6 +262,12 @@ pub fn derive_value_methods(input: TokenStream) -> TokenStream {
         panic!("This derive macro only works on enums");
     }
 
+    let value_expressions = quote! {
+        [#(#dscr_expressions),*]
+    };
+
+    let variant_len = dscr_expressions.len();
+
     let repr_type_ident = syn::Ident::new(&repr_type, proc_macro2::Span::call_site());
     let repr_type_ident_func = syn::Ident::new(
         &format!("value_{repr_type}"),
@@ -267,9 +276,23 @@ pub fn derive_value_methods(input: TokenStream) -> TokenStream {
 
     let gen = quote! {
         impl #enum_name {
+            pub const MAX: #repr_type_ident = Self::max_value();
+            pub const VARIANTS: usize = #variant_len;
             pub const fn #repr_type_ident_func(&self) -> #repr_type_ident { unsafe { core::mem::transmute(*self) } }
             pub const fn value_word(&self) -> usize { self.#repr_type_ident_func() as usize }
             pub const fn value_qword(&self) -> u64 { self.#repr_type_ident_func() as u64 }
+            pub const fn max_value() -> #repr_type_ident {
+                let values = #value_expressions;
+                let mut i = 1;
+                let mut max = values[0];
+                while i < values.len() {
+                    if values[i] > max {
+                        max = values[i];
+                    }
+                    i = i + 1;
+                }
+                max
+            }
         }
     };
     gen.into()

From 91514d42194a1d308208926b2ced8e3b22ea85a6 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 18 Sep 2023 09:35:36 +0000
Subject: [PATCH 253/310] Implement protocol stage 1

---
 Cargo.toml                                  |   1 +
 server/src/engine/mem/buf.rs                | 127 +++++-
 server/src/engine/mem/mod.rs                |   2 +-
 server/src/engine/mod.rs                    |   1 +
 server/src/engine/net/mod.rs                |  27 ++
 server/src/engine/net/protocol/handshake.rs | 406 ++++++++++++++++++++
 server/src/engine/net/protocol/mod.rs       |  29 ++
 server/src/engine/net/protocol/tests.rs     | 197 ++++++++++
 server/src/engine/ql/dml/upd.rs             |   2 +-
 9 files changed, 777 insertions(+), 15 deletions(-)
 create mode 100644 server/src/engine/net/mod.rs
 create mode 100644 server/src/engine/net/protocol/handshake.rs
 create mode 100644 server/src/engine/net/protocol/mod.rs
 create mode 100644 server/src/engine/net/protocol/tests.rs

diff --git a/Cargo.toml b/Cargo.toml
index d5b6c93c..f3b41279 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,4 +1,5 @@
 [workspace]
+resolver = "2"
 members = [
     "cli",
     "server",
diff --git a/server/src/engine/mem/buf.rs b/server/src/engine/mem/buf.rs
index 0909c4a4..cc4d34ab 100644
--- a/server/src/engine/mem/buf.rs
+++ b/server/src/engine/mem/buf.rs
@@ -29,36 +29,137 @@ use core::{ptr, slice};
 #[derive(Debug)]
 pub struct BufferedScanner<'a> {
     d: &'a [u8],
-    i: usize,
+    __cursor: usize,
 }
 
 impl<'a> BufferedScanner<'a> {
     pub const fn new(d: &'a [u8]) -> Self {
-        Self { d, i: 0 }
+        unsafe { Self::new_with_cursor(d, 0) }
+    }
+    pub const unsafe fn new_with_cursor(d: &'a [u8], i: usize) -> Self {
+        Self { d, __cursor: i }
     }
     pub const fn remaining(&self) -> usize {
-        self.d.len() - self.i
+        self.d.len() - self.__cursor
     }
     pub const fn consumed(&self) -> usize {
-        self.i
+        self.__cursor
     }
     pub const fn cursor(&self) -> usize {
-        self.i
-    }
-    pub(crate) fn has_left(&self, sizeof: usize) -> bool {
-        self.remaining() >= sizeof
+        self.__cursor
     }
-    unsafe fn _cursor(&self) -> *const u8 {
-        self.d.as_ptr().add(self.i)
+    pub fn current(&self) -> &[u8] {
+        &self.d[self.__cursor..]
     }
     pub fn eof(&self) -> bool {
         self.remaining() == 0
     }
+    pub fn has_left(&self, sizeof: usize) -> bool {
+        self.remaining() >= sizeof
+    }
+    pub fn matches_cursor_rounded(&self, f: impl Fn(u8) -> bool) -> bool {
+        f(self.d[self.d.len().min(self.__cursor)])
+    }
+    pub fn matches_cursor_rounded_and_not_eof(&self, f: impl Fn(u8) -> bool) -> bool {
+        self.matches_cursor_rounded(f) & !self.eof()
+    }
+}
+
+impl<'a> BufferedScanner<'a> {
+    pub unsafe fn set_cursor(&mut self, i: usize) {
+        self.__cursor = i;
+    }
+    pub unsafe fn move_ahead_by(&mut self, by: usize) {
+        self._incr(by)
+    }
+    pub unsafe fn move_back(&mut self) {
+        self.move_back_by(1)
+    }
+    pub unsafe fn move_back_by(&mut self, by: usize) {
+        self.__cursor -= by;
+    }
     unsafe fn _incr(&mut self, by: usize) {
-        self.i += by;
+        self.__cursor += by;
     }
-    pub fn current(&self) -> &[u8] {
-        &self.d[self.i..]
+    unsafe fn _cursor(&self) -> *const u8 {
+        self.d.as_ptr().add(self.__cursor)
+    }
+}
+
+impl<'a> BufferedScanner<'a> {
+    pub fn try_next_byte(&mut self) -> Option<u8> {
+        if self.eof() {
+            None
+        } else {
+            Some(unsafe { self.next_byte() })
+        }
+    }
+    pub fn try_next_block<const N: usize>(&mut self) -> Option<[u8; N]> {
+        if self.has_left(N) {
+            Some(unsafe { self.next_chunk() })
+        } else {
+            None
+        }
+    }
+    pub fn try_next_variable_block(&'a mut self, len: usize) -> Option<&'a [u8]> {
+        if self.has_left(len) {
+            Some(unsafe { self.next_chunk_variable(len) })
+        } else {
+            None
+        }
+    }
+}
+
+pub enum BufferedReadResult<T> {
+    Value(T),
+    NeedMore,
+    Error,
+}
+
+impl<'a> BufferedScanner<'a> {
+    /// Attempt to parse a `\n` terminated (we move past the LF, so you can't see it)
+    ///
+    /// If we were unable to read in the integer, then the cursor will be restored to its starting position
+    // TODO(@ohsayan): optimize
+    pub fn try_next_ascii_u64_lf_separated(&mut self) -> BufferedReadResult<u64> {
+        let mut okay = true;
+        let start = self.cursor();
+        let mut ret = 0u64;
+        while self.matches_cursor_rounded_and_not_eof(|b| b != b'\n') & okay {
+            let b = self.d[self.cursor()];
+            okay &= b.is_ascii_digit();
+            ret = match ret.checked_mul(10) {
+                Some(r) => r,
+                None => {
+                    okay = false;
+                    break;
+                }
+            };
+            ret = match ret.checked_add((b & 0x0F) as u64) {
+                Some(r) => r,
+                None => {
+                    okay = false;
+                    break;
+                }
+            };
+            unsafe { self._incr(1) }
+        }
+        let payload_ok = okay;
+        let null_ok = self.matches_cursor_rounded_and_not_eof(|b| b == b'\n');
+        okay &= null_ok;
+        unsafe { self._incr(okay as _) }; // skip LF
+        if okay {
+            BufferedReadResult::Value(ret)
+        } else {
+            unsafe { self.set_cursor(start) }
+            if payload_ok {
+                // payload was ok, but we missed a null
+                BufferedReadResult::NeedMore
+            } else {
+                // payload was NOT ok
+                BufferedReadResult::Error
+            }
+        }
     }
 }
 
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index 5b150612..6e1d6e87 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -25,7 +25,7 @@
 */
 
 mod astr;
-mod buf;
+pub mod buf;
 mod ll;
 mod stackop;
 mod uarray;
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 4d43a88f..c4646524 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -34,6 +34,7 @@ mod error;
 mod fractal;
 mod idx;
 mod mem;
+mod net;
 mod ql;
 mod storage;
 mod sync;
diff --git a/server/src/engine/net/mod.rs b/server/src/engine/net/mod.rs
new file mode 100644
index 00000000..d5c17871
--- /dev/null
+++ b/server/src/engine/net/mod.rs
@@ -0,0 +1,27 @@
+/*
+ * Created on Fri Sep 15 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod protocol;
diff --git a/server/src/engine/net/protocol/handshake.rs b/server/src/engine/net/protocol/handshake.rs
new file mode 100644
index 00000000..4400b9c3
--- /dev/null
+++ b/server/src/engine/net/protocol/handshake.rs
@@ -0,0 +1,406 @@
+/*
+ * Created on Mon Sep 18 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::{
+        engine::mem::buf::{BufferedReadResult, BufferedScanner},
+        util::compiler,
+    },
+    std::slice,
+};
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, sky_macros::EnumMethods)]
+#[repr(u8)]
+/// Low-level protocol errors
+pub enum ProtocolError {
+    /// packet has incorrect structure
+    CorruptedHSPacket = 0,
+    /// incorrect handshake version
+    RejectHSVersion = 1,
+    /// invalid protocol version
+    RejectProtocol = 2,
+    /// invalid exchange mode
+    RejectExchangeMode = 3,
+    /// invalid query mode
+    RejectQueryMode = 4,
+    /// invalid auth details
+    ///
+    /// **NB**: this can be due to either an incorrect auth flag, or incorrect auth data or disallowed auth mode. we keep it
+    /// in one error for purposes of security
+    RejectAuth = 5,
+}
+
+/*
+    handshake meta
+*/
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, sky_macros::EnumMethods)]
+#[repr(u8)]
+/// the handshake version
+pub enum HandshakeVersion {
+    /// Skyhash/2.0 HS
+    Original = 0,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, sky_macros::EnumMethods)]
+#[repr(u8)]
+/// the skyhash protocol version
+pub enum ProtocolVersion {
+    /// Skyhash/2.0 protocol
+    Original = 0,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, sky_macros::EnumMethods)]
+#[repr(u8)]
+/// the data exchange mode
+pub enum DataExchangeMode {
+    /// query-time data exchange mode
+    QueryTime = 0,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, sky_macros::EnumMethods)]
+#[repr(u8)]
+/// the query mode
+pub enum QueryMode {
+    /// BQL-1 query mode
+    Bql1 = 0,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, sky_macros::EnumMethods)]
+#[repr(u8)]
+/// the authentication mode
+pub enum AuthMode {
+    Anonymous = 0,
+    Password = 1,
+}
+
+impl AuthMode {
+    unsafe fn from_raw(v: u8) -> Self {
+        core::mem::transmute(v)
+    }
+    /// returns the minimum number of metadata bytes need to parse the payload for this auth mode
+    const fn min_payload_bytes(&self) -> usize {
+        match self {
+            Self::Anonymous => 1,
+            Self::Password => 4,
+        }
+    }
+}
+
+/*
+    client handshake
+*/
+
+/// The handshake state
+#[derive(Debug, PartialEq, Clone)]
+pub enum HandshakeState {
+    /// we just began the handshake
+    Initial,
+    /// we just processed the static block
+    StaticBlock(CHandshakeStatic),
+    /// Expecting some more auth meta
+    ExpectingMetaForVariableBlock {
+        /// static block
+        static_hs: CHandshakeStatic,
+        /// uname len
+        uname_l: usize,
+    },
+    /// we're expecting to finish the handshake
+    ExpectingVariableBlock {
+        /// static block
+        static_hs: CHandshakeStatic,
+        /// uname len
+        uname_l: usize,
+        /// pwd len
+        pwd_l: usize,
+    },
+}
+
+impl Default for HandshakeState {
+    fn default() -> Self {
+        Self::Initial
+    }
+}
+
+/// The static segment of the handshake
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct CHandshakeStatic {
+    /// the handshake version
+    hs_version: HandshakeVersion,
+    /// protocol version
+    protocol: ProtocolVersion,
+    /// exchange mode
+    exchange_mode: DataExchangeMode,
+    /// query mode
+    query_mode: QueryMode,
+    /// authentication mode
+    auth_mode: AuthMode,
+}
+
+impl CHandshakeStatic {
+    pub const fn new(
+        hs_version: HandshakeVersion,
+        protocol: ProtocolVersion,
+        exchange_mode: DataExchangeMode,
+        query_mode: QueryMode,
+        auth_mode: AuthMode,
+    ) -> Self {
+        Self {
+            hs_version,
+            protocol,
+            exchange_mode,
+            query_mode,
+            auth_mode,
+        }
+    }
+}
+
+/// handshake authentication
+// TODO(@ohsayan): enum?
+#[derive(Debug, PartialEq)]
+pub struct CHandshakeAuth<'a> {
+    username: &'a [u8],
+    password: &'a [u8],
+}
+
+impl<'a> CHandshakeAuth<'a> {
+    pub fn new(username: &'a [u8], password: &'a [u8]) -> Self {
+        Self { username, password }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum HandshakeResult<'a> {
+    /// Finished handshake
+    Completed(CHandshake<'a>),
+    /// Update handshake state
+    ///
+    /// **NOTE:** expect does not take into account the current amount of buffered data (hence the unbuffered part must be computed!)
+    ChangeState {
+        new_state: HandshakeState,
+        expect: usize,
+    },
+    /// An error occurred
+    Error(ProtocolError),
+}
+
+/// The client's handshake record
+#[derive(Debug, PartialEq)]
+pub struct CHandshake<'a> {
+    /// the static segment of the handshake
+    hs_static: CHandshakeStatic,
+    /// the auth section of the dynamic segment of the handshake
+    hs_auth: Option<CHandshakeAuth<'a>>,
+}
+
+impl<'a> CHandshake<'a> {
+    pub const INITIAL_READ: usize = 6;
+    const CLIENT_HELLO: u8 = b'H';
+    pub fn new(hs_static: CHandshakeStatic, hs_auth: Option<CHandshakeAuth<'a>>) -> Self {
+        Self { hs_static, hs_auth }
+    }
+    /// Resume handshake with the given state and buffer
+    pub fn resume_with(
+        scanner: &mut BufferedScanner<'a>,
+        state: HandshakeState,
+    ) -> HandshakeResult<'a> {
+        match state {
+            // nothing buffered yet
+            HandshakeState::Initial => Self::resume_initial(scanner),
+            // buffered static block
+            HandshakeState::StaticBlock(static_block) => {
+                Self::resume_at_auth_metadata1(scanner, static_block)
+            }
+            // buffered some auth meta
+            HandshakeState::ExpectingMetaForVariableBlock { static_hs, uname_l } => {
+                Self::resume_at_auth_metadata2(scanner, static_hs, uname_l)
+            }
+            // buffered full auth meta
+            HandshakeState::ExpectingVariableBlock {
+                static_hs,
+                uname_l,
+                pwd_l,
+            } => Self::resume_at_variable_block_payload(scanner, static_hs, uname_l, pwd_l),
+        }
+    }
+}
+
+impl<'a> CHandshake<'a> {
+    /// Resume from the initial state (nothing buffered yet)
+    fn resume_initial(scanner: &mut BufferedScanner<'a>) -> HandshakeResult<'a> {
+        // get our block
+        if cfg!(debug_assertions) {
+            if scanner.remaining() < Self::INITIAL_READ {
+                return HandshakeResult::ChangeState {
+                    new_state: HandshakeState::Initial,
+                    expect: Self::INITIAL_READ,
+                };
+            }
+        } else {
+            assert!(scanner.remaining() >= Self::INITIAL_READ);
+        }
+        let buf: [u8; CHandshake::INITIAL_READ] = unsafe { scanner.next_chunk() };
+        let invalid_first_byte = buf[0] != Self::CLIENT_HELLO;
+        let invalid_hs_version = buf[1] > HandshakeVersion::MAX;
+        let invalid_proto_version = buf[2] > ProtocolVersion::MAX;
+        let invalid_exchange_mode = buf[3] > DataExchangeMode::MAX;
+        let invalid_query_mode = buf[4] > QueryMode::MAX;
+        let invalid_auth_mode = buf[5] > AuthMode::MAX;
+        // check block
+        if compiler::unlikely(
+            invalid_first_byte
+                | invalid_hs_version
+                | invalid_proto_version
+                | invalid_exchange_mode
+                | invalid_query_mode
+                | invalid_auth_mode,
+        ) {
+            static ERROR: [ProtocolError; 6] = [
+                ProtocolError::CorruptedHSPacket,
+                ProtocolError::RejectHSVersion,
+                ProtocolError::RejectProtocol,
+                ProtocolError::RejectExchangeMode,
+                ProtocolError::RejectQueryMode,
+                ProtocolError::RejectAuth,
+            ];
+            return HandshakeResult::Error(
+                ERROR[((invalid_first_byte as u8 * 1)
+                    | (invalid_hs_version as u8 * 2)
+                    | (invalid_proto_version as u8 * 3)
+                    | (invalid_exchange_mode as u8 * 4)
+                    | (invalid_query_mode as u8 * 5)
+                    | (invalid_auth_mode as u8) * 6) as usize
+                    - 1usize],
+            );
+        }
+        // init header
+        let static_header = CHandshakeStatic::new(
+            HandshakeVersion::Original,
+            ProtocolVersion::Original,
+            DataExchangeMode::QueryTime,
+            QueryMode::Bql1,
+            unsafe {
+                // UNSAFE(@ohsayan): already checked
+                AuthMode::from_raw(buf[5])
+            },
+        );
+        // check if we have auth data
+        Self::resume_at_auth_metadata1(scanner, static_header)
+    }
+    fn resume_at_variable_block_payload(
+        scanner: &mut BufferedScanner<'a>,
+        static_hs: CHandshakeStatic,
+        uname_l: usize,
+        pwd_l: usize,
+    ) -> HandshakeResult<'a> {
+        if scanner.has_left(uname_l + pwd_l) {
+            // we're done here
+            return unsafe {
+                // UNSAFE(@ohsayan): we just checked buffered size
+                let uname = slice::from_raw_parts(scanner.current().as_ptr(), uname_l);
+                let pwd = slice::from_raw_parts(scanner.current().as_ptr().add(uname_l), pwd_l);
+                scanner.move_ahead_by(uname_l + pwd_l);
+                HandshakeResult::Completed(Self::new(
+                    static_hs,
+                    Some(CHandshakeAuth::new(uname, pwd)),
+                ))
+            };
+        }
+        HandshakeResult::ChangeState {
+            new_state: HandshakeState::ExpectingVariableBlock {
+                static_hs,
+                uname_l,
+                pwd_l,
+            },
+            expect: (uname_l + pwd_l),
+        }
+    }
+}
+
+impl<'a> CHandshake<'a> {
+    /// Resume parsing at the first part of the auth metadata
+    fn resume_at_auth_metadata1(
+        scanner: &mut BufferedScanner<'a>,
+        static_header: CHandshakeStatic,
+    ) -> HandshakeResult<'a> {
+        // now let's see if we have buffered enough data for auth
+        if scanner.remaining() < static_header.auth_mode.min_payload_bytes() {
+            // we need more data
+            return HandshakeResult::ChangeState {
+                new_state: HandshakeState::StaticBlock(static_header),
+                expect: static_header.auth_mode.min_payload_bytes(),
+            };
+        }
+        // we seem to have enough data for this auth mode
+        match static_header.auth_mode {
+            AuthMode::Anonymous => {
+                if unsafe { scanner.next_byte() } == 0 {
+                    // matched
+                    return HandshakeResult::Completed(Self::new(static_header, None));
+                }
+                // we can only accept a NUL byte
+                return HandshakeResult::Error(ProtocolError::RejectAuth);
+            }
+            AuthMode::Password => {}
+        }
+        // let us see if we can parse the username length
+        let uname_l = match scanner.try_next_ascii_u64_lf_separated() {
+            BufferedReadResult::NeedMore => {
+                return HandshakeResult::ChangeState {
+                    new_state: HandshakeState::StaticBlock(static_header),
+                    expect: AuthMode::Password.min_payload_bytes(), // 2 for uname_l and 2 for pwd_l
+                };
+            }
+            BufferedReadResult::Value(v) => v as usize,
+            BufferedReadResult::Error => {
+                return HandshakeResult::Error(ProtocolError::CorruptedHSPacket)
+            }
+        };
+        Self::resume_at_auth_metadata2(scanner, static_header, uname_l)
+    }
+    /// Resume at trying to get the final part of the auth metadata
+    fn resume_at_auth_metadata2(
+        scanner: &mut BufferedScanner<'a>,
+        static_hs: CHandshakeStatic,
+        uname_l: usize,
+    ) -> HandshakeResult<'a> {
+        // we just have to get the password len
+        let pwd_l = match scanner.try_next_ascii_u64_lf_separated() {
+            BufferedReadResult::Value(v) => v as usize,
+            BufferedReadResult::NeedMore => {
+                // newline missing (or maybe there's more?)
+                return HandshakeResult::ChangeState {
+                    new_state: HandshakeState::ExpectingMetaForVariableBlock { static_hs, uname_l },
+                    expect: uname_l + 2, // space for username + password len
+                };
+            }
+            BufferedReadResult::Error => {
+                return HandshakeResult::Error(ProtocolError::CorruptedHSPacket)
+            }
+        };
+        Self::resume_at_variable_block_payload(scanner, static_hs, uname_l, pwd_l)
+    }
+}
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
new file mode 100644
index 00000000..ec08551e
--- /dev/null
+++ b/server/src/engine/net/protocol/mod.rs
@@ -0,0 +1,29 @@
+/*
+ * Created on Fri Sep 15 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod handshake;
+#[cfg(test)]
+mod tests;
diff --git a/server/src/engine/net/protocol/tests.rs b/server/src/engine/net/protocol/tests.rs
new file mode 100644
index 00000000..91185cdb
--- /dev/null
+++ b/server/src/engine/net/protocol/tests.rs
@@ -0,0 +1,197 @@
+/*
+ * Created on Mon Sep 18 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::mem::BufferedScanner;
+use crate::engine::net::protocol::handshake::{
+    AuthMode, CHandshake, CHandshakeAuth, CHandshakeStatic, DataExchangeMode, HandshakeResult,
+    HandshakeState, HandshakeVersion, ProtocolVersion, QueryMode,
+};
+
+const FULL_HANDSHAKE_NO_AUTH: [u8; 7] = [b'H', 0, 0, 0, 0, 0, 0];
+const FULL_HANDSHAKE_WITH_AUTH: [u8; 23] = *b"H\0\0\0\0\x015\n8\nsayanpass1234";
+
+const STATIC_HANDSHAKE_NO_AUTH: CHandshakeStatic = CHandshakeStatic::new(
+    HandshakeVersion::Original,
+    ProtocolVersion::Original,
+    DataExchangeMode::QueryTime,
+    QueryMode::Bql1,
+    AuthMode::Anonymous,
+);
+
+const STATIC_HANDSHAKE_WITH_AUTH: CHandshakeStatic = CHandshakeStatic::new(
+    HandshakeVersion::Original,
+    ProtocolVersion::Original,
+    DataExchangeMode::QueryTime,
+    QueryMode::Bql1,
+    AuthMode::Password,
+);
+
+/*
+    handshake with no state changes
+*/
+
+#[test]
+fn parse_staged_no_auth() {
+    for i in 0..FULL_HANDSHAKE_NO_AUTH.len() {
+        let buf = &FULL_HANDSHAKE_NO_AUTH[..i + 1];
+        let mut scanner = BufferedScanner::new(buf);
+        let result = CHandshake::resume_with(&mut scanner, HandshakeState::Initial);
+        match buf.len() {
+            1..=5 => {
+                assert_eq!(
+                    result,
+                    HandshakeResult::ChangeState {
+                        new_state: HandshakeState::Initial,
+                        expect: CHandshake::INITIAL_READ,
+                    }
+                );
+            }
+            6 => {
+                assert_eq!(
+                    result,
+                    HandshakeResult::ChangeState {
+                        new_state: HandshakeState::StaticBlock(STATIC_HANDSHAKE_NO_AUTH),
+                        expect: 1,
+                    }
+                );
+            }
+            7 => {
+                assert_eq!(
+                    result,
+                    HandshakeResult::Completed(CHandshake::new(STATIC_HANDSHAKE_NO_AUTH, None))
+                );
+            }
+            _ => unreachable!(),
+        }
+    }
+}
+
+#[test]
+fn parse_staged_with_auth() {
+    for i in 0..FULL_HANDSHAKE_WITH_AUTH.len() {
+        let buf = &FULL_HANDSHAKE_WITH_AUTH[..i + 1];
+        let mut s = BufferedScanner::new(buf);
+        let ref mut scanner = s;
+        let result = CHandshake::resume_with(scanner, HandshakeState::Initial);
+        match buf.len() {
+            1..=5 => {
+                assert_eq!(
+                    result,
+                    HandshakeResult::ChangeState {
+                        new_state: HandshakeState::Initial,
+                        expect: CHandshake::INITIAL_READ
+                    }
+                );
+            }
+            6..=9 => {
+                // might seem funny that we don't parse the second integer at all, but it's because
+                // of the relatively small size of the integers
+                assert_eq!(
+                    result,
+                    HandshakeResult::ChangeState {
+                        new_state: HandshakeState::StaticBlock(STATIC_HANDSHAKE_WITH_AUTH),
+                        expect: 4
+                    }
+                );
+            }
+            10..=22 => {
+                assert_eq!(
+                    result,
+                    HandshakeResult::ChangeState {
+                        new_state: HandshakeState::ExpectingVariableBlock {
+                            static_hs: STATIC_HANDSHAKE_WITH_AUTH,
+                            uname_l: 5,
+                            pwd_l: 8
+                        },
+                        expect: 13,
+                    }
+                );
+            }
+            23 => {
+                assert_eq!(
+                    result,
+                    HandshakeResult::Completed(CHandshake::new(
+                        STATIC_HANDSHAKE_WITH_AUTH,
+                        Some(CHandshakeAuth::new(b"sayan", b"pass1234"))
+                    ))
+                );
+            }
+            _ => unreachable!(),
+        }
+    }
+}
+
+/*
+    handshake with state changes
+*/
+
+fn run_state_changes_return_rounds(src: &[u8], expected_final_handshake: CHandshake) -> usize {
+    let mut rounds = 0;
+    let hs;
+    let mut state = HandshakeState::default();
+    let mut cursor = 0;
+    let mut expect_many = CHandshake::INITIAL_READ;
+    loop {
+        rounds += 1;
+        let buf = &src[..cursor + expect_many];
+        let mut scanner = unsafe { BufferedScanner::new_with_cursor(buf, cursor) };
+        match CHandshake::resume_with(&mut scanner, state) {
+            HandshakeResult::ChangeState { new_state, expect } => {
+                state = new_state;
+                expect_many = expect;
+                cursor = scanner.cursor();
+            }
+            HandshakeResult::Completed(c) => {
+                hs = c;
+                assert_eq!(hs, expected_final_handshake);
+                break;
+            }
+            HandshakeResult::Error(e) => panic!("unexpected handshake error: {:?}", e),
+        }
+    }
+    rounds
+}
+
+#[test]
+fn parse_no_auth_with_state_updates() {
+    let rounds = run_state_changes_return_rounds(
+        &FULL_HANDSHAKE_NO_AUTH,
+        CHandshake::new(STATIC_HANDSHAKE_NO_AUTH, None),
+    );
+    assert_eq!(rounds, 2); // r1 = initial, r2 = auth NUL
+}
+
+#[test]
+fn parse_auth_with_state_updates() {
+    let rounds = run_state_changes_return_rounds(
+        &FULL_HANDSHAKE_WITH_AUTH,
+        CHandshake::new(
+            STATIC_HANDSHAKE_WITH_AUTH,
+            Some(CHandshakeAuth::new(b"sayan", b"pass1234")),
+        ),
+    );
+    assert_eq!(rounds, 3); // r1 = initial read, r2 = lengths, r3 = items
+}
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index f91ae41a..8e159a14 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -238,7 +238,7 @@ mod impls {
     }
     #[cfg(test)]
     mod test {
-        use super::{super::AssignmentExpression, ASTNode, QueryResult, QueryData, State};
+        use super::{super::AssignmentExpression, ASTNode, QueryData, QueryResult, State};
         impl<'a> ASTNode<'a> for AssignmentExpression<'a> {
             // important: upstream must verify this
             const VERIFY: bool = true;

From 7238f0c0e840bbdb25094dfc8201a1c87d934424 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 18 Sep 2023 18:02:05 +0000
Subject: [PATCH 254/310] Impl Skyhash/2 data exchange

---
 server/src/engine/mem/buf.rs                  |   2 +-
 server/src/engine/net/mod.rs                  |   9 +
 .../src/engine/net/protocol/data_exchange.rs  | 192 ++++++++++++++++++
 server/src/engine/net/protocol/mod.rs         |  76 +++++++
 server/src/engine/net/protocol/tests.rs       |  81 +++++++-
 5 files changed, 352 insertions(+), 8 deletions(-)
 create mode 100644 server/src/engine/net/protocol/data_exchange.rs

diff --git a/server/src/engine/mem/buf.rs b/server/src/engine/mem/buf.rs
index cc4d34ab..4870772a 100644
--- a/server/src/engine/mem/buf.rs
+++ b/server/src/engine/mem/buf.rs
@@ -58,7 +58,7 @@ impl<'a> BufferedScanner<'a> {
         self.remaining() >= sizeof
     }
     pub fn matches_cursor_rounded(&self, f: impl Fn(u8) -> bool) -> bool {
-        f(self.d[self.d.len().min(self.__cursor)])
+        f(self.d[(self.d.len() - 1).min(self.__cursor)])
     }
     pub fn matches_cursor_rounded_and_not_eof(&self, f: impl Fn(u8) -> bool) -> bool {
         self.matches_cursor_rounded(f) & !self.eof()
diff --git a/server/src/engine/net/mod.rs b/server/src/engine/net/mod.rs
index d5c17871..9975ae49 100644
--- a/server/src/engine/net/mod.rs
+++ b/server/src/engine/net/mod.rs
@@ -24,4 +24,13 @@
  *
 */
 
+use tokio::io::{AsyncRead, AsyncWrite};
 mod protocol;
+
+pub trait Socket: AsyncWrite + AsyncRead + Unpin {}
+pub type IoResult<T> = Result<T, std::io::Error>;
+
+enum QLoopReturn {
+    Fin,
+    ConnectionRst,
+}
diff --git a/server/src/engine/net/protocol/data_exchange.rs b/server/src/engine/net/protocol/data_exchange.rs
new file mode 100644
index 00000000..2ffd9177
--- /dev/null
+++ b/server/src/engine/net/protocol/data_exchange.rs
@@ -0,0 +1,192 @@
+/*
+ * Created on Mon Sep 18 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/*
+    Skyhash/2 Data Exchange Packets
+    ---
+    1. Client side packet:
+    a. SQ
+    S<len>\n<payload>
+    b. PQ
+    P<count>\n(<pld len>\n<pld>)*
+
+    TODO(@ohsayan): Restore pipeline impl
+*/
+
+use crate::{engine::mem::BufferedScanner, util::compiler};
+use std::slice;
+
+#[derive(Debug, PartialEq)]
+pub struct CSQuery<'a> {
+    query: &'a [u8],
+}
+
+impl<'a> CSQuery<'a> {
+    pub(super) const fn new(query: &'a [u8]) -> Self {
+        Self { query }
+    }
+    pub const fn query(&self) -> &'a [u8] {
+        self.query
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum CSQueryState {
+    Initial,
+    SizeSegmentPart(u64),
+    WaitingForFullBlock(usize),
+}
+
+impl Default for CSQueryState {
+    fn default() -> Self {
+        Self::Initial
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum CSQueryExchangeResult<'a> {
+    Completed(CSQuery<'a>),
+    ChangeState(CSQueryState, usize),
+    PacketError,
+}
+
+impl<'a> CSQuery<'a> {
+    pub fn resume_with(
+        scanner: &mut BufferedScanner<'a>,
+        state: CSQueryState,
+    ) -> CSQueryExchangeResult<'a> {
+        match state {
+            CSQueryState::Initial => Self::resume_initial(scanner),
+            CSQueryState::SizeSegmentPart(part) => Self::resume_at_meta_segment(scanner, part),
+            CSQueryState::WaitingForFullBlock(size) => Self::resume_at_data_segment(scanner, size),
+        }
+    }
+}
+
+enum LFTIntParseResult {
+    Value(u64),
+    Partial(u64),
+    Error,
+}
+
+fn parse_lf_separated(
+    scanner: &mut BufferedScanner,
+    previously_buffered: u64,
+) -> LFTIntParseResult {
+    let mut ret = previously_buffered;
+    let mut okay = true;
+    while scanner.matches_cursor_rounded_and_not_eof(|b| b != b'\n') & okay {
+        let b = unsafe { scanner.next_byte() };
+        okay &= b.is_ascii_digit();
+        ret = match ret.checked_mul(10) {
+            Some(r) => r,
+            None => return LFTIntParseResult::Error,
+        };
+        ret = match ret.checked_add((b & 0x0F) as u64) {
+            Some(r) => r,
+            None => return LFTIntParseResult::Error,
+        };
+    }
+    let payload_ok = okay;
+    let lf_ok = scanner.matches_cursor_rounded_and_not_eof(|b| b == b'\n');
+    unsafe { scanner.move_ahead_by(lf_ok as usize) }
+    if payload_ok & lf_ok {
+        LFTIntParseResult::Value(ret)
+    } else {
+        if payload_ok {
+            LFTIntParseResult::Partial(ret)
+        } else {
+            LFTIntParseResult::Error
+        }
+    }
+}
+
+impl<'a> CSQuery<'a> {
+    pub const PREEMPTIVE_READ: usize = 4;
+    const FIRST_BYTE: u8 = b'S';
+    fn resume_initial(scanner: &mut BufferedScanner<'a>) -> CSQueryExchangeResult<'a> {
+        if cfg!(debug_assertions) {
+            if scanner.remaining() < Self::PREEMPTIVE_READ {
+                return CSQueryExchangeResult::ChangeState(
+                    CSQueryState::Initial,
+                    Self::PREEMPTIVE_READ,
+                );
+            }
+        } else {
+            assert!(scanner.remaining() >= Self::PREEMPTIVE_READ);
+        }
+        // get our block
+        let first_byte = unsafe { scanner.next_byte() };
+        // be optimistic and check first byte later
+        let size_of_query = match parse_lf_separated(scanner, 0) {
+            LFTIntParseResult::Value(v) => v as usize,
+            LFTIntParseResult::Partial(v) => {
+                if compiler::unlikely(first_byte != Self::FIRST_BYTE) {
+                    return CSQueryExchangeResult::PacketError;
+                } else {
+                    // expect at least 1 LF and at least 1 query byte
+                    return CSQueryExchangeResult::ChangeState(CSQueryState::SizeSegmentPart(v), 2);
+                }
+            }
+            LFTIntParseResult::Error => {
+                // that's pretty much over
+                return CSQueryExchangeResult::PacketError;
+            }
+        };
+        if compiler::unlikely(first_byte != Self::FIRST_BYTE) {
+            return CSQueryExchangeResult::PacketError;
+        }
+        Self::resume_at_data_segment(scanner, size_of_query)
+    }
+    fn resume_at_meta_segment(
+        scanner: &mut BufferedScanner<'a>,
+        previous: u64,
+    ) -> CSQueryExchangeResult<'a> {
+        match parse_lf_separated(scanner, previous) {
+            LFTIntParseResult::Value(v) => Self::resume_at_data_segment(scanner, v as usize),
+            LFTIntParseResult::Partial(p) => {
+                CSQueryExchangeResult::ChangeState(CSQueryState::SizeSegmentPart(p), 2)
+            }
+            LFTIntParseResult::Error => CSQueryExchangeResult::PacketError,
+        }
+    }
+    fn resume_at_data_segment(
+        scanner: &mut BufferedScanner<'a>,
+        size: usize,
+    ) -> CSQueryExchangeResult<'a> {
+        if scanner.has_left(size) {
+            let slice;
+            unsafe {
+                // UNSAFE(@ohsayan): checked len at branch
+                slice = slice::from_raw_parts(scanner.current().as_ptr(), size);
+                scanner.move_ahead_by(size);
+            }
+            CSQueryExchangeResult::Completed(CSQuery::new(slice))
+        } else {
+            CSQueryExchangeResult::ChangeState(CSQueryState::WaitingForFullBlock(size), size)
+        }
+    }
+}
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index ec08551e..ba794028 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -24,6 +24,82 @@
  *
 */
 
+mod data_exchange;
 mod handshake;
 #[cfg(test)]
 mod tests;
+
+use {
+    self::handshake::{CHandshake, HandshakeResult, HandshakeState},
+    super::{IoResult, QLoopReturn, Socket},
+    crate::engine::mem::BufferedScanner,
+    bytes::{Buf, BytesMut},
+    tokio::io::{AsyncReadExt, BufWriter},
+};
+
+pub async fn query_loop<S: Socket>(
+    con: &mut BufWriter<S>,
+    buf: &mut BytesMut,
+) -> IoResult<QLoopReturn> {
+    // handshake
+    match do_handshake(con, buf).await? {
+        Some(ret) => return Ok(ret),
+        None => {}
+    }
+    // done handshaking
+    loop {
+        let read_many = con.read_buf(buf).await?;
+        if let Some(t) = see_if_connection_terminates(read_many, buf) {
+            return Ok(t);
+        }
+        todo!()
+    }
+}
+
+fn see_if_connection_terminates(read_many: usize, buf: &[u8]) -> Option<QLoopReturn> {
+    if read_many == 0 {
+        // that's a connection termination
+        if buf.is_empty() {
+            // nice termination
+            return Some(QLoopReturn::Fin);
+        } else {
+            return Some(QLoopReturn::ConnectionRst);
+        }
+    }
+    None
+}
+
+async fn do_handshake<S: Socket>(
+    con: &mut BufWriter<S>,
+    buf: &mut BytesMut,
+) -> IoResult<Option<QLoopReturn>> {
+    let mut expected = CHandshake::INITIAL_READ;
+    let mut state = HandshakeState::default();
+    let mut cursor = 0;
+    let handshake;
+    loop {
+        let read_many = con.read_buf(buf).await?;
+        if let Some(t) = see_if_connection_terminates(read_many, buf) {
+            return Ok(Some(t));
+        }
+        if buf.len() < expected {
+            continue;
+        }
+        let mut scanner = unsafe { BufferedScanner::new_with_cursor(buf, cursor) };
+        match handshake::CHandshake::resume_with(&mut scanner, state) {
+            HandshakeResult::Completed(hs) => {
+                handshake = hs;
+                break;
+            }
+            HandshakeResult::ChangeState { new_state, expect } => {
+                expected = expect;
+                state = new_state;
+                cursor = scanner.cursor();
+            }
+            HandshakeResult::Error(_) => todo!(),
+        }
+    }
+    dbg!(handshake);
+    buf.advance(cursor);
+    Ok(None)
+}
diff --git a/server/src/engine/net/protocol/tests.rs b/server/src/engine/net/protocol/tests.rs
index 91185cdb..e2159deb 100644
--- a/server/src/engine/net/protocol/tests.rs
+++ b/server/src/engine/net/protocol/tests.rs
@@ -24,12 +24,21 @@
  *
 */
 
-use crate::engine::mem::BufferedScanner;
-use crate::engine::net::protocol::handshake::{
-    AuthMode, CHandshake, CHandshakeAuth, CHandshakeStatic, DataExchangeMode, HandshakeResult,
-    HandshakeState, HandshakeVersion, ProtocolVersion, QueryMode,
+use crate::engine::{
+    mem::BufferedScanner,
+    net::protocol::{
+        data_exchange::{CSQuery, CSQueryExchangeResult, CSQueryState},
+        handshake::{
+            AuthMode, CHandshake, CHandshakeAuth, CHandshakeStatic, DataExchangeMode,
+            HandshakeResult, HandshakeState, HandshakeVersion, ProtocolVersion, QueryMode,
+        },
+    },
 };
 
+/*
+    client handshake
+*/
+
 const FULL_HANDSHAKE_NO_AUTH: [u8; 7] = [b'H', 0, 0, 0, 0, 0, 0];
 const FULL_HANDSHAKE_WITH_AUTH: [u8; 23] = *b"H\0\0\0\0\x015\n8\nsayanpass1234";
 
@@ -150,7 +159,6 @@ fn parse_staged_with_auth() {
 
 fn run_state_changes_return_rounds(src: &[u8], expected_final_handshake: CHandshake) -> usize {
     let mut rounds = 0;
-    let hs;
     let mut state = HandshakeState::default();
     let mut cursor = 0;
     let mut expect_many = CHandshake::INITIAL_READ;
@@ -164,8 +172,7 @@ fn run_state_changes_return_rounds(src: &[u8], expected_final_handshake: CHandsh
                 expect_many = expect;
                 cursor = scanner.cursor();
             }
-            HandshakeResult::Completed(c) => {
-                hs = c;
+            HandshakeResult::Completed(hs) => {
                 assert_eq!(hs, expected_final_handshake);
                 break;
             }
@@ -195,3 +202,63 @@ fn parse_auth_with_state_updates() {
     );
     assert_eq!(rounds, 3); // r1 = initial read, r2 = lengths, r3 = items
 }
+
+/*
+    QT-DEX/SQ
+*/
+
+const FULL_SQ: [u8; 116] = *b"S111\nSELECT username, email, bio, profile_pic, following, followers, FROM mysocialapp.users WHERE username = 'sayan'";
+const SQ_FULL: CSQuery<'static> = CSQuery::new(
+    b"SELECT username, email, bio, profile_pic, following, followers, FROM mysocialapp.users WHERE username = 'sayan'"
+);
+
+#[test]
+fn staged_qt_dex_sq() {
+    for i in 0..FULL_SQ.len() {
+        let buf = &FULL_SQ[..i + 1];
+        let mut scanner = BufferedScanner::new(buf);
+        let result = CSQuery::resume_with(&mut scanner, CSQueryState::default());
+        match buf.len() {
+            1..=3 => assert_eq!(
+                result,
+                CSQueryExchangeResult::ChangeState(CSQueryState::Initial, CSQuery::PREEMPTIVE_READ)
+            ),
+            4 => assert_eq!(
+                result,
+                CSQueryExchangeResult::ChangeState(CSQueryState::SizeSegmentPart(111), 2)
+            ),
+            5..=115 => assert_eq!(
+                result,
+                CSQueryExchangeResult::ChangeState(CSQueryState::WaitingForFullBlock(111), 111),
+            ),
+            116 => assert_eq!(result, CSQueryExchangeResult::Completed(SQ_FULL)),
+            _ => unreachable!(),
+        }
+    }
+}
+
+#[test]
+fn staged_with_status_switch_qt_dex_sq() {
+    let mut cursor = 0;
+    let mut expect = CSQuery::PREEMPTIVE_READ;
+    let mut state = CSQueryState::default();
+    let mut rounds = 0;
+    loop {
+        rounds += 1;
+        let buf = &FULL_SQ[..cursor + expect];
+        let mut scanner = unsafe { BufferedScanner::new_with_cursor(buf, cursor) };
+        match CSQuery::resume_with(&mut scanner, state) {
+            CSQueryExchangeResult::Completed(c) => {
+                assert_eq!(c, SQ_FULL);
+                break;
+            }
+            CSQueryExchangeResult::ChangeState(new_state, _expect) => {
+                state = new_state;
+                expect = _expect;
+                cursor = scanner.cursor();
+            }
+            CSQueryExchangeResult::PacketError => panic!("packet error"),
+        }
+    }
+    assert_eq!(rounds, 3);
+}

From 5e068c0e9bad992cbb01a41edcf16fd6b18f4b2b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 19 Sep 2023 14:58:17 +0000
Subject: [PATCH 255/310] Generalize scanner

---
 server/src/engine/mem/mod.rs                  |   4 +-
 server/src/engine/mem/{buf.rs => scanner.rs}  | 104 ++++++++++++------
 .../src/engine/net/protocol/data_exchange.rs  |   4 +-
 server/src/engine/net/protocol/handshake.rs   |   6 +-
 4 files changed, 75 insertions(+), 43 deletions(-)
 rename server/src/engine/mem/{buf.rs => scanner.rs} (69%)

diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index 6e1d6e87..290c0597 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -25,8 +25,8 @@
 */
 
 mod astr;
-pub mod buf;
 mod ll;
+pub mod scanner;
 mod stackop;
 mod uarray;
 mod vinline;
@@ -37,8 +37,8 @@ mod tests;
 // re-exports
 pub use {
     astr::AStr,
-    buf::BufferedScanner,
     ll::CachePadded,
+    scanner::BufferedScanner,
     stackop::ByteStack,
     uarray::UArray,
     vinline::VInline,
diff --git a/server/src/engine/mem/buf.rs b/server/src/engine/mem/scanner.rs
similarity index 69%
rename from server/src/engine/mem/buf.rs
rename to server/src/engine/mem/scanner.rs
index 4870772a..892a4548 100644
--- a/server/src/engine/mem/buf.rs
+++ b/server/src/engine/mem/scanner.rs
@@ -26,17 +26,19 @@
 
 use core::{ptr, slice};
 
-#[derive(Debug)]
-pub struct BufferedScanner<'a> {
-    d: &'a [u8],
+pub type BufferedScanner<'a> = Scanner<'a, u8>;
+
+#[derive(Debug, PartialEq)]
+pub struct Scanner<'a, T> {
+    d: &'a [T],
     __cursor: usize,
 }
 
-impl<'a> BufferedScanner<'a> {
-    pub const fn new(d: &'a [u8]) -> Self {
+impl<'a, T> Scanner<'a, T> {
+    pub const fn new(d: &'a [T]) -> Self {
         unsafe { Self::new_with_cursor(d, 0) }
     }
-    pub const unsafe fn new_with_cursor(d: &'a [u8], i: usize) -> Self {
+    pub const unsafe fn new_with_cursor(d: &'a [T], i: usize) -> Self {
         Self { d, __cursor: i }
     }
     pub const fn remaining(&self) -> usize {
@@ -48,27 +50,33 @@ impl<'a> BufferedScanner<'a> {
     pub const fn cursor(&self) -> usize {
         self.__cursor
     }
-    pub fn current(&self) -> &[u8] {
+    pub fn current(&self) -> &[T] {
         &self.d[self.__cursor..]
     }
+    pub const fn cursor_ptr(&self) -> *const T {
+        unsafe { self.d.as_ptr().add(self.__cursor) }
+    }
     pub fn eof(&self) -> bool {
         self.remaining() == 0
     }
     pub fn has_left(&self, sizeof: usize) -> bool {
         self.remaining() >= sizeof
     }
-    pub fn matches_cursor_rounded(&self, f: impl Fn(u8) -> bool) -> bool {
-        f(self.d[(self.d.len() - 1).min(self.__cursor)])
+    pub fn matches_cursor_rounded(&self, f: impl Fn(&T) -> bool) -> bool {
+        f(&self.d[(self.d.len() - 1).min(self.__cursor)])
     }
-    pub fn matches_cursor_rounded_and_not_eof(&self, f: impl Fn(u8) -> bool) -> bool {
+    pub fn matches_cursor_rounded_and_not_eof(&self, f: impl Fn(&T) -> bool) -> bool {
         self.matches_cursor_rounded(f) & !self.eof()
     }
 }
 
-impl<'a> BufferedScanner<'a> {
+impl<'a, T> Scanner<'a, T> {
     pub unsafe fn set_cursor(&mut self, i: usize) {
         self.__cursor = i;
     }
+    pub unsafe fn move_ahead(&mut self) {
+        self.move_back_by(1)
+    }
     pub unsafe fn move_ahead_by(&mut self, by: usize) {
         self._incr(by)
     }
@@ -81,12 +89,12 @@ impl<'a> BufferedScanner<'a> {
     unsafe fn _incr(&mut self, by: usize) {
         self.__cursor += by;
     }
-    unsafe fn _cursor(&self) -> *const u8 {
+    unsafe fn _cursor(&self) -> *const T {
         self.d.as_ptr().add(self.__cursor)
     }
 }
 
-impl<'a> BufferedScanner<'a> {
+impl<'a> Scanner<'a, u8> {
     pub fn try_next_byte(&mut self) -> Option<u8> {
         if self.eof() {
             None
@@ -116,54 +124,78 @@ pub enum BufferedReadResult<T> {
     Error,
 }
 
-impl<'a> BufferedScanner<'a> {
+impl<'a> Scanner<'a, u8> {
+    pub fn trim_ahead(&mut self, f: impl Fn(u8) -> bool) {
+        while self.matches_cursor_rounded_and_not_eof(|b| f(*b)) {
+            unsafe { self.move_ahead() }
+        }
+    }
+    pub fn move_ahead_if_matches(&mut self, f: impl Fn(u8) -> bool) {
+        unsafe { self.move_back_by(self.matches_cursor_rounded_and_not_eof(|b| f(*b)) as _) }
+    }
     /// Attempt to parse a `\n` terminated (we move past the LF, so you can't see it)
     ///
     /// If we were unable to read in the integer, then the cursor will be restored to its starting position
     // TODO(@ohsayan): optimize
-    pub fn try_next_ascii_u64_lf_separated(&mut self) -> BufferedReadResult<u64> {
+    pub fn try_next_ascii_u64_lf_separated_with_result(&mut self) -> BufferedReadResult<u64> {
         let mut okay = true;
         let start = self.cursor();
+        let ret = self.extract_integer(&mut okay);
+        let payload_ok = okay;
+        let lf = self.matches_cursor_rounded_and_not_eof(|b| *b == b'\n');
+        okay &= lf;
+        unsafe { self._incr(okay as _) }; // skip LF
+        if okay {
+            BufferedReadResult::Value(ret)
+        } else {
+            unsafe { self.set_cursor(start) }
+            if payload_ok {
+                // payload was ok, but we missed a null
+                BufferedReadResult::NeedMore
+            } else {
+                // payload was NOT ok
+                BufferedReadResult::Error
+            }
+        }
+    }
+    pub fn try_next_ascii_u64_lf_separated(&mut self) -> Option<u64> {
+        let start = self.cursor();
+        let mut okay = true;
+        let ret = self.extract_integer(&mut okay);
+        let lf = self.matches_cursor_rounded_and_not_eof(|b| *b == b'\n');
+        if okay & lf {
+            Some(ret)
+        } else {
+            unsafe { self.set_cursor(start) }
+            None
+        }
+    }
+    pub fn extract_integer(&mut self, okay: &mut bool) -> u64 {
         let mut ret = 0u64;
-        while self.matches_cursor_rounded_and_not_eof(|b| b != b'\n') & okay {
+        while self.matches_cursor_rounded_and_not_eof(|b| *b != b'\n') & *okay {
             let b = self.d[self.cursor()];
-            okay &= b.is_ascii_digit();
+            *okay &= b.is_ascii_digit();
             ret = match ret.checked_mul(10) {
                 Some(r) => r,
                 None => {
-                    okay = false;
+                    *okay = false;
                     break;
                 }
             };
             ret = match ret.checked_add((b & 0x0F) as u64) {
                 Some(r) => r,
                 None => {
-                    okay = false;
+                    *okay = false;
                     break;
                 }
             };
             unsafe { self._incr(1) }
         }
-        let payload_ok = okay;
-        let null_ok = self.matches_cursor_rounded_and_not_eof(|b| b == b'\n');
-        okay &= null_ok;
-        unsafe { self._incr(okay as _) }; // skip LF
-        if okay {
-            BufferedReadResult::Value(ret)
-        } else {
-            unsafe { self.set_cursor(start) }
-            if payload_ok {
-                // payload was ok, but we missed a null
-                BufferedReadResult::NeedMore
-            } else {
-                // payload was NOT ok
-                BufferedReadResult::Error
-            }
-        }
+        ret
     }
 }
 
-impl<'a> BufferedScanner<'a> {
+impl<'a> Scanner<'a, u8> {
     pub unsafe fn next_u64_le(&mut self) -> u64 {
         u64::from_le_bytes(self.next_chunk())
     }
diff --git a/server/src/engine/net/protocol/data_exchange.rs b/server/src/engine/net/protocol/data_exchange.rs
index 2ffd9177..8b024e04 100644
--- a/server/src/engine/net/protocol/data_exchange.rs
+++ b/server/src/engine/net/protocol/data_exchange.rs
@@ -98,7 +98,7 @@ fn parse_lf_separated(
 ) -> LFTIntParseResult {
     let mut ret = previously_buffered;
     let mut okay = true;
-    while scanner.matches_cursor_rounded_and_not_eof(|b| b != b'\n') & okay {
+    while scanner.matches_cursor_rounded_and_not_eof(|b| *b != b'\n') & okay {
         let b = unsafe { scanner.next_byte() };
         okay &= b.is_ascii_digit();
         ret = match ret.checked_mul(10) {
@@ -111,7 +111,7 @@ fn parse_lf_separated(
         };
     }
     let payload_ok = okay;
-    let lf_ok = scanner.matches_cursor_rounded_and_not_eof(|b| b == b'\n');
+    let lf_ok = scanner.matches_cursor_rounded_and_not_eof(|b| *b == b'\n');
     unsafe { scanner.move_ahead_by(lf_ok as usize) }
     if payload_ok & lf_ok {
         LFTIntParseResult::Value(ret)
diff --git a/server/src/engine/net/protocol/handshake.rs b/server/src/engine/net/protocol/handshake.rs
index 4400b9c3..a5d9616d 100644
--- a/server/src/engine/net/protocol/handshake.rs
+++ b/server/src/engine/net/protocol/handshake.rs
@@ -26,7 +26,7 @@
 
 use {
     crate::{
-        engine::mem::buf::{BufferedReadResult, BufferedScanner},
+        engine::mem::scanner::{BufferedReadResult, BufferedScanner},
         util::compiler,
     },
     std::slice,
@@ -367,7 +367,7 @@ impl<'a> CHandshake<'a> {
             AuthMode::Password => {}
         }
         // let us see if we can parse the username length
-        let uname_l = match scanner.try_next_ascii_u64_lf_separated() {
+        let uname_l = match scanner.try_next_ascii_u64_lf_separated_with_result() {
             BufferedReadResult::NeedMore => {
                 return HandshakeResult::ChangeState {
                     new_state: HandshakeState::StaticBlock(static_header),
@@ -388,7 +388,7 @@ impl<'a> CHandshake<'a> {
         uname_l: usize,
     ) -> HandshakeResult<'a> {
         // we just have to get the password len
-        let pwd_l = match scanner.try_next_ascii_u64_lf_separated() {
+        let pwd_l = match scanner.try_next_ascii_u64_lf_separated_with_result() {
             BufferedReadResult::Value(v) => v as usize,
             BufferedReadResult::NeedMore => {
                 // newline missing (or maybe there's more?)

From a390120231ab364c078e9a763bd592432e2e7d85 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 19 Sep 2023 18:57:05 +0000
Subject: [PATCH 256/310] Cleanup lexer impl

---
 Cargo.toml                                    |   2 +-
 server/src/engine/core/dml/mod.rs             |   4 +-
 server/src/engine/core/dml/upd.rs             |  87 +-
 server/src/engine/core/index/key.rs           |  41 +-
 server/src/engine/core/index/mod.rs           |   6 +-
 server/src/engine/core/tests/dml/mod.rs       |   4 +-
 server/src/engine/data/cell.rs                |  29 +-
 server/src/engine/data/dict.rs                |  15 +-
 server/src/engine/data/lit.rs                 | 508 +++++-----
 server/src/engine/data/macros.rs              |  60 --
 server/src/engine/data/mod.rs                 |   3 -
 server/src/engine/data/spec.rs                | 310 -------
 server/src/engine/data/tests/mod.rs           |  16 +-
 server/src/engine/mem/mod.rs                  |   6 +
 server/src/engine/mem/scanner.rs              | 326 +++++--
 server/src/engine/mem/tests/mod.rs            |   1 +
 server/src/engine/mem/tests/scanner.rs        | 249 +++++
 server/src/engine/net/mod.rs                  |   2 +-
 .../src/engine/net/protocol/data_exchange.rs  |  10 +-
 server/src/engine/net/protocol/handshake.rs   |  26 +-
 server/src/engine/ql/ast/mod.rs               |  46 +-
 server/src/engine/ql/benches.rs               |   2 +-
 server/src/engine/ql/dml/mod.rs               |   8 +-
 server/src/engine/ql/dml/upd.rs               |   6 +-
 server/src/engine/ql/lex/mod.rs               | 878 ++++++++----------
 server/src/engine/ql/lex/raw.rs               | 181 +---
 server/src/engine/ql/macros.rs                |   8 +
 server/src/engine/ql/tests.rs                 |   8 +-
 server/src/engine/ql/tests/dml_tests.rs       | 166 +---
 server/src/engine/ql/tests/lexer_tests.rs     | 510 ++++------
 server/src/engine/ql/tests/schema_tests.rs    | 101 +-
 server/src/engine/ql/tests/structure_syn.rs   |  42 +-
 32 files changed, 1619 insertions(+), 2042 deletions(-)
 delete mode 100644 server/src/engine/data/macros.rs
 delete mode 100644 server/src/engine/data/spec.rs
 create mode 100644 server/src/engine/mem/tests/scanner.rs

diff --git a/Cargo.toml b/Cargo.toml
index f3b41279..448c2321 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,5 @@
 [workspace]
-resolver = "2"
+resolver = "1"
 members = [
     "cli",
     "server",
diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index bb65edbf..4cf41c44 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -32,7 +32,7 @@ mod upd;
 use crate::{
     engine::{
         core::model::Model,
-        data::{lit::LitIR, spec::DataspecMeta1D, tag::DataTag},
+        data::{lit::Lit, tag::DataTag},
         error::{Error, QueryResult},
         ql::dml::WhereClause,
     },
@@ -47,7 +47,7 @@ impl Model {
     pub(self) fn resolve_where<'a>(
         &self,
         where_clause: &mut WhereClause<'a>,
-    ) -> QueryResult<LitIR<'a>> {
+    ) -> QueryResult<Lit<'a>> {
         match where_clause.clauses_mut().remove(self.p_key().as_bytes()) {
             Some(clause)
                 if clause.filter_hint_none()
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index 3586168f..95bd6089 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -33,8 +33,7 @@ use {
             core::{self, model::delta::DataDeltaKind, query_meta::AssignmentOperator},
             data::{
                 cell::Datacell,
-                lit::LitIR,
-                spec::{Dataspec1D, DataspecMeta1D},
+                lit::Lit,
                 tag::{DataTag, TagClass},
             },
             error::{Error, QueryResult},
@@ -49,51 +48,51 @@ use {
 };
 
 #[inline(always)]
-unsafe fn dc_op_fail(_: &Datacell, _: LitIR) -> (bool, Datacell) {
+unsafe fn dc_op_fail(_: &Datacell, _: Lit) -> (bool, Datacell) {
     (false, Datacell::null())
 }
 // bool
-unsafe fn dc_op_bool_ass(_: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    (true, Datacell::new_bool(rhs.read_bool_uck()))
+unsafe fn dc_op_bool_ass(_: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    (true, Datacell::new_bool(rhs.bool()))
 }
 // uint
-unsafe fn dc_op_uint_ass(_: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    (true, Datacell::new_uint(rhs.read_uint_uck()))
+unsafe fn dc_op_uint_ass(_: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    (true, Datacell::new_uint(rhs.uint()))
 }
-unsafe fn dc_op_uint_add(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let (sum, of) = dc.read_uint().overflowing_add(rhs.read_uint_uck());
+unsafe fn dc_op_uint_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let (sum, of) = dc.read_uint().overflowing_add(rhs.uint());
     (of, Datacell::new_uint(sum))
 }
-unsafe fn dc_op_uint_sub(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let (diff, of) = dc.read_uint().overflowing_sub(rhs.read_uint_uck());
+unsafe fn dc_op_uint_sub(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let (diff, of) = dc.read_uint().overflowing_sub(rhs.uint());
     (of, Datacell::new_uint(diff))
 }
-unsafe fn dc_op_uint_mul(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let (prod, of) = dc.read_uint().overflowing_mul(rhs.read_uint_uck());
+unsafe fn dc_op_uint_mul(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let (prod, of) = dc.read_uint().overflowing_mul(rhs.uint());
     (of, Datacell::new_uint(prod))
 }
-unsafe fn dc_op_uint_div(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let (quo, of) = dc.read_uint().overflowing_div(rhs.read_uint_uck());
+unsafe fn dc_op_uint_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let (quo, of) = dc.read_uint().overflowing_div(rhs.uint());
     (of, Datacell::new_uint(quo))
 }
 // sint
-unsafe fn dc_op_sint_ass(_: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    (true, Datacell::new_sint(rhs.read_sint_uck()))
+unsafe fn dc_op_sint_ass(_: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    (true, Datacell::new_sint(rhs.sint()))
 }
-unsafe fn dc_op_sint_add(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let (sum, of) = dc.read_sint().overflowing_add(rhs.read_sint_uck());
+unsafe fn dc_op_sint_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let (sum, of) = dc.read_sint().overflowing_add(rhs.sint());
     (of, Datacell::new_sint(sum))
 }
-unsafe fn dc_op_sint_sub(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let (diff, of) = dc.read_sint().overflowing_sub(rhs.read_sint_uck());
+unsafe fn dc_op_sint_sub(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let (diff, of) = dc.read_sint().overflowing_sub(rhs.sint());
     (of, Datacell::new_sint(diff))
 }
-unsafe fn dc_op_sint_mul(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let (prod, of) = dc.read_sint().overflowing_mul(rhs.read_sint_uck());
+unsafe fn dc_op_sint_mul(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let (prod, of) = dc.read_sint().overflowing_mul(rhs.sint());
     (of, Datacell::new_sint(prod))
 }
-unsafe fn dc_op_sint_div(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let (quo, of) = dc.read_sint().overflowing_div(rhs.read_sint_uck());
+unsafe fn dc_op_sint_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let (quo, of) = dc.read_sint().overflowing_div(rhs.sint());
     (of, Datacell::new_sint(quo))
 }
 /*
@@ -106,28 +105,28 @@ unsafe fn dc_op_sint_div(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
     --
     TODO(@ohsayan): account for float32 overflow
 */
-unsafe fn dc_op_float_ass(_: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    (true, Datacell::new_float(rhs.read_float_uck()))
+unsafe fn dc_op_float_ass(_: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    (true, Datacell::new_float(rhs.float()))
 }
-unsafe fn dc_op_float_add(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let sum = dc.read_float() + rhs.read_float_uck();
+unsafe fn dc_op_float_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let sum = dc.read_float() + rhs.float();
     (true, Datacell::new_float(sum))
 }
-unsafe fn dc_op_float_sub(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let diff = dc.read_float() - rhs.read_float_uck();
+unsafe fn dc_op_float_sub(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let diff = dc.read_float() - rhs.float();
     (true, Datacell::new_float(diff))
 }
-unsafe fn dc_op_float_mul(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let prod = dc.read_float() - rhs.read_float_uck();
+unsafe fn dc_op_float_mul(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let prod = dc.read_float() - rhs.float();
     (true, Datacell::new_float(prod))
 }
-unsafe fn dc_op_float_div(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let quo = dc.read_float() * rhs.read_float_uck();
+unsafe fn dc_op_float_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let quo = dc.read_float() * rhs.float();
     (true, Datacell::new_float(quo))
 }
 // binary
-unsafe fn dc_op_bin_ass(_dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let new_bin = rhs.read_bin_uck();
+unsafe fn dc_op_bin_ass(_dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let new_bin = rhs.bin();
     let mut v = Vec::new();
     if v.try_reserve_exact(new_bin.len()).is_err() {
         return dc_op_fail(_dc, rhs);
@@ -135,8 +134,8 @@ unsafe fn dc_op_bin_ass(_dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
     v.extend_from_slice(new_bin);
     (true, Datacell::new_bin(v.into_boxed_slice()))
 }
-unsafe fn dc_op_bin_add(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let push_into_bin = rhs.read_bin_uck();
+unsafe fn dc_op_bin_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let push_into_bin = rhs.bin();
     let mut bin = Vec::new();
     if compiler::unlikely(bin.try_reserve_exact(push_into_bin.len()).is_err()) {
         return dc_op_fail(dc, rhs);
@@ -146,8 +145,8 @@ unsafe fn dc_op_bin_add(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
     (true, Datacell::new_bin(bin.into_boxed_slice()))
 }
 // string
-unsafe fn dc_op_str_ass(_dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let new_str = rhs.read_str_uck();
+unsafe fn dc_op_str_ass(_dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let new_str = rhs.str();
     let mut v = String::new();
     if v.try_reserve_exact(new_str.len()).is_err() {
         return dc_op_fail(_dc, rhs);
@@ -155,8 +154,8 @@ unsafe fn dc_op_str_ass(_dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
     v.push_str(new_str);
     (true, Datacell::new_str(v.into_boxed_str()))
 }
-unsafe fn dc_op_str_add(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
-    let push_into_str = rhs.read_str_uck();
+unsafe fn dc_op_str_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let push_into_str = rhs.str();
     let mut str = String::new();
     if compiler::unlikely(str.try_reserve_exact(push_into_str.len()).is_err()) {
         return dc_op_fail(dc, rhs);
@@ -166,7 +165,7 @@ unsafe fn dc_op_str_add(dc: &Datacell, rhs: LitIR) -> (bool, Datacell) {
     (true, Datacell::new_str(str.into_boxed_str()))
 }
 
-static OPERATOR: [unsafe fn(&Datacell, LitIR) -> (bool, Datacell); {
+static OPERATOR: [unsafe fn(&Datacell, Lit) -> (bool, Datacell); {
     TagClass::MAX as usize * AssignmentOperator::VARIANTS
 }] = [
     // bool
diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
index 470168c9..f8d8280e 100644
--- a/server/src/engine/core/index/key.rs
+++ b/server/src/engine/core/index/key.rs
@@ -26,13 +26,12 @@
 
 use crate::engine::mem::ZERO_BLOCK;
 #[cfg(test)]
-use crate::{engine::data::spec::Dataspec1D, util::test_utils};
+use crate::util::test_utils;
 use {
     crate::engine::{
         data::{
             cell::Datacell,
-            lit::LitIR,
-            spec::DataspecMeta1D,
+            lit::Lit,
             tag::{DataTag, TagUnique},
         },
         idx::meta::Comparable,
@@ -245,22 +244,22 @@ impl Hash for PrimaryIndexKey {
     }
 }
 
-impl<'a> PartialEq<LitIR<'a>> for PrimaryIndexKey {
-    fn eq(&self, key: &LitIR<'a>) -> bool {
+impl<'a> PartialEq<Lit<'a>> for PrimaryIndexKey {
+    fn eq(&self, key: &Lit<'a>) -> bool {
         debug_assert!(key.kind().tag_unique().is_unique());
         self.tag == key.kind().tag_unique() && self.virtual_block() == key.__vdata()
     }
 }
 
-impl<'a> Comparable<LitIR<'a>> for PrimaryIndexKey {
-    fn cmp_eq(&self, key: &LitIR<'a>) -> bool {
-        <PrimaryIndexKey as PartialEq<LitIR>>::eq(self, key)
+impl<'a> Comparable<Lit<'a>> for PrimaryIndexKey {
+    fn cmp_eq(&self, key: &Lit<'a>) -> bool {
+        <PrimaryIndexKey as PartialEq<Lit>>::eq(self, key)
     }
 }
 
-impl<'a> Comparable<PrimaryIndexKey> for LitIR<'a> {
+impl<'a> Comparable<PrimaryIndexKey> for Lit<'a> {
     fn cmp_eq(&self, key: &PrimaryIndexKey) -> bool {
-        <PrimaryIndexKey as PartialEq<LitIR>>::eq(key, self)
+        <PrimaryIndexKey as PartialEq<Lit>>::eq(key, self)
     }
 }
 
@@ -333,16 +332,16 @@ fn check_pk_eq_hash() {
 fn check_pk_lit_eq_hash() {
     let state = test_utils::randomstate();
     let data = [
-        LitIR::UnsignedInt(100),
-        LitIR::SignedInt(-100),
-        LitIR::Bin(b"binary bro"),
-        LitIR::Str("string bro"),
+        Lit::new_uint(100),
+        Lit::new_sint(-100),
+        Lit::new_bin(b"binary bro"),
+        Lit::new_str("string bro"),
     ];
-    for litir in data {
-        let pk = PrimaryIndexKey::try_from_dc(Datacell::from(litir.clone())).unwrap();
-        assert_eq!(pk, litir);
+    for lit in data {
+        let pk = PrimaryIndexKey::try_from_dc(Datacell::from(lit.clone())).unwrap();
+        assert_eq!(pk, lit);
         assert_eq!(
-            test_utils::hash_rs(&state, &litir),
+            test_utils::hash_rs(&state, &lit),
             test_utils::hash_rs(&state, &pk)
         );
     }
@@ -352,7 +351,7 @@ fn check_pk_lit_eq_hash() {
 fn check_pk_extremes() {
     let state = test_utils::randomstate();
     let d1 = PrimaryIndexKey::try_from_dc(Datacell::new_uint(u64::MAX)).unwrap();
-    let d2 = PrimaryIndexKey::try_from_dc(Datacell::from(LitIR::UnsignedInt(u64::MAX))).unwrap();
+    let d2 = PrimaryIndexKey::try_from_dc(Datacell::from(Lit::new_uint(u64::MAX))).unwrap();
     assert_eq!(d1, d2);
     assert_eq!(d1.uint().unwrap(), u64::MAX);
     assert_eq!(d2.uint().unwrap(), u64::MAX);
@@ -360,7 +359,7 @@ fn check_pk_extremes() {
         test_utils::hash_rs(&state, &d1),
         test_utils::hash_rs(&state, &d2)
     );
-    assert_eq!(d1, LitIR::UnsignedInt(u64::MAX));
-    assert_eq!(d2, LitIR::UnsignedInt(u64::MAX));
+    assert_eq!(d1, Lit::new_uint(u64::MAX));
+    assert_eq!(d2, Lit::new_uint(u64::MAX));
     assert_eq!(d1.uint().unwrap(), u64::MAX);
 }
diff --git a/server/src/engine/core/index/mod.rs b/server/src/engine/core/index/mod.rs
index 4fafe5f3..ab40fbc5 100644
--- a/server/src/engine/core/index/mod.rs
+++ b/server/src/engine/core/index/mod.rs
@@ -28,7 +28,7 @@ mod key;
 mod row;
 
 use crate::engine::{
-    data::lit::LitIR,
+    data::lit::Lit,
     idx::{IndexBaseSpec, IndexMTRaw, MTIndex},
     sync::atm::Guard,
 };
@@ -49,12 +49,12 @@ impl PrimaryIndex {
             data: IndexMTRaw::idx_init(),
         }
     }
-    pub fn remove<'a>(&self, key: LitIR<'a>, g: &Guard) -> bool {
+    pub fn remove<'a>(&self, key: Lit<'a>, g: &Guard) -> bool {
         self.data.mt_delete(&key, g)
     }
     pub fn select<'a, 'v, 't: 'v, 'g: 't>(
         &'t self,
-        key: LitIR<'a>,
+        key: Lit<'a>,
         g: &'g Guard,
     ) -> Option<&'v Row> {
         self.data.mt_get_element(&key, g)
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index b2bcdc0e..9d9fa230 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -31,7 +31,7 @@ mod update;
 
 use crate::engine::{
     core::{dml, index::Row, model::Model},
-    data::{cell::Datacell, lit::LitIR},
+    data::{cell::Datacell, lit::Lit},
     error::QueryResult,
     fractal::GlobalInstanceLike,
     ql::{
@@ -75,7 +75,7 @@ fn _exec_only_read_key_and_then<T>(
         let _irm = mdl.intent_read_model();
         let row = mdl
             .primary_index()
-            .select(LitIR::from(key_name), &guard)
+            .select(Lit::from(key_name), &guard)
             .unwrap()
             .clone();
         drop(guard);
diff --git a/server/src/engine/data/cell.rs b/server/src/engine/data/cell.rs
index 95017988..1cce866d 100644
--- a/server/src/engine/data/cell.rs
+++ b/server/src/engine/data/cell.rs
@@ -30,8 +30,7 @@ use {
     crate::engine::{
         self,
         data::{
-            lit::{Lit, LitIR},
-            spec::{Dataspec1D, DataspecMeta1D},
+            lit::Lit,
             tag::{CUTag, DataTag, TagClass},
         },
         mem::{DwordNN, DwordQN, NativeQword, SpecialPaddedWord, WordIO},
@@ -228,8 +227,8 @@ direct_from! {
     }
 }
 
-impl<'a> From<LitIR<'a>> for Datacell {
-    fn from(l: LitIR<'a>) -> Self {
+impl<'a> From<Lit<'a>> for Datacell {
+    fn from(l: Lit<'a>) -> Self {
         match l.kind().tag_class() {
             tag if tag < TagClass::Bin => unsafe {
                 // UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type doesn't need any advanced construction
@@ -241,7 +240,7 @@ impl<'a> From<LitIR<'a>> for Datacell {
             },
             TagClass::Bin | TagClass::Str => unsafe {
                 // UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type requires a new heap for construction
-                let mut bin = ManuallyDrop::new(l.read_bin_uck().to_owned().into_boxed_slice());
+                let mut bin = ManuallyDrop::new(l.bin().to_owned().into_boxed_slice());
                 Datacell::new(
                     CUTag::from(l.kind()),
                     DataRaw::word(DwordQN::dwordqn_store_qw_nw(
@@ -269,12 +268,6 @@ impl From<i32> for Datacell {
     }
 }
 
-impl<'a> From<Lit<'a>> for Datacell {
-    fn from(l: Lit<'a>) -> Self {
-        Self::from(l.as_ir())
-    }
-}
-
 impl<const N: usize> From<[Datacell; N]> for Datacell {
     fn from(l: [Datacell; N]) -> Self {
         Self::new_list(l.into())
@@ -459,17 +452,17 @@ impl Clone for Datacell {
 #[derive(Debug)]
 pub struct VirtualDatacell<'a> {
     dc: ManuallyDrop<Datacell>,
-    _lt: PhantomData<LitIR<'a>>,
+    _lt: PhantomData<Lit<'a>>,
 }
 
 impl<'a> VirtualDatacell<'a> {
-    pub fn new(litir: LitIR<'a>) -> Self {
+    pub fn new(lit: Lit<'a>) -> Self {
         Self {
             dc: ManuallyDrop::new(unsafe {
                 // UNSAFE(@ohsayan): this is a "reference" to a "virtual" aka fake DC. this just works because of memory layouts
                 Datacell::new(
-                    CUTag::from(litir.kind()),
-                    DataRaw::word(litir.data().dwordqn_promote()),
+                    CUTag::from(lit.kind()),
+                    DataRaw::word(lit.data().dwordqn_promote()),
                 )
             }),
             _lt: PhantomData,
@@ -477,8 +470,8 @@ impl<'a> VirtualDatacell<'a> {
     }
 }
 
-impl<'a> From<LitIR<'a>> for VirtualDatacell<'a> {
-    fn from(l: LitIR<'a>) -> Self {
+impl<'a> From<Lit<'a>> for VirtualDatacell<'a> {
+    fn from(l: Lit<'a>) -> Self {
         Self::new(l)
     }
 }
@@ -504,6 +497,6 @@ impl<'a> Clone for VirtualDatacell<'a> {
 
 #[test]
 fn virtual_dc_damn() {
-    let dc = LitIR::Str("hello, world");
+    let dc = Lit::new_str("hello, world");
     assert_eq!(VirtualDatacell::from(dc), Datacell::from("hello, world"));
 }
diff --git a/server/src/engine/data/dict.rs b/server/src/engine/data/dict.rs
index 6dc1894f..e67cd71d 100644
--- a/server/src/engine/data/dict.rs
+++ b/server/src/engine/data/dict.rs
@@ -26,10 +26,7 @@
 
 use {
     crate::engine::{
-        data::{
-            cell::Datacell,
-            lit::{Lit, LitIR},
-        },
+        data::{cell::Datacell, lit::Lit},
         idx::STIndex,
     },
     std::collections::HashMap,
@@ -181,15 +178,9 @@ fn rmerge_metadata_prepare_patch(
     impls
 */
 
-impl<'a> From<LitIR<'a>> for DictEntryGeneric {
-    fn from(l: LitIR<'a>) -> Self {
-        Self::Data(Datacell::from(l))
-    }
-}
-
 impl<'a> From<Lit<'a>> for DictEntryGeneric {
-    fn from(value: Lit<'a>) -> Self {
-        Self::Data(Datacell::from(value))
+    fn from(l: Lit<'a>) -> Self {
+        Self::Data(Datacell::from(l))
     }
 }
 
diff --git a/server/src/engine/data/lit.rs b/server/src/engine/data/lit.rs
index d40b02ff..e48e3d6d 100644
--- a/server/src/engine/data/lit.rs
+++ b/server/src/engine/data/lit.rs
@@ -1,5 +1,5 @@
 /*
- * Created on Sun Feb 26 2023
+ * Created on Wed Sep 20 2023
  *
  * This file is a part of Skytable
  * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@@ -25,166 +25,186 @@
 */
 
 use {
-    super::{
-        spec::{Dataspec1D, DataspecMeta1D, DataspecMethods1D, DataspecRaw1D},
-        tag::{DataTag, FullTag, TagUnique},
+    crate::engine::{
+        data::tag::{DataTag, FullTag, TagClass, TagUnique},
+        mem::{DwordQN, SpecialPaddedWord},
     },
-    crate::engine::mem::{DwordQN, SpecialPaddedWord, WordIO},
     core::{
         fmt,
         hash::{Hash, Hasher},
         marker::PhantomData,
-        mem::{self, ManuallyDrop},
-        slice,
+        mem::ManuallyDrop,
+        slice, str,
     },
 };
 
 /*
-    Lit
+    NOTE(@ohsayan): Heinous hackery that should not ever be repeated. Just don't touch anything here.
 */
 
+/// A literal representation
 pub struct Lit<'a> {
-    data: SpecialPaddedWord,
     tag: FullTag,
+    dtc: u8,
+    word: SpecialPaddedWord,
     _lt: PhantomData<&'a [u8]>,
 }
 
 impl<'a> Lit<'a> {
-    pub fn as_ir(&'a self) -> LitIR<'a> {
-        unsafe {
-            // UNSAFE(@ohsayan): 'tis the lifetime. 'tis the savior
-            mem::transmute_copy(self)
-        }
+    /// Create a new bool literal
+    pub fn new_bool(b: bool) -> Self {
+        Self::_quad(b as _, FullTag::BOOL)
     }
-}
-
-impl<'a> DataspecMeta1D for Lit<'a> {
-    type Tag = FullTag;
-    type Target = SpecialPaddedWord;
-    type StringItem = Box<str>;
-    fn new(flag: Self::Tag, data: Self::Target) -> Self {
-        Self {
-            data,
-            tag: flag,
-            _lt: PhantomData,
-        }
+    /// Create a new unsigned integer
+    pub fn new_uint(u: u64) -> Self {
+        Self::_quad(u, FullTag::UINT)
     }
-    fn kind(&self) -> Self::Tag {
-        self.tag
+    /// Create a new signed integer
+    pub fn new_sint(s: i64) -> Self {
+        Self::_quad(s as _, FullTag::SINT)
+    }
+    /// Create a new float64
+    pub fn new_float(f: f64) -> Self {
+        Self::_quad(f.to_bits(), FullTag::FLOAT)
     }
-    fn data(&self) -> Self::Target {
+    /// Returns a "shallow clone"
+    ///
+    /// This function will fall apart if lifetimes aren't handled correctly (aka will segfault)
+    pub fn as_ir(&'a self) -> Lit<'a> {
         unsafe {
-            // UNSAFE(@ohsayan): This function doesn't create any clones, so we're good
-            mem::transmute_copy(self)
+            // UNSAFE(@ohsayan): this is a dirty, uncanny and wild hack that everyone should be forbidden from doing
+            let mut slf: Lit<'a> = core::mem::transmute_copy(self);
+            slf.dtc = Self::DTC_NONE;
+            slf
         }
     }
 }
 
-/*
-    UNSAFE(@ohsayan): Safety checks:
-    - Heap str: yes
-    - Heap bin: no
-    - Drop str: yes, dealloc
-    - Drop bin: not needed
-    - Clone str: yes, alloc
-    - Clone bin: not needed
-*/
-unsafe impl<'a> DataspecRaw1D for Lit<'a> {
-    const HEAP_STR: bool = true;
-    const HEAP_BIN: bool = false;
-    unsafe fn drop_str(&mut self) {
-        let (len, ptr) = self.data().load();
-        drop(String::from_raw_parts(ptr, len, len));
+impl<'a> Lit<'a> {
+    /// Attempt to read a bool
+    pub fn try_bool(&self) -> Option<bool> {
+        (self.tag.tag_class() == TagClass::Bool).then_some(unsafe {
+            // UNSAFE(@ohsayan): +tagck
+            self.bool()
+        })
     }
-    unsafe fn drop_bin(&mut self) {}
-    unsafe fn clone_str(s: &str) -> Self::Target {
-        let new_string = ManuallyDrop::new(s.to_owned().into_boxed_str());
-        WordIO::store((new_string.len(), new_string.as_ptr()))
+    /// Attempt to read an unsigned integer
+    pub fn try_uint(&self) -> Option<u64> {
+        (self.tag.tag_class() == TagClass::UnsignedInt).then_some(unsafe {
+            // UNSAFE(@ohsayan): +tagck
+            self.uint()
+        })
     }
-    unsafe fn clone_bin(b: &[u8]) -> Self::Target {
-        WordIO::store((b.len(), b.as_ptr()))
+    /// Attempt to read a signed integer
+    pub fn try_sint(&self) -> Option<i64> {
+        (self.tag.tag_class() == TagClass::SignedInt).then_some(unsafe {
+            // UNSAFE(@ohsayan): +tagck
+            self.sint()
+        })
     }
-}
-
-/*
-    UNSAFE(@ohsayan): Safety checks:
-    - We LEAK memory because, duh
-    - We don't touch our own targets, ever (well, I'm a bad boy so I do touch it in fmt::Debug)
-*/
-unsafe impl<'a> Dataspec1D for Lit<'a> {
-    fn Str(s: Box<str>) -> Self {
-        let md = ManuallyDrop::new(s);
-        Self::new(FullTag::STR, WordIO::store((md.len(), md.as_ptr())))
+    /// Attempt to read a float
+    pub fn try_float(&self) -> Option<f64> {
+        (self.tag.tag_class() == TagClass::Float).then_some(unsafe {
+            // UNSAFE(@ohsayan): +tagck
+            self.float()
+        })
     }
-}
-
-/*
-    UNSAFE(@ohsayan):
-    - No target touch
-*/
-unsafe impl<'a> DataspecMethods1D for Lit<'a> {}
-
-impl<'a, T: DataspecMethods1D> PartialEq<T> for Lit<'a> {
-    fn eq(&self, other: &T) -> bool {
-        <Self as DataspecMethods1D>::self_eq(self, other)
+    /// Read a bool directly. This function isn't exactly unsafe, but we want to provide a type preserving API
+    pub unsafe fn bool(&self) -> bool {
+        self.uint() == 1
     }
-}
-impl<'a> fmt::Debug for Lit<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let mut f = f.debug_struct("Lit");
-        f.field("tag", &self.tag);
-        self.self_fmt_debug_data("data", &mut f);
-        f.field("_lt", &self._lt);
-        f.finish()
+    /// Read an unsigned integer directly. This function isn't exactly unsafe, but we want to provide a type
+    /// preserving API
+    pub unsafe fn uint(&self) -> u64 {
+        self.word.dwordqn_load_qw_nw().0
     }
-}
-
-impl<'a> Drop for Lit<'a> {
-    fn drop(&mut self) {
-        self.self_drop();
+    /// Read a signed integer directly. This function isn't exactly unsafe, but we want to provide a type
+    /// preserving API
+    pub unsafe fn sint(&self) -> i64 {
+        self.uint() as _
     }
-}
-
-impl<'a> Clone for Lit<'a> {
-    fn clone(&self) -> Self {
-        self.self_clone()
+    /// Read a floating point number directly. This function isn't exactly unsafe, but we want to provide a type
+    /// preserving API
+    pub unsafe fn float(&self) -> f64 {
+        f64::from_bits(self.uint())
     }
 }
 
-impl<'a> ToString for Lit<'a> {
-    fn to_string(&self) -> String {
-        <Self as DataspecMethods1D>::to_string_debug(self)
+impl<'a> Lit<'a> {
+    /// Attempt to read a binary value
+    pub fn try_bin(&self) -> Option<&'a [u8]> {
+        (self.tag.tag_class() == TagClass::Bin).then(|| unsafe {
+            // UNSAFE(@ohsayan): +tagck
+            self.bin()
+        })
     }
-}
-
-direct_from! {
-    Lit<'a> => {
-        bool as Bool,
-        u64 as UnsignedInt,
-        i64 as SignedInt,
-        f64 as Float,
-        &'a str as Str,
-        String as Str,
-        Box<str> as Str,
-        &'a [u8] as Bin,
+    /// Attempt to read a string value
+    pub fn try_str(&self) -> Option<&'a str> {
+        (self.tag.tag_class() == TagClass::Str).then(|| unsafe {
+            // UNSAFE(@ohsayan): +tagck
+            self.str()
+        })
+    }
+    /// Read a string value directly
+    ///
+    /// ## Safety
+    /// The underlying repr MUST be a string. Otherwise you'll segfault or cause other library functions to misbehave
+    pub unsafe fn str(&self) -> &'a str {
+        str::from_utf8_unchecked(self.bin())
+    }
+    /// Read a binary value directly
+    ///
+    /// ## Safety
+    /// The underlying repr MUST be a string. Otherwise you'll segfault
+    pub unsafe fn bin(&self) -> &'a [u8] {
+        let (q, n) = self.word.dwordqn_load_qw_nw();
+        slice::from_raw_parts(n as *const u8 as *mut u8, q as _)
     }
 }
 
-/*
-    LitIR
-*/
-
-/// ☢️TRAIT WARNING☢️: The [`Hash`] implementation is strictly intended for usage with [`crate::engine::core`] components ONLY. This will FAIL and PRODUCE INCORRECT results
-/// when used elsewhere
-pub struct LitIR<'a> {
-    tag: FullTag,
-    data: SpecialPaddedWord,
-    _lt: PhantomData<&'a str>,
+impl<'a> Lit<'a> {
+    /// Create a new string (referenced)
+    pub fn new_str(s: &'a str) -> Self {
+        unsafe {
+            /*
+                UNSAFE(@ohsayan): the mut cast is just for typesake so it doesn't matter while we also set DTC
+                to none so it shouldn't matter anyway
+            */
+            Self::_str(s.as_ptr() as *mut u8, s.len(), Self::DTC_NONE)
+        }
+    }
+    /// Create a new boxed string
+    pub fn new_boxed_str(s: Box<str>) -> Self {
+        let mut md = ManuallyDrop::new(s); // mut -> aliasing!
+        unsafe {
+            // UNSAFE(@ohsayan): correct aliasing, and DTC to destroy heap
+            Self::_str(md.as_mut_ptr(), md.len(), Self::DTC_HSTR)
+        }
+    }
+    /// Create a new string
+    pub fn new_string(s: String) -> Self {
+        Self::new_boxed_str(s.into_boxed_str())
+    }
+    /// Create a new binary (referenced)
+    pub fn new_bin(b: &'a [u8]) -> Self {
+        unsafe {
+            // UNSAFE(@ohsayan): mut cast is once again just a typesake change
+            Self::_wide_word(b.as_ptr() as *mut _, b.len(), Self::DTC_NONE, FullTag::BIN)
+        }
+    }
 }
 
-impl<'a> LitIR<'a> {
-    pub fn __vdata(&self) -> &[u8] {
-        let (vlen, data) = self.data().dwordqn_load_qw_nw();
+impl<'a> Lit<'a> {
+    /// Returns the type of this literal
+    pub fn kind(&self) -> FullTag {
+        self.tag
+    }
+    /// Returns the internal representation of this type
+    pub unsafe fn data(&self) -> &SpecialPaddedWord {
+        &self.word
+    }
+    pub fn __vdata(&self) -> &'a [u8] {
+        let (vlen, data) = self.word.dwordqn_load_qw_nw();
         let len = vlen as usize * (self.kind().tag_unique() >= TagUnique::Bin) as usize;
         unsafe {
             // UNSAFE(@ohsayan): either because of static or lt
@@ -193,136 +213,192 @@ impl<'a> LitIR<'a> {
     }
 }
 
-impl<'a> Hash for LitIR<'a> {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.tag.tag_unique().hash(state);
-        self.__vdata().hash(state);
-    }
-}
-
-impl<'a> DataspecMeta1D for LitIR<'a> {
-    type Target = SpecialPaddedWord;
-    type StringItem = &'a str;
-    type Tag = FullTag;
-    fn new(flag: Self::Tag, data: Self::Target) -> Self {
+impl<'a> Lit<'a> {
+    const DTC_NONE: u8 = 0;
+    const DTC_HSTR: u8 = 1;
+    unsafe fn _new(tag: FullTag, dtc: u8, word: SpecialPaddedWord) -> Self {
         Self {
-            tag: flag,
-            data,
+            tag,
+            dtc,
+            word,
             _lt: PhantomData,
         }
     }
-    fn kind(&self) -> Self::Tag {
-        self.tag
-    }
-    fn data(&self) -> Self::Target {
+    fn _quad(quad: u64, tag: FullTag) -> Self {
         unsafe {
-            // UNSAFE(@ohsayan): We can freely copy our stack because everything is already allocated
-            mem::transmute_copy(self)
+            // UNSAFE(@ohsayan): we initialize the correct bit pattern
+            Self::_new(tag, Self::DTC_NONE, SpecialPaddedWord::new_quad(quad))
         }
     }
-}
-
-/*
-    UNSAFE(@ohsayan): Safety:
-    - Heap str: no
-    - Heap bin: no
-    - Drop str: no
-    - Drop bin: no
-    - Clone str: stack
-    - Clone bin: stack
-*/
-unsafe impl<'a> DataspecRaw1D for LitIR<'a> {
-    const HEAP_STR: bool = false;
-    const HEAP_BIN: bool = false;
-    unsafe fn drop_str(&mut self) {}
-    unsafe fn drop_bin(&mut self) {}
-    unsafe fn clone_str(s: &str) -> Self::Target {
-        WordIO::store((s.len(), s.as_ptr()))
+    unsafe fn _wide_word(ptr: *mut u8, len: usize, dtc: u8, tag: FullTag) -> Self {
+        Self::_new(tag, dtc, SpecialPaddedWord::new(len as _, ptr as _))
     }
-    unsafe fn clone_bin(b: &[u8]) -> Self::Target {
-        WordIO::store((b.len(), b.as_ptr()))
+    unsafe fn _str(ptr: *mut u8, len: usize, dtc: u8) -> Self {
+        Self::_wide_word(ptr, len, dtc, FullTag::STR)
     }
-}
-
-/*
-    UNSAFE(@ohsayan): Safety:
-    - No touches :)
-*/
-unsafe impl<'a> Dataspec1D for LitIR<'a> {
-    fn Str(s: Self::StringItem) -> Self {
-        Self::new(FullTag::STR, WordIO::store((s.len(), s.as_ptr())))
+    unsafe fn _drop_zero(_: SpecialPaddedWord) {}
+    unsafe fn _drop_hstr(word: SpecialPaddedWord) {
+        let (a, b) = word.dwordqn_load_qw_nw();
+        drop(Vec::from_raw_parts(
+            b as *const u8 as *mut u8,
+            a as _,
+            a as _,
+        ));
     }
 }
 
-impl<'a> ToString for LitIR<'a> {
-    fn to_string(&self) -> String {
-        <Self as DataspecMethods1D>::to_string_debug(self)
+impl<'a> Drop for Lit<'a> {
+    fn drop(&mut self) {
+        static DFN: [unsafe fn(SpecialPaddedWord); 2] = [Lit::_drop_zero, Lit::_drop_hstr];
+        unsafe { DFN[self.dtc as usize](core::mem::transmute_copy(&self.word)) }
     }
 }
 
-/*
-    UNSAFE(@ohsayan): Safety:
-    - No touches
-*/
-unsafe impl<'a> DataspecMethods1D for LitIR<'a> {}
-
-impl<'a, T: DataspecMethods1D> PartialEq<T> for LitIR<'a> {
-    fn eq(&self, other: &T) -> bool {
-        <Self as DataspecMethods1D>::self_eq(self, other)
+impl<'a> Clone for Lit<'a> {
+    fn clone(&self) -> Lit<'a> {
+        static CFN: [unsafe fn(SpecialPaddedWord) -> SpecialPaddedWord; 2] = unsafe {
+            [
+                |stack| core::mem::transmute(stack),
+                |hstr| {
+                    let (q, n) = hstr.dwordqn_load_qw_nw();
+                    let mut md = ManuallyDrop::new(
+                        slice::from_raw_parts(n as *const u8, q as usize).to_owned(),
+                    );
+                    md.shrink_to_fit();
+                    SpecialPaddedWord::new(q, md.as_mut_ptr() as _)
+                },
+            ]
+        };
+        unsafe {
+            Self::_new(
+                self.tag,
+                self.dtc,
+                CFN[self.dtc as usize](core::mem::transmute_copy(&self.word)),
+            )
+        }
     }
 }
 
-impl<'a> fmt::Debug for LitIR<'a> {
+impl<'a> fmt::Debug for Lit<'a> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let mut f = f.debug_struct("LitIR");
-        f.field("tag", &self.tag);
-        self.self_fmt_debug_data("data", &mut f);
-        f.field("_lt", &self._lt);
-        f.finish()
+        let mut field = f.debug_struct("Lit");
+        field.field("tag", &self.tag);
+        unsafe {
+            macro_rules! d {
+                ($expr:expr) => {{
+                    field.field("data", &$expr);
+                }};
+            }
+            match self.tag.tag_class() {
+                TagClass::Bool => d!(self.bool()),
+                TagClass::UnsignedInt => d!(self.uint()),
+                TagClass::SignedInt => d!(self.sint()),
+                TagClass::Float => d!(self.float()),
+                TagClass::Bin => d!(self.bin()),
+                TagClass::Str => d!(self.str()),
+                TagClass::List => panic!("found 2D in 1D"),
+            }
+        }
+        field.finish()
     }
 }
 
-impl<'a> Drop for LitIR<'a> {
-    fn drop(&mut self) {
-        self.self_drop();
+impl<'a> Hash for Lit<'a> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.tag.tag_unique().hash(state);
+        self.__vdata().hash(state);
     }
 }
 
-impl<'a> Clone for LitIR<'a> {
-    fn clone(&self) -> Self {
-        self.self_clone()
+impl<'a> PartialEq for Lit<'a> {
+    fn eq(&self, other: &Self) -> bool {
+        unsafe {
+            // UNSAFE(@ohsayan): +tagck
+            match (self.tag.tag_class(), other.tag.tag_class()) {
+                (TagClass::Bool, TagClass::Bool) => self.bool() == other.bool(),
+                (TagClass::UnsignedInt, TagClass::UnsignedInt) => self.uint() == other.uint(),
+                (TagClass::SignedInt, TagClass::SignedInt) => self.sint() == other.sint(),
+                (TagClass::Float, TagClass::Float) => self.float() == other.float(),
+                (TagClass::Bin, TagClass::Bin) => self.bin() == other.bin(),
+                (TagClass::Str, TagClass::Str) => self.str() == other.str(),
+                _ => false,
+            }
+        }
     }
 }
 
 direct_from! {
-    LitIR<'a> => {
-        bool as Bool,
-        u64 as UnsignedInt,
-        i64 as SignedInt,
-        f64 as Float,
-        &'a str as Str,
-        &'a [u8] as Bin,
+    Lit<'a> => {
+        bool as new_bool,
+        u64 as new_uint,
+        i64 as new_sint,
+        f64 as new_float,
+        &'a str as new_str,
+        String as new_string,
+        Box<str> as new_boxed_str,
+        &'a [u8] as new_bin,
+    }
+}
+
+impl<'a> ToString for Lit<'a> {
+    fn to_string(&self) -> String {
+        unsafe {
+            match self.kind().tag_class() {
+                TagClass::Bool => self.bool().to_string(),
+                TagClass::UnsignedInt => self.uint().to_string(),
+                TagClass::SignedInt => self.sint().to_string(),
+                TagClass::Float => self.float().to_string(),
+                TagClass::Bin => format!("{:?}", self.bin()),
+                TagClass::Str => format!("{:?}", self.str()),
+                TagClass::List => panic!("found 2D in 1D"),
+            }
+        }
     }
 }
 
 #[test]
-fn tlit() {
-    let str1 = Lit::Str("hello".into());
-    let str2 = str1.clone();
-    assert_eq!(str1, str2);
-    assert_eq!(str1.str(), "hello");
-    assert_eq!(str2.str(), "hello");
-    drop(str1);
-    assert_eq!(str2.str(), "hello");
+fn stk_variants() {
+    let stk1 = [
+        Lit::new_bool(true),
+        Lit::new_uint(u64::MAX),
+        Lit::new_sint(i64::MIN),
+        Lit::new_float(f64::MIN),
+        Lit::new_str("hello"),
+        Lit::new_bin(b"world"),
+    ];
+    let stk2 = stk1.clone();
+    assert_eq!(stk1, stk2);
 }
 
 #[test]
-fn tlitir() {
-    let str1 = LitIR::Str("hello");
-    let str2 = str1.clone();
-    assert_eq!(str1, str2);
-    assert_eq!(str1.str(), "hello");
-    assert_eq!(str2.str(), "hello");
-    drop(str1);
-    assert_eq!(str2.str(), "hello");
+fn hp_variants() {
+    let hp1 = [
+        Lit::new_string("hello".into()),
+        Lit::new_string("world".into()),
+    ];
+    let hp2 = hp1.clone();
+    assert_eq!(hp1, hp2);
+}
+
+#[test]
+fn lt_link() {
+    let l = Lit::new_string("hello".into());
+    let l_ir = l.as_ir();
+    assert_eq!(l, l_ir);
+}
+
+#[test]
+fn token_array_lt_test() {
+    let tokens = vec![Lit::new_string("hello".to_string()), Lit::new_str("hi")];
+    #[derive(Debug)]
+    pub struct SelectStatement<'a> {
+        primary_key: Lit<'a>,
+        shorthand: Lit<'a>,
+    }
+    let select_stmt = SelectStatement {
+        primary_key: tokens[0].as_ir(),
+        shorthand: tokens[1].as_ir(),
+    };
+    drop(select_stmt);
+    drop(tokens);
 }
diff --git a/server/src/engine/data/macros.rs b/server/src/engine/data/macros.rs
deleted file mode 100644
index 99e552b6..00000000
--- a/server/src/engine/data/macros.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Created on Mon Feb 27 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/// This is a pretty complex macro that emulates the behavior of an enumeration by making use of flags and macro hacks. You might literally feel it's like a lang match, but nope,
-/// there's a lot of wizardry beneath. Well, it's important to know that it works and you shouldn't touch it UNLESS YOU ABSOLUTELY KNOW what you're doing
-macro_rules! match_data {
-    (match ref $dataitem:ident $tail:tt) => {match_data!(@branch [ #[deny(unreachable_patterns)] match crate::engine::data::tag::DataTag::tag_class(&crate::engine::data::spec::DataspecMeta1D::kind($dataitem))] $dataitem [] $tail)};
-    (match $dataitem:ident $tail:tt) => {match_data!(@branch [ #[deny(unreachable_patterns)] match crate::engine::data::tag::DataTag::tag_class(&crate::engine::data::spec::DataspecMeta1D::kind(&$dataitem))] $dataitem [] $tail)};
-    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] {}) => {match_data!(@defeat0 $decl [$($branch)*])};
-    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $name:ident::$variant:ident($capture:ident) => $ret:expr, $($tail:tt)*}) => {
-        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::tag::TagClass::$variant => {let $capture = unsafe { /* UNSAFE(@ohsayan): flagck */ match_data!(@extract $name $dataitem $variant) }; $ret},] {$($tail)*})
-    };
-    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $name:ident::$variant:ident(_) => $ret:expr, $($tail:tt)*}) => {
-        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::tag::TagClass::$variant => $ret,] {$($tail)*})
-    };
-    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $name:ident::$variant:ident($capture:ident) if $guard:expr => $ret:expr, $($tail:tt)*}) => {
-        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::tag::TagClass::$variant if { let $capture = unsafe { /* UNSAFE(@ohsayan): flagck */ match_data!(@extract $name $dataitem $variant) }; $guard } => {
-            let $capture = unsafe { /* UNSAFE(@ohsayan): flagck */  match_data!(@extract $name $dataitem $variant) }; let _ = &$capture; $ret}, ] {$($tail)*}
-        )
-    };
-    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $name:ident::$variant:ident(_) if $guard:expr => $ret:expr, $($tail:tt)*}) => {
-        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* crate::engine::data::tag::TagClass::$variant if $guard => $ret,] {$($tail)*})
-    };
-    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* _ => $ret:expr, $($tail:tt)*}) => {
-        match_data!(@branch $decl $dataitem [$($branch)* $(#[$attr])* _ => $ret,] {$($tail)*})
-    };
-    (@branch $decl:tt $dataitem:ident [$($branch:tt)*] { $(#[$attr:meta])* $capture:ident => $ret:expr, $($tail:tt)* }) => {
-        match_data!(@branch $decl $dataitem [ $($branch)* $(#[$attr])* $capture => { $ret},] {$($tail:tt)*})
-    };
-    (@defeat0 [$($decl:tt)*] [$($branch:tt)*]) => {$($decl)* { $($branch)* }};
-    (@extract $name:ident $dataitem:ident Bool) => {<$name as crate::engine::data::spec::Dataspec1D>::read_bool_uck(&$dataitem)};
-    (@extract $name:ident $dataitem:ident UnsignedInt) => {<$name as crate::engine::data::spec::Dataspec1D>::read_uint_uck(&$dataitem)};
-    (@extract $name:ident $dataitem:ident SignedInt) => {<$name as crate::engine::data::spec::Dataspec1D>::read_sint_uck(&$dataitem)};
-    (@extract $name:ident $dataitem:ident Float) => {<$name as crate::engine::data::spec::Dataspec1D>::read_float_uck(&$dataitem)};
-    (@extract $name:ident $dataitem:ident Bin) => {<$name as crate::engine::data::spec::Dataspec1D>::read_bin_uck(&$dataitem)};
-    (@extract $name:ident $dataitem:ident Str) => {<$name as crate::engine::data::spec::Dataspec1D>::read_str_uck(&$dataitem)};
-}
diff --git a/server/src/engine/data/mod.rs b/server/src/engine/data/mod.rs
index 97dedc69..be15bd35 100644
--- a/server/src/engine/data/mod.rs
+++ b/server/src/engine/data/mod.rs
@@ -24,12 +24,9 @@
  *
 */
 
-#[macro_use]
-mod macros;
 pub mod cell;
 pub mod dict;
 pub mod lit;
-pub mod spec;
 pub mod tag;
 pub mod uuid;
 // test
diff --git a/server/src/engine/data/spec.rs b/server/src/engine/data/spec.rs
deleted file mode 100644
index 9a9e71c2..00000000
--- a/server/src/engine/data/spec.rs
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Created on Sun Feb 26 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
-    So, I woke up and chose violence. God bless me and the stack memory. What I've done here is a sin. Do not follow my footsteps here if you want to write safe and maintainable code.
-    -- @ohsayan
-*/
-
-use {
-    super::tag::{DataTag, TagClass},
-    crate::engine::mem::{DwordQN, WordIO},
-    core::{fmt, mem, slice},
-};
-
-#[inline(always)]
-fn when_then<T, F: FnOnce() -> T>(cond: bool, then: F) -> Option<T> {
-    cond.then(then)
-}
-
-/// Information about the type that implements the dataspec traits
-pub trait DataspecMeta1D: Sized {
-    // assoc
-    type Tag: DataTag;
-    /// The target must be able to store (atleast) a native dword
-    type Target: DwordQN;
-    /// The string item. This helps us remain correct with the dtors
-    type StringItem;
-    // fn
-    /// Create a new instance. Usually allocates zero memory *directly*
-    fn new(tag: Self::Tag, data: Self::Target) -> Self;
-    /// Returns the reduced dataflag
-    fn kind(&self) -> Self::Tag;
-    /// Returns the data stack
-    fn data(&self) -> Self::Target;
-}
-
-/// Unsafe dtor/ctor impls for dataspec items. We have no clue about these things, the implementor must take care of them
-///
-/// ## Safety
-///
-/// - Your dtors MUST BE correct
-pub unsafe trait DataspecRaw1D: DataspecMeta1D {
-    /// Is the string heap allocated...anywhere down the line?
-    const HEAP_STR: bool;
-    /// Is the binary heap allocated...anywhere down the line?
-    const HEAP_BIN: bool;
-    /// Drop the string, if you need a dtor
-    unsafe fn drop_str(&mut self);
-    /// Drop the binary, if you need a dtor
-    unsafe fn drop_bin(&mut self);
-    /// Clone the string object. Note, we literally HAVE NO IDEA about what you're doing here
-    unsafe fn clone_str(s: &str) -> Self::Target;
-    /// Clone the binary object. Again, NOT A DAMN CLUE about whay you're doing down there
-    unsafe fn clone_bin(b: &[u8]) -> Self::Target;
-}
-
-/// Functions that can be used to read/write to/from dataspec objects
-///
-/// ## Safety
-/// - You must touch your targets by yourself
-pub unsafe trait Dataspec1D: DataspecMeta1D + DataspecRaw1D {
-    // store
-    /// Store a new bool. This function is always safe to call
-    #[allow(non_snake_case)]
-    fn Bool(b: bool) -> Self {
-        Self::new(Self::Tag::BOOL, WordIO::store(b))
-    }
-    /// Store a new uint. This function is always safe to call
-    #[allow(non_snake_case)]
-    fn UnsignedInt(u: u64) -> Self {
-        Self::new(Self::Tag::UINT, WordIO::store(u))
-    }
-    /// Store a new sint. This function is always safe to call
-    #[allow(non_snake_case)]
-    fn SignedInt(s: i64) -> Self {
-        Self::new(Self::Tag::SINT, WordIO::store(s))
-    }
-    /// Store a new float. This function is always safe to call
-    #[allow(non_snake_case)]
-    fn Float(f: f64) -> Self {
-        Self::new(Self::Tag::FLOAT, WordIO::store(f.to_bits()))
-    }
-    /// Store a new binary. This function is always safe to call
-    #[allow(non_snake_case)]
-    fn Bin(b: &[u8]) -> Self {
-        Self::new(Self::Tag::BIN, WordIO::store((b.len(), b.as_ptr())))
-    }
-
-    /// Store a new string. Now, I won't talk about this one's safety because it depends on the implementor
-    #[allow(non_snake_case)]
-    fn Str(s: Self::StringItem) -> Self;
-
-    // load
-    // bool
-    /// Load a bool (this is unsafe for logical verity)
-    unsafe fn read_bool_uck(&self) -> bool {
-        self.data().load()
-    }
-    /// Load a bool
-    fn read_bool_try(&self) -> Option<bool> {
-        when_then(self.kind().tag_class() == TagClass::Bool, || unsafe {
-            // UNSAFE(@ohsayan): we've verified the flag. but lol because this isn't actually unsafe
-            self.read_bool_uck()
-        })
-    }
-    /// Load a bool
-    /// ## Panics
-    /// If you're not a bool, you panic
-    fn bool(&self) -> bool {
-        self.read_bool_try().unwrap()
-    }
-    // uint
-    /// Load a uint (this is unsafe for logical verity)
-    unsafe fn read_uint_uck(&self) -> u64 {
-        self.data().load()
-    }
-    /// Load a uint
-    fn read_uint_try(&self) -> Option<u64> {
-        when_then(
-            self.kind().tag_class() == TagClass::UnsignedInt,
-            || unsafe {
-                // UNSAFE(@ohsayan): we've verified the flag. but lol because this isn't actually unsafe
-                self.read_uint_uck()
-            },
-        )
-    }
-    /// Load a uint
-    /// ## Panics
-    /// If you're not a uint, you panic
-    fn uint(&self) -> u64 {
-        self.read_uint_try().unwrap()
-    }
-    // sint
-    /// Load a sint (unsafe for logical verity)
-    unsafe fn read_sint_uck(&self) -> i64 {
-        self.data().load()
-    }
-    /// Load a sint
-    fn read_sint_try(&self) -> Option<i64> {
-        when_then(self.kind().tag_class() == TagClass::SignedInt, || unsafe {
-            // UNSAFE(@ohsayan): we've verified the flag. but lol because this isn't actually unsafe
-            self.read_sint_uck()
-        })
-    }
-    /// Load a sint and panic if we're not a sint
-    fn sint(&self) -> i64 {
-        self.read_sint_try().unwrap()
-    }
-    // float
-    /// Load a float (unsafe for logical verity)
-    unsafe fn read_float_uck(&self) -> f64 {
-        self.data().load()
-    }
-    /// Load a float
-    fn read_float_try(&self) -> Option<f64> {
-        when_then(self.kind().tag_class() == TagClass::Float, || unsafe {
-            self.read_float_uck()
-        })
-    }
-    /// Load a float and panic if we aren't one
-    fn float(&self) -> f64 {
-        self.read_float_try().unwrap()
-    }
-    // bin
-    /// Load a binary
-    ///
-    /// ## Safety
-    /// Are you a binary? Did you store it correctly? Are you a victim of segfaults?
-    unsafe fn read_bin_uck(&self) -> &[u8] {
-        let (l, p) = self.data().load();
-        slice::from_raw_parts(p, l)
-    }
-    /// Load a bin
-    fn read_bin_try(&self) -> Option<&[u8]> {
-        when_then(self.kind().tag_class() == TagClass::Bin, || unsafe {
-            self.read_bin_uck()
-        })
-    }
-    /// Load a bin or panic if we aren't one
-    fn bin(&self) -> &[u8] {
-        self.read_bin_try().unwrap()
-    }
-    // str
-    /// Load a str
-    ///
-    /// ## Safety
-    /// Are you a str? Did you store it correctly? Are you a victim of segfaults?
-    unsafe fn read_str_uck(&self) -> &str {
-        mem::transmute(self.read_bin_uck())
-    }
-    /// Load a str
-    fn read_str_try(&self) -> Option<&str> {
-        when_then(self.kind().tag_class() == TagClass::Str, || unsafe {
-            self.read_str_uck()
-        })
-    }
-    /// Load a str and panic if we aren't one
-    fn str(&self) -> &str {
-        self.read_str_try().unwrap()
-    }
-}
-
-/// Common impls
-///
-/// ## Safety
-/// - You are not touching your target
-pub unsafe trait DataspecMethods1D: Dataspec1D {
-    fn self_drop(&mut self) {
-        match self.kind().tag_class() {
-            TagClass::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
-                // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
-                <Self as DataspecRaw1D>::drop_str(self)
-            },
-            TagClass::Bin if <Self as DataspecRaw1D>::HEAP_BIN => unsafe {
-                // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
-                <Self as DataspecRaw1D>::drop_bin(self)
-            },
-            _ => {}
-        }
-    }
-    fn self_clone(&self) -> Self {
-        let data = match self.kind().tag_class() {
-            TagClass::Str if <Self as DataspecRaw1D>::HEAP_STR => unsafe {
-                // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
-                <Self as DataspecRaw1D>::clone_str(Dataspec1D::read_str_uck(self))
-            },
-            TagClass::Bin if <Self as DataspecRaw1D>::HEAP_BIN => unsafe {
-                // UNSAFE(@ohsayan): we are heap allocated, and we're calling the implementor's definition
-                <Self as DataspecRaw1D>::clone_bin(Dataspec1D::read_bin_uck(self))
-            },
-            _ => self.data(),
-        };
-        Self::new(self.kind(), data)
-    }
-    fn self_eq(&self, other: &impl DataspecMethods1D) -> bool {
-        unsafe {
-            // UNSAFE(@ohsayan): we are checking our flags
-            match (self.kind().tag_class(), other.kind().tag_class()) {
-                (TagClass::Bool, TagClass::Bool) => self.read_bool_uck() == other.read_bool_uck(),
-                (TagClass::UnsignedInt, TagClass::UnsignedInt) => {
-                    self.read_uint_uck() == other.read_uint_uck()
-                }
-                (TagClass::SignedInt, TagClass::SignedInt) => {
-                    self.read_sint_uck() == other.read_sint_uck()
-                }
-                (TagClass::Float, TagClass::Float) => {
-                    self.read_float_uck() == other.read_float_uck()
-                }
-                (TagClass::Bin, TagClass::Bin) => self.read_bin_uck() == other.read_bin_uck(),
-                (TagClass::Str, TagClass::Str) => self.read_str_uck() == other.read_str_uck(),
-                _ => false,
-            }
-        }
-    }
-    fn self_fmt_debug_data(&self, data_field: &str, f: &mut fmt::DebugStruct) {
-        macro_rules! fmtdebug {
-            ($($(#[$attr:meta])* $match:pat => $ret:expr),* $(,)?) => {
-                match self.kind().tag_class() {$($(#[$attr])* $match => { let _x = $ret; f.field(data_field, &_x) },)*}
-            }
-        }
-        unsafe {
-            // UNSAFE(@ohsayan): we are checking our flags
-            fmtdebug!(
-                TagClass::Bool => self.read_bool_uck(),
-                TagClass::UnsignedInt => self.read_uint_uck(),
-                TagClass::SignedInt => self.read_sint_uck(),
-                TagClass::Float => self.read_float_uck(),
-                TagClass::Bin => self.read_bin_uck(),
-                TagClass::Str => self.read_str_uck(),
-                #[allow(unreachable_code)]
-                TagClass::List => unreachable!("found 2D data in 1D"),
-            )
-        };
-    }
-    #[rustfmt::skip]
-    fn to_string_debug(&self) -> String {
-        match_data!(match ref self {
-            Self::Bool(b) => b.to_string(),
-            Self::UnsignedInt(u) => u.to_string(),
-            Self::SignedInt(s) => s.to_string(),
-            Self::Float(f) => f.to_string(),
-            Self::Bin(b) => format!("{:?}", b),
-            Self::Str(s) => format!("{:?}", s),
-            Self::List(_) => unreachable!("found 2D data in 1D"),
-        })
-    }
-}
diff --git a/server/src/engine/data/tests/mod.rs b/server/src/engine/data/tests/mod.rs
index 4c931392..1586b8e4 100644
--- a/server/src/engine/data/tests/mod.rs
+++ b/server/src/engine/data/tests/mod.rs
@@ -25,21 +25,11 @@
 */
 
 mod md_dict_tests;
-use super::{
-    lit::{Lit, LitIR},
-    spec::Dataspec1D,
-};
-
-#[test]
-fn t_largest_int_litir() {
-    let x = LitIR::UnsignedInt(u64::MAX);
-    let y = LitIR::UnsignedInt(u64::MAX);
-    assert_eq!(x, y);
-}
+use super::lit::Lit;
 
 #[test]
 fn t_largest_int_lit() {
-    let x = Lit::UnsignedInt(u64::MAX);
-    let y = Lit::UnsignedInt(u64::MAX);
+    let x = Lit::new_uint(u64::MAX);
+    let y = Lit::new_uint(u64::MAX);
     assert_eq!(x, y);
 }
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index 290c0597..eca293bc 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -69,6 +69,12 @@ impl SpecialPaddedWord {
     pub const unsafe fn new(a: u64, b: usize) -> Self {
         Self { a, b }
     }
+    pub fn new_quad(a: u64) -> Self {
+        Self {
+            a,
+            b: ZERO_BLOCK.as_ptr() as usize,
+        }
+    }
 }
 
 pub trait StatelessLen {
diff --git a/server/src/engine/mem/scanner.rs b/server/src/engine/mem/scanner.rs
index 892a4548..f761b340 100644
--- a/server/src/engine/mem/scanner.rs
+++ b/server/src/engine/mem/scanner.rs
@@ -29,190 +29,408 @@ use core::{ptr, slice};
 pub type BufferedScanner<'a> = Scanner<'a, u8>;
 
 #[derive(Debug, PartialEq)]
+/// A scanner over a slice buffer `[T]`
 pub struct Scanner<'a, T> {
     d: &'a [T],
     __cursor: usize,
 }
 
 impl<'a, T> Scanner<'a, T> {
+    /// Create a new scanner, starting at position 0
     pub const fn new(d: &'a [T]) -> Self {
-        unsafe { Self::new_with_cursor(d, 0) }
+        unsafe {
+            // UNSAFE(@ohsayan): starting with 0 is always correct
+            Self::new_with_cursor(d, 0)
+        }
     }
+    /// Create a new scanner, starting with the given position
+    ///
+    /// ## Safety
+    ///
+    /// `i` must be a valid index into the given slice
     pub const unsafe fn new_with_cursor(d: &'a [T], i: usize) -> Self {
         Self { d, __cursor: i }
     }
+}
+
+impl<'a, T> Scanner<'a, T> {
+    pub const fn buffer_len(&self) -> usize {
+        self.d.len()
+    }
+    /// Returns the remaining number of **items**
     pub const fn remaining(&self) -> usize {
-        self.d.len() - self.__cursor
+        self.buffer_len() - self.__cursor
     }
+    /// Returns the number of items consumed by the scanner
     pub const fn consumed(&self) -> usize {
         self.__cursor
     }
+    /// Returns the current cursor position
     pub const fn cursor(&self) -> usize {
         self.__cursor
     }
-    pub fn current(&self) -> &[T] {
+    /// Returns the buffer from the current position
+    pub fn current_buffer(&self) -> &[T] {
         &self.d[self.__cursor..]
     }
+    /// Returns the ptr to the cursor
+    ///
+    /// WARNING: The pointer might be invalid!
     pub const fn cursor_ptr(&self) -> *const T {
-        unsafe { self.d.as_ptr().add(self.__cursor) }
+        unsafe {
+            // UNSAFE(@ohsayan): assuming that the cursor is correctly initialized, this is always fine
+            self.d.as_ptr().add(self.__cursor)
+        }
     }
+    /// Returns true if the scanner has reached eof
     pub fn eof(&self) -> bool {
         self.remaining() == 0
     }
+    /// Returns true if the scanner has atleast `sizeof` bytes remaining
     pub fn has_left(&self, sizeof: usize) -> bool {
         self.remaining() >= sizeof
     }
-    pub fn matches_cursor_rounded(&self, f: impl Fn(&T) -> bool) -> bool {
-        f(&self.d[(self.d.len() - 1).min(self.__cursor)])
+    /// Returns true if the rounded cursor matches the predicate
+    pub fn rounded_cursor_matches(&self, f: impl Fn(&T) -> bool) -> bool {
+        f(&self.d[self.rounded_cursor()])
+    }
+    /// Same as `rounded_cursor_matches`, but with the added guarantee that no rounding was done
+    pub fn rounded_cursor_not_eof_matches(&self, f: impl Fn(&T) -> bool) -> bool {
+        self.rounded_cursor_matches(f) & !self.eof()
     }
-    pub fn matches_cursor_rounded_and_not_eof(&self, f: impl Fn(&T) -> bool) -> bool {
-        self.matches_cursor_rounded(f) & !self.eof()
+    /// A shorthand for equality in `rounded_cursor_not_eof_matches`
+    pub fn rounded_cursor_not_eof_equals(&self, v_t: T) -> bool
+    where
+        T: PartialEq,
+    {
+        self.rounded_cursor_matches(|v| v_t.eq(v)) & !self.eof()
     }
 }
 
 impl<'a, T> Scanner<'a, T> {
+    /// Manually set the cursor position
+    ///
+    /// ## Safety
+    /// The index must be valid
     pub unsafe fn set_cursor(&mut self, i: usize) {
         self.__cursor = i;
     }
-    pub unsafe fn move_ahead(&mut self) {
-        self.move_back_by(1)
+    /// Increment the cursor
+    ///
+    /// ## Safety
+    /// The buffer must not have reached EOF
+    pub unsafe fn incr_cursor(&mut self) {
+        self.incr_cursor_by(1)
     }
-    pub unsafe fn move_ahead_by(&mut self, by: usize) {
-        self._incr(by)
+    /// Increment the cursor by the given amount
+    ///
+    /// ## Safety
+    /// The buffer must have atleast `by` remaining
+    pub unsafe fn incr_cursor_by(&mut self, by: usize) {
+        self.__cursor += by;
     }
-    pub unsafe fn move_back(&mut self) {
-        self.move_back_by(1)
+    /// Increment the cursor if the given the condition is satisfied
+    ///
+    /// ## Safety
+    /// Custom logic should ensure only legal cursor increments
+    pub unsafe fn incr_cursor_if(&mut self, iff: bool) {
+        self.incr_cursor_by(iff as _)
     }
-    pub unsafe fn move_back_by(&mut self, by: usize) {
+    /// Decrement the cursor
+    ///
+    /// ## Safety
+    /// The cursor must **not be at 0**
+    pub unsafe fn decr_cursor(&mut self) {
+        self.decr_cursor_by(1)
+    }
+    /// Decrement the cursor by the given amount
+    ///
+    /// ## Safety
+    /// Should not overflow (overflow safety is ... nevermind)
+    pub unsafe fn decr_cursor_by(&mut self, by: usize) {
         self.__cursor -= by;
     }
-    unsafe fn _incr(&mut self, by: usize) {
-        self.__cursor += by;
+    /// Returns the current cursor
+    ///
+    /// ## Safety
+    /// Buffer should NOT be at EOF
+    pub unsafe fn deref_cursor(&self) -> T
+    where
+        T: Copy,
+    {
+        *self.cursor_ptr()
+    }
+    /// Returns the rounded cursor
+    pub fn rounded_cursor(&self) -> usize {
+        (self.buffer_len() - 1).min(self.__cursor)
     }
-    unsafe fn _cursor(&self) -> *const T {
-        self.d.as_ptr().add(self.__cursor)
+    /// Returns the current cursor value with rounding
+    pub fn rounded_cursor_value(&self) -> T
+    where
+        T: Copy,
+    {
+        self.d[self.rounded_cursor()]
     }
 }
 
 impl<'a> Scanner<'a, u8> {
+    /// Attempt to parse the next byte
     pub fn try_next_byte(&mut self) -> Option<u8> {
         if self.eof() {
             None
         } else {
-            Some(unsafe { self.next_byte() })
+            Some(unsafe {
+                // UNSAFE(@ohsayan): +remaining check
+                self.next_byte()
+            })
         }
     }
+    /// Attempt to parse the next block
     pub fn try_next_block<const N: usize>(&mut self) -> Option<[u8; N]> {
         if self.has_left(N) {
-            Some(unsafe { self.next_chunk() })
+            Some(unsafe {
+                // UNSAFE(@ohsayan): +remaining check
+                self.next_chunk()
+            })
         } else {
             None
         }
     }
-    pub fn try_next_variable_block(&'a mut self, len: usize) -> Option<&'a [u8]> {
+    /// Attempt to parse the next block (variable)
+    pub fn try_next_variable_block(&mut self, len: usize) -> Option<&'a [u8]> {
         if self.has_left(len) {
-            Some(unsafe { self.next_chunk_variable(len) })
+            Some(unsafe {
+                // UNSAFE(@ohsayan): +remaining check
+                self.next_chunk_variable(len)
+            })
         } else {
             None
         }
     }
 }
 
-pub enum BufferedReadResult<T> {
+/// Incomplete buffered reads
+#[derive(Debug, PartialEq)]
+pub enum ScannerDecodeResult<T> {
+    /// The value was decoded
     Value(T),
+    /// We need more data to determine if we have the correct value
     NeedMore,
+    /// Found an error while decoding a value
     Error,
 }
 
 impl<'a> Scanner<'a, u8> {
+    /// Keep moving the cursor ahead while the predicate returns true
     pub fn trim_ahead(&mut self, f: impl Fn(u8) -> bool) {
-        while self.matches_cursor_rounded_and_not_eof(|b| f(*b)) {
-            unsafe { self.move_ahead() }
+        while self.rounded_cursor_not_eof_matches(|b| f(*b)) {
+            unsafe {
+                // UNSAFE(@ohsayan): not eof
+                self.incr_cursor()
+            }
         }
     }
-    pub fn move_ahead_if_matches(&mut self, f: impl Fn(u8) -> bool) {
-        unsafe { self.move_back_by(self.matches_cursor_rounded_and_not_eof(|b| f(*b)) as _) }
-    }
-    /// Attempt to parse a `\n` terminated (we move past the LF, so you can't see it)
+    /// Attempt to parse a `\n` terminated integer (we move past the LF, so you can't see it)
     ///
     /// If we were unable to read in the integer, then the cursor will be restored to its starting position
     // TODO(@ohsayan): optimize
-    pub fn try_next_ascii_u64_lf_separated_with_result(&mut self) -> BufferedReadResult<u64> {
+    pub fn try_next_ascii_u64_lf_separated_with_result_or_restore_cursor(
+        &mut self,
+    ) -> ScannerDecodeResult<u64> {
+        self.try_next_ascii_u64_lf_separated_with_result_or::<true>()
+    }
+    pub fn try_next_ascii_u64_lf_separated_with_result(&mut self) -> ScannerDecodeResult<u64> {
+        self.try_next_ascii_u64_lf_separated_with_result_or::<false>()
+    }
+    pub fn try_next_ascii_u64_lf_separated_with_result_or<const RESTORE_CURSOR: bool>(
+        &mut self,
+    ) -> ScannerDecodeResult<u64> {
         let mut okay = true;
         let start = self.cursor();
-        let ret = self.extract_integer(&mut okay);
+        let ret = self.try_next_ascii_u64_stop_at_lf(&mut okay);
         let payload_ok = okay;
-        let lf = self.matches_cursor_rounded_and_not_eof(|b| *b == b'\n');
+        let lf = self.rounded_cursor_not_eof_matches(|b| *b == b'\n');
         okay &= lf;
-        unsafe { self._incr(okay as _) }; // skip LF
+        unsafe {
+            // UNSAFE(@ohsayan): not eof
+            // skip LF
+            self.incr_cursor_if(okay)
+        };
         if okay {
-            BufferedReadResult::Value(ret)
+            ScannerDecodeResult::Value(ret)
         } else {
-            unsafe { self.set_cursor(start) }
+            if RESTORE_CURSOR {
+                unsafe {
+                    // UNSAFE(@ohsayan): we correctly restore the cursor
+                    self.set_cursor(start)
+                }
+            }
             if payload_ok {
                 // payload was ok, but we missed a null
-                BufferedReadResult::NeedMore
+                ScannerDecodeResult::NeedMore
             } else {
                 // payload was NOT ok
-                BufferedReadResult::Error
+                ScannerDecodeResult::Error
             }
         }
     }
+    /// Attempt to parse a LF terminated integer (we move past the LF)
+    /// If we were unable to read in the integer, then the cursor will be restored to its starting position
+    pub fn try_next_ascii_u64_lf_separated_or_restore_cursor(&mut self) -> Option<u64> {
+        self.try_next_ascii_u64_lf_separated_or::<true>()
+    }
     pub fn try_next_ascii_u64_lf_separated(&mut self) -> Option<u64> {
+        self.try_next_ascii_u64_lf_separated_or::<false>()
+    }
+    pub fn try_next_ascii_u64_lf_separated_or<const RESTORE_CURSOR: bool>(
+        &mut self,
+    ) -> Option<u64> {
         let start = self.cursor();
         let mut okay = true;
-        let ret = self.extract_integer(&mut okay);
-        let lf = self.matches_cursor_rounded_and_not_eof(|b| *b == b'\n');
+        let ret = self.try_next_ascii_u64_stop_at_lf(&mut okay);
+        let lf = self.rounded_cursor_not_eof_matches(|b| *b == b'\n');
+        unsafe {
+            // UNSAFE(@ohsayan): not eof
+            self.incr_cursor_if(lf & okay)
+        }
         if okay & lf {
             Some(ret)
         } else {
-            unsafe { self.set_cursor(start) }
+            if RESTORE_CURSOR {
+                unsafe {
+                    // UNSAFE(@ohsayan): we correctly restore the cursor
+                    self.set_cursor(start)
+                }
+            }
             None
         }
     }
-    pub fn extract_integer(&mut self, okay: &mut bool) -> u64 {
+    /// Extracts whatever integer is possible using the current bytestream, stopping at a LF (but **not** skipping it)
+    pub fn try_next_ascii_u64_stop_at_lf(&mut self, g_okay: &mut bool) -> u64 {
+        self.try_next_ascii_u64_stop_at::<true>(g_okay, |byte| byte != b'\n')
+    }
+    /// Extracts whatever integer is possible using the current bytestream, stopping only when either an overflow occurs or when
+    /// the closure returns false
+    pub fn try_next_ascii_u64_stop_at<const ASCII_CHECK: bool>(
+        &mut self,
+        g_okay: &mut bool,
+        keep_going_if: impl Fn(u8) -> bool,
+    ) -> u64 {
         let mut ret = 0u64;
-        while self.matches_cursor_rounded_and_not_eof(|b| *b != b'\n') & *okay {
+        let mut okay = true;
+        while self.rounded_cursor_not_eof_matches(|b| keep_going_if(*b)) & okay {
             let b = self.d[self.cursor()];
-            *okay &= b.is_ascii_digit();
+            if ASCII_CHECK {
+                okay &= b.is_ascii_digit();
+            }
             ret = match ret.checked_mul(10) {
                 Some(r) => r,
                 None => {
-                    *okay = false;
+                    okay = false;
                     break;
                 }
             };
             ret = match ret.checked_add((b & 0x0F) as u64) {
                 Some(r) => r,
                 None => {
-                    *okay = false;
+                    okay = false;
                     break;
                 }
             };
-            unsafe { self._incr(1) }
+            unsafe {
+                // UNSAFE(@ohsayan): loop invariant
+                self.incr_cursor_by(1)
+            }
         }
+        *g_okay &= okay;
         ret
     }
 }
 
 impl<'a> Scanner<'a, u8> {
+    /// Attempt to parse the next [`i64`] value, stopping and skipping the STOP_BYTE
+    ///
+    /// WARNING: The cursor is NOT reversed
+    pub fn try_next_ascii_i64_separated_by<const STOP_BYTE: u8>(&mut self) -> (bool, i64) {
+        let (okay, int) = self.try_next_ascii_i64_stop_at(|b| b == STOP_BYTE);
+        let lf = self.rounded_cursor_not_eof_equals(STOP_BYTE);
+        unsafe {
+            // UNSAFE(@ohsayan): not eof
+            self.incr_cursor_if(lf & okay)
+        }
+        (lf & okay, int)
+    }
+    /// Attempt to parse the next [`i64`] value, stopping at the stop condition or stopping if an error occurred
+    ///
+    /// WARNING: It is NOT guaranteed that the stop condition was met
+    pub fn try_next_ascii_i64_stop_at(&mut self, stop_if: impl Fn(u8) -> bool) -> (bool, i64) {
+        let mut ret = 0i64;
+        // check if we have a direction
+        let current = self.rounded_cursor_value();
+        let direction_negative = current == b'-';
+        // skip negative
+        unsafe {
+            // UNSAFE(@ohsayan): not eof
+            self.incr_cursor_if(direction_negative)
+        }
+        let mut okay = direction_negative | current.is_ascii_digit() & !self.eof();
+        while self.rounded_cursor_not_eof_matches(|b| !stop_if(*b)) & okay {
+            let byte = unsafe {
+                // UNSAFE(@ohsayan): loop invariant
+                self.next_byte()
+            };
+            okay &= byte.is_ascii_digit();
+            ret = match ret.checked_mul(10) {
+                Some(r) => r,
+                None => {
+                    okay = false;
+                    break;
+                }
+            };
+            if direction_negative {
+                ret = match ret.checked_sub((byte & 0x0f) as i64) {
+                    Some(r) => r,
+                    None => {
+                        okay = false;
+                        break;
+                    }
+                };
+            } else {
+                ret = match ret.checked_add((byte & 0x0f) as i64) {
+                    Some(r) => r,
+                    None => {
+                        okay = false;
+                        break;
+                    }
+                }
+            }
+        }
+        (okay, ret)
+    }
+}
+
+impl<'a> Scanner<'a, u8> {
+    /// Load the next [`u64`] LE
     pub unsafe fn next_u64_le(&mut self) -> u64 {
         u64::from_le_bytes(self.next_chunk())
     }
+    /// Load the next block
     pub unsafe fn next_chunk<const N: usize>(&mut self) -> [u8; N] {
         let mut b = [0u8; N];
-        ptr::copy_nonoverlapping(self._cursor(), b.as_mut_ptr(), N);
-        self._incr(N);
+        ptr::copy_nonoverlapping(self.cursor_ptr(), b.as_mut_ptr(), N);
+        self.incr_cursor_by(N);
         b
     }
-    pub unsafe fn next_chunk_variable(&mut self, size: usize) -> &[u8] {
-        let r = slice::from_raw_parts(self._cursor(), size);
-        self._incr(size);
+    /// Load the next variable-sized block
+    pub unsafe fn next_chunk_variable(&mut self, size: usize) -> &'a [u8] {
+        let r = slice::from_raw_parts(self.cursor_ptr(), size);
+        self.incr_cursor_by(size);
         r
     }
+    /// Load the next byte
     pub unsafe fn next_byte(&mut self) -> u8 {
-        let r = *self._cursor();
-        self._incr(1);
+        let r = *self.cursor_ptr();
+        self.incr_cursor_by(1);
         r
     }
 }
diff --git a/server/src/engine/mem/tests/mod.rs b/server/src/engine/mem/tests/mod.rs
index efe152a8..b689c053 100644
--- a/server/src/engine/mem/tests/mod.rs
+++ b/server/src/engine/mem/tests/mod.rs
@@ -25,6 +25,7 @@
 */
 
 use super::*;
+mod scanner;
 mod word;
 
 mod vinline {
diff --git a/server/src/engine/mem/tests/scanner.rs b/server/src/engine/mem/tests/scanner.rs
new file mode 100644
index 00000000..e1b2ee94
--- /dev/null
+++ b/server/src/engine/mem/tests/scanner.rs
@@ -0,0 +1,249 @@
+/*
+ * Created on Wed Sep 20 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::mem::scanner::{BufferedScanner, ScannerDecodeResult};
+
+fn s(b: &[u8]) -> BufferedScanner {
+    BufferedScanner::new(b)
+}
+
+/*
+    lf separated
+*/
+
+#[test]
+fn read_u64_lf_separated() {
+    let mut s = s(b"18446744073709551615\n");
+    assert_eq!(
+        s.try_next_ascii_u64_lf_separated_or_restore_cursor()
+            .unwrap(),
+        u64::MAX
+    );
+    assert_eq!(s.cursor(), s.buffer_len());
+}
+
+#[test]
+fn read_u64_lf_separated_missing() {
+    let mut s = s(b"18446744073709551615");
+    assert!(s
+        .try_next_ascii_u64_lf_separated_or_restore_cursor()
+        .is_none());
+    assert_eq!(s.cursor(), 0);
+}
+
+#[test]
+fn read_u64_lf_separated_invalid() {
+    let mut scn = s(b"1844674407370955161A\n");
+    assert!(scn
+        .try_next_ascii_u64_lf_separated_or_restore_cursor()
+        .is_none());
+    assert_eq!(scn.cursor(), 0);
+    let mut scn = s(b"?1844674407370955161A\n");
+    assert!(scn
+        .try_next_ascii_u64_lf_separated_or_restore_cursor()
+        .is_none());
+    assert_eq!(scn.cursor(), 0);
+}
+
+#[test]
+fn read_u64_lf_separated_zero() {
+    let mut s = s(b"0\n");
+    assert_eq!(
+        s.try_next_ascii_u64_lf_separated_or_restore_cursor()
+            .unwrap(),
+        0
+    );
+    assert_eq!(s.cursor(), s.buffer_len());
+}
+
+#[test]
+fn read_u64_lf_overflow() {
+    let mut s = s(b"184467440737095516155\n");
+    assert!(s
+        .try_next_ascii_u64_lf_separated_or_restore_cursor()
+        .is_none());
+    assert_eq!(s.cursor(), 0);
+}
+
+/*
+    lf separated allow unbuffered
+*/
+
+#[test]
+fn incomplete_read_u64_okay() {
+    let mut scn = s(b"18446744073709551615\n");
+    assert_eq!(
+        scn.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor(),
+        ScannerDecodeResult::Value(u64::MAX)
+    );
+    assert_eq!(scn.cursor(), scn.buffer_len());
+}
+
+#[test]
+fn incomplete_read_u64_missing_lf() {
+    let mut scn = s(b"18446744073709551615");
+    assert_eq!(
+        scn.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor(),
+        ScannerDecodeResult::NeedMore
+    );
+    assert_eq!(scn.cursor(), 0);
+}
+
+#[test]
+fn incomplete_read_u64_lf_error() {
+    let mut scn = s(b"1844674407370955161A\n");
+    assert_eq!(
+        scn.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor(),
+        ScannerDecodeResult::Error
+    );
+    assert_eq!(scn.cursor(), 0);
+    let mut scn = s(b"?1844674407370955161A\n");
+    assert_eq!(
+        scn.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor(),
+        ScannerDecodeResult::Error
+    );
+    assert_eq!(scn.cursor(), 0);
+}
+
+#[test]
+fn incomplete_read_u64_lf_zero() {
+    let mut scn = s(b"0\n");
+    assert_eq!(
+        scn.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor(),
+        ScannerDecodeResult::Value(0)
+    )
+}
+
+#[test]
+fn incomplete_read_u64_lf_overflow() {
+    let mut s = s(b"184467440737095516155\n");
+    assert_eq!(
+        s.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor(),
+        ScannerDecodeResult::Error
+    );
+    assert_eq!(s.cursor(), 0);
+}
+
+/*
+    lf separated i64
+*/
+
+fn concat(a: impl ToString, b: impl ToString) -> Vec<u8> {
+    let (a, b) = (a.to_string(), b.to_string());
+    let mut s = String::with_capacity(a.len() + b.len());
+    s.push_str(a.as_str());
+    s.push_str(b.as_str());
+    s.into_bytes()
+}
+
+#[test]
+fn read_i64_lf_separated_okay() {
+    let buf = concat(i64::MAX, "\n");
+    let mut scn = s(&buf);
+    assert_eq!(
+        scn.try_next_ascii_i64_separated_by::<b'\n'>(),
+        (true, i64::MAX)
+    );
+    assert_eq!(scn.cursor(), scn.buffer_len());
+    let buf = concat(i64::MIN, "\n");
+    let mut scn = s(&buf);
+    assert_eq!(
+        scn.try_next_ascii_i64_separated_by::<b'\n'>(),
+        (true, i64::MIN)
+    );
+    assert_eq!(scn.cursor(), scn.buffer_len());
+}
+
+#[test]
+fn read_i64_lf_separated_missing() {
+    let buf = concat(i64::MAX, "");
+    let mut scn = s(&buf);
+    assert_eq!(
+        scn.try_next_ascii_i64_separated_by::<b'\n'>(),
+        (false, i64::MAX)
+    );
+    assert_eq!(scn.cursor(), scn.buffer_len());
+    let buf = concat(i64::MIN, "");
+    let mut scn = s(&buf);
+    assert_eq!(
+        scn.try_next_ascii_i64_separated_by::<b'\n'>(),
+        (false, i64::MIN)
+    );
+    assert_eq!(scn.cursor(), scn.buffer_len());
+}
+
+#[test]
+fn read_i64_lf_separated_invalid() {
+    let buf = concat(i64::MAX, "A\n");
+    let mut scn = s(&buf);
+    assert_eq!(
+        scn.try_next_ascii_i64_separated_by::<b'\n'>(),
+        (false, i64::MAX)
+    );
+    assert_eq!(scn.cursor(), scn.buffer_len() - 1);
+    let buf = concat("A", format!("{}\n", i64::MIN));
+    let mut scn = s(&buf);
+    assert_eq!(scn.try_next_ascii_i64_separated_by::<b'\n'>(), (false, 0));
+    assert_eq!(scn.cursor(), 0);
+}
+
+#[test]
+fn read_i64_lf_overflow() {
+    let buf = concat(u64::MAX, "\n");
+    let mut scn = s(&buf);
+    assert_eq!(
+        scn.try_next_ascii_i64_separated_by::<b'\n'>(),
+        (false, 1844674407370955161)
+    );
+    assert_eq!(scn.cursor(), scn.buffer_len() - 1);
+}
+
+#[test]
+fn read_i64_lf_underflow() {
+    let buf = concat(i64::MIN, "1\n");
+    let mut scn = s(&buf);
+    assert_eq!(
+        scn.try_next_ascii_i64_separated_by::<b'\n'>(),
+        (false, -9223372036854775808)
+    );
+    assert_eq!(scn.cursor(), scn.buffer_len() - 1);
+}
+
+#[test]
+fn rounding() {
+    let mut scanner = s(b"123");
+    for i in 1..=u8::MAX {
+        match i {
+            1..=3 => {
+                assert_eq!(scanner.try_next_byte().unwrap(), (i + b'0'));
+            }
+            _ => {
+                assert_eq!(scanner.rounded_cursor_value(), b'3');
+            }
+        }
+    }
+    assert_eq!(scanner.cursor(), scanner.buffer_len());
+}
diff --git a/server/src/engine/net/mod.rs b/server/src/engine/net/mod.rs
index 9975ae49..213135b9 100644
--- a/server/src/engine/net/mod.rs
+++ b/server/src/engine/net/mod.rs
@@ -30,7 +30,7 @@ mod protocol;
 pub trait Socket: AsyncWrite + AsyncRead + Unpin {}
 pub type IoResult<T> = Result<T, std::io::Error>;
 
-enum QLoopReturn {
+pub enum QLoopReturn {
     Fin,
     ConnectionRst,
 }
diff --git a/server/src/engine/net/protocol/data_exchange.rs b/server/src/engine/net/protocol/data_exchange.rs
index 8b024e04..b964ca9c 100644
--- a/server/src/engine/net/protocol/data_exchange.rs
+++ b/server/src/engine/net/protocol/data_exchange.rs
@@ -98,7 +98,7 @@ fn parse_lf_separated(
 ) -> LFTIntParseResult {
     let mut ret = previously_buffered;
     let mut okay = true;
-    while scanner.matches_cursor_rounded_and_not_eof(|b| *b != b'\n') & okay {
+    while scanner.rounded_cursor_not_eof_matches(|b| *b != b'\n') & okay {
         let b = unsafe { scanner.next_byte() };
         okay &= b.is_ascii_digit();
         ret = match ret.checked_mul(10) {
@@ -111,8 +111,8 @@ fn parse_lf_separated(
         };
     }
     let payload_ok = okay;
-    let lf_ok = scanner.matches_cursor_rounded_and_not_eof(|b| *b == b'\n');
-    unsafe { scanner.move_ahead_by(lf_ok as usize) }
+    let lf_ok = scanner.rounded_cursor_not_eof_matches(|b| *b == b'\n');
+    unsafe { scanner.incr_cursor_by(lf_ok as usize) }
     if payload_ok & lf_ok {
         LFTIntParseResult::Value(ret)
     } else {
@@ -181,8 +181,8 @@ impl<'a> CSQuery<'a> {
             let slice;
             unsafe {
                 // UNSAFE(@ohsayan): checked len at branch
-                slice = slice::from_raw_parts(scanner.current().as_ptr(), size);
-                scanner.move_ahead_by(size);
+                slice = slice::from_raw_parts(scanner.current_buffer().as_ptr(), size);
+                scanner.incr_cursor_by(size);
             }
             CSQueryExchangeResult::Completed(CSQuery::new(slice))
         } else {
diff --git a/server/src/engine/net/protocol/handshake.rs b/server/src/engine/net/protocol/handshake.rs
index a5d9616d..c1d0545b 100644
--- a/server/src/engine/net/protocol/handshake.rs
+++ b/server/src/engine/net/protocol/handshake.rs
@@ -26,7 +26,7 @@
 
 use {
     crate::{
-        engine::mem::scanner::{BufferedReadResult, BufferedScanner},
+        engine::mem::scanner::{BufferedScanner, ScannerDecodeResult},
         util::compiler,
     },
     std::slice,
@@ -320,9 +320,10 @@ impl<'a> CHandshake<'a> {
             // we're done here
             return unsafe {
                 // UNSAFE(@ohsayan): we just checked buffered size
-                let uname = slice::from_raw_parts(scanner.current().as_ptr(), uname_l);
-                let pwd = slice::from_raw_parts(scanner.current().as_ptr().add(uname_l), pwd_l);
-                scanner.move_ahead_by(uname_l + pwd_l);
+                let uname = slice::from_raw_parts(scanner.current_buffer().as_ptr(), uname_l);
+                let pwd =
+                    slice::from_raw_parts(scanner.current_buffer().as_ptr().add(uname_l), pwd_l);
+                scanner.incr_cursor_by(uname_l + pwd_l);
                 HandshakeResult::Completed(Self::new(
                     static_hs,
                     Some(CHandshakeAuth::new(uname, pwd)),
@@ -367,15 +368,16 @@ impl<'a> CHandshake<'a> {
             AuthMode::Password => {}
         }
         // let us see if we can parse the username length
-        let uname_l = match scanner.try_next_ascii_u64_lf_separated_with_result() {
-            BufferedReadResult::NeedMore => {
+        let uname_l = match scanner.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor()
+        {
+            ScannerDecodeResult::NeedMore => {
                 return HandshakeResult::ChangeState {
                     new_state: HandshakeState::StaticBlock(static_header),
                     expect: AuthMode::Password.min_payload_bytes(), // 2 for uname_l and 2 for pwd_l
                 };
             }
-            BufferedReadResult::Value(v) => v as usize,
-            BufferedReadResult::Error => {
+            ScannerDecodeResult::Value(v) => v as usize,
+            ScannerDecodeResult::Error => {
                 return HandshakeResult::Error(ProtocolError::CorruptedHSPacket)
             }
         };
@@ -388,16 +390,16 @@ impl<'a> CHandshake<'a> {
         uname_l: usize,
     ) -> HandshakeResult<'a> {
         // we just have to get the password len
-        let pwd_l = match scanner.try_next_ascii_u64_lf_separated_with_result() {
-            BufferedReadResult::Value(v) => v as usize,
-            BufferedReadResult::NeedMore => {
+        let pwd_l = match scanner.try_next_ascii_u64_lf_separated_with_result_or_restore_cursor() {
+            ScannerDecodeResult::Value(v) => v as usize,
+            ScannerDecodeResult::NeedMore => {
                 // newline missing (or maybe there's more?)
                 return HandshakeResult::ChangeState {
                     new_state: HandshakeState::ExpectingMetaForVariableBlock { static_hs, uname_l },
                     expect: uname_l + 2, // space for username + password len
                 };
             }
-            BufferedReadResult::Error => {
+            ScannerDecodeResult::Error => {
                 return HandshakeResult::Error(ProtocolError::CorruptedHSPacket)
             }
         };
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 1ec298db..4fb2ebd6 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -36,7 +36,7 @@ use {
     },
     crate::{
         engine::{
-            data::{cell::Datacell, lit::LitIR},
+            data::{cell::Datacell, lit::Lit},
             error::{Error, QueryResult},
         },
         util::{compiler, MaybeInit},
@@ -162,7 +162,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     ///
     /// ## Safety
     /// - Must ensure that `Self::can_read_lit_rounded` is true
-    pub unsafe fn read_cursor_lit_unchecked(&mut self) -> LitIR<'a> {
+    pub unsafe fn read_cursor_lit_unchecked(&mut self) -> Lit<'a> {
         let tok = self.read();
         Qd::read_lit(&mut self.d, tok)
     }
@@ -171,7 +171,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     ///
     /// ## Safety
     /// - Must ensure that `Self::can_read_lit_from` is true for the token
-    pub unsafe fn read_lit_unchecked_from(&mut self, tok: &'a Token<'a>) -> LitIR<'a> {
+    pub unsafe fn read_lit_unchecked_from(&mut self, tok: &'a Token<'a>) -> Lit<'a> {
         Qd::read_lit(&mut self.d, tok)
     }
     #[inline(always)]
@@ -274,7 +274,7 @@ pub trait QueryData<'a> {
     ///
     /// ## Safety
     /// The current token **must match** the signature of a lit
-    unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a>;
+    unsafe fn read_lit(&mut self, tok: &'a Token) -> Lit<'a>;
     /// Read a lit using the given token and then copy it into a [`DataType`]
     ///
     /// ## Safety
@@ -299,7 +299,7 @@ impl<'a> QueryData<'a> for InplaceData {
         tok.is_lit()
     }
     #[inline(always)]
-    unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a> {
+    unsafe fn read_lit(&mut self, tok: &'a Token) -> Lit<'a> {
         tok.uck_read_lit().as_ir()
     }
     #[inline(always)]
@@ -312,42 +312,6 @@ impl<'a> QueryData<'a> for InplaceData {
     }
 }
 
-#[derive(Debug)]
-pub struct SubstitutedData<'a> {
-    data: &'a [LitIR<'a>],
-}
-impl<'a> SubstitutedData<'a> {
-    #[inline(always)]
-    pub const fn new(src: &'a [LitIR<'a>]) -> Self {
-        Self { data: src }
-    }
-}
-
-impl<'a> QueryData<'a> for SubstitutedData<'a> {
-    #[inline(always)]
-    fn can_read_lit_from(&self, tok: &Token) -> bool {
-        Token![?].eq(tok) && self.nonzero()
-    }
-    #[inline(always)]
-    unsafe fn read_lit(&mut self, tok: &'a Token) -> LitIR<'a> {
-        debug_assert!(Token![?].eq(tok));
-        let ret = self.data[0].clone();
-        self.data = &self.data[1..];
-        ret
-    }
-    #[inline(always)]
-    unsafe fn read_data_type(&mut self, tok: &'a Token) -> Datacell {
-        debug_assert!(Token![?].eq(tok));
-        let ret = self.data[0].clone();
-        self.data = &self.data[1..];
-        Datacell::from(ret)
-    }
-    #[inline(always)]
-    fn nonzero(&self) -> bool {
-        !self.data.is_empty()
-    }
-}
-
 /*
     AST
 */
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index 848c4d7d..53fd6613 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -77,7 +77,7 @@ mod lexer {
     #[bench]
     fn lex_raw_literal(b: &mut Bencher) {
         let src = b"\r44\ne69b10ffcc250ae5091dec6f299072e23b0b41d6a739";
-        let expected = vec![Token::Lit(Lit::Bin(
+        let expected = vec![Token::Lit(Lit::new_bin(
             b"e69b10ffcc250ae5091dec6f299072e23b0b41d6a739",
         ))];
         b.iter(|| assert_eq!(lex_insecure(src).unwrap(), expected));
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index 11645848..76c9087c 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -39,7 +39,7 @@ use {
         ast::{QueryData, State},
         lex::Ident,
     },
-    crate::{engine::data::lit::LitIR, util::compiler},
+    crate::{engine::data::lit::Lit, util::compiler},
     std::collections::HashMap,
 };
 
@@ -59,13 +59,13 @@ fn u(b: bool) -> u8 {
 #[derive(Debug, PartialEq)]
 pub struct RelationalExpr<'a> {
     pub(super) lhs: Ident<'a>,
-    pub(super) rhs: LitIR<'a>,
+    pub(super) rhs: Lit<'a>,
     pub(super) opc: u8,
 }
 
 impl<'a> RelationalExpr<'a> {
     #[inline(always)]
-    pub(super) fn new(lhs: Ident<'a>, rhs: LitIR<'a>, opc: u8) -> RelationalExpr<'a> {
+    pub(super) fn new(lhs: Ident<'a>, rhs: Lit<'a>, opc: u8) -> RelationalExpr<'a> {
         Self { lhs, rhs, opc }
     }
     pub(super) const OP_EQ: u8 = 1;
@@ -77,7 +77,7 @@ impl<'a> RelationalExpr<'a> {
     pub fn filter_hint_none(&self) -> bool {
         self.opc == Self::OP_EQ
     }
-    pub fn rhs(&self) -> LitIR<'a> {
+    pub fn rhs(&self) -> Lit<'a> {
         self.rhs.clone()
     }
     #[inline(always)]
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index 8e159a14..f6a59d4a 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -31,7 +31,7 @@ use {
     crate::{
         engine::{
             core::query_meta::AssignmentOperator,
-            data::lit::LitIR,
+            data::lit::Lit,
             error::{Error, QueryResult},
             ql::{
                 ast::{Entity, QueryData, State},
@@ -60,13 +60,13 @@ pub struct AssignmentExpression<'a> {
     /// the LHS ident
     pub lhs: Ident<'a>,
     /// the RHS lit
-    pub rhs: LitIR<'a>,
+    pub rhs: Lit<'a>,
     /// operator
     pub operator_fn: AssignmentOperator,
 }
 
 impl<'a> AssignmentExpression<'a> {
-    pub fn new(lhs: Ident<'a>, rhs: LitIR<'a>, operator_fn: AssignmentOperator) -> Self {
+    pub fn new(lhs: Ident<'a>, rhs: Lit<'a>, operator_fn: AssignmentOperator) -> Self {
         Self {
             lhs,
             rhs,
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index 6c5ca70c..3d4f9363 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -25,553 +25,471 @@
 */
 
 mod raw;
+pub use raw::{Ident, Keyword, Symbol, Token};
 
 use {
-    self::raw::RawLexer,
     crate::{
         engine::{
-            data::{
-                lit::{Lit, LitIR},
-                spec::Dataspec1D,
-            },
+            data::lit::Lit,
             error::{Error, QueryResult},
+            mem::BufferedScanner,
         },
         util::compiler,
     },
-    core::{fmt, ops::BitOr, slice, str},
+    core::slice,
+    raw::{kwof, symof},
 };
 
-pub use self::raw::{Ident, Keyword, Symbol, Token};
-pub type Slice<'a> = &'a [u8];
-
 /*
-    Lexer impls
+    basic lexer definition
 */
 
-#[derive(Debug)]
-/// This implements the `opmode-dev` for BlueQL
-pub struct InsecureLexer<'a> {
-    base: RawLexer<'a>,
+type Slice<'a> = &'a [u8];
+
+#[derive(Debug, PartialEq)]
+/// The internal lexer impl
+pub struct Lexer<'a> {
+    token_buffer: BufferedScanner<'a>,
+    tokens: Vec<Token<'a>>,
+    last_error: Option<Error>,
 }
 
-impl<'a> InsecureLexer<'a> {
-    #[inline(always)]
-    pub const fn new(src: Slice<'a>) -> Self {
+impl<'a> Lexer<'a> {
+    /// Initialize a new lexer
+    fn new(src: &'a [u8]) -> Self {
         Self {
-            base: RawLexer::new(src),
+            token_buffer: BufferedScanner::new(src),
+            tokens: Vec::new(),
+            last_error: None,
         }
     }
-    #[inline(always)]
-    pub fn lex(src: Slice<'a>) -> QueryResult<Vec<Token<'a>>> {
-        let mut slf = Self::new(src);
-        slf._lex();
-        let RawLexer {
-            tokens, last_error, ..
-        } = slf.base;
-        match last_error {
-            None => Ok(tokens),
-            Some(e) => Err(e),
-        }
+    /// set an error
+    #[inline(never)]
+    #[cold]
+    fn set_error(&mut self, e: Error) {
+        self.last_error = Some(e);
     }
-    #[inline(always)]
-    fn _lex(&mut self) {
-        let slf = &mut self.base;
-        while slf.not_exhausted() && slf.no_error() {
-            match unsafe {
-                // UNSAFE(@ohsayan): Verified non-null from pre
-                slf.deref_cursor()
-            } {
-                byte if byte.is_ascii_alphabetic() => slf.scan_ident_or_keyword(),
-                #[cfg(test)]
-                byte if byte == b'\x01' => {
-                    slf.push_token(Token::IgnorableComma);
-                    unsafe {
-                        // UNSAFE(@ohsayan): All good here. Already read the token
-                        slf.incr_cursor();
-                    }
-                }
-                byte if byte.is_ascii_digit() => Self::scan_unsigned_integer(slf),
-                b'\r' => Self::scan_binary_literal(slf),
-                b'-' => Self::scan_signed_integer(slf),
-                qs @ (b'\'' | b'"') => Self::scan_quoted_string(slf, qs),
-                // blank space or an arbitrary byte
-                b' ' | b'\n' | b'\t' => slf.trim_ahead(),
-                b => slf.scan_byte(b),
-            }
-        }
+    /// push in a new token
+    fn push_token(&mut self, t: impl Into<Token<'a>>) {
+        self.tokens.push(t.into())
+    }
+    fn no_error(&self) -> bool {
+        self.last_error.is_none()
     }
 }
 
-// high-level methods
-impl<'a> InsecureLexer<'a> {
-    #[inline(always)]
-    fn scan_signed_integer(slf: &mut RawLexer<'a>) {
+impl<'a> Lexer<'a> {
+    /// Scan an identifier
+    fn scan_ident(&mut self) -> Slice<'a> {
+        let s = self.token_buffer.cursor_ptr();
         unsafe {
-            // UNSAFE(@ohsayan): We hit an integer hence this was called
-            slf.incr_cursor();
-        }
-        if slf.peek_is(|b| b.is_ascii_digit()) {
-            // we have some digits
-            let start = unsafe {
-                // UNSAFE(@ohsayan): Take the (-) into the parse
-                // TODO(@ohsayan): we can maybe look at a more efficient way later
-                slf.cursor().sub(1)
-            };
-            while slf.peek_is_and_forward(|b| b.is_ascii_digit()) {}
-            let wseof = slf.peek_is(|char| !char.is_ascii_alphabetic()) || slf.exhausted();
-            match unsafe {
-                // UNSAFE(@ohsayan): a sequence of ASCII bytes in the integer range will always be correct unicode
-                str::from_utf8_unchecked(slice::from_raw_parts(
-                    start,
-                    // UNSAFE(@ohsayan): valid cursor and start pointers
-                    slf.cursor().offset_from(start) as usize,
-                ))
-            }
-            .parse::<i64>()
+            while self
+                .token_buffer
+                .rounded_cursor_not_eof_matches(|b| b.is_ascii_alphanumeric() || *b == b'_')
             {
-                Ok(num) if compiler::likely(wseof) => {
-                    slf.push_token(Lit::SignedInt(num));
-                }
-                _ => {
-                    compiler::cold_call(|| slf.set_error(Error::LexInvalidLiteral));
-                }
+                // UNSAFE(@ohsayan): increment cursor, this is valid
+                self.token_buffer.incr_cursor();
             }
-        } else {
-            slf.push_token(Token![-]);
+            // UNSAFE(@ohsayan): valid slice and ptrs
+            slice::from_raw_parts(
+                s,
+                self.token_buffer.current_buffer().as_ptr().offset_from(s) as usize,
+            )
         }
     }
-    #[inline(always)]
-    fn scan_unsigned_integer(slf: &mut RawLexer<'a>) {
-        let s = slf.cursor();
-
-        while slf.peek_is(|b| b.is_ascii_digit()) {
-            unsafe {
-                // UNSAFE(@ohsayan): since we're going ahead, this is correct (until EOA)
-                slf.incr_cursor();
+    /// Scan an identifier or keyword
+    fn scan_ident_or_keyword(&mut self) {
+        let s = self.scan_ident();
+        let st = s.to_ascii_lowercase();
+        match kwof(&st) {
+            Some(kw) => self.tokens.push(kw.into()),
+            // FIXME(@ohsayan): Uh, mind fixing this? The only advantage is that I can keep the graph *memory* footprint small
+            None if st == b"true" || st == b"false" => {
+                self.push_token(Lit::new_bool(st == b"true"))
             }
+            None => self.tokens.push(unsafe {
+                // UNSAFE(@ohsayan): scan_ident only returns a valid ident which is always a string
+                Token::Ident(Ident::new(s))
+            }),
         }
-        /*
-            1234; // valid
-            1234} // valid
-            1234{ // invalid
-            1234, // valid
-            1234a // invalid
-        */
-        let wseof = slf.peek_is(|char| !char.is_ascii_alphabetic()) || slf.exhausted();
-        match unsafe {
-            /*
-                UNSAFE(@ohsayan):
-                (1) Valid cursor and start pointer (since we copy it from the cursor which is correct)
-                (2) All ASCII alphabetic bytes are captured, hence this will always be a correct unicode string
-            */
-            str::from_utf8_unchecked(slice::from_raw_parts(
-                s,
-                slf.cursor().offset_from(s) as usize,
-            ))
+    }
+    fn scan_byte(&mut self, byte: u8) {
+        match symof(byte) {
+            Some(tok) => self.push_token(tok),
+            None => return self.set_error(Error::LexUnexpectedByte),
         }
-        .parse()
-        {
-            Ok(num) if compiler::likely(wseof) => {
-                slf.tokens.push(Token::Lit(Lit::UnsignedInt(num)))
-            }
-            _ => slf.set_error(Error::LexInvalidLiteral),
+        unsafe {
+            // UNSAFE(@ohsayan): we are sent a byte, so fw cursor
+            self.token_buffer.incr_cursor();
         }
     }
+}
 
-    #[inline(always)]
-    fn scan_binary_literal(slf: &mut RawLexer<'a>) {
-        unsafe {
-            // UNSAFE(@ohsayan): cursor increment since we hit the marker byte (CR)
-            slf.incr_cursor();
-        }
-        let mut size = 0usize;
-        let mut okay = true;
-        while slf.not_exhausted()
-            && unsafe {
-                // UNSAFE(@ohsayan): verified non-exhaustion
-                slf.deref_cursor() != b'\n'
-            }
-            && okay
-        {
-            /*
-                Don't ask me how stupid this is. Like, I was probably in some "mood" when I wrote this
-                and it works duh, but isn't the most elegant of things (could I have just used a parse?
-                nah, I'm just a hardcore numeric normie)
-                -- Sayan
-            */
+impl<'a> Lexer<'a> {
+    fn trim_ahead(&mut self) {
+        self.token_buffer
+            .trim_ahead(|b| (b == b' ') | (b == b'\n') | (b == b'\t'))
+    }
+}
+
+/*
+    Insecure lexer
+*/
+
+pub struct InsecureLexer<'a> {
+    l: Lexer<'a>,
+}
+
+impl<'a> InsecureLexer<'a> {
+    pub fn lex(src: &'a [u8]) -> QueryResult<Vec<Token<'a>>> {
+        let slf = Self { l: Lexer::new(src) };
+        slf._lex()
+    }
+    fn _lex(mut self) -> QueryResult<Vec<Token<'a>>> {
+        while !self.l.token_buffer.eof() & self.l.no_error() {
             let byte = unsafe {
-                // UNSAFE(@ohsayan): The pre invariant guarantees that this is correct
-                slf.deref_cursor()
+                // UNSAFE(@ohsayan): loop invariant
+                self.l.token_buffer.deref_cursor()
             };
-            okay &= byte.is_ascii_digit();
-            let (prod, of_flag) = size.overflowing_mul(10);
-            okay &= !of_flag;
-            let (sum, of_flag) = prod.overflowing_add((byte & 0x0F) as _);
-            size = sum;
-            okay &= !of_flag;
-            unsafe {
-                // UNSAFE(@ohsayan): We just read something, so this is fine (until EOA)
-                slf.incr_cursor();
+            match byte {
+                #[cfg(test)]
+                byte if byte == b'\x01' => {
+                    self.l.push_token(Token::IgnorableComma);
+                    unsafe {
+                        // UNSAFE(@ohsayan): All good here. Already read the token
+                        self.l.token_buffer.incr_cursor();
+                    }
+                }
+                // ident
+                byte if byte.is_ascii_alphabetic() | (byte == b'_') => {
+                    self.l.scan_ident_or_keyword()
+                }
+                // uint
+                byte if byte.is_ascii_digit() => self.scan_unsigned_integer(),
+                // sint
+                b'-' => {
+                    unsafe {
+                        // UNSAFE(@ohsayan): loop invariant
+                        self.l.token_buffer.incr_cursor()
+                    };
+                    self.scan_signed_integer();
+                }
+                // binary
+                b'\r' => {
+                    unsafe {
+                        // UNSAFE(@ohsayan): loop invariant
+                        self.l.token_buffer.incr_cursor()
+                    }
+                    self.scan_binary()
+                }
+                // string
+                quote_style @ (b'"' | b'\'') => {
+                    unsafe {
+                        // UNSAFE(@ohsayan): loop invariant
+                        self.l.token_buffer.incr_cursor()
+                    }
+                    self.scan_quoted_string(quote_style)
+                }
+                // whitespace
+                b' ' | b'\n' | b'\t' => self.l.trim_ahead(),
+                // some random byte
+                byte => self.l.scan_byte(byte),
             }
         }
-        okay &= slf.peek_eq_and_forward(b'\n');
-        okay &= slf.remaining() >= size;
-        if compiler::likely(okay) {
-            unsafe {
-                // UNSAFE(@ohsayan): Correct cursor and length (from above we know that we have enough bytes)
-                slf.push_token(Lit::Bin(slice::from_raw_parts(slf.cursor(), size)));
-                // UNSAFE(@ohsayan): Correct length increment
-                slf.incr_cursor_by(size);
-            }
-        } else {
-            slf.set_error(Error::LexInvalidLiteral);
+        match self.l.last_error {
+            None => Ok(self.l.tokens),
+            Some(e) => Err(e),
         }
     }
-    #[inline(always)]
-    fn scan_quoted_string(slf: &mut RawLexer<'a>, quote_style: u8) {
-        debug_assert!(
-            unsafe {
-                // UNSAFE(@ohsayan): yessir, we just hit this byte. if called elsewhere, this function will crash and burn (or simply, segfault)
-                slf.deref_cursor()
-            } == quote_style,
-            "illegal call to scan_quoted_string"
-        );
-        unsafe {
-            // UNSAFE(@ohsayan): Increment this cursor (this is correct since we just hit the quote)
-            slf.incr_cursor()
+}
+
+impl<'a> InsecureLexer<'a> {
+    fn scan_binary(&mut self) {
+        let Some(len) = self
+            .l
+            .token_buffer
+            .try_next_ascii_u64_lf_separated_or_restore_cursor()
+        else {
+            self.l.set_error(Error::LexInvalidLiteral);
+            return;
+        };
+        let len = len as usize;
+        match self.l.token_buffer.try_next_variable_block(len) {
+            Some(block) => self.l.push_token(Lit::new_bin(block)),
+            None => self.l.set_error(Error::LexInvalidLiteral),
         }
+    }
+    fn scan_quoted_string(&mut self, quote_style: u8) {
+        // cursor is at beginning of `"`; we need to scan until the end of quote or an escape
         let mut buf = Vec::new();
-        unsafe {
-            while slf.peek_neq(quote_style) {
-                // UNSAFE(@ohsayan): deref is good since peek passed
-                match slf.deref_cursor() {
-                    b if b != b'\\' => {
-                        buf.push(b);
-                    }
-                    _ => {
-                        // UNSAFE(@ohsayan): we read one byte, so this should work
-                        slf.incr_cursor();
-                        if slf.exhausted() {
-                            break;
+        while self
+            .l
+            .token_buffer
+            .rounded_cursor_not_eof_matches(|b| *b != quote_style)
+        {
+            let byte = unsafe {
+                // UNSAFE(@ohsayan): loop invariant
+                self.l.token_buffer.next_byte()
+            };
+            match byte {
+                b'\\' => {
+                    // hmm, this might be an escape (either `\\` or `\"`)
+                    if self
+                        .l
+                        .token_buffer
+                        .rounded_cursor_not_eof_matches(|b| *b == quote_style || *b == b'\\')
+                    {
+                        // ignore escaped byte
+                        unsafe {
+                            buf.push(self.l.token_buffer.next_byte());
                         }
-                        // UNSAFE(@ohsayan): correct because of the above branch
-                        let b = slf.deref_cursor();
-                        let quote = b == quote_style;
-                        let bs = b == b'\\';
-                        if quote | bs {
-                            buf.push(b);
-                        } else {
-                            break; // what on good earth is that escape?
+                    } else {
+                        // this is not allowed
+                        unsafe {
+                            // UNSAFE(@ohsayan): we move the cursor ahead, now we're moving it back
+                            self.l.token_buffer.decr_cursor()
                         }
+                        self.l.set_error(Error::LexInvalidLiteral);
+                        return;
                     }
                 }
-                /*
-                    UNSAFE(@ohsayan): This is correct because:
-                    (a) If we are in arm 1: we move the cursor ahead from the `\` byte (the branch doesn't do it)
-                    (b) If we are in arm 2: we don't skip the second quote byte in the branch, hence this is correct
-                */
-                slf.incr_cursor();
+                _ => buf.push(byte),
             }
-            let terminated = slf.peek_eq_and_forward(quote_style);
-            match String::from_utf8(buf) {
-                Ok(st) if terminated => slf.tokens.push(Token::Lit(st.into_boxed_str().into())),
-                _ => slf.set_error(Error::LexInvalidLiteral),
+        }
+        let ended_with_quote = self
+            .l
+            .token_buffer
+            .rounded_cursor_not_eof_equals(quote_style);
+        // skip quote
+        unsafe {
+            // UNSAFE(@ohsayan): not eof
+            self.l.token_buffer.incr_cursor_if(ended_with_quote)
+        }
+        match String::from_utf8(buf) {
+            Ok(s) if ended_with_quote => self.l.push_token(Lit::new_string(s)),
+            Err(_) | Ok(_) => self.l.set_error(Error::LexInvalidLiteral),
+        }
+    }
+    fn scan_unsigned_integer(&mut self) {
+        let mut okay = true;
+        // extract integer
+        let int = self
+            .l
+            .token_buffer
+            .try_next_ascii_u64_stop_at::<false>(&mut okay, |b| b.is_ascii_digit());
+        /*
+            see if we ended at a correct byte:
+            iff the integer has an alphanumeric byte at the end is the integer invalid
+        */
+        if compiler::unlikely(
+            !okay
+                | self
+                    .l
+                    .token_buffer
+                    .rounded_cursor_not_eof_matches(u8::is_ascii_alphanumeric),
+        ) {
+            self.l.set_error(Error::LexInvalidLiteral);
+        } else {
+            self.l.push_token(Lit::new_uint(int))
+        }
+    }
+    fn scan_signed_integer(&mut self) {
+        if self.l.token_buffer.rounded_cursor_value().is_ascii_digit() {
+            unsafe {
+                // UNSAFE(@ohsayan): the cursor was moved ahead, now we're moving it back
+                self.l.token_buffer.decr_cursor()
             }
+            let (okay, int) = self
+                .l
+                .token_buffer
+                .try_next_ascii_i64_stop_at(|b| !b.is_ascii_digit());
+            if okay
+                & !self
+                    .l
+                    .token_buffer
+                    .rounded_cursor_value()
+                    .is_ascii_alphabetic()
+            {
+                self.l.push_token(Lit::new_sint(int))
+            } else {
+                self.l.set_error(Error::LexInvalidLiteral)
+            }
+        } else {
+            self.l.push_token(Token![-]);
         }
     }
 }
 
+/*
+    secure
+*/
+
 #[derive(Debug)]
-/// This lexer implements the `opmod-safe` for BlueQL
-pub struct SafeLexer<'a> {
-    base: RawLexer<'a>,
+pub struct SecureLexer<'a> {
+    l: Lexer<'a>,
+    param_buffer: BufferedScanner<'a>,
 }
 
-impl<'a> SafeLexer<'a> {
-    #[inline(always)]
-    pub const fn new(src: Slice<'a>) -> Self {
+impl<'a> SecureLexer<'a> {
+    pub fn new(src: &'a [u8], query_window: usize) -> Self {
         Self {
-            base: RawLexer::new(src),
+            l: Lexer::new(&src[..query_window]),
+            param_buffer: BufferedScanner::new(&src[query_window..]),
         }
     }
-    #[inline(always)]
-    pub fn lex(src: Slice<'a>) -> QueryResult<Vec<Token>> {
-        Self::new(src)._lex()
+    pub fn lex(src: &'a [u8], query_window: usize) -> QueryResult<Vec<Token<'a>>> {
+        Self::new(src, query_window)._lex()
     }
-    #[inline(always)]
-    fn _lex(self) -> QueryResult<Vec<Token<'a>>> {
-        let Self { base: mut l } = self;
-        while l.not_exhausted() && l.no_error() {
+}
+
+impl<'a> SecureLexer<'a> {
+    fn _lex(mut self) -> QueryResult<Vec<Token<'a>>> {
+        while self.l.no_error() & !self.l.token_buffer.eof() {
             let b = unsafe {
-                // UNSAFE(@ohsayan): This is correct because of the pre invariant
-                l.deref_cursor()
+                // UNSAFE(@ohsayan): loop invariant
+                self.l.token_buffer.deref_cursor()
             };
             match b {
-                // ident or kw
-                b if b.is_ascii_alphabetic() => l.scan_ident_or_keyword(),
-                // extra terminal chars
-                b'\n' | b'\t' | b' ' => l.trim_ahead(),
-                // arbitrary byte
-                b => l.scan_byte(b),
+                b if b.is_ascii_alphabetic() | (b == b'_') => self.l.scan_ident_or_keyword(),
+                b'?' => {
+                    // a parameter: null, bool, sint, uint, float, binary, string
+                    const TYPE: [&str; 8] = [
+                        "null", "bool", "uint", "sint", "float", "binary", "string", "ERROR",
+                    ];
+                    // skip the param byte
+                    unsafe {
+                        // UNSAFE(@ohsayan): loop invariant
+                        self.l.token_buffer.incr_cursor()
+                    }
+                    // find target
+                    let ecc_code = SCAN_PARAM.len() - 1;
+                    let target_code = self.param_buffer.rounded_cursor_value();
+                    let target_fn = target_code.min(ecc_code as u8);
+                    // forward if we have target
+                    unsafe {
+                        self.param_buffer
+                            .incr_cursor_by((target_code == target_fn) as _)
+                    }
+                    // check requirements
+                    let has_enough = self
+                        .param_buffer
+                        .has_left(SCAN_PARAM_EXPECT[target_fn as usize] as _);
+                    let final_target =
+                        (has_enough as u8 * target_fn) | (!has_enough as u8 * ecc_code as u8);
+                    // exec
+                    let final_target = final_target as usize;
+                    unsafe {
+                        if final_target >= SCAN_PARAM.len() {
+                            impossible!()
+                        }
+                    }
+                    unsafe {
+                        // UNSAFE(@ohsayan): our computation above ensures that we're meeting the expected target
+                        SCAN_PARAM[final_target](&mut self)
+                    }
+                }
+                b' ' | b'\t' | b'\n' => self.l.trim_ahead(),
+                sym => self.l.scan_byte(sym),
             }
         }
-        let RawLexer {
-            last_error, tokens, ..
-        } = l;
-        match last_error {
-            None => Ok(tokens),
+        match self.l.last_error {
+            None => Ok(self.l.tokens),
             Some(e) => Err(e),
         }
     }
 }
 
-const ALLOW_UNSIGNED: bool = false;
-const ALLOW_SIGNED: bool = true;
-
-pub trait NumberDefinition: Sized + fmt::Debug + Copy + Clone + BitOr<Self, Output = Self> {
-    const ALLOW_SIGNED: bool;
-    fn mul_of(&self, v: u8) -> (Self, bool);
-    fn add_of(&self, v: u8) -> (Self, bool);
-    fn sub_of(&self, v: u8) -> (Self, bool);
-    fn qualified_max_length() -> usize;
-    fn zero() -> Self;
-    fn b(self, b: bool) -> Self;
-}
-
-macro_rules! impl_number_def {
-	($(
-        $ty:ty {$supports_signed:ident, $qualified_max_length:expr}),* $(,)?
-    ) => {
-		$(impl NumberDefinition for $ty {
-			const ALLOW_SIGNED: bool = $supports_signed;
-            #[inline(always)] fn zero() -> Self { 0 }
-            #[inline(always)] fn b(self, b: bool) -> Self { b as Self * self }
-			#[inline(always)]
-			fn mul_of(&self, v: u8) -> ($ty, bool) { <$ty>::overflowing_mul(*self, v as $ty) }
-			#[inline(always)]
-			fn add_of(&self, v: u8) -> ($ty, bool) { <$ty>::overflowing_add(*self, v as $ty) }
-			#[inline(always)]
-			fn sub_of(&self, v: u8) -> ($ty, bool) { <$ty>::overflowing_sub(*self, v as $ty) }
-            #[inline(always)] fn qualified_max_length() -> usize { $qualified_max_length }
-		})*
-	}
-}
-
-#[cfg(target_pointer_width = "64")]
-const SZ_USIZE: usize = 20;
-#[cfg(target_pointer_width = "32")]
-const SZ_USIZE: usize = 10;
-#[cfg(target_pointer_width = "64")]
-const SZ_ISIZE: usize = 20;
-#[cfg(target_pointer_width = "32")]
-const SZ_ISIZE: usize = 11;
-
-impl_number_def! {
-    usize {ALLOW_SIGNED, SZ_USIZE},
-    // 255
-    u8 {ALLOW_UNSIGNED, 3},
-    // 65536
-    u16 {ALLOW_UNSIGNED, 5},
-    // 4294967296
-    u32 {ALLOW_UNSIGNED, 10},
-    // 18446744073709551616
-    u64 {ALLOW_UNSIGNED, 20},
-    // signed
-    isize {ALLOW_SIGNED, SZ_ISIZE},
-    // -128
-    i8 {ALLOW_SIGNED, 4},
-    // -32768
-    i16 {ALLOW_SIGNED, 6},
-    // -2147483648
-    i32 {ALLOW_SIGNED, 11},
-    // -9223372036854775808
-    i64 {ALLOW_SIGNED, 20},
-}
-
-#[inline(always)]
-pub(super) fn decode_num_ub<N>(src: &[u8], flag: &mut bool, cnt: &mut usize) -> N
-where
-    N: NumberDefinition,
-{
-    let l = src.len();
-    let mut okay = !src.is_empty();
-    let mut i = 0;
-    let mut number = N::zero();
-    let mut nx_stop = false;
-
-    let is_signed = if N::ALLOW_SIGNED {
-        let loc_s = i < l && src[i] == b'-';
-        i += loc_s as usize;
-        okay &= (i + 2) <= l; // [-][digit][LF]
-        loc_s
-    } else {
-        false
-    };
-
-    while i < l && okay && !nx_stop {
-        // potential exit
-        nx_stop = src[i] == b'\n';
-        // potential entry
-        let mut local_ok = src[i].is_ascii_digit();
-        let (p, p_of) = number.mul_of(10);
-        local_ok &= !p_of;
-        let lfret = if N::ALLOW_SIGNED && is_signed {
-            let (d, d_of) = p.sub_of(src[i] & 0x0f);
-            local_ok &= !d_of;
-            d
-        } else {
-            let (s, s_of) = p.add_of(src[i] & 0x0f);
-            local_ok &= !s_of;
-            s
-        };
-        // reassign or assign
-        let reassign = number.b(nx_stop);
-        let assign = lfret.b(!nx_stop);
-        number = reassign | assign;
-        okay &= local_ok | nx_stop;
-        i += okay as usize;
-    }
-    if N::ALLOW_SIGNED {
-        number = number.b(okay);
-    }
-    okay &= nx_stop;
-    *cnt += i;
-    *flag &= okay;
-    number
-}
-
-#[derive(Debug, PartialEq)]
-/// Data constructed from `opmode-safe`
-pub struct SafeQueryData<'a> {
-    p: Box<[LitIR<'a>]>,
-    t: Vec<Token<'a>>,
-}
-
-impl<'a> SafeQueryData<'a> {
-    #[cfg(test)]
-    pub fn new_test(p: Box<[LitIR<'a>]>, t: Vec<Token<'a>>) -> Self {
-        Self { p, t }
-    }
-    #[inline(always)]
-    pub fn parse_data(pf: Slice<'a>, pf_sz: usize) -> QueryResult<Box<[LitIR<'a>]>> {
-        Self::p_revloop(pf, pf_sz)
-    }
-    #[inline(always)]
-    pub fn parse(qf: Slice<'a>, pf: Slice<'a>, pf_sz: usize) -> QueryResult<Self> {
-        let q = SafeLexer::lex(qf);
-        let p = Self::p_revloop(pf, pf_sz);
-        match (q, p) {
-            (Ok(q), Ok(p)) => Ok(Self { t: q, p }),
-            // first error
-            (Err(e), _) | (_, Err(e)) => Err(e),
-        }
-    }
-    #[inline]
-    pub(super) fn p_revloop(mut src: Slice<'a>, size: usize) -> QueryResult<Box<[LitIR<'a>]>> {
-        static LITIR_TF: [for<'a> fn(Slice<'a>, &mut usize, &mut Vec<LitIR<'a>>) -> bool; 7] = [
-            SafeQueryData::uint,  // tc: 0
-            SafeQueryData::sint,  // tc: 1
-            SafeQueryData::bool,  // tc: 2
-            SafeQueryData::float, // tc: 3
-            SafeQueryData::bin,   // tc: 4
-            SafeQueryData::str,   // tc: 5
-            |_, _, _| false,      // ecc: 6
-        ];
-        let nonpadded_offset = (LITIR_TF.len() - 2) as u8;
-        let ecc_offset = LITIR_TF.len() - 1;
-        let mut okay = true;
-        let mut data = Vec::with_capacity(size);
-        while src.len() >= 3 && okay {
-            let tc = src[0];
-            okay &= tc <= nonpadded_offset;
-            let mx = ecc_offset.min(tc as usize);
-            let mut i_ = 1;
-            okay &= LITIR_TF[mx](&src[1..], &mut i_, &mut data);
-            src = &src[i_..];
-        }
-        okay &= src.is_empty() && data.len() == size;
-        if compiler::likely(okay) {
-            Ok(data.into_boxed_slice())
-        } else {
-            Err(Error::LexInvalidEscapedLiteral)
-        }
-    }
-}
-
-// low level methods
-impl<'b> SafeQueryData<'b> {
-    #[inline(always)]
-    fn mxple<'a>(src: Slice<'a>, cnt: &mut usize, flag: &mut bool) -> Slice<'a> {
-        // find payload length
-        let mut i = 0;
-        let payload_len = decode_num_ub::<usize>(src, flag, &mut i);
-        let src = &src[i..];
-        // find payload
-        *flag &= src.len() >= payload_len;
-        let mx_extract = payload_len.min(src.len());
-        // incr cursor
-        i += mx_extract;
-        *cnt += i;
-        unsafe {
-            // UNSAFE(@ohsayan): src is correct (guaranteed). even if the decoded length returns an error we still remain within bounds of the EOA
-            slice::from_raw_parts(src.as_ptr(), mx_extract)
-        }
-    }
-    #[inline(always)]
-    pub(super) fn uint<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
-        let mut b = true;
-        let r = decode_num_ub(src, &mut b, cnt);
-        data.push(LitIR::UnsignedInt(r));
-        b
-    }
-    #[inline(always)]
-    pub(super) fn sint<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
-        let mut b = true;
-        let r = decode_num_ub(src, &mut b, cnt);
-        data.push(LitIR::SignedInt(r));
-        b
-    }
-    #[inline(always)]
-    pub(super) fn bool<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
-        // `true\n` or `false\n`
-        let mx = 6.min(src.len());
-        let slice = &src[..mx];
-        let v_true = slice.starts_with(b"true\n");
-        let v_false = slice.starts_with(b"false\n");
-        let incr = v_true as usize * 5 + v_false as usize * 6;
-        data.push(LitIR::Bool(v_true));
-        *cnt += incr;
-        v_true | v_false
-    }
-    #[inline(always)]
-    pub(super) fn float<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
-        let mut okay = true;
-        let payload = Self::mxple(src, cnt, &mut okay);
-        match String::from_utf8_lossy(payload).parse() {
-            Ok(p) if compiler::likely(okay) => {
-                data.push(LitIR::Float(p));
+const SCAN_PARAM_EXPECT: [u8; 8] = [0, 1, 2, 2, 2, 2, 2, 0];
+static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
+    [
+        // null
+        |s| s.l.push_token(Token![null]),
+        // bool
+        |slf| {
+            let nb = slf.param_buffer.next_byte();
+            slf.l.push_token(Token::Lit(Lit::new_bool(nb == 1)));
+            if nb > 1 {
+                slf.l.set_error(Error::LexInvalidEscapedLiteral);
             }
-            _ => {}
-        }
-        okay
-    }
-    #[inline(always)]
-    pub(super) fn bin<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
-        let mut okay = true;
-        let payload = Self::mxple(src, cnt, &mut okay);
-        data.push(LitIR::Bin(payload));
-        okay
-    }
-    #[inline(always)]
-    pub(super) fn str<'a>(src: Slice<'a>, cnt: &mut usize, data: &mut Vec<LitIR<'a>>) -> bool {
-        let mut okay = true;
-        let payload = Self::mxple(src, cnt, &mut okay);
-        match str::from_utf8(payload) {
-            Ok(s) if compiler::likely(okay) => {
-                data.push(LitIR::Str(s));
-                true
+        },
+        // uint
+        |slf| match slf
+            .param_buffer
+            .try_next_ascii_u64_lf_separated_or_restore_cursor()
+        {
+            Some(int) => slf.l.push_token(Lit::new_uint(int)),
+            None => slf.l.set_error(Error::LexInvalidEscapedLiteral),
+        },
+        // sint
+        |slf| {
+            let (okay, int) = slf.param_buffer.try_next_ascii_i64_separated_by::<b'\n'>();
+            if okay {
+                slf.l.push_token(Lit::new_sint(int))
+            } else {
+                slf.l.set_error(Error::LexInvalidLiteral)
             }
-            _ => false,
-        }
-    }
-}
+        },
+        // float
+        |slf| {
+            let Some(size_of_body) = slf
+                .param_buffer
+                .try_next_ascii_u64_lf_separated_or_restore_cursor()
+            else {
+                slf.l.set_error(Error::LexInvalidEscapedLiteral);
+                return;
+            };
+            let body = match slf
+                .param_buffer
+                .try_next_variable_block(size_of_body as usize)
+            {
+                Some(body) => body,
+                None => {
+                    slf.l.set_error(Error::LexInvalidEscapedLiteral);
+                    return;
+                }
+            };
+            match core::str::from_utf8(body).map(core::str::FromStr::from_str) {
+                Ok(Ok(fp)) => slf.l.push_token(Lit::new_float(fp)),
+                _ => slf.l.set_error(Error::LexInvalidEscapedLiteral),
+            }
+        },
+        // binary
+        |slf| {
+            let Some(size_of_body) = slf
+                .param_buffer
+                .try_next_ascii_u64_lf_separated_or_restore_cursor()
+            else {
+                slf.l.set_error(Error::LexInvalidEscapedLiteral);
+                return;
+            };
+            match slf
+                .param_buffer
+                .try_next_variable_block(size_of_body as usize)
+            {
+                Some(block) => slf.l.push_token(Lit::new_bin(block)),
+                None => slf.l.set_error(Error::LexInvalidEscapedLiteral),
+            }
+        },
+        // string
+        |slf| {
+            let Some(size_of_body) = slf
+                .param_buffer
+                .try_next_ascii_u64_lf_separated_or_restore_cursor()
+            else {
+                slf.l.set_error(Error::LexInvalidEscapedLiteral);
+                return;
+            };
+            match slf
+                .param_buffer
+                .try_next_variable_block(size_of_body as usize)
+                .map(core::str::from_utf8)
+            {
+                // TODO(@ohsayan): obliterate this alloc
+                Some(Ok(s)) => slf.l.push_token(Lit::new_string(s.to_owned())),
+                _ => slf.l.set_error(Error::LexInvalidEscapedLiteral),
+            }
+        },
+        // ecc
+        |s| s.l.set_error(Error::LexInvalidEscapedLiteral),
+    ]
+};
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index 74061272..3fb83abd 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -25,12 +25,8 @@
 */
 
 use {
-    super::Slice,
-    crate::engine::{
-        data::{lit::Lit, spec::Dataspec1D},
-        error::Error,
-    },
-    core::{borrow::Borrow, fmt, ops::Deref, slice, str},
+    crate::engine::data::lit::Lit,
+    core::{borrow::Borrow, fmt, ops::Deref, str},
 };
 
 #[repr(transparent)]
@@ -367,176 +363,3 @@ impl<'a> AsRef<Token<'a>> for Token<'a> {
         self
     }
 }
-
-#[derive(Debug)]
-pub struct RawLexer<'a> {
-    c: *const u8,
-    e: *const u8,
-    pub(super) tokens: Vec<Token<'a>>,
-    pub(super) last_error: Option<Error>,
-}
-
-// ctor
-impl<'a> RawLexer<'a> {
-    #[inline(always)]
-    pub(super) const fn new(src: Slice<'a>) -> Self {
-        Self {
-            c: src.as_ptr(),
-            e: unsafe {
-                // UNSAFE(@ohsayan): Always safe (<= EOA)
-                src.as_ptr().add(src.len())
-            },
-            last_error: None,
-            tokens: Vec::new(),
-        }
-    }
-}
-
-// meta
-impl<'a> RawLexer<'a> {
-    #[inline(always)]
-    pub(super) const fn cursor(&self) -> *const u8 {
-        self.c
-    }
-    #[inline(always)]
-    pub(super) const fn data_end_ptr(&self) -> *const u8 {
-        self.e
-    }
-    #[inline(always)]
-    pub(super) fn not_exhausted(&self) -> bool {
-        self.data_end_ptr() > self.cursor()
-    }
-    #[inline(always)]
-    pub(super) fn exhausted(&self) -> bool {
-        self.cursor() == self.data_end_ptr()
-    }
-    #[inline(always)]
-    pub(super) fn remaining(&self) -> usize {
-        unsafe {
-            // UNSAFE(@ohsayan): valid ptrs
-            self.e.offset_from(self.c) as usize
-        }
-    }
-    #[inline(always)]
-    pub(super) unsafe fn deref_cursor(&self) -> u8 {
-        *self.cursor()
-    }
-    #[inline(always)]
-    pub(super) unsafe fn incr_cursor_by(&mut self, by: usize) {
-        debug_assert!(self.remaining() >= by);
-        self.c = self.cursor().add(by)
-    }
-    #[inline(always)]
-    pub(super) unsafe fn incr_cursor(&mut self) {
-        self.incr_cursor_by(1)
-    }
-    #[inline(always)]
-    unsafe fn incr_cursor_if(&mut self, iff: bool) {
-        self.incr_cursor_by(iff as usize)
-    }
-    #[inline(always)]
-    pub(super) fn push_token(&mut self, token: impl Into<Token<'a>>) {
-        self.tokens.push(token.into())
-    }
-    #[inline(always)]
-    pub(super) fn peek_is(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
-        self.not_exhausted()
-            && unsafe {
-                // UNSAFE(@ohsayan): verified cursor is nonnull
-                f(self.deref_cursor())
-            }
-    }
-    #[inline(always)]
-    pub(super) fn peek_is_and_forward(&mut self, f: impl FnOnce(u8) -> bool) -> bool {
-        let did_fw = self.not_exhausted()
-            && unsafe {
-                // UNSAFE(@ohsayan): verified ptr
-                f(self.deref_cursor())
-            };
-        unsafe {
-            // UNSAFE(@ohsayan): increment cursor
-            self.incr_cursor_if(did_fw);
-        }
-        did_fw
-    }
-    #[inline(always)]
-    fn peek_eq_and_forward_or_eof(&mut self, eq: u8) -> bool {
-        unsafe {
-            // UNSAFE(@ohsayan): verified cursor
-            let eq = self.not_exhausted() && self.deref_cursor() == eq;
-            // UNSAFE(@ohsayan): incr cursor if matched
-            self.incr_cursor_if(eq);
-            eq | self.exhausted()
-        }
-    }
-    #[inline(always)]
-    pub(super) fn peek_neq(&self, b: u8) -> bool {
-        self.not_exhausted()
-            && unsafe {
-                // UNSAFE(@ohsayan): verified cursor
-                self.deref_cursor() != b
-            }
-    }
-    #[inline(always)]
-    pub(super) fn peek_eq_and_forward(&mut self, b: u8) -> bool {
-        unsafe {
-            // UNSAFE(@ohsayan): verified cursor
-            let r = self.not_exhausted() && self.deref_cursor() == b;
-            self.incr_cursor_if(r);
-            r
-        }
-    }
-    #[inline(always)]
-    pub(super) fn trim_ahead(&mut self) {
-        while self.peek_is_and_forward(|b| b == b' ' || b == b'\t' || b == b'\n') {}
-    }
-    #[inline(always)]
-    pub(super) fn set_error(&mut self, e: Error) {
-        self.last_error = Some(e);
-    }
-    #[inline(always)]
-    pub(super) fn no_error(&self) -> bool {
-        self.last_error.is_none()
-    }
-}
-
-// high level methods
-impl<'a> RawLexer<'a> {
-    #[inline(always)]
-    pub(super) fn scan_ident(&mut self) -> Slice<'a> {
-        let s = self.cursor();
-        unsafe {
-            while self.peek_is(|b| b.is_ascii_alphanumeric() || b == b'_') {
-                // UNSAFE(@ohsayan): increment cursor, this is valid
-                self.incr_cursor();
-            }
-            // UNSAFE(@ohsayan): valid slice and ptrs
-            slice::from_raw_parts(s, self.cursor().offset_from(s) as usize)
-        }
-    }
-    #[inline(always)]
-    pub(super) fn scan_ident_or_keyword(&mut self) {
-        let s = self.scan_ident();
-        let st = s.to_ascii_lowercase();
-        match kwof(&st) {
-            Some(kw) => self.tokens.push(kw.into()),
-            // FIXME(@ohsayan): Uh, mind fixing this? The only advantage is that I can keep the graph *memory* footprint small
-            None if st == b"true" || st == b"false" => self.push_token(Lit::Bool(st == b"true")),
-            None => self.tokens.push(unsafe {
-                // UNSAFE(@ohsayan): scan_ident only returns a valid ident which is always a string
-                Token::Ident(Ident::new(s))
-            }),
-        }
-    }
-    #[inline(always)]
-    pub(super) fn scan_byte(&mut self, byte: u8) {
-        match symof(byte) {
-            Some(tok) => self.push_token(tok),
-            None => return self.set_error(Error::LexUnexpectedByte),
-        }
-        unsafe {
-            // UNSAFE(@ohsayan): we are sent a byte, so fw cursor
-            self.incr_cursor();
-        }
-    }
-}
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 7daa66ac..b5e7cde5 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -331,3 +331,11 @@ macro_rules! build_lut {
         }
     }
 }
+
+#[cfg(test)]
+macro_rules! into_vec {
+    ($ty:ty => ($($v:expr),* $(,)?)) => {{
+        let v: Vec<$ty> = std::vec![$($v.into(),)*];
+        v
+    }}
+}
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 259d97da..ebcef8d1 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    super::lex::{InsecureLexer, SafeLexer, Symbol, Token},
+    super::lex::{InsecureLexer, SecureLexer, Symbol, Token},
     crate::{
         engine::{data::cell::Datacell, error::QueryResult},
         util::test_utils,
@@ -44,10 +44,8 @@ mod structure_syn;
 pub fn lex_insecure(src: &[u8]) -> QueryResult<Vec<Token<'_>>> {
     InsecureLexer::lex(src)
 }
-#[inline(always)]
-/// Uses the [`SafeLexer`] to lex the given input
-pub fn lex_secure(src: &[u8]) -> QueryResult<Vec<Token>> {
-    SafeLexer::lex(src)
+pub fn lex_secure<'a>(src: &'a [u8], query_window: usize) -> QueryResult<Vec<Token<'a>>> {
+    SecureLexer::lex(src, query_window)
 }
 
 pub trait NullableData<T> {
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 2b0cc879..eeb7ec3a 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -27,13 +27,7 @@
 use super::*;
 mod list_parse {
     use super::*;
-    use crate::engine::{
-        data::{lit::LitIR, spec::Dataspec1D},
-        ql::{
-            ast::{parse_ast_node_full, traits::ASTNode, State, SubstitutedData},
-            dml::ins::List,
-        },
-    };
+    use crate::engine::ql::{ast::parse_ast_node_full, dml::ins::List};
 
     #[test]
     fn list_mini() {
@@ -58,28 +52,6 @@ mod list_parse {
         assert_eq!(r.as_slice(), into_array![1, 2, 3, 4])
     }
     #[test]
-    fn list_param() {
-        let tok = lex_secure(
-            b"
-                [?, ?, ?, ?]
-            ",
-        )
-        .unwrap();
-        let data = [
-            LitIR::UnsignedInt(1),
-            LitIR::UnsignedInt(2),
-            LitIR::UnsignedInt(3),
-            LitIR::UnsignedInt(4),
-        ];
-        let mut state = State::new(&tok[1..], SubstitutedData::new(&data));
-        assert_eq!(
-            <List as ASTNode>::from_state(&mut state)
-                .unwrap()
-                .into_inner(),
-            into_array![1, 2, 3, 4]
-        )
-    }
-    #[test]
     fn list_pro() {
         let tok = lex_insecure(
             b"
@@ -104,40 +76,6 @@ mod list_parse {
         )
     }
     #[test]
-    fn list_pro_param() {
-        let tok = lex_secure(
-            b"
-                [
-                    [?, ?],
-                    [?, ?],
-                    [?, ?],
-                    []
-                ]
-            ",
-        )
-        .unwrap();
-        let data = [
-            LitIR::UnsignedInt(1),
-            LitIR::UnsignedInt(2),
-            LitIR::UnsignedInt(3),
-            LitIR::UnsignedInt(4),
-            LitIR::UnsignedInt(5),
-            LitIR::UnsignedInt(6),
-        ];
-        let mut state = State::new(&tok[1..], SubstitutedData::new(&data));
-        assert_eq!(
-            <List as ASTNode>::from_state(&mut state)
-                .unwrap()
-                .into_inner(),
-            into_array![
-                into_array![1, 2],
-                into_array![3, 4],
-                into_array![5, 6],
-                into_array![]
-            ]
-        )
-    }
-    #[test]
     fn list_pro_max() {
         let tok = lex_insecure(
             b"
@@ -161,46 +99,6 @@ mod list_parse {
             ]
         )
     }
-    #[test]
-    fn list_pro_max_param() {
-        let tok = lex_secure(
-            b"
-                [
-                    [[?, ?], [?, ?]],
-                    [[], [?, ?]],
-                    [[?, ?], [?, ?]],
-                    [[?, ?], []]
-                ]
-            ",
-        )
-        .unwrap();
-        let data = [
-            LitIR::UnsignedInt(1),
-            LitIR::UnsignedInt(1),
-            LitIR::UnsignedInt(2),
-            LitIR::UnsignedInt(2),
-            LitIR::UnsignedInt(4),
-            LitIR::UnsignedInt(4),
-            LitIR::UnsignedInt(5),
-            LitIR::UnsignedInt(5),
-            LitIR::UnsignedInt(6),
-            LitIR::UnsignedInt(6),
-            LitIR::UnsignedInt(7),
-            LitIR::UnsignedInt(7),
-        ];
-        let mut state = State::new(&tok[1..], SubstitutedData::new(&data));
-        assert_eq!(
-            <List as ASTNode>::from_state(&mut state)
-                .unwrap()
-                .into_inner(),
-            into_array![
-                into_array![into_array![1, 1], into_array![2, 2]],
-                into_array![into_array![], into_array![4, 4]],
-                into_array![into_array![5, 5], into_array![6, 6]],
-                into_array![into_array![7, 7], into_array![]],
-            ]
-        )
-    }
 }
 
 mod tuple_syntax {
@@ -599,7 +497,7 @@ mod stmt_select {
     use {
         super::*,
         crate::engine::{
-            data::{lit::LitIR, spec::Dataspec1D},
+            data::lit::Lit,
             ql::{
                 ast::{parse_ast_node_full, Entity},
                 dml::{sel::SelectStatement, RelationalExpr},
@@ -622,7 +520,7 @@ mod stmt_select {
             true,
             dict! {
                 Ident::from("username") => RelationalExpr::new(
-                    Ident::from("username"), LitIR::Str("sayan"), RelationalExpr::OP_EQ
+                    Ident::from("username"), Lit::new_str("sayan"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -643,7 +541,7 @@ mod stmt_select {
             false,
             dict! {
                 Ident::from("username") => RelationalExpr::new(
-                    Ident::from("username"), LitIR::Str("sayan"), RelationalExpr::OP_EQ
+                    Ident::from("username"), Lit::new_str("sayan"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -664,7 +562,7 @@ mod stmt_select {
             false,
             dict! {
                 Ident::from("username") => RelationalExpr::new(
-                    Ident::from("username"), LitIR::Str("sayan"), RelationalExpr::OP_EQ
+                    Ident::from("username"), Lit::new_str("sayan"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -685,7 +583,7 @@ mod stmt_select {
             false,
             dict! {
                 Ident::from("username") => RelationalExpr::new(
-                    Ident::from("username"), LitIR::Str("sayan"), RelationalExpr::OP_EQ
+                    Ident::from("username"), Lit::new_str("sayan"), RelationalExpr::OP_EQ
                 ),
             },
         );
@@ -697,7 +595,7 @@ mod expression_tests {
         super::*,
         crate::engine::{
             core::query_meta::AssignmentOperator,
-            data::{lit::LitIR, spec::Dataspec1D},
+            data::lit::Lit,
             ql::{ast::parse_ast_node_full, dml::upd::AssignmentExpression, lex::Ident},
         },
     };
@@ -709,7 +607,7 @@ mod expression_tests {
             r,
             AssignmentExpression::new(
                 Ident::from("username"),
-                LitIR::Str("sayan"),
+                Lit::new_str("sayan"),
                 AssignmentOperator::Assign
             )
         );
@@ -722,7 +620,7 @@ mod expression_tests {
             r,
             AssignmentExpression::new(
                 Ident::from("followers"),
-                LitIR::UnsignedInt(100),
+                Lit::new_uint(100),
                 AssignmentOperator::AddAssign
             )
         );
@@ -735,7 +633,7 @@ mod expression_tests {
             r,
             AssignmentExpression::new(
                 Ident::from("following"),
-                LitIR::UnsignedInt(150),
+                Lit::new_uint(150),
                 AssignmentOperator::SubAssign
             )
         );
@@ -748,7 +646,7 @@ mod expression_tests {
             r,
             AssignmentExpression::new(
                 Ident::from("product_qty"),
-                LitIR::UnsignedInt(2),
+                Lit::new_uint(2),
                 AssignmentOperator::MulAssign
             )
         );
@@ -761,7 +659,7 @@ mod expression_tests {
             r,
             AssignmentExpression::new(
                 Ident::from("image_crop_factor"),
-                LitIR::UnsignedInt(2),
+                Lit::new_uint(2),
                 AssignmentOperator::DivAssign
             )
         );
@@ -772,7 +670,7 @@ mod update_statement {
         super::*,
         crate::engine::{
             core::query_meta::AssignmentOperator,
-            data::{lit::LitIR, spec::Dataspec1D},
+            data::lit::Lit,
             ql::{
                 ast::{parse_ast_node_full, Entity},
                 dml::{
@@ -796,13 +694,13 @@ mod update_statement {
             Entity::Single(Ident::from("app")),
             vec![AssignmentExpression::new(
                 Ident::from("notes"),
-                LitIR::Str("this is my new note"),
+                Lit::new_str("this is my new note"),
                 AssignmentOperator::AddAssign,
             )],
             WhereClause::new(dict! {
                 Ident::from("username") => RelationalExpr::new(
                     Ident::from("username"),
-                    LitIR::Str("sayan"),
+                    Lit::new_str("sayan"),
                     RelationalExpr::OP_EQ
                 )
             }),
@@ -829,19 +727,19 @@ mod update_statement {
             vec![
                 AssignmentExpression::new(
                     Ident::from("notes"),
-                    LitIR::Str("this is my new note"),
+                    Lit::new_str("this is my new note"),
                     AssignmentOperator::AddAssign,
                 ),
                 AssignmentExpression::new(
                     Ident::from("email"),
-                    LitIR::Str("sayan@example.com"),
+                    Lit::new_str("sayan@example.com"),
                     AssignmentOperator::Assign,
                 ),
             ],
             WhereClause::new(dict! {
                 Ident::from("username") => RelationalExpr::new(
                     Ident::from("username"),
-                    LitIR::Str("sayan"),
+                    Lit::new_str("sayan"),
                     RelationalExpr::OP_EQ
                 )
             }),
@@ -853,7 +751,7 @@ mod delete_stmt {
     use {
         super::*,
         crate::engine::{
-            data::{lit::LitIR, spec::Dataspec1D},
+            data::lit::Lit,
             ql::{
                 ast::{parse_ast_node_full, Entity},
                 dml::{del::DeleteStatement, RelationalExpr},
@@ -875,7 +773,7 @@ mod delete_stmt {
             dict! {
                 Ident::from("username") => RelationalExpr::new(
                     Ident::from("username"),
-                    LitIR::Str("sayan"),
+                    Lit::new_str("sayan"),
                     RelationalExpr::OP_EQ
                 )
             },
@@ -898,7 +796,7 @@ mod delete_stmt {
             dict! {
                 Ident::from("username") => RelationalExpr::new(
                     Ident::from("username"),
-                    LitIR::Str("sayan"),
+                    Lit::new_str("sayan"),
                     RelationalExpr::OP_EQ
                 )
             },
@@ -913,7 +811,7 @@ mod relational_expr {
     use {
         super::*,
         crate::engine::{
-            data::{lit::LitIR, spec::Dataspec1D},
+            data::lit::Lit,
             ql::{ast::parse_ast_node_full, dml::RelationalExpr, lex::Ident},
         },
     };
@@ -925,7 +823,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: LitIR::UnsignedInt(10),
+                rhs: Lit::new_uint(10),
                 lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_EQ
             }
@@ -938,7 +836,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: LitIR::UnsignedInt(10),
+                rhs: Lit::new_uint(10),
                 lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_NE
             }
@@ -951,7 +849,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: LitIR::UnsignedInt(10),
+                rhs: Lit::new_uint(10),
                 lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_GT
             }
@@ -964,7 +862,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: LitIR::UnsignedInt(10),
+                rhs: Lit::new_uint(10),
                 lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_GE
             }
@@ -977,7 +875,7 @@ mod relational_expr {
         assert_eq!(
             r,
             RelationalExpr {
-                rhs: LitIR::UnsignedInt(10),
+                rhs: Lit::new_uint(10),
                 lhs: Ident::from("primary_key"),
                 opc: RelationalExpr::OP_LT
             }
@@ -991,7 +889,7 @@ mod relational_expr {
             r,
             RelationalExpr::new(
                 Ident::from("primary_key"),
-                LitIR::UnsignedInt(10),
+                Lit::new_uint(10),
                 RelationalExpr::OP_LE
             )
         );
@@ -1001,7 +899,7 @@ mod where_clause {
     use {
         super::*,
         crate::engine::{
-            data::{lit::LitIR, spec::Dataspec1D},
+            data::lit::Lit,
             ql::{
                 ast::parse_ast_node_full,
                 dml::{RelationalExpr, WhereClause},
@@ -1020,7 +918,7 @@ mod where_clause {
         let expected = WhereClause::new(dict! {
             Ident::from("x") => RelationalExpr::new(
                 Ident::from("x"),
-                LitIR::UnsignedInt(100),
+                Lit::new_uint(100),
                 RelationalExpr::OP_EQ
             )
         });
@@ -1037,12 +935,12 @@ mod where_clause {
         let expected = WhereClause::new(dict! {
             Ident::from("userid") => RelationalExpr::new(
                 Ident::from("userid"),
-                LitIR::UnsignedInt(100),
+                Lit::new_uint(100),
                 RelationalExpr::OP_EQ
             ),
             Ident::from("pass") => RelationalExpr::new(
                 Ident::from("pass"),
-                LitIR::Str("password"),
+                Lit::new_str("password"),
                 RelationalExpr::OP_EQ
             )
         });
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index b680a080..0af6afc0 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -27,12 +27,9 @@
 use {
     super::{
         super::lex::{Ident, Token},
-        lex_insecure,
-    },
-    crate::engine::{
-        data::{lit::Lit, spec::Dataspec1D},
-        error::Error,
+        lex_insecure, lex_secure,
     },
+    crate::engine::{data::lit::Lit, error::Error},
 };
 
 macro_rules! v(
@@ -59,7 +56,7 @@ fn lex_unsigned_int() {
     let number = v!("123456");
     assert_eq!(
         lex_insecure(&number).unwrap(),
-        vec![Token::Lit(Lit::UnsignedInt(123456))]
+        vec![Token::Lit(Lit::new_uint(123456))]
     );
 }
 #[test]
@@ -67,16 +64,19 @@ fn lex_signed_int() {
     let number = v!("-123456");
     assert_eq!(
         lex_insecure(&number).unwrap(),
-        vec![Token::Lit(Lit::SignedInt(-123456))]
+        vec![Token::Lit(Lit::new_sint(-123456))]
     );
 }
 #[test]
 fn lex_bool() {
     let (t, f) = v!("true", "false");
-    assert_eq!(lex_insecure(&t).unwrap(), vec![Token::Lit(Lit::Bool(true))]);
+    assert_eq!(
+        lex_insecure(&t).unwrap(),
+        vec![Token::Lit(Lit::new_bool(true))]
+    );
     assert_eq!(
         lex_insecure(&f).unwrap(),
-        vec![Token::Lit(Lit::Bool(false))]
+        vec![Token::Lit(Lit::new_bool(false))]
     );
 }
 #[test]
@@ -84,12 +84,12 @@ fn lex_string() {
     let s = br#" "hello, world" "#;
     assert_eq!(
         lex_insecure(s).unwrap(),
-        vec![Token::Lit(Lit::Str("hello, world".into()))]
+        vec![Token::Lit(Lit::new_string("hello, world".into()))]
     );
     let s = br#" 'hello, world' "#;
     assert_eq!(
         lex_insecure(s).unwrap(),
-        vec![Token::Lit(Lit::Str("hello, world".into()))]
+        vec![Token::Lit(Lit::new_string("hello, world".into()))]
     );
 }
 #[test]
@@ -97,12 +97,12 @@ fn lex_string_test_escape_quote() {
     let s = br#" "\"hello world\"" "#; // == "hello world"
     assert_eq!(
         lex_insecure(s).unwrap(),
-        vec![Token::Lit(Lit::Str("\"hello world\"".into()))]
+        vec![Token::Lit(Lit::new_string("\"hello world\"".into()))]
     );
     let s = br#" '\'hello world\'' "#; // == 'hello world'
     assert_eq!(
         lex_insecure(s).unwrap(),
-        vec![Token::Lit(Lit::Str("'hello world'".into()))]
+        vec![Token::Lit(Lit::new_string("'hello world'".into()))]
     );
 }
 #[test]
@@ -110,12 +110,12 @@ fn lex_string_use_different_quote_style() {
     let s = br#" "he's on it" "#;
     assert_eq!(
         lex_insecure(s).unwrap(),
-        vec![Token::Lit(Lit::Str("he's on it".into()))]
+        vec![Token::Lit(Lit::new_string("he's on it".into()))]
     );
     let s = br#" 'he thinks that "that girl" fixed it' "#;
     assert_eq!(
         lex_insecure(s).unwrap(),
-        vec![Token::Lit(Lit::Str(
+        vec![Token::Lit(Lit::new_string(
             "he thinks that \"that girl\" fixed it".into()
         ))]
     )
@@ -125,18 +125,18 @@ fn lex_string_escape_bs() {
     let s = v!(r#" "windows has c:\\" "#);
     assert_eq!(
         lex_insecure(&s).unwrap(),
-        vec![Token::Lit(Lit::Str("windows has c:\\".into()))]
+        vec![Token::Lit(Lit::new_string("windows has c:\\".into()))]
     );
     let s = v!(r#" 'windows has c:\\' "#);
     assert_eq!(
         lex_insecure(&s).unwrap(),
-        vec![Token::Lit(Lit::Str("windows has c:\\".into()))]
+        vec![Token::Lit(Lit::new_string("windows has c:\\".into()))]
     );
     let lol = v!(r#"'\\\\\\\\\\'"#);
     let lexed = lex_insecure(&lol).unwrap();
     assert_eq!(
         lexed,
-        vec![Token::Lit(Lit::Str("\\".repeat(5).into_boxed_str()))],
+        vec![Token::Lit(Lit::new_string("\\".repeat(5)))],
         "lol"
     )
 }
@@ -156,352 +156,166 @@ fn lex_string_unclosed() {
 fn lex_unsafe_literal_mini() {
     let usl = lex_insecure("\r0\n".as_bytes()).unwrap();
     assert_eq!(usl.len(), 1);
-    assert_eq!(Token::Lit(Lit::Bin(b"")), usl[0]);
+    assert_eq!(Token::Lit(Lit::new_bin(b"")), usl[0]);
 }
 #[test]
 fn lex_unsafe_literal() {
     let usl = lex_insecure("\r9\nabcdefghi".as_bytes()).unwrap();
     assert_eq!(usl.len(), 1);
-    assert_eq!(Token::Lit(Lit::Bin(b"abcdefghi")), usl[0]);
+    assert_eq!(Token::Lit(Lit::new_bin(b"abcdefghi")), usl[0]);
 }
 #[test]
 fn lex_unsafe_literal_pro() {
     let usl = lex_insecure("\r18\nabcdefghi123456789".as_bytes()).unwrap();
     assert_eq!(usl.len(), 1);
-    assert_eq!(Token::Lit(Lit::Bin(b"abcdefghi123456789")), usl[0]);
+    assert_eq!(Token::Lit(Lit::new_bin(b"abcdefghi123456789")), usl[0]);
 }
 
-mod num_tests {
-    use crate::engine::ql::lex::decode_num_ub as ubdc;
-    mod uint8 {
-        use super::*;
-        #[test]
-        fn ndecub_u8_ok() {
-            const SRC: &[u8] = b"123\n";
-            let mut i = 0;
-            let mut b = true;
-            let x = ubdc::<u8>(SRC, &mut b, &mut i);
-            assert!(b);
-            assert_eq!(i, SRC.len());
-            assert_eq!(x, 123);
-        }
-        #[test]
-        fn ndecub_u8_lb() {
-            const SRC: &[u8] = b"0\n";
-            let mut i = 0;
-            let mut b = true;
-            let x = ubdc::<u8>(SRC, &mut b, &mut i);
-            assert!(b);
-            assert_eq!(i, SRC.len());
-            assert_eq!(x, 0);
-        }
-        #[test]
-        fn ndecub_u8_ub() {
-            const SRC: &[u8] = b"255\n";
-            let mut i = 0;
-            let mut b = true;
-            let x = ubdc::<u8>(SRC, &mut b, &mut i);
-            assert!(b);
-            assert_eq!(i, SRC.len());
-            assert_eq!(x, 255);
-        }
-        #[test]
-        fn ndecub_u8_ub_of() {
-            const SRC: &[u8] = b"256\n";
-            let mut i = 0;
-            let mut b = true;
-            let x = ubdc::<u8>(SRC, &mut b, &mut i);
-            assert!(!b);
-            assert_eq!(i, 2);
-            assert_eq!(x, 0);
-        }
-    }
-    mod sint8 {
-        use super::*;
-        #[test]
-        pub(crate) fn ndecub_i8_ok() {
-            const SRC: &[u8] = b"-123\n";
-            let mut i = 0;
-            let mut b = true;
-            let x = ubdc::<i8>(SRC, &mut b, &mut i);
-            assert!(b);
-            assert_eq!(i, SRC.len());
-            assert_eq!(x, -123);
-        }
-        #[test]
-        pub(crate) fn ndecub_i8_lb() {
-            const SRC: &[u8] = b"-128\n";
-            let mut i = 0;
-            let mut b = true;
-            let x = ubdc::<i8>(SRC, &mut b, &mut i);
-            assert!(b);
-            assert_eq!(i, SRC.len());
-            assert_eq!(x, -128);
-        }
-        #[test]
-        pub(crate) fn ndecub_i8_lb_of() {
-            const SRC: &[u8] = b"-129\n";
-            let mut i = 0;
-            let mut b = true;
-            let x = ubdc::<i8>(SRC, &mut b, &mut i);
-            assert!(!b);
-            assert_eq!(i, 3);
-            assert_eq!(x, 0);
-        }
-        #[test]
-        pub(crate) fn ndecub_i8_ub() {
-            const SRC: &[u8] = b"127\n";
-            let mut i = 0;
-            let mut b = true;
-            let x = ubdc::<i8>(SRC, &mut b, &mut i);
-            assert!(b);
-            assert_eq!(i, SRC.len());
-            assert_eq!(x, 127);
-        }
-        #[test]
-        pub(crate) fn ndecub_i8_ub_of() {
-            const SRC: &[u8] = b"128\n";
-            let mut i = 0;
-            let mut b = true;
-            let x = ubdc::<i8>(SRC, &mut b, &mut i);
-            assert!(!b);
-            assert_eq!(i, 2);
-            assert_eq!(x, 0);
-        }
-    }
+/*
+    safe query tests
+*/
+
+fn make_safe_query(a: &[u8], b: &[u8]) -> (Vec<u8>, usize) {
+    let mut s = Vec::with_capacity(a.len() + b.len());
+    s.extend(a);
+    s.extend(b);
+    (s, a.len())
 }
 
-mod safequery_params {
-    use crate::engine::{
-        data::{lit::LitIR, spec::Dataspec1D},
-        ql::lex::SafeQueryData,
-    };
-    use rand::seq::SliceRandom;
-    #[test]
-    fn param_uint() {
-        let src = b"12345\n";
-        let mut d = Vec::new();
-        let mut i = 0;
-        assert!(SafeQueryData::uint(src, &mut i, &mut d));
-        assert_eq!(i, src.len());
-        assert_eq!(d, vec![LitIR::UnsignedInt(12345)]);
-    }
-    #[test]
-    fn param_sint() {
-        let src = b"-12345\n";
-        let mut d = Vec::new();
-        let mut i = 0;
-        assert!(SafeQueryData::sint(src, &mut i, &mut d));
-        assert_eq!(i, src.len());
-        assert_eq!(d, vec![LitIR::SignedInt(-12345)]);
-    }
-    #[test]
-    fn param_bool_true() {
-        let src = b"true\n";
-        let mut d = Vec::new();
-        let mut i = 0;
-        assert!(SafeQueryData::bool(src, &mut i, &mut d));
-        assert_eq!(i, src.len());
-        assert_eq!(d, vec![LitIR::Bool(true)]);
-    }
-    #[test]
-    fn param_bool_false() {
-        let src = b"false\n";
-        let mut d = Vec::new();
-        let mut i = 0;
-        assert!(SafeQueryData::bool(src, &mut i, &mut d));
-        assert_eq!(i, src.len());
-        assert_eq!(d, vec![LitIR::Bool(false)]);
-    }
-    #[test]
-    fn param_float() {
-        let src = b"4\n3.14";
-        let mut d = Vec::new();
-        let mut i = 0;
-        assert!(SafeQueryData::float(src, &mut i, &mut d));
-        assert_eq!(i, src.len());
-        assert_eq!(d, vec![LitIR::Float(3.14)]);
-    }
-    #[test]
-    fn param_bin() {
-        let src = b"5\nsayan";
-        let mut d = Vec::new();
-        let mut i = 0;
-        assert!(SafeQueryData::bin(src, &mut i, &mut d));
-        assert_eq!(i, src.len());
-        assert_eq!(d, vec![LitIR::Bin(b"sayan")]);
-    }
-    #[test]
-    fn param_str() {
-        let src = b"5\nsayan";
-        let mut d = Vec::new();
-        let mut i = 0;
-        assert!(SafeQueryData::str(src, &mut i, &mut d));
-        assert_eq!(i, src.len());
-        assert_eq!(d, vec![LitIR::Str("sayan")]);
-    }
-    #[test]
-    fn param_full_uint() {
-        let src = b"\x0012345\n";
-        let r = SafeQueryData::p_revloop(src, 1).unwrap();
-        assert_eq!(r.as_ref(), [LitIR::UnsignedInt(12345)]);
-    }
-    #[test]
-    fn param_full_sint() {
-        let src = b"\x01-12345\n";
-        let r = SafeQueryData::p_revloop(src, 1).unwrap();
-        assert_eq!(r.as_ref(), [LitIR::SignedInt(-12345)]);
-    }
-    #[test]
-    fn param_full_bool() {
-        let src = b"\x02true\n";
-        let r = SafeQueryData::p_revloop(src, 1).unwrap();
-        assert_eq!(r.as_ref(), [LitIR::Bool(true)]);
-        let src = b"\x02false\n";
-        let r = SafeQueryData::p_revloop(src, 1).unwrap();
-        assert_eq!(r.as_ref(), [LitIR::Bool(false)]);
-    }
-    #[test]
-    fn param_full_float() {
-        let src = b"\x034\n3.14";
-        let r = SafeQueryData::p_revloop(src, 1).unwrap();
-        assert_eq!(r.as_ref(), [LitIR::Float(3.14)]);
-        let src = b"\x035\n-3.14";
-        let r = SafeQueryData::p_revloop(src, 1).unwrap();
-        assert_eq!(r.as_ref(), [LitIR::Float(-3.14)]);
-    }
-    #[test]
-    fn param_full_bin() {
-        let src = b"\x0412\nhello, world";
-        let r = SafeQueryData::p_revloop(src, 1).unwrap();
-        assert_eq!(r.as_ref(), [LitIR::Bin(b"hello, world")]);
-    }
-    #[test]
-    fn param_full_str() {
-        let src = b"\x0512\nhello, world";
-        let r = SafeQueryData::p_revloop(src, 1).unwrap();
-        assert_eq!(r.as_ref(), [LitIR::Str("hello, world")]);
-    }
-    #[test]
-    fn params_mix() {
-        let mut rng = rand::thread_rng();
-        const DATA: [&[u8]; 6] = [
-            b"\x0012345\n",
-            b"\x01-12345\n",
-            b"\x02true\n",
-            b"\x0311\n12345.67890",
-            b"\x0430\none two three four five binary",
-            b"\x0527\none two three four five str",
-        ];
-        let retmap: [LitIR; 6] = [
-            LitIR::UnsignedInt(12345),
-            LitIR::SignedInt(-12345),
-            LitIR::Bool(true),
-            LitIR::Float(12345.67890),
-            LitIR::Bin(b"one two three four five binary"),
-            LitIR::Str("one two three four five str"),
-        ];
-        for _ in 0..DATA.len().pow(2) {
-            let mut local_data = DATA;
-            local_data.shuffle(&mut rng);
-            let ret: Vec<LitIR> = local_data
-                .iter()
-                .map(|v| retmap[v[0] as usize].clone())
-                .collect();
-            let src: Vec<u8> = local_data.into_iter().flat_map(|v| v.to_owned()).collect();
-            let r = SafeQueryData::p_revloop(&src, 6).unwrap();
-            assert_eq!(r.as_ref(), ret);
-        }
-    }
+#[test]
+fn safe_query_all_literals() {
+    let (query, query_window) = make_safe_query(
+        b"? ? ? ? ? ? ?",
+        b"\x00\x01\x01\x021234\n\x03-1234\n\x049\n1234.5678\x0513\nbinarywithlf\n\x065\nsayan",
+    );
+    let ret = lex_secure(&query, query_window).unwrap();
+    assert_eq!(
+        ret,
+        into_vec![Token<'static> => (
+            Token![null],
+            Lit::new_bool(true),
+            Lit::new_uint(1234),
+            Lit::new_sint(-1234),
+            Lit::new_float(1234.5678),
+            Lit::new_bin(b"binarywithlf\n"),
+            Lit::new_string("sayan".into()),
+        )],
+    );
 }
 
-mod safequery_full_param {
-    use crate::engine::{
-        data::{lit::LitIR, spec::Dataspec1D},
-        ql::lex::{Ident, SafeQueryData, Token},
-    };
-    #[test]
-    fn p_mini() {
-        let query = b"select * from myapp where username = ?";
-        let params = b"\x055\nsayan";
-        let sq = SafeQueryData::parse(query, params, 1).unwrap();
-        assert_eq!(
-            sq,
-            SafeQueryData::new_test(
-                vec![LitIR::Str("sayan")].into_boxed_slice(),
-                vec![
-                    Token![select],
-                    Token![*],
-                    Token![from],
-                    Token::Ident(Ident::from("myapp")),
-                    Token![where],
-                    Token::Ident(Ident::from("username")),
-                    Token![=],
-                    Token![?]
-                ]
-            )
-        );
-    }
-    #[test]
-    fn p() {
-        let query = b"select * from myapp where username = ? and pass = ?";
-        let params = b"\x055\nsayan\x048\npass1234";
-        let sq = SafeQueryData::parse(query, params, 2).unwrap();
-        assert_eq!(
-            sq,
-            SafeQueryData::new_test(
-                vec![LitIR::Str("sayan"), LitIR::Bin(b"pass1234")].into_boxed_slice(),
-                vec![
-                    Token![select],
-                    Token![*],
-                    Token![from],
-                    Token::Ident(Ident::from("myapp")),
-                    Token![where],
-                    Token::Ident(Ident::from("username")),
-                    Token![=],
-                    Token![?],
-                    Token![and],
-                    Token::Ident(Ident::from("pass")),
-                    Token![=],
-                    Token![?]
-                ]
-            )
-        );
-    }
-    #[test]
-    fn p_pro() {
-        let query = b"select $notes[~?] from myapp where username = ? and pass = ?";
-        let params = b"\x00100\n\x055\nsayan\x048\npass1234";
-        let sq = SafeQueryData::parse(query, params, 3).unwrap();
+const SFQ_NULL: &[u8] = b"\x00";
+const SFQ_BOOL_FALSE: &[u8] = b"\x01\0";
+const SFQ_BOOL_TRUE: &[u8] = b"\x01\x01";
+const SFQ_UINT: &[u8] = b"\x0218446744073709551615\n";
+const SFQ_SINT: &[u8] = b"\x03-9223372036854775808\n";
+const SFQ_FLOAT: &[u8] = b"\x0411\n3.141592654";
+const SFQ_BINARY: &[u8] = "\x0546\ncringe😃😄😁😆😅😂🤣😊😸😺".as_bytes();
+const SFQ_STRING: &[u8] = "\x0646\ncringe😃😄😁😆😅😂🤣😊😸😺".as_bytes();
+
+#[test]
+fn safe_query_null() {
+    let (query, query_window) = make_safe_query(b"?", SFQ_NULL);
+    let r = lex_secure(&query, query_window).unwrap();
+    assert_eq!(r, vec![Token![null]])
+}
+
+#[test]
+fn safe_query_bool() {
+    let (query, query_window) = make_safe_query(b"?", SFQ_BOOL_FALSE);
+    let b_false = lex_secure(&query, query_window).unwrap();
+    let (query, query_window) = make_safe_query(b"?", SFQ_BOOL_TRUE);
+    let b_true = lex_secure(&query, query_window).unwrap();
+    assert_eq!(
+        [b_false, b_true].concat(),
+        vec![
+            Token::from(Lit::new_bool(false)),
+            Token::from(Lit::new_bool(true))
+        ]
+    );
+}
+
+#[test]
+fn safe_query_uint() {
+    let (query, query_window) = make_safe_query(b"?", SFQ_UINT);
+    let int = lex_secure(&query, query_window).unwrap();
+    assert_eq!(int, vec![Token::Lit(Lit::new_uint(u64::MAX))]);
+}
+
+#[test]
+fn safe_query_sint() {
+    let (query, query_window) = make_safe_query(b"?", SFQ_SINT);
+    let int = lex_secure(&query, query_window).unwrap();
+    assert_eq!(int, vec![Token::Lit(Lit::new_sint(i64::MIN))]);
+}
+
+#[test]
+fn safe_query_float() {
+    let (query, query_window) = make_safe_query(b"?", SFQ_FLOAT);
+    let float = lex_secure(&query, query_window).unwrap();
+    assert_eq!(float, vec![Token::Lit(Lit::new_float(3.141592654))]);
+}
+
+#[test]
+fn safe_query_binary() {
+    let (query, query_window) = make_safe_query(b"?", SFQ_BINARY);
+    let binary = lex_secure(&query, query_window).unwrap();
+    assert_eq!(
+        binary,
+        vec![Token::Lit(Lit::new_bin(
+            "cringe😃😄😁😆😅😂🤣😊😸😺".as_bytes()
+        ))]
+    );
+}
+
+#[test]
+fn safe_query_string() {
+    let (query, query_window) = make_safe_query(b"?", SFQ_STRING);
+    let binary = lex_secure(&query, query_window).unwrap();
+    assert_eq!(
+        binary,
+        vec![Token::Lit(Lit::new_string(
+            "cringe😃😄😁😆😅😂🤣😊😸😺".to_owned().into()
+        ))]
+    );
+}
+
+#[test]
+fn safe_params_shuffled() {
+    let expected = [
+        (SFQ_NULL, Token![null]),
+        (SFQ_BOOL_FALSE, Token::Lit(Lit::new_bool(false))),
+        (SFQ_BOOL_TRUE, Token::Lit(Lit::new_bool(true))),
+        (SFQ_UINT, Token::Lit(Lit::new_uint(u64::MAX))),
+        (SFQ_SINT, Token::Lit(Lit::new_sint(i64::MIN))),
+        (SFQ_FLOAT, Token::Lit(Lit::new_float(3.141592654))),
+        (
+            SFQ_BINARY,
+            Token::Lit(Lit::new_bin("cringe😃😄😁😆😅😂🤣😊😸😺".as_bytes())),
+        ),
+        (
+            SFQ_STRING,
+            Token::Lit(Lit::new_string(
+                "cringe😃😄😁😆😅😂🤣😊😸😺".to_owned().into(),
+            )),
+        ),
+    ];
+    let mut rng = crate::util::test_utils::randomizer();
+    for _ in 0..expected.len().pow(2) {
+        let mut this_expected = expected.clone();
+        crate::util::test_utils::shuffle_slice(&mut this_expected, &mut rng);
+        let param_segment: Vec<u8> = this_expected
+            .iter()
+            .map(|(raw, _)| raw.to_vec())
+            .flatten()
+            .collect();
+        let (query, query_window) = make_safe_query(b"? ? ? ? ? ? ? ?", &param_segment);
+        let ret = lex_secure(&query, query_window).unwrap();
         assert_eq!(
-            sq,
-            SafeQueryData::new_test(
-                vec![
-                    LitIR::UnsignedInt(100),
-                    LitIR::Str("sayan"),
-                    LitIR::Bin(b"pass1234")
-                ]
-                .into_boxed_slice(),
-                vec![
-                    Token![select],
-                    Token![$],
-                    Token::Ident(Ident::from("notes")),
-                    Token![open []],
-                    Token![~],
-                    Token![?],
-                    Token![close []],
-                    Token![from],
-                    Token::Ident(Ident::from("myapp")),
-                    Token![where],
-                    Token::Ident(Ident::from("username")),
-                    Token![=],
-                    Token![?],
-                    Token![and],
-                    Token::Ident(Ident::from("pass")),
-                    Token![=],
-                    Token![?]
-                ]
-            )
-        );
+            ret,
+            this_expected
+                .into_iter()
+                .map(|(_, expected)| expected)
+                .collect::<Vec<_>>()
+        )
     }
 }
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 32f26f1a..d9927fe6 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -26,7 +26,7 @@
 
 use {
     super::{super::lex::Ident, lex_insecure, *},
-    crate::engine::data::{lit::Lit, spec::Dataspec1D},
+    crate::engine::data::lit::Lit,
 };
 mod inspect {
     use {
@@ -71,7 +71,7 @@ mod alter_space {
     use {
         super::*,
         crate::engine::{
-            data::{lit::Lit, spec::Dataspec1D},
+            data::lit::Lit,
             ql::{ast::parse_ast_node_full, ddl::alt::AlterSpace},
         },
     };
@@ -98,8 +98,8 @@ mod alter_space {
             AlterSpace::new(
                 Ident::from("mymodel"),
                 null_dict! {
-                    "max_entry" => Lit::UnsignedInt(1000),
-                    "driver" => Lit::Str("ts-0.8".into())
+                    "max_entry" => Lit::new_uint(1000),
+                    "driver" => Lit::new_string("ts-0.8".into())
                 }
             )
         );
@@ -130,9 +130,9 @@ mod tymeta {
         assert_eq!(
             tymeta,
             null_dict! {
-                "hello" => Lit::Str("world".into()),
-                "loading" => Lit::Bool(true),
-                "size" => Lit::UnsignedInt(100)
+                "hello" => Lit::new_string("world".into()),
+                "loading" => Lit::new_bool(true),
+                "size" => Lit::new_uint(100)
             }
         );
     }
@@ -154,8 +154,8 @@ mod tymeta {
         assert_eq!(
             final_ret,
             null_dict! {
-                "maxlen" => Lit::UnsignedInt(100),
-                "unique" => Lit::Bool(true)
+                "maxlen" => Lit::new_uint(100),
+                "unique" => Lit::new_bool(true)
             }
         )
     }
@@ -179,10 +179,10 @@ mod tymeta {
         assert_eq!(
             final_ret,
             null_dict! {
-                "maxlen" => Lit::UnsignedInt(100),
-                "unique" => Lit::Bool(true),
+                "maxlen" => Lit::new_uint(100),
+                "unique" => Lit::new_bool(true),
                 "this" => null_dict! {
-                    "is" => Lit::Str("cool".into())
+                    "is" => Lit::new_string("cool".into())
                 }
             }
         )
@@ -209,7 +209,7 @@ mod layer {
             vec![LayerSpec::new(
                 Ident::from("string"),
                 null_dict! {
-                    "maxlen" => Lit::UnsignedInt(100)
+                    "maxlen" => Lit::new_uint(100)
                 }
             )]
         );
@@ -237,8 +237,8 @@ mod layer {
                 LayerSpec::new(
                     Ident::from("list"),
                     null_dict! {
-                        "unique" => Lit::Bool(true),
-                        "maxlen" => Lit::UnsignedInt(10),
+                        "unique" => Lit::new_bool(true),
+                        "maxlen" => Lit::new_uint(10),
                     }
                 )
             ]
@@ -257,15 +257,15 @@ mod layer {
                 LayerSpec::new(
                     Ident::from("string"),
                     null_dict! {
-                        "ascii_only" => Lit::Bool(true),
-                        "maxlen" => Lit::UnsignedInt(255)
+                        "ascii_only" => Lit::new_bool(true),
+                        "maxlen" => Lit::new_uint(255)
                     }
                 ),
                 LayerSpec::new(
                     Ident::from("list"),
                     null_dict! {
-                        "unique" => Lit::Bool(true),
-                        "maxlen" => Lit::UnsignedInt(10),
+                        "unique" => Lit::new_bool(true),
+                        "maxlen" => Lit::new_uint(10),
                     }
                 )
             ]
@@ -289,10 +289,13 @@ mod layer {
             LayerSpec::new(
                 Ident::from("list"),
                 null_dict! {
-                    "maxlen" => Lit::UnsignedInt(100),
+                    "maxlen" => Lit::new_uint(100),
                 },
             ),
-            LayerSpec::new(Ident::from("list"), null_dict!("unique" => Lit::Bool(true))),
+            LayerSpec::new(
+                Ident::from("list"),
+                null_dict!("unique" => Lit::new_bool(true)),
+            ),
         ];
         fuzz_tokens(tok.as_slice(), |should_pass, new_tok| {
             let layers = parse_ast_node_multiple_full::<LayerSpec>(&new_tok);
@@ -360,8 +363,8 @@ mod fields {
                 [LayerSpec::new(
                     Ident::from("string"),
                     null_dict! {
-                        "maxlen" => Lit::UnsignedInt(10),
-                        "ascii_only" => Lit::Bool(true),
+                        "maxlen" => Lit::new_uint(10),
+                        "ascii_only" => Lit::new_bool(true),
                     }
                 )]
                 .into(),
@@ -393,14 +396,14 @@ mod fields {
                     LayerSpec::new(
                         Ident::from("string"),
                         null_dict! {
-                            "maxlen" => Lit::UnsignedInt(255),
-                            "ascii_only" => Lit::Bool(true),
+                            "maxlen" => Lit::new_uint(255),
+                            "ascii_only" => Lit::new_bool(true),
                         }
                     ),
                     LayerSpec::new(
                         Ident::from("list"),
                         null_dict! {
-                            "unique" => Lit::Bool(true)
+                            "unique" => Lit::new_bool(true)
                         }
                     ),
                 ]
@@ -555,7 +558,7 @@ mod schemas {
                             LayerSpec::new(
                                 Ident::from("list"),
                                 null_dict! {
-                                    "unique" => Lit::Bool(true)
+                                    "unique" => Lit::new_bool(true)
                                 }
                             )
                         ],
@@ -624,7 +627,7 @@ mod schemas {
                             LayerSpec::new(
                                 Ident::from("list"),
                                 null_dict! {
-                                    "unique" => Lit::Bool(true)
+                                    "unique" => Lit::new_bool(true)
                                 }
                             )
                         ],
@@ -634,9 +637,9 @@ mod schemas {
                 ],
                 null_dict! {
                     "env" => null_dict! {
-                        "free_user_limit" => Lit::UnsignedInt(100),
+                        "free_user_limit" => Lit::new_uint(100),
                     },
-                    "storage_driver" => Lit::Str("skyheap".into()),
+                    "storage_driver" => Lit::new_string("skyheap".into()),
                 }
             )
         )
@@ -679,7 +682,7 @@ mod dict_field_syntax {
                 Ident::from("username"),
                 vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
                 null_dict! {
-                    "nullable" => Lit::Bool(false),
+                    "nullable" => Lit::new_bool(false),
                 },
             )
         );
@@ -707,13 +710,13 @@ mod dict_field_syntax {
                 vec![LayerSpec::new(
                     Ident::from("string"),
                     null_dict! {
-                        "minlen" => Lit::UnsignedInt(6),
-                        "maxlen" => Lit::UnsignedInt(255),
+                        "minlen" => Lit::new_uint(6),
+                        "maxlen" => Lit::new_uint(255),
                     }
                 )],
                 null_dict! {
-                    "nullable" => Lit::Bool(false),
-                    "jingle_bells" => Lit::Str("snow".into()),
+                    "nullable" => Lit::new_bool(false),
+                    "jingle_bells" => Lit::new_string("snow".into()),
                 },
             )
         );
@@ -744,19 +747,19 @@ mod dict_field_syntax {
                     LayerSpec::new(
                         Ident::from("string"),
                         null_dict! {
-                            "ascii_only" => Lit::Bool(true),
+                            "ascii_only" => Lit::new_bool(true),
                         }
                     ),
                     LayerSpec::new(
                         Ident::from("list"),
                         null_dict! {
-                            "unique" => Lit::Bool(true),
+                            "unique" => Lit::new_bool(true),
                         }
                     )
                 ],
                 null_dict! {
-                    "nullable" => Lit::Bool(true),
-                    "jingle_bells" => Lit::Str("snow".into()),
+                    "nullable" => Lit::new_bool(true),
+                    "jingle_bells" => Lit::new_string("snow".into()),
                 },
             )
         );
@@ -863,7 +866,7 @@ mod alter_model_add {
                         Ident::from("myfield"),
                         [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {
-                            "nullable" => Lit::Bool(true)
+                            "nullable" => Lit::new_bool(true)
                         },
                     )]
                     .into()
@@ -889,7 +892,7 @@ mod alter_model_add {
                         Ident::from("myfield"),
                         [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {
-                            "nullable" => Lit::Bool(true)
+                            "nullable" => Lit::new_bool(true)
                         },
                     )]
                     .into()
@@ -930,7 +933,7 @@ mod alter_model_add {
                             Ident::from("myfield"),
                             [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                             null_dict! {
-                                "nullable" => Lit::Bool(true)
+                                "nullable" => Lit::new_bool(true)
                             },
                         ),
                         ExpandedField::new(
@@ -939,19 +942,19 @@ mod alter_model_add {
                                 LayerSpec::new(
                                     Ident::from("string"),
                                     null_dict! {
-                                        "maxlen" => Lit::UnsignedInt(255)
+                                        "maxlen" => Lit::new_uint(255)
                                     }
                                 ),
                                 LayerSpec::new(
                                     Ident::from("list"),
                                     null_dict! {
-                                       "unique" => Lit::Bool(true)
+                                       "unique" => Lit::new_bool(true)
                                     },
                                 )
                             ]
                             .into(),
                             null_dict! {
-                                "nullable" => Lit::Bool(false)
+                                "nullable" => Lit::new_bool(false)
                             },
                         )
                     ]
@@ -1042,7 +1045,7 @@ mod alter_model_update {
                         Ident::from("myfield"),
                         [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                         null_dict! {
-                            "nullable" => Lit::Bool(true)
+                            "nullable" => Lit::new_bool(true)
                         },
                     )]
                     .into()
@@ -1077,7 +1080,7 @@ mod alter_model_update {
                             Ident::from("myfield"),
                             [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                             null_dict! {
-                                "nullable" => Lit::Bool(true)
+                                "nullable" => Lit::new_bool(true)
                             },
                         ),
                         ExpandedField::new(
@@ -1120,14 +1123,14 @@ mod alter_model_update {
                             Ident::from("myfield"),
                             [LayerSpec::new(Ident::from("string"), null_dict! {})].into(),
                             null_dict! {
-                                "nullable" => Lit::Bool(true)
+                                "nullable" => Lit::new_bool(true)
                             },
                         ),
                         ExpandedField::new(
                             Ident::from("myfield2"),
                             [LayerSpec::new(
                                 Ident::from("string"),
-                                null_dict! {"maxlen" => Lit::UnsignedInt(255)}
+                                null_dict! {"maxlen" => Lit::new_uint(255)}
                             )]
                             .into(),
                             null_dict! {},
diff --git a/server/src/engine/ql/tests/structure_syn.rs b/server/src/engine/ql/tests/structure_syn.rs
index 3a17af63..960a0cac 100644
--- a/server/src/engine/ql/tests/structure_syn.rs
+++ b/server/src/engine/ql/tests/structure_syn.rs
@@ -27,7 +27,7 @@
 use {
     super::*,
     crate::engine::{
-        data::{lit::Lit, spec::Dataspec1D, DictGeneric},
+        data::{lit::Lit, DictGeneric},
         ql::{ast::parse_ast_node_full, ddl::syn::DictBasic},
     },
 };
@@ -56,7 +56,7 @@ mod dict {
             br#"{name: "sayan"}"#,
             br#"{name: "sayan",}"#,
         };
-        let r = null_dict!("name" => Lit::Str("sayan".into()));
+        let r = null_dict!("name" => Lit::new_string("sayan".into()));
         multi_assert_eq!(d1, d2 => r);
     }
     #[test]
@@ -78,9 +78,9 @@ mod dict {
             "#,
         };
         let r = null_dict! (
-            "name" => Lit::Str("sayan".into()),
-            "verified" => Lit::Bool(true),
-            "burgers" => Lit::UnsignedInt(152),
+            "name" => Lit::new_string("sayan".into()),
+            "verified" => Lit::new_bool(true),
+            "burgers" => Lit::new_uint(152),
         );
         multi_assert_eq!(d1, d2 => r);
     }
@@ -119,11 +119,11 @@ mod dict {
         };
         multi_assert_eq!(
             d1, d2, d3 => null_dict! {
-                "name" => Lit::Str("sayan".into()),
+                "name" => Lit::new_string("sayan".into()),
                 "notes" => null_dict! {
-                    "burgers" => Lit::Str("all the time, extra mayo".into()),
-                    "taco" => Lit::Bool(true),
-                    "pretzels" => Lit::UnsignedInt(1),
+                    "burgers" => Lit::new_string("all the time, extra mayo".into()),
+                    "taco" => Lit::new_bool(true),
+                    "pretzels" => Lit::new_uint(1),
                 }
             }
         );
@@ -178,7 +178,7 @@ mod dict {
                     "now" => null_dict! {
                         "this" => null_dict! {
                             "is" => null_dict! {
-                                "ridiculous" => Lit::Bool(true),
+                                "ridiculous" => Lit::new_bool(true),
                             }
                         }
                     }
@@ -207,16 +207,16 @@ mod dict {
             }
         ";
         let ret_dict = null_dict! {
-            "the_tradition_is" => Lit::Str("hello, world".into()),
+            "the_tradition_is" => Lit::new_string("hello, world".into()),
             "could_have_been" => null_dict! {
-                "this" => Lit::Bool(true),
-                "or_maybe_this" => Lit::UnsignedInt(100),
-                "even_this" => Lit::Str("hello, universe!".into()),
+                "this" => Lit::new_bool(true),
+                "or_maybe_this" => Lit::new_uint(100),
+                "even_this" => Lit::new_string("hello, universe!".into()),
             },
-            "but_oh_well" => Lit::Str("it continues to be the 'annoying' phrase".into()),
+            "but_oh_well" => Lit::new_string("it continues to be the 'annoying' phrase".into()),
             "lorem" => null_dict! {
                 "ipsum" => null_dict! {
-                    "dolor" => Lit::Str("sit amet".into())
+                    "dolor" => Lit::new_string("sit amet".into())
                 }
             }
         };
@@ -258,7 +258,7 @@ mod null_dict_tests {
             assert_eq!(
                 d,
                 null_dict! {
-                    "this_is_non_null" => Lit::Str("hello".into()),
+                    "this_is_non_null" => Lit::new_string("hello".into()),
                     "but_this_is_null" => Null,
                 }
             )
@@ -279,8 +279,8 @@ mod null_dict_tests {
             assert_eq!(
                 d,
                 null_dict! {
-                    "a_string" => Lit::Str("this is a string".into()),
-                    "num" => Lit::UnsignedInt(1234),
+                    "a_string" => Lit::new_string("this is a string".into()),
+                    "num" => Lit::new_uint(1234),
                     "a_dict" => null_dict! {
                         "a_null" => Null,
                     }
@@ -304,8 +304,8 @@ mod null_dict_tests {
             assert_eq!(
                 d,
                 null_dict! {
-                    "a_string" => Lit::Str("this is a string".into()),
-                    "num" => Lit::UnsignedInt(1234),
+                    "a_string" => Lit::new_string("this is a string".into()),
+                    "num" => Lit::new_uint(1234),
                     "a_dict" => null_dict! {
                         "a_null" => Null,
                     },

From 597a49a91b15ab8ca2cd3c044cd3502b8ce65347 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 20 Sep 2023 18:30:21 +0000
Subject: [PATCH 257/310] Param by default in protocol

---
 .../src/engine/net/protocol/data_exchange.rs  | 192 -----------
 server/src/engine/net/protocol/exchange.rs    | 318 ++++++++++++++++++
 server/src/engine/net/protocol/handshake.rs   |  15 +-
 server/src/engine/net/protocol/mod.rs         |   2 +-
 server/src/engine/net/protocol/tests.rs       | 112 ++++--
 server/src/engine/ql/benches.rs               |   5 +-
 6 files changed, 408 insertions(+), 236 deletions(-)
 delete mode 100644 server/src/engine/net/protocol/data_exchange.rs
 create mode 100644 server/src/engine/net/protocol/exchange.rs

diff --git a/server/src/engine/net/protocol/data_exchange.rs b/server/src/engine/net/protocol/data_exchange.rs
deleted file mode 100644
index b964ca9c..00000000
--- a/server/src/engine/net/protocol/data_exchange.rs
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Created on Mon Sep 18 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
-    Skyhash/2 Data Exchange Packets
-    ---
-    1. Client side packet:
-    a. SQ
-    S<len>\n<payload>
-    b. PQ
-    P<count>\n(<pld len>\n<pld>)*
-
-    TODO(@ohsayan): Restore pipeline impl
-*/
-
-use crate::{engine::mem::BufferedScanner, util::compiler};
-use std::slice;
-
-#[derive(Debug, PartialEq)]
-pub struct CSQuery<'a> {
-    query: &'a [u8],
-}
-
-impl<'a> CSQuery<'a> {
-    pub(super) const fn new(query: &'a [u8]) -> Self {
-        Self { query }
-    }
-    pub const fn query(&self) -> &'a [u8] {
-        self.query
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub enum CSQueryState {
-    Initial,
-    SizeSegmentPart(u64),
-    WaitingForFullBlock(usize),
-}
-
-impl Default for CSQueryState {
-    fn default() -> Self {
-        Self::Initial
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub enum CSQueryExchangeResult<'a> {
-    Completed(CSQuery<'a>),
-    ChangeState(CSQueryState, usize),
-    PacketError,
-}
-
-impl<'a> CSQuery<'a> {
-    pub fn resume_with(
-        scanner: &mut BufferedScanner<'a>,
-        state: CSQueryState,
-    ) -> CSQueryExchangeResult<'a> {
-        match state {
-            CSQueryState::Initial => Self::resume_initial(scanner),
-            CSQueryState::SizeSegmentPart(part) => Self::resume_at_meta_segment(scanner, part),
-            CSQueryState::WaitingForFullBlock(size) => Self::resume_at_data_segment(scanner, size),
-        }
-    }
-}
-
-enum LFTIntParseResult {
-    Value(u64),
-    Partial(u64),
-    Error,
-}
-
-fn parse_lf_separated(
-    scanner: &mut BufferedScanner,
-    previously_buffered: u64,
-) -> LFTIntParseResult {
-    let mut ret = previously_buffered;
-    let mut okay = true;
-    while scanner.rounded_cursor_not_eof_matches(|b| *b != b'\n') & okay {
-        let b = unsafe { scanner.next_byte() };
-        okay &= b.is_ascii_digit();
-        ret = match ret.checked_mul(10) {
-            Some(r) => r,
-            None => return LFTIntParseResult::Error,
-        };
-        ret = match ret.checked_add((b & 0x0F) as u64) {
-            Some(r) => r,
-            None => return LFTIntParseResult::Error,
-        };
-    }
-    let payload_ok = okay;
-    let lf_ok = scanner.rounded_cursor_not_eof_matches(|b| *b == b'\n');
-    unsafe { scanner.incr_cursor_by(lf_ok as usize) }
-    if payload_ok & lf_ok {
-        LFTIntParseResult::Value(ret)
-    } else {
-        if payload_ok {
-            LFTIntParseResult::Partial(ret)
-        } else {
-            LFTIntParseResult::Error
-        }
-    }
-}
-
-impl<'a> CSQuery<'a> {
-    pub const PREEMPTIVE_READ: usize = 4;
-    const FIRST_BYTE: u8 = b'S';
-    fn resume_initial(scanner: &mut BufferedScanner<'a>) -> CSQueryExchangeResult<'a> {
-        if cfg!(debug_assertions) {
-            if scanner.remaining() < Self::PREEMPTIVE_READ {
-                return CSQueryExchangeResult::ChangeState(
-                    CSQueryState::Initial,
-                    Self::PREEMPTIVE_READ,
-                );
-            }
-        } else {
-            assert!(scanner.remaining() >= Self::PREEMPTIVE_READ);
-        }
-        // get our block
-        let first_byte = unsafe { scanner.next_byte() };
-        // be optimistic and check first byte later
-        let size_of_query = match parse_lf_separated(scanner, 0) {
-            LFTIntParseResult::Value(v) => v as usize,
-            LFTIntParseResult::Partial(v) => {
-                if compiler::unlikely(first_byte != Self::FIRST_BYTE) {
-                    return CSQueryExchangeResult::PacketError;
-                } else {
-                    // expect at least 1 LF and at least 1 query byte
-                    return CSQueryExchangeResult::ChangeState(CSQueryState::SizeSegmentPart(v), 2);
-                }
-            }
-            LFTIntParseResult::Error => {
-                // that's pretty much over
-                return CSQueryExchangeResult::PacketError;
-            }
-        };
-        if compiler::unlikely(first_byte != Self::FIRST_BYTE) {
-            return CSQueryExchangeResult::PacketError;
-        }
-        Self::resume_at_data_segment(scanner, size_of_query)
-    }
-    fn resume_at_meta_segment(
-        scanner: &mut BufferedScanner<'a>,
-        previous: u64,
-    ) -> CSQueryExchangeResult<'a> {
-        match parse_lf_separated(scanner, previous) {
-            LFTIntParseResult::Value(v) => Self::resume_at_data_segment(scanner, v as usize),
-            LFTIntParseResult::Partial(p) => {
-                CSQueryExchangeResult::ChangeState(CSQueryState::SizeSegmentPart(p), 2)
-            }
-            LFTIntParseResult::Error => CSQueryExchangeResult::PacketError,
-        }
-    }
-    fn resume_at_data_segment(
-        scanner: &mut BufferedScanner<'a>,
-        size: usize,
-    ) -> CSQueryExchangeResult<'a> {
-        if scanner.has_left(size) {
-            let slice;
-            unsafe {
-                // UNSAFE(@ohsayan): checked len at branch
-                slice = slice::from_raw_parts(scanner.current_buffer().as_ptr(), size);
-                scanner.incr_cursor_by(size);
-            }
-            CSQueryExchangeResult::Completed(CSQuery::new(slice))
-        } else {
-            CSQueryExchangeResult::ChangeState(CSQueryState::WaitingForFullBlock(size), size)
-        }
-    }
-}
diff --git a/server/src/engine/net/protocol/exchange.rs b/server/src/engine/net/protocol/exchange.rs
new file mode 100644
index 00000000..739757b7
--- /dev/null
+++ b/server/src/engine/net/protocol/exchange.rs
@@ -0,0 +1,318 @@
+/*
+ * Created on Wed Sep 20 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::mem::BufferedScanner;
+
+pub const EXCHANGE_MIN_SIZE: usize = b"S1\nh".len();
+pub(super) const STATE_READ_INITIAL: QueryTimeExchangeResult<'static> =
+    QueryTimeExchangeResult::ChangeState {
+        new_state: QueryTimeExchangeState::Initial,
+        expect_more: EXCHANGE_MIN_SIZE,
+    };
+pub(super) const STATE_ERROR: QueryTimeExchangeResult<'static> = QueryTimeExchangeResult::Error;
+
+#[derive(Debug, PartialEq)]
+/// State of a query time exchange
+pub enum QueryTimeExchangeState {
+    /// beginning of exchange
+    Initial,
+    /// SQ (part of packet size)
+    SQ1Meta1Partial { packet_size_part: u64 },
+    /// SQ (part of Q window)
+    SQ2Meta2Partial {
+        size_of_static_frame: usize,
+        packet_size: usize,
+        q_window_part: u64,
+    },
+    /// SQ waiting for block
+    SQ3FinalizeWaitingForBlock {
+        dataframe_size: usize,
+        q_window: usize,
+    },
+}
+
+impl Default for QueryTimeExchangeState {
+    fn default() -> Self {
+        Self::Initial
+    }
+}
+
+#[derive(Debug, PartialEq)]
+/// Result after attempting to complete (or terminate) a query time exchange
+pub enum QueryTimeExchangeResult<'a> {
+    /// We completed the exchange and yielded a [`SQuery`]
+    SQCompleted(SQuery<'a>),
+    /// We're changing states
+    ChangeState {
+        new_state: QueryTimeExchangeState,
+        expect_more: usize,
+    },
+    /// We hit an error and need to terminate this exchange
+    Error,
+}
+
+/// Resume a query time exchange
+pub fn resume<'a>(
+    scanner: &mut BufferedScanner<'a>,
+    state: QueryTimeExchangeState,
+) -> QueryTimeExchangeResult<'a> {
+    if cfg!(debug_assertions) {
+        if !scanner.has_left(EXCHANGE_MIN_SIZE) {
+            return STATE_READ_INITIAL;
+        }
+    } else {
+        assert!(scanner.has_left(EXCHANGE_MIN_SIZE));
+    }
+    match state {
+        QueryTimeExchangeState::Initial => {
+            // attempt to read atleast one byte
+            if cfg!(debug_assertions) {
+                match scanner.try_next_byte() {
+                    Some(b'S') => SQuery::resume_initial(scanner),
+                    Some(_) => return STATE_ERROR,
+                    None => return STATE_READ_INITIAL,
+                }
+            } else {
+                match unsafe { scanner.next_byte() } {
+                    b'S' => SQuery::resume_initial(scanner),
+                    _ => return STATE_ERROR,
+                }
+            }
+        }
+        QueryTimeExchangeState::SQ1Meta1Partial { packet_size_part } => {
+            SQuery::resume_at_sq1_meta1_partial(scanner, packet_size_part)
+        }
+        QueryTimeExchangeState::SQ2Meta2Partial {
+            packet_size,
+            q_window_part,
+            size_of_static_frame,
+        } => SQuery::resume_at_sq2_meta2_partial(
+            scanner,
+            size_of_static_frame,
+            packet_size,
+            q_window_part,
+        ),
+        QueryTimeExchangeState::SQ3FinalizeWaitingForBlock {
+            dataframe_size,
+            q_window,
+        } => SQuery::resume_at_final(scanner, q_window, dataframe_size),
+    }
+}
+
+/*
+    SQ
+*/
+
+enum LFTIntParseResult {
+    Value(u64),
+    Partial(u64),
+    Error,
+}
+
+fn parse_lf_separated(
+    scanner: &mut BufferedScanner,
+    previously_buffered: u64,
+) -> LFTIntParseResult {
+    let mut ret = previously_buffered;
+    let mut okay = true;
+    while scanner.rounded_cursor_not_eof_matches(|b| *b != b'\n') & okay {
+        let b = unsafe { scanner.next_byte() };
+        okay &= b.is_ascii_digit();
+        ret = match ret.checked_mul(10) {
+            Some(r) => r,
+            None => return LFTIntParseResult::Error,
+        };
+        ret = match ret.checked_add((b & 0x0F) as u64) {
+            Some(r) => r,
+            None => return LFTIntParseResult::Error,
+        };
+    }
+    let payload_ok = okay;
+    let lf_ok = scanner.rounded_cursor_not_eof_matches(|b| *b == b'\n');
+    unsafe { scanner.incr_cursor_if(lf_ok) }
+    if payload_ok & lf_ok {
+        LFTIntParseResult::Value(ret)
+    } else {
+        if payload_ok {
+            LFTIntParseResult::Partial(ret)
+        } else {
+            LFTIntParseResult::Error
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub struct SQuery<'a> {
+    q: &'a [u8],
+    q_window: usize,
+}
+
+impl<'a> SQuery<'a> {
+    pub(super) fn new(q: &'a [u8], q_window: usize) -> Self {
+        Self { q, q_window }
+    }
+    pub fn q(&self) -> &'a [u8] {
+        self.q
+    }
+    pub fn q_window(&self) -> usize {
+        self.q_window
+    }
+    pub fn query(&self) -> &'a [u8] {
+        &self.q[..self.q_window]
+    }
+    pub fn query_str(&self) -> Option<&'a str> {
+        core::str::from_utf8(self.query()).ok()
+    }
+    pub fn params(&self) -> &'a [u8] {
+        &self.q[self.q_window..]
+    }
+    pub fn params_str(&self) -> Option<&'a str> {
+        core::str::from_utf8(self.params()).ok()
+    }
+}
+
+impl<'a> SQuery<'a> {
+    /// We're touching this packet for the first time
+    fn resume_initial(scanner: &mut BufferedScanner<'a>) -> QueryTimeExchangeResult<'a> {
+        Self::resume_at_sq1_meta1_partial(scanner, 0)
+    }
+    /// We found some part of meta1, and need to resume
+    fn resume_at_sq1_meta1_partial(
+        scanner: &mut BufferedScanner<'a>,
+        prev: u64,
+    ) -> QueryTimeExchangeResult<'a> {
+        match parse_lf_separated(scanner, prev) {
+            LFTIntParseResult::Value(packet_size) => {
+                // we got the packet size; can we get the q window?
+                Self::resume_at_sq2_meta2_partial(
+                    scanner,
+                    scanner.cursor(),
+                    packet_size as usize,
+                    0,
+                )
+            }
+            LFTIntParseResult::Partial(partial_packet_size) => {
+                // we couldn't get the packet size
+                QueryTimeExchangeResult::ChangeState {
+                    new_state: QueryTimeExchangeState::SQ1Meta1Partial {
+                        packet_size_part: partial_packet_size,
+                    },
+                    expect_more: 3, // 1LF + 1ASCII + 1LF
+                }
+            }
+            LFTIntParseResult::Error => STATE_ERROR,
+        }
+    }
+    /// We found some part of meta2, and need to resume
+    fn resume_at_sq2_meta2_partial(
+        scanner: &mut BufferedScanner<'a>,
+        static_size: usize,
+        packet_size: usize,
+        prev_qw_buffered: u64,
+    ) -> QueryTimeExchangeResult<'a> {
+        let start = scanner.cursor();
+        match parse_lf_separated(scanner, prev_qw_buffered) {
+            LFTIntParseResult::Value(q_window) => {
+                // we got the q window; can we complete the exchange?
+                Self::resume_at_final(
+                    scanner,
+                    q_window as usize,
+                    Self::compute_df_size(scanner, static_size, packet_size),
+                )
+            }
+            LFTIntParseResult::Partial(q_window_partial) => {
+                // not enough bytes for getting Q window
+                QueryTimeExchangeResult::ChangeState {
+                    new_state: QueryTimeExchangeState::SQ2Meta2Partial {
+                        packet_size,
+                        q_window_part: q_window_partial,
+                        size_of_static_frame: static_size,
+                    },
+                    // we passed cursor - start bytes out of the packet, so expect this more
+                    expect_more: packet_size - (scanner.cursor() - start),
+                }
+            }
+            LFTIntParseResult::Error => STATE_ERROR,
+        }
+    }
+    /// We got all our meta and need the dataframe
+    fn resume_at_final(
+        scanner: &mut BufferedScanner<'a>,
+        q_window: usize,
+        dataframe_size: usize,
+    ) -> QueryTimeExchangeResult<'a> {
+        if scanner.has_left(dataframe_size) {
+            // we have sufficient bytes for the dataframe
+            unsafe {
+                // UNSAFE(@ohsayan): +lenck
+                QueryTimeExchangeResult::SQCompleted(SQuery::new(
+                    scanner.next_chunk_variable(dataframe_size),
+                    q_window,
+                ))
+            }
+        } else {
+            // not enough bytes for the dataframe
+            QueryTimeExchangeResult::ChangeState {
+                new_state: QueryTimeExchangeState::SQ3FinalizeWaitingForBlock {
+                    dataframe_size,
+                    q_window,
+                },
+                expect_more: Self::compute_df_remaining(scanner, dataframe_size), // dataframe
+            }
+        }
+    }
+}
+
+impl<'a> SQuery<'a> {
+    fn compute_df_size(scanner: &BufferedScanner, static_size: usize, packet_size: usize) -> usize {
+        packet_size - scanner.cursor() + static_size
+    }
+    fn compute_df_remaining(scanner: &BufferedScanner<'_>, df_size: usize) -> usize {
+        (scanner.cursor() + df_size) - scanner.buffer_len()
+    }
+}
+
+pub(super) fn create_simple_query<const N: usize>(query: &str, params: [&str; N]) -> Vec<u8> {
+    let mut buf = vec![];
+    let query_size_as_string = query.len().to_string();
+    let size_of_variable_section = query.len()
+        + params.iter().map(|l| l.len()).sum::<usize>()
+        + query_size_as_string.len()
+        + 1;
+    // segment 1
+    buf.push(b'S');
+    buf.extend(size_of_variable_section.to_string().as_bytes());
+    buf.push(b'\n');
+    // segment
+    buf.extend(query_size_as_string.as_bytes());
+    buf.push(b'\n');
+    // dataframe
+    buf.extend(query.as_bytes());
+    params
+        .into_iter()
+        .for_each(|param| buf.extend(param.as_bytes()));
+    buf
+}
diff --git a/server/src/engine/net/protocol/handshake.rs b/server/src/engine/net/protocol/handshake.rs
index c1d0545b..3b1cc2e4 100644
--- a/server/src/engine/net/protocol/handshake.rs
+++ b/server/src/engine/net/protocol/handshake.rs
@@ -24,12 +24,9 @@
  *
 */
 
-use {
-    crate::{
-        engine::mem::scanner::{BufferedScanner, ScannerDecodeResult},
-        util::compiler,
-    },
-    std::slice,
+use crate::{
+    engine::mem::scanner::{BufferedScanner, ScannerDecodeResult},
+    util::compiler,
 };
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy, sky_macros::EnumMethods)]
@@ -320,10 +317,8 @@ impl<'a> CHandshake<'a> {
             // we're done here
             return unsafe {
                 // UNSAFE(@ohsayan): we just checked buffered size
-                let uname = slice::from_raw_parts(scanner.current_buffer().as_ptr(), uname_l);
-                let pwd =
-                    slice::from_raw_parts(scanner.current_buffer().as_ptr().add(uname_l), pwd_l);
-                scanner.incr_cursor_by(uname_l + pwd_l);
+                let uname = scanner.next_chunk_variable(uname_l);
+                let pwd = scanner.next_chunk_variable(pwd_l);
                 HandshakeResult::Completed(Self::new(
                     static_hs,
                     Some(CHandshakeAuth::new(uname, pwd)),
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index ba794028..81bc3f2f 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -24,7 +24,7 @@
  *
 */
 
-mod data_exchange;
+mod exchange;
 mod handshake;
 #[cfg(test)]
 mod tests;
diff --git a/server/src/engine/net/protocol/tests.rs b/server/src/engine/net/protocol/tests.rs
index e2159deb..6c05058d 100644
--- a/server/src/engine/net/protocol/tests.rs
+++ b/server/src/engine/net/protocol/tests.rs
@@ -27,7 +27,7 @@
 use crate::engine::{
     mem::BufferedScanner,
     net::protocol::{
-        data_exchange::{CSQuery, CSQueryExchangeResult, CSQueryState},
+        exchange::{self, create_simple_query, QueryTimeExchangeResult, QueryTimeExchangeState},
         handshake::{
             AuthMode, CHandshake, CHandshakeAuth, CHandshakeStatic, DataExchangeMode,
             HandshakeResult, HandshakeState, HandshakeVersion, ProtocolVersion, QueryMode,
@@ -207,58 +207,112 @@ fn parse_auth_with_state_updates() {
     QT-DEX/SQ
 */
 
-const FULL_SQ: [u8; 116] = *b"S111\nSELECT username, email, bio, profile_pic, following, followers, FROM mysocialapp.users WHERE username = 'sayan'";
-const SQ_FULL: CSQuery<'static> = CSQuery::new(
-    b"SELECT username, email, bio, profile_pic, following, followers, FROM mysocialapp.users WHERE username = 'sayan'"
-);
+const SQ: &str = "select * from myspace.mymodel where username = ?";
 
 #[test]
-fn staged_qt_dex_sq() {
-    for i in 0..FULL_SQ.len() {
-        let buf = &FULL_SQ[..i + 1];
-        let mut scanner = BufferedScanner::new(buf);
-        let result = CSQuery::resume_with(&mut scanner, CSQueryState::default());
-        match buf.len() {
-            1..=3 => assert_eq!(
+fn qtdex_simple_query() {
+    let query = create_simple_query(SQ, ["sayan"]);
+    let mut fin = 52;
+    for i in 0..query.len() {
+        let mut scanner = BufferedScanner::new(&query[..i + 1]);
+        let result = exchange::resume(&mut scanner, Default::default());
+        match scanner.buffer_len() {
+            1..=3 => assert_eq!(result, exchange::STATE_READ_INITIAL),
+            4 => assert_eq!(
                 result,
-                CSQueryExchangeResult::ChangeState(CSQueryState::Initial, CSQuery::PREEMPTIVE_READ)
+                QueryTimeExchangeResult::ChangeState {
+                    new_state: QueryTimeExchangeState::SQ2Meta2Partial {
+                        size_of_static_frame: 4,
+                        packet_size: 56,
+                        q_window_part: 0,
+                    },
+                    expect_more: 56,
+                }
             ),
-            4 => assert_eq!(
+            5 => assert_eq!(
+                result,
+                QueryTimeExchangeResult::ChangeState {
+                    new_state: QueryTimeExchangeState::SQ2Meta2Partial {
+                        size_of_static_frame: 4,
+                        packet_size: 56,
+                        q_window_part: 4,
+                    },
+                    expect_more: 55,
+                }
+            ),
+            6 => assert_eq!(
                 result,
-                CSQueryExchangeResult::ChangeState(CSQueryState::SizeSegmentPart(111), 2)
+                QueryTimeExchangeResult::ChangeState {
+                    new_state: QueryTimeExchangeState::SQ2Meta2Partial {
+                        size_of_static_frame: 4,
+                        packet_size: 56,
+                        q_window_part: 48,
+                    },
+                    expect_more: 54,
+                }
             ),
-            5..=115 => assert_eq!(
+            7 => assert_eq!(
                 result,
-                CSQueryExchangeResult::ChangeState(CSQueryState::WaitingForFullBlock(111), 111),
+                QueryTimeExchangeResult::ChangeState {
+                    new_state: QueryTimeExchangeState::SQ3FinalizeWaitingForBlock {
+                        dataframe_size: 53,
+                        q_window: 48,
+                    },
+                    expect_more: 53,
+                }
             ),
-            116 => assert_eq!(result, CSQueryExchangeResult::Completed(SQ_FULL)),
+            8..=59 => {
+                assert_eq!(
+                    result,
+                    QueryTimeExchangeResult::ChangeState {
+                        new_state: QueryTimeExchangeState::SQ3FinalizeWaitingForBlock {
+                            dataframe_size: 53,
+                            q_window: 48
+                        },
+                        expect_more: fin,
+                    }
+                );
+                fin -= 1;
+            }
+            60 => match result {
+                QueryTimeExchangeResult::SQCompleted(sq) => {
+                    assert_eq!(sq.query_str().unwrap(), SQ);
+                    assert_eq!(sq.params_str().unwrap(), "sayan");
+                }
+                _ => unreachable!(),
+            },
             _ => unreachable!(),
         }
     }
 }
 
 #[test]
-fn staged_with_status_switch_qt_dex_sq() {
+fn qtdex_simple_query_update_state() {
+    let query = create_simple_query(SQ, ["sayan"]);
+    let mut state = QueryTimeExchangeState::default();
     let mut cursor = 0;
-    let mut expect = CSQuery::PREEMPTIVE_READ;
-    let mut state = CSQueryState::default();
+    let mut expected = 0;
     let mut rounds = 0;
     loop {
         rounds += 1;
-        let buf = &FULL_SQ[..cursor + expect];
+        let buf = &query[..expected + cursor];
         let mut scanner = unsafe { BufferedScanner::new_with_cursor(buf, cursor) };
-        match CSQuery::resume_with(&mut scanner, state) {
-            CSQueryExchangeResult::Completed(c) => {
-                assert_eq!(c, SQ_FULL);
+        match exchange::resume(&mut scanner, state) {
+            QueryTimeExchangeResult::SQCompleted(sq) => {
+                assert_eq!(sq.query_str().unwrap(), SQ);
+                assert_eq!(sq.params_str().unwrap(), "sayan");
                 break;
             }
-            CSQueryExchangeResult::ChangeState(new_state, _expect) => {
+            QueryTimeExchangeResult::ChangeState {
+                new_state,
+                expect_more,
+            } => {
+                expected = expect_more;
                 state = new_state;
-                expect = _expect;
-                cursor = scanner.cursor();
             }
-            CSQueryExchangeResult::PacketError => panic!("packet error"),
+            QueryTimeExchangeResult::Error => panic!("hit error!"),
         }
+        cursor = scanner.cursor();
     }
     assert_eq!(rounds, 3);
 }
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
index 53fd6613..5bd97a3e 100644
--- a/server/src/engine/ql/benches.rs
+++ b/server/src/engine/ql/benches.rs
@@ -45,10 +45,7 @@ use {
 mod lexer {
     use {
         super::*,
-        crate::engine::{
-            data::{lit::Lit, spec::Dataspec1D},
-            ql::lex::Token,
-        },
+        crate::engine::{data::lit::Lit, ql::lex::Token},
     };
     #[bench]
     fn lex_number(b: &mut Bencher) {

From 6a01d9d513b86077e7342d2095afcd90c770d83c Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 22 Sep 2023 06:48:56 +0000
Subject: [PATCH 258/310] Basic DCL impls

---
 server/src/engine/core/index/mod.rs           |   6 +-
 server/src/engine/data/dict.rs                |   9 +
 server/src/engine/error.rs                    |   1 +
 server/src/engine/fractal/config.rs           | 138 +++++++++++++--
 server/src/engine/fractal/mod.rs              |  14 +-
 server/src/engine/fractal/test_utils.rs       |   7 +-
 server/src/engine/ql/dcl.rs                   | 140 +++++++++++++++
 server/src/engine/ql/ddl/syn.rs               |  10 ++
 server/src/engine/ql/macros.rs                |   7 +
 server/src/engine/ql/mod.rs                   |   1 +
 server/src/engine/ql/tests.rs                 |   1 +
 server/src/engine/ql/tests/dcl.rs             |  44 +++++
 .../src/engine/storage/v1/header_impl/mod.rs  |   4 +
 server/src/engine/storage/v1/memfs.rs         | 161 +++++++++++-------
 server/src/engine/storage/v1/mod.rs           |   3 +-
 server/src/engine/storage/v1/rw.rs            |   8 +
 server/src/engine/storage/v1/start_stop.rs    |   2 +-
 server/src/engine/storage/v1/sysdb.rs         |  96 +++++++++++
 18 files changed, 565 insertions(+), 87 deletions(-)
 create mode 100644 server/src/engine/ql/dcl.rs
 create mode 100644 server/src/engine/ql/tests/dcl.rs
 create mode 100644 server/src/engine/storage/v1/sysdb.rs

diff --git a/server/src/engine/core/index/mod.rs b/server/src/engine/core/index/mod.rs
index ab40fbc5..a92b9741 100644
--- a/server/src/engine/core/index/mod.rs
+++ b/server/src/engine/core/index/mod.rs
@@ -52,11 +52,7 @@ impl PrimaryIndex {
     pub fn remove<'a>(&self, key: Lit<'a>, g: &Guard) -> bool {
         self.data.mt_delete(&key, g)
     }
-    pub fn select<'a, 'v, 't: 'v, 'g: 't>(
-        &'t self,
-        key: Lit<'a>,
-        g: &'g Guard,
-    ) -> Option<&'v Row> {
+    pub fn select<'a, 'v, 't: 'v, 'g: 't>(&'t self, key: Lit<'a>, g: &'g Guard) -> Option<&'v Row> {
         self.data.mt_get_element(&key, g)
     }
     pub fn __raw_index(&self) -> &IndexMTRaw<row::Row> {
diff --git a/server/src/engine/data/dict.rs b/server/src/engine/data/dict.rs
index e67cd71d..9295f051 100644
--- a/server/src/engine/data/dict.rs
+++ b/server/src/engine/data/dict.rs
@@ -45,6 +45,15 @@ pub enum DictEntryGeneric {
     Map(DictGeneric),
 }
 
+impl DictEntryGeneric {
+    pub fn as_dict_mut(&mut self) -> Option<&mut DictGeneric> {
+        match self {
+            Self::Map(m) => Some(m),
+            _ => None,
+        }
+    }
+}
+
 /*
     patchsets
 */
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index e4fd079d..ecf1c60e 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -93,6 +93,7 @@ pub enum Error {
     TransactionalError,
     /// storage subsystem error
     StorageSubsystemError,
+    SysAuthError,
 }
 
 direct_from! {
diff --git a/server/src/engine/fractal/config.rs b/server/src/engine/fractal/config.rs
index 5b898ec0..6090c532 100644
--- a/server/src/engine/fractal/config.rs
+++ b/server/src/engine/fractal/config.rs
@@ -24,24 +24,134 @@
  *
 */
 
-use crate::engine::storage::v1::header_meta::HostRunMode;
+use {
+    crate::engine::{
+        error::{Error, QueryResult},
+        storage::v1::header_meta::HostRunMode,
+    },
+    parking_lot::RwLock,
+    std::collections::{hash_map::Entry, HashMap},
+};
 
-pub struct ServerConfig {
-    host_settings_version: u32,
-    host_run_mode: HostRunMode,
-    host_startup_counter: u64,
+#[derive(Debug)]
+/// The global system configuration
+pub struct SysConfig {
+    auth_data: RwLock<Option<SysAuth>>,
+    host_data: SysHostData,
 }
 
-impl ServerConfig {
-    pub fn new(
-        host_settings_version: u32,
-        host_run_mode: HostRunMode,
-        host_startup_counter: u64,
-    ) -> Self {
+impl SysConfig {
+    /// Initialize a new system config
+    pub fn new(auth_data: RwLock<Option<SysAuth>>, host_data: SysHostData) -> Self {
         Self {
-            host_settings_version,
-            host_run_mode,
-            host_startup_counter,
+            auth_data,
+            host_data,
         }
     }
+    #[cfg(test)]
+    /// A test-mode default setting with auth disabled
+    pub(super) fn test_default() -> Self {
+        Self {
+            auth_data: RwLock::new(None),
+            host_data: SysHostData::new(0, HostRunMode::Prod, 0),
+        }
+    }
+    /// Returns a handle to the authentication data
+    pub fn auth_data(&self) -> &RwLock<Option<SysAuth>> {
+        &self.auth_data
+    }
+    /// Returns a reference to host data
+    pub fn host_data(&self) -> &SysHostData {
+        &self.host_data
+    }
+}
+
+#[derive(Debug, PartialEq)]
+/// The host data section (system.host)
+pub struct SysHostData {
+    startup_counter: u64,
+    run_mode: HostRunMode,
+    settings_version: u32,
+}
+
+impl SysHostData {
+    /// New [`SysHostData`]
+    pub fn new(startup_counter: u64, run_mode: HostRunMode, settings_version: u32) -> Self {
+        Self {
+            startup_counter,
+            run_mode,
+            settings_version,
+        }
+    }
+    pub fn startup_counter(&self) -> u64 {
+        self.startup_counter
+    }
+    pub fn run_mode(&self) -> HostRunMode {
+        self.run_mode
+    }
+    pub fn settings_version(&self) -> u32 {
+        self.settings_version
+    }
+}
+
+/*
+    auth
+*/
+
+#[derive(Debug, PartialEq)]
+/// The auth data section (system.auth)
+pub struct SysAuth {
+    root_key: Box<[u8]>,
+    users: HashMap<Box<str>, SysAuthUser>,
+}
+
+impl SysAuth {
+    /// New [`SysAuth`] with the given settings
+    pub fn new(root_key: Box<[u8]>, users: HashMap<Box<str>, SysAuthUser>) -> Self {
+        Self { root_key, users }
+    }
+    /// Create a new user with the given details
+    pub fn create_new_user(&mut self, username: &str, password: &str) -> QueryResult<()> {
+        match self.users.entry(username.into()) {
+            Entry::Vacant(ve) => {
+                ve.insert(SysAuthUser::new(
+                    rcrypt::hash(password, rcrypt::DEFAULT_COST)
+                        .unwrap()
+                        .into_boxed_slice(),
+                ));
+                Ok(())
+            }
+            Entry::Occupied(_) => Err(Error::SysAuthError),
+        }
+    }
+    /// Verify the user with the given details
+    pub fn verify_user(&self, username: &str, password: &str) -> QueryResult<()> {
+        match self.users.get(username) {
+            Some(user) if rcrypt::verify(password, user.key()).unwrap() => Ok(()),
+            Some(_) | None => Err(Error::SysAuthError),
+        }
+    }
+    pub fn root_key(&self) -> &[u8] {
+        &self.root_key
+    }
+    pub fn users(&self) -> &HashMap<Box<str>, SysAuthUser> {
+        &self.users
+    }
+}
+
+#[derive(Debug, PartialEq)]
+/// The auth user
+pub struct SysAuthUser {
+    key: Box<[u8]>,
+}
+
+impl SysAuthUser {
+    /// Create a new [`SysAuthUser`]
+    fn new(key: Box<[u8]>) -> Self {
+        Self { key }
+    }
+    /// Get the key
+    pub fn key(&self) -> &[u8] {
+        self.key.as_ref()
+    }
 }
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index 26f5d1db..e5978542 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -43,7 +43,7 @@ mod mgr;
 pub mod test_utils;
 mod util;
 pub use {
-    config::ServerConfig,
+    config::SysConfig,
     drivers::FractalModelDriver,
     mgr::{CriticalTask, GenericTask, Task},
     util::FractalToken,
@@ -71,7 +71,7 @@ pub struct GlobalStateStart {
 /// Must be called iff this is the only thread calling it
 pub unsafe fn enable_and_start_all(
     gns: GlobalNS,
-    config: config::ServerConfig,
+    config: config::SysConfig,
     gns_driver: GNSTransactionDriverAnyFS<LocalFS>,
     model_drivers: ModelDrivers,
 ) -> GlobalStateStart {
@@ -124,6 +124,8 @@ pub trait GlobalInstanceLike {
             observed_len,
         )))
     }
+    // config handle
+    fn sys_cfg(&self) -> &config::SysConfig;
 }
 
 impl GlobalInstanceLike for Global {
@@ -146,6 +148,10 @@ impl GlobalInstanceLike for Global {
     fn get_max_delta_size(&self) -> usize {
         self._get_max_delta_size()
     }
+    // sys
+    fn sys_cfg(&self) -> &config::SysConfig {
+        &self.get_state().config
+    }
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -194,7 +200,7 @@ struct GlobalState {
     gns_driver: drivers::FractalGNSDriver,
     mdl_driver: RwLock<ModelDrivers>,
     task_mgr: mgr::FractalMgr,
-    config: config::ServerConfig,
+    config: config::SysConfig,
 }
 
 impl GlobalState {
@@ -203,7 +209,7 @@ impl GlobalState {
         gns_driver: drivers::FractalGNSDriver,
         mdl_driver: RwLock<ModelDrivers>,
         task_mgr: mgr::FractalMgr,
-        config: config::ServerConfig,
+        config: config::SysConfig,
     ) -> Self {
         Self {
             gns,
diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
index 9789754a..03321645 100644
--- a/server/src/engine/fractal/test_utils.rs
+++ b/server/src/engine/fractal/test_utils.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    super::{CriticalTask, GenericTask, GlobalInstanceLike, Task},
+    super::{CriticalTask, GenericTask, GlobalInstanceLike, SysConfig, Task},
     crate::engine::{
         core::GlobalNS,
         storage::v1::{
@@ -45,6 +45,7 @@ pub struct TestGlobal<Fs: RawFSInterface = VirtualFS> {
     lp_queue: RwLock<Vec<Task<GenericTask>>>,
     max_delta_size: usize,
     txn_driver: Mutex<GNSTransactionDriverAnyFS<Fs>>,
+    sys_cfg: super::SysConfig,
 }
 
 impl<Fs: RawFSInterface> TestGlobal<Fs> {
@@ -59,6 +60,7 @@ impl<Fs: RawFSInterface> TestGlobal<Fs> {
             lp_queue: RwLock::default(),
             max_delta_size,
             txn_driver: Mutex::new(txn_driver),
+            sys_cfg: SysConfig::test_default(),
         }
     }
 }
@@ -110,6 +112,9 @@ impl<Fs: RawFSInterface> GlobalInstanceLike for TestGlobal<Fs> {
     fn get_max_delta_size(&self) -> usize {
         100
     }
+    fn sys_cfg(&self) -> &super::config::SysConfig {
+        &self.sys_cfg
+    }
 }
 
 impl<Fs: RawFSInterface> Drop for TestGlobal<Fs> {
diff --git a/server/src/engine/ql/dcl.rs b/server/src/engine/ql/dcl.rs
new file mode 100644
index 00000000..0456dc1a
--- /dev/null
+++ b/server/src/engine/ql/dcl.rs
@@ -0,0 +1,140 @@
+/*
+ * Created on Thu Sep 21 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    data::{
+        tag::{DataTag, TagClass},
+        DictGeneric,
+    },
+    error::{Error, QueryResult},
+    ql::{
+        ast::{traits, QueryData, State},
+        ddl::syn,
+    },
+};
+
+fn parse<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<UserMeta<'a>> {
+    /*
+        user add [username] with { password: [password], ... }
+                 ^cursor
+        7 tokens
+    */
+    if state.remaining() < 7 {
+        return Err(Error::QLInvalidSyntax);
+    }
+    let token_buffer = state.current();
+    // initial sig
+    let signature_okay = token_buffer[0].is_lit()
+        & token_buffer[1].eq(&Token![with])
+        & token_buffer[2].eq(&Token![open {}]);
+    // get props
+    state.poison_if_not(signature_okay);
+    state.cursor_ahead_by(2);
+    let Some(dict) = syn::parse_dict(state) else {
+        return Err(Error::QLInvalidCollectionSyntax);
+    };
+    let maybe_username = unsafe {
+        // UNSAFE(@ohsayan): the dict parse ensures state correctness
+        token_buffer[0].uck_read_lit()
+    };
+    state.poison_if_not(maybe_username.kind().tag_class() == TagClass::Str);
+    if state.not_exhausted() & !state.okay() {
+        // we shouldn't have more tokens
+        return Err(Error::QLInvalidSyntax);
+    }
+    Ok(UserMeta {
+        username: unsafe {
+            // UNSAFE(@ohsayan): +tagck in state
+            maybe_username.str()
+        },
+        options: dict,
+    })
+}
+
+struct UserMeta<'a> {
+    username: &'a str,
+    options: DictGeneric,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct UserAdd<'a> {
+    username: &'a str,
+    options: DictGeneric,
+}
+
+impl<'a> UserAdd<'a> {
+    pub(in crate::engine::ql) fn new(username: &'a str, options: DictGeneric) -> Self {
+        Self { username, options }
+    }
+    /// Parse a `user add` DCL command
+    ///
+    /// MUSTENDSTREAM: YES
+    pub fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        parse(state).map(|UserMeta { username, options }: UserMeta| Self::new(username, options))
+    }
+}
+
+impl<'a> traits::ASTNode<'a> for UserAdd<'a> {
+    fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        Self::parse(state)
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub struct UserDel<'a> {
+    username: &'a str,
+}
+
+impl<'a> UserDel<'a> {
+    pub(in crate::engine::ql) fn new(username: &'a str) -> Self {
+        Self { username }
+    }
+    /// Parse a `user del` DCL command
+    ///
+    /// MUSTENDSTREAM: YES
+    pub fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        if state.can_read_lit_rounded() & (state.remaining() == 1) {
+            let lit = unsafe {
+                // UNSAFE(@ohsayan): +boundck
+                state.read_cursor_lit_unchecked()
+            };
+            state.cursor_ahead();
+            if lit.kind().tag_class() == TagClass::Str {
+                return Ok(Self::new(unsafe {
+                    // UNSAFE(@ohsayan): +tagck
+                    lit.str()
+                }));
+            }
+        }
+        Err(Error::QLInvalidSyntax)
+    }
+}
+
+impl<'a> traits::ASTNode<'a> for UserDel<'a> {
+    fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        Self::parse(state)
+    }
+}
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index da376e1d..dd256224 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -231,6 +231,16 @@ pub(super) fn rfold_dict<'a, Qd: QueryData<'a>>(
     _rfold_dict::<Qd, NoBreakpoint>(mstate, state, dict);
 }
 
+pub fn parse_dict<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> Option<DictGeneric> {
+    let mut d = DictGeneric::new();
+    rfold_dict(DictFoldState::OB, state, &mut d);
+    if state.okay() {
+        Some(d)
+    } else {
+        None
+    }
+}
+
 pub(super) fn rfold_tymeta<'a, Qd: QueryData<'a>>(
     mstate: DictFoldState,
     state: &mut State<'a, Qd>,
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index b5e7cde5..0e6dae05 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -339,3 +339,10 @@ macro_rules! into_vec {
         v
     }}
 }
+
+#[cfg(test)]
+macro_rules! lit {
+    ($lit:expr) => {
+        $crate::engine::data::lit::Lit::from($lit)
+    };
+}
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index c5a14b7a..02c1368f 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -30,6 +30,7 @@ pub(super) mod ast;
 #[cfg(feature = "nightly")]
 #[cfg(test)]
 mod benches;
+pub(super) mod dcl;
 pub(super) mod ddl;
 pub(super) mod dml;
 pub(super) mod lex;
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index ebcef8d1..5efcf9ba 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -33,6 +33,7 @@ use {
     rand::{self, Rng},
 };
 
+mod dcl;
 mod dml_tests;
 mod entity;
 mod lexer_tests;
diff --git a/server/src/engine/ql/tests/dcl.rs b/server/src/engine/ql/tests/dcl.rs
new file mode 100644
index 00000000..35db503c
--- /dev/null
+++ b/server/src/engine/ql/tests/dcl.rs
@@ -0,0 +1,44 @@
+/*
+ * Created on Fri Sep 22 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::ql::{ast, dcl, tests::lex_insecure};
+
+#[test]
+fn create_user_simple() {
+    let query = lex_insecure(b"create user 'sayan' with { password: 'mypass123' }").unwrap();
+    let q = ast::parse_ast_node_full::<dcl::UserAdd>(&query[2..]).unwrap();
+    assert_eq!(
+        q,
+        dcl::UserAdd::new("sayan", into_dict!("password" => lit!("mypass123")))
+    )
+}
+
+#[test]
+fn delete_user_simple() {
+    let query = lex_insecure(b"delete user 'monster'").unwrap();
+    let q = ast::parse_ast_node_full::<dcl::UserDel>(&query[2..]).unwrap();
+    assert_eq!(q, dcl::UserDel::new("monster"))
+}
diff --git a/server/src/engine/storage/v1/header_impl/mod.rs b/server/src/engine/storage/v1/header_impl/mod.rs
index b8e1ae1f..59a74e3b 100644
--- a/server/src/engine/storage/v1/header_impl/mod.rs
+++ b/server/src/engine/storage/v1/header_impl/mod.rs
@@ -73,6 +73,7 @@ mod dr;
 pub enum FileScope {
     Journal = 0,
     DataBatch = 1,
+    FlatmapData = 2,
 }
 
 impl FileScope {
@@ -80,6 +81,7 @@ impl FileScope {
         Some(match id {
             0 => Self::Journal,
             1 => Self::DataBatch,
+            2 => Self::FlatmapData,
             _ => return None,
         })
     }
@@ -96,6 +98,7 @@ impl FileScope {
 pub enum FileSpecifier {
     GNSTxnLog = 0,
     TableDataBatch = 1,
+    SysDB = 2,
     #[cfg(test)]
     TestTransactionLog = 0xFF,
 }
@@ -105,6 +108,7 @@ impl FileSpecifier {
         Some(match v {
             0 => Self::GNSTxnLog,
             1 => Self::TableDataBatch,
+            2 => Self::SysDB,
             #[cfg(test)]
             0xFF => Self::TestTransactionLog,
             _ => return None,
diff --git a/server/src/engine/storage/v1/memfs.rs b/server/src/engine/storage/v1/memfs.rs
index d6480519..15a0a2a6 100644
--- a/server/src/engine/storage/v1/memfs.rs
+++ b/server/src/engine/storage/v1/memfs.rs
@@ -37,7 +37,10 @@ use {
     },
     parking_lot::RwLock,
     std::{
-        collections::{hash_map::Entry, HashMap},
+        collections::{
+            hash_map::{Entry, OccupiedEntry},
+            HashMap,
+        },
         io::{Error, ErrorKind},
     },
 };
@@ -74,6 +77,23 @@ impl VNode {
     }
 }
 
+/*
+    errors
+*/
+
+fn err_item_is_not_file<T>() -> super::SDSSResult<T> {
+    Err(Error::new(ErrorKind::InvalidInput, "found directory, not a file").into())
+}
+fn err_file_in_dir_path<T>() -> super::SDSSResult<T> {
+    Err(Error::new(ErrorKind::InvalidInput, "found file in directory path").into())
+}
+fn err_dir_missing_in_path<T>() -> super::SDSSResult<T> {
+    Err(Error::new(ErrorKind::InvalidInput, "could not find directory in path").into())
+}
+fn err_could_not_find_item<T>() -> super::SDSSResult<T> {
+    Err(Error::new(ErrorKind::NotFound, "could not find item").into())
+}
+
 /*
     vfs impl:
     - nested directory structure
@@ -97,6 +117,32 @@ pub struct VirtualFS;
 
 impl RawFSInterface for VirtualFS {
     type File = VFileDescriptor;
+    fn fs_rename_file(from: &str, to: &str) -> SDSSResult<()> {
+        // get file data
+        let data = with_file(from, |f| Ok(f.data.clone()))?;
+        // create new file
+        let file = VirtualFS::fs_fopen_or_create_rw(to)?;
+        match file {
+            RawFileOpen::Created(mut c) => {
+                c.fw_write_all(&data)?;
+            }
+            RawFileOpen::Existing(mut e) => {
+                e.fw_truncate_to(0)?;
+                e.fw_write_all(&data)?;
+            }
+        }
+        // delete old file
+        Self::fs_remove_file(from)
+    }
+    fn fs_remove_file(fpath: &str) -> SDSSResult<()> {
+        handle_item_mut(fpath, |e| match e.get() {
+            VNode::File(_) => {
+                e.remove();
+                Ok(())
+            }
+            _ => return err_item_is_not_file(),
+        })
+    }
     fn fs_create_dir(fpath: &str) -> super::SDSSResult<()> {
         // get vfs
         let mut vfs = VFS.write();
@@ -109,14 +155,8 @@ impl RawFSInterface for VirtualFS {
                 Some(VNode::Dir(d)) => {
                     current = d;
                 }
-                Some(VNode::File(_)) => {
-                    return Err(Error::new(ErrorKind::InvalidInput, "found file in path").into())
-                }
-                None => {
-                    return Err(
-                        Error::new(ErrorKind::NotFound, "could not find directory in path").into(),
-                    )
-                }
+                Some(VNode::File(_)) => return err_file_in_dir_path(),
+                None => return err_dir_missing_in_path(),
             }
         }
         match current.entry(target.into()) {
@@ -146,9 +186,7 @@ impl RawFSInterface for VirtualFS {
                     }
                     return create_ahead(ahead, d);
                 }
-                Some(VNode::File(_)) => {
-                    return Err(Error::new(ErrorKind::InvalidInput, "found file in path").into())
-                }
+                Some(VNode::File(_)) => return err_file_in_dir_path(),
                 None => {
                     let _ = current.insert(this.into(), VNode::Dir(into_dict!()));
                     let dir = current.get_mut(this).unwrap().as_dir_mut().unwrap();
@@ -177,11 +215,7 @@ impl RawFSInterface for VirtualFS {
                     f.write = true;
                     Ok(RawFileOpen::Existing(VFileDescriptor(fpath.into())))
                 }
-                VNode::Dir(_) => {
-                    return Err(
-                        Error::new(ErrorKind::InvalidInput, "found directory, not a file").into(),
-                    )
-                }
+                VNode::Dir(_) => return err_item_is_not_file(),
             },
             Entry::Vacant(v) => {
                 v.insert(VNode::File(VFile::new(true, true, vec![], 0)));
@@ -198,14 +232,8 @@ fn find_target_dir_mut<'a>(
     for component in components {
         match current.get_mut(component) {
             Some(VNode::Dir(d)) => current = d,
-            Some(VNode::File(_)) => {
-                return Err(Error::new(ErrorKind::InvalidInput, "found file in path").into())
-            }
-            None => {
-                return Err(
-                    Error::new(ErrorKind::NotFound, "could not find directory in path").into(),
-                )
-            }
+            Some(VNode::File(_)) => return err_file_in_dir_path(),
+            None => return err_dir_missing_in_path(),
         }
     }
     Ok(current)
@@ -218,20 +246,17 @@ fn find_target_dir<'a>(
     for component in components {
         match current.get(component) {
             Some(VNode::Dir(d)) => current = d,
-            Some(VNode::File(_)) => {
-                return Err(Error::new(ErrorKind::InvalidInput, "found file in path").into())
-            }
-            None => {
-                return Err(
-                    Error::new(ErrorKind::NotFound, "could not find directory in path").into(),
-                )
-            }
+            Some(VNode::File(_)) => return err_file_in_dir_path(),
+            None => return err_dir_missing_in_path(),
         }
     }
     Ok(current)
 }
 
-fn delete_dir(fpath: &str, allow_if_non_empty: bool) -> Result<(), super::SDSSError> {
+fn handle_item_mut<T>(
+    fpath: &str,
+    f: impl Fn(OccupiedEntry<Box<str>, VNode>) -> super::SDSSResult<T>,
+) -> super::SDSSResult<T> {
     let mut vfs = VFS.write();
     let mut current = &mut *vfs;
     // process components
@@ -241,33 +266,45 @@ fn delete_dir(fpath: &str, allow_if_non_empty: bool) -> Result<(), super::SDSSEr
             Some(VNode::Dir(dir)) => {
                 current = dir;
             }
-            Some(VNode::File(_)) => {
-                return Err(Error::new(ErrorKind::InvalidInput, "found file in path").into())
-            }
-            None => {
-                return Err(
-                    Error::new(ErrorKind::NotFound, "could not find directory in path").into(),
-                )
-            }
+            Some(VNode::File(_)) => return err_file_in_dir_path(),
+            None => return err_dir_missing_in_path(),
         }
     }
     match current.entry(target.into()) {
-        Entry::Occupied(dir) => match dir.get() {
-            VNode::Dir(d) => {
-                if allow_if_non_empty || d.is_empty() {
-                    dir.remove();
-                    return Ok(());
-                }
-                return Err(Error::new(ErrorKind::InvalidInput, "directory is not empty").into());
-            }
-            VNode::File(_) => {
-                return Err(Error::new(ErrorKind::InvalidInput, "found file in path").into())
+        Entry::Occupied(item) => return f(item),
+        Entry::Vacant(_) => return err_could_not_find_item(),
+    }
+}
+fn handle_item<T>(fpath: &str, f: impl Fn(&VNode) -> super::SDSSResult<T>) -> super::SDSSResult<T> {
+    let vfs = VFS.read();
+    let mut current = &*vfs;
+    // process components
+    let (target, components) = split_target_and_components(fpath);
+    for component in components {
+        match current.get(component) {
+            Some(VNode::Dir(dir)) => {
+                current = dir;
             }
-        },
-        Entry::Vacant(_) => {
-            return Err(Error::new(ErrorKind::NotFound, "could not find directory in path").into())
+            Some(VNode::File(_)) => return err_file_in_dir_path(),
+            None => return err_dir_missing_in_path(),
         }
     }
+    match current.get(target) {
+        Some(item) => return f(item),
+        None => return err_could_not_find_item(),
+    }
+}
+fn delete_dir(fpath: &str, allow_if_non_empty: bool) -> Result<(), super::SDSSError> {
+    handle_item_mut(fpath, |node| match node.get() {
+        VNode::Dir(d) => {
+            if allow_if_non_empty || d.is_empty() {
+                node.remove();
+                return Ok(());
+            }
+            return Err(Error::new(ErrorKind::InvalidInput, "directory is not empty").into());
+        }
+        VNode::File(_) => return err_file_in_dir_path(),
+    })
 }
 
 /*
@@ -318,9 +355,7 @@ fn with_file_mut<T>(fpath: &str, mut f: impl FnMut(&mut VFile) -> SDSSResult<T>)
     let target_dir = find_target_dir_mut(components, &mut vfs)?;
     match target_dir.get_mut(target_file) {
         Some(VNode::File(file)) => f(file),
-        Some(VNode::Dir(_)) => {
-            return Err(Error::new(ErrorKind::InvalidInput, "found directory, not a file").into())
-        }
+        Some(VNode::Dir(_)) => return err_item_is_not_file(),
         None => return Err(Error::from(ErrorKind::NotFound).into()),
     }
 }
@@ -331,9 +366,7 @@ fn with_file<T>(fpath: &str, mut f: impl FnMut(&VFile) -> SDSSResult<T>) -> SDSS
     let target_dir = find_target_dir(components, &vfs)?;
     match target_dir.get(target_file) {
         Some(VNode::File(file)) => f(file),
-        Some(VNode::Dir(_)) => {
-            return Err(Error::new(ErrorKind::InvalidInput, "found directory, not a file").into())
-        }
+        Some(VNode::Dir(_)) => return err_item_is_not_file(),
         None => return Err(Error::from(ErrorKind::NotFound).into()),
     }
 }
@@ -445,6 +478,12 @@ pub struct NullFS;
 pub struct NullFile;
 impl RawFSInterface for NullFS {
     type File = NullFile;
+    fn fs_rename_file(_: &str, _: &str) -> SDSSResult<()> {
+        Ok(())
+    }
+    fn fs_remove_file(_: &str) -> SDSSResult<()> {
+        Ok(())
+    }
     fn fs_create_dir(_: &str) -> SDSSResult<()> {
         Ok(())
     }
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index 555b8239..07045708 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -31,6 +31,7 @@ mod batch_jrnl;
 mod journal;
 mod loader;
 mod rw;
+mod sysdb;
 // hl
 pub mod inf;
 mod start_stop;
@@ -94,7 +95,7 @@ pub enum SDSSError {
     /// A corrupted file
     CorruptedFile(&'static str),
     // process errors
-    StartupError(&'static str),
+    OtherError(&'static str),
     // header
     /// The entire header is corrupted
     HeaderDecodeCorruptedHeader,
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 09406b5f..8174f4f4 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -74,6 +74,8 @@ pub enum RawFileOpen<F> {
 pub trait RawFSInterface {
     const NOT_NULL: bool = true;
     type File: RawFileInterface;
+    fn fs_remove_file(fpath: &str) -> SDSSResult<()>;
+    fn fs_rename_file(from: &str, to: &str) -> SDSSResult<()>;
     fn fs_create_dir(fpath: &str) -> SDSSResult<()>;
     fn fs_create_dir_all(fpath: &str) -> SDSSResult<()>;
     fn fs_delete_dir(fpath: &str) -> SDSSResult<()>;
@@ -141,6 +143,12 @@ pub struct LocalFS;
 
 impl RawFSInterface for LocalFS {
     type File = File;
+    fn fs_remove_file(fpath: &str) -> SDSSResult<()> {
+        cvt(fs::remove_file(fpath))
+    }
+    fn fs_rename_file(from: &str, to: &str) -> SDSSResult<()> {
+        cvt(fs::rename(from, to))
+    }
     fn fs_create_dir(fpath: &str) -> SDSSResult<()> {
         cvt(fs::create_dir(fpath))
     }
diff --git a/server/src/engine/storage/v1/start_stop.rs b/server/src/engine/storage/v1/start_stop.rs
index 54bbe3b4..11121523 100644
--- a/server/src/engine/storage/v1/start_stop.rs
+++ b/server/src/engine/storage/v1/start_stop.rs
@@ -122,7 +122,7 @@ impl StartStop {
             (ReadNX::Read(_, time_start), ReadNX::Read(_, time_stop))
                 if time_start == time_stop => {}
             (ReadNX::Created(_), ReadNX::Created(_)) => {}
-            _ => return Err(SDSSError::StartupError(EMSG_FAILED_VERIFY)),
+            _ => return Err(SDSSError::OtherError(EMSG_FAILED_VERIFY)),
         }
         start_file
             .file_mut()
diff --git a/server/src/engine/storage/v1/sysdb.rs b/server/src/engine/storage/v1/sysdb.rs
new file mode 100644
index 00000000..917f1028
--- /dev/null
+++ b/server/src/engine/storage/v1/sysdb.rs
@@ -0,0 +1,96 @@
+/*
+ * Created on Fri Sep 22 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    data::{cell::Datacell, DictEntryGeneric, DictGeneric},
+    fractal::SysConfig,
+    storage::v1::{
+        header_meta::{FileScope, FileSpecifier, FileSpecifierVersion},
+        RawFSInterface, SDSSError, SDSSFileIO, SDSSResult,
+    },
+};
+
+const SYSDB_PATH: &str = "sys.db";
+const SYSDB_COW_PATH: &str = "sys.db.cow";
+
+pub fn sync_system_database<Fs: RawFSInterface>(cfg: &SysConfig) -> SDSSResult<()> {
+    // get auth data
+    let auth_data = cfg.auth_data().read();
+    // prepare our flat file
+    let mut map: DictGeneric = into_dict!(
+        "host" => DictEntryGeneric::Map(into_dict!(
+            "settings_version" => Datacell::new_uint(cfg.host_data().settings_version() as _),
+            "startup_counter" => Datacell::new_uint(cfg.host_data().startup_counter() as _),
+        )),
+        "auth" => DictGeneric::new(),
+    );
+    let auth_key = map.get_mut("auth").unwrap();
+    match &*auth_data {
+        None => *auth_key = Datacell::null().into(),
+        Some(auth) => {
+            let auth_key = auth_key.as_dict_mut().unwrap();
+            auth_key.insert(
+                "root".into(),
+                DictEntryGeneric::Data(Datacell::new_bin(auth.root_key().into())),
+            );
+            auth_key.insert(
+                "users".into(),
+                DictEntryGeneric::Map(
+                    auth.users()
+                        .iter()
+                        .map(|(username, user)| {
+                            (
+                                username.to_owned(),
+                                DictEntryGeneric::Data(Datacell::new_list(vec![
+                                    Datacell::new_bin(user.key().into()),
+                                ])),
+                            )
+                        })
+                        .collect(),
+                ),
+            );
+        }
+    }
+    // open file
+    let mut file = SDSSFileIO::<Fs>::open_or_create_perm_rw::<false>(
+        SYSDB_COW_PATH,
+        FileScope::FlatmapData,
+        FileSpecifier::SysDB,
+        FileSpecifierVersion::__new(0),
+        cfg.host_data().settings_version(),
+        cfg.host_data().run_mode(),
+        cfg.host_data().startup_counter(),
+    )?
+    .into_created()
+    .ok_or(SDSSError::OtherError(
+        "sys.db.cow already exists. please remove this file.",
+    ))?;
+    // write
+    let buf = super::inf::enc::enc_dict_full::<super::inf::map::GenericDictSpec>(&map);
+    file.fsynced_write(&buf)?;
+    // replace
+    Fs::fs_rename_file(SYSDB_COW_PATH, SYSDB_PATH)
+}

From 190220127c6f452dba53b6b7aaabac836d0be2d4 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 22 Sep 2023 18:10:35 +0000
Subject: [PATCH 259/310] Implement new config module

---
 Cargo.lock                     |   20 +
 server/Cargo.toml              |    1 +
 server/src/engine/config.rs    | 1005 ++++++++++++++++++++++++++++++++
 server/src/engine/macros.rs    |    5 +
 server/src/engine/mod.rs       |    1 +
 server/src/engine/ql/macros.rs |    5 -
 server/src/util/os.rs          |    8 +-
 7 files changed, 1039 insertions(+), 6 deletions(-)
 create mode 100644 server/src/engine/config.rs

diff --git a/Cargo.lock b/Cargo.lock
index 42b12021..0ab36bfd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1280,6 +1280,19 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_yaml"
+version = "0.9.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574"
+dependencies = [
+ "indexmap",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
 [[package]]
 name = "sha1"
 version = "0.10.5"
@@ -1394,6 +1407,7 @@ dependencies = [
  "rcrypt",
  "regex",
  "serde",
+ "serde_yaml",
  "sky_macros",
  "skytable 0.8.0 (git+https://github.com/skytable/client-rust?branch=next)",
  "tokio",
@@ -1684,6 +1698,12 @@ version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
 
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa"
+
 [[package]]
 name = "utf8parse"
 version = "0.2.1"
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 284cb58d..c0eb8280 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -29,6 +29,7 @@ toml = "0.7.6"
 base64 = "0.21.2"
 uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics"] }
 crc = "3.0.1"
+serde_yaml = "0.9"
 
 [target.'cfg(all(not(target_env = "msvc"), not(miri)))'.dependencies]
 # external deps
diff --git a/server/src/engine/config.rs b/server/src/engine/config.rs
new file mode 100644
index 00000000..6aac6458
--- /dev/null
+++ b/server/src/engine/config.rs
@@ -0,0 +1,1005 @@
+/*
+ * Created on Fri Sep 22 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::util::os::SysIOError,
+    core::fmt,
+    serde::{
+        de::{self, Deserializer, Visitor},
+        Deserialize,
+    },
+    std::{collections::HashMap, fs},
+};
+
+/*
+    misc
+*/
+
+#[derive(Debug, PartialEq)]
+pub struct ModifyGuard<T> {
+    val: T,
+    modified: bool,
+}
+
+impl<T> ModifyGuard<T> {
+    pub const fn new(val: T) -> Self {
+        Self {
+            val,
+            modified: false,
+        }
+    }
+    pub const fn modified(me: &Self) -> bool {
+        me.modified
+    }
+    pub const fn same(me: &Self) -> bool {
+        !me.modified
+    }
+}
+
+impl<T> core::ops::Deref for ModifyGuard<T> {
+    type Target = T;
+    fn deref(&self) -> &Self::Target {
+        &self.val
+    }
+}
+
+impl<T> core::ops::DerefMut for ModifyGuard<T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.modified = true;
+        &mut self.val
+    }
+}
+
+/*
+    configuration
+*/
+
+#[derive(Debug, PartialEq)]
+/// The final configuration that can be used to start up all services
+pub struct Configuration {
+    endpoints: ConfigEndpoint,
+    mode: ConfigMode,
+    system: ConfigSystem,
+}
+
+impl Configuration {
+    const DEFAULT_HOST: &'static str = "127.0.0.1";
+    const DEFAULT_PORT_TCP: u16 = 2003;
+    const DEFAULT_RELIABILITY_SVC_PING: u64 = 5 * 60;
+    pub fn default_dev_mode() -> Self {
+        Self {
+            endpoints: ConfigEndpoint::Insecure(ConfigEndpointTcp {
+                host: Self::DEFAULT_HOST.to_owned(),
+                port: Self::DEFAULT_PORT_TCP,
+            }),
+            mode: ConfigMode::Dev,
+            system: ConfigSystem {
+                reliability_system_window: Self::DEFAULT_RELIABILITY_SVC_PING,
+                auth: false,
+            },
+        }
+    }
+}
+
+// endpoint config
+
+#[derive(Debug, PartialEq)]
+/// Endpoint configuration (TCP/TLS/TCP+TLS)
+pub enum ConfigEndpoint {
+    Insecure(ConfigEndpointTcp),
+    Secure(ConfigEndpointTls),
+    Multi(ConfigEndpointTcp, ConfigEndpointTls),
+}
+
+#[derive(Debug, PartialEq, Clone)]
+/// TCP endpoint configuration
+pub struct ConfigEndpointTcp {
+    host: String,
+    port: u16,
+}
+
+#[derive(Debug, PartialEq)]
+/// TLS endpoint configuration
+pub struct ConfigEndpointTls {
+    tcp: ConfigEndpointTcp,
+    cert: String,
+    private_key: String,
+}
+
+/*
+    config mode
+*/
+
+#[derive(Debug, PartialEq)]
+/// The configuration mode
+pub enum ConfigMode {
+    /// In [`ConfigMode::Dev`] we're allowed to be more relaxed with settings
+    Dev,
+    /// In [`ConfigMode::Prod`] we're more stringent with settings
+    Prod,
+}
+
+impl<'de> serde::Deserialize<'de> for ConfigMode {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        struct StringVisitor;
+        impl<'de> Visitor<'de> for StringVisitor {
+            type Value = ConfigMode;
+            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+                formatter.write_str("a string 'dev' or 'prod'")
+            }
+            fn visit_str<E>(self, value: &str) -> Result<ConfigMode, E>
+            where
+                E: de::Error,
+            {
+                match value {
+                    "dev" => Ok(ConfigMode::Dev),
+                    "prod" => Ok(ConfigMode::Prod),
+                    _ => Err(de::Error::custom(format!(
+                        "expected 'dev' or 'prod', got {}",
+                        value
+                    ))),
+                }
+            }
+        }
+        deserializer.deserialize_str(StringVisitor)
+    }
+}
+
+/*
+    config system
+*/
+
+#[derive(Debug, PartialEq)]
+/// System configuration settings
+pub struct ConfigSystem {
+    /// time window in seconds for the reliability system to kick-in automatically
+    reliability_system_window: u64,
+    /// if or not auth is enabled
+    auth: bool,
+}
+
+/**
+    decoded configuration
+    ---
+    the "raw" configuration that we got from the user. not validated
+*/
+#[derive(Debug, PartialEq, Deserialize)]
+pub struct DecodedConfiguration {
+    system: Option<DecodedSystemConfig>,
+    endpoints: Option<DecodedEPConfig>,
+}
+
+impl Default for DecodedConfiguration {
+    fn default() -> Self {
+        Self {
+            system: Default::default(),
+            endpoints: Default::default(),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Deserialize)]
+/// Decoded system configuration
+pub struct DecodedSystemConfig {
+    auth_enabled: Option<bool>,
+    mode: Option<ConfigMode>,
+    rs_window: Option<u64>,
+}
+
+#[derive(Debug, PartialEq, Deserialize)]
+/// Decoded endpoint configuration
+pub struct DecodedEPConfig {
+    secure: Option<DecodedEPSecureConfig>,
+    insecure: Option<DecodedEPInsecureConfig>,
+}
+
+#[derive(Debug, PartialEq, Deserialize)]
+/// Decoded secure port configuration
+pub struct DecodedEPSecureConfig {
+    host: String,
+    port: u16,
+    cert: String,
+    pass: String,
+}
+
+#[derive(Debug, PartialEq, Deserialize)]
+/// Decoded insecure port configuration
+pub struct DecodedEPInsecureConfig {
+    host: String,
+    port: u16,
+}
+
+impl DecodedEPInsecureConfig {
+    pub fn new(host: &str, port: u16) -> Self {
+        Self {
+            host: host.to_owned(),
+            port,
+        }
+    }
+}
+
+/*
+    errors and misc
+*/
+
+/// Configuration result
+pub type ConfigResult<T> = Result<T, ConfigError>;
+
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+/// A configuration error (with an optional error origin source)
+pub struct ConfigError {
+    source: Option<ConfigSource>,
+    kind: ConfigErrorKind,
+}
+
+impl ConfigError {
+    /// Init config error
+    fn _new(source: Option<ConfigSource>, kind: ConfigErrorKind) -> Self {
+        Self { kind, source }
+    }
+    /// New config error with no source
+    fn new(kind: ConfigErrorKind) -> Self {
+        Self::_new(None, kind)
+    }
+    /// New config error with the given source
+    fn with_src(source: ConfigSource, kind: ConfigErrorKind) -> Self {
+        Self::_new(Some(source), kind)
+    }
+}
+
+impl fmt::Display for ConfigError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match &self.source {
+            Some(src) => write!(f, "config error in {}: ", src.as_str())?,
+            None => write!(f, "config error: ")?,
+        }
+        match &self.kind {
+            ConfigErrorKind::Conflict => write!(
+                f,
+                "conflicting settings. please choose either CLI or ENV or configuration file"
+            ),
+            ConfigErrorKind::ErrorString(e) => write!(f, "{e}"),
+            ConfigErrorKind::IoError(e) => write!(
+                f,
+                "an I/O error occurred while reading a configuration related file: `{e}`",
+            ),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+/// The configuration source
+pub enum ConfigSource {
+    /// Command-line
+    Cli,
+    /// Environment variabels
+    Env,
+    /// Configuration file
+    File,
+}
+
+impl ConfigSource {
+    fn as_str(&self) -> &'static str {
+        match self {
+            ConfigSource::Cli => "CLI",
+            ConfigSource::Env => "ENV",
+            ConfigSource::File => "config file",
+        }
+    }
+}
+
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+/// Type of configuration error
+pub enum ConfigErrorKind {
+    /// Conflict between different setting modes (more than one of CLI/ENV/FILE was provided)
+    Conflict,
+    /// A custom error output
+    ErrorString(String),
+    /// An I/O error related to configuration
+    IoError(SysIOError),
+}
+
+direct_from! {
+    ConfigErrorKind => {
+        SysIOError as IoError,
+    }
+}
+
+impl From<std::io::Error> for ConfigError {
+    fn from(value: std::io::Error) -> Self {
+        Self::new(ConfigErrorKind::IoError(value.into()))
+    }
+}
+
+/// A configuration source implementation
+trait ConfigurationSource {
+    const KEY_TLS_CERT: &'static str;
+    const KEY_TLS_KEY: &'static str;
+    const KEY_AUTH: &'static str;
+    const KEY_ENDPOINTS: &'static str;
+    const KEY_RUN_MODE: &'static str;
+    const KEY_SERVICE_WINDOW: &'static str;
+    const SOURCE: ConfigSource;
+    /// Formats an error `Invalid value for {key}`
+    fn err_invalid_value_for(key: &str) -> ConfigError {
+        ConfigError::with_src(
+            Self::SOURCE,
+            ConfigErrorKind::ErrorString(format!("Invalid value for {key}")),
+        )
+    }
+    /// Formats an error `Too many values for {key}`
+    fn err_too_many_values_for(key: &str) -> ConfigError {
+        ConfigError::with_src(
+            Self::SOURCE,
+            ConfigErrorKind::ErrorString(format!("Too many values for {key}")),
+        )
+    }
+    /// Formats the custom error directly
+    fn custom_err(error: String) -> ConfigError {
+        ConfigError::with_src(Self::SOURCE, ConfigErrorKind::ErrorString(error))
+    }
+}
+
+/// Check if there are any duplicate values
+fn argck_duplicate_values<CS: ConfigurationSource>(
+    v: &[String],
+    key: &'static str,
+) -> ConfigResult<()> {
+    if v.len() != 1 {
+        return Err(CS::err_too_many_values_for(key));
+    }
+    Ok(())
+}
+
+/*
+    decode helpers
+*/
+
+/// Protocol to be used by a given endpoint
+enum ConnectionProtocol {
+    Tcp,
+    Tls,
+}
+
+/// Parse an endpoint (`protocol@host:port`)
+fn parse_endpoint(source: ConfigSource, s: &str) -> ConfigResult<(ConnectionProtocol, &str, u16)> {
+    let err = || {
+        Err(ConfigError::with_src(
+            source,
+            ConfigErrorKind::ErrorString(format!(
+                "invalid endpoint syntax. should be `protocol@hostname:port`"
+            )),
+        ))
+    };
+    let x = s.split("@").collect::<Vec<&str>>();
+    if x.len() != 2 {
+        return err();
+    }
+    let [protocol, hostport] = [x[0], x[1]];
+    let hostport = hostport.split(":").collect::<Vec<&str>>();
+    if hostport.len() != 2 {
+        return err();
+    }
+    let [host, port] = [hostport[0], hostport[1]];
+    let Ok(port) = port.parse::<u16>() else {
+        return err();
+    };
+    let protocol = match protocol {
+        "tcp" => ConnectionProtocol::Tcp,
+        "tls" => ConnectionProtocol::Tls,
+        _ => return err(),
+    };
+    Ok((protocol, host, port))
+}
+
+/// Decode a TLS endpoint (read in cert and private key)
+fn decode_tls_ep(
+    cert_path: &str,
+    key_path: &str,
+    host: &str,
+    port: u16,
+) -> ConfigResult<DecodedEPSecureConfig> {
+    let tls_key = fs::read_to_string(key_path)?;
+    let tls_cert = fs::read_to_string(cert_path)?;
+    Ok(DecodedEPSecureConfig {
+        host: host.into(),
+        port,
+        cert: tls_cert,
+        pass: tls_key,
+    })
+}
+
+/// Helper for decoding a TLS endpoint (we read in the cert and private key)
+fn arg_decode_tls_endpoint<CS: ConfigurationSource>(
+    args: &mut HashMap<String, Vec<String>>,
+    host: &str,
+    port: u16,
+) -> ConfigResult<DecodedEPSecureConfig> {
+    let (Some(tls_key), Some(tls_cert)) =
+        (args.remove(CS::KEY_TLS_KEY), args.remove(CS::KEY_TLS_CERT))
+    else {
+        return Err(ConfigError::with_src(
+            ConfigSource::Cli,
+            ConfigErrorKind::ErrorString(format!(
+                "must supply values for both `{}` and `{}` when using TLS",
+                CS::KEY_TLS_CERT,
+                CS::KEY_TLS_KEY
+            )),
+        ));
+    };
+    argck_duplicate_values::<CS>(&tls_cert, CS::KEY_TLS_CERT)?;
+    argck_duplicate_values::<CS>(&tls_key, CS::KEY_TLS_KEY)?;
+    Ok(decode_tls_ep(&tls_cert[0], &tls_key[0], host, port)?)
+}
+
+/*
+    decode options
+*/
+
+/// Check the auth mode. We currently only allow `pwd`
+fn arg_decode_auth<CS: ConfigurationSource>(
+    args: &[String],
+    config: &mut ModifyGuard<DecodedConfiguration>,
+) -> ConfigResult<()> {
+    argck_duplicate_values::<CS>(&args, CS::KEY_AUTH)?;
+    match args[0].as_str() {
+        "pwd" => match config.system.as_mut() {
+            Some(cfg) => cfg.auth_enabled = Some(true),
+            _ => {
+                config.system = Some(DecodedSystemConfig {
+                    auth_enabled: Some(true),
+                    mode: None,
+                    rs_window: None,
+                })
+            }
+        },
+        _ => return Err(CS::err_invalid_value_for(CS::KEY_AUTH)),
+    }
+    Ok(())
+}
+
+/// Decode the endpoints (`protocol@host:port`)
+fn arg_decode_endpoints<CS: ConfigurationSource>(
+    args: &mut HashMap<String, Vec<String>>,
+    config: &mut ModifyGuard<DecodedConfiguration>,
+) -> ConfigResult<()> {
+    let mut insecure_ep = None;
+    let mut secure_ep = None;
+    let Some(endpoints) = args.remove(CS::KEY_ENDPOINTS) else {
+        return Ok(());
+    };
+    if endpoints.len() > 2 {
+        return Err(CS::err_too_many_values_for(CS::KEY_ENDPOINTS));
+    }
+    for ep in endpoints {
+        let (proto, host, port) = parse_endpoint(CS::SOURCE, &ep)?;
+        match proto {
+            ConnectionProtocol::Tcp if insecure_ep.is_none() => {
+                insecure_ep = Some(DecodedEPInsecureConfig::new(host, port));
+            }
+            ConnectionProtocol::Tls if secure_ep.is_none() => {
+                secure_ep = Some(arg_decode_tls_endpoint::<CS>(args, host, port)?);
+            }
+            _ => {
+                return Err(CS::custom_err(format!(
+                    "duplicate endpoints specified in `{}`",
+                    CS::KEY_ENDPOINTS
+                )));
+            }
+        }
+    }
+    match config.endpoints.as_mut() {
+        Some(ep) => {
+            ep.secure = secure_ep;
+            ep.insecure = insecure_ep;
+        }
+        None => {}
+    }
+    Ok(())
+}
+
+/// Decode the run mode:
+/// - Dev OR
+/// - Prod
+fn arg_decode_mode<CS: ConfigurationSource>(
+    mode: &[String],
+    config: &mut ModifyGuard<DecodedConfiguration>,
+) -> ConfigResult<()> {
+    argck_duplicate_values::<CS>(&mode, CS::KEY_RUN_MODE)?;
+    let mode = match mode[0].as_str() {
+        "dev" => ConfigMode::Dev,
+        "prod" => ConfigMode::Prod,
+        _ => return Err(CS::err_invalid_value_for(CS::KEY_RUN_MODE)),
+    };
+    match config.system.as_mut() {
+        Some(s) => s.mode = Some(mode),
+        None => {
+            config.system = Some(DecodedSystemConfig {
+                auth_enabled: None,
+                mode: Some(mode),
+                rs_window: None,
+            })
+        }
+    }
+    Ok(())
+}
+
+/// Decode the service time window
+fn arg_decode_rs_window<CS: ConfigurationSource>(
+    mode: &[String],
+    config: &mut ModifyGuard<DecodedConfiguration>,
+) -> ConfigResult<()> {
+    argck_duplicate_values::<CS>(&mode, CS::KEY_SERVICE_WINDOW)?;
+    match mode[0].parse::<u64>() {
+        Ok(n) => match config.system.as_mut() {
+            Some(sys) => sys.rs_window = Some(n),
+            None => {
+                config.system = Some(DecodedSystemConfig {
+                    auth_enabled: None,
+                    mode: None,
+                    rs_window: Some(n),
+                })
+            }
+        },
+        Err(_) => return Err(CS::err_invalid_value_for(CS::KEY_SERVICE_WINDOW)),
+    }
+    Ok(())
+}
+
+/*
+    CLI args process
+*/
+
+/// CLI help message
+const CLI_HELP: &str ="\
+Usage: skyd [OPTION]...
+
+skyd is the Skytable database server daemon and can be used to serve database requests.
+
+Flags:
+  -h, --help                 Display this help menu and exit.
+  -v, --version              Display the version number and exit.
+
+Options:
+  --tlscert <path>           Specify the path to the TLS certificate.
+  --tlskey <path>            Define the path to the TLS private key.
+  --endpoint <definition>    Designate an endpoint. Format: protocol@host:port.
+                             This option can be repeated to define multiple endpoints.
+  --service-window <seconds> Establish the time window for the background service in seconds.
+  --auth <plugin_name>       Identify the authentication plugin by name.
+  --mode <dev/prod>          Set the operational mode. Note: This option is mandatory.
+
+Examples:
+  skyd --mode=dev --endpoint=tcp@127.0.0.1:2003
+
+Notes:
+  Ensure the 'mode' is always provided, as it is essential for the application's correct functioning.
+
+For further assistance, refer to the official documentation here: https://docs.skytable.org
+";
+
+#[derive(Debug, PartialEq)]
+/// Return from parsing CLI configuration
+pub enum CLIConfigParseReturn<T> {
+    /// No changes
+    Default,
+    /// Output help menu
+    Help,
+    /// Output version
+    Version,
+    /// We yielded a config
+    YieldedConfig(T),
+}
+
+/// Parse CLI args:
+/// - `--{option} {value}`
+/// - `--{option}={value}`
+pub fn parse_cli_args(
+    src: impl Iterator<Item = String>,
+) -> ConfigResult<CLIConfigParseReturn<HashMap<String, Vec<String>>>> {
+    let mut args_iter = src.into_iter().skip(1);
+    let mut cli_args: HashMap<String, Vec<String>> = HashMap::new();
+    while let Some(arg) = args_iter.next() {
+        if arg == "--help" {
+            return Ok(CLIConfigParseReturn::Help);
+        }
+        if arg == "--version" {
+            return Ok(CLIConfigParseReturn::Version);
+        }
+        if !arg.starts_with("--") {
+            return Err(ConfigError::with_src(
+                ConfigSource::Cli,
+                ConfigErrorKind::ErrorString(format!("unexpected argument `{arg}`")),
+            ));
+        }
+        // x=1
+        let splits_arg_and_value = arg.split("=").collect::<Vec<&str>>();
+        if (splits_arg_and_value.len() == 2) & (arg.len() >= 5) {
+            // --{n}={x}; good
+            cli_args.insert(
+                splits_arg_and_value[0].into(),
+                vec![splits_arg_and_value[1].into()],
+            );
+            continue;
+        } else {
+            if splits_arg_and_value.len() != 1 {
+                // that's an invalid argument since the split is either `x=y` or `x` and we don't have any args
+                // with special characters
+                return Err(ConfigError::with_src(
+                    ConfigSource::Cli,
+                    ConfigErrorKind::ErrorString(format!("incorrectly formatted argument `{arg}`")),
+                ));
+            }
+        }
+        let Some(value) = args_iter.next() else {
+            return Err(ConfigError::with_src(
+                ConfigSource::Cli,
+                ConfigErrorKind::ErrorString(format!("missing value for option `{arg}`")),
+            ));
+        };
+        // merge duplicates into a vec
+        match cli_args.get_mut(&arg) {
+            Some(cli) => {
+                cli.push(value);
+            }
+            None => {
+                cli_args.insert(arg, vec![value]);
+            }
+        }
+    }
+    if cli_args.is_empty() {
+        Ok(CLIConfigParseReturn::Default)
+    } else {
+        Ok(CLIConfigParseReturn::YieldedConfig(cli_args))
+    }
+}
+
+/*
+    env args process
+*/
+
+/// Parse environment variables
+pub fn parse_env_args(
+    keys: impl Iterator<Item = &'static str>,
+) -> ConfigResult<Option<HashMap<String, Vec<String>>>> {
+    let mut ret = HashMap::new();
+    for key in keys {
+        let var = match std::env::var(key) {
+            Ok(v) => v,
+            Err(e) => match e {
+                std::env::VarError::NotPresent => continue,
+                std::env::VarError::NotUnicode(_) => {
+                    return Err(ConfigError::with_src(
+                        ConfigSource::Env,
+                        ConfigErrorKind::ErrorString(format!("invalid value for `{key}`")),
+                    ))
+                }
+            },
+        };
+        let splits: Vec<_> = var.split(",").map(ToString::to_string).collect();
+        ret.insert(key.into(), splits);
+    }
+    if ret.is_empty() {
+        Ok(None)
+    } else {
+        Ok(Some(ret))
+    }
+}
+
+/*
+    apply config changes
+*/
+
+/// Apply the configuration changes to the given mutable config
+fn apply_config_changes<CS: ConfigurationSource>(
+    args: &mut HashMap<String, Vec<String>>,
+    config: &mut ModifyGuard<DecodedConfiguration>,
+) -> ConfigResult<()> {
+    enum DecodeKind {
+        Simple {
+            key: &'static str,
+            f: fn(&[String], &mut ModifyGuard<DecodedConfiguration>) -> ConfigResult<()>,
+        },
+        Complex {
+            f: fn(
+                &mut HashMap<String, Vec<String>>,
+                &mut ModifyGuard<DecodedConfiguration>,
+            ) -> ConfigResult<()>,
+        },
+    }
+    let decode_tasks = [
+        // auth
+        DecodeKind::Simple {
+            key: CS::KEY_AUTH,
+            f: arg_decode_auth::<CS>,
+        },
+        // mode
+        DecodeKind::Simple {
+            key: CS::KEY_RUN_MODE,
+            f: arg_decode_mode::<CS>,
+        },
+        // service time window
+        DecodeKind::Simple {
+            key: CS::KEY_SERVICE_WINDOW,
+            f: arg_decode_rs_window::<CS>,
+        },
+        // endpoints
+        DecodeKind::Complex {
+            f: arg_decode_endpoints::<CS>,
+        },
+    ];
+    for task in decode_tasks {
+        match task {
+            DecodeKind::Simple { key, f } => match args.get(key) {
+                Some(args) => (f)(&args, config)?,
+                None => {}
+            },
+            DecodeKind::Complex { f } => (f)(args, config)?,
+        }
+    }
+    Ok(())
+}
+
+/*
+    config source impls
+*/
+
+pub struct CSCommandLine;
+impl CSCommandLine {
+    const ARG_CONFIG_FILE: &'static str = "--config";
+}
+impl ConfigurationSource for CSCommandLine {
+    const KEY_TLS_CERT: &'static str = "--tlsert";
+    const KEY_TLS_KEY: &'static str = "--tlskey";
+    const KEY_AUTH: &'static str = "--auth";
+    const KEY_ENDPOINTS: &'static str = "--endpoint";
+    const KEY_RUN_MODE: &'static str = "--mode";
+    const KEY_SERVICE_WINDOW: &'static str = "--service-window";
+    const SOURCE: ConfigSource = ConfigSource::Cli;
+}
+
+pub struct CSEnvArgs;
+impl ConfigurationSource for CSEnvArgs {
+    const KEY_TLS_CERT: &'static str = "SKYDB_TLS_CERT";
+    const KEY_TLS_KEY: &'static str = "SKYDB_TLS_KEY";
+    const KEY_AUTH: &'static str = "SKYDB_AUTH";
+    const KEY_ENDPOINTS: &'static str = "SKYDB_ENDPOINTS";
+    const KEY_RUN_MODE: &'static str = "SKYDB_RUN_MODE";
+    const KEY_SERVICE_WINDOW: &'static str = "SKYDB_SERVICE_WINDOW";
+    const SOURCE: ConfigSource = ConfigSource::Env;
+}
+
+pub struct CSConfigFile;
+impl ConfigurationSource for CSConfigFile {
+    const KEY_TLS_CERT: &'static str = "endpoints.secure.cert";
+    const KEY_TLS_KEY: &'static str = "endpoints.secure.key";
+    const KEY_AUTH: &'static str = "system.auth";
+    const KEY_ENDPOINTS: &'static str = "endpoints";
+    const KEY_RUN_MODE: &'static str = "system.mode";
+    const KEY_SERVICE_WINDOW: &'static str = "system.service_window";
+    const SOURCE: ConfigSource = ConfigSource::File;
+}
+
+/*
+    validate configuration
+*/
+
+macro_rules! if_some {
+    ($target:expr => $then:expr) => {
+        if let Some(x) = $target {
+            $then(x);
+        }
+    };
+}
+
+macro_rules! err_if {
+    ($(if $cond:expr => $error:expr),*) => {
+        $(if $cond { return Err($error) })*
+    }
+}
+
+/// Validate the configuration, and prepare the final configuration
+fn validate_configuration<CS: ConfigurationSource>(
+    DecodedConfiguration { system, endpoints }: DecodedConfiguration,
+) -> ConfigResult<Configuration> {
+    // initialize our default configuration
+    let mut config = Configuration::default_dev_mode();
+    // mutate
+    if_some!(
+        system => |system: DecodedSystemConfig| {
+            if_some!(system.auth_enabled => |auth| config.system.auth = auth);
+            if_some!(system.mode => |mode| config.mode = mode);
+            if_some!(system.rs_window => |window| config.system.reliability_system_window = window);
+        }
+    );
+    if_some!(
+        endpoints => |ep: DecodedEPConfig| {
+            let has_insecure = ep.insecure.is_some();
+            if_some!(ep.insecure => |insecure: DecodedEPInsecureConfig| {
+                config.endpoints = ConfigEndpoint::Insecure(ConfigEndpointTcp { host: insecure.host, port: insecure.port });
+            });
+            if_some!(ep.secure => |secure: DecodedEPSecureConfig| {
+                let secure_ep = ConfigEndpointTls {
+                    tcp: ConfigEndpointTcp {
+                        host: secure.host,
+                        port: secure.port
+                    },
+                    cert: secure.cert,
+                    private_key: secure.pass
+                };
+                match &config.endpoints {
+                    ConfigEndpoint::Insecure(is) => if has_insecure {
+                        // an insecure EP was defined by the user, so set to multi
+                        config.endpoints = ConfigEndpoint::Multi(is.clone(), secure_ep)
+                    } else {
+                        // only secure EP was defined by the user
+                        config.endpoints = ConfigEndpoint::Secure(secure_ep);
+                    },
+                    _ => {}
+                }
+            })
+        }
+    );
+    // now check a few things
+    err_if!(
+        if config.system.reliability_system_window == 0 => ConfigError::with_src(
+            CS::SOURCE,
+            ConfigErrorKind::ErrorString("invalid value for service window. must be nonzero".into())
+        )
+    );
+    Ok(config)
+}
+
+/*
+    actual configuration check and exec
+*/
+
+/// The return from parsing a configuration file
+pub enum ConfigReturn {
+    /// No configuration was provided. Need to use default
+    Default,
+    /// Don't need to do anything. We've output a message and we're good to exit
+    HelpMessage,
+    /// A configuration that we have fully validated was provided
+    Config(Configuration),
+}
+
+/// Apply the changes and validate the configuration
+fn apply_and_validate<CS: ConfigurationSource>(
+    mut args: HashMap<String, Vec<String>>,
+) -> ConfigResult<ConfigReturn> {
+    let mut modcfg = ModifyGuard::new(DecodedConfiguration::default());
+    apply_config_changes::<CS>(&mut args, &mut modcfg)?;
+    if ModifyGuard::modified(&modcfg) {
+        validate_configuration::<CS>(modcfg.val).map(ConfigReturn::Config)
+    } else {
+        Ok(ConfigReturn::Default)
+    }
+}
+
+/// Check the configuration. We look through:
+/// - CLI args
+/// - ENV variables
+/// - Config file (if any)
+pub fn check_configuration() -> ConfigResult<ConfigReturn> {
+    // read in our environment variables
+    let env_args = parse_env_args(
+        [
+            CSEnvArgs::KEY_AUTH,
+            CSEnvArgs::KEY_ENDPOINTS,
+            CSEnvArgs::KEY_RUN_MODE,
+            CSEnvArgs::KEY_SERVICE_WINDOW,
+            CSEnvArgs::KEY_TLS_CERT,
+            CSEnvArgs::KEY_TLS_KEY,
+        ]
+        .into_iter(),
+    )?;
+    // read in our CLI args (since that can tell us whether we need a configuration file)
+    let read_cli_args = parse_cli_args(std::env::args().skip(1))?;
+    let cli_args = match read_cli_args {
+        CLIConfigParseReturn::Default => {
+            // no options were provided in the CLI
+            None
+        }
+        CLIConfigParseReturn::Help => {
+            // just output the help menu
+            println!("{CLI_HELP}");
+            return Ok(ConfigReturn::HelpMessage);
+        }
+        CLIConfigParseReturn::Version => {
+            // just output the version
+            println!("Skytable Database Server (skyd) v{}", libsky::VERSION);
+            return Ok(ConfigReturn::HelpMessage);
+        }
+        CLIConfigParseReturn::YieldedConfig(cfg) => Some(cfg),
+    };
+    match cli_args {
+        Some(cfg_from_cli) => {
+            // we have some CLI args
+            match cfg_from_cli.get(CSCommandLine::ARG_CONFIG_FILE) {
+                Some(cfg_file) => return check_config_file(&cfg_from_cli, &env_args, cfg_file),
+                None => {
+                    // no config file; check if there is a conflict with environment args
+                    if env_args.is_some() {
+                        // as we feared
+                        return Err(ConfigError::with_src(
+                            ConfigSource::Cli,
+                            ConfigErrorKind::Conflict,
+                        ));
+                    }
+                    return apply_and_validate::<CSCommandLine>(cfg_from_cli);
+                }
+            }
+        }
+        None => {
+            // no CLI args; but do we have anything from env?
+            match env_args {
+                Some(args) => {
+                    return apply_and_validate::<CSEnvArgs>(args);
+                }
+                None => {
+                    // no env args or cli args; we're running on default
+                    return Ok(ConfigReturn::Default);
+                }
+            }
+        }
+    }
+}
+
+/// Check the configuration file
+fn check_config_file(
+    cfg_from_cli: &HashMap<String, Vec<String>>,
+    env_args: &Option<HashMap<String, Vec<String>>>,
+    cfg_file: &Vec<String>,
+) -> ConfigResult<ConfigReturn> {
+    if cfg_from_cli.len() == 1 && env_args.is_none() {
+        // yes, we only have the config file
+        argck_duplicate_values::<CSCommandLine>(&cfg_file, CSCommandLine::ARG_CONFIG_FILE)?;
+        // read the config file
+        let file = fs::read_to_string(&cfg_file[0])?;
+        let config_from_file: DecodedConfiguration = serde_yaml::from_str(&file).map_err(|e| {
+            ConfigError::with_src(
+                ConfigSource::File,
+                ConfigErrorKind::ErrorString(format!(
+                    "failed to parse YAML config file with error: `{e}`"
+                )),
+            )
+        })?;
+        // done here
+        return validate_configuration::<CSConfigFile>(config_from_file).map(ConfigReturn::Config);
+    } else {
+        // so there are more configuration options + a config file? (and maybe even env?)
+        return Err(ConfigError::with_src(
+            ConfigSource::Cli,
+            ConfigErrorKind::Conflict,
+        ));
+    }
+}
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index 91058d0d..7f101aa2 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -24,6 +24,11 @@
  *
 */
 
+#[cfg(test)]
+macro_rules! into_array {
+    ($($e:expr),* $(,)?) => { [$($e.into()),*] };
+}
+
 macro_rules! extract {
     ($src:expr, $what:pat => $ret:expr) => {
         if let $what = $src {
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index c4646524..d7eea010 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -28,6 +28,7 @@
 
 #[macro_use]
 mod macros;
+mod config;
 mod core;
 mod data;
 mod error;
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 0e6dae05..b278ec00 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -294,11 +294,6 @@ macro_rules! dict_nullable {
     }};
 }
 
-#[cfg(test)]
-macro_rules! into_array {
-    ($($e:expr),* $(,)?) => { [$($e.into()),*] };
-}
-
 #[cfg(test)]
 macro_rules! into_array_nullable {
     ($($e:expr),* $(,)?) => { [$($crate::engine::ql::tests::nullable_datatype($e)),*] };
diff --git a/server/src/util/os.rs b/server/src/util/os.rs
index ffb5234c..478f67ba 100644
--- a/server/src/util/os.rs
+++ b/server/src/util/os.rs
@@ -35,7 +35,7 @@ use {
     crate::IoResult,
     std::{
         ffi::OsStr,
-        fs,
+        fmt, fs,
         path::Path,
         time::{SystemTime, UNIX_EPOCH},
     },
@@ -58,6 +58,12 @@ impl From<std::io::ErrorKind> for SysIOError {
     }
 }
 
+impl fmt::Display for SysIOError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
 #[cfg(test)]
 impl PartialEq for SysIOError {
     fn eq(&self, other: &Self) -> bool {

From 9102834623554ab770d6054cf759ab0f3d794c8f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 23 Sep 2023 15:03:38 +0000
Subject: [PATCH 260/310] Add config tests

---
 server/src/engine/config.rs            | 303 +++++++++++++++++--------
 server/src/engine/fractal/mgr.rs       |   6 +-
 server/src/engine/fractal/mod.rs       |  10 +-
 server/src/engine/mod.rs               |   3 +
 server/src/engine/storage/v1/loader.rs |   5 +-
 server/src/engine/tests/mod.rs         | 232 +++++++++++++++++++
 server/src/util/test_utils.rs          |  12 +
 7 files changed, 476 insertions(+), 95 deletions(-)
 create mode 100644 server/src/engine/tests/mod.rs

diff --git a/server/src/engine/config.rs b/server/src/engine/config.rs
index 6aac6458..75c0bdba 100644
--- a/server/src/engine/config.rs
+++ b/server/src/engine/config.rs
@@ -38,6 +38,8 @@ use {
     misc
 */
 
+pub type ParsedRawArgs = std::collections::HashMap<String, Vec<String>>;
+
 #[derive(Debug, PartialEq)]
 pub struct ModifyGuard<T> {
     val: T,
@@ -86,6 +88,13 @@ pub struct Configuration {
 }
 
 impl Configuration {
+    pub fn new(endpoints: ConfigEndpoint, mode: ConfigMode, system: ConfigSystem) -> Self {
+        Self {
+            endpoints,
+            mode,
+            system,
+        }
+    }
     const DEFAULT_HOST: &'static str = "127.0.0.1";
     const DEFAULT_PORT_TCP: u16 = 2003;
     const DEFAULT_RELIABILITY_SVC_PING: u64 = 5 * 60;
@@ -121,6 +130,12 @@ pub struct ConfigEndpointTcp {
     port: u16,
 }
 
+impl ConfigEndpointTcp {
+    pub fn new(host: String, port: u16) -> Self {
+        Self { host, port }
+    }
+}
+
 #[derive(Debug, PartialEq)]
 /// TLS endpoint configuration
 pub struct ConfigEndpointTls {
@@ -129,6 +144,16 @@ pub struct ConfigEndpointTls {
     private_key: String,
 }
 
+impl ConfigEndpointTls {
+    pub fn new(tcp: ConfigEndpointTcp, cert: String, private_key: String) -> Self {
+        Self {
+            tcp,
+            cert,
+            private_key,
+        }
+    }
+}
+
 /*
     config mode
 */
@@ -184,6 +209,15 @@ pub struct ConfigSystem {
     auth: bool,
 }
 
+impl ConfigSystem {
+    pub fn new(reliability_system_window: u64, auth: bool) -> Self {
+        Self {
+            reliability_system_window,
+            auth,
+        }
+    }
+}
+
 /**
     decoded configuration
     ---
@@ -340,7 +374,7 @@ impl From<std::io::Error> for ConfigError {
 }
 
 /// A configuration source implementation
-trait ConfigurationSource {
+pub(super) trait ConfigurationSource {
     const KEY_TLS_CERT: &'static str;
     const KEY_TLS_KEY: &'static str;
     const KEY_AUTH: &'static str;
@@ -439,21 +473,24 @@ fn decode_tls_ep(
 
 /// Helper for decoding a TLS endpoint (we read in the cert and private key)
 fn arg_decode_tls_endpoint<CS: ConfigurationSource>(
-    args: &mut HashMap<String, Vec<String>>,
+    args: &mut ParsedRawArgs,
     host: &str,
     port: u16,
 ) -> ConfigResult<DecodedEPSecureConfig> {
-    let (Some(tls_key), Some(tls_cert)) =
-        (args.remove(CS::KEY_TLS_KEY), args.remove(CS::KEY_TLS_CERT))
-    else {
-        return Err(ConfigError::with_src(
-            ConfigSource::Cli,
-            ConfigErrorKind::ErrorString(format!(
-                "must supply values for both `{}` and `{}` when using TLS",
-                CS::KEY_TLS_CERT,
-                CS::KEY_TLS_KEY
-            )),
-        ));
+    let _cert = args.remove(CS::KEY_TLS_CERT);
+    let _key = args.remove(CS::KEY_TLS_KEY);
+    let (tls_cert, tls_key) = match (_cert, _key) {
+        (Some(cert), Some(key)) => (cert, key),
+        _ => {
+            return Err(ConfigError::with_src(
+                ConfigSource::Cli,
+                ConfigErrorKind::ErrorString(format!(
+                    "must supply values for both `{}` and `{}` when using TLS",
+                    CS::KEY_TLS_CERT,
+                    CS::KEY_TLS_KEY
+                )),
+            ));
+        }
     };
     argck_duplicate_values::<CS>(&tls_cert, CS::KEY_TLS_CERT)?;
     argck_duplicate_values::<CS>(&tls_key, CS::KEY_TLS_KEY)?;
@@ -488,11 +525,11 @@ fn arg_decode_auth<CS: ConfigurationSource>(
 
 /// Decode the endpoints (`protocol@host:port`)
 fn arg_decode_endpoints<CS: ConfigurationSource>(
-    args: &mut HashMap<String, Vec<String>>,
+    args: &mut ParsedRawArgs,
     config: &mut ModifyGuard<DecodedConfiguration>,
 ) -> ConfigResult<()> {
-    let mut insecure_ep = None;
-    let mut secure_ep = None;
+    let mut insecure = None;
+    let mut secure = None;
     let Some(endpoints) = args.remove(CS::KEY_ENDPOINTS) else {
         return Ok(());
     };
@@ -502,11 +539,11 @@ fn arg_decode_endpoints<CS: ConfigurationSource>(
     for ep in endpoints {
         let (proto, host, port) = parse_endpoint(CS::SOURCE, &ep)?;
         match proto {
-            ConnectionProtocol::Tcp if insecure_ep.is_none() => {
-                insecure_ep = Some(DecodedEPInsecureConfig::new(host, port));
+            ConnectionProtocol::Tcp if insecure.is_none() => {
+                insecure = Some(DecodedEPInsecureConfig::new(host, port));
             }
-            ConnectionProtocol::Tls if secure_ep.is_none() => {
-                secure_ep = Some(arg_decode_tls_endpoint::<CS>(args, host, port)?);
+            ConnectionProtocol::Tls if secure.is_none() => {
+                secure = Some(arg_decode_tls_endpoint::<CS>(args, host, port)?);
             }
             _ => {
                 return Err(CS::custom_err(format!(
@@ -516,12 +553,8 @@ fn arg_decode_endpoints<CS: ConfigurationSource>(
             }
         }
     }
-    match config.endpoints.as_mut() {
-        Some(ep) => {
-            ep.secure = secure_ep;
-            ep.insecure = insecure_ep;
-        }
-        None => {}
+    if insecure.is_some() | secure.is_some() {
+        config.endpoints = Some(DecodedEPConfig { secure, insecure });
     }
     Ok(())
 }
@@ -579,7 +612,7 @@ fn arg_decode_rs_window<CS: ConfigurationSource>(
 */
 
 /// CLI help message
-const CLI_HELP: &str ="\
+pub(super) const CLI_HELP: &str ="\
 Usage: skyd [OPTION]...
 
 skyd is the Skytable database server daemon and can be used to serve database requests.
@@ -602,6 +635,7 @@ Examples:
 
 Notes:
   Ensure the 'mode' is always provided, as it is essential for the application's correct functioning.
+  When either of `--help` or `--version` is provided, all other options and flags are ignored.
 
 For further assistance, refer to the official documentation here: https://docs.skytable.org
 ";
@@ -619,19 +653,30 @@ pub enum CLIConfigParseReturn<T> {
     YieldedConfig(T),
 }
 
+impl<T> CLIConfigParseReturn<T> {
+    #[cfg(test)]
+    pub fn into_config(self) -> T {
+        match self {
+            Self::YieldedConfig(yc) => yc,
+            _ => panic!(),
+        }
+    }
+}
+
 /// Parse CLI args:
 /// - `--{option} {value}`
 /// - `--{option}={value}`
-pub fn parse_cli_args(
-    src: impl Iterator<Item = String>,
-) -> ConfigResult<CLIConfigParseReturn<HashMap<String, Vec<String>>>> {
+pub fn parse_cli_args<'a, T: 'a + AsRef<str>>(
+    src: impl Iterator<Item = T>,
+) -> ConfigResult<CLIConfigParseReturn<ParsedRawArgs>> {
     let mut args_iter = src.into_iter().skip(1);
-    let mut cli_args: HashMap<String, Vec<String>> = HashMap::new();
+    let mut cli_args: ParsedRawArgs = HashMap::new();
     while let Some(arg) = args_iter.next() {
-        if arg == "--help" {
+        let arg = arg.as_ref();
+        if arg == "--help" || arg == "-h" {
             return Ok(CLIConfigParseReturn::Help);
         }
-        if arg == "--version" {
+        if arg == "--version" || arg == "-v" {
             return Ok(CLIConfigParseReturn::Version);
         }
         if !arg.starts_with("--") {
@@ -641,37 +686,37 @@ pub fn parse_cli_args(
             ));
         }
         // x=1
+        let arg_key;
+        let arg_val;
         let splits_arg_and_value = arg.split("=").collect::<Vec<&str>>();
         if (splits_arg_and_value.len() == 2) & (arg.len() >= 5) {
             // --{n}={x}; good
-            cli_args.insert(
-                splits_arg_and_value[0].into(),
-                vec![splits_arg_and_value[1].into()],
-            );
-            continue;
+            arg_key = splits_arg_and_value[0];
+            arg_val = splits_arg_and_value[1].to_string();
+        } else if splits_arg_and_value.len() != 1 {
+            // that's an invalid argument since the split is either `x=y` or `x` and we don't have any args
+            // with special characters
+            return Err(ConfigError::with_src(
+                ConfigSource::Cli,
+                ConfigErrorKind::ErrorString(format!("incorrectly formatted argument `{arg}`")),
+            ));
         } else {
-            if splits_arg_and_value.len() != 1 {
-                // that's an invalid argument since the split is either `x=y` or `x` and we don't have any args
-                // with special characters
+            let Some(value) = args_iter.next() else {
                 return Err(ConfigError::with_src(
                     ConfigSource::Cli,
-                    ConfigErrorKind::ErrorString(format!("incorrectly formatted argument `{arg}`")),
+                    ConfigErrorKind::ErrorString(format!("missing value for option `{arg}`")),
                 ));
-            }
+            };
+            arg_key = arg;
+            arg_val = value.as_ref().to_string();
         }
-        let Some(value) = args_iter.next() else {
-            return Err(ConfigError::with_src(
-                ConfigSource::Cli,
-                ConfigErrorKind::ErrorString(format!("missing value for option `{arg}`")),
-            ));
-        };
         // merge duplicates into a vec
-        match cli_args.get_mut(&arg) {
+        match cli_args.get_mut(arg_key) {
             Some(cli) => {
-                cli.push(value);
+                cli.push(arg_val.to_string());
             }
             None => {
-                cli_args.insert(arg, vec![value]);
+                cli_args.insert(arg_key.to_string(), vec![arg_val.to_string()]);
             }
         }
     }
@@ -687,12 +732,18 @@ pub fn parse_cli_args(
 */
 
 /// Parse environment variables
-pub fn parse_env_args(
-    keys: impl Iterator<Item = &'static str>,
-) -> ConfigResult<Option<HashMap<String, Vec<String>>>> {
+pub fn parse_env_args() -> ConfigResult<Option<ParsedRawArgs>> {
+    const KEYS: [&str; 6] = [
+        CSEnvArgs::KEY_AUTH,
+        CSEnvArgs::KEY_ENDPOINTS,
+        CSEnvArgs::KEY_RUN_MODE,
+        CSEnvArgs::KEY_SERVICE_WINDOW,
+        CSEnvArgs::KEY_TLS_CERT,
+        CSEnvArgs::KEY_TLS_KEY,
+    ];
     let mut ret = HashMap::new();
-    for key in keys {
-        let var = match std::env::var(key) {
+    for key in KEYS {
+        let var = match get_var(key) {
             Ok(v) => v,
             Err(e) => match e {
                 std::env::VarError::NotPresent => continue,
@@ -720,7 +771,7 @@ pub fn parse_env_args(
 
 /// Apply the configuration changes to the given mutable config
 fn apply_config_changes<CS: ConfigurationSource>(
-    args: &mut HashMap<String, Vec<String>>,
+    args: &mut ParsedRawArgs,
     config: &mut ModifyGuard<DecodedConfiguration>,
 ) -> ConfigResult<()> {
     enum DecodeKind {
@@ -729,10 +780,7 @@ fn apply_config_changes<CS: ConfigurationSource>(
             f: fn(&[String], &mut ModifyGuard<DecodedConfiguration>) -> ConfigResult<()>,
         },
         Complex {
-            f: fn(
-                &mut HashMap<String, Vec<String>>,
-                &mut ModifyGuard<DecodedConfiguration>,
-            ) -> ConfigResult<()>,
+            f: fn(&mut ParsedRawArgs, &mut ModifyGuard<DecodedConfiguration>) -> ConfigResult<()>,
         },
     }
     let decode_tasks = [
@@ -759,13 +807,23 @@ fn apply_config_changes<CS: ConfigurationSource>(
     for task in decode_tasks {
         match task {
             DecodeKind::Simple { key, f } => match args.get(key) {
-                Some(args) => (f)(&args, config)?,
+                Some(values_for_arg) => {
+                    (f)(&values_for_arg, config)?;
+                    args.remove(key);
+                }
                 None => {}
             },
             DecodeKind::Complex { f } => (f)(args, config)?,
         }
     }
-    Ok(())
+    if args.is_empty() {
+        Ok(())
+    } else {
+        Err(ConfigError::with_src(
+            CS::SOURCE,
+            ConfigErrorKind::ErrorString("found unknown arguments".into()),
+        ))
+    }
 }
 
 /*
@@ -777,7 +835,7 @@ impl CSCommandLine {
     const ARG_CONFIG_FILE: &'static str = "--config";
 }
 impl ConfigurationSource for CSCommandLine {
-    const KEY_TLS_CERT: &'static str = "--tlsert";
+    const KEY_TLS_CERT: &'static str = "--tlscert";
     const KEY_TLS_KEY: &'static str = "--tlskey";
     const KEY_AUTH: &'static str = "--auth";
     const KEY_ENDPOINTS: &'static str = "--endpoint";
@@ -863,7 +921,7 @@ fn validate_configuration<CS: ConfigurationSource>(
                         // only secure EP was defined by the user
                         config.endpoints = ConfigEndpoint::Secure(secure_ep);
                     },
-                    _ => {}
+                    _ => unreachable!()
                 }
             })
         }
@@ -883,18 +941,29 @@ fn validate_configuration<CS: ConfigurationSource>(
 */
 
 /// The return from parsing a configuration file
+#[derive(Debug, PartialEq)]
 pub enum ConfigReturn {
     /// No configuration was provided. Need to use default
     Default,
     /// Don't need to do anything. We've output a message and we're good to exit
-    HelpMessage,
+    HelpMessage(String),
     /// A configuration that we have fully validated was provided
     Config(Configuration),
 }
 
+impl ConfigReturn {
+    #[cfg(test)]
+    pub fn into_config(self) -> Configuration {
+        match self {
+            Self::Config(c) => c,
+            _ => panic!(),
+        }
+    }
+}
+
 /// Apply the changes and validate the configuration
-fn apply_and_validate<CS: ConfigurationSource>(
-    mut args: HashMap<String, Vec<String>>,
+pub(super) fn apply_and_validate<CS: ConfigurationSource>(
+    mut args: ParsedRawArgs,
 ) -> ConfigResult<ConfigReturn> {
     let mut modcfg = ModifyGuard::new(DecodedConfiguration::default());
     apply_config_changes::<CS>(&mut args, &mut modcfg)?;
@@ -905,39 +974,93 @@ fn apply_and_validate<CS: ConfigurationSource>(
     }
 }
 
+/*
+    just some test hacks
+*/
+
+#[cfg(test)]
+thread_local! {
+    static CLI_SRC: std::cell::RefCell<Option<Vec<String>>> = std::cell::RefCell::new(None);
+    static ENV_SRC: std::cell::RefCell<Option<HashMap<String, String>>> = std::cell::RefCell::new(None);
+}
+#[cfg(test)]
+pub(super) fn set_cli_src(cli: Vec<String>) {
+    CLI_SRC.with(|args| *args.borrow_mut() = Some(cli))
+}
+#[cfg(test)]
+pub(super) fn set_env_src(variables: Vec<String>) {
+    ENV_SRC.with(|env| {
+        let variables = variables
+            .into_iter()
+            .map(|var| {
+                var.split("=")
+                    .map(ToString::to_string)
+                    .collect::<Vec<String>>()
+            })
+            .map(|mut vars| (vars.remove(0), vars.remove(0)))
+            .collect();
+        *env.borrow_mut() = Some(variables);
+    })
+}
+fn get_var(name: &str) -> Result<String, std::env::VarError> {
+    let var;
+    #[cfg(test)]
+    {
+        var = ENV_SRC.with(|venv| {
+            let ret = {
+                match venv.borrow_mut().as_mut() {
+                    None => return Err(std::env::VarError::NotPresent),
+                    Some(env_store) => match env_store.remove(name) {
+                        Some(var) => Ok(var),
+                        None => Err(std::env::VarError::NotPresent),
+                    },
+                }
+            };
+            ret
+        });
+    }
+    #[cfg(not(test))]
+    {
+        var = std::env::var(name);
+    }
+    var
+}
+fn get_cli_src() -> Vec<String> {
+    let src;
+    #[cfg(test)]
+    {
+        src = CLI_SRC
+            .with(|args| args.borrow_mut().take())
+            .unwrap_or(vec![]);
+    }
+    #[cfg(not(test))]
+    {
+        src = std::env::args().collect();
+    }
+    src
+}
+
 /// Check the configuration. We look through:
 /// - CLI args
 /// - ENV variables
 /// - Config file (if any)
 pub fn check_configuration() -> ConfigResult<ConfigReturn> {
     // read in our environment variables
-    let env_args = parse_env_args(
-        [
-            CSEnvArgs::KEY_AUTH,
-            CSEnvArgs::KEY_ENDPOINTS,
-            CSEnvArgs::KEY_RUN_MODE,
-            CSEnvArgs::KEY_SERVICE_WINDOW,
-            CSEnvArgs::KEY_TLS_CERT,
-            CSEnvArgs::KEY_TLS_KEY,
-        ]
-        .into_iter(),
-    )?;
+    let env_args = parse_env_args()?;
     // read in our CLI args (since that can tell us whether we need a configuration file)
-    let read_cli_args = parse_cli_args(std::env::args().skip(1))?;
+    let read_cli_args = parse_cli_args(get_cli_src().into_iter())?;
     let cli_args = match read_cli_args {
         CLIConfigParseReturn::Default => {
             // no options were provided in the CLI
             None
         }
-        CLIConfigParseReturn::Help => {
-            // just output the help menu
-            println!("{CLI_HELP}");
-            return Ok(ConfigReturn::HelpMessage);
-        }
+        CLIConfigParseReturn::Help => return Ok(ConfigReturn::HelpMessage(CLI_HELP.into())),
         CLIConfigParseReturn::Version => {
             // just output the version
-            println!("Skytable Database Server (skyd) v{}", libsky::VERSION);
-            return Ok(ConfigReturn::HelpMessage);
+            return Ok(ConfigReturn::HelpMessage(format!(
+                "Skytable Database Server (skyd) v{}",
+                libsky::VERSION
+            )));
         }
         CLIConfigParseReturn::YieldedConfig(cfg) => Some(cfg),
     };
@@ -976,8 +1099,8 @@ pub fn check_configuration() -> ConfigResult<ConfigReturn> {
 
 /// Check the configuration file
 fn check_config_file(
-    cfg_from_cli: &HashMap<String, Vec<String>>,
-    env_args: &Option<HashMap<String, Vec<String>>>,
+    cfg_from_cli: &ParsedRawArgs,
+    env_args: &Option<ParsedRawArgs>,
     cfg_file: &Vec<String>,
 ) -> ConfigResult<ConfigReturn> {
     if cfg_from_cli.len() == 1 && env_args.is_none() {
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 32614e5e..5e7861df 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -147,11 +147,13 @@ impl FractalMgr {
         hp_receiver: UnboundedReceiver<Task<CriticalTask>>,
     ) -> FractalServiceHandles {
         let fractal_mgr = global.get_state().fractal_mgr();
+        let global_1 = global.__global_clone();
+        let global_2 = global.__global_clone();
         let hp_handle = tokio::spawn(async move {
-            FractalMgr::hp_executor_svc(fractal_mgr, global, hp_receiver).await
+            FractalMgr::hp_executor_svc(fractal_mgr, global_1, hp_receiver).await
         });
         let lp_handle = tokio::spawn(async move {
-            FractalMgr::general_executor_svc(fractal_mgr, global, lp_receiver).await
+            FractalMgr::general_executor_svc(fractal_mgr, global_2, lp_receiver).await
         });
         FractalServiceHandles {
             hp_handle,
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index e5978542..e8cb8536 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -90,7 +90,7 @@ pub unsafe fn enable_and_start_all(
     GLOBAL = MaybeUninit::new(global_state);
     let token = Global::new();
     GlobalStateStart {
-        global: token,
+        global: token.__global_clone(),
         mgr_handles: mgr::FractalMgr::start_all(token, lp_recv, hp_recv),
     }
 }
@@ -154,7 +154,7 @@ impl GlobalInstanceLike for Global {
     }
 }
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug)]
 /// A handle to the global state
 pub struct Global(());
 
@@ -162,6 +162,12 @@ impl Global {
     unsafe fn new() -> Self {
         Self(())
     }
+    fn __global_clone(&self) -> Self {
+        unsafe {
+            // UNSAFE(@ohsayan): safe to call within this module
+            Self::new()
+        }
+    }
     fn get_state(&self) -> &'static GlobalState {
         unsafe { GLOBAL.assume_init_ref() }
     }
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index d7eea010..db600b35 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -40,3 +40,6 @@ mod ql;
 mod storage;
 mod sync;
 mod txn;
+// test
+#[cfg(test)]
+mod tests;
diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs
index 2c3876e5..60a01418 100644
--- a/server/src/engine/storage/v1/loader.rs
+++ b/server/src/engine/storage/v1/loader.rs
@@ -37,16 +37,19 @@ const GNS_FILE_PATH: &str = "gns.db-tlog";
 pub struct SEInitState {
     pub txn_driver: GNSTransactionDriverAnyFS<super::LocalFS>,
     pub model_drivers: ModelDrivers,
+    pub gns: GlobalNS,
 }
 
 impl SEInitState {
     pub fn new(
         txn_driver: GNSTransactionDriverAnyFS<super::LocalFS>,
         model_drivers: ModelDrivers,
+        gns: GlobalNS,
     ) -> Self {
         Self {
             txn_driver,
             model_drivers,
+            gns,
         }
     }
     pub fn try_init(
@@ -87,7 +90,7 @@ impl SEInitState {
                 );
             }
         }
-        Ok(SEInitState::new(gns_txn_driver, model_drivers))
+        Ok(SEInitState::new(gns_txn_driver, model_drivers, gns))
     }
     fn model_path(
         space_name: &str,
diff --git a/server/src/engine/tests/mod.rs b/server/src/engine/tests/mod.rs
new file mode 100644
index 00000000..96c89d9b
--- /dev/null
+++ b/server/src/engine/tests/mod.rs
@@ -0,0 +1,232 @@
+/*
+ * Created on Sat Sep 23 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod cfg {
+    use crate::{
+        engine::config::{
+            self, CLIConfigParseReturn, ConfigEndpoint, ConfigEndpointTcp, ConfigEndpointTls,
+            ConfigMode, ConfigReturn, ConfigSystem, Configuration, ParsedRawArgs,
+        },
+        util::test_utils::with_files,
+    };
+
+    /*
+        CLI tests
+    */
+
+    fn extract_cli_args(payload: &str) -> std::collections::HashMap<String, Vec<String>> {
+        extract_cli_args_raw(payload).into_config()
+    }
+    fn extract_cli_args_raw(
+        payload: &str,
+    ) -> CLIConfigParseReturn<std::collections::HashMap<String, Vec<String>>> {
+        config::parse_cli_args(payload.split_ascii_whitespace().map_while(|item| {
+            let mut item = item.trim();
+            if item.ends_with("\n") {
+                item = &item[..item.len() - 1];
+            }
+            if item.is_empty() {
+                None
+            } else {
+                Some(item)
+            }
+        }))
+        .unwrap()
+    }
+    #[test]
+    fn parse_cli_args_simple() {
+        let payload = "skyd --mode dev --endpoint tcp@localhost:2003";
+        let cfg = extract_cli_args(payload);
+        let expected: ParsedRawArgs = into_dict! {
+            "--mode" => vec!["dev".into()],
+            "--endpoint" => vec!["tcp@localhost:2003".into()]
+        };
+        assert_eq!(cfg, expected);
+    }
+    #[test]
+    fn parse_cli_args_packed() {
+        let payload = "skyd --mode=dev --endpoint=tcp@localhost:2003";
+        let cfg = extract_cli_args(payload);
+        let expected: ParsedRawArgs = into_dict! {
+            "--mode" => vec!["dev".into()],
+            "--endpoint" => vec!["tcp@localhost:2003".into()]
+        };
+        assert_eq!(cfg, expected);
+    }
+    #[test]
+    fn parse_cli_args_multi() {
+        let payload = "skyd --mode=dev --endpoint tcp@localhost:2003";
+        let cfg = extract_cli_args(payload);
+        let expected: ParsedRawArgs = into_dict! {
+            "--mode" => vec!["dev".into()],
+            "--endpoint" => vec!["tcp@localhost:2003".into()]
+        };
+        assert_eq!(cfg, expected);
+    }
+    #[test]
+    fn parse_validate_cli_args() {
+        with_files(
+            ["__cli_args_test_private.key", "__cli_args_test_cert.pem"],
+            |[pkey, cert]| {
+                let payload = format!(
+                    "skyd --mode=dev \
+                --endpoint tcp@127.0.0.1:2003 \
+                --endpoint tls@127.0.0.2:2004 \
+                --service-window=600 \
+                --tlskey {pkey} \
+                --tlscert {cert} \
+                --auth pwd"
+                );
+                let cfg = extract_cli_args(&payload);
+                let ret = config::apply_and_validate::<config::CSCommandLine>(cfg)
+                    .unwrap()
+                    .into_config();
+                assert_eq!(
+                    ret,
+                    Configuration::new(
+                        ConfigEndpoint::Multi(
+                            ConfigEndpointTcp::new("127.0.0.1".into(), 2003),
+                            ConfigEndpointTls::new(
+                                ConfigEndpointTcp::new("127.0.0.2".into(), 2004),
+                                "".into(),
+                                "".into()
+                            )
+                        ),
+                        ConfigMode::Dev,
+                        ConfigSystem::new(600, true)
+                    )
+                )
+            },
+        );
+    }
+    #[test]
+    fn parse_validate_cli_args_help_and_version() {
+        let pl1 = "skyd --help";
+        let pl2 = "skyd --version";
+        let ret1 = extract_cli_args_raw(pl1);
+        let ret2 = extract_cli_args_raw(pl2);
+        assert_eq!(ret1, CLIConfigParseReturn::Help);
+        assert_eq!(ret2, CLIConfigParseReturn::Version);
+        config::set_cli_src(vec!["skyd".into(), "--help".into()]);
+        let ret3 = config::check_configuration().unwrap();
+        config::set_cli_src(vec!["skyd".into(), "--version".into()]);
+        let ret4 = config::check_configuration().unwrap();
+        assert_eq!(
+            ret3,
+            ConfigReturn::HelpMessage(config::CLI_HELP.to_string())
+        );
+        assert_eq!(
+            ret4,
+            ConfigReturn::HelpMessage(format!(
+                "Skytable Database Server (skyd) v{}",
+                libsky::VERSION
+            ))
+        );
+    }
+
+    /*
+        env tests
+    */
+
+    fn vars_to_args(variables: &[String]) -> ParsedRawArgs {
+        variables
+            .iter()
+            .map(|var| {
+                var.split("=")
+                    .map(ToString::to_string)
+                    .collect::<Vec<String>>()
+            })
+            .map(|mut v| {
+                let key = v.remove(0);
+                let values = v.remove(0).split(",").map(ToString::to_string).collect();
+                (key, values)
+            })
+            .collect()
+    }
+    #[test]
+    fn parse_env_args_simple() {
+        let variables = [
+            format!("SKYDB_TLS_CERT=/var/skytable/keys/cert.pem"),
+            format!("SKYDB_TLS_KEY=/var/skytable/keys/private.key"),
+            format!("SKYDB_AUTH=pwd"),
+            format!("SKYDB_ENDPOINTS=tcp@localhost:8080"),
+            format!("SKYDB_RUN_MODE=dev"),
+            format!("SKYDB_SERVICE_WINDOW=600"),
+        ];
+        let expected_args = vars_to_args(&variables);
+        config::set_env_src(variables.into());
+        let args = config::parse_env_args().unwrap().unwrap();
+        assert_eq!(args, expected_args);
+    }
+    #[test]
+    fn parse_env_args_multi() {
+        let variables = [
+            format!("SKYDB_TLS_CERT=/var/skytable/keys/cert.pem"),
+            format!("SKYDB_TLS_KEY=/var/skytable/keys/private.key"),
+            format!("SKYDB_AUTH=pwd"),
+            format!("SKYDB_ENDPOINTS=tcp@localhost:8080,tls@localhost:8081"),
+            format!("SKYDB_RUN_MODE=dev"),
+            format!("SKYDB_SERVICE_WINDOW=600"),
+        ];
+        let expected_args = vars_to_args(&variables);
+        config::set_env_src(variables.into());
+        let args = config::parse_env_args().unwrap().unwrap();
+        assert_eq!(args, expected_args);
+    }
+    #[test]
+    fn parse_validate_env_args() {
+        with_files(
+            ["__env_args_test_cert.pem", "__env_args_test_private.key"],
+            |[cert, key]| {
+                let variables = [
+                    format!("SKYDB_TLS_CERT={cert}"),
+                    format!("SKYDB_TLS_KEY={key}"),
+                    format!("SKYDB_AUTH=pwd"),
+                    format!("SKYDB_ENDPOINTS=tcp@localhost:8080,tls@localhost:8081"),
+                    format!("SKYDB_RUN_MODE=dev"),
+                    format!("SKYDB_SERVICE_WINDOW=600"),
+                ];
+                config::set_env_src(variables.into());
+                let cfg = config::check_configuration().unwrap().into_config();
+                assert_eq!(
+                    cfg,
+                    Configuration::new(
+                        ConfigEndpoint::Multi(
+                            ConfigEndpointTcp::new("localhost".into(), 8080),
+                            ConfigEndpointTls::new(
+                                ConfigEndpointTcp::new("localhost".into(), 8081),
+                                "".into(),
+                                "".into()
+                            )
+                        ),
+                        ConfigMode::Dev,
+                        ConfigSystem::new(600, true)
+                    )
+                )
+            },
+        );
+    }
+}
diff --git a/server/src/util/test_utils.rs b/server/src/util/test_utils.rs
index 59beb02f..962431fd 100644
--- a/server/src/util/test_utils.rs
+++ b/server/src/util/test_utils.rs
@@ -44,6 +44,18 @@ pub fn shuffle_slice<T>(slice: &mut [T], rng: &mut impl Rng) {
     slice.shuffle(rng)
 }
 
+pub fn with_files<const N: usize, T>(files: [&str; N], f: impl Fn([&str; N]) -> T) -> T {
+    use std::fs;
+    for file in files {
+        let _ = fs::File::create(file);
+    }
+    let r = f(files);
+    for file in files {
+        let _ = fs::remove_file(file);
+    }
+    r
+}
+
 pub fn wait_for_key(msg: &str) {
     use std::io::{self, Write};
     print!("{msg}");

From fac2802849fd1cff6a35e6d39ecf15bf4dbd40b3 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 23 Sep 2023 17:47:05 +0000
Subject: [PATCH 261/310] Wire up queries to SE

---
 server/src/engine/core/model/mod.rs     | 34 ++++++++++++++++++++++---
 server/src/engine/core/space.rs         | 32 ++++++++++++++++++-----
 server/src/engine/fractal/mgr.rs        | 27 +++++++++++++++++++-
 server/src/engine/fractal/mod.rs        | 20 ++++++++++++++-
 server/src/engine/fractal/test_utils.rs |  9 +++++++
 server/src/engine/storage/v1/loader.rs  | 18 +++++++++++--
 server/src/engine/storage/v1/mod.rs     |  2 +-
 7 files changed, 128 insertions(+), 14 deletions(-)

diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 405e0641..2c689e46 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -40,7 +40,7 @@ use {
             uuid::Uuid,
         },
         error::{Error, QueryResult},
-        fractal::GlobalInstanceLike,
+        fractal::{GenericTask, GlobalInstanceLike, Task},
         idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
         mem::VInline,
         ql::ddl::{
@@ -216,17 +216,38 @@ impl Model {
                 return Err(Error::QPDdlObjectAlreadyExists);
             }
             if G::FS_IS_NON_NULL {
-                // prepare txn
                 let irm = model.intent_read_model();
                 let mut txn_driver = global.namespace_txn_driver().lock();
+                // prepare txn
                 let txn = gnstxn::CreateModelTxn::new(
                     gnstxn::SpaceIDRef::new(space_name, space),
                     model_name,
                     &model,
                     &irm,
                 );
+                // attempt to initialize driver
+                global.initialize_model_driver(
+                    space_name,
+                    space.get_uuid(),
+                    model_name,
+                    model.get_uuid(),
+                )?;
                 // commit txn
-                txn_driver.try_commit(txn)?;
+                match txn_driver.try_commit(txn) {
+                    Ok(()) => {}
+                    Err(e) => {
+                        // failed to commit, delete this
+                        global.taskmgr_post_standard_priority(Task::new(
+                            GenericTask::delete_model_dir(
+                                space_name,
+                                space.get_uuid(),
+                                model_name,
+                                model.get_uuid(),
+                            ),
+                        ));
+                        return Err(e.into());
+                    }
+                }
             }
             // update global state
             let _ = w_space.st_insert(model_name.into(), model);
@@ -253,6 +274,13 @@ impl Model {
                 ));
                 // commit txn
                 global.namespace_txn_driver().lock().try_commit(txn)?;
+                // ask for cleanup
+                global.taskmgr_post_standard_priority(Task::new(GenericTask::delete_model_dir(
+                    space_name,
+                    space.get_uuid(),
+                    model_name,
+                    model.get_uuid(),
+                )));
             }
             // update global state
             let _ = w_space.st_delete(model_name);
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index ec07bf43..6d4b8496 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -29,9 +29,10 @@ use {
         core::{model::Model, RWLIdx},
         data::{dict, uuid::Uuid, DictEntryGeneric, DictGeneric},
         error::{Error, QueryResult},
-        fractal::GlobalInstanceLike,
+        fractal::{GenericTask, GlobalInstanceLike, Task},
         idx::{IndexST, STIndex},
         ql::ddl::{alt::AlterSpace, crt::CreateSpace, drop::DropSpace},
+        storage::v1::{loader::SEInitState, RawFSInterface},
         txn::gns as gnstxn,
     },
     parking_lot::RwLock,
@@ -203,12 +204,25 @@ impl Space {
         }
         // commit txn
         if G::FS_IS_NON_NULL {
-            // prepare and commit txn
+            // prepare txn
             let s_read = space.metadata().dict().read();
-            global
-                .namespace_txn_driver()
-                .lock()
-                .try_commit(gnstxn::CreateSpaceTxn::new(&s_read, &space_name, &space))?;
+            let txn = gnstxn::CreateSpaceTxn::new(&s_read, &space_name, &space);
+            // try to create space for...the space
+            G::FileSystem::fs_create_dir_all(&SEInitState::space_dir(
+                &space_name,
+                space.get_uuid(),
+            ))?;
+            // commit txn
+            match global.namespace_txn_driver().lock().try_commit(txn) {
+                Ok(()) => {}
+                Err(e) => {
+                    // tell fractal to clean it up sometime
+                    global.taskmgr_post_standard_priority(Task::new(
+                        GenericTask::delete_space_dir(&space_name, space.get_uuid()),
+                    ));
+                    return Err(e.into());
+                }
+            }
         }
         // update global state
         let _ = wl.st_insert(space_name, space);
@@ -272,7 +286,13 @@ impl Space {
         if G::FS_IS_NON_NULL {
             // prepare txn
             let txn = gnstxn::DropSpaceTxn::new(gnstxn::SpaceIDRef::new(&space_name, space));
+            // commit txn
             global.namespace_txn_driver().lock().try_commit(txn)?;
+            // ask for cleanup
+            global.taskmgr_post_standard_priority(Task::new(GenericTask::delete_space_dir(
+                &space_name,
+                space.get_uuid(),
+            )));
         }
         drop(space_w);
         let _ = wgns.st_delete(space_name.as_str());
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 5e7861df..38cd532f 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -27,7 +27,10 @@
 use {
     super::ModelUniqueID,
     crate::{
-        engine::core::model::{delta::DataDelta, Model},
+        engine::{
+            core::model::{delta::DataDelta, Model},
+            data::uuid::Uuid,
+        },
         util::os,
     },
     std::path::PathBuf,
@@ -64,6 +67,28 @@ pub enum GenericTask {
     DeleteDirAll(PathBuf),
 }
 
+impl GenericTask {
+    pub fn delete_model_dir(
+        space_name: &str,
+        space_uuid: Uuid,
+        model_name: &str,
+        model_uuid: Uuid,
+    ) -> Self {
+        Self::DeleteDirAll(
+            crate::engine::storage::v1::loader::SEInitState::model_dir(
+                space_name, space_uuid, model_name, model_uuid,
+            )
+            .into(),
+        )
+    }
+    pub fn delete_space_dir(space_name: &str, space_uuid: Uuid) -> Self {
+        Self::DeleteDirAll(
+            crate::engine::storage::v1::loader::SEInitState::space_dir(space_name, space_uuid)
+                .into(),
+        )
+    }
+}
+
 /// A critical task
 pub enum CriticalTask {
     /// Write a new data batch
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index e8cb8536..090f88de 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -28,7 +28,7 @@ use {
     super::{
         core::GlobalNS,
         data::uuid::Uuid,
-        storage::v1::{LocalFS, RawFSInterface},
+        storage::v1::{LocalFS, RawFSInterface, SDSSResult},
         txn::gns::GNSTransactionDriverAnyFS,
     },
     parking_lot::{Mutex, RwLock},
@@ -108,6 +108,14 @@ pub trait GlobalInstanceLike {
     // global namespace
     fn namespace(&self) -> &GlobalNS;
     fn namespace_txn_driver(&self) -> &Mutex<GNSTransactionDriverAnyFS<Self::FileSystem>>;
+    // model drivers
+    fn initialize_model_driver(
+        &self,
+        space_name: &str,
+        space_uuid: Uuid,
+        model_name: &str,
+        model_uuid: Uuid,
+    ) -> SDSSResult<()>;
     // taskmgr
     fn taskmgr_post_high_priority(&self, task: Task<CriticalTask>);
     fn taskmgr_post_standard_priority(&self, task: Task<GenericTask>);
@@ -152,6 +160,16 @@ impl GlobalInstanceLike for Global {
     fn sys_cfg(&self) -> &config::SysConfig {
         &self.get_state().config
     }
+    // model
+    fn initialize_model_driver(
+        &self,
+        _space_name: &str,
+        _space_uuid: Uuid,
+        _model_name: &str,
+        _model_uuid: Uuid,
+    ) -> SDSSResult<()> {
+        todo!()
+    }
 }
 
 #[derive(Debug)]
diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
index 03321645..75043e23 100644
--- a/server/src/engine/fractal/test_utils.rs
+++ b/server/src/engine/fractal/test_utils.rs
@@ -115,6 +115,15 @@ impl<Fs: RawFSInterface> GlobalInstanceLike for TestGlobal<Fs> {
     fn sys_cfg(&self) -> &super::config::SysConfig {
         &self.sys_cfg
     }
+    fn initialize_model_driver(
+        &self,
+        _space_name: &str,
+        _space_uuid: crate::engine::data::uuid::Uuid,
+        _model_name: &str,
+        _model_uuid: crate::engine::data::uuid::Uuid,
+    ) -> crate::engine::storage::v1::SDSSResult<()> {
+        todo!()
+    }
 }
 
 impl<Fs: RawFSInterface> Drop for TestGlobal<Fs> {
diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs
index 60a01418..0a21b218 100644
--- a/server/src/engine/storage/v1/loader.rs
+++ b/server/src/engine/storage/v1/loader.rs
@@ -92,12 +92,26 @@ impl SEInitState {
         }
         Ok(SEInitState::new(gns_txn_driver, model_drivers, gns))
     }
-    fn model_path(
+    pub fn model_path(
         space_name: &str,
         space_uuid: Uuid,
         model_name: &str,
         model_uuid: Uuid,
     ) -> String {
-        format!("data/{space_name}-{space_uuid}/{model_name}-{model_uuid}/data.db-btlog")
+        format!(
+            "{}/data.db-btlog",
+            Self::model_dir(space_name, space_uuid, model_name, model_uuid)
+        )
+    }
+    pub fn model_dir(
+        space_name: &str,
+        space_uuid: Uuid,
+        model_name: &str,
+        model_uuid: Uuid,
+    ) -> String {
+        format!("data/{space_name}-{space_uuid}/mdl_{model_name}-{model_uuid}")
+    }
+    pub fn space_dir(space_name: &str, space_uuid: Uuid) -> String {
+        format!("data/{space_name}-{space_uuid}")
     }
 }
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index 07045708..c84f3f21 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -29,7 +29,7 @@ mod header_impl;
 // impls
 mod batch_jrnl;
 mod journal;
-mod loader;
+pub(in crate::engine) mod loader;
 mod rw;
 mod sysdb;
 // hl

From 6df09c194c78e31e2fce151f69c062346dceaac1 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 24 Sep 2023 05:33:13 +0000
Subject: [PATCH 262/310] Initialize storage drivers on init

Also fixed an issue with NullFS where the null
flag was not set.
---
 server/src/engine/fractal/drivers.rs          |  22 +--
 server/src/engine/fractal/mgr.rs              |   4 +-
 server/src/engine/fractal/mod.rs              |  46 ++++--
 server/src/engine/fractal/test_utils.rs       |  35 +++-
 server/src/engine/idx/stord/mod.rs            |  13 +-
 .../src/engine/storage/v1/batch_jrnl/mod.rs   |  42 ++---
 server/src/engine/storage/v1/loader.rs        |  12 +-
 server/src/engine/storage/v1/memfs.rs         |  47 +++++-
 server/src/engine/storage/v1/mod.rs           |   2 +-
 server/src/engine/storage/v1/rw.rs            | 155 +++++++++++++++---
 10 files changed, 289 insertions(+), 89 deletions(-)

diff --git a/server/src/engine/fractal/drivers.rs b/server/src/engine/fractal/drivers.rs
index c003c8c2..87ac096a 100644
--- a/server/src/engine/fractal/drivers.rs
+++ b/server/src/engine/fractal/drivers.rs
@@ -27,7 +27,7 @@
 use {
     super::util,
     crate::engine::{
-        storage::v1::{data_batch::DataBatchPersistDriver, LocalFS},
+        storage::v1::{data_batch::DataBatchPersistDriver, RawFSInterface},
         txn::gns::GNSTransactionDriverAnyFS,
     },
     parking_lot::Mutex,
@@ -35,39 +35,39 @@ use {
 };
 
 /// GNS driver
-pub(super) struct FractalGNSDriver {
+pub(super) struct FractalGNSDriver<Fs: RawFSInterface> {
     status: util::Status,
-    txn_driver: Mutex<GNSTransactionDriverAnyFS<LocalFS>>,
+    txn_driver: Mutex<GNSTransactionDriverAnyFS<Fs>>,
 }
 
-impl FractalGNSDriver {
-    pub(super) fn new(txn_driver: GNSTransactionDriverAnyFS<LocalFS>) -> Self {
+impl<Fs: RawFSInterface> FractalGNSDriver<Fs> {
+    pub(super) fn new(txn_driver: GNSTransactionDriverAnyFS<Fs>) -> Self {
         Self {
             status: util::Status::new_okay(),
             txn_driver: Mutex::new(txn_driver),
         }
     }
-    pub fn txn_driver(&self) -> &Mutex<GNSTransactionDriverAnyFS<LocalFS>> {
+    pub fn txn_driver(&self) -> &Mutex<GNSTransactionDriverAnyFS<Fs>> {
         &self.txn_driver
     }
 }
 
 /// Model driver
-pub struct FractalModelDriver {
+pub struct FractalModelDriver<Fs: RawFSInterface> {
     hooks: Arc<FractalModelHooks>,
-    batch_driver: Mutex<DataBatchPersistDriver<LocalFS>>,
+    batch_driver: Mutex<DataBatchPersistDriver<Fs>>,
 }
 
-impl FractalModelDriver {
+impl<Fs: RawFSInterface> FractalModelDriver<Fs> {
     /// Initialize a model driver with default settings
-    pub fn init(batch_driver: DataBatchPersistDriver<LocalFS>) -> Self {
+    pub fn init(batch_driver: DataBatchPersistDriver<Fs>) -> Self {
         Self {
             hooks: Arc::new(FractalModelHooks::new()),
             batch_driver: Mutex::new(batch_driver),
         }
     }
     /// Returns a reference to the batch persist driver
-    pub fn batch_driver(&self) -> &Mutex<DataBatchPersistDriver<LocalFS>> {
+    pub fn batch_driver(&self) -> &Mutex<DataBatchPersistDriver<Fs>> {
         &self.batch_driver
     }
 }
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 38cd532f..72f46d29 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use crate::engine::storage::v1::LocalFS;
+
 use {
     super::ModelUniqueID,
     crate::{
@@ -316,7 +318,7 @@ impl FractalMgr {
     fn try_write_model_data_batch(
         model: &Model,
         observed_size: usize,
-        mdl_driver: &super::FractalModelDriver,
+        mdl_driver: &super::FractalModelDriver<LocalFS>,
     ) -> crate::engine::error::QueryResult<()> {
         if observed_size == 0 {
             // no changes, all good
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index 090f88de..c01e3fcb 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -28,7 +28,10 @@ use {
     super::{
         core::GlobalNS,
         data::uuid::Uuid,
-        storage::v1::{LocalFS, RawFSInterface, SDSSResult},
+        storage::{
+            self,
+            v1::{LocalFS, RawFSInterface, SDSSResult},
+        },
         txn::gns::GNSTransactionDriverAnyFS,
     },
     parking_lot::{Mutex, RwLock},
@@ -49,7 +52,7 @@ pub use {
     util::FractalToken,
 };
 
-pub type ModelDrivers = HashMap<ModelUniqueID, drivers::FractalModelDriver>;
+pub type ModelDrivers<Fs> = HashMap<ModelUniqueID, drivers::FractalModelDriver<Fs>>;
 
 static mut GLOBAL: MaybeUninit<GlobalState> = MaybeUninit::uninit();
 
@@ -73,7 +76,7 @@ pub unsafe fn enable_and_start_all(
     gns: GlobalNS,
     config: config::SysConfig,
     gns_driver: GNSTransactionDriverAnyFS<LocalFS>,
-    model_drivers: ModelDrivers,
+    model_drivers: ModelDrivers<LocalFS>,
 ) -> GlobalStateStart {
     let model_cnt_on_boot = model_drivers.len();
     let gns_driver = drivers::FractalGNSDriver::new(gns_driver);
@@ -163,12 +166,29 @@ impl GlobalInstanceLike for Global {
     // model
     fn initialize_model_driver(
         &self,
-        _space_name: &str,
-        _space_uuid: Uuid,
-        _model_name: &str,
-        _model_uuid: Uuid,
+        space_name: &str,
+        space_uuid: Uuid,
+        model_name: &str,
+        model_uuid: Uuid,
     ) -> SDSSResult<()> {
-        todo!()
+        // create dir
+        LocalFS::fs_create_dir(&storage::v1::loader::SEInitState::model_dir(
+            space_name, space_uuid, model_name, model_uuid,
+        ))?;
+        // init driver
+        let driver = storage::v1::data_batch::create(
+            &storage::v1::loader::SEInitState::model_path(
+                space_name, space_uuid, model_name, model_uuid,
+            ),
+            self.sys_cfg().host_data().settings_version(),
+            self.sys_cfg().host_data().run_mode(),
+            self.sys_cfg().host_data().startup_counter(),
+        )?;
+        self.get_state().mdl_driver.write().insert(
+            ModelUniqueID::new(space_name, model_name, model_uuid),
+            drivers::FractalModelDriver::init(driver),
+        );
+        Ok(())
     }
 }
 
@@ -221,8 +241,8 @@ impl Global {
 /// The global state
 struct GlobalState {
     gns: GlobalNS,
-    gns_driver: drivers::FractalGNSDriver,
-    mdl_driver: RwLock<ModelDrivers>,
+    gns_driver: drivers::FractalGNSDriver<LocalFS>,
+    mdl_driver: RwLock<ModelDrivers<LocalFS>>,
     task_mgr: mgr::FractalMgr,
     config: config::SysConfig,
 }
@@ -230,8 +250,8 @@ struct GlobalState {
 impl GlobalState {
     fn new(
         gns: GlobalNS,
-        gns_driver: drivers::FractalGNSDriver,
-        mdl_driver: RwLock<ModelDrivers>,
+        gns_driver: drivers::FractalGNSDriver<LocalFS>,
+        mdl_driver: RwLock<ModelDrivers<LocalFS>>,
         task_mgr: mgr::FractalMgr,
         config: config::SysConfig,
     ) -> Self {
@@ -243,7 +263,7 @@ impl GlobalState {
             config,
         }
     }
-    pub(self) fn get_mdl_drivers(&self) -> &RwLock<ModelDrivers> {
+    pub(self) fn get_mdl_drivers(&self) -> &RwLock<ModelDrivers<LocalFS>> {
         &self.mdl_driver
     }
     pub(self) fn fractal_mgr(&self) -> &mgr::FractalMgr {
diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
index 75043e23..69e52c42 100644
--- a/server/src/engine/fractal/test_utils.rs
+++ b/server/src/engine/fractal/test_utils.rs
@@ -25,9 +25,13 @@
 */
 
 use {
-    super::{CriticalTask, GenericTask, GlobalInstanceLike, SysConfig, Task},
+    super::{
+        CriticalTask, FractalModelDriver, GenericTask, GlobalInstanceLike, ModelUniqueID,
+        SysConfig, Task,
+    },
     crate::engine::{
         core::GlobalNS,
+        data::uuid::Uuid,
         storage::v1::{
             header_meta::HostRunMode,
             memfs::{NullFS, VirtualFS},
@@ -36,6 +40,7 @@ use {
         txn::gns::GNSTransactionDriverAnyFS,
     },
     parking_lot::{Mutex, RwLock},
+    std::collections::HashMap,
 };
 
 /// A `test` mode global implementation
@@ -45,6 +50,7 @@ pub struct TestGlobal<Fs: RawFSInterface = VirtualFS> {
     lp_queue: RwLock<Vec<Task<GenericTask>>>,
     max_delta_size: usize,
     txn_driver: Mutex<GNSTransactionDriverAnyFS<Fs>>,
+    model_drivers: RwLock<HashMap<ModelUniqueID, FractalModelDriver<Fs>>>,
     sys_cfg: super::SysConfig,
 }
 
@@ -60,6 +66,7 @@ impl<Fs: RawFSInterface> TestGlobal<Fs> {
             lp_queue: RwLock::default(),
             max_delta_size,
             txn_driver: Mutex::new(txn_driver),
+            model_drivers: RwLock::default(),
             sys_cfg: SysConfig::test_default(),
         }
     }
@@ -117,12 +124,28 @@ impl<Fs: RawFSInterface> GlobalInstanceLike for TestGlobal<Fs> {
     }
     fn initialize_model_driver(
         &self,
-        _space_name: &str,
-        _space_uuid: crate::engine::data::uuid::Uuid,
-        _model_name: &str,
-        _model_uuid: crate::engine::data::uuid::Uuid,
+        space_name: &str,
+        space_uuid: Uuid,
+        model_name: &str,
+        model_uuid: Uuid,
     ) -> crate::engine::storage::v1::SDSSResult<()> {
-        todo!()
+        // create model dir
+        Fs::fs_create_dir(&crate::engine::storage::v1::loader::SEInitState::model_dir(
+            space_name, space_uuid, model_name, model_uuid,
+        ))?;
+        let driver = crate::engine::storage::v1::data_batch::create(
+            &crate::engine::storage::v1::loader::SEInitState::model_path(
+                space_name, space_uuid, model_name, model_uuid,
+            ),
+            self.sys_cfg().host_data().settings_version(),
+            self.sys_cfg().host_data().run_mode(),
+            self.sys_cfg().host_data().startup_counter(),
+        )?;
+        self.model_drivers.write().insert(
+            ModelUniqueID::new(space_name, model_name, model_uuid),
+            FractalModelDriver::init(driver),
+        );
+        Ok(())
     }
 }
 
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index d053caf5..22400507 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -399,6 +399,15 @@ impl<K: AsKey, V: AsValue, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
                 &(e.as_ref()).read_value().v
             })
     }
+    fn _get_mut<Q: ?Sized>(&mut self, k: &Q) -> Option<&mut V>
+    where
+        K: Borrow<Q>,
+        Q: AsKey,
+    {
+        self.m
+            .get_mut(unsafe { IndexSTSeqDllQref::from_ref(k) })
+            .map(|e| unsafe { &mut e.as_mut().v })
+    }
     #[inline(always)]
     fn _update<Q: ?Sized>(&mut self, k: &Q, v: V) -> Option<V>
     where
@@ -629,12 +638,12 @@ where
         self._get(key).cloned()
     }
 
-    fn st_get_mut<Q>(&mut self, _: &Q) -> Option<&mut V>
+    fn st_get_mut<Q>(&mut self, k: &Q) -> Option<&mut V>
     where
         K: AsKey + Borrow<Q>,
         Q: ?Sized + AsKey,
     {
-        todo!()
+        self._get_mut(k)
     }
 
     fn st_update<Q>(&mut self, key: &Q, val: V) -> bool
diff --git a/server/src/engine/storage/v1/batch_jrnl/mod.rs b/server/src/engine/storage/v1/batch_jrnl/mod.rs
index 0632e27e..b5a3de3c 100644
--- a/server/src/engine/storage/v1/batch_jrnl/mod.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/mod.rs
@@ -45,26 +45,39 @@ pub(super) use restore::{DecodedBatchEvent, DecodedBatchEventKind, NormalBatch};
 pub use {persist::DataBatchPersistDriver, restore::DataBatchRestoreDriver};
 
 use {
-    super::{
-        header_meta,
-        rw::{FileOpen, SDSSFileIO},
-        RawFSInterface, SDSSResult,
-    },
+    super::{header_meta, rw::SDSSFileIO, RawFSInterface, SDSSResult},
     crate::engine::core::model::Model,
 };
 
 const LOG_SPECIFIER_VERSION: header_meta::FileSpecifierVersion =
     header_meta::FileSpecifierVersion::__new(0);
 
-pub fn open_or_reinit<Fs: RawFSInterface>(
+/// Re-initialize an existing batch journal and read all its data into model
+pub fn reinit<Fs: RawFSInterface>(
     name: &str,
     model: &Model,
+) -> SDSSResult<DataBatchPersistDriver<Fs>> {
+    let (_header, f) = SDSSFileIO::<Fs>::open::<false>(
+        name,
+        header_meta::FileScope::Journal,
+        header_meta::FileSpecifier::TableDataBatch,
+        LOG_SPECIFIER_VERSION,
+    )?;
+    // restore
+    let mut restore_driver = DataBatchRestoreDriver::new(f)?;
+    restore_driver.read_data_batch_into_model(model)?;
+    DataBatchPersistDriver::new(restore_driver.into_file(), false)
+}
+
+/// Create a new batch journal
+pub fn create<Fs: RawFSInterface>(
+    path: &str,
     host_setting_version: u32,
     host_run_mode: header_meta::HostRunMode,
     host_startup_counter: u64,
 ) -> SDSSResult<DataBatchPersistDriver<Fs>> {
-    let f = SDSSFileIO::<Fs>::open_or_create_perm_rw::<false>(
-        name,
+    let f = SDSSFileIO::<Fs>::create(
+        path,
         header_meta::FileScope::Journal,
         header_meta::FileSpecifier::TableDataBatch,
         LOG_SPECIFIER_VERSION,
@@ -72,16 +85,5 @@ pub fn open_or_reinit<Fs: RawFSInterface>(
         host_run_mode,
         host_startup_counter,
     )?;
-    match f {
-        FileOpen::Created(new_file) => Ok(DataBatchPersistDriver::new(new_file, true)?),
-        FileOpen::Existing(existing, _) => {
-            // restore
-            let mut restore_driver = DataBatchRestoreDriver::new(existing)?;
-            restore_driver.read_data_batch_into_model(model)?;
-            Ok(DataBatchPersistDriver::new(
-                restore_driver.into_file(),
-                false,
-            )?)
-        }
-    }
+    DataBatchPersistDriver::new(f, true)
 }
diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs
index 0a21b218..1fad9678 100644
--- a/server/src/engine/storage/v1/loader.rs
+++ b/server/src/engine/storage/v1/loader.rs
@@ -36,14 +36,14 @@ const GNS_FILE_PATH: &str = "gns.db-tlog";
 
 pub struct SEInitState {
     pub txn_driver: GNSTransactionDriverAnyFS<super::LocalFS>,
-    pub model_drivers: ModelDrivers,
+    pub model_drivers: ModelDrivers<LocalFS>,
     pub gns: GlobalNS,
 }
 
 impl SEInitState {
     pub fn new(
         txn_driver: GNSTransactionDriverAnyFS<super::LocalFS>,
-        model_drivers: ModelDrivers,
+        model_drivers: ModelDrivers<LocalFS>,
         gns: GlobalNS,
     ) -> Self {
         Self {
@@ -70,13 +70,7 @@ impl SEInitState {
             let space_uuid = space.get_uuid();
             for (model_name, model) in space.models().read().iter() {
                 let path = Self::model_path(space_name, space_uuid, model_name, model.get_uuid());
-                let persist_driver = match batch_jrnl::open_or_reinit(
-                    &path,
-                    model,
-                    host_setting_version,
-                    host_run_mode,
-                    host_startup_counter,
-                ) {
+                let persist_driver = match batch_jrnl::reinit(&path, model) {
                     Ok(j) => j,
                     Err(e) => {
                         return Err(e.with_extra(format!(
diff --git a/server/src/engine/storage/v1/memfs.rs b/server/src/engine/storage/v1/memfs.rs
index 15a0a2a6..adce7539 100644
--- a/server/src/engine/storage/v1/memfs.rs
+++ b/server/src/engine/storage/v1/memfs.rs
@@ -57,7 +57,7 @@ type ComponentIter<'a> = std::iter::Take<std::vec::IntoIter<&'a str>>;
 */
 
 #[derive(Debug)]
-enum VNode {
+pub(super) enum VNode {
     Dir(HashMap<Box<str>, Self>),
     File(VFile),
 }
@@ -223,6 +223,44 @@ impl RawFSInterface for VirtualFS {
             }
         }
     }
+    fn fs_fcreate_rw(fpath: &str) -> SDSSResult<Self::File> {
+        let mut vfs = VFS.write();
+        let (target_file, components) = split_target_and_components(fpath);
+        let target_dir = find_target_dir_mut(components, &mut vfs)?;
+        match target_dir.entry(target_file.into()) {
+            Entry::Occupied(k) => {
+                match k.get() {
+                    VNode::Dir(_) => {
+                        return Err(Error::new(
+                            ErrorKind::AlreadyExists,
+                            "found directory with same name where file was to be created",
+                        )
+                        .into());
+                    }
+                    VNode::File(_) => {
+                        // the file already exists
+                        return Err(Error::new(
+                            ErrorKind::AlreadyExists,
+                            "the file already exists",
+                        )
+                        .into());
+                    }
+                }
+            }
+            Entry::Vacant(v) => {
+                // no file exists, we can create this
+                v.insert(VNode::File(VFile::new(true, true, vec![], 0)));
+                Ok(VFileDescriptor(fpath.into()))
+            }
+        }
+    }
+    fn fs_fopen_rw(fpath: &str) -> SDSSResult<Self::File> {
+        with_file_mut(fpath, |f| {
+            f.read = true;
+            f.write = true;
+            Ok(VFileDescriptor(fpath.into()))
+        })
+    }
 }
 
 fn find_target_dir_mut<'a>(
@@ -477,6 +515,7 @@ impl RawFileInterfaceExt for VFileDescriptor {
 pub struct NullFS;
 pub struct NullFile;
 impl RawFSInterface for NullFS {
+    const NOT_NULL: bool = false;
     type File = NullFile;
     fn fs_rename_file(_: &str, _: &str) -> SDSSResult<()> {
         Ok(())
@@ -499,6 +538,12 @@ impl RawFSInterface for NullFS {
     fn fs_fopen_or_create_rw(_: &str) -> SDSSResult<RawFileOpen<Self::File>> {
         Ok(RawFileOpen::Created(NullFile))
     }
+    fn fs_fopen_rw(_: &str) -> SDSSResult<Self::File> {
+        Ok(NullFile)
+    }
+    fn fs_fcreate_rw(_: &str) -> SDSSResult<Self::File> {
+        Ok(NullFile)
+    }
 }
 impl RawFileInterfaceRead for NullFile {
     fn fr_read_exact(&mut self, _: &mut [u8]) -> SDSSResult<()> {
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index c84f3f21..0dbb413c 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -47,7 +47,7 @@ pub use {
     rw::{LocalFS, RawFSInterface, SDSSFileIO},
 };
 pub mod data_batch {
-    pub use super::batch_jrnl::{DataBatchPersistDriver, DataBatchRestoreDriver};
+    pub use super::batch_jrnl::{create, reinit, DataBatchPersistDriver, DataBatchRestoreDriver};
 }
 pub mod header_meta {
     pub use super::header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode};
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 8174f4f4..b040f142 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -71,16 +71,34 @@ pub enum RawFileOpen<F> {
     Existing(F),
 }
 
+/// The specification for a file system interface (our own abstraction over the fs)
 pub trait RawFSInterface {
+    /// asserts that the file system is not a null filesystem (like `/dev/null` for example)
     const NOT_NULL: bool = true;
+    /// the file descriptor that is returned by the file system when a file is opened
     type File: RawFileInterface;
+    /// Remove a file
     fn fs_remove_file(fpath: &str) -> SDSSResult<()>;
+    /// Rename a file
     fn fs_rename_file(from: &str, to: &str) -> SDSSResult<()>;
+    /// Create a directory
     fn fs_create_dir(fpath: &str) -> SDSSResult<()>;
+    /// Create a directory and all corresponding path components
     fn fs_create_dir_all(fpath: &str) -> SDSSResult<()>;
+    /// Delete a directory
     fn fs_delete_dir(fpath: &str) -> SDSSResult<()>;
+    /// Delete a directory and recursively remove all (if any) children
     fn fs_delete_dir_all(fpath: &str) -> SDSSResult<()>;
+    /// Open or create a file in R/W mode
+    ///
+    /// This will:
+    /// - Create a file if it doesn't exist
+    /// - Open a file it it does exist
     fn fs_fopen_or_create_rw(fpath: &str) -> SDSSResult<RawFileOpen<Self::File>>;
+    /// Open an existing file
+    fn fs_fopen_rw(fpath: &str) -> SDSSResult<Self::File>;
+    /// Create a new file
+    fn fs_fcreate_rw(fpath: &str) -> SDSSResult<Self::File>;
 }
 
 /// A file (well, probably) that can be used for RW operations along with advanced write and extended operations (such as seeking)
@@ -174,6 +192,18 @@ impl RawFSInterface for LocalFS {
             Ok(RawFileOpen::Existing(f))
         }
     }
+    fn fs_fcreate_rw(fpath: &str) -> SDSSResult<Self::File> {
+        let f = File::options()
+            .create_new(true)
+            .read(true)
+            .write(true)
+            .open(fpath)?;
+        Ok(f)
+    }
+    fn fs_fopen_rw(fpath: &str) -> SDSSResult<Self::File> {
+        let f = File::options().read(true).write(true).open(fpath)?;
+        Ok(f)
+    }
 }
 
 impl RawFileInterface for File {
@@ -357,6 +387,95 @@ pub struct SDSSFileIO<Fs: RawFSInterface> {
 }
 
 impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
+    /// Open an existing SDSS file
+    ///
+    /// **IMPORTANT: File position: end-of-header-section**
+    pub fn open<const REWRITE_MODIFY_COUNTER: bool>(
+        file_path: &str,
+        file_scope: FileScope,
+        file_specifier: FileSpecifier,
+        file_specifier_version: FileSpecifierVersion,
+    ) -> SDSSResult<(SDSSHeader, Self)> {
+        let f = Fs::fs_fopen_rw(file_path)?;
+        Self::_sdss_fopen::<REWRITE_MODIFY_COUNTER>(
+            f,
+            file_scope,
+            file_specifier,
+            file_specifier_version,
+        )
+    }
+    /// internal SDSS fopen routine
+    fn _sdss_fopen<const REWRITE_MODIFY_COUNTER: bool>(
+        mut f: <Fs as RawFSInterface>::File,
+        file_scope: FileScope,
+        file_specifier: FileSpecifier,
+        file_specifier_version: FileSpecifierVersion,
+    ) -> Result<(SDSSHeader, SDSSFileIO<Fs>), SDSSError> {
+        let mut header_raw = [0u8; SDSSHeaderRaw::header_size()];
+        f.fr_read_exact(&mut header_raw)?;
+        let header = SDSSHeaderRaw::decode_noverify(header_raw)
+            .ok_or(SDSSError::HeaderDecodeCorruptedHeader)?;
+        header.verify(file_scope, file_specifier, file_specifier_version)?;
+        let mut f = Self::_new(f);
+        if REWRITE_MODIFY_COUNTER {
+            // since we updated this file, let us update the header
+            let mut new_header = header.clone();
+            new_header.dr_rs_mut().bump_modify_count();
+            f.seek_from_start(0)?;
+            f.fsynced_write(new_header.encoded().array().as_ref())?;
+            f.seek_from_start(SDSSHeaderRaw::header_size() as _)?;
+        }
+        Ok((header, f))
+    }
+    /// Create a new SDSS file
+    ///
+    /// **IMPORTANT: File position: end-of-header-section**
+    pub fn create(
+        file_path: &str,
+        file_scope: FileScope,
+        file_specifier: FileSpecifier,
+        file_specifier_version: FileSpecifierVersion,
+        host_setting_version: u32,
+        host_run_mode: HostRunMode,
+        host_startup_counter: u64,
+    ) -> SDSSResult<Self> {
+        let f = Fs::fs_fcreate_rw(file_path)?;
+        Self::_sdss_fcreate(
+            file_scope,
+            file_specifier,
+            file_specifier_version,
+            host_setting_version,
+            host_run_mode,
+            host_startup_counter,
+            f,
+        )
+    }
+    /// Internal SDSS fcreate routine
+    fn _sdss_fcreate(
+        file_scope: FileScope,
+        file_specifier: FileSpecifier,
+        file_specifier_version: FileSpecifierVersion,
+        host_setting_version: u32,
+        host_run_mode: HostRunMode,
+        host_startup_counter: u64,
+        f: <Fs as RawFSInterface>::File,
+    ) -> Result<SDSSFileIO<Fs>, SDSSError> {
+        let data = SDSSHeaderRaw::new_auto(
+            file_scope,
+            file_specifier,
+            file_specifier_version,
+            host_setting_version,
+            host_run_mode,
+            host_startup_counter,
+            0,
+        )
+        .array();
+        let mut f = Self::_new(f);
+        f.fsynced_write(&data)?;
+        Ok(f)
+    }
+    /// Create a new SDSS file or re-open an existing file and verify
+    ///
     /// **IMPORTANT: File position: end-of-header-section**
     pub fn open_or_create_perm_rw<const REWRITE_MODIFY_COUNTER: bool>(
         file_path: &str,
@@ -370,39 +489,25 @@ impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
         let f = Fs::fs_fopen_or_create_rw(file_path)?;
         match f {
             RawFileOpen::Created(f) => {
-                // since this file was just created, we need to append the header
-                let data = SDSSHeaderRaw::new_auto(
+                let f = Self::_sdss_fcreate(
                     file_scope,
                     file_specifier,
                     file_specifier_version,
                     host_setting_version,
                     host_run_mode,
                     host_startup_counter,
-                    0,
-                )
-                .array();
-                let mut f = Self::_new(f);
-                f.fsynced_write(&data)?;
+                    f,
+                )?;
                 Ok(FileOpen::Created(f))
             }
-            RawFileOpen::Existing(mut f) => {
-                // this is an existing file. decoded the header
-                let mut header_raw = [0u8; SDSSHeaderRaw::header_size()];
-                f.fr_read_exact(&mut header_raw)?;
-                let header = SDSSHeaderRaw::decode_noverify(header_raw)
-                    .ok_or(SDSSError::HeaderDecodeCorruptedHeader)?;
-                // now validate the header
-                header.verify(file_scope, file_specifier, file_specifier_version)?;
-                let mut f = Self::_new(f);
-                if REWRITE_MODIFY_COUNTER {
-                    // since we updated this file, let us update the header
-                    let mut new_header = header.clone();
-                    new_header.dr_rs_mut().bump_modify_count();
-                    f.seek_from_start(0)?;
-                    f.fsynced_write(new_header.encoded().array().as_ref())?;
-                    f.seek_from_start(SDSSHeaderRaw::header_size() as _)?;
-                }
-                Ok(FileOpen::Existing(f, header))
+            RawFileOpen::Existing(f) => {
+                let (f, header) = Self::_sdss_fopen::<REWRITE_MODIFY_COUNTER>(
+                    f,
+                    file_scope,
+                    file_specifier,
+                    file_specifier_version,
+                )?;
+                Ok(FileOpen::Existing(header, f))
             }
         }
     }

From c35e35b9c8245e62259fd6fb271e15fd850cb038 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 24 Sep 2023 07:49:10 +0000
Subject: [PATCH 263/310] Only attempt to restore data if new instance

---
 server/src/engine/fractal/test_utils.rs     | 32 ++++----
 server/src/engine/storage/v1/journal.rs     | 42 ++---------
 server/src/engine/storage/v1/loader.rs      | 81 +++++++++++++++------
 server/src/engine/storage/v1/memfs.rs       | 18 ++---
 server/src/engine/storage/v1/rw.rs          | 52 +++++++------
 server/src/engine/storage/v1/tests/batch.rs |  7 +-
 server/src/engine/storage/v1/tests/rw.rs    |  4 +-
 server/src/engine/storage/v1/tests/tx.rs    |  1 +
 server/src/engine/txn/gns/mod.rs            | 28 ++-----
 9 files changed, 133 insertions(+), 132 deletions(-)

diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
index 69e52c42..018ee678 100644
--- a/server/src/engine/fractal/test_utils.rs
+++ b/server/src/engine/fractal/test_utils.rs
@@ -32,10 +32,13 @@ use {
     crate::engine::{
         core::GlobalNS,
         data::uuid::Uuid,
-        storage::v1::{
-            header_meta::HostRunMode,
-            memfs::{NullFS, VirtualFS},
-            RawFSInterface,
+        storage::{
+            self,
+            v1::{
+                header_meta::HostRunMode,
+                memfs::{NullFS, VirtualFS},
+                RawFSInterface,
+            },
         },
         txn::gns::GNSTransactionDriverAnyFS,
     },
@@ -75,15 +78,10 @@ impl<Fs: RawFSInterface> TestGlobal<Fs> {
 impl<Fs: RawFSInterface> TestGlobal<Fs> {
     pub fn new_with_driver_id(log_name: &str) -> Self {
         let gns = GlobalNS::empty();
-        let driver = GNSTransactionDriverAnyFS::open_or_reinit_with_name(
-            &gns,
-            log_name,
-            0,
-            HostRunMode::Prod,
-            0,
-        )
-        .unwrap();
-        Self::new(gns, 0, driver)
+        let driver = storage::v1::loader::open_gns_driver(log_name, 0, HostRunMode::Dev, 0, &gns)
+            .unwrap()
+            .into_inner();
+        Self::new(gns, 0, GNSTransactionDriverAnyFS::new(driver))
     }
 }
 
@@ -128,13 +126,13 @@ impl<Fs: RawFSInterface> GlobalInstanceLike for TestGlobal<Fs> {
         space_uuid: Uuid,
         model_name: &str,
         model_uuid: Uuid,
-    ) -> crate::engine::storage::v1::SDSSResult<()> {
+    ) -> storage::v1::SDSSResult<()> {
         // create model dir
-        Fs::fs_create_dir(&crate::engine::storage::v1::loader::SEInitState::model_dir(
+        Fs::fs_create_dir(&storage::v1::loader::SEInitState::model_dir(
             space_name, space_uuid, model_name, model_uuid,
         ))?;
-        let driver = crate::engine::storage::v1::data_batch::create(
-            &crate::engine::storage::v1::loader::SEInitState::model_path(
+        let driver = storage::v1::data_batch::create(
+            &storage::v1::loader::SEInitState::model_path(
                 space_name, space_uuid, model_name, model_uuid,
             ),
             self.sys_cfg().host_data().settings_version(),
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index 7336b932..07a4bc3e 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -41,12 +41,10 @@
   - FIXME(@ohsayan): we will probably (naively) need to dynamically reposition the cursor in case the metadata is corrupted as well
 */
 
-use super::rw::RawFSInterface;
-
 use {
     super::{
         header_impl::{FileSpecifierVersion, HostRunMode, SDSSHeaderRaw},
-        rw::{FileOpen, SDSSFileIO},
+        rw::{FileOpen, RawFSInterface, SDSSFileIO},
         SDSSError, SDSSResult,
     },
     crate::{
@@ -59,34 +57,6 @@ use {
 const CRC: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
 const RECOVERY_BLOCK_AUTO_THRESHOLD: usize = 5;
 
-/// A journal to `/dev/null` (app. level impl)
-#[cfg(test)]
-pub fn null_journal<TA: JournalAdapter>(
-    log_file_name: &str,
-    log_kind: FileSpecifier,
-    log_kind_version: FileSpecifierVersion,
-    host_setting_version: u32,
-    host_run_mode: HostRunMode,
-    host_startup_counter: u64,
-    _: &TA::GlobalState,
-) -> JournalWriter<super::memfs::NullFS, TA> {
-    let FileOpen::Created(journal) =
-        SDSSFileIO::<super::memfs::NullFS>::open_or_create_perm_rw::<false>(
-            log_file_name,
-            FileScope::Journal,
-            log_kind,
-            log_kind_version,
-            host_setting_version,
-            host_run_mode,
-            host_startup_counter,
-        )
-        .unwrap()
-    else {
-        panic!()
-    };
-    JournalWriter::new(journal, 0, true).unwrap()
-}
-
 pub fn open_journal<TA: JournalAdapter, Fs: RawFSInterface>(
     log_file_name: &str,
     log_kind: FileSpecifier,
@@ -95,7 +65,7 @@ pub fn open_journal<TA: JournalAdapter, Fs: RawFSInterface>(
     host_run_mode: HostRunMode,
     host_startup_counter: u64,
     gs: &TA::GlobalState,
-) -> SDSSResult<JournalWriter<Fs, TA>> {
+) -> SDSSResult<FileOpen<JournalWriter<Fs, TA>>> {
     macro_rules! open_file {
         ($modify:literal) => {
             SDSSFileIO::<Fs>::open_or_create_perm_rw::<$modify>(
@@ -116,11 +86,13 @@ pub fn open_journal<TA: JournalAdapter, Fs: RawFSInterface>(
         open_file!(true)
     }?;
     let file = match f {
-        FileOpen::Created(f) => return JournalWriter::new(f, 0, true),
-        FileOpen::Existing(file, _) => file,
+        FileOpen::Created(f) => return Ok(FileOpen::Created(JournalWriter::new(f, 0, true)?)),
+        FileOpen::Existing((file, _header)) => file,
     };
     let (file, last_txn) = JournalReader::<TA, Fs>::scroll(file, gs)?;
-    JournalWriter::new(file, last_txn, false)
+    Ok(FileOpen::Existing(JournalWriter::new(
+        file, last_txn, false,
+    )?))
 }
 
 /// The journal adapter
diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs
index 1fad9678..b73c6fef 100644
--- a/server/src/engine/storage/v1/loader.rs
+++ b/server/src/engine/storage/v1/loader.rs
@@ -28,13 +28,20 @@ use crate::engine::{
     core::GlobalNS,
     data::uuid::Uuid,
     fractal::{FractalModelDriver, ModelDrivers, ModelUniqueID},
-    storage::v1::{batch_jrnl, header_meta::HostRunMode, LocalFS, SDSSErrorContext, SDSSResult},
-    txn::gns::GNSTransactionDriverAnyFS,
+    storage::v1::{
+        batch_jrnl, header_meta,
+        journal::{self, JournalWriter},
+        rw::{FileOpen, RawFSInterface},
+        LocalFS, SDSSErrorContext, SDSSResult,
+    },
+    txn::gns::{GNSAdapter, GNSTransactionDriverAnyFS},
 };
 
 const GNS_FILE_PATH: &str = "gns.db-tlog";
+const GNS_LOG_VERSION_CODE: u32 = 0;
 
 pub struct SEInitState {
+    pub new_instance: bool,
     pub txn_driver: GNSTransactionDriverAnyFS<super::LocalFS>,
     pub model_drivers: ModelDrivers<LocalFS>,
     pub gns: GlobalNS,
@@ -42,11 +49,13 @@ pub struct SEInitState {
 
 impl SEInitState {
     pub fn new(
+        new_instance: bool,
         txn_driver: GNSTransactionDriverAnyFS<super::LocalFS>,
         model_drivers: ModelDrivers<LocalFS>,
         gns: GlobalNS,
     ) -> Self {
         Self {
+            new_instance,
             txn_driver,
             model_drivers,
             gns,
@@ -54,37 +63,47 @@ impl SEInitState {
     }
     pub fn try_init(
         host_setting_version: u32,
-        host_run_mode: HostRunMode,
+        host_run_mode: header_meta::HostRunMode,
         host_startup_counter: u64,
     ) -> SDSSResult<Self> {
         let gns = GlobalNS::empty();
-        let gns_txn_driver = GNSTransactionDriverAnyFS::<LocalFS>::open_or_reinit_with_name(
-            &gns,
+        let gns_txn_driver = open_gns_driver(
             GNS_FILE_PATH,
             host_setting_version,
             host_run_mode,
             host_startup_counter,
+            &gns,
         )?;
+        let new_instance = gns_txn_driver.is_created();
         let mut model_drivers = ModelDrivers::new();
-        for (space_name, space) in gns.spaces().read().iter() {
-            let space_uuid = space.get_uuid();
-            for (model_name, model) in space.models().read().iter() {
-                let path = Self::model_path(space_name, space_uuid, model_name, model.get_uuid());
-                let persist_driver = match batch_jrnl::reinit(&path, model) {
-                    Ok(j) => j,
-                    Err(e) => {
-                        return Err(e.with_extra(format!(
-                            "failed to restore model data from journal in `{path}`"
-                        )))
-                    }
-                };
-                let _ = model_drivers.insert(
-                    ModelUniqueID::new(space_name, model_name, model.get_uuid()),
-                    FractalModelDriver::init(persist_driver),
-                );
+        if !new_instance {
+            // this is an existing instance, so read in all data
+            for (space_name, space) in gns.spaces().read().iter() {
+                let space_uuid = space.get_uuid();
+                for (model_name, model) in space.models().read().iter() {
+                    let path =
+                        Self::model_path(space_name, space_uuid, model_name, model.get_uuid());
+                    let persist_driver = match batch_jrnl::reinit(&path, model) {
+                        Ok(j) => j,
+                        Err(e) => {
+                            return Err(e.with_extra(format!(
+                                "failed to restore model data from journal in `{path}`"
+                            )))
+                        }
+                    };
+                    let _ = model_drivers.insert(
+                        ModelUniqueID::new(space_name, model_name, model.get_uuid()),
+                        FractalModelDriver::init(persist_driver),
+                    );
+                }
             }
         }
-        Ok(SEInitState::new(gns_txn_driver, model_drivers, gns))
+        Ok(SEInitState::new(
+            new_instance,
+            GNSTransactionDriverAnyFS::new(gns_txn_driver.into_inner()),
+            model_drivers,
+            gns,
+        ))
     }
     pub fn model_path(
         space_name: &str,
@@ -109,3 +128,21 @@ impl SEInitState {
         format!("data/{space_name}-{space_uuid}")
     }
 }
+
+pub fn open_gns_driver<Fs: RawFSInterface>(
+    path: &str,
+    host_setting_version: u32,
+    host_run_mode: header_meta::HostRunMode,
+    host_startup_counter: u64,
+    gns: &GlobalNS,
+) -> SDSSResult<FileOpen<JournalWriter<Fs, GNSAdapter>>> {
+    journal::open_journal::<GNSAdapter, Fs>(
+        path,
+        header_meta::FileSpecifier::GNSTxnLog,
+        header_meta::FileSpecifierVersion::__new(GNS_LOG_VERSION_CODE),
+        host_setting_version,
+        host_run_mode,
+        host_startup_counter,
+        gns,
+    )
+}
diff --git a/server/src/engine/storage/v1/memfs.rs b/server/src/engine/storage/v1/memfs.rs
index adce7539..9dd11fa9 100644
--- a/server/src/engine/storage/v1/memfs.rs
+++ b/server/src/engine/storage/v1/memfs.rs
@@ -28,8 +28,8 @@ use {
     crate::engine::{
         storage::v1::{
             rw::{
-                RawFSInterface, RawFileInterface, RawFileInterfaceExt, RawFileInterfaceRead,
-                RawFileInterfaceWrite, RawFileInterfaceWriteExt, RawFileOpen,
+                FileOpen, RawFSInterface, RawFileInterface, RawFileInterfaceExt,
+                RawFileInterfaceRead, RawFileInterfaceWrite, RawFileInterfaceWriteExt,
             },
             SDSSResult,
         },
@@ -123,10 +123,10 @@ impl RawFSInterface for VirtualFS {
         // create new file
         let file = VirtualFS::fs_fopen_or_create_rw(to)?;
         match file {
-            RawFileOpen::Created(mut c) => {
+            FileOpen::Created(mut c) => {
                 c.fw_write_all(&data)?;
             }
-            RawFileOpen::Existing(mut e) => {
+            FileOpen::Existing(mut e) => {
                 e.fw_truncate_to(0)?;
                 e.fw_write_all(&data)?;
             }
@@ -203,7 +203,7 @@ impl RawFSInterface for VirtualFS {
     fn fs_delete_dir_all(fpath: &str) -> super::SDSSResult<()> {
         delete_dir(fpath, true)
     }
-    fn fs_fopen_or_create_rw(fpath: &str) -> super::SDSSResult<super::rw::RawFileOpen<Self::File>> {
+    fn fs_fopen_or_create_rw(fpath: &str) -> super::SDSSResult<super::rw::FileOpen<Self::File>> {
         let mut vfs = VFS.write();
         // components
         let (target_file, components) = split_target_and_components(fpath);
@@ -213,13 +213,13 @@ impl RawFSInterface for VirtualFS {
                 VNode::File(f) => {
                     f.read = true;
                     f.write = true;
-                    Ok(RawFileOpen::Existing(VFileDescriptor(fpath.into())))
+                    Ok(FileOpen::Existing(VFileDescriptor(fpath.into())))
                 }
                 VNode::Dir(_) => return err_item_is_not_file(),
             },
             Entry::Vacant(v) => {
                 v.insert(VNode::File(VFile::new(true, true, vec![], 0)));
-                Ok(RawFileOpen::Created(VFileDescriptor(fpath.into())))
+                Ok(FileOpen::Created(VFileDescriptor(fpath.into())))
             }
         }
     }
@@ -535,8 +535,8 @@ impl RawFSInterface for NullFS {
     fn fs_delete_dir_all(_: &str) -> SDSSResult<()> {
         Ok(())
     }
-    fn fs_fopen_or_create_rw(_: &str) -> SDSSResult<RawFileOpen<Self::File>> {
-        Ok(RawFileOpen::Created(NullFile))
+    fn fs_fopen_or_create_rw(_: &str) -> SDSSResult<FileOpen<Self::File>> {
+        Ok(FileOpen::Created(NullFile))
     }
     fn fs_fopen_rw(_: &str) -> SDSSResult<Self::File> {
         Ok(NullFile)
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index b040f142..fbabc1b8 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -24,8 +24,6 @@
  *
 */
 
-use std::marker::PhantomData;
-
 use {
     super::{
         header_impl::{
@@ -40,35 +38,45 @@ use {
     std::{
         fs::{self, File},
         io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write},
+        marker::PhantomData,
     },
 };
 
 #[derive(Debug)]
 /// Log whether
-pub enum FileOpen<F> {
-    Created(F),
-    Existing(F, SDSSHeader),
+pub enum FileOpen<CF, EF = CF> {
+    Created(CF),
+    Existing(EF),
 }
 
-impl<F> FileOpen<F> {
-    pub fn into_existing(self) -> Option<(F, SDSSHeader)> {
+impl<CF, EF> FileOpen<CF, EF> {
+    pub const fn is_created(&self) -> bool {
+        matches!(self, Self::Created(_))
+    }
+    pub const fn is_existing(&self) -> bool {
+        !self.is_created()
+    }
+    pub fn into_existing(self) -> Option<EF> {
         match self {
-            Self::Existing(f, h) => Some((f, h)),
+            Self::Existing(e) => Some(e),
             Self::Created(_) => None,
         }
     }
-    pub fn into_created(self) -> Option<F> {
+    pub fn into_created(self) -> Option<CF> {
         match self {
             Self::Created(f) => Some(f),
-            Self::Existing(_, _) => None,
+            Self::Existing(_) => None,
         }
     }
 }
 
-#[derive(Debug)]
-pub enum RawFileOpen<F> {
-    Created(F),
-    Existing(F),
+impl<F> FileOpen<F> {
+    pub fn into_inner(self) -> F {
+        match self {
+            Self::Created(f) => f,
+            Self::Existing(f) => f,
+        }
+    }
 }
 
 /// The specification for a file system interface (our own abstraction over the fs)
@@ -94,7 +102,7 @@ pub trait RawFSInterface {
     /// This will:
     /// - Create a file if it doesn't exist
     /// - Open a file it it does exist
-    fn fs_fopen_or_create_rw(fpath: &str) -> SDSSResult<RawFileOpen<Self::File>>;
+    fn fs_fopen_or_create_rw(fpath: &str) -> SDSSResult<FileOpen<Self::File>>;
     /// Open an existing file
     fn fs_fopen_rw(fpath: &str) -> SDSSResult<Self::File>;
     /// Create a new file
@@ -179,7 +187,7 @@ impl RawFSInterface for LocalFS {
     fn fs_delete_dir_all(fpath: &str) -> SDSSResult<()> {
         cvt(fs::remove_dir_all(fpath))
     }
-    fn fs_fopen_or_create_rw(fpath: &str) -> SDSSResult<RawFileOpen<Self::File>> {
+    fn fs_fopen_or_create_rw(fpath: &str) -> SDSSResult<FileOpen<Self::File>> {
         let f = File::options()
             .create(true)
             .read(true)
@@ -187,9 +195,9 @@ impl RawFSInterface for LocalFS {
             .open(fpath)?;
         let md = f.metadata()?;
         if md.len() == 0 {
-            Ok(RawFileOpen::Created(f))
+            Ok(FileOpen::Created(f))
         } else {
-            Ok(RawFileOpen::Existing(f))
+            Ok(FileOpen::Existing(f))
         }
     }
     fn fs_fcreate_rw(fpath: &str) -> SDSSResult<Self::File> {
@@ -485,10 +493,10 @@ impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
         host_setting_version: u32,
         host_run_mode: HostRunMode,
         host_startup_counter: u64,
-    ) -> SDSSResult<FileOpen<Self>> {
+    ) -> SDSSResult<FileOpen<Self, (Self, SDSSHeader)>> {
         let f = Fs::fs_fopen_or_create_rw(file_path)?;
         match f {
-            RawFileOpen::Created(f) => {
+            FileOpen::Created(f) => {
                 let f = Self::_sdss_fcreate(
                     file_scope,
                     file_specifier,
@@ -500,14 +508,14 @@ impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
                 )?;
                 Ok(FileOpen::Created(f))
             }
-            RawFileOpen::Existing(f) => {
+            FileOpen::Existing(f) => {
                 let (f, header) = Self::_sdss_fopen::<REWRITE_MODIFY_COUNTER>(
                     f,
                     file_scope,
                     file_specifier,
                     file_specifier_version,
                 )?;
-                Ok(FileOpen::Existing(header, f))
+                Ok(FileOpen::Existing((header, f)))
             }
         }
     }
diff --git a/server/src/engine/storage/v1/tests/batch.rs b/server/src/engine/storage/v1/tests/batch.rs
index 5f63a1ef..e38d4113 100644
--- a/server/src/engine/storage/v1/tests/batch.rs
+++ b/server/src/engine/storage/v1/tests/batch.rs
@@ -55,7 +55,10 @@ fn pkey(v: impl Into<Datacell>) -> PrimaryIndexKey {
     PrimaryIndexKey::try_from_dc(v.into()).unwrap()
 }
 
-fn open_file(fpath: &str) -> FileOpen<SDSSFileIO<VirtualFS>> {
+fn open_file(
+    fpath: &str,
+) -> FileOpen<SDSSFileIO<VirtualFS>, (SDSSFileIO<VirtualFS>, super::super::header_impl::SDSSHeader)>
+{
     SDSSFileIO::open_or_create_perm_rw::<false>(
         fpath,
         FileScope::DataBatch,
@@ -71,7 +74,7 @@ fn open_file(fpath: &str) -> FileOpen<SDSSFileIO<VirtualFS>> {
 fn open_batch_data(fpath: &str, mdl: &Model) -> DataBatchPersistDriver<VirtualFS> {
     match open_file(fpath) {
         FileOpen::Created(f) => DataBatchPersistDriver::new(f, true),
-        FileOpen::Existing(f, _) => {
+        FileOpen::Existing((f, _header)) => {
             let mut dbr = DataBatchRestoreDriver::new(f).unwrap();
             dbr.read_data_batch_into_model(mdl).unwrap();
             DataBatchPersistDriver::new(dbr.into_file(), false)
diff --git a/server/src/engine/storage/v1/tests/rw.rs b/server/src/engine/storage/v1/tests/rw.rs
index ff70d03a..4df4ce87 100644
--- a/server/src/engine/storage/v1/tests/rw.rs
+++ b/server/src/engine/storage/v1/tests/rw.rs
@@ -43,7 +43,7 @@ fn create_delete() {
         )
         .unwrap();
         match f {
-            FileOpen::Existing(_, _) => panic!(),
+            FileOpen::Existing(_) => panic!(),
             FileOpen::Created(_) => {}
         };
     }
@@ -58,7 +58,7 @@ fn create_delete() {
     )
     .unwrap();
     let h = match open {
-        FileOpen::Existing(_, header) => header,
+        FileOpen::Existing((_, header)) => header,
         _ => panic!(),
     };
     assert_eq!(h.gr_mdr().file_scope(), FileScope::Journal);
diff --git a/server/src/engine/storage/v1/tests/tx.rs b/server/src/engine/storage/v1/tests/tx.rs
index 01c24403..5bf01ec0 100644
--- a/server/src/engine/storage/v1/tests/tx.rs
+++ b/server/src/engine/storage/v1/tests/tx.rs
@@ -143,6 +143,7 @@ fn open_log(
         1,
         &db,
     )
+    .map(|v| v.into_inner())
 }
 
 #[test]
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index 85905315..07479d46 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -34,7 +34,6 @@ use {
             data::uuid::Uuid,
             mem::BufferedScanner,
             storage::v1::{
-                self, header_meta,
                 inf::{self, PersistObject},
                 JournalAdapter, JournalWriter, LocalFS, RawFSInterface, SDSSResult,
             },
@@ -64,35 +63,18 @@ pub type GNSTransactionDriverNullZero =
 #[cfg(test)]
 pub type GNSTransactionDriverVFS = GNSTransactionDriverAnyFS<VirtualFS>;
 
-const CURRENT_LOG_VERSION: u32 = 0;
-
 /// The GNS transaction driver is used to handle DDL transactions
-pub struct GNSTransactionDriverAnyFS<F: RawFSInterface = LocalFS> {
-    journal: JournalWriter<F, GNSAdapter>,
+pub struct GNSTransactionDriverAnyFS<Fs: RawFSInterface = LocalFS> {
+    journal: JournalWriter<Fs, GNSAdapter>,
 }
 
 impl<Fs: RawFSInterface> GNSTransactionDriverAnyFS<Fs> {
+    pub fn new(journal: JournalWriter<Fs, GNSAdapter>) -> Self {
+        Self { journal }
+    }
     pub fn __journal_mut(&mut self) -> &mut JournalWriter<Fs, GNSAdapter> {
         &mut self.journal
     }
-    pub fn open_or_reinit_with_name(
-        gns: &GlobalNS,
-        log_file_name: &str,
-        host_setting_version: u32,
-        host_run_mode: header_meta::HostRunMode,
-        host_startup_counter: u64,
-    ) -> TransactionResult<Self> {
-        let journal = v1::open_journal(
-            log_file_name,
-            header_meta::FileSpecifier::GNSTxnLog,
-            header_meta::FileSpecifierVersion::__new(CURRENT_LOG_VERSION),
-            host_setting_version,
-            host_run_mode,
-            host_startup_counter,
-            gns,
-        )?;
-        Ok(Self { journal })
-    }
     /// Attempts to commit the given event into the journal, handling any possible recovery triggers and returning
     /// errors (if any)
     pub fn try_commit<GE: GNSEvent>(&mut self, gns_event: GE) -> TransactionResult<()> {

From c4d51ac8e7146e9a8eb30a95810640d9ea677f27 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 24 Sep 2023 10:15:32 +0000
Subject: [PATCH 264/310] Ensure full auth config is read

---
 server/src/engine/config.rs    | 260 +++++++++++++++++++++------------
 server/src/engine/tests/mod.rs |  70 ++++++++-
 2 files changed, 226 insertions(+), 104 deletions(-)

diff --git a/server/src/engine/config.rs b/server/src/engine/config.rs
index 75c0bdba..b6fd2606 100644
--- a/server/src/engine/config.rs
+++ b/server/src/engine/config.rs
@@ -27,10 +27,7 @@
 use {
     crate::util::os::SysIOError,
     core::fmt,
-    serde::{
-        de::{self, Deserializer, Visitor},
-        Deserialize,
-    },
+    serde::Deserialize,
     std::{collections::HashMap, fs},
 };
 
@@ -39,6 +36,7 @@ use {
 */
 
 pub type ParsedRawArgs = std::collections::HashMap<String, Vec<String>>;
+pub const ROOT_PASSWORD_MIN_LEN: usize = 16;
 
 #[derive(Debug, PartialEq)]
 pub struct ModifyGuard<T> {
@@ -85,14 +83,21 @@ pub struct Configuration {
     endpoints: ConfigEndpoint,
     mode: ConfigMode,
     system: ConfigSystem,
+    auth: Option<ConfigAuth>,
 }
 
 impl Configuration {
-    pub fn new(endpoints: ConfigEndpoint, mode: ConfigMode, system: ConfigSystem) -> Self {
+    pub fn new(
+        endpoints: ConfigEndpoint,
+        mode: ConfigMode,
+        system: ConfigSystem,
+        auth: Option<ConfigAuth>,
+    ) -> Self {
         Self {
             endpoints,
             mode,
             system,
+            auth,
         }
     }
     const DEFAULT_HOST: &'static str = "127.0.0.1";
@@ -107,8 +112,8 @@ impl Configuration {
             mode: ConfigMode::Dev,
             system: ConfigSystem {
                 reliability_system_window: Self::DEFAULT_RELIABILITY_SVC_PING,
-                auth: false,
             },
+            auth: None,
         }
     }
 }
@@ -158,44 +163,17 @@ impl ConfigEndpointTls {
     config mode
 */
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Deserialize)]
 /// The configuration mode
 pub enum ConfigMode {
     /// In [`ConfigMode::Dev`] we're allowed to be more relaxed with settings
+    #[serde(rename = "dev")]
     Dev,
     /// In [`ConfigMode::Prod`] we're more stringent with settings
+    #[serde(rename = "prod")]
     Prod,
 }
 
-impl<'de> serde::Deserialize<'de> for ConfigMode {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        struct StringVisitor;
-        impl<'de> Visitor<'de> for StringVisitor {
-            type Value = ConfigMode;
-            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
-                formatter.write_str("a string 'dev' or 'prod'")
-            }
-            fn visit_str<E>(self, value: &str) -> Result<ConfigMode, E>
-            where
-                E: de::Error,
-            {
-                match value {
-                    "dev" => Ok(ConfigMode::Dev),
-                    "prod" => Ok(ConfigMode::Prod),
-                    _ => Err(de::Error::custom(format!(
-                        "expected 'dev' or 'prod', got {}",
-                        value
-                    ))),
-                }
-            }
-        }
-        deserializer.deserialize_str(StringVisitor)
-    }
-}
-
 /*
     config system
 */
@@ -205,19 +183,38 @@ impl<'de> serde::Deserialize<'de> for ConfigMode {
 pub struct ConfigSystem {
     /// time window in seconds for the reliability system to kick-in automatically
     reliability_system_window: u64,
-    /// if or not auth is enabled
-    auth: bool,
 }
 
 impl ConfigSystem {
-    pub fn new(reliability_system_window: u64, auth: bool) -> Self {
+    pub fn new(reliability_system_window: u64) -> Self {
         Self {
             reliability_system_window,
-            auth,
         }
     }
 }
 
+/*
+    config auth
+*/
+
+#[derive(Debug, PartialEq, Deserialize)]
+pub enum AuthDriver {
+    #[serde(rename = "pwd")]
+    Pwd,
+}
+
+#[derive(Debug, PartialEq, Deserialize)]
+pub struct ConfigAuth {
+    plugin: AuthDriver,
+    root_key: String,
+}
+
+impl ConfigAuth {
+    pub fn new(plugin: AuthDriver, root_key: String) -> Self {
+        Self { plugin, root_key }
+    }
+}
+
 /**
     decoded configuration
     ---
@@ -227,6 +224,7 @@ impl ConfigSystem {
 pub struct DecodedConfiguration {
     system: Option<DecodedSystemConfig>,
     endpoints: Option<DecodedEPConfig>,
+    auth: Option<DecodedAuth>,
 }
 
 impl Default for DecodedConfiguration {
@@ -234,14 +232,20 @@ impl Default for DecodedConfiguration {
         Self {
             system: Default::default(),
             endpoints: Default::default(),
+            auth: None,
         }
     }
 }
 
+#[derive(Debug, PartialEq, Deserialize)]
+pub struct DecodedAuth {
+    plugin: AuthDriver,
+    root_pass: String,
+}
+
 #[derive(Debug, PartialEq, Deserialize)]
 /// Decoded system configuration
 pub struct DecodedSystemConfig {
-    auth_enabled: Option<bool>,
     mode: Option<ConfigMode>,
     rs_window: Option<u64>,
 }
@@ -259,7 +263,7 @@ pub struct DecodedEPSecureConfig {
     host: String,
     port: u16,
     cert: String,
-    pass: String,
+    private_key: String,
 }
 
 #[derive(Debug, PartialEq, Deserialize)]
@@ -375,9 +379,10 @@ impl From<std::io::Error> for ConfigError {
 
 /// A configuration source implementation
 pub(super) trait ConfigurationSource {
+    const KEY_AUTH_DRIVER: &'static str;
+    const KEY_AUTH_ROOT_PASSWORD: &'static str;
     const KEY_TLS_CERT: &'static str;
     const KEY_TLS_KEY: &'static str;
-    const KEY_AUTH: &'static str;
     const KEY_ENDPOINTS: &'static str;
     const KEY_RUN_MODE: &'static str;
     const KEY_SERVICE_WINDOW: &'static str;
@@ -467,7 +472,7 @@ fn decode_tls_ep(
         host: host.into(),
         port,
         cert: tls_cert,
-        pass: tls_key,
+        private_key: tls_key,
     })
 }
 
@@ -501,25 +506,33 @@ fn arg_decode_tls_endpoint<CS: ConfigurationSource>(
     decode options
 */
 
-/// Check the auth mode. We currently only allow `pwd`
 fn arg_decode_auth<CS: ConfigurationSource>(
-    args: &[String],
+    src_args: &mut ParsedRawArgs,
     config: &mut ModifyGuard<DecodedConfiguration>,
 ) -> ConfigResult<()> {
-    argck_duplicate_values::<CS>(&args, CS::KEY_AUTH)?;
-    match args[0].as_str() {
-        "pwd" => match config.system.as_mut() {
-            Some(cfg) => cfg.auth_enabled = Some(true),
-            _ => {
-                config.system = Some(DecodedSystemConfig {
-                    auth_enabled: Some(true),
-                    mode: None,
-                    rs_window: None,
-                })
-            }
-        },
-        _ => return Err(CS::err_invalid_value_for(CS::KEY_AUTH)),
-    }
+    let (Some(auth_driver), Some(mut root_key)) = (
+        src_args.remove(CS::KEY_AUTH_DRIVER),
+        src_args.remove(CS::KEY_AUTH_ROOT_PASSWORD),
+    ) else {
+        return Err(ConfigError::with_src(
+            CS::SOURCE,
+            ConfigErrorKind::ErrorString(format!(
+                "to enable auth, you must provide values for both {} and {}",
+                CS::KEY_AUTH_DRIVER,
+                CS::KEY_AUTH_ROOT_PASSWORD
+            )),
+        ));
+    };
+    argck_duplicate_values::<CS>(&auth_driver, CS::KEY_AUTH_DRIVER)?;
+    argck_duplicate_values::<CS>(&root_key, CS::KEY_AUTH_DRIVER)?;
+    let auth_plugin = match auth_driver[0].as_str() {
+        "pwd" => AuthDriver::Pwd,
+        _ => return Err(CS::err_invalid_value_for(CS::KEY_AUTH_DRIVER)),
+    };
+    config.auth = Some(DecodedAuth {
+        plugin: auth_plugin,
+        root_pass: root_key.remove(0),
+    });
     Ok(())
 }
 
@@ -576,7 +589,6 @@ fn arg_decode_mode<CS: ConfigurationSource>(
         Some(s) => s.mode = Some(mode),
         None => {
             config.system = Some(DecodedSystemConfig {
-                auth_enabled: None,
                 mode: Some(mode),
                 rs_window: None,
             })
@@ -596,7 +608,6 @@ fn arg_decode_rs_window<CS: ConfigurationSource>(
             Some(sys) => sys.rs_window = Some(n),
             None => {
                 config.system = Some(DecodedSystemConfig {
-                    auth_enabled: None,
                     mode: None,
                     rs_window: Some(n),
                 })
@@ -612,7 +623,7 @@ fn arg_decode_rs_window<CS: ConfigurationSource>(
 */
 
 /// CLI help message
-pub(super) const CLI_HELP: &str ="\
+pub(super) const CLI_HELP: &str = "\
 Usage: skyd [OPTION]...
 
 skyd is the Skytable database server daemon and can be used to serve database requests.
@@ -622,20 +633,23 @@ Flags:
   -v, --version              Display the version number and exit.
 
 Options:
-  --tlscert <path>           Specify the path to the TLS certificate.
-  --tlskey <path>            Define the path to the TLS private key.
-  --endpoint <definition>    Designate an endpoint. Format: protocol@host:port.
-                             This option can be repeated to define multiple endpoints.
-  --service-window <seconds> Establish the time window for the background service in seconds.
-  --auth <plugin_name>       Identify the authentication plugin by name.
-  --mode <dev/prod>          Set the operational mode. Note: This option is mandatory.
+  --tlscert <path>            Specify the path to the TLS certificate.
+  --tlskey <path>             Define the path to the TLS private key.
+  --endpoint <definition>     Designate an endpoint. Format: protocol@host:port.
+                              This option can be repeated to define multiple endpoints.
+  --service-window <seconds>  Establish the time window for the background service in seconds.
+  --auth <plugin_name>        Identify the authentication plugin by name.
+  --mode <dev/prod>           Set the operational mode. Note: This option is mandatory.
+  --auth-plugin <plugin>      Set the auth plugin. `pwd` is a supported option
+  --auth-root-password <pass> Set the root password
 
 Examples:
-  skyd --mode=dev --endpoint=tcp@127.0.0.1:2003
+  skyd --mode=dev --endpoint tcp@127.0.0.1:2003
 
 Notes:
-  Ensure the 'mode' is always provided, as it is essential for the application's correct functioning.
-  When either of `--help` or `--version` is provided, all other options and flags are ignored.
+  - When no mode is provided, `--mode=dev` is defaulted to
+  - When either of `-h` or `-v` is provided, all other options and flags are ignored.
+  - When `--auth-plugin` is provided, you must provide a value for `--auth-root-password`
 
 For further assistance, refer to the official documentation here: https://docs.skytable.org
 ";
@@ -733,8 +747,9 @@ pub fn parse_cli_args<'a, T: 'a + AsRef<str>>(
 
 /// Parse environment variables
 pub fn parse_env_args() -> ConfigResult<Option<ParsedRawArgs>> {
-    const KEYS: [&str; 6] = [
-        CSEnvArgs::KEY_AUTH,
+    const KEYS: [&str; 7] = [
+        CSEnvArgs::KEY_AUTH_DRIVER,
+        CSEnvArgs::KEY_AUTH_ROOT_PASSWORD,
         CSEnvArgs::KEY_ENDPOINTS,
         CSEnvArgs::KEY_RUN_MODE,
         CSEnvArgs::KEY_SERVICE_WINDOW,
@@ -743,7 +758,7 @@ pub fn parse_env_args() -> ConfigResult<Option<ParsedRawArgs>> {
     ];
     let mut ret = HashMap::new();
     for key in KEYS {
-        let var = match get_var(key) {
+        let var = match get_var_from_store(key) {
             Ok(v) => v,
             Err(e) => match e {
                 std::env::VarError::NotPresent => continue,
@@ -785,8 +800,7 @@ fn apply_config_changes<CS: ConfigurationSource>(
     }
     let decode_tasks = [
         // auth
-        DecodeKind::Simple {
-            key: CS::KEY_AUTH,
+        DecodeKind::Complex {
             f: arg_decode_auth::<CS>,
         },
         // mode
@@ -835,9 +849,10 @@ impl CSCommandLine {
     const ARG_CONFIG_FILE: &'static str = "--config";
 }
 impl ConfigurationSource for CSCommandLine {
+    const KEY_AUTH_DRIVER: &'static str = "--auth-plugin";
+    const KEY_AUTH_ROOT_PASSWORD: &'static str = "--auth-root-password";
     const KEY_TLS_CERT: &'static str = "--tlscert";
     const KEY_TLS_KEY: &'static str = "--tlskey";
-    const KEY_AUTH: &'static str = "--auth";
     const KEY_ENDPOINTS: &'static str = "--endpoint";
     const KEY_RUN_MODE: &'static str = "--mode";
     const KEY_SERVICE_WINDOW: &'static str = "--service-window";
@@ -846,9 +861,10 @@ impl ConfigurationSource for CSCommandLine {
 
 pub struct CSEnvArgs;
 impl ConfigurationSource for CSEnvArgs {
+    const KEY_AUTH_DRIVER: &'static str = "SKYDB_AUTH_PLUGIN";
+    const KEY_AUTH_ROOT_PASSWORD: &'static str = "SKYDB_AUTH_ROOT_PASSWORD";
     const KEY_TLS_CERT: &'static str = "SKYDB_TLS_CERT";
     const KEY_TLS_KEY: &'static str = "SKYDB_TLS_KEY";
-    const KEY_AUTH: &'static str = "SKYDB_AUTH";
     const KEY_ENDPOINTS: &'static str = "SKYDB_ENDPOINTS";
     const KEY_RUN_MODE: &'static str = "SKYDB_RUN_MODE";
     const KEY_SERVICE_WINDOW: &'static str = "SKYDB_SERVICE_WINDOW";
@@ -857,9 +873,10 @@ impl ConfigurationSource for CSEnvArgs {
 
 pub struct CSConfigFile;
 impl ConfigurationSource for CSConfigFile {
+    const KEY_AUTH_DRIVER: &'static str = "auth.plugin";
+    const KEY_AUTH_ROOT_PASSWORD: &'static str = "auth.root_password";
     const KEY_TLS_CERT: &'static str = "endpoints.secure.cert";
     const KEY_TLS_KEY: &'static str = "endpoints.secure.key";
-    const KEY_AUTH: &'static str = "system.auth";
     const KEY_ENDPOINTS: &'static str = "endpoints";
     const KEY_RUN_MODE: &'static str = "system.mode";
     const KEY_SERVICE_WINDOW: &'static str = "system.service_window";
@@ -886,14 +903,17 @@ macro_rules! err_if {
 
 /// Validate the configuration, and prepare the final configuration
 fn validate_configuration<CS: ConfigurationSource>(
-    DecodedConfiguration { system, endpoints }: DecodedConfiguration,
+    DecodedConfiguration {
+        system,
+        endpoints,
+        auth,
+    }: DecodedConfiguration,
 ) -> ConfigResult<Configuration> {
     // initialize our default configuration
     let mut config = Configuration::default_dev_mode();
     // mutate
     if_some!(
         system => |system: DecodedSystemConfig| {
-            if_some!(system.auth_enabled => |auth| config.system.auth = auth);
             if_some!(system.mode => |mode| config.mode = mode);
             if_some!(system.rs_window => |window| config.system.reliability_system_window = window);
         }
@@ -911,7 +931,7 @@ fn validate_configuration<CS: ConfigurationSource>(
                         port: secure.port
                     },
                     cert: secure.cert,
-                    private_key: secure.pass
+                    private_key: secure.private_key
                 };
                 match &config.endpoints {
                     ConfigEndpoint::Insecure(is) => if has_insecure {
@@ -926,6 +946,20 @@ fn validate_configuration<CS: ConfigurationSource>(
             })
         }
     );
+    if let Some(auth) = auth {
+        if auth.root_pass.len() < ROOT_PASSWORD_MIN_LEN {
+            return Err(ConfigError::with_src(
+                CS::SOURCE,
+                ConfigErrorKind::ErrorString(format!(
+                    "root password must have atleast {ROOT_PASSWORD_MIN_LEN} characters"
+                )),
+            ));
+        }
+        config.auth = Some(ConfigAuth {
+            plugin: auth.plugin,
+            root_key: auth.root_pass,
+        });
+    }
     // now check a few things
     err_if!(
         if config.system.reliability_system_window == 0 => ConfigError::with_src(
@@ -982,6 +1016,7 @@ pub(super) fn apply_and_validate<CS: ConfigurationSource>(
 thread_local! {
     static CLI_SRC: std::cell::RefCell<Option<Vec<String>>> = std::cell::RefCell::new(None);
     static ENV_SRC: std::cell::RefCell<Option<HashMap<String, String>>> = std::cell::RefCell::new(None);
+    static FILE_SRC: std::cell::RefCell<Option<String>> = std::cell::RefCell::new(None);
 }
 #[cfg(test)]
 pub(super) fn set_cli_src(cli: Vec<String>) {
@@ -1002,7 +1037,26 @@ pub(super) fn set_env_src(variables: Vec<String>) {
         *env.borrow_mut() = Some(variables);
     })
 }
-fn get_var(name: &str) -> Result<String, std::env::VarError> {
+#[cfg(test)]
+pub(super) fn set_file_src(src: &str) {
+    FILE_SRC.with(|s| {
+        s.borrow_mut().replace(src.to_string());
+    })
+}
+fn get_file_from_store(filename: &str) -> ConfigResult<String> {
+    let _f = filename;
+    let f;
+    #[cfg(test)]
+    {
+        f = Ok(FILE_SRC.with(|f| f.borrow().clone().unwrap()));
+    }
+    #[cfg(not(test))]
+    {
+        f = Ok(fs::read_to_string(filename)?);
+    }
+    f
+}
+fn get_var_from_store(name: &str) -> Result<String, std::env::VarError> {
     let var;
     #[cfg(test)]
     {
@@ -1025,7 +1079,7 @@ fn get_var(name: &str) -> Result<String, std::env::VarError> {
     }
     var
 }
-fn get_cli_src() -> Vec<String> {
+fn get_cli_from_store() -> Vec<String> {
     let src;
     #[cfg(test)]
     {
@@ -1048,7 +1102,7 @@ pub fn check_configuration() -> ConfigResult<ConfigReturn> {
     // read in our environment variables
     let env_args = parse_env_args()?;
     // read in our CLI args (since that can tell us whether we need a configuration file)
-    let read_cli_args = parse_cli_args(get_cli_src().into_iter())?;
+    let read_cli_args = parse_cli_args(get_cli_from_store().into_iter())?;
     let cli_args = match read_cli_args {
         CLIConfigParseReturn::Default => {
             // no options were provided in the CLI
@@ -1107,15 +1161,29 @@ fn check_config_file(
         // yes, we only have the config file
         argck_duplicate_values::<CSCommandLine>(&cfg_file, CSCommandLine::ARG_CONFIG_FILE)?;
         // read the config file
-        let file = fs::read_to_string(&cfg_file[0])?;
-        let config_from_file: DecodedConfiguration = serde_yaml::from_str(&file).map_err(|e| {
-            ConfigError::with_src(
-                ConfigSource::File,
-                ConfigErrorKind::ErrorString(format!(
-                    "failed to parse YAML config file with error: `{e}`"
-                )),
-            )
-        })?;
+        let file = get_file_from_store(&cfg_file[0])?;
+        let mut config_from_file: DecodedConfiguration =
+            serde_yaml::from_str(&file).map_err(|e| {
+                ConfigError::with_src(
+                    ConfigSource::File,
+                    ConfigErrorKind::ErrorString(format!(
+                        "failed to parse YAML config file with error: `{e}`"
+                    )),
+                )
+            })?;
+        // read in the TLS certs (if any)
+        match config_from_file.endpoints.as_mut() {
+            Some(ep) => match ep.secure.as_mut() {
+                Some(secure_ep) => {
+                    let cert = fs::read_to_string(&secure_ep.cert)?;
+                    let private_key = fs::read_to_string(&secure_ep.private_key)?;
+                    secure_ep.cert = cert;
+                    secure_ep.private_key = private_key;
+                }
+                None => {}
+            },
+            None => {}
+        }
         // done here
         return validate_configuration::<CSConfigFile>(config_from_file).map(ConfigReturn::Config);
     } else {
diff --git a/server/src/engine/tests/mod.rs b/server/src/engine/tests/mod.rs
index 96c89d9b..5e421df3 100644
--- a/server/src/engine/tests/mod.rs
+++ b/server/src/engine/tests/mod.rs
@@ -27,8 +27,9 @@
 mod cfg {
     use crate::{
         engine::config::{
-            self, CLIConfigParseReturn, ConfigEndpoint, ConfigEndpointTcp, ConfigEndpointTls,
-            ConfigMode, ConfigReturn, ConfigSystem, Configuration, ParsedRawArgs,
+            self, AuthDriver, CLIConfigParseReturn, ConfigAuth, ConfigEndpoint, ConfigEndpointTcp,
+            ConfigEndpointTls, ConfigMode, ConfigReturn, ConfigSystem, Configuration,
+            ParsedRawArgs,
         },
         util::test_utils::with_files,
     };
@@ -98,7 +99,9 @@ mod cfg {
                 --service-window=600 \
                 --tlskey {pkey} \
                 --tlscert {cert} \
-                --auth pwd"
+                --auth-plugin pwd \
+                --auth-root-password password12345678
+                "
                 );
                 let cfg = extract_cli_args(&payload);
                 let ret = config::apply_and_validate::<config::CSCommandLine>(cfg)
@@ -116,7 +119,8 @@ mod cfg {
                             )
                         ),
                         ConfigMode::Dev,
-                        ConfigSystem::new(600, true)
+                        ConfigSystem::new(600),
+                        Some(ConfigAuth::new(AuthDriver::Pwd, "password12345678".into()))
                     )
                 )
             },
@@ -171,7 +175,8 @@ mod cfg {
         let variables = [
             format!("SKYDB_TLS_CERT=/var/skytable/keys/cert.pem"),
             format!("SKYDB_TLS_KEY=/var/skytable/keys/private.key"),
-            format!("SKYDB_AUTH=pwd"),
+            format!("SKYDB_AUTH_PLUGIN=pwd"),
+            format!("SKYDB_AUTH_ROOT_PASSWORD=password12345678"),
             format!("SKYDB_ENDPOINTS=tcp@localhost:8080"),
             format!("SKYDB_RUN_MODE=dev"),
             format!("SKYDB_SERVICE_WINDOW=600"),
@@ -186,7 +191,8 @@ mod cfg {
         let variables = [
             format!("SKYDB_TLS_CERT=/var/skytable/keys/cert.pem"),
             format!("SKYDB_TLS_KEY=/var/skytable/keys/private.key"),
-            format!("SKYDB_AUTH=pwd"),
+            format!("SKYDB_AUTH_PLUGIN=pwd"),
+            format!("SKYDB_AUTH_ROOT_PASSWORD=password12345678"),
             format!("SKYDB_ENDPOINTS=tcp@localhost:8080,tls@localhost:8081"),
             format!("SKYDB_RUN_MODE=dev"),
             format!("SKYDB_SERVICE_WINDOW=600"),
@@ -202,9 +208,10 @@ mod cfg {
             ["__env_args_test_cert.pem", "__env_args_test_private.key"],
             |[cert, key]| {
                 let variables = [
+                    format!("SKYDB_AUTH_PLUGIN=pwd"),
+                    format!("SKYDB_AUTH_ROOT_PASSWORD=password12345678"),
                     format!("SKYDB_TLS_CERT={cert}"),
                     format!("SKYDB_TLS_KEY={key}"),
-                    format!("SKYDB_AUTH=pwd"),
                     format!("SKYDB_ENDPOINTS=tcp@localhost:8080,tls@localhost:8081"),
                     format!("SKYDB_RUN_MODE=dev"),
                     format!("SKYDB_SERVICE_WINDOW=600"),
@@ -223,10 +230,57 @@ mod cfg {
                             )
                         ),
                         ConfigMode::Dev,
-                        ConfigSystem::new(600, true)
+                        ConfigSystem::new(600),
+                        Some(ConfigAuth::new(AuthDriver::Pwd, "password12345678".into()))
                     )
                 )
             },
         );
     }
+    const CONFIG_FILE: &str = "\
+system:
+  mode: dev
+  rs_window: 600
+
+auth:
+  plugin: pwd
+  root_pass: password12345678
+
+endpoints:
+  secure:
+    host: 127.0.0.1
+    port: 2004
+    cert: ._test_sample_cert.pem
+    private_key: ._test_sample_private.key
+  insecure:
+    host: 127.0.0.1
+    port: 2003
+    ";
+    #[test]
+    fn test_config_file() {
+        with_files(
+            ["._test_sample_cert.pem", "._test_sample_private.key"],
+            |_| {
+                config::set_cli_src(vec!["skyd".into(), "--config=config.yml".into()]);
+                config::set_file_src(CONFIG_FILE);
+                let cfg = config::check_configuration().unwrap().into_config();
+                assert_eq!(
+                    cfg,
+                    Configuration::new(
+                        ConfigEndpoint::Multi(
+                            ConfigEndpointTcp::new("127.0.0.1".into(), 2003),
+                            ConfigEndpointTls::new(
+                                ConfigEndpointTcp::new("127.0.0.1".into(), 2004),
+                                "".into(),
+                                "".into()
+                            )
+                        ),
+                        ConfigMode::Dev,
+                        ConfigSystem::new(600),
+                        Some(ConfigAuth::new(AuthDriver::Pwd, "password12345678".into()))
+                    )
+                )
+            },
+        )
+    }
 }

From ef3d71f5931405a797fb9b21b3ae15ad1a5de547 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 26 Sep 2023 10:29:03 +0000
Subject: [PATCH 265/310] Optimize header implementation

The SDSS header impls were way too complex, and unnecessarily linked
to the file system API abstractions. Now we have a more decoupled API
with optional variable headers.
---
 server/src/engine/fractal/config.rs           |  14 +-
 server/src/engine/fractal/mod.rs              |  10 +-
 server/src/engine/fractal/test_utils.rs       |  13 +-
 server/src/engine/storage/header.rs           |   4 +-
 .../src/engine/storage/v1/batch_jrnl/mod.rs   |  29 +-
 .../src/engine/storage/v1/header_impl/dr.rs   | 472 ----------------
 .../src/engine/storage/v1/header_impl/gr.rs   | 490 -----------------
 .../src/engine/storage/v1/header_impl/mod.rs  | 338 ------------
 .../src/engine/storage/v1/header_impl/sr.rs   |  85 ---
 server/src/engine/storage/v1/journal.rs       |  38 +-
 server/src/engine/storage/v1/loader.rs        |  31 +-
 server/src/engine/storage/v1/mod.rs           |   7 +-
 server/src/engine/storage/v1/rw.rs            | 142 +----
 server/src/engine/storage/v1/spec.rs          | 517 ++++++++++++++++++
 server/src/engine/storage/v1/sysdb.rs         |  23 +-
 server/src/engine/storage/v1/tests/batch.rs   |  16 +-
 server/src/engine/storage/v1/tests/rw.rs      |  28 +-
 server/src/engine/storage/v1/tests/tx.rs      |  15 +-
 server/src/engine/storage/versions/mod.rs     |  10 +-
 19 files changed, 590 insertions(+), 1692 deletions(-)
 delete mode 100644 server/src/engine/storage/v1/header_impl/dr.rs
 delete mode 100644 server/src/engine/storage/v1/header_impl/gr.rs
 delete mode 100644 server/src/engine/storage/v1/header_impl/mod.rs
 delete mode 100644 server/src/engine/storage/v1/header_impl/sr.rs
 create mode 100644 server/src/engine/storage/v1/spec.rs

diff --git a/server/src/engine/fractal/config.rs b/server/src/engine/fractal/config.rs
index 6090c532..4eb08478 100644
--- a/server/src/engine/fractal/config.rs
+++ b/server/src/engine/fractal/config.rs
@@ -25,10 +25,7 @@
 */
 
 use {
-    crate::engine::{
-        error::{Error, QueryResult},
-        storage::v1::header_meta::HostRunMode,
-    },
+    crate::engine::error::{Error, QueryResult},
     parking_lot::RwLock,
     std::collections::{hash_map::Entry, HashMap},
 };
@@ -53,7 +50,7 @@ impl SysConfig {
     pub(super) fn test_default() -> Self {
         Self {
             auth_data: RwLock::new(None),
-            host_data: SysHostData::new(0, HostRunMode::Prod, 0),
+            host_data: SysHostData::new(0, 0),
         }
     }
     /// Returns a handle to the authentication data
@@ -70,25 +67,20 @@ impl SysConfig {
 /// The host data section (system.host)
 pub struct SysHostData {
     startup_counter: u64,
-    run_mode: HostRunMode,
     settings_version: u32,
 }
 
 impl SysHostData {
     /// New [`SysHostData`]
-    pub fn new(startup_counter: u64, run_mode: HostRunMode, settings_version: u32) -> Self {
+    pub fn new(startup_counter: u64, settings_version: u32) -> Self {
         Self {
             startup_counter,
-            run_mode,
             settings_version,
         }
     }
     pub fn startup_counter(&self) -> u64 {
         self.startup_counter
     }
-    pub fn run_mode(&self) -> HostRunMode {
-        self.run_mode
-    }
     pub fn settings_version(&self) -> u32 {
         self.settings_version
     }
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index c01e3fcb..72b34aac 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -176,14 +176,10 @@ impl GlobalInstanceLike for Global {
             space_name, space_uuid, model_name, model_uuid,
         ))?;
         // init driver
-        let driver = storage::v1::data_batch::create(
-            &storage::v1::loader::SEInitState::model_path(
+        let driver =
+            storage::v1::data_batch::create(&storage::v1::loader::SEInitState::model_path(
                 space_name, space_uuid, model_name, model_uuid,
-            ),
-            self.sys_cfg().host_data().settings_version(),
-            self.sys_cfg().host_data().run_mode(),
-            self.sys_cfg().host_data().startup_counter(),
-        )?;
+            ))?;
         self.get_state().mdl_driver.write().insert(
             ModelUniqueID::new(space_name, model_name, model_uuid),
             drivers::FractalModelDriver::init(driver),
diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
index 018ee678..5f9c826e 100644
--- a/server/src/engine/fractal/test_utils.rs
+++ b/server/src/engine/fractal/test_utils.rs
@@ -35,7 +35,6 @@ use {
         storage::{
             self,
             v1::{
-                header_meta::HostRunMode,
                 memfs::{NullFS, VirtualFS},
                 RawFSInterface,
             },
@@ -78,7 +77,7 @@ impl<Fs: RawFSInterface> TestGlobal<Fs> {
 impl<Fs: RawFSInterface> TestGlobal<Fs> {
     pub fn new_with_driver_id(log_name: &str) -> Self {
         let gns = GlobalNS::empty();
-        let driver = storage::v1::loader::open_gns_driver(log_name, 0, HostRunMode::Dev, 0, &gns)
+        let driver = storage::v1::loader::open_gns_driver(log_name, &gns)
             .unwrap()
             .into_inner();
         Self::new(gns, 0, GNSTransactionDriverAnyFS::new(driver))
@@ -131,14 +130,10 @@ impl<Fs: RawFSInterface> GlobalInstanceLike for TestGlobal<Fs> {
         Fs::fs_create_dir(&storage::v1::loader::SEInitState::model_dir(
             space_name, space_uuid, model_name, model_uuid,
         ))?;
-        let driver = storage::v1::data_batch::create(
-            &storage::v1::loader::SEInitState::model_path(
+        let driver =
+            storage::v1::data_batch::create(&storage::v1::loader::SEInitState::model_path(
                 space_name, space_uuid, model_name, model_uuid,
-            ),
-            self.sys_cfg().host_data().settings_version(),
-            self.sys_cfg().host_data().run_mode(),
-            self.sys_cfg().host_data().startup_counter(),
-        )?;
+            ))?;
         self.model_drivers.write().insert(
             ModelUniqueID::new(space_name, model_name, model_uuid),
             FractalModelDriver::init(driver),
diff --git a/server/src/engine/storage/header.rs b/server/src/engine/storage/header.rs
index 2cf9371e..129bd893 100644
--- a/server/src/engine/storage/header.rs
+++ b/server/src/engine/storage/header.rs
@@ -370,7 +370,7 @@ impl StaticRecordUVRaw {
         if u64::from_le(slf.data.read_qword(Self::OFFSET_P0)) != SR0_MAGIC {
             return None;
         }
-        let sr1_header_version = HeaderVersion::__new(slf.data.read_dword(Self::OFFSET_P1));
+        let sr1_header_version = HeaderVersion::__new(slf.data.read_dword(Self::OFFSET_P1) as _);
         let sr2_ptr = HostPointerWidth::try_new_with_val(slf.data.read_byte(Self::OFFSET_P2))?; // p2: ptr width
         let sr3_endian = HostEndian::try_new_with_val(slf.data.read_byte(Self::OFFSET_P3))?; // p3: endian
         let sr4_arch = HostArch::try_new_with_val(slf.data.read_byte(Self::OFFSET_P4))?; // p4: arch
@@ -393,7 +393,7 @@ impl StaticRecordUVRaw {
         self.data.read_qword(Self::OFFSET_P0)
     }
     pub const fn read_p1_header_version(&self) -> HeaderVersion {
-        HeaderVersion::__new(self.data.read_dword(Self::OFFSET_P1))
+        HeaderVersion::__new(self.data.read_dword(Self::OFFSET_P1) as _)
     }
     pub const fn read_p2_ptr_width(&self) -> HostPointerWidth {
         HostPointerWidth::new_with_val(self.data.read_byte(Self::OFFSET_P2))
diff --git a/server/src/engine/storage/v1/batch_jrnl/mod.rs b/server/src/engine/storage/v1/batch_jrnl/mod.rs
index b5a3de3c..45f793bd 100644
--- a/server/src/engine/storage/v1/batch_jrnl/mod.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/mod.rs
@@ -45,24 +45,16 @@ pub(super) use restore::{DecodedBatchEvent, DecodedBatchEventKind, NormalBatch};
 pub use {persist::DataBatchPersistDriver, restore::DataBatchRestoreDriver};
 
 use {
-    super::{header_meta, rw::SDSSFileIO, RawFSInterface, SDSSResult},
+    super::{rw::SDSSFileIO, spec, RawFSInterface, SDSSResult},
     crate::engine::core::model::Model,
 };
 
-const LOG_SPECIFIER_VERSION: header_meta::FileSpecifierVersion =
-    header_meta::FileSpecifierVersion::__new(0);
-
 /// Re-initialize an existing batch journal and read all its data into model
 pub fn reinit<Fs: RawFSInterface>(
     name: &str,
     model: &Model,
 ) -> SDSSResult<DataBatchPersistDriver<Fs>> {
-    let (_header, f) = SDSSFileIO::<Fs>::open::<false>(
-        name,
-        header_meta::FileScope::Journal,
-        header_meta::FileSpecifier::TableDataBatch,
-        LOG_SPECIFIER_VERSION,
-    )?;
+    let (f, _header) = SDSSFileIO::<Fs>::open::<spec::DataBatchJournalV1>(name)?;
     // restore
     let mut restore_driver = DataBatchRestoreDriver::new(f)?;
     restore_driver.read_data_batch_into_model(model)?;
@@ -70,20 +62,7 @@ pub fn reinit<Fs: RawFSInterface>(
 }
 
 /// Create a new batch journal
-pub fn create<Fs: RawFSInterface>(
-    path: &str,
-    host_setting_version: u32,
-    host_run_mode: header_meta::HostRunMode,
-    host_startup_counter: u64,
-) -> SDSSResult<DataBatchPersistDriver<Fs>> {
-    let f = SDSSFileIO::<Fs>::create(
-        path,
-        header_meta::FileScope::Journal,
-        header_meta::FileSpecifier::TableDataBatch,
-        LOG_SPECIFIER_VERSION,
-        host_setting_version,
-        host_run_mode,
-        host_startup_counter,
-    )?;
+pub fn create<Fs: RawFSInterface>(path: &str) -> SDSSResult<DataBatchPersistDriver<Fs>> {
+    let f = SDSSFileIO::<Fs>::create::<spec::DataBatchJournalV1>(path)?;
     DataBatchPersistDriver::new(f, true)
 }
diff --git a/server/src/engine/storage/v1/header_impl/dr.rs b/server/src/engine/storage/v1/header_impl/dr.rs
deleted file mode 100644
index 8ec3cff8..00000000
--- a/server/src/engine/storage/v1/header_impl/dr.rs
+++ /dev/null
@@ -1,472 +0,0 @@
-/*
- * Created on Thu May 25 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{
-    engine::{
-        mem::ByteStack,
-        storage::{
-            header::{HostArch, HostEndian, HostOS, HostPointerWidth},
-            v1::{header_impl::FileSpecifierVersion, SDSSError, SDSSResult},
-            versions::{self, DriverVersion, ServerVersion},
-        },
-    },
-    util,
-};
-
-/*
-    Dynamic record (1/2): Host signature
-    ---
-    - 8B: Server version
-    - 8B: Driver version
-    - 4B: File specifier ID
-    - 1B: Endian
-    - 1B: Pointer width
-    - 1B: Arch
-    - 1B: OS
-*/
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct DRHostSignature {
-    server_version: ServerVersion,
-    driver_version: DriverVersion,
-    file_specifier_version: FileSpecifierVersion,
-    endian: HostEndian,
-    ptr_width: HostPointerWidth,
-    arch: HostArch,
-    os: HostOS,
-}
-
-impl DRHostSignature {
-    pub fn verify(&self, expected_file_specifier_version: FileSpecifierVersion) -> SDSSResult<()> {
-        if self.server_version() != versions::v1::V1_SERVER_VERSION {
-            return Err(SDSSError::HeaderDecodeServerVersionMismatch);
-        }
-        if self.driver_version() != versions::v1::V1_DRIVER_VERSION {
-            return Err(SDSSError::HeaderDecodeDriverVersionMismatch);
-        }
-        if self.file_specifier_version() != expected_file_specifier_version {
-            return Err(SDSSError::HeaderDecodeDataMismatch);
-        }
-        Ok(())
-    }
-}
-
-impl DRHostSignature {
-    /// Decode the [`DRHostSignature`] from the given bytes
-    ///
-    /// **☢ WARNING ☢: This only decodes; it doesn't validate expected values!**
-    pub fn decode_noverify(bytes: [u8; sizeof!(DRHostSignatureRaw)]) -> Option<Self> {
-        let ns = ByteStack::new(bytes);
-        let server_version = ServerVersion::__new(u64::from_le(
-            ns.read_qword(DRHostSignatureRaw::DRHS_OFFSET_P0),
-        ));
-        let driver_version = DriverVersion::__new(u64::from_le(
-            ns.read_qword(DRHostSignatureRaw::DRHS_OFFSET_P1),
-        ));
-        let file_specifier_id = FileSpecifierVersion::__new(u32::from_le(
-            ns.read_dword(DRHostSignatureRaw::DRHS_OFFSET_P2),
-        ));
-        let endian =
-            HostEndian::try_new_with_val(ns.read_byte(DRHostSignatureRaw::DRHS_OFFSET_P3))?;
-        let ptr_width =
-            HostPointerWidth::try_new_with_val(ns.read_byte(DRHostSignatureRaw::DRHS_OFFSET_P4))?;
-        let arch = HostArch::try_new_with_val(ns.read_byte(DRHostSignatureRaw::DRHS_OFFSET_P5))?;
-        let os = HostOS::try_new_with_val(ns.read_byte(DRHostSignatureRaw::DRHS_OFFSET_P6))?;
-        Some(Self::new(
-            server_version,
-            driver_version,
-            file_specifier_id,
-            endian,
-            ptr_width,
-            arch,
-            os,
-        ))
-    }
-}
-
-impl DRHostSignature {
-    pub const fn new(
-        server_version: ServerVersion,
-        driver_version: DriverVersion,
-        file_specifier_version: FileSpecifierVersion,
-        endian: HostEndian,
-        ptr_width: HostPointerWidth,
-        arch: HostArch,
-        os: HostOS,
-    ) -> Self {
-        Self {
-            server_version,
-            driver_version,
-            file_specifier_version,
-            endian,
-            ptr_width,
-            arch,
-            os,
-        }
-    }
-    pub const fn server_version(&self) -> ServerVersion {
-        self.server_version
-    }
-    pub const fn driver_version(&self) -> DriverVersion {
-        self.driver_version
-    }
-    pub const fn file_specifier_version(&self) -> FileSpecifierVersion {
-        self.file_specifier_version
-    }
-    pub const fn endian(&self) -> HostEndian {
-        self.endian
-    }
-    pub const fn ptr_width(&self) -> HostPointerWidth {
-        self.ptr_width
-    }
-    pub const fn arch(&self) -> HostArch {
-        self.arch
-    }
-    pub const fn os(&self) -> HostOS {
-        self.os
-    }
-    pub const fn encoded(&self) -> DRHostSignatureRaw {
-        DRHostSignatureRaw::new_full(
-            self.server_version(),
-            self.driver_version(),
-            self.file_specifier_version(),
-            self.endian(),
-            self.ptr_width(),
-            self.arch(),
-            self.os(),
-        )
-    }
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct DRHostSignatureRaw {
-    pub(super) data: ByteStack<24>,
-}
-
-impl DRHostSignatureRaw {
-    const DRHS_OFFSET_P0: usize = 0;
-    const DRHS_OFFSET_P1: usize = sizeof!(u64);
-    const DRHS_OFFSET_P2: usize = Self::DRHS_OFFSET_P1 + sizeof!(u64);
-    const DRHS_OFFSET_P3: usize = Self::DRHS_OFFSET_P2 + sizeof!(u32);
-    const DRHS_OFFSET_P4: usize = Self::DRHS_OFFSET_P3 + 1;
-    const DRHS_OFFSET_P5: usize = Self::DRHS_OFFSET_P4 + 1;
-    const DRHS_OFFSET_P6: usize = Self::DRHS_OFFSET_P5 + 1;
-    const _ENSURE: () = assert!(Self::DRHS_OFFSET_P6 == sizeof!(Self) - 1);
-    pub const fn new_auto(file_specifier_version: FileSpecifierVersion) -> Self {
-        Self::new(
-            versions::v1::V1_SERVER_VERSION,
-            versions::v1::V1_DRIVER_VERSION,
-            file_specifier_version,
-        )
-    }
-    pub const fn new(
-        server_version: ServerVersion,
-        driver_version: DriverVersion,
-        file_specifier_id: FileSpecifierVersion,
-    ) -> Self {
-        Self::new_full(
-            server_version,
-            driver_version,
-            file_specifier_id,
-            HostEndian::new(),
-            HostPointerWidth::new(),
-            HostArch::new(),
-            HostOS::new(),
-        )
-    }
-    pub const fn new_full(
-        server_version: ServerVersion,
-        driver_version: DriverVersion,
-        file_specifier_id: FileSpecifierVersion,
-        endian: HostEndian,
-        ptr_width: HostPointerWidth,
-        arch: HostArch,
-        os: HostOS,
-    ) -> Self {
-        let _ = Self::_ENSURE;
-        let bytes: [u8; 24] = unsafe {
-            let [qw_a, qw_b]: [u64; 2] = core::mem::transmute([
-                server_version.little_endian(),
-                driver_version.little_endian(),
-            ]);
-            let dw: u32 = core::mem::transmute([
-                endian.value_u8(),
-                ptr_width.value_u8(),
-                arch.value_u8(),
-                os.value_u8(),
-            ]);
-            let qw_c: u64 = core::mem::transmute([(file_specifier_id.0.to_le(), dw.to_le())]);
-            core::mem::transmute([qw_a, qw_b, qw_c])
-        };
-        Self {
-            data: ByteStack::new(bytes),
-        }
-    }
-}
-
-impl DRHostSignatureRaw {
-    pub const fn read_p0_server_version(&self) -> ServerVersion {
-        ServerVersion::__new(self.data.read_qword(Self::DRHS_OFFSET_P0))
-    }
-    pub const fn read_p1_driver_version(&self) -> DriverVersion {
-        DriverVersion::__new(self.data.read_qword(Self::DRHS_OFFSET_P1))
-    }
-    pub const fn read_p2_file_specifier_id(&self) -> FileSpecifierVersion {
-        FileSpecifierVersion::__new(self.data.read_dword(Self::DRHS_OFFSET_P2))
-    }
-    pub const fn read_p3_endian(&self) -> HostEndian {
-        HostEndian::new_with_val(self.data.read_byte(Self::DRHS_OFFSET_P3))
-    }
-    pub const fn read_p4_pointer_width(&self) -> HostPointerWidth {
-        HostPointerWidth::new_with_val(self.data.read_byte(Self::DRHS_OFFSET_P4))
-    }
-    pub const fn read_p5_arch(&self) -> HostArch {
-        HostArch::new_with_val(self.data.read_byte(Self::DRHS_OFFSET_P5))
-    }
-    pub const fn read_p6_os(&self) -> HostOS {
-        HostOS::new_with_val(self.data.read_byte(Self::DRHS_OFFSET_P6))
-    }
-    pub const fn decoded(&self) -> DRHostSignature {
-        DRHostSignature::new(
-            self.read_p0_server_version(),
-            self.read_p1_driver_version(),
-            self.read_p2_file_specifier_id(),
-            self.read_p3_endian(),
-            self.read_p4_pointer_width(),
-            self.read_p5_arch(),
-            self.read_p6_os(),
-        )
-    }
-}
-
-/*
-    Dynamic record (2/2): Runtime signature
-    ---
-    - 8B: Dynamic record modify count
-    - 16B: Host epoch time
-    - 16B: Host uptime
-    - 1B: Host name length
-    - 255B: Host name (nulled)
-    = 296B
-*/
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct DRRuntimeSignature {
-    modify_count: u64,
-    epoch_time: u128,
-    host_uptime: u128,
-    host_name_length: u8,
-    host_name_raw: [u8; 255],
-}
-
-impl DRRuntimeSignature {
-    pub fn verify(&self) -> SDSSResult<()> {
-        let et = util::os::get_epoch_time();
-        if self.epoch_time() > et || self.host_uptime() > et {
-            // a file from the future?
-            return Err(SDSSError::HeaderTimeConflict);
-        }
-        Ok(())
-    }
-    pub fn decode_noverify(bytes: [u8; sizeof!(DRRuntimeSignatureRaw)]) -> Option<Self> {
-        let bytes = ByteStack::new(bytes);
-        // check
-        let modify_count = u64::from_le(bytes.read_qword(DRRuntimeSignatureRaw::DRRS_OFFSET_P0));
-        let epoch_time = u128::from_le(bytes.read_xmmword(DRRuntimeSignatureRaw::DRRS_OFFSET_P1));
-        let host_uptime = u128::from_le(bytes.read_xmmword(DRRuntimeSignatureRaw::DRRS_OFFSET_P2));
-        let host_name_length = bytes.read_byte(DRRuntimeSignatureRaw::DRRS_OFFSET_P3);
-        let host_name_raw =
-            util::copy_slice_to_array(&bytes.slice()[DRRuntimeSignatureRaw::DRRS_OFFSET_P4..]);
-        if cfg!(debug_assertions) {
-            assert_eq!(
-                255 - host_name_raw.iter().filter(|b| **b == 0u8).count(),
-                host_name_length as _
-            );
-        }
-        Some(Self {
-            modify_count,
-            epoch_time,
-            host_uptime,
-            host_name_length,
-            host_name_raw,
-        })
-    }
-}
-
-impl DRRuntimeSignature {
-    pub const fn new(
-        modify_count: u64,
-        epoch_time: u128,
-        host_uptime: u128,
-        host_name_length: u8,
-        host_name_raw: [u8; 255],
-    ) -> Self {
-        Self {
-            modify_count,
-            epoch_time,
-            host_uptime,
-            host_name_length,
-            host_name_raw,
-        }
-    }
-    pub const fn modify_count(&self) -> u64 {
-        self.modify_count
-    }
-    pub const fn epoch_time(&self) -> u128 {
-        self.epoch_time
-    }
-    pub const fn host_uptime(&self) -> u128 {
-        self.host_uptime
-    }
-    pub const fn host_name_length(&self) -> u8 {
-        self.host_name_length
-    }
-    pub const fn host_name_raw(&self) -> [u8; 255] {
-        self.host_name_raw
-    }
-    pub fn host_name(&self) -> &[u8] {
-        &self.host_name_raw[..self.host_name_length() as usize]
-    }
-    pub fn encoded(&self) -> DRRuntimeSignatureRaw {
-        DRRuntimeSignatureRaw::new(
-            self.modify_count(),
-            self.epoch_time(),
-            self.host_uptime(),
-            self.host_name_length(),
-            self.host_name_raw(),
-        )
-    }
-    pub fn set_modify_count(&mut self, new: u64) {
-        self.modify_count = new;
-    }
-    pub fn bump_modify_count(&mut self) {
-        self.modify_count += 1;
-    }
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct DRRuntimeSignatureRaw {
-    pub(super) data: ByteStack<296>,
-}
-
-impl DRRuntimeSignatureRaw {
-    const DRRS_OFFSET_P0: usize = 0;
-    const DRRS_OFFSET_P1: usize = sizeof!(u64);
-    const DRRS_OFFSET_P2: usize = Self::DRRS_OFFSET_P1 + sizeof!(u128);
-    const DRRS_OFFSET_P3: usize = Self::DRRS_OFFSET_P2 + sizeof!(u128);
-    const DRRS_OFFSET_P4: usize = Self::DRRS_OFFSET_P3 + 1;
-    const _ENSURE: () = assert!(Self::DRRS_OFFSET_P4 == sizeof!(Self) - 255);
-    pub fn new_auto(modify_count: u64) -> Self {
-        let hostname = crate::util::os::get_hostname();
-        Self::new(
-            modify_count,
-            crate::util::os::get_epoch_time(),
-            crate::util::os::get_uptime(),
-            hostname.len(),
-            hostname.raw(),
-        )
-    }
-    pub fn new(
-        modify_count: u64,
-        host_epoch_time: u128,
-        host_uptime: u128,
-        host_name_length: u8,
-        host_name: [u8; 255],
-    ) -> Self {
-        let _ = Self::_ENSURE;
-        let mut data = [0u8; 296];
-        data[Self::DRRS_OFFSET_P0..Self::DRRS_OFFSET_P1]
-            .copy_from_slice(&modify_count.to_le_bytes());
-        data[Self::DRRS_OFFSET_P1..Self::DRRS_OFFSET_P2]
-            .copy_from_slice(&host_epoch_time.to_le_bytes());
-        data[Self::DRRS_OFFSET_P2..Self::DRRS_OFFSET_P3]
-            .copy_from_slice(&host_uptime.to_le_bytes());
-        data[Self::DRRS_OFFSET_P3] = host_name_length;
-        data[Self::DRRS_OFFSET_P4..].copy_from_slice(&host_name);
-        Self {
-            data: ByteStack::new(data),
-        }
-    }
-    pub fn decoded(&self) -> DRRuntimeSignature {
-        DRRuntimeSignature::new(
-            self.read_p0_modify_count(),
-            self.read_p1_epoch_time(),
-            self.read_p2_uptime(),
-            self.read_p3_host_name_length() as _,
-            util::copy_slice_to_array(self.read_p4_host_name_raw_null()),
-        )
-    }
-    pub const fn read_p0_modify_count(&self) -> u64 {
-        self.data.read_qword(Self::DRRS_OFFSET_P0)
-    }
-    pub const fn read_p1_epoch_time(&self) -> u128 {
-        self.data.read_xmmword(Self::DRRS_OFFSET_P1)
-    }
-    pub const fn read_p2_uptime(&self) -> u128 {
-        self.data.read_xmmword(Self::DRRS_OFFSET_P2)
-    }
-    pub const fn read_p3_host_name_length(&self) -> usize {
-        self.data.read_byte(Self::DRRS_OFFSET_P3) as _
-    }
-    pub fn read_p4_host_name_raw_null(&self) -> &[u8] {
-        &self.data.slice()[Self::DRRS_OFFSET_P4..]
-    }
-    pub fn read_host_name(&self) -> &[u8] {
-        &self.data.slice()
-            [Self::DRRS_OFFSET_P4..Self::DRRS_OFFSET_P4 + self.read_p3_host_name_length()]
-    }
-}
-
-#[test]
-fn test_dr_host_signature_encode_decode() {
-    const TARGET: DRHostSignature = DRHostSignature::new(
-        crate::engine::storage::versions::v1::V1_SERVER_VERSION,
-        crate::engine::storage::versions::v1::V1_DRIVER_VERSION,
-        FileSpecifierVersion::__new(u32::MAX - 3),
-        HostEndian::new(),
-        HostPointerWidth::new(),
-        HostArch::new(),
-        HostOS::new(),
-    );
-    let encoded = TARGET.encoded();
-    let decoded = encoded.decoded();
-    assert_eq!(decoded, TARGET);
-}
-
-#[test]
-fn test_dr_runtime_signature_encoded_decode() {
-    const TARGET: DRRuntimeSignature = DRRuntimeSignature::new(
-        u64::MAX - 3,
-        u128::MAX - u32::MAX as u128,
-        u128::MAX - u32::MAX as u128,
-        "skycloud".len() as _,
-        util::copy_str_to_array("skycloud"),
-    );
-    let encoded = TARGET.encoded();
-    let decoded = encoded.decoded();
-    assert_eq!(decoded, TARGET);
-    assert_eq!(decoded.host_name(), b"skycloud");
-}
diff --git a/server/src/engine/storage/v1/header_impl/gr.rs b/server/src/engine/storage/v1/header_impl/gr.rs
deleted file mode 100644
index e7a0a076..00000000
--- a/server/src/engine/storage/v1/header_impl/gr.rs
+++ /dev/null
@@ -1,490 +0,0 @@
-/*
- * Created on Thu May 25 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{
-    engine::{
-        mem::ByteStack,
-        storage::{
-            v1::{
-                header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
-                SDSSError, SDSSResult,
-            },
-            versions::{self, DriverVersion, ServerVersion},
-        },
-    },
-    util,
-};
-
-/*
-    Genesis record (1/2)
-    ---
-    Metadata record (8B x 3 + (4B x 2)):
-    +----------+----------+----------+---------+
-    |  Server  |  Driver  |   File   |File|Spec|
-    |  version |  Version |   Scope  |Spec|ID  |
-    +----------+----------+----------+---------+
-    0, 63
-*/
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct GRMetadataRecord {
-    server_version: ServerVersion,
-    driver_version: DriverVersion,
-    file_scope: FileScope,
-    file_spec: FileSpecifier,
-    file_spec_id: FileSpecifierVersion,
-}
-
-impl GRMetadataRecord {
-    pub fn verify(
-        &self,
-        expected_file_scope: FileScope,
-        expected_file_specifier: FileSpecifier,
-        expected_file_specifier_version: FileSpecifierVersion,
-    ) -> SDSSResult<()> {
-        if self.server_version() != versions::v1::V1_SERVER_VERSION {
-            return Err(SDSSError::HeaderDecodeServerVersionMismatch);
-        }
-        if self.driver_version() != versions::v1::V1_DRIVER_VERSION {
-            return Err(SDSSError::HeaderDecodeDriverVersionMismatch);
-        }
-        let okay = self.file_scope() == expected_file_scope
-            && self.file_spec() == expected_file_specifier
-            && self.file_spec_id() == expected_file_specifier_version;
-        if okay {
-            Ok(())
-        } else {
-            Err(SDSSError::HeaderDecodeDataMismatch)
-        }
-    }
-}
-
-impl GRMetadataRecord {
-    pub const fn new(
-        server_version: ServerVersion,
-        driver_version: DriverVersion,
-        file_scope: FileScope,
-        file_spec: FileSpecifier,
-        file_spec_id: FileSpecifierVersion,
-    ) -> Self {
-        Self {
-            server_version,
-            driver_version,
-            file_scope,
-            file_spec,
-            file_spec_id,
-        }
-    }
-    pub const fn server_version(&self) -> ServerVersion {
-        self.server_version
-    }
-    pub const fn driver_version(&self) -> DriverVersion {
-        self.driver_version
-    }
-    pub const fn file_scope(&self) -> FileScope {
-        self.file_scope
-    }
-    pub const fn file_spec(&self) -> FileSpecifier {
-        self.file_spec
-    }
-    pub const fn file_spec_id(&self) -> FileSpecifierVersion {
-        self.file_spec_id
-    }
-    pub const fn encoded(&self) -> GRMetadataRecordRaw {
-        GRMetadataRecordRaw::new_full(
-            self.server_version(),
-            self.driver_version(),
-            self.file_scope(),
-            self.file_spec(),
-            self.file_spec_id(),
-        )
-    }
-}
-
-#[derive(Clone)]
-pub struct GRMetadataRecordRaw {
-    pub(super) data: ByteStack<32>,
-}
-
-impl GRMetadataRecordRaw {
-    /// Decodes a given metadata record, validating all data for correctness.
-    ///
-    /// **☢ WARNING ☢: This only decodes; it doesn't validate expected values!**
-    pub fn decode_noverify(data: [u8; 32]) -> Option<GRMetadataRecord> {
-        let data = ByteStack::new(data);
-        let server_version =
-            ServerVersion::__new(u64::from_le(data.read_qword(Self::MDR_OFFSET_P0)));
-        let driver_version =
-            DriverVersion::__new(u64::from_le(data.read_qword(Self::MDR_OFFSET_P1)));
-        let file_scope = FileScope::try_new(u64::from_le(data.read_qword(Self::MDR_OFFSET_P2)))?;
-        let file_spec = FileSpecifier::try_new(u32::from_le(data.read_dword(Self::MDR_OFFSET_P3)))?;
-        let file_spec_id =
-            FileSpecifierVersion::__new(u32::from_le(data.read_dword(Self::MDR_OFFSET_P4)));
-        Some(GRMetadataRecord::new(
-            server_version,
-            driver_version,
-            file_scope,
-            file_spec,
-            file_spec_id,
-        ))
-    }
-}
-
-impl GRMetadataRecordRaw {
-    const MDR_OFFSET_P0: usize = 0;
-    const MDR_OFFSET_P1: usize = sizeof!(u64);
-    const MDR_OFFSET_P2: usize = Self::MDR_OFFSET_P1 + sizeof!(u64);
-    const MDR_OFFSET_P3: usize = Self::MDR_OFFSET_P2 + sizeof!(u64);
-    const MDR_OFFSET_P4: usize = Self::MDR_OFFSET_P3 + sizeof!(u32);
-    const _ENSURE: () = assert!(Self::MDR_OFFSET_P4 == (sizeof!(Self) - sizeof!(u32)));
-    pub const fn empty_buffer() -> [u8; sizeof!(Self)] {
-        [0u8; sizeof!(Self)]
-    }
-    pub const fn new_auto(
-        scope: FileScope,
-        specifier: FileSpecifier,
-        specifier_id: FileSpecifierVersion,
-    ) -> Self {
-        Self::new_full(
-            versions::v1::V1_SERVER_VERSION,
-            versions::v1::V1_DRIVER_VERSION,
-            scope,
-            specifier,
-            specifier_id,
-        )
-    }
-    pub const fn new_full(
-        server_version: ServerVersion,
-        driver_version: DriverVersion,
-        scope: FileScope,
-        specifier: FileSpecifier,
-        specifier_id: FileSpecifierVersion,
-    ) -> Self {
-        let _ = Self::_ENSURE;
-        let mut ret = [0u8; 32];
-        let mut i = 0;
-        // read buf
-        let server_version = server_version.little_endian();
-        let driver_version = driver_version.little_endian();
-        let file_scope = scope.value_qword().to_le_bytes();
-        // specifier + specifier ID
-        let file_specifier_and_id: u64 = unsafe {
-            core::mem::transmute([
-                (specifier.value_u8() as u32).to_le(),
-                specifier_id.0.to_le(),
-            ])
-        };
-        let file_specifier_and_id = file_specifier_and_id.to_le_bytes();
-        while i < sizeof!(u64) {
-            ret[i] = server_version[i];
-            ret[i + sizeof!(u64, 1)] = driver_version[i];
-            ret[i + sizeof!(u64, 2)] = file_scope[i];
-            ret[i + sizeof!(u64, 3)] = file_specifier_and_id[i];
-            i += 1;
-        }
-        Self {
-            data: ByteStack::new(ret),
-        }
-    }
-    pub const fn new(
-        scope: FileScope,
-        specifier: FileSpecifier,
-        specifier_id: FileSpecifierVersion,
-    ) -> Self {
-        Self::new_full(
-            versions::v1::V1_SERVER_VERSION,
-            versions::v1::V1_DRIVER_VERSION,
-            scope,
-            specifier,
-            specifier_id,
-        )
-    }
-}
-
-impl GRMetadataRecordRaw {
-    pub const fn read_p0_server_version(&self) -> ServerVersion {
-        ServerVersion::__new(self.data.read_qword(Self::MDR_OFFSET_P0))
-    }
-    pub const fn read_p1_driver_version(&self) -> DriverVersion {
-        DriverVersion::__new(self.data.read_qword(Self::MDR_OFFSET_P1))
-    }
-    pub const fn read_p2_file_scope(&self) -> FileScope {
-        FileScope::new(self.data.read_qword(Self::MDR_OFFSET_P2))
-    }
-    pub const fn read_p3_file_spec(&self) -> FileSpecifier {
-        FileSpecifier::new(self.data.read_dword(Self::MDR_OFFSET_P3))
-    }
-    pub const fn read_p4_file_spec_version(&self) -> FileSpecifierVersion {
-        FileSpecifierVersion(self.data.read_dword(Self::MDR_OFFSET_P4))
-    }
-}
-
-/*
-    Genesis Record (2/2)
-    ---
-    Host record (?B; > 56B):
-    - 16B: Host epoch time in nanoseconds
-    - 16B: Host uptime in nanoseconds
-    - 08B:
-     - 04B: Host setting version ID
-     - 04B: Host run mode
-    - 08B: Host startup counter
-    - 01B: Host name length
-    - 255B: Host name
-    = 304B
-*/
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct GRHostRecord {
-    epoch_time: u128,
-    uptime: u128,
-    setting_version: u32,
-    run_mode: HostRunMode,
-    startup_counter: u64,
-    hostname_len: u8,
-    hostname_raw: [u8; 255],
-}
-
-impl GRHostRecord {
-    /// Verified: N/A
-    /// To verify: N/A
-    pub fn verify(&self) -> SDSSResult<()> {
-        Ok(())
-    }
-}
-
-impl GRHostRecord {
-    pub fn decode_noverify(bytes: [u8; sizeof!(GRHostRecordRaw)]) -> Option<Self> {
-        let ns = ByteStack::new(bytes);
-        let epoch_time = u128::from_le(ns.read_xmmword(GRHostRecordRaw::GRHR_OFFSET_P0));
-        let uptime = u128::from_le(ns.read_xmmword(GRHostRecordRaw::GRHR_OFFSET_P1));
-        let setting_version = u32::from_le(ns.read_dword(GRHostRecordRaw::GRHR_OFFSET_P2));
-        let run_mode = HostRunMode::try_new_with_val(u32::from_le(
-            ns.read_dword(GRHostRecordRaw::GRHR_OFFSET_P3),
-        ))?;
-        let startup_counter = u64::from_le(ns.read_qword(GRHostRecordRaw::GRHR_OFFSET_P4));
-        let host_name_len = ns.read_byte(GRHostRecordRaw::GRHR_OFFSET_P5);
-        let host_name_raw =
-            util::copy_slice_to_array(&ns.slice()[GRHostRecordRaw::GRHR_OFFSET_P6..]);
-        Some(Self::new(
-            epoch_time,
-            uptime,
-            setting_version,
-            run_mode,
-            startup_counter,
-            host_name_len,
-            host_name_raw,
-        ))
-    }
-}
-
-impl GRHostRecord {
-    pub const fn new(
-        epoch_time: u128,
-        uptime: u128,
-        setting_version: u32,
-        run_mode: HostRunMode,
-        startup_counter: u64,
-        hostname_len: u8,
-        hostname: [u8; 255],
-    ) -> Self {
-        Self {
-            epoch_time,
-            uptime,
-            setting_version,
-            run_mode,
-            startup_counter,
-            hostname_len,
-            hostname_raw: hostname,
-        }
-    }
-    pub fn epoch_time(&self) -> u128 {
-        self.epoch_time
-    }
-    pub fn uptime(&self) -> u128 {
-        self.uptime
-    }
-    pub fn setting_version(&self) -> u32 {
-        self.setting_version
-    }
-    pub fn run_mode(&self) -> HostRunMode {
-        self.run_mode
-    }
-    pub fn startup_counter(&self) -> u64 {
-        self.startup_counter
-    }
-    pub fn hostname_len(&self) -> u8 {
-        self.hostname_len
-    }
-    pub fn hostname_raw(&self) -> [u8; 255] {
-        self.hostname_raw
-    }
-    pub fn encoded(&self) -> GRHostRecordRaw {
-        GRHostRecordRaw::new(
-            self.epoch_time(),
-            self.uptime(),
-            self.setting_version(),
-            self.run_mode(),
-            self.startup_counter(),
-            self.hostname_len(),
-            self.hostname_raw(),
-        )
-    }
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct GRHostRecordRaw {
-    pub(super) data: ByteStack<304>,
-}
-
-impl GRHostRecordRaw {
-    const GRHR_OFFSET_P0: usize = 0;
-    const GRHR_OFFSET_P1: usize = sizeof!(u128);
-    const GRHR_OFFSET_P2: usize = Self::GRHR_OFFSET_P1 + sizeof!(u128);
-    const GRHR_OFFSET_P3: usize = Self::GRHR_OFFSET_P2 + sizeof!(u32);
-    const GRHR_OFFSET_P4: usize = Self::GRHR_OFFSET_P3 + sizeof!(u32);
-    const GRHR_OFFSET_P5: usize = Self::GRHR_OFFSET_P4 + sizeof!(u64);
-    const GRHR_OFFSET_P6: usize = Self::GRHR_OFFSET_P5 + 1;
-    const _ENSURE: () = assert!(Self::GRHR_OFFSET_P6 == sizeof!(Self) - 255);
-    pub fn new_auto(setting_version: u32, run_mode: HostRunMode, startup_counter: u64) -> Self {
-        let hostname = crate::util::os::get_hostname();
-        Self::new(
-            crate::util::os::get_epoch_time(),
-            crate::util::os::get_uptime(),
-            setting_version,
-            run_mode,
-            startup_counter,
-            hostname.len(),
-            hostname.raw(),
-        )
-    }
-    pub fn new(
-        p0_epoch_time: u128,
-        p1_uptime: u128,
-        p2_setting_version: u32,
-        p3_run_mode: HostRunMode,
-        p4_host_startup_counter: u64,
-        p5_host_name_length: u8,
-        p6_host_name_raw: [u8; 255],
-    ) -> Self {
-        let _ = Self::_ENSURE;
-        let mut data = [0u8; sizeof!(Self)];
-        data[Self::GRHR_OFFSET_P0..Self::GRHR_OFFSET_P1]
-            .copy_from_slice(&p0_epoch_time.to_le_bytes());
-        data[Self::GRHR_OFFSET_P1..Self::GRHR_OFFSET_P2].copy_from_slice(&p1_uptime.to_le_bytes());
-        data[Self::GRHR_OFFSET_P2..Self::GRHR_OFFSET_P3]
-            .copy_from_slice(&p2_setting_version.to_le_bytes());
-        data[Self::GRHR_OFFSET_P3..Self::GRHR_OFFSET_P4]
-            .copy_from_slice(&(p3_run_mode.value_u8() as u32).to_le_bytes());
-        data[Self::GRHR_OFFSET_P4..Self::GRHR_OFFSET_P5]
-            .copy_from_slice(&p4_host_startup_counter.to_le_bytes());
-        data[Self::GRHR_OFFSET_P5] = p5_host_name_length;
-        data[Self::GRHR_OFFSET_P6..].copy_from_slice(&p6_host_name_raw);
-        Self {
-            data: ByteStack::new(data),
-        }
-    }
-    pub const fn read_p0_epoch_time(&self) -> u128 {
-        self.data.read_xmmword(Self::GRHR_OFFSET_P0)
-    }
-    pub const fn read_p1_uptime(&self) -> u128 {
-        self.data.read_xmmword(Self::GRHR_OFFSET_P1)
-    }
-    pub const fn read_p2_setting_version_id(&self) -> u32 {
-        self.data.read_dword(Self::GRHR_OFFSET_P2)
-    }
-    pub const fn read_p3_run_mode(&self) -> HostRunMode {
-        HostRunMode::new_with_val(self.data.read_dword(Self::GRHR_OFFSET_P3))
-    }
-    pub const fn read_p4_startup_counter(&self) -> u64 {
-        self.data.read_qword(Self::GRHR_OFFSET_P4)
-    }
-    pub const fn read_p5_host_name_length(&self) -> usize {
-        self.data.read_byte(Self::GRHR_OFFSET_P5) as _
-    }
-    pub fn read_p6_host_name_raw(&self) -> &[u8] {
-        &self.data.slice()[Self::GRHR_OFFSET_P6..]
-    }
-    pub fn read_host_name(&self) -> &[u8] {
-        &self.data.slice()
-            [Self::GRHR_OFFSET_P6..Self::GRHR_OFFSET_P6 + self.read_p5_host_name_length()]
-    }
-    pub fn decoded(&self) -> GRHostRecord {
-        GRHostRecord::new(
-            self.read_p0_epoch_time(),
-            self.read_p1_uptime(),
-            self.read_p2_setting_version_id(),
-            self.read_p3_run_mode(),
-            self.read_p4_startup_counter(),
-            self.read_p5_host_name_length() as _,
-            util::copy_slice_to_array(self.read_p6_host_name_raw()),
-        )
-    }
-}
-
-#[test]
-fn test_metadata_record_encode_decode() {
-    let md = GRMetadataRecordRaw::new(
-        FileScope::Journal,
-        FileSpecifier::GNSTxnLog,
-        FileSpecifierVersion(1),
-    );
-    assert_eq!(md.read_p0_server_version(), versions::v1::V1_SERVER_VERSION);
-    assert_eq!(md.read_p1_driver_version(), versions::v1::V1_DRIVER_VERSION);
-    assert_eq!(md.read_p2_file_scope(), FileScope::Journal);
-    assert_eq!(md.read_p3_file_spec(), FileSpecifier::GNSTxnLog);
-    assert_eq!(md.read_p4_file_spec_version(), FileSpecifierVersion(1));
-}
-
-#[test]
-fn test_host_record_encode_decode() {
-    const HOST_UPTIME: u128 = u128::MAX - 434324903;
-    const HOST_SETTING_VERSION_ID: u32 = 245;
-    const HOST_RUN_MODE: HostRunMode = HostRunMode::Prod;
-    const HOST_STARTUP_COUNTER: u64 = u32::MAX as _;
-    const HOST_NAME: &str = "skycloud";
-    use std::time::*;
-    let time = SystemTime::now()
-        .duration_since(UNIX_EPOCH)
-        .unwrap()
-        .as_nanos();
-    let hr = GRHostRecordRaw::new(
-        time,
-        HOST_UPTIME,
-        HOST_SETTING_VERSION_ID,
-        HOST_RUN_MODE,
-        HOST_STARTUP_COUNTER,
-        HOST_NAME.len() as _,
-        crate::util::copy_str_to_array(HOST_NAME),
-    );
-    assert_eq!(hr.read_p0_epoch_time(), time);
-    assert_eq!(hr.read_p1_uptime(), HOST_UPTIME);
-    assert_eq!(hr.read_p2_setting_version_id(), HOST_SETTING_VERSION_ID);
-    assert_eq!(hr.read_p3_run_mode(), HOST_RUN_MODE);
-    assert_eq!(hr.read_p4_startup_counter(), HOST_STARTUP_COUNTER);
-    assert_eq!(hr.read_p5_host_name_length(), HOST_NAME.len());
-    assert_eq!(hr.read_host_name(), HOST_NAME.as_bytes());
-}
diff --git a/server/src/engine/storage/v1/header_impl/mod.rs b/server/src/engine/storage/v1/header_impl/mod.rs
deleted file mode 100644
index 59a74e3b..00000000
--- a/server/src/engine/storage/v1/header_impl/mod.rs
+++ /dev/null
@@ -1,338 +0,0 @@
-/*
- * Created on Mon May 15 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
- * SDSS Header layout:
- *
- * +--------------------------------------------------------------+
- * |                                                              |
- * |                        STATIC RECORD                         |
- * |                            128B                              |
- * +--------------------------------------------------------------+
- * +--------------------------------------------------------------+
- * |                                                              |
- * |                                                              |
- * |                       GENESIS RECORD                         |
- * |                         (256+56+?)B                          |
- * |        +--------------------------------------------+        |
- * |        |                                            |        |
- * |        |              METADATA RECORD               |        |
- * |        |                   256B                     |        |
- * |        +--------------------------------------------+        |
- * |        +--------------------------------------------+        |
- * |        |                                            |        |
- * |        |               HOST RECORD                  |        |
- * |        |                  >56B                      |        |
- * |        +--------------------------------------------+        |
- * |                                                              |
- * +--------------------------------------------------------------+
- * +--------------------------------------------------------------+
- * |                       DYNAMIC RECORD                         |
- * |                           >56B                               |
- * +--------------------------------------------------------------+
- * Note: The entire part of the header is little endian encoded
-*/
-
-use crate::util::copy_slice_to_array as cp;
-
-use super::SDSSResult;
-
-// (1) sr
-mod sr;
-// (2) gr
-mod gr;
-// (3) dr
-mod dr;
-
-/// The file scope
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
-pub enum FileScope {
-    Journal = 0,
-    DataBatch = 1,
-    FlatmapData = 2,
-}
-
-impl FileScope {
-    pub const fn try_new(id: u64) -> Option<Self> {
-        Some(match id {
-            0 => Self::Journal,
-            1 => Self::DataBatch,
-            2 => Self::FlatmapData,
-            _ => return None,
-        })
-    }
-    pub const fn new(id: u64) -> Self {
-        match Self::try_new(id) {
-            Some(v) => v,
-            None => panic!("unknown filescope"),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
-#[repr(u8)]
-pub enum FileSpecifier {
-    GNSTxnLog = 0,
-    TableDataBatch = 1,
-    SysDB = 2,
-    #[cfg(test)]
-    TestTransactionLog = 0xFF,
-}
-
-impl FileSpecifier {
-    pub const fn try_new(v: u32) -> Option<Self> {
-        Some(match v {
-            0 => Self::GNSTxnLog,
-            1 => Self::TableDataBatch,
-            2 => Self::SysDB,
-            #[cfg(test)]
-            0xFF => Self::TestTransactionLog,
-            _ => return None,
-        })
-    }
-    pub const fn new(v: u32) -> Self {
-        match Self::try_new(v) {
-            Some(v) => v,
-            _ => panic!("unknown filespecifier"),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-pub struct FileSpecifierVersion(u32);
-impl FileSpecifierVersion {
-    pub const fn __new(v: u32) -> Self {
-        Self(v)
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
-#[repr(u8)]
-pub enum HostRunMode {
-    Dev = 0,
-    Prod = 1,
-}
-
-impl HostRunMode {
-    pub const fn try_new_with_val(v: u32) -> Option<Self> {
-        Some(match v {
-            0 => Self::Dev,
-            1 => Self::Prod,
-            _ => return None,
-        })
-    }
-    pub const fn new_with_val(v: u32) -> Self {
-        match Self::try_new_with_val(v) {
-            Some(v) => v,
-            None => panic!("unknown hostrunmode"),
-        }
-    }
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct SDSSHeader {
-    // static record
-    sr: sr::StaticRecord,
-    // genesis record
-    gr_mdr: gr::GRMetadataRecord,
-    gr_hr: gr::GRHostRecord,
-    // dynamic record
-    dr_hs: dr::DRHostSignature,
-    dr_rs: dr::DRRuntimeSignature,
-}
-
-impl SDSSHeader {
-    pub fn verify(
-        &self,
-        expected_file_scope: FileScope,
-        expected_file_specifier: FileSpecifier,
-        expected_file_specifier_version: FileSpecifierVersion,
-    ) -> SDSSResult<()> {
-        self.sr().verify()?;
-        self.gr_mdr().verify(
-            expected_file_scope,
-            expected_file_specifier,
-            expected_file_specifier_version,
-        )?;
-        self.gr_hr().verify()?;
-        self.dr_hs().verify(expected_file_specifier_version)?;
-        self.dr_rs().verify()?;
-        Ok(())
-    }
-}
-
-impl SDSSHeader {
-    pub const fn new(
-        sr: sr::StaticRecord,
-        gr_mdr: gr::GRMetadataRecord,
-        gr_hr: gr::GRHostRecord,
-        dr_hs: dr::DRHostSignature,
-        dr_rs: dr::DRRuntimeSignature,
-    ) -> Self {
-        Self {
-            sr,
-            gr_mdr,
-            gr_hr,
-            dr_hs,
-            dr_rs,
-        }
-    }
-    pub fn sr(&self) -> &sr::StaticRecord {
-        &self.sr
-    }
-    pub fn gr_mdr(&self) -> &gr::GRMetadataRecord {
-        &self.gr_mdr
-    }
-    pub fn gr_hr(&self) -> &gr::GRHostRecord {
-        &self.gr_hr
-    }
-    pub fn dr_hs(&self) -> &dr::DRHostSignature {
-        &self.dr_hs
-    }
-    pub fn dr_rs(&self) -> &dr::DRRuntimeSignature {
-        &self.dr_rs
-    }
-    pub fn dr_rs_mut(&mut self) -> &mut dr::DRRuntimeSignature {
-        &mut self.dr_rs
-    }
-    pub fn encoded(&self) -> SDSSHeaderRaw {
-        SDSSHeaderRaw::new_full(
-            self.sr.encoded(),
-            self.gr_mdr().encoded(),
-            self.gr_hr().encoded(),
-            self.dr_hs().encoded(),
-            self.dr_rs().encoded(),
-        )
-    }
-}
-
-#[derive(Clone)]
-pub struct SDSSHeaderRaw {
-    sr: sr::StaticRecordRaw,
-    gr_0_mdr: gr::GRMetadataRecordRaw,
-    gr_1_hr: gr::GRHostRecordRaw,
-    dr_0_hs: dr::DRHostSignatureRaw,
-    dr_1_rs: dr::DRRuntimeSignatureRaw,
-}
-
-impl SDSSHeaderRaw {
-    const OFFSET_SR0: usize = 0;
-    const OFFSET_SR1: usize = sizeof!(sr::StaticRecordRaw);
-    const OFFSET_SR2: usize = Self::OFFSET_SR1 + sizeof!(gr::GRMetadataRecordRaw);
-    const OFFSET_SR3: usize = Self::OFFSET_SR2 + sizeof!(gr::GRHostRecordRaw);
-    const OFFSET_SR4: usize = Self::OFFSET_SR3 + sizeof!(dr::DRHostSignatureRaw);
-    pub fn new_auto(
-        gr_mdr_scope: FileScope,
-        gr_mdr_specifier: FileSpecifier,
-        gr_mdr_specifier_id: FileSpecifierVersion,
-        gr_hr_setting_version: u32,
-        gr_hr_run_mode: HostRunMode,
-        gr_hr_startup_counter: u64,
-        dr_rts_modify_count: u64,
-    ) -> Self {
-        Self::new_full(
-            sr::StaticRecordRaw::new_auto(),
-            gr::GRMetadataRecordRaw::new_auto(gr_mdr_scope, gr_mdr_specifier, gr_mdr_specifier_id),
-            gr::GRHostRecordRaw::new_auto(
-                gr_hr_setting_version,
-                gr_hr_run_mode,
-                gr_hr_startup_counter,
-            ),
-            dr::DRHostSignatureRaw::new_auto(gr_mdr_specifier_id),
-            dr::DRRuntimeSignatureRaw::new_auto(dr_rts_modify_count),
-        )
-    }
-    pub fn new_full(
-        sr: sr::StaticRecordRaw,
-        gr_mdr: gr::GRMetadataRecordRaw,
-        gr_hr: gr::GRHostRecordRaw,
-        dr_hs: dr::DRHostSignatureRaw,
-        dr_rs: dr::DRRuntimeSignatureRaw,
-    ) -> Self {
-        Self {
-            sr,
-            gr_0_mdr: gr_mdr,
-            gr_1_hr: gr_hr,
-            dr_0_hs: dr_hs,
-            dr_1_rs: dr_rs,
-        }
-    }
-    pub fn new(
-        sr: sr::StaticRecordRaw,
-        gr_0_mdr: gr::GRMetadataRecordRaw,
-        gr_1_hr: gr::GRHostRecordRaw,
-        dr_hs: dr::DRHostSignatureRaw,
-        dr_rs: dr::DRRuntimeSignatureRaw,
-    ) -> Self {
-        Self {
-            sr,
-            gr_0_mdr,
-            gr_1_hr,
-            dr_0_hs: dr_hs,
-            dr_1_rs: dr_rs,
-        }
-    }
-    pub fn get0_sr(&self) -> &[u8] {
-        self.sr.base.get_ref()
-    }
-    pub fn get1_dr_0_mdr(&self) -> &[u8] {
-        self.gr_0_mdr.data.slice()
-    }
-    pub fn get1_dr_1_hr_0(&self) -> &[u8] {
-        self.gr_1_hr.data.slice()
-    }
-    pub const fn header_size() -> usize {
-        sizeof!(sr::StaticRecordRaw)
-            + sizeof!(gr::GRMetadataRecordRaw)
-            + sizeof!(gr::GRHostRecordRaw)
-            + sizeof!(dr::DRHostSignatureRaw)
-            + sizeof!(dr::DRRuntimeSignatureRaw)
-    }
-    pub fn array(&self) -> [u8; Self::header_size()] {
-        let mut data = [0u8; Self::header_size()];
-        data[Self::OFFSET_SR0..Self::OFFSET_SR1].copy_from_slice(self.sr.base.get_ref());
-        data[Self::OFFSET_SR1..Self::OFFSET_SR2].copy_from_slice(self.gr_0_mdr.data.slice());
-        data[Self::OFFSET_SR2..Self::OFFSET_SR3].copy_from_slice(self.gr_1_hr.data.slice());
-        data[Self::OFFSET_SR3..Self::OFFSET_SR4].copy_from_slice(self.dr_0_hs.data.slice());
-        data[Self::OFFSET_SR4..].copy_from_slice(self.dr_1_rs.data.slice());
-        data
-    }
-    /// **☢ WARNING ☢: This only decodes; it doesn't validate expected values!**
-    pub fn decode_noverify(slice: [u8; Self::header_size()]) -> Option<SDSSHeader> {
-        let sr =
-            sr::StaticRecordRaw::decode_noverify(cp(&slice[Self::OFFSET_SR0..Self::OFFSET_SR1]))?;
-        let gr_mdr = gr::GRMetadataRecordRaw::decode_noverify(cp(
-            &slice[Self::OFFSET_SR1..Self::OFFSET_SR2]
-        ))?;
-        let gr_hr =
-            gr::GRHostRecord::decode_noverify(cp(&slice[Self::OFFSET_SR2..Self::OFFSET_SR3]))?;
-        let dr_sig =
-            dr::DRHostSignature::decode_noverify(cp(&slice[Self::OFFSET_SR3..Self::OFFSET_SR4]))?;
-        let dr_rt = dr::DRRuntimeSignature::decode_noverify(cp(&slice[Self::OFFSET_SR4..]))?;
-        Some(SDSSHeader::new(sr, gr_mdr, gr_hr, dr_sig, dr_rt))
-    }
-}
diff --git a/server/src/engine/storage/v1/header_impl/sr.rs b/server/src/engine/storage/v1/header_impl/sr.rs
deleted file mode 100644
index 40b591f1..00000000
--- a/server/src/engine/storage/v1/header_impl/sr.rs
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Created on Thu May 25 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::engine::storage::{
-    header::{StaticRecordUV, StaticRecordUVRaw},
-    v1::{SDSSError, SDSSResult},
-    versions,
-};
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct StaticRecord {
-    sr: StaticRecordUV,
-}
-
-impl StaticRecord {
-    /// Verified:
-    /// - header version
-    ///
-    /// Need to verify: N/A
-    pub fn verify(&self) -> SDSSResult<()> {
-        if self.sr().header_version() == versions::v1::V1_HEADER_VERSION {
-            Ok(())
-        } else {
-            return Err(SDSSError::HeaderDecodeHeaderVersionMismatch);
-        }
-    }
-}
-
-impl StaticRecord {
-    pub const fn new(sr: StaticRecordUV) -> Self {
-        Self { sr }
-    }
-    pub const fn encoded(&self) -> StaticRecordRaw {
-        StaticRecordRaw {
-            base: self.sr.encoded(),
-        }
-    }
-    pub const fn sr(&self) -> &StaticRecordUV {
-        &self.sr
-    }
-}
-
-/// Static record
-#[derive(Clone)]
-pub struct StaticRecordRaw {
-    pub(super) base: StaticRecordUVRaw,
-}
-
-impl StaticRecordRaw {
-    pub const fn new_auto() -> Self {
-        Self::new(StaticRecordUVRaw::create(versions::v1::V1_HEADER_VERSION))
-    }
-    pub const fn new(base: StaticRecordUVRaw) -> Self {
-        Self { base }
-    }
-    pub const fn empty_buffer() -> [u8; sizeof!(Self)] {
-        [0u8; sizeof!(Self)]
-    }
-    pub fn decode_noverify(buf: [u8; sizeof!(Self)]) -> Option<StaticRecord> {
-        StaticRecordUVRaw::decode_from_bytes(buf).map(StaticRecord::new)
-    }
-}
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index 07a4bc3e..2c4aa730 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -43,49 +43,21 @@
 
 use {
     super::{
-        header_impl::{FileSpecifierVersion, HostRunMode, SDSSHeaderRaw},
         rw::{FileOpen, RawFSInterface, SDSSFileIO},
-        SDSSError, SDSSResult,
-    },
-    crate::{
-        engine::storage::v1::header_impl::{FileScope, FileSpecifier},
-        util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy, Threshold},
+        spec, SDSSError, SDSSResult,
     },
+    crate::util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy, Threshold},
     std::marker::PhantomData,
 };
 
 const CRC: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
 const RECOVERY_BLOCK_AUTO_THRESHOLD: usize = 5;
 
-pub fn open_journal<TA: JournalAdapter, Fs: RawFSInterface>(
+pub fn open_journal<TA: JournalAdapter, Fs: RawFSInterface, F: spec::FileSpec>(
     log_file_name: &str,
-    log_kind: FileSpecifier,
-    log_kind_version: FileSpecifierVersion,
-    host_setting_version: u32,
-    host_run_mode: HostRunMode,
-    host_startup_counter: u64,
     gs: &TA::GlobalState,
 ) -> SDSSResult<FileOpen<JournalWriter<Fs, TA>>> {
-    macro_rules! open_file {
-        ($modify:literal) => {
-            SDSSFileIO::<Fs>::open_or_create_perm_rw::<$modify>(
-                log_file_name,
-                FileScope::Journal,
-                log_kind,
-                log_kind_version,
-                host_setting_version,
-                host_run_mode,
-                host_startup_counter,
-            )
-        };
-    }
-    // HACK(@ohsayan): until generic const exprs are stabilized, we're in a state of hell
-    let f = if TA::DENY_NONAPPEND {
-        open_file!(false)
-    } else {
-        open_file!(true)
-    }?;
-    let file = match f {
+    let file = match SDSSFileIO::<Fs>::open_or_create_perm_rw::<F>(log_file_name)? {
         FileOpen::Created(f) => return Ok(FileOpen::Created(JournalWriter::new(f, 0, true)?)),
         FileOpen::Existing((file, _header)) => file,
     };
@@ -218,7 +190,7 @@ pub struct JournalReader<TA, Fs: RawFSInterface> {
 
 impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
     pub fn new(log_file: SDSSFileIO<Fs>) -> SDSSResult<Self> {
-        let log_size = log_file.file_length()? - SDSSHeaderRaw::header_size() as u64;
+        let log_size = log_file.file_length()? - spec::SDSSStaticHeaderV1Compact::SIZE as u64;
         Ok(Self {
             log_file,
             log_size,
diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs
index b73c6fef..acf7f248 100644
--- a/server/src/engine/storage/v1/loader.rs
+++ b/server/src/engine/storage/v1/loader.rs
@@ -29,10 +29,10 @@ use crate::engine::{
     data::uuid::Uuid,
     fractal::{FractalModelDriver, ModelDrivers, ModelUniqueID},
     storage::v1::{
-        batch_jrnl, header_meta,
+        batch_jrnl,
         journal::{self, JournalWriter},
         rw::{FileOpen, RawFSInterface},
-        LocalFS, SDSSErrorContext, SDSSResult,
+        spec, LocalFS, SDSSErrorContext, SDSSResult,
     },
     txn::gns::{GNSAdapter, GNSTransactionDriverAnyFS},
 };
@@ -61,19 +61,9 @@ impl SEInitState {
             gns,
         }
     }
-    pub fn try_init(
-        host_setting_version: u32,
-        host_run_mode: header_meta::HostRunMode,
-        host_startup_counter: u64,
-    ) -> SDSSResult<Self> {
+    pub fn try_init() -> SDSSResult<Self> {
         let gns = GlobalNS::empty();
-        let gns_txn_driver = open_gns_driver(
-            GNS_FILE_PATH,
-            host_setting_version,
-            host_run_mode,
-            host_startup_counter,
-            &gns,
-        )?;
+        let gns_txn_driver = open_gns_driver(GNS_FILE_PATH, &gns)?;
         let new_instance = gns_txn_driver.is_created();
         let mut model_drivers = ModelDrivers::new();
         if !new_instance {
@@ -131,18 +121,7 @@ impl SEInitState {
 
 pub fn open_gns_driver<Fs: RawFSInterface>(
     path: &str,
-    host_setting_version: u32,
-    host_run_mode: header_meta::HostRunMode,
-    host_startup_counter: u64,
     gns: &GlobalNS,
 ) -> SDSSResult<FileOpen<JournalWriter<Fs, GNSAdapter>>> {
-    journal::open_journal::<GNSAdapter, Fs>(
-        path,
-        header_meta::FileSpecifier::GNSTxnLog,
-        header_meta::FileSpecifierVersion::__new(GNS_LOG_VERSION_CODE),
-        host_setting_version,
-        host_run_mode,
-        host_startup_counter,
-        gns,
-    )
+    journal::open_journal::<GNSAdapter, Fs, spec::GNSTransactionLogV1>(path, gns)
 }
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index 0dbb413c..4c2a67b2 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -24,13 +24,12 @@
  *
 */
 
-// raw
-mod header_impl;
 // impls
 mod batch_jrnl;
 mod journal;
 pub(in crate::engine) mod loader;
 mod rw;
+pub mod spec;
 mod sysdb;
 // hl
 pub mod inf;
@@ -49,9 +48,6 @@ pub use {
 pub mod data_batch {
     pub use super::batch_jrnl::{create, reinit, DataBatchPersistDriver, DataBatchRestoreDriver};
 }
-pub mod header_meta {
-    pub use super::header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode};
-}
 
 use crate::{engine::txn::TransactionError, util::os::SysIOError as IoError};
 
@@ -138,6 +134,7 @@ pub enum SDSSError {
     /// An error with more context
     // TODO(@ohsayan): avoid the box; we'll clean this up soon
     Extra(Box<Self>, String),
+    HeaderDecodeVersionMismatch,
 }
 
 impl From<TransactionError> for SDSSError {
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index fbabc1b8..7dee56e5 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -26,9 +26,7 @@
 
 use {
     super::{
-        header_impl::{
-            FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode, SDSSHeader, SDSSHeaderRaw,
-        },
+        spec::{FileSpec, Header},
         SDSSResult,
     },
     crate::{
@@ -395,134 +393,36 @@ pub struct SDSSFileIO<Fs: RawFSInterface> {
 }
 
 impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
-    /// Open an existing SDSS file
-    ///
-    /// **IMPORTANT: File position: end-of-header-section**
-    pub fn open<const REWRITE_MODIFY_COUNTER: bool>(
-        file_path: &str,
-        file_scope: FileScope,
-        file_specifier: FileSpecifier,
-        file_specifier_version: FileSpecifierVersion,
-    ) -> SDSSResult<(SDSSHeader, Self)> {
-        let f = Fs::fs_fopen_rw(file_path)?;
-        Self::_sdss_fopen::<REWRITE_MODIFY_COUNTER>(
-            f,
-            file_scope,
-            file_specifier,
-            file_specifier_version,
-        )
-    }
-    /// internal SDSS fopen routine
-    fn _sdss_fopen<const REWRITE_MODIFY_COUNTER: bool>(
-        mut f: <Fs as RawFSInterface>::File,
-        file_scope: FileScope,
-        file_specifier: FileSpecifier,
-        file_specifier_version: FileSpecifierVersion,
-    ) -> Result<(SDSSHeader, SDSSFileIO<Fs>), SDSSError> {
-        let mut header_raw = [0u8; SDSSHeaderRaw::header_size()];
-        f.fr_read_exact(&mut header_raw)?;
-        let header = SDSSHeaderRaw::decode_noverify(header_raw)
-            .ok_or(SDSSError::HeaderDecodeCorruptedHeader)?;
-        header.verify(file_scope, file_specifier, file_specifier_version)?;
-        let mut f = Self::_new(f);
-        if REWRITE_MODIFY_COUNTER {
-            // since we updated this file, let us update the header
-            let mut new_header = header.clone();
-            new_header.dr_rs_mut().bump_modify_count();
-            f.seek_from_start(0)?;
-            f.fsynced_write(new_header.encoded().array().as_ref())?;
-            f.seek_from_start(SDSSHeaderRaw::header_size() as _)?;
-        }
-        Ok((header, f))
-    }
-    /// Create a new SDSS file
-    ///
-    /// **IMPORTANT: File position: end-of-header-section**
-    pub fn create(
-        file_path: &str,
-        file_scope: FileScope,
-        file_specifier: FileSpecifier,
-        file_specifier_version: FileSpecifierVersion,
-        host_setting_version: u32,
-        host_run_mode: HostRunMode,
-        host_startup_counter: u64,
-    ) -> SDSSResult<Self> {
-        let f = Fs::fs_fcreate_rw(file_path)?;
-        Self::_sdss_fcreate(
-            file_scope,
-            file_specifier,
-            file_specifier_version,
-            host_setting_version,
-            host_run_mode,
-            host_startup_counter,
-            f,
-        )
-    }
-    /// Internal SDSS fcreate routine
-    fn _sdss_fcreate(
-        file_scope: FileScope,
-        file_specifier: FileSpecifier,
-        file_specifier_version: FileSpecifierVersion,
-        host_setting_version: u32,
-        host_run_mode: HostRunMode,
-        host_startup_counter: u64,
-        f: <Fs as RawFSInterface>::File,
-    ) -> Result<SDSSFileIO<Fs>, SDSSError> {
-        let data = SDSSHeaderRaw::new_auto(
-            file_scope,
-            file_specifier,
-            file_specifier_version,
-            host_setting_version,
-            host_run_mode,
-            host_startup_counter,
-            0,
-        )
-        .array();
-        let mut f = Self::_new(f);
-        f.fsynced_write(&data)?;
+    pub fn open<F: FileSpec>(fpath: &str) -> SDSSResult<(Self, F::Header)> {
+        let mut f = Self::_new(Fs::fs_fopen_rw(fpath)?);
+        let header = F::Header::decode_verify(&mut f, F::DECODE_DATA, F::VERIFY_DATA)?;
+        Ok((f, header))
+    }
+    pub fn create<F: FileSpec>(fpath: &str) -> SDSSResult<Self> {
+        let mut f = Self::_new(Fs::fs_fcreate_rw(fpath)?);
+        F::Header::encode(&mut f, F::ENCODE_DATA)?;
         Ok(f)
     }
-    /// Create a new SDSS file or re-open an existing file and verify
-    ///
-    /// **IMPORTANT: File position: end-of-header-section**
-    pub fn open_or_create_perm_rw<const REWRITE_MODIFY_COUNTER: bool>(
-        file_path: &str,
-        file_scope: FileScope,
-        file_specifier: FileSpecifier,
-        file_specifier_version: FileSpecifierVersion,
-        host_setting_version: u32,
-        host_run_mode: HostRunMode,
-        host_startup_counter: u64,
-    ) -> SDSSResult<FileOpen<Self, (Self, SDSSHeader)>> {
-        let f = Fs::fs_fopen_or_create_rw(file_path)?;
-        match f {
-            FileOpen::Created(f) => {
-                let f = Self::_sdss_fcreate(
-                    file_scope,
-                    file_specifier,
-                    file_specifier_version,
-                    host_setting_version,
-                    host_run_mode,
-                    host_startup_counter,
-                    f,
-                )?;
+    pub fn open_or_create_perm_rw<F: FileSpec>(
+        fpath: &str,
+    ) -> SDSSResult<FileOpen<Self, (Self, F::Header)>> {
+        match Fs::fs_fopen_or_create_rw(fpath)? {
+            FileOpen::Created(c) => {
+                let mut f = Self::_new(c);
+                F::Header::encode(&mut f, F::ENCODE_DATA)?;
                 Ok(FileOpen::Created(f))
             }
-            FileOpen::Existing(f) => {
-                let (f, header) = Self::_sdss_fopen::<REWRITE_MODIFY_COUNTER>(
-                    f,
-                    file_scope,
-                    file_specifier,
-                    file_specifier_version,
-                )?;
-                Ok(FileOpen::Existing((header, f)))
+            FileOpen::Existing(e) => {
+                let mut f = Self::_new(e);
+                let header = F::Header::decode_verify(&mut f, F::DECODE_DATA, F::VERIFY_DATA)?;
+                Ok(FileOpen::Existing((f, header)))
             }
         }
     }
 }
 
 impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
-    fn _new(f: Fs::File) -> Self {
+    pub fn _new(f: Fs::File) -> Self {
         Self {
             f,
             _fs: PhantomData,
diff --git a/server/src/engine/storage/v1/spec.rs b/server/src/engine/storage/v1/spec.rs
new file mode 100644
index 00000000..6a448f3b
--- /dev/null
+++ b/server/src/engine/storage/v1/spec.rs
@@ -0,0 +1,517 @@
+/*
+ * Created on Mon Sep 25 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/*
+    Header specification
+    ---
+    We utilize two different kinds of headers:
+    - Static header - Mostly to avoid data corruption
+    - Variable header - For preserving dynamic information
+*/
+
+use {
+    super::{
+        rw::{RawFSInterface, SDSSFileIO},
+        SDSSResult,
+    },
+    crate::{
+        engine::storage::{
+            header::{HostArch, HostEndian, HostOS, HostPointerWidth},
+            v1::SDSSError,
+            versions::{self, DriverVersion, HeaderVersion, ServerVersion},
+        },
+        util::os,
+    },
+    std::{
+        mem::{transmute, ManuallyDrop},
+        ops::Range,
+    },
+};
+
+/*
+    meta
+*/
+
+/// The file scope
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+pub enum FileScope {
+    Journal = 0,
+    DataBatch = 1,
+    FlatmapData = 2,
+}
+
+impl FileScope {
+    pub const fn try_new(id: u64) -> Option<Self> {
+        Some(match id {
+            0 => Self::Journal,
+            1 => Self::DataBatch,
+            2 => Self::FlatmapData,
+            _ => return None,
+        })
+    }
+    pub const fn new(id: u64) -> Self {
+        match Self::try_new(id) {
+            Some(v) => v,
+            None => panic!("unknown filescope"),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+#[repr(u8)]
+pub enum FileSpecifier {
+    GNSTxnLog = 0,
+    TableDataBatch = 1,
+    SysDB = 2,
+    #[cfg(test)]
+    TestTransactionLog = 0xFF,
+}
+
+impl FileSpecifier {
+    pub const fn try_new(v: u32) -> Option<Self> {
+        Some(match v {
+            0 => Self::GNSTxnLog,
+            1 => Self::TableDataBatch,
+            2 => Self::SysDB,
+            #[cfg(test)]
+            0xFF => Self::TestTransactionLog,
+            _ => return None,
+        })
+    }
+    pub const fn new(v: u32) -> Self {
+        match Self::try_new(v) {
+            Some(v) => v,
+            _ => panic!("unknown filespecifier"),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct FileSpecifierVersion(u16);
+impl FileSpecifierVersion {
+    pub const fn __new(v: u16) -> Self {
+        Self(v)
+    }
+}
+
+const SDSS_MAGIC: u64 = 0x4F48534159414E21;
+
+/// Specification for a SDSS file
+pub trait FileSpec {
+    /// The header spec for the file
+    type Header: Header;
+    /// Encode data
+    const ENCODE_DATA: <Self::Header as Header>::EncodeArgs;
+    /// Decode data
+    const DECODE_DATA: <Self::Header as Header>::DecodeArgs;
+    /// Verify data
+    const VERIFY_DATA: <Self::Header as Header>::DecodeVerifyArgs;
+}
+
+/*
+    file spec impls
+*/
+
+#[cfg(test)]
+pub struct TestFile;
+#[cfg(test)]
+impl FileSpec for TestFile {
+    type Header = SDSSStaticHeaderV1Compact;
+    const ENCODE_DATA: <Self::Header as Header>::EncodeArgs = (
+        FileScope::FlatmapData,
+        FileSpecifier::TestTransactionLog,
+        FileSpecifierVersion::__new(0),
+    );
+    const DECODE_DATA: <Self::Header as Header>::DecodeArgs = ();
+    const VERIFY_DATA: <Self::Header as Header>::DecodeVerifyArgs = Self::ENCODE_DATA;
+}
+
+/// The file specification for the GNS transaction log (impl v1)
+pub struct GNSTransactionLogV1;
+impl FileSpec for GNSTransactionLogV1 {
+    type Header = SDSSStaticHeaderV1Compact;
+    const ENCODE_DATA: <Self::Header as Header>::EncodeArgs = (
+        FileScope::Journal,
+        FileSpecifier::GNSTxnLog,
+        FileSpecifierVersion::__new(0),
+    );
+    const DECODE_DATA: <Self::Header as Header>::DecodeArgs = ();
+    const VERIFY_DATA: <Self::Header as Header>::DecodeVerifyArgs = Self::ENCODE_DATA;
+}
+
+/// The file specification for a journal batch
+pub struct DataBatchJournalV1;
+impl FileSpec for DataBatchJournalV1 {
+    type Header = SDSSStaticHeaderV1Compact;
+    const ENCODE_DATA: <Self::Header as Header>::EncodeArgs = (
+        FileScope::DataBatch,
+        FileSpecifier::TableDataBatch,
+        FileSpecifierVersion::__new(0),
+    );
+    const DECODE_DATA: <Self::Header as Header>::DecodeArgs = ();
+    const VERIFY_DATA: <Self::Header as Header>::DecodeVerifyArgs = Self::ENCODE_DATA;
+}
+
+/// The file specification for the system db
+pub struct SysDBV1;
+impl FileSpec for SysDBV1 {
+    type Header = SDSSStaticHeaderV1Compact;
+    const ENCODE_DATA: <Self::Header as Header>::EncodeArgs = (
+        FileScope::FlatmapData,
+        FileSpecifier::SysDB,
+        FileSpecifierVersion::__new(0),
+    );
+    const DECODE_DATA: <Self::Header as Header>::DecodeArgs = ();
+    const VERIFY_DATA: <Self::Header as Header>::DecodeVerifyArgs = Self::ENCODE_DATA;
+}
+
+/*
+    header spec
+*/
+
+/// SDSS Header specification
+pub trait Header: Sized {
+    /// Encode arguments
+    type EncodeArgs;
+    /// Decode arguments
+    type DecodeArgs;
+    /// Decode verify arguments
+    type DecodeVerifyArgs;
+    /// Encode the header
+    fn encode<Fs: RawFSInterface>(f: &mut SDSSFileIO<Fs>, args: Self::EncodeArgs)
+        -> SDSSResult<()>;
+    /// Decode the header
+    fn decode<Fs: RawFSInterface>(
+        f: &mut SDSSFileIO<Fs>,
+        args: Self::DecodeArgs,
+    ) -> SDSSResult<Self>;
+    /// Verify the header
+    fn verify(&self, args: Self::DecodeVerifyArgs) -> SDSSResult<()>;
+    /// Decode and verify the header
+    fn decode_verify<Fs: RawFSInterface>(
+        f: &mut SDSSFileIO<Fs>,
+        d_args: Self::DecodeArgs,
+        v_args: Self::DecodeVerifyArgs,
+    ) -> SDSSResult<Self> {
+        let h = Self::decode(f, d_args)?;
+        h.verify(v_args)?;
+        Ok(h)
+    }
+}
+
+/*
+    header impls
+*/
+
+unsafe fn memcpy<const N: usize>(src: &[u8]) -> [u8; N] {
+    let mut dst = [0u8; N];
+    src.as_ptr().copy_to_nonoverlapping(dst.as_mut_ptr(), N);
+    dst
+}
+
+macro_rules! var {
+    (let $($name:ident),* $(,)?) => {
+        $(let $name;)*
+    }
+}
+
+/*
+    Compact SDSS Header v1
+    ---
+    - 1: Magic block (16B): magic + header version
+    - 2: Static block (40B):
+        - 2.1: Genesis static record (24B)
+            - 2.1.1: Software information (16B)
+                - Server version (8B)
+                - Driver version (8B)
+            - 2.1.2: Host information (4B):
+                - OS (1B)
+                - Arch (1B)
+                - Pointer width (1B)
+                - Endian (1B)
+            - 2.1.3: File information (4B):
+                - File class (1B)
+                - File specifier (1B)
+                - File specifier version (2B)
+        - 2.2: Genesis runtime record (16B)
+            - Host epoch (16B)
+    - 3: Padding block (8B)
+*/
+
+#[repr(align(8))]
+#[derive(Debug, PartialEq)]
+pub struct SDSSStaticHeaderV1Compact {
+    // 1 magic block
+    magic_header_version: HeaderVersion,
+    // 2.1.1
+    genesis_static_sw_server_version: ServerVersion,
+    genesis_static_sw_driver_version: DriverVersion,
+    // 2.1.2
+    genesis_static_host_os: HostOS,
+    genesis_static_host_arch: HostArch,
+    genesis_static_host_ptr_width: HostPointerWidth,
+    genesis_static_host_endian: HostEndian,
+    // 2.1.3
+    genesis_static_file_class: FileScope,
+    genesis_static_file_specifier: FileSpecifier,
+    genesis_static_file_specifier_version: FileSpecifierVersion,
+    // 2.2
+    genesis_runtime_epoch_time: u128,
+    // 3
+    genesis_padding_block: [u8; 8],
+}
+
+impl SDSSStaticHeaderV1Compact {
+    pub const SIZE: usize = 64;
+    /// Decode and validate the full header block (validate ONLY; you must verify yourself)
+    ///
+    /// Notes:
+    /// - Time might be inconsistent; verify
+    /// - Compatibility requires additional intervention
+    /// - If padding block was not zeroed, handle
+    /// - No file metadata and is verified. Check!
+    ///
+    fn _decode(block: [u8; 64]) -> SDSSResult<Self> {
+        var!(let raw_magic, raw_header_version, raw_server_version, raw_driver_version, raw_host_os, raw_host_arch,
+            raw_host_ptr_width, raw_host_endian, raw_file_class, raw_file_specifier, raw_file_specifier_version,
+            raw_runtime_epoch_time, raw_paddding_block,
+        );
+        macro_rules! u64 {
+            ($pos:expr) => {
+                u64::from_le_bytes(memcpy(&block[$pos]))
+            };
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): all segments are correctly accessed (aligned to u8)
+            raw_magic = u64!(Self::SEG1_MAGIC);
+            raw_header_version = HeaderVersion::__new(u64!(Self::SEG1_HEADER_VERSION));
+            raw_server_version = ServerVersion::__new(u64!(Self::SEG2_REC1_SERVER_VERSION));
+            raw_driver_version = DriverVersion::__new(u64!(Self::SEG2_REC1_DRIVER_VERSION));
+            raw_host_os = block[Self::SEG2_REC1_HOST_OS];
+            raw_host_arch = block[Self::SEG2_REC1_HOST_ARCH];
+            raw_host_ptr_width = block[Self::SEG2_REC1_HOST_PTR_WIDTH];
+            raw_host_endian = block[Self::SEG2_REC1_HOST_ENDIAN];
+            raw_file_class = block[Self::SEG2_REC1_FILE_CLASS];
+            raw_file_specifier = block[Self::SEG2_REC1_FILE_SPECIFIER];
+            raw_file_specifier_version = FileSpecifierVersion::__new(u16::from_le_bytes(memcpy(
+                &block[Self::SEG2_REC1_FILE_SPECIFIER_VERSION],
+            )));
+            raw_runtime_epoch_time =
+                u128::from_le_bytes(memcpy(&block[Self::SEG2_REC2_RUNTIME_EPOCH_TIME]));
+            raw_paddding_block = memcpy::<8>(&block[Self::SEG3_PADDING_BLK]);
+        }
+        macro_rules! okay {
+            ($($expr:expr),* $(,)?) => {
+                $(($expr) &)*true
+            }
+        }
+        let okay_header_version = raw_header_version == versions::CURRENT_HEADER_VERSION;
+        let okay_server_version = raw_server_version == versions::CURRENT_SERVER_VERSION;
+        let okay_driver_version = raw_driver_version == versions::CURRENT_DRIVER_VERSION;
+        let okay = okay!(
+            // 1.1 mgblk
+            raw_magic == SDSS_MAGIC,
+            okay_header_version,
+            // 2.1.1
+            okay_server_version,
+            okay_driver_version,
+            // 2.1.2
+            raw_host_os <= HostOS::MAX,
+            raw_host_arch <= HostArch::MAX,
+            raw_host_ptr_width <= HostPointerWidth::MAX,
+            raw_host_endian <= HostEndian::MAX,
+            // 2.1.3
+            raw_file_class <= FileScope::MAX,
+            raw_file_specifier <= FileSpecifier::MAX,
+        );
+        if okay {
+            Ok(unsafe {
+                // UNSAFE(@ohsayan): the block ranges are very well defined
+                Self {
+                    // 1.1
+                    magic_header_version: raw_header_version,
+                    // 2.1.1
+                    genesis_static_sw_server_version: raw_server_version,
+                    genesis_static_sw_driver_version: raw_driver_version,
+                    // 2.1.2
+                    genesis_static_host_os: transmute(raw_host_os),
+                    genesis_static_host_arch: transmute(raw_host_arch),
+                    genesis_static_host_ptr_width: transmute(raw_host_ptr_width),
+                    genesis_static_host_endian: transmute(raw_host_endian),
+                    // 2.1.3
+                    genesis_static_file_class: transmute(raw_file_class),
+                    genesis_static_file_specifier: transmute(raw_file_specifier),
+                    genesis_static_file_specifier_version: raw_file_specifier_version,
+                    // 2.2
+                    genesis_runtime_epoch_time: raw_runtime_epoch_time,
+                    // 3
+                    genesis_padding_block: raw_paddding_block,
+                }
+            })
+        } else {
+            let version_okay = okay_header_version & okay_server_version & okay_driver_version;
+            let md = ManuallyDrop::new([
+                SDSSError::HeaderDecodeCorruptedHeader,
+                SDSSError::HeaderDecodeVersionMismatch,
+            ]);
+            Err(unsafe {
+                // UNSAFE(@ohsayan): while not needed, md for drop safety + correct index
+                md.as_ptr().add(!version_okay as usize).read()
+            })
+        }
+    }
+}
+
+impl SDSSStaticHeaderV1Compact {
+    const SEG1_MAGIC: Range<usize> = 0..8;
+    const SEG1_HEADER_VERSION: Range<usize> = 8..16;
+    const SEG2_REC1_SERVER_VERSION: Range<usize> = 16..24;
+    const SEG2_REC1_DRIVER_VERSION: Range<usize> = 24..32;
+    const SEG2_REC1_HOST_OS: usize = 32;
+    const SEG2_REC1_HOST_ARCH: usize = 33;
+    const SEG2_REC1_HOST_PTR_WIDTH: usize = 34;
+    const SEG2_REC1_HOST_ENDIAN: usize = 35;
+    const SEG2_REC1_FILE_CLASS: usize = 36;
+    const SEG2_REC1_FILE_SPECIFIER: usize = 37;
+    const SEG2_REC1_FILE_SPECIFIER_VERSION: Range<usize> = 38..40;
+    const SEG2_REC2_RUNTIME_EPOCH_TIME: Range<usize> = 40..56;
+    const SEG3_PADDING_BLK: Range<usize> = 56..64;
+    fn _encode(
+        file_class: FileScope,
+        file_specifier: FileSpecifier,
+        file_specifier_version: FileSpecifierVersion,
+        epoch_time: u128,
+        padding_block: [u8; 8],
+    ) -> [u8; 64] {
+        let mut ret = [0; 64];
+        // 1. mgblk
+        ret[Self::SEG1_MAGIC].copy_from_slice(&SDSS_MAGIC.to_le_bytes());
+        ret[Self::SEG1_HEADER_VERSION]
+            .copy_from_slice(&versions::CURRENT_HEADER_VERSION.little_endian_u64());
+        // 2.1.1
+        ret[Self::SEG2_REC1_SERVER_VERSION]
+            .copy_from_slice(&versions::CURRENT_SERVER_VERSION.little_endian());
+        ret[Self::SEG2_REC1_DRIVER_VERSION]
+            .copy_from_slice(&versions::CURRENT_DRIVER_VERSION.little_endian());
+        // 2.1.2
+        ret[Self::SEG2_REC1_HOST_OS] = HostOS::new().value_u8();
+        ret[Self::SEG2_REC1_HOST_ARCH] = HostArch::new().value_u8();
+        ret[Self::SEG2_REC1_HOST_PTR_WIDTH] = HostPointerWidth::new().value_u8();
+        ret[Self::SEG2_REC1_HOST_ENDIAN] = HostEndian::new().value_u8();
+        // 2.1.3
+        ret[Self::SEG2_REC1_FILE_CLASS] = file_class.value_u8();
+        ret[Self::SEG2_REC1_FILE_SPECIFIER] = file_specifier.value_u8();
+        ret[Self::SEG2_REC1_FILE_SPECIFIER_VERSION]
+            .copy_from_slice(&file_specifier_version.0.to_le_bytes());
+        // 2.2
+        ret[Self::SEG2_REC2_RUNTIME_EPOCH_TIME].copy_from_slice(&epoch_time.to_le_bytes());
+        // 3
+        ret[Self::SEG3_PADDING_BLK].copy_from_slice(&padding_block);
+        ret
+    }
+    pub fn _encode_auto(
+        file_class: FileScope,
+        file_specifier: FileSpecifier,
+        file_specifier_version: FileSpecifierVersion,
+    ) -> [u8; 64] {
+        let epoch_time = os::get_epoch_time();
+        Self::_encode(
+            file_class,
+            file_specifier,
+            file_specifier_version,
+            epoch_time,
+            [0; 8],
+        )
+    }
+}
+
+impl SDSSStaticHeaderV1Compact {
+    pub fn header_version(&self) -> HeaderVersion {
+        self.magic_header_version
+    }
+    pub fn server_version(&self) -> ServerVersion {
+        self.genesis_static_sw_server_version
+    }
+    pub fn driver_version(&self) -> DriverVersion {
+        self.genesis_static_sw_driver_version
+    }
+    pub fn host_os(&self) -> HostOS {
+        self.genesis_static_host_os
+    }
+    pub fn host_arch(&self) -> HostArch {
+        self.genesis_static_host_arch
+    }
+    pub fn host_ptr_width(&self) -> HostPointerWidth {
+        self.genesis_static_host_ptr_width
+    }
+    pub fn host_endian(&self) -> HostEndian {
+        self.genesis_static_host_endian
+    }
+    pub fn file_class(&self) -> FileScope {
+        self.genesis_static_file_class
+    }
+    pub fn file_specifier(&self) -> FileSpecifier {
+        self.genesis_static_file_specifier
+    }
+    pub fn file_specifier_version(&self) -> FileSpecifierVersion {
+        self.genesis_static_file_specifier_version
+    }
+    pub fn epoch_time(&self) -> u128 {
+        self.genesis_runtime_epoch_time
+    }
+    pub fn padding_block(&self) -> [u8; 8] {
+        self.genesis_padding_block
+    }
+}
+
+impl Header for SDSSStaticHeaderV1Compact {
+    type EncodeArgs = (FileScope, FileSpecifier, FileSpecifierVersion);
+    type DecodeArgs = ();
+    type DecodeVerifyArgs = Self::EncodeArgs;
+    fn encode<Fs: RawFSInterface>(
+        f: &mut SDSSFileIO<Fs>,
+        (scope, spec, spec_v): Self::EncodeArgs,
+    ) -> SDSSResult<()> {
+        let b = Self::_encode_auto(scope, spec, spec_v);
+        f.fsynced_write(&b)
+    }
+    fn decode<Fs: RawFSInterface>(f: &mut SDSSFileIO<Fs>, _: Self::DecodeArgs) -> SDSSResult<Self> {
+        let mut buf = [0u8; 64];
+        f.read_to_buffer(&mut buf)?;
+        Self::_decode(buf)
+    }
+    fn verify(&self, (scope, spec, spec_v): Self::DecodeVerifyArgs) -> SDSSResult<()> {
+        if (self.file_class() == scope)
+            & (self.file_specifier() == spec)
+            & (self.file_specifier_version() == spec_v)
+        {
+            Ok(())
+        } else {
+            Err(SDSSError::HeaderDecodeDataMismatch)
+        }
+    }
+}
diff --git a/server/src/engine/storage/v1/sysdb.rs b/server/src/engine/storage/v1/sysdb.rs
index 917f1028..f1d28f51 100644
--- a/server/src/engine/storage/v1/sysdb.rs
+++ b/server/src/engine/storage/v1/sysdb.rs
@@ -27,10 +27,7 @@
 use crate::engine::{
     data::{cell::Datacell, DictEntryGeneric, DictGeneric},
     fractal::SysConfig,
-    storage::v1::{
-        header_meta::{FileScope, FileSpecifier, FileSpecifierVersion},
-        RawFSInterface, SDSSError, SDSSFileIO, SDSSResult,
-    },
+    storage::v1::{spec, RawFSInterface, SDSSError, SDSSFileIO, SDSSResult},
 };
 
 const SYSDB_PATH: &str = "sys.db";
@@ -75,19 +72,11 @@ pub fn sync_system_database<Fs: RawFSInterface>(cfg: &SysConfig) -> SDSSResult<(
         }
     }
     // open file
-    let mut file = SDSSFileIO::<Fs>::open_or_create_perm_rw::<false>(
-        SYSDB_COW_PATH,
-        FileScope::FlatmapData,
-        FileSpecifier::SysDB,
-        FileSpecifierVersion::__new(0),
-        cfg.host_data().settings_version(),
-        cfg.host_data().run_mode(),
-        cfg.host_data().startup_counter(),
-    )?
-    .into_created()
-    .ok_or(SDSSError::OtherError(
-        "sys.db.cow already exists. please remove this file.",
-    ))?;
+    let mut file = SDSSFileIO::<Fs>::open_or_create_perm_rw::<spec::SysDBV1>(SYSDB_COW_PATH)?
+        .into_created()
+        .ok_or(SDSSError::OtherError(
+            "sys.db.cow already exists. please remove this file.",
+        ))?;
     // write
     let buf = super::inf::enc::enc_dict_full::<super::inf::map::GenericDictSpec>(&map);
     file.fsynced_write(&buf)?;
diff --git a/server/src/engine/storage/v1/tests/batch.rs b/server/src/engine/storage/v1/tests/batch.rs
index e38d4113..2418883e 100644
--- a/server/src/engine/storage/v1/tests/batch.rs
+++ b/server/src/engine/storage/v1/tests/batch.rs
@@ -41,9 +41,9 @@ use {
                     DataBatchPersistDriver, DataBatchRestoreDriver, DecodedBatchEvent,
                     DecodedBatchEventKind, NormalBatch,
                 },
-                header_meta::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
                 memfs::VirtualFS,
                 rw::{FileOpen, SDSSFileIO},
+                spec,
             },
         },
         util::test_utils,
@@ -57,18 +57,8 @@ fn pkey(v: impl Into<Datacell>) -> PrimaryIndexKey {
 
 fn open_file(
     fpath: &str,
-) -> FileOpen<SDSSFileIO<VirtualFS>, (SDSSFileIO<VirtualFS>, super::super::header_impl::SDSSHeader)>
-{
-    SDSSFileIO::open_or_create_perm_rw::<false>(
-        fpath,
-        FileScope::DataBatch,
-        FileSpecifier::TableDataBatch,
-        FileSpecifierVersion::__new(0),
-        0,
-        HostRunMode::Dev,
-        1,
-    )
-    .unwrap()
+) -> FileOpen<SDSSFileIO<VirtualFS>, (SDSSFileIO<VirtualFS>, spec::SDSSStaticHeaderV1Compact)> {
+    SDSSFileIO::open_or_create_perm_rw::<spec::DataBatchJournalV1>(fpath).unwrap()
 }
 
 fn open_batch_data(fpath: &str, mdl: &Model) -> DataBatchPersistDriver<VirtualFS> {
diff --git a/server/src/engine/storage/v1/tests/rw.rs b/server/src/engine/storage/v1/tests/rw.rs
index 4df4ce87..d964cf81 100644
--- a/server/src/engine/storage/v1/tests/rw.rs
+++ b/server/src/engine/storage/v1/tests/rw.rs
@@ -25,21 +25,15 @@
 */
 
 use crate::engine::storage::v1::{
-    header_impl::{FileScope, FileSpecifier, FileSpecifierVersion, HostRunMode},
     rw::{FileOpen, SDSSFileIO},
+    spec,
 };
 
 #[test]
 fn create_delete() {
     {
-        let f = SDSSFileIO::<super::VirtualFS>::open_or_create_perm_rw::<false>(
+        let f = SDSSFileIO::<super::VirtualFS>::open_or_create_perm_rw::<spec::TestFile>(
             "hello_world.db-tlog",
-            FileScope::Journal,
-            FileSpecifier::GNSTxnLog,
-            FileSpecifierVersion::__new(0),
-            0,
-            HostRunMode::Prod,
-            0,
         )
         .unwrap();
         match f {
@@ -47,24 +41,12 @@ fn create_delete() {
             FileOpen::Created(_) => {}
         };
     }
-    let open = SDSSFileIO::<super::VirtualFS>::open_or_create_perm_rw::<false>(
+    let open = SDSSFileIO::<super::VirtualFS>::open_or_create_perm_rw::<spec::TestFile>(
         "hello_world.db-tlog",
-        FileScope::Journal,
-        FileSpecifier::GNSTxnLog,
-        FileSpecifierVersion::__new(0),
-        0,
-        HostRunMode::Prod,
-        0,
     )
     .unwrap();
-    let h = match open {
-        FileOpen::Existing((_, header)) => header,
+    let _ = match open {
+        FileOpen::Existing(_) => {}
         _ => panic!(),
     };
-    assert_eq!(h.gr_mdr().file_scope(), FileScope::Journal);
-    assert_eq!(h.gr_mdr().file_spec(), FileSpecifier::GNSTxnLog);
-    assert_eq!(h.gr_mdr().file_spec_id(), FileSpecifierVersion::__new(0));
-    assert_eq!(h.gr_hr().run_mode(), HostRunMode::Prod);
-    assert_eq!(h.gr_hr().setting_version(), 0);
-    assert_eq!(h.gr_hr().startup_counter(), 0);
 }
diff --git a/server/src/engine/storage/v1/tests/tx.rs b/server/src/engine/storage/v1/tests/tx.rs
index 5bf01ec0..45140c4d 100644
--- a/server/src/engine/storage/v1/tests/tx.rs
+++ b/server/src/engine/storage/v1/tests/tx.rs
@@ -27,9 +27,8 @@
 use {
     crate::{
         engine::storage::v1::{
-            header_impl::{FileSpecifier, FileSpecifierVersion, HostRunMode},
             journal::{self, JournalAdapter, JournalWriter},
-            SDSSError, SDSSResult,
+            spec, SDSSError, SDSSResult,
         },
         util,
     },
@@ -134,16 +133,8 @@ fn open_log(
     log_name: &str,
     db: &Database,
 ) -> SDSSResult<JournalWriter<super::VirtualFS, DatabaseTxnAdapter>> {
-    journal::open_journal::<DatabaseTxnAdapter, super::VirtualFS>(
-        log_name,
-        FileSpecifier::TestTransactionLog,
-        FileSpecifierVersion::__new(0),
-        0,
-        HostRunMode::Prod,
-        1,
-        &db,
-    )
-    .map(|v| v.into_inner())
+    journal::open_journal::<DatabaseTxnAdapter, super::VirtualFS, spec::TestFile>(log_name, db)
+        .map(|v| v.into_inner())
 }
 
 #[test]
diff --git a/server/src/engine/storage/versions/mod.rs b/server/src/engine/storage/versions/mod.rs
index 5a7d3bb0..23650e3d 100644
--- a/server/src/engine/storage/versions/mod.rs
+++ b/server/src/engine/storage/versions/mod.rs
@@ -28,18 +28,22 @@
 
 pub mod server_version;
 
+pub const CURRENT_SERVER_VERSION: ServerVersion = v1::V1_SERVER_VERSION;
+pub const CURRENT_DRIVER_VERSION: DriverVersion = v1::V1_DRIVER_VERSION;
+pub const CURRENT_HEADER_VERSION: HeaderVersion = v1::V1_HEADER_VERSION;
+
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
 /// The header version
 ///
 /// The header version is part of the static record and *barely* changes (almost like once in a light year)
-pub struct HeaderVersion(u32);
+pub struct HeaderVersion(u64);
 
 impl HeaderVersion {
-    pub const fn __new(v: u32) -> Self {
+    pub const fn __new(v: u64) -> Self {
         Self(v)
     }
     pub const fn little_endian_u64(&self) -> [u8; 8] {
-        (self.0 as u64).to_le_bytes()
+        self.0.to_le_bytes()
     }
 }
 

From 0d167765418eb55e2a09c9ca9a876dbcf4dab62f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 26 Sep 2023 17:14:26 +0000
Subject: [PATCH 266/310] Enable sysdb init

---
 server/src/engine/config.rs             |   8 +-
 server/src/engine/data/cell.rs          |  36 ++++
 server/src/engine/data/dict.rs          |  12 ++
 server/src/engine/fractal/config.rs     |  55 +++++-
 server/src/engine/fractal/mod.rs        |   3 +-
 server/src/engine/fractal/test_utils.rs |   6 +-
 server/src/engine/storage/v1/mod.rs     |  10 +-
 server/src/engine/storage/v1/rw.rs      |   6 +
 server/src/engine/storage/v1/sysdb.rs   | 218 +++++++++++++++++++++---
 server/src/engine/storage/v1/tests.rs   | 144 ++++++++++++++++
 10 files changed, 452 insertions(+), 46 deletions(-)

diff --git a/server/src/engine/config.rs b/server/src/engine/config.rs
index b6fd2606..d0df75fc 100644
--- a/server/src/engine/config.rs
+++ b/server/src/engine/config.rs
@@ -197,16 +197,16 @@ impl ConfigSystem {
     config auth
 */
 
-#[derive(Debug, PartialEq, Deserialize)]
+#[derive(Debug, PartialEq, Deserialize, Clone, Copy)]
 pub enum AuthDriver {
     #[serde(rename = "pwd")]
     Pwd,
 }
 
-#[derive(Debug, PartialEq, Deserialize)]
+#[derive(Debug, PartialEq, Deserialize, Clone)]
 pub struct ConfigAuth {
-    plugin: AuthDriver,
-    root_key: String,
+    pub plugin: AuthDriver,
+    pub root_key: String,
 }
 
 impl ConfigAuth {
diff --git a/server/src/engine/data/cell.rs b/server/src/engine/data/cell.rs
index 1cce866d..842df8a2 100644
--- a/server/src/engine/data/cell.rs
+++ b/server/src/engine/data/cell.rs
@@ -90,6 +90,16 @@ impl Datacell {
     pub fn uint(&self) -> u64 {
         self.try_uint().unwrap()
     }
+    pub fn into_uint(self) -> Option<u64> {
+        if self.kind() != TagClass::UnsignedInt {
+            return None;
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): +tagck
+            let md = ManuallyDrop::new(self);
+            Some(md.data.word.dwordnn_load_qw())
+        }
+    }
     // sint
     pub fn new_sint(i: i64) -> Self {
         unsafe {
@@ -158,6 +168,21 @@ impl Datacell {
     pub fn bin(&self) -> &[u8] {
         self.try_bin().unwrap()
     }
+    pub fn into_bin(self) -> Option<Vec<u8>> {
+        if self.kind() != TagClass::Bin {
+            return None;
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): no double free + tagck
+            let md = ManuallyDrop::new(self);
+            let (a, b) = md.data.word.dwordqn_load_qw_nw();
+            Some(Vec::from_raw_parts(
+                b as *const u8 as *mut u8,
+                a as usize,
+                a as usize,
+            ))
+        }
+    }
     // str
     pub fn new_str(s: Box<str>) -> Self {
         let mut md = ManuallyDrop::new(s.into_boxed_bytes());
@@ -201,6 +226,17 @@ impl Datacell {
     pub fn list(&self) -> &RwLock<Vec<Self>> {
         self.try_list().unwrap()
     }
+    pub fn into_list(self) -> Option<Vec<Datacell>> {
+        if self.kind() != TagClass::List {
+            return None;
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): +tagck +avoid double free
+            let md = ManuallyDrop::new(self);
+            let rwl = core::ptr::read(&md.data.rwl);
+            Some(ManuallyDrop::into_inner(rwl).into_inner())
+        }
+    }
     pub unsafe fn new_qw(qw: u64, tag: CUTag) -> Datacell {
         Self::new(
             tag,
diff --git a/server/src/engine/data/dict.rs b/server/src/engine/data/dict.rs
index 9295f051..bdce358a 100644
--- a/server/src/engine/data/dict.rs
+++ b/server/src/engine/data/dict.rs
@@ -52,6 +52,18 @@ impl DictEntryGeneric {
             _ => None,
         }
     }
+    pub fn into_dict(self) -> Option<DictGeneric> {
+        match self {
+            Self::Map(m) => Some(m),
+            _ => None,
+        }
+    }
+    pub fn into_data(self) -> Option<Datacell> {
+        match self {
+            Self::Data(d) => Some(d),
+            _ => None,
+        }
+    }
 }
 
 /*
diff --git a/server/src/engine/fractal/config.rs b/server/src/engine/fractal/config.rs
index 4eb08478..1609171d 100644
--- a/server/src/engine/fractal/config.rs
+++ b/server/src/engine/fractal/config.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use crate::engine::config::ConfigAuth;
+
 use {
     crate::engine::error::{Error, QueryResult},
     parking_lot::RwLock,
@@ -33,28 +35,53 @@ use {
 #[derive(Debug)]
 /// The global system configuration
 pub struct SysConfig {
-    auth_data: RwLock<Option<SysAuth>>,
+    auth_data: Option<RwLock<SysAuth>>,
     host_data: SysHostData,
 }
 
+impl PartialEq for SysConfig {
+    fn eq(&self, other: &Self) -> bool {
+        self.host_data == other.host_data
+            && match (self.auth_data.as_ref(), other.auth_data.as_ref()) {
+                (None, None) => true,
+                (Some(a), Some(b)) => &*a.read() == &*b.read(),
+                _ => false,
+            }
+    }
+}
+
 impl SysConfig {
     /// Initialize a new system config
-    pub fn new(auth_data: RwLock<Option<SysAuth>>, host_data: SysHostData) -> Self {
+    pub fn new(auth_data: Option<RwLock<SysAuth>>, host_data: SysHostData) -> Self {
         Self {
             auth_data,
             host_data,
         }
     }
+    pub fn new_auth(new_auth: Option<ConfigAuth>, host_data: SysHostData) -> Self {
+        match new_auth {
+            Some(ConfigAuth { root_key, .. }) => Self::new(
+                Some(RwLock::new(SysAuth::new(
+                    rcrypt::hash(root_key, rcrypt::DEFAULT_COST)
+                        .unwrap()
+                        .into_boxed_slice(),
+                    Default::default(),
+                ))),
+                host_data,
+            ),
+            None => Self::new(None, host_data),
+        }
+    }
     #[cfg(test)]
     /// A test-mode default setting with auth disabled
     pub(super) fn test_default() -> Self {
         Self {
-            auth_data: RwLock::new(None),
+            auth_data: None,
             host_data: SysHostData::new(0, 0),
         }
     }
     /// Returns a handle to the authentication data
-    pub fn auth_data(&self) -> &RwLock<Option<SysAuth>> {
+    pub fn auth_data(&self) -> &Option<RwLock<SysAuth>> {
         &self.auth_data
     }
     /// Returns a reference to host data
@@ -78,9 +105,20 @@ impl SysHostData {
             settings_version,
         }
     }
+    /// Returns the startup counter
+    ///
+    /// Note:
+    /// - If this is `0` -> this is the first boot
+    /// - If this is `1` -> this is the second boot (... and so on)
     pub fn startup_counter(&self) -> u64 {
         self.startup_counter
     }
+    /// Returns the settings version
+    ///
+    /// Note:
+    /// - If this is `0` -> this is the initial setting (first boot)
+    ///
+    /// If it stays at 0, this means that the settings were never changed
     pub fn settings_version(&self) -> u32 {
         self.settings_version
     }
@@ -118,6 +156,13 @@ impl SysAuth {
     }
     /// Verify the user with the given details
     pub fn verify_user(&self, username: &str, password: &str) -> QueryResult<()> {
+        if username == "root" {
+            if rcrypt::verify(password, self.root_key()).unwrap() {
+                return Ok(());
+            } else {
+                return Err(Error::SysAuthError);
+            }
+        }
         match self.users.get(username) {
             Some(user) if rcrypt::verify(password, user.key()).unwrap() => Ok(()),
             Some(_) | None => Err(Error::SysAuthError),
@@ -139,7 +184,7 @@ pub struct SysAuthUser {
 
 impl SysAuthUser {
     /// Create a new [`SysAuthUser`]
-    fn new(key: Box<[u8]>) -> Self {
+    pub fn new(key: Box<[u8]>) -> Self {
         Self { key }
     }
     /// Get the key
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index 72b34aac..46159de8 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -39,14 +39,13 @@ use {
     tokio::sync::mpsc::unbounded_channel,
 };
 
-mod config;
+pub mod config;
 mod drivers;
 mod mgr;
 #[cfg(test)]
 pub mod test_utils;
 mod util;
 pub use {
-    config::SysConfig,
     drivers::FractalModelDriver,
     mgr::{CriticalTask, GenericTask, Task},
     util::FractalToken,
diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
index 5f9c826e..b312db99 100644
--- a/server/src/engine/fractal/test_utils.rs
+++ b/server/src/engine/fractal/test_utils.rs
@@ -26,8 +26,8 @@
 
 use {
     super::{
-        CriticalTask, FractalModelDriver, GenericTask, GlobalInstanceLike, ModelUniqueID,
-        SysConfig, Task,
+        config::SysConfig, CriticalTask, FractalModelDriver, GenericTask, GlobalInstanceLike,
+        ModelUniqueID, Task,
     },
     crate::engine::{
         core::GlobalNS,
@@ -53,7 +53,7 @@ pub struct TestGlobal<Fs: RawFSInterface = VirtualFS> {
     max_delta_size: usize,
     txn_driver: Mutex<GNSTransactionDriverAnyFS<Fs>>,
     model_drivers: RwLock<HashMap<ModelUniqueID, FractalModelDriver<Fs>>>,
-    sys_cfg: super::SysConfig,
+    sys_cfg: SysConfig,
 }
 
 impl<Fs: RawFSInterface> TestGlobal<Fs> {
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index 4c2a67b2..9f11804d 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -93,14 +93,10 @@ pub enum SDSSError {
     // process errors
     OtherError(&'static str),
     // header
+    /// version mismatch
+    HeaderDecodeVersionMismatch,
     /// The entire header is corrupted
     HeaderDecodeCorruptedHeader,
-    /// The header versions don't match
-    HeaderDecodeHeaderVersionMismatch,
-    /// The driver versions don't match
-    HeaderDecodeDriverVersionMismatch,
-    /// The server versions don't match
-    HeaderDecodeServerVersionMismatch,
     /// Expected header values were not matched with the current header
     HeaderDecodeDataMismatch,
     /// The time in the [header/dynrec/rtsig] is in the future
@@ -134,7 +130,7 @@ pub enum SDSSError {
     /// An error with more context
     // TODO(@ohsayan): avoid the box; we'll clean this up soon
     Extra(Box<Self>, String),
-    HeaderDecodeVersionMismatch,
+    SysDBCorrupted,
 }
 
 impl From<TransactionError> for SDSSError {
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 7dee56e5..be4233d8 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -458,4 +458,10 @@ impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
         let mut r = [0; 1];
         self.read_to_buffer(&mut r).map(|_| r[0])
     }
+    pub fn load_remaining_into_buffer(&mut self) -> SDSSResult<Vec<u8>> {
+        let len = self.file_length()? - self.retrieve_cursor()?;
+        let mut buf = vec![0; len as usize];
+        self.read_to_buffer(&mut buf)?;
+        Ok(buf)
+    }
 }
diff --git a/server/src/engine/storage/v1/sysdb.rs b/server/src/engine/storage/v1/sysdb.rs
index f1d28f51..d21501d6 100644
--- a/server/src/engine/storage/v1/sysdb.rs
+++ b/server/src/engine/storage/v1/sysdb.rs
@@ -24,38 +24,144 @@
  *
 */
 
-use crate::engine::{
-    data::{cell::Datacell, DictEntryGeneric, DictGeneric},
-    fractal::SysConfig,
-    storage::v1::{spec, RawFSInterface, SDSSError, SDSSFileIO, SDSSResult},
+use {
+    super::{rw::FileOpen, SDSSError},
+    crate::engine::{
+        config::ConfigAuth,
+        data::{cell::Datacell, DictEntryGeneric, DictGeneric},
+        fractal::config::{SysAuth, SysAuthUser, SysConfig, SysHostData},
+        storage::v1::{inf, spec, RawFSInterface, SDSSFileIO, SDSSResult},
+    },
+    parking_lot::RwLock,
+    std::collections::HashMap,
 };
 
 const SYSDB_PATH: &str = "sys.db";
 const SYSDB_COW_PATH: &str = "sys.db.cow";
+const SYS_KEY_AUTH: &str = "auth";
+const SYS_KEY_AUTH_ROOT: &str = "root";
+const SYS_KEY_AUTH_USERS: &str = "users";
+const SYS_KEY_SYS: &str = "sys";
+const SYS_KEY_SYS_STARTUP_COUNTER: &str = "sc";
+const SYS_KEY_SYS_SETTINGS_VERSION: &str = "sv";
 
-pub fn sync_system_database<Fs: RawFSInterface>(cfg: &SysConfig) -> SDSSResult<()> {
-    // get auth data
-    let auth_data = cfg.auth_data().read();
+#[derive(Debug, PartialEq)]
+/// The system store init state
+pub enum SystemStoreInitState {
+    /// No system store was present. it was created
+    Created,
+    /// The system store was present, but no new changes were applied
+    Unchanged,
+    /// The system store was present, root settings were updated
+    UpdatedRoot,
+    /// the system store was present, auth was previously enabled but is now disabled
+    UpdatedAuthDisabled,
+    /// the system store was present, auth was previously disabled but is now enabled
+    UpdatedAuthEnabled,
+}
+
+#[derive(Debug, PartialEq)]
+/// Result of initializing the system store (sysdb)
+pub struct SystemStoreInit {
+    pub store: SysConfig,
+    pub state: SystemStoreInitState,
+}
+
+impl SystemStoreInit {
+    pub fn new(store: SysConfig, state: SystemStoreInitState) -> Self {
+        Self { store, state }
+    }
+}
+
+/// Open the system database
+///
+/// - If it doesn't exist, create it
+/// - If it exists, look for config changes and sync them
+pub fn open_system_database<Fs: RawFSInterface>(
+    auth: Option<ConfigAuth>,
+) -> SDSSResult<SystemStoreInit> {
+    open_or_reinit_system_database::<Fs>(auth, SYSDB_PATH, SYSDB_COW_PATH)
+}
+
+/// Open or re-initialize the system database
+pub fn open_or_reinit_system_database<Fs: RawFSInterface>(
+    auth: Option<ConfigAuth>,
+    sysdb_path: &str,
+    sysdb_path_cow: &str,
+) -> SDSSResult<SystemStoreInit> {
+    let (ex, _) = match SDSSFileIO::<Fs>::open_or_create_perm_rw::<spec::SysDBV1>(sysdb_path)? {
+        FileOpen::Created(new_sysdb) => {
+            let syscfg = SysConfig::new_auth(auth, SysHostData::new(0, 0));
+            sync_system_database_to(&syscfg, new_sysdb)?;
+            return Ok(SystemStoreInit::new(syscfg, SystemStoreInitState::Created));
+        }
+        FileOpen::Existing(ex) => ex,
+    };
+    let last_syscfg = decode_system_database(ex)?;
+    let mut state = SystemStoreInitState::Unchanged;
+    match (last_syscfg.auth_data(), &auth) {
+        (Some(last_auth), Some(new_auth)) => {
+            let last_auth = last_auth.read();
+            if last_auth.verify_user("root", &new_auth.root_key).is_err() {
+                // the root password was changed
+                state = SystemStoreInitState::UpdatedRoot;
+            }
+        }
+        (Some(_), None) => {
+            state = SystemStoreInitState::UpdatedAuthDisabled;
+        }
+        (None, Some(_)) => {
+            state = SystemStoreInitState::UpdatedAuthEnabled;
+        }
+        (None, None) => {}
+    }
+    let new_syscfg = SysConfig::new_auth(
+        auth,
+        SysHostData::new(
+            last_syscfg.host_data().startup_counter() + 1,
+            last_syscfg.host_data().settings_version()
+                + !matches!(state, SystemStoreInitState::Unchanged) as u32,
+        ),
+    );
+    // sync
+    let cow_file = SDSSFileIO::<Fs>::create::<spec::SysDBV1>(sysdb_path_cow)?;
+    sync_system_database_to(&new_syscfg, cow_file)?;
+    // replace
+    Fs::fs_rename_file(sysdb_path_cow, sysdb_path)?;
+    Ok(SystemStoreInit::new(new_syscfg, state))
+}
+
+/// Sync the system database to the given file
+pub fn sync_system_database_to<Fs: RawFSInterface>(
+    cfg: &SysConfig,
+    mut f: SDSSFileIO<Fs>,
+) -> SDSSResult<()> {
     // prepare our flat file
     let mut map: DictGeneric = into_dict!(
-        "host" => DictEntryGeneric::Map(into_dict!(
-            "settings_version" => Datacell::new_uint(cfg.host_data().settings_version() as _),
-            "startup_counter" => Datacell::new_uint(cfg.host_data().startup_counter() as _),
+        SYS_KEY_SYS => DictEntryGeneric::Map(into_dict!(
+            SYS_KEY_SYS_SETTINGS_VERSION => Datacell::new_uint(cfg.host_data().settings_version() as _),
+            SYS_KEY_SYS_STARTUP_COUNTER => Datacell::new_uint(cfg.host_data().startup_counter() as _),
         )),
-        "auth" => DictGeneric::new(),
+        SYS_KEY_AUTH => DictGeneric::new(),
     );
-    let auth_key = map.get_mut("auth").unwrap();
-    match &*auth_data {
-        None => *auth_key = Datacell::null().into(),
+    let auth_key = map.get_mut(SYS_KEY_AUTH).unwrap();
+    match cfg.auth_data() {
+        None => {
+            *auth_key = DictEntryGeneric::Map(
+                into_dict!(SYS_KEY_AUTH_ROOT => Datacell::null(), SYS_KEY_AUTH_USERS => Datacell::null()),
+            )
+        }
         Some(auth) => {
+            let auth = auth.read();
             let auth_key = auth_key.as_dict_mut().unwrap();
             auth_key.insert(
-                "root".into(),
+                SYS_KEY_AUTH_ROOT.into(),
                 DictEntryGeneric::Data(Datacell::new_bin(auth.root_key().into())),
             );
             auth_key.insert(
-                "users".into(),
+                SYS_KEY_AUTH_USERS.into(),
                 DictEntryGeneric::Map(
+                    // username -> [..settings]
                     auth.users()
                         .iter()
                         .map(|(username, user)| {
@@ -71,15 +177,77 @@ pub fn sync_system_database<Fs: RawFSInterface>(cfg: &SysConfig) -> SDSSResult<(
             );
         }
     }
-    // open file
-    let mut file = SDSSFileIO::<Fs>::open_or_create_perm_rw::<spec::SysDBV1>(SYSDB_COW_PATH)?
-        .into_created()
-        .ok_or(SDSSError::OtherError(
-            "sys.db.cow already exists. please remove this file.",
-        ))?;
     // write
     let buf = super::inf::enc::enc_dict_full::<super::inf::map::GenericDictSpec>(&map);
-    file.fsynced_write(&buf)?;
-    // replace
-    Fs::fs_rename_file(SYSDB_COW_PATH, SYSDB_PATH)
+    f.fsynced_write(&buf)
+}
+
+fn rkey<T>(
+    d: &mut DictGeneric,
+    key: &str,
+    transform: impl Fn(DictEntryGeneric) -> Option<T>,
+) -> SDSSResult<T> {
+    match d.remove(key).map(transform) {
+        Some(Some(k)) => Ok(k),
+        _ => Err(SDSSError::SysDBCorrupted),
+    }
+}
+
+/// Decode the system database
+pub fn decode_system_database<Fs: RawFSInterface>(mut f: SDSSFileIO<Fs>) -> SDSSResult<SysConfig> {
+    let rem = f.load_remaining_into_buffer()?;
+    let mut store = inf::dec::dec_dict_full::<inf::map::GenericDictSpec>(&rem)?;
+    // find auth and sys stores
+    let mut auth_store = rkey(&mut store, SYS_KEY_AUTH, DictEntryGeneric::into_dict)?;
+    let mut sys_store = rkey(&mut store, SYS_KEY_SYS, DictEntryGeneric::into_dict)?;
+    // get our auth
+    let auth_root = rkey(&mut auth_store, SYS_KEY_AUTH_ROOT, |dict| {
+        let data = dict.into_data()?;
+        match data.kind() {
+            _ if data.is_null() => Some(None),
+            _ => data.into_bin().map(Some),
+        }
+    })?;
+    let auth_users = rkey(&mut auth_store, SYS_KEY_AUTH_USERS, |dict| match dict {
+        DictEntryGeneric::Data(dc) if dc.is_null() => Some(None),
+        DictEntryGeneric::Map(m) => Some(Some(m)),
+        _ => None,
+    })?;
+    let sys_auth = match (auth_root, auth_users) {
+        (Some(root_pass), Some(users)) => {
+            let mut usermap = HashMap::new();
+            for (user_name, user) in users {
+                let mut user_data = user
+                    .into_data()
+                    .and_then(|d| d.into_list())
+                    .ok_or(SDSSError::SysDBCorrupted)?;
+                if user_data.len() != 1 {
+                    return Err(SDSSError::SysDBCorrupted);
+                }
+                let password = user_data
+                    .remove(0)
+                    .into_bin()
+                    .ok_or(SDSSError::SysDBCorrupted)?;
+                usermap.insert(user_name, SysAuthUser::new(password.into_boxed_slice()));
+            }
+            Some(RwLock::new(SysAuth::new(
+                root_pass.into_boxed_slice(),
+                usermap,
+            )))
+        }
+        (None, None) => None,
+        _ => return Err(SDSSError::SysDBCorrupted),
+    };
+    // get our sys
+    let sv = rkey(&mut sys_store, SYS_KEY_SYS_SETTINGS_VERSION, |de| {
+        de.into_data()?.into_uint()
+    })?;
+    let sc = rkey(&mut sys_store, SYS_KEY_SYS_STARTUP_COUNTER, |de| {
+        de.into_data()?.into_uint()
+    })?;
+    if !(sys_store.is_empty() & auth_store.is_empty() & store.is_empty()) {
+        // the stores have more keys than we expected. something is wrong here
+        return Err(SDSSError::SysDBCorrupted);
+    }
+    Ok(SysConfig::new(sys_auth, SysHostData::new(sc, sv as u32)))
 }
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index 4c72da2d..5bfd8817 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -29,3 +29,147 @@ type VirtualFS = super::memfs::VirtualFS;
 mod batch;
 mod rw;
 mod tx;
+
+mod sysdb {
+    use {
+        super::{
+            super::sysdb::{self, SystemStoreInitState},
+            VirtualFS as VFS,
+        },
+        crate::engine::config::{AuthDriver, ConfigAuth},
+    };
+    #[test]
+    fn simple_open_close() {
+        {
+            let syscfg_new = sysdb::open_or_reinit_system_database::<VFS>(
+                None,
+                "sysdb_test_1.db",
+                "sysdb_test_1.cow.db",
+            )
+            .unwrap();
+            assert_eq!(syscfg_new.state, SystemStoreInitState::Created);
+            assert!(syscfg_new.store.auth_data().is_none());
+            assert_eq!(syscfg_new.store.host_data().settings_version(), 0);
+            assert_eq!(syscfg_new.store.host_data().startup_counter(), 0);
+        }
+        let syscfg_restore = sysdb::open_or_reinit_system_database::<VFS>(
+            None,
+            "sysdb_test_1.db",
+            "sysdb_test_1.cow.db",
+        )
+        .unwrap();
+        assert_eq!(syscfg_restore.state, SystemStoreInitState::Unchanged);
+        assert!(syscfg_restore.store.auth_data().is_none());
+        assert_eq!(syscfg_restore.store.host_data().settings_version(), 0);
+        assert_eq!(syscfg_restore.store.host_data().startup_counter(), 1);
+    }
+    #[test]
+    fn with_auth_nochange() {
+        let auth = ConfigAuth::new(AuthDriver::Pwd, "password12345678".to_string());
+        {
+            let syscfg_new = sysdb::open_or_reinit_system_database::<VFS>(
+                Some(auth.clone()),
+                "sysdb_test_2.db",
+                "sysdb_test_2.cow.db",
+            )
+            .unwrap();
+            assert_eq!(syscfg_new.state, SystemStoreInitState::Created);
+            assert!(syscfg_new
+                .store
+                .auth_data()
+                .as_ref()
+                .unwrap()
+                .read()
+                .verify_user("root", "password12345678")
+                .is_ok());
+            assert_eq!(syscfg_new.store.host_data().startup_counter(), 0);
+            assert_eq!(syscfg_new.store.host_data().settings_version(), 0);
+        }
+        // now reboot
+        let syscfg_new = sysdb::open_or_reinit_system_database::<VFS>(
+            Some(auth),
+            "sysdb_test_2.db",
+            "sysdb_test_2.cow.db",
+        )
+        .unwrap();
+        assert_eq!(syscfg_new.state, SystemStoreInitState::Unchanged);
+        assert!(syscfg_new
+            .store
+            .auth_data()
+            .as_ref()
+            .unwrap()
+            .read()
+            .verify_user("root", "password12345678")
+            .is_ok());
+        assert_eq!(syscfg_new.store.host_data().startup_counter(), 1);
+        assert_eq!(syscfg_new.store.host_data().settings_version(), 0);
+    }
+    #[test]
+    fn disable_auth() {
+        {
+            let auth = ConfigAuth::new(AuthDriver::Pwd, "password12345678".to_string());
+            let syscfg_new = sysdb::open_or_reinit_system_database::<VFS>(
+                Some(auth),
+                "sysdb_test_3.db",
+                "sysdb_test_3.cow.db",
+            )
+            .unwrap();
+            assert_eq!(syscfg_new.state, SystemStoreInitState::Created);
+            assert!(syscfg_new
+                .store
+                .auth_data()
+                .as_ref()
+                .unwrap()
+                .read()
+                .verify_user("root", "password12345678")
+                .is_ok());
+            assert_eq!(syscfg_new.store.host_data().startup_counter(), 0);
+            assert_eq!(syscfg_new.store.host_data().settings_version(), 0);
+        }
+        // reboot
+        let sysdb_cfg = sysdb::open_or_reinit_system_database::<VFS>(
+            None,
+            "sysdb_test_3.db",
+            "sysdb_test_3.cow.db",
+        )
+        .unwrap();
+        assert_eq!(sysdb_cfg.state, SystemStoreInitState::UpdatedAuthDisabled);
+        assert!(sysdb_cfg.store.auth_data().is_none());
+        assert_eq!(sysdb_cfg.store.host_data().startup_counter(), 1);
+        assert_eq!(sysdb_cfg.store.host_data().settings_version(), 1);
+    }
+    #[test]
+    fn enable_auth() {
+        {
+            let sysdb_cfg = sysdb::open_or_reinit_system_database::<VFS>(
+                None,
+                "sysdb_test_4.db",
+                "sysdb_test_4.cow.db",
+            )
+            .unwrap();
+            assert_eq!(sysdb_cfg.state, SystemStoreInitState::Created);
+            assert!(sysdb_cfg.store.auth_data().is_none());
+            assert_eq!(sysdb_cfg.store.host_data().startup_counter(), 0);
+            assert_eq!(sysdb_cfg.store.host_data().settings_version(), 0);
+        }
+        // reboot
+        let auth = ConfigAuth::new(AuthDriver::Pwd, "password12345678".to_string());
+        let syscfg_new = sysdb::open_or_reinit_system_database::<VFS>(
+            Some(auth),
+            "sysdb_test_4.db",
+            "sysdb_test_4.cow.db",
+        )
+        .unwrap();
+        assert_eq!(syscfg_new.state, SystemStoreInitState::UpdatedAuthEnabled);
+        assert!(syscfg_new
+            .store
+            .auth_data()
+            .as_ref()
+            .unwrap()
+            .read()
+            .verify_user("root", "password12345678")
+            .is_ok());
+        assert_eq!(syscfg_new.store.host_data().startup_counter(), 1);
+        assert_eq!(syscfg_new.store.host_data().settings_version(), 1);
+    }
+}

From d2faa19140d487d90b79f301392aa64785366cd5 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 27 Sep 2023 14:56:56 +0000
Subject: [PATCH 267/310] Remove retry loop in SE recovery algorithm

I must have done this in a hurry, but this algorithm has a lot of
inconsistencies and it's safer to avoid this for now. I'll implement
a more robust error detection and correction algorithm later.
---
 .../src/engine/storage/v1/batch_jrnl/mod.rs   |  2 -
 .../engine/storage/v1/batch_jrnl/restore.rs   | 10 ++--
 server/src/engine/storage/v1/journal.rs       | 47 ++++++++-----------
 3 files changed, 22 insertions(+), 37 deletions(-)

diff --git a/server/src/engine/storage/v1/batch_jrnl/mod.rs b/server/src/engine/storage/v1/batch_jrnl/mod.rs
index 45f793bd..0348c7a6 100644
--- a/server/src/engine/storage/v1/batch_jrnl/mod.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/mod.rs
@@ -37,8 +37,6 @@ const MARKER_END_OF_BATCH: u8 = 0xFD;
 const MARKER_ACTUAL_BATCH_EVENT: u8 = 0xFE;
 /// recovery batch event marker
 const MARKER_RECOVERY_EVENT: u8 = 0xFF;
-/// recovery threshold
-const RECOVERY_THRESHOLD: usize = 10;
 
 #[cfg(test)]
 pub(super) use restore::{DecodedBatchEvent, DecodedBatchEventKind, NormalBatch};
diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs
index f7bf0dc1..3475cd0b 100644
--- a/server/src/engine/storage/v1/batch_jrnl/restore.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs
@@ -27,7 +27,7 @@
 use {
     super::{
         MARKER_ACTUAL_BATCH_EVENT, MARKER_BATCH_CLOSED, MARKER_BATCH_REOPEN, MARKER_END_OF_BATCH,
-        MARKER_RECOVERY_EVENT, RECOVERY_THRESHOLD,
+        MARKER_RECOVERY_EVENT,
     },
     crate::engine::{
         core::{
@@ -414,12 +414,8 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
         )))
     }
     fn attempt_recover_data_batch(&mut self) -> SDSSResult<()> {
-        let mut max_threshold = RECOVERY_THRESHOLD;
-        while max_threshold != 0 && self.f.has_left(1) {
-            if let Ok(MARKER_RECOVERY_EVENT) = self.f.inner_file().read_byte() {
-                return Ok(());
-            }
-            max_threshold -= 1;
+        if let Ok(MARKER_RECOVERY_EVENT) = self.f.inner_file().read_byte() {
+            return Ok(());
         }
         Err(SDSSError::DataBatchRestoreCorruptedBatch)
     }
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index 2c4aa730..4dbaaadb 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -46,12 +46,11 @@ use {
         rw::{FileOpen, RawFSInterface, SDSSFileIO},
         spec, SDSSError, SDSSResult,
     },
-    crate::util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy, Threshold},
+    crate::util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy},
     std::marker::PhantomData,
 };
 
 const CRC: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
-const RECOVERY_BLOCK_AUTO_THRESHOLD: usize = 5;
 
 pub fn open_journal<TA: JournalAdapter, Fs: RawFSInterface, F: spec::FileSpec>(
     log_file_name: &str,
@@ -284,26 +283,22 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
     /// once we **sucessfully** finish processing a normal (aka server event origin) event and not a non-normal branch)
     fn try_recover_journal_strategy_simple_reverse(&mut self) -> SDSSResult<()> {
         debug_assert!(TA::RECOVERY_PLUGIN, "recovery plugin not enabled");
-        let mut threshold = Threshold::<RECOVERY_BLOCK_AUTO_THRESHOLD>::new();
-        while threshold.not_busted() & self.has_remaining_bytes(JournalEntryMetadata::SIZE as _) {
-            self.__record_read_bytes(JournalEntryMetadata::SIZE); // FIXME(@ohsayan): don't assume read length?
-            let mut entry_buf = [0u8; JournalEntryMetadata::SIZE];
-            if self.log_file.read_to_buffer(&mut entry_buf).is_err() {
-                threshold.bust_one();
-                continue;
-            }
-            let entry = JournalEntryMetadata::decode(entry_buf);
-            let okay = (entry.event_id == self.evid as u128)
-                & (entry.event_crc == 0)
-                & (entry.event_payload_len == 0)
-                & (entry.event_source_md == EventSourceMarker::RECOVERY_REVERSE_LAST_JOURNAL);
-            if okay {
-                return Ok(());
-            }
-            self._incr_evid();
-            threshold.bust_one();
+        self.__record_read_bytes(JournalEntryMetadata::SIZE); // FIXME(@ohsayan): don't assume read length?
+        let mut entry_buf = [0u8; JournalEntryMetadata::SIZE];
+        if self.log_file.read_to_buffer(&mut entry_buf).is_err() {
+            return Err(SDSSError::JournalCorrupted);
+        }
+        let entry = JournalEntryMetadata::decode(entry_buf);
+        let okay = (entry.event_id == self.evid as u128)
+            & (entry.event_crc == 0)
+            & (entry.event_payload_len == 0)
+            & (entry.event_source_md == EventSourceMarker::RECOVERY_REVERSE_LAST_JOURNAL);
+        self._incr_evid();
+        if okay {
+            return Ok(());
+        } else {
+            Err(SDSSError::JournalCorrupted)
         }
-        Err(SDSSError::JournalCorrupted)
     }
     /// Read and apply all events in the given log file to the global state, returning the (open file, last event ID)
     pub fn scroll(file: SDSSFileIO<Fs>, gs: &TA::GlobalState) -> SDSSResult<(SDSSFileIO<Fs>, u64)> {
@@ -396,15 +391,11 @@ impl<Fs: RawFSInterface, TA: JournalAdapter> JournalWriter<Fs, TA> {
 
 impl<Fs: RawFSInterface, TA> JournalWriter<Fs, TA> {
     pub fn appendrec_journal_reverse_entry(&mut self) -> SDSSResult<()> {
-        let mut threshold = Threshold::<RECOVERY_BLOCK_AUTO_THRESHOLD>::new();
         let mut entry =
             JournalEntryMetadata::new(0, EventSourceMarker::RECOVERY_REVERSE_LAST_JOURNAL, 0, 0);
-        while threshold.not_busted() {
-            entry.event_id = self._incr_id() as u128;
-            if self.log_file.fsynced_write(&entry.encoded()).is_ok() {
-                return Ok(());
-            }
-            threshold.bust_one();
+        entry.event_id = self._incr_id() as u128;
+        if self.log_file.fsynced_write(&entry.encoded()).is_ok() {
+            return Ok(());
         }
         Err(SDSSError::JournalWRecoveryStageOneFailCritical)
     }

From 5ba82a6cf0c6bcf739905ff75ac30be99ec623b6 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 28 Sep 2023 10:18:20 +0000
Subject: [PATCH 268/310] Force users to set root password

---
 server/src/engine/config.rs           |  74 +++++----
 server/src/engine/fractal/config.rs   |  48 +++---
 server/src/engine/storage/v1/sysdb.rs | 212 ++++++++++++--------------
 server/src/engine/storage/v1/tests.rs | 152 ++++++------------
 server/src/engine/tests/mod.rs        |   6 +-
 5 files changed, 203 insertions(+), 289 deletions(-)

diff --git a/server/src/engine/config.rs b/server/src/engine/config.rs
index d0df75fc..19397155 100644
--- a/server/src/engine/config.rs
+++ b/server/src/engine/config.rs
@@ -83,7 +83,7 @@ pub struct Configuration {
     endpoints: ConfigEndpoint,
     mode: ConfigMode,
     system: ConfigSystem,
-    auth: Option<ConfigAuth>,
+    auth: ConfigAuth,
 }
 
 impl Configuration {
@@ -91,7 +91,7 @@ impl Configuration {
         endpoints: ConfigEndpoint,
         mode: ConfigMode,
         system: ConfigSystem,
-        auth: Option<ConfigAuth>,
+        auth: ConfigAuth,
     ) -> Self {
         Self {
             endpoints,
@@ -103,7 +103,7 @@ impl Configuration {
     const DEFAULT_HOST: &'static str = "127.0.0.1";
     const DEFAULT_PORT_TCP: u16 = 2003;
     const DEFAULT_RELIABILITY_SVC_PING: u64 = 5 * 60;
-    pub fn default_dev_mode() -> Self {
+    pub fn default_dev_mode(auth: DecodedAuth) -> Self {
         Self {
             endpoints: ConfigEndpoint::Insecure(ConfigEndpointTcp {
                 host: Self::DEFAULT_HOST.to_owned(),
@@ -113,7 +113,7 @@ impl Configuration {
             system: ConfigSystem {
                 reliability_system_window: Self::DEFAULT_RELIABILITY_SVC_PING,
             },
-            auth: None,
+            auth: ConfigAuth::new(auth.plugin, auth.root_pass),
         }
     }
 }
@@ -634,7 +634,7 @@ Flags:
 
 Options:
   --tlscert <path>            Specify the path to the TLS certificate.
-  --tlskey <path>             Define the path to the TLS private key.
+  --tlskey <path>             Specify the path to the TLS private key.
   --endpoint <definition>     Designate an endpoint. Format: protocol@host:port.
                               This option can be repeated to define multiple endpoints.
   --service-window <seconds>  Establish the time window for the background service in seconds.
@@ -644,12 +644,12 @@ Options:
   --auth-root-password <pass> Set the root password
 
 Examples:
-  skyd --mode=dev --endpoint tcp@127.0.0.1:2003
+  skyd --mode=dev --auth-root-password \"password12345678\"
 
 Notes:
-  - When no mode is provided, `--mode=dev` is defaulted to
-  - When either of `-h` or `-v` is provided, all other options and flags are ignored.
-  - When `--auth-plugin` is provided, you must provide a value for `--auth-root-password`
+  - If no `--mode` is provided, we default to `dev`
+  - You must provide `--auth-root-password` to set the default root password
+  - To use TLS, you must provide both `--tlscert` and `--tlskey`
 
 For further assistance, refer to the official documentation here: https://docs.skytable.org
 ";
@@ -787,8 +787,8 @@ pub fn parse_env_args() -> ConfigResult<Option<ParsedRawArgs>> {
 /// Apply the configuration changes to the given mutable config
 fn apply_config_changes<CS: ConfigurationSource>(
     args: &mut ParsedRawArgs,
-    config: &mut ModifyGuard<DecodedConfiguration>,
-) -> ConfigResult<()> {
+) -> ConfigResult<ModifyGuard<DecodedConfiguration>> {
+    let mut config = ModifyGuard::new(DecodedConfiguration::default());
     enum DecodeKind {
         Simple {
             key: &'static str,
@@ -822,21 +822,21 @@ fn apply_config_changes<CS: ConfigurationSource>(
         match task {
             DecodeKind::Simple { key, f } => match args.get(key) {
                 Some(values_for_arg) => {
-                    (f)(&values_for_arg, config)?;
+                    (f)(&values_for_arg, &mut config)?;
                     args.remove(key);
                 }
                 None => {}
             },
-            DecodeKind::Complex { f } => (f)(args, config)?,
+            DecodeKind::Complex { f } => (f)(args, &mut config)?,
         }
     }
-    if args.is_empty() {
-        Ok(())
-    } else {
+    if !args.is_empty() {
         Err(ConfigError::with_src(
             CS::SOURCE,
             ConfigErrorKind::ErrorString("found unknown arguments".into()),
         ))
+    } else {
+        Ok(config)
     }
 }
 
@@ -896,7 +896,7 @@ macro_rules! if_some {
 }
 
 macro_rules! err_if {
-    ($(if $cond:expr => $error:expr),*) => {
+    ($(if $cond:expr => $error:expr),* $(,)?) => {
         $(if $cond { return Err($error) })*
     }
 }
@@ -909,8 +909,17 @@ fn validate_configuration<CS: ConfigurationSource>(
         auth,
     }: DecodedConfiguration,
 ) -> ConfigResult<Configuration> {
+    let Some(auth) = auth else {
+        return Err(ConfigError::with_src(
+            CS::SOURCE,
+            ConfigErrorKind::ErrorString(format!(
+                "root account must be configured with {}",
+                CS::KEY_AUTH_ROOT_PASSWORD
+            )),
+        ));
+    };
     // initialize our default configuration
-    let mut config = Configuration::default_dev_mode();
+    let mut config = Configuration::default_dev_mode(auth);
     // mutate
     if_some!(
         system => |system: DecodedSystemConfig| {
@@ -946,26 +955,16 @@ fn validate_configuration<CS: ConfigurationSource>(
             })
         }
     );
-    if let Some(auth) = auth {
-        if auth.root_pass.len() < ROOT_PASSWORD_MIN_LEN {
-            return Err(ConfigError::with_src(
-                CS::SOURCE,
-                ConfigErrorKind::ErrorString(format!(
-                    "root password must have atleast {ROOT_PASSWORD_MIN_LEN} characters"
-                )),
-            ));
-        }
-        config.auth = Some(ConfigAuth {
-            plugin: auth.plugin,
-            root_key: auth.root_pass,
-        });
-    }
     // now check a few things
     err_if!(
         if config.system.reliability_system_window == 0 => ConfigError::with_src(
             CS::SOURCE,
-            ConfigErrorKind::ErrorString("invalid value for service window. must be nonzero".into())
-        )
+            ConfigErrorKind::ErrorString("invalid value for service window. must be nonzero".into()),
+        ),
+        if config.auth.root_key.len() <= ROOT_PASSWORD_MIN_LEN => ConfigError::with_src(
+            CS::SOURCE,
+            ConfigErrorKind::ErrorString("the root password must have at least 16 characters".into()),
+        ),
     );
     Ok(config)
 }
@@ -999,10 +998,9 @@ impl ConfigReturn {
 pub(super) fn apply_and_validate<CS: ConfigurationSource>(
     mut args: ParsedRawArgs,
 ) -> ConfigResult<ConfigReturn> {
-    let mut modcfg = ModifyGuard::new(DecodedConfiguration::default());
-    apply_config_changes::<CS>(&mut args, &mut modcfg)?;
-    if ModifyGuard::modified(&modcfg) {
-        validate_configuration::<CS>(modcfg.val).map(ConfigReturn::Config)
+    let cfg = apply_config_changes::<CS>(&mut args)?;
+    if ModifyGuard::modified(&cfg) {
+        validate_configuration::<CS>(cfg.val).map(ConfigReturn::Config)
     } else {
         Ok(ConfigReturn::Default)
     }
diff --git a/server/src/engine/fractal/config.rs b/server/src/engine/fractal/config.rs
index 1609171d..40188c47 100644
--- a/server/src/engine/fractal/config.rs
+++ b/server/src/engine/fractal/config.rs
@@ -35,53 +35,53 @@ use {
 #[derive(Debug)]
 /// The global system configuration
 pub struct SysConfig {
-    auth_data: Option<RwLock<SysAuth>>,
+    auth_data: RwLock<SysAuth>,
     host_data: SysHostData,
 }
 
 impl PartialEq for SysConfig {
     fn eq(&self, other: &Self) -> bool {
-        self.host_data == other.host_data
-            && match (self.auth_data.as_ref(), other.auth_data.as_ref()) {
-                (None, None) => true,
-                (Some(a), Some(b)) => &*a.read() == &*b.read(),
-                _ => false,
-            }
+        self.host_data == other.host_data && self.auth_data.read().eq(&other.auth_data.read())
     }
 }
 
 impl SysConfig {
     /// Initialize a new system config
-    pub fn new(auth_data: Option<RwLock<SysAuth>>, host_data: SysHostData) -> Self {
+    pub fn new(auth_data: RwLock<SysAuth>, host_data: SysHostData) -> Self {
         Self {
             auth_data,
             host_data,
         }
     }
-    pub fn new_auth(new_auth: Option<ConfigAuth>, host_data: SysHostData) -> Self {
-        match new_auth {
-            Some(ConfigAuth { root_key, .. }) => Self::new(
-                Some(RwLock::new(SysAuth::new(
-                    rcrypt::hash(root_key, rcrypt::DEFAULT_COST)
-                        .unwrap()
-                        .into_boxed_slice(),
-                    Default::default(),
-                ))),
-                host_data,
-            ),
-            None => Self::new(None, host_data),
-        }
+    pub fn new_full(new_auth: ConfigAuth, host_data: SysHostData) -> Self {
+        Self::new(
+            RwLock::new(SysAuth::new(
+                rcrypt::hash(new_auth.root_key, rcrypt::DEFAULT_COST)
+                    .unwrap()
+                    .into_boxed_slice(),
+                Default::default(),
+            )),
+            host_data,
+        )
+    }
+    pub fn new_auth(new_auth: ConfigAuth) -> Self {
+        Self::new_full(new_auth, SysHostData::new(0, 0))
     }
     #[cfg(test)]
-    /// A test-mode default setting with auth disabled
+    /// A test-mode default setting with the root password set to `password12345678`
     pub(super) fn test_default() -> Self {
         Self {
-            auth_data: None,
+            auth_data: RwLock::new(SysAuth::new(
+                rcrypt::hash("password12345678", rcrypt::DEFAULT_COST)
+                    .unwrap()
+                    .into_boxed_slice(),
+                Default::default(),
+            )),
             host_data: SysHostData::new(0, 0),
         }
     }
     /// Returns a handle to the authentication data
-    pub fn auth_data(&self) -> &Option<RwLock<SysAuth>> {
+    pub fn auth_data(&self) -> &RwLock<SysAuth> {
         &self.auth_data
     }
     /// Returns a reference to host data
diff --git a/server/src/engine/storage/v1/sysdb.rs b/server/src/engine/storage/v1/sysdb.rs
index d21501d6..7bb70c55 100644
--- a/server/src/engine/storage/v1/sysdb.rs
+++ b/server/src/engine/storage/v1/sysdb.rs
@@ -54,10 +54,6 @@ pub enum SystemStoreInitState {
     Unchanged,
     /// The system store was present, root settings were updated
     UpdatedRoot,
-    /// the system store was present, auth was previously enabled but is now disabled
-    UpdatedAuthDisabled,
-    /// the system store was present, auth was previously disabled but is now enabled
-    UpdatedAuthEnabled,
 }
 
 #[derive(Debug, PartialEq)]
@@ -77,56 +73,55 @@ impl SystemStoreInit {
 ///
 /// - If it doesn't exist, create it
 /// - If it exists, look for config changes and sync them
-pub fn open_system_database<Fs: RawFSInterface>(
-    auth: Option<ConfigAuth>,
-) -> SDSSResult<SystemStoreInit> {
+pub fn open_system_database<Fs: RawFSInterface>(auth: ConfigAuth) -> SDSSResult<SystemStoreInit> {
     open_or_reinit_system_database::<Fs>(auth, SYSDB_PATH, SYSDB_COW_PATH)
 }
 
 /// Open or re-initialize the system database
 pub fn open_or_reinit_system_database<Fs: RawFSInterface>(
-    auth: Option<ConfigAuth>,
+    auth: ConfigAuth,
     sysdb_path: &str,
     sysdb_path_cow: &str,
 ) -> SDSSResult<SystemStoreInit> {
-    let (ex, _) = match SDSSFileIO::<Fs>::open_or_create_perm_rw::<spec::SysDBV1>(sysdb_path)? {
-        FileOpen::Created(new_sysdb) => {
-            let syscfg = SysConfig::new_auth(auth, SysHostData::new(0, 0));
-            sync_system_database_to(&syscfg, new_sysdb)?;
-            return Ok(SystemStoreInit::new(syscfg, SystemStoreInitState::Created));
+    let sysdb_file = match SDSSFileIO::<Fs>::open_or_create_perm_rw::<spec::SysDBV1>(sysdb_path)? {
+        FileOpen::Created(new) => {
+            // init new syscfg
+            let new_syscfg = SysConfig::new_auth(auth);
+            sync_system_database_to(&new_syscfg, new)?;
+            return Ok(SystemStoreInit::new(
+                new_syscfg,
+                SystemStoreInitState::Created,
+            ));
         }
-        FileOpen::Existing(ex) => ex,
+        FileOpen::Existing((ex, _)) => ex,
     };
-    let last_syscfg = decode_system_database(ex)?;
-    let mut state = SystemStoreInitState::Unchanged;
-    match (last_syscfg.auth_data(), &auth) {
-        (Some(last_auth), Some(new_auth)) => {
-            let last_auth = last_auth.read();
-            if last_auth.verify_user("root", &new_auth.root_key).is_err() {
-                // the root password was changed
-                state = SystemStoreInitState::UpdatedRoot;
-            }
-        }
-        (Some(_), None) => {
-            state = SystemStoreInitState::UpdatedAuthDisabled;
-        }
-        (None, Some(_)) => {
-            state = SystemStoreInitState::UpdatedAuthEnabled;
-        }
-        (None, None) => {}
+    let prev_sysdb = decode_system_database(sysdb_file)?;
+    let state;
+    // see if settings have changed
+    if prev_sysdb
+        .auth_data()
+        .read()
+        .verify_user("root", &auth.root_key)
+        .is_ok()
+    {
+        state = SystemStoreInitState::Unchanged;
+    } else {
+        state = SystemStoreInitState::UpdatedRoot;
     }
-    let new_syscfg = SysConfig::new_auth(
+    // create new config
+    let new_syscfg = SysConfig::new_full(
         auth,
         SysHostData::new(
-            last_syscfg.host_data().startup_counter() + 1,
-            last_syscfg.host_data().settings_version()
+            prev_sysdb.host_data().startup_counter() + 1,
+            prev_sysdb.host_data().settings_version()
                 + !matches!(state, SystemStoreInitState::Unchanged) as u32,
         ),
     );
     // sync
-    let cow_file = SDSSFileIO::<Fs>::create::<spec::SysDBV1>(sysdb_path_cow)?;
-    sync_system_database_to(&new_syscfg, cow_file)?;
-    // replace
+    sync_system_database_to(
+        &new_syscfg,
+        SDSSFileIO::<Fs>::create::<spec::SysDBV1>(sysdb_path_cow)?,
+    )?;
     Fs::fs_rename_file(sysdb_path_cow, sysdb_path)?;
     Ok(SystemStoreInit::new(new_syscfg, state))
 }
@@ -145,38 +140,29 @@ pub fn sync_system_database_to<Fs: RawFSInterface>(
         SYS_KEY_AUTH => DictGeneric::new(),
     );
     let auth_key = map.get_mut(SYS_KEY_AUTH).unwrap();
-    match cfg.auth_data() {
-        None => {
-            *auth_key = DictEntryGeneric::Map(
-                into_dict!(SYS_KEY_AUTH_ROOT => Datacell::null(), SYS_KEY_AUTH_USERS => Datacell::null()),
-            )
-        }
-        Some(auth) => {
-            let auth = auth.read();
-            let auth_key = auth_key.as_dict_mut().unwrap();
-            auth_key.insert(
-                SYS_KEY_AUTH_ROOT.into(),
-                DictEntryGeneric::Data(Datacell::new_bin(auth.root_key().into())),
-            );
-            auth_key.insert(
-                SYS_KEY_AUTH_USERS.into(),
-                DictEntryGeneric::Map(
-                    // username -> [..settings]
-                    auth.users()
-                        .iter()
-                        .map(|(username, user)| {
-                            (
-                                username.to_owned(),
-                                DictEntryGeneric::Data(Datacell::new_list(vec![
-                                    Datacell::new_bin(user.key().into()),
-                                ])),
-                            )
-                        })
-                        .collect(),
-                ),
-            );
-        }
-    }
+    let auth = cfg.auth_data().read();
+    let auth_key = auth_key.as_dict_mut().unwrap();
+    auth_key.insert(
+        SYS_KEY_AUTH_ROOT.into(),
+        DictEntryGeneric::Data(Datacell::new_bin(auth.root_key().into())),
+    );
+    auth_key.insert(
+        SYS_KEY_AUTH_USERS.into(),
+        DictEntryGeneric::Map(
+            // username -> [..settings]
+            auth.users()
+                .iter()
+                .map(|(username, user)| {
+                    (
+                        username.to_owned(),
+                        DictEntryGeneric::Data(Datacell::new_list(vec![Datacell::new_bin(
+                            user.key().into(),
+                        )])),
+                    )
+                })
+                .collect(),
+        ),
+    );
     // write
     let buf = super::inf::enc::enc_dict_full::<super::inf::map::GenericDictSpec>(&map);
     f.fsynced_write(&buf)
@@ -195,59 +181,49 @@ fn rkey<T>(
 
 /// Decode the system database
 pub fn decode_system_database<Fs: RawFSInterface>(mut f: SDSSFileIO<Fs>) -> SDSSResult<SysConfig> {
-    let rem = f.load_remaining_into_buffer()?;
-    let mut store = inf::dec::dec_dict_full::<inf::map::GenericDictSpec>(&rem)?;
-    // find auth and sys stores
-    let mut auth_store = rkey(&mut store, SYS_KEY_AUTH, DictEntryGeneric::into_dict)?;
-    let mut sys_store = rkey(&mut store, SYS_KEY_SYS, DictEntryGeneric::into_dict)?;
-    // get our auth
-    let auth_root = rkey(&mut auth_store, SYS_KEY_AUTH_ROOT, |dict| {
-        let data = dict.into_data()?;
-        match data.kind() {
-            _ if data.is_null() => Some(None),
-            _ => data.into_bin().map(Some),
-        }
+    let mut sysdb_data =
+        inf::dec::dec_dict_full::<inf::map::GenericDictSpec>(&f.load_remaining_into_buffer()?)?;
+    // get our auth and sys stores
+    let mut auth_store = rkey(&mut sysdb_data, SYS_KEY_AUTH, DictEntryGeneric::into_dict)?;
+    let mut sys_store = rkey(&mut sysdb_data, SYS_KEY_SYS, DictEntryGeneric::into_dict)?;
+    // load auth store
+    let root_key = rkey(&mut auth_store, SYS_KEY_AUTH_ROOT, |d| {
+        d.into_data()?.into_bin()
     })?;
-    let auth_users = rkey(&mut auth_store, SYS_KEY_AUTH_USERS, |dict| match dict {
-        DictEntryGeneric::Data(dc) if dc.is_null() => Some(None),
-        DictEntryGeneric::Map(m) => Some(Some(m)),
-        _ => None,
-    })?;
-    let sys_auth = match (auth_root, auth_users) {
-        (Some(root_pass), Some(users)) => {
-            let mut usermap = HashMap::new();
-            for (user_name, user) in users {
-                let mut user_data = user
-                    .into_data()
-                    .and_then(|d| d.into_list())
-                    .ok_or(SDSSError::SysDBCorrupted)?;
-                if user_data.len() != 1 {
-                    return Err(SDSSError::SysDBCorrupted);
-                }
-                let password = user_data
-                    .remove(0)
-                    .into_bin()
-                    .ok_or(SDSSError::SysDBCorrupted)?;
-                usermap.insert(user_name, SysAuthUser::new(password.into_boxed_slice()));
-            }
-            Some(RwLock::new(SysAuth::new(
-                root_pass.into_boxed_slice(),
-                usermap,
-            )))
+    let users = rkey(
+        &mut auth_store,
+        SYS_KEY_AUTH_USERS,
+        DictEntryGeneric::into_dict,
+    )?;
+    // load users
+    let mut loaded_users = HashMap::new();
+    for (username, userdata) in users {
+        let mut userdata = userdata
+            .into_data()
+            .and_then(Datacell::into_list)
+            .ok_or(SDSSError::SysDBCorrupted)?;
+        if userdata.len() != 1 {
+            return Err(SDSSError::SysDBCorrupted);
         }
-        (None, None) => None,
-        _ => return Err(SDSSError::SysDBCorrupted),
-    };
-    // get our sys
-    let sv = rkey(&mut sys_store, SYS_KEY_SYS_SETTINGS_VERSION, |de| {
-        de.into_data()?.into_uint()
+        let user_password = userdata
+            .remove(0)
+            .into_bin()
+            .ok_or(SDSSError::SysDBCorrupted)?;
+        loaded_users.insert(username, SysAuthUser::new(user_password.into_boxed_slice()));
+    }
+    let sys_auth = SysAuth::new(root_key.into_boxed_slice(), loaded_users);
+    // load sys data
+    let sc = rkey(&mut sys_store, SYS_KEY_SYS_STARTUP_COUNTER, |d| {
+        d.into_data()?.into_uint()
     })?;
-    let sc = rkey(&mut sys_store, SYS_KEY_SYS_STARTUP_COUNTER, |de| {
-        de.into_data()?.into_uint()
+    let sv = rkey(&mut sys_store, SYS_KEY_SYS_SETTINGS_VERSION, |d| {
+        d.into_data()?.into_uint()
     })?;
-    if !(sys_store.is_empty() & auth_store.is_empty() & store.is_empty()) {
-        // the stores have more keys than we expected. something is wrong here
+    if !(sysdb_data.is_empty() & auth_store.is_empty() & sys_store.is_empty()) {
         return Err(SDSSError::SysDBCorrupted);
     }
-    Ok(SysConfig::new(sys_auth, SysHostData::new(sc, sv as u32)))
+    Ok(SysConfig::new(
+        RwLock::new(sys_auth),
+        SysHostData::new(sc, sv as u32),
+    ))
 }
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index 5bfd8817..b6aa9802 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -38,138 +38,78 @@ mod sysdb {
         },
         crate::engine::config::{AuthDriver, ConfigAuth},
     };
-    #[test]
-    fn simple_open_close() {
-        {
-            let syscfg_new = sysdb::open_or_reinit_system_database::<VFS>(
-                None,
-                "sysdb_test_1.db",
-                "sysdb_test_1.cow.db",
-            )
-            .unwrap();
-            assert_eq!(syscfg_new.state, SystemStoreInitState::Created);
-            assert!(syscfg_new.store.auth_data().is_none());
-            assert_eq!(syscfg_new.store.host_data().settings_version(), 0);
-            assert_eq!(syscfg_new.store.host_data().startup_counter(), 0);
-        }
-        let syscfg_restore = sysdb::open_or_reinit_system_database::<VFS>(
-            None,
-            "sysdb_test_1.db",
-            "sysdb_test_1.cow.db",
-        )
-        .unwrap();
-        assert_eq!(syscfg_restore.state, SystemStoreInitState::Unchanged);
-        assert!(syscfg_restore.store.auth_data().is_none());
-        assert_eq!(syscfg_restore.store.host_data().settings_version(), 0);
-        assert_eq!(syscfg_restore.store.host_data().startup_counter(), 1);
+    fn open_sysdb(
+        auth_config: ConfigAuth,
+        sysdb_path: &str,
+        sysdb_cow_path: &str,
+    ) -> sysdb::SystemStoreInit {
+        sysdb::open_or_reinit_system_database::<VFS>(auth_config, sysdb_path, sysdb_cow_path)
+            .unwrap()
     }
     #[test]
-    fn with_auth_nochange() {
-        let auth = ConfigAuth::new(AuthDriver::Pwd, "password12345678".to_string());
-        {
-            let syscfg_new = sysdb::open_or_reinit_system_database::<VFS>(
-                Some(auth.clone()),
-                "sysdb_test_2.db",
-                "sysdb_test_2.cow.db",
+    fn open_close() {
+        let open = |auth_config| {
+            open_sysdb(
+                auth_config,
+                "open_close_test.sys.db",
+                "open_close_test.sys.cow.db",
             )
-            .unwrap();
-            assert_eq!(syscfg_new.state, SystemStoreInitState::Created);
-            assert!(syscfg_new
+        };
+        let auth_config = ConfigAuth::new(AuthDriver::Pwd, "password12345678".into());
+        {
+            let config = open(auth_config.clone());
+            assert_eq!(config.state, SystemStoreInitState::Created);
+            assert!(config
                 .store
                 .auth_data()
-                .as_ref()
-                .unwrap()
                 .read()
                 .verify_user("root", "password12345678")
                 .is_ok());
-            assert_eq!(syscfg_new.store.host_data().startup_counter(), 0);
-            assert_eq!(syscfg_new.store.host_data().settings_version(), 0);
+            assert_eq!(config.store.host_data().settings_version(), 0);
+            assert_eq!(config.store.host_data().startup_counter(), 0);
         }
-        // now reboot
-        let syscfg_new = sysdb::open_or_reinit_system_database::<VFS>(
-            Some(auth),
-            "sysdb_test_2.db",
-            "sysdb_test_2.cow.db",
-        )
-        .unwrap();
-        assert_eq!(syscfg_new.state, SystemStoreInitState::Unchanged);
-        assert!(syscfg_new
+        // reboot
+        let config = open(auth_config);
+        assert_eq!(config.state, SystemStoreInitState::Unchanged);
+        assert!(config
             .store
             .auth_data()
-            .as_ref()
-            .unwrap()
             .read()
             .verify_user("root", "password12345678")
             .is_ok());
-        assert_eq!(syscfg_new.store.host_data().startup_counter(), 1);
-        assert_eq!(syscfg_new.store.host_data().settings_version(), 0);
+        assert_eq!(config.store.host_data().settings_version(), 0);
+        assert_eq!(config.store.host_data().startup_counter(), 1);
     }
     #[test]
-    fn disable_auth() {
-        {
-            let auth = ConfigAuth::new(AuthDriver::Pwd, "password12345678".to_string());
-            let syscfg_new = sysdb::open_or_reinit_system_database::<VFS>(
-                Some(auth),
-                "sysdb_test_3.db",
-                "sysdb_test_3.cow.db",
+    fn open_change_root_password() {
+        let open = |auth_config| {
+            open_sysdb(
+                auth_config,
+                "open_change_root_password.sys.db",
+                "open_change_root_password.sys.cow.db",
             )
-            .unwrap();
-            assert_eq!(syscfg_new.state, SystemStoreInitState::Created);
-            assert!(syscfg_new
+        };
+        {
+            let config = open(ConfigAuth::new(AuthDriver::Pwd, "password12345678".into()));
+            assert_eq!(config.state, SystemStoreInitState::Created);
+            assert!(config
                 .store
                 .auth_data()
-                .as_ref()
-                .unwrap()
                 .read()
                 .verify_user("root", "password12345678")
                 .is_ok());
-            assert_eq!(syscfg_new.store.host_data().startup_counter(), 0);
-            assert_eq!(syscfg_new.store.host_data().settings_version(), 0);
-        }
-        // reboot
-        let sysdb_cfg = sysdb::open_or_reinit_system_database::<VFS>(
-            None,
-            "sysdb_test_3.db",
-            "sysdb_test_3.cow.db",
-        )
-        .unwrap();
-        assert_eq!(sysdb_cfg.state, SystemStoreInitState::UpdatedAuthDisabled);
-        assert!(sysdb_cfg.store.auth_data().is_none());
-        assert_eq!(sysdb_cfg.store.host_data().startup_counter(), 1);
-        assert_eq!(sysdb_cfg.store.host_data().settings_version(), 1);
-    }
-    #[test]
-    fn enable_auth() {
-        {
-            let sysdb_cfg = sysdb::open_or_reinit_system_database::<VFS>(
-                None,
-                "sysdb_test_4.db",
-                "sysdb_test_4.cow.db",
-            )
-            .unwrap();
-            assert_eq!(sysdb_cfg.state, SystemStoreInitState::Created);
-            assert!(sysdb_cfg.store.auth_data().is_none());
-            assert_eq!(sysdb_cfg.store.host_data().startup_counter(), 0);
-            assert_eq!(sysdb_cfg.store.host_data().settings_version(), 0);
+            assert_eq!(config.store.host_data().settings_version(), 0);
+            assert_eq!(config.store.host_data().startup_counter(), 0);
         }
-        // reboot
-        let auth = ConfigAuth::new(AuthDriver::Pwd, "password12345678".to_string());
-        let syscfg_new = sysdb::open_or_reinit_system_database::<VFS>(
-            Some(auth),
-            "sysdb_test_4.db",
-            "sysdb_test_4.cow.db",
-        )
-        .unwrap();
-        assert_eq!(syscfg_new.state, SystemStoreInitState::UpdatedAuthEnabled);
-        assert!(syscfg_new
+        let config = open(ConfigAuth::new(AuthDriver::Pwd, "password23456789".into()));
+        assert_eq!(config.state, SystemStoreInitState::UpdatedRoot);
+        assert!(config
             .store
             .auth_data()
-            .as_ref()
-            .unwrap()
             .read()
-            .verify_user("root", "password12345678")
+            .verify_user("root", "password23456789")
             .is_ok());
-        assert_eq!(syscfg_new.store.host_data().startup_counter(), 1);
-        assert_eq!(syscfg_new.store.host_data().settings_version(), 1);
+        assert_eq!(config.store.host_data().settings_version(), 1);
+        assert_eq!(config.store.host_data().startup_counter(), 1);
     }
 }
diff --git a/server/src/engine/tests/mod.rs b/server/src/engine/tests/mod.rs
index 5e421df3..092acf3a 100644
--- a/server/src/engine/tests/mod.rs
+++ b/server/src/engine/tests/mod.rs
@@ -120,7 +120,7 @@ mod cfg {
                         ),
                         ConfigMode::Dev,
                         ConfigSystem::new(600),
-                        Some(ConfigAuth::new(AuthDriver::Pwd, "password12345678".into()))
+                        ConfigAuth::new(AuthDriver::Pwd, "password12345678".into())
                     )
                 )
             },
@@ -231,7 +231,7 @@ mod cfg {
                         ),
                         ConfigMode::Dev,
                         ConfigSystem::new(600),
-                        Some(ConfigAuth::new(AuthDriver::Pwd, "password12345678".into()))
+                        ConfigAuth::new(AuthDriver::Pwd, "password12345678".into())
                     )
                 )
             },
@@ -277,7 +277,7 @@ endpoints:
                         ),
                         ConfigMode::Dev,
                         ConfigSystem::new(600),
-                        Some(ConfigAuth::new(AuthDriver::Pwd, "password12345678".into()))
+                        ConfigAuth::new(AuthDriver::Pwd, "password12345678".into())
                     )
                 )
             },

From be540a7ded484741e3b0891bf678f3b172bed9b5 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 29 Sep 2023 09:34:22 +0000
Subject: [PATCH 269/310] Add net impls

Also cleaned up error impls
---
 server/src/engine/config.rs                   |  48 +++-
 server/src/engine/error.rs                    | 130 +++++++++-
 server/src/engine/fractal/mgr.rs              |   4 +-
 server/src/engine/fractal/mod.rs              |  28 ++-
 server/src/engine/mod.rs                      |   6 +
 server/src/engine/net/mod.rs                  | 236 +++++++++++++++++-
 server/src/engine/net/protocol/mod.rs         |   4 +-
 .../engine/storage/v1/batch_jrnl/persist.rs   |  12 +-
 .../engine/storage/v1/batch_jrnl/restore.rs   |  32 +--
 server/src/engine/storage/v1/inf/map.rs       |  25 +-
 server/src/engine/storage/v1/inf/mod.rs       |  10 +-
 server/src/engine/storage/v1/inf/obj.rs       |   8 +-
 server/src/engine/storage/v1/journal.rs       |  18 +-
 server/src/engine/storage/v1/loader.rs        |  15 +-
 server/src/engine/storage/v1/mod.rs           |  80 ++----
 server/src/engine/storage/v1/rw.rs            |  13 +-
 server/src/engine/storage/v1/spec.rs          |  10 +-
 server/src/engine/storage/v1/start_stop.rs    | 150 -----------
 server/src/engine/storage/v1/sysdb.rs         |  12 +-
 server/src/engine/storage/v1/tests/tx.rs      |  12 +-
 server/src/engine/tests/mod.rs                |  28 ++-
 server/src/engine/txn/gns/model.rs            |   4 +-
 22 files changed, 555 insertions(+), 330 deletions(-)
 delete mode 100644 server/src/engine/storage/v1/start_stop.rs

diff --git a/server/src/engine/config.rs b/server/src/engine/config.rs
index 19397155..4c75f009 100644
--- a/server/src/engine/config.rs
+++ b/server/src/engine/config.rs
@@ -147,14 +147,21 @@ pub struct ConfigEndpointTls {
     tcp: ConfigEndpointTcp,
     cert: String,
     private_key: String,
+    pkey_pass: String,
 }
 
 impl ConfigEndpointTls {
-    pub fn new(tcp: ConfigEndpointTcp, cert: String, private_key: String) -> Self {
+    pub fn new(
+        tcp: ConfigEndpointTcp,
+        cert: String,
+        private_key: String,
+        pkey_pass: String,
+    ) -> Self {
         Self {
             tcp,
             cert,
             private_key,
+            pkey_pass,
         }
     }
 }
@@ -264,6 +271,7 @@ pub struct DecodedEPSecureConfig {
     port: u16,
     cert: String,
     private_key: String,
+    pkey_passphrase: String,
 }
 
 #[derive(Debug, PartialEq, Deserialize)]
@@ -383,6 +391,7 @@ pub(super) trait ConfigurationSource {
     const KEY_AUTH_ROOT_PASSWORD: &'static str;
     const KEY_TLS_CERT: &'static str;
     const KEY_TLS_KEY: &'static str;
+    const KEY_TLS_PKEY_PASS: &'static str;
     const KEY_ENDPOINTS: &'static str;
     const KEY_RUN_MODE: &'static str;
     const KEY_SERVICE_WINDOW: &'static str;
@@ -463,16 +472,19 @@ fn parse_endpoint(source: ConfigSource, s: &str) -> ConfigResult<(ConnectionProt
 fn decode_tls_ep(
     cert_path: &str,
     key_path: &str,
+    pkey_pass: &str,
     host: &str,
     port: u16,
 ) -> ConfigResult<DecodedEPSecureConfig> {
     let tls_key = fs::read_to_string(key_path)?;
     let tls_cert = fs::read_to_string(cert_path)?;
+    let tls_priv_key_passphrase = fs::read_to_string(pkey_pass)?;
     Ok(DecodedEPSecureConfig {
         host: host.into(),
         port,
         cert: tls_cert,
         private_key: tls_key,
+        pkey_passphrase: tls_priv_key_passphrase,
     })
 }
 
@@ -484,22 +496,31 @@ fn arg_decode_tls_endpoint<CS: ConfigurationSource>(
 ) -> ConfigResult<DecodedEPSecureConfig> {
     let _cert = args.remove(CS::KEY_TLS_CERT);
     let _key = args.remove(CS::KEY_TLS_KEY);
-    let (tls_cert, tls_key) = match (_cert, _key) {
-        (Some(cert), Some(key)) => (cert, key),
+    let _passphrase = args.remove(CS::KEY_TLS_PKEY_PASS);
+    let (tls_cert, tls_key, tls_passphrase) = match (_cert, _key, _passphrase) {
+        (Some(cert), Some(key), Some(pass)) => (cert, key, pass),
         _ => {
             return Err(ConfigError::with_src(
                 ConfigSource::Cli,
                 ConfigErrorKind::ErrorString(format!(
-                    "must supply values for both `{}` and `{}` when using TLS",
+                    "must supply values for `{}`, `{}` and `{}` when using TLS",
                     CS::KEY_TLS_CERT,
-                    CS::KEY_TLS_KEY
+                    CS::KEY_TLS_KEY,
+                    CS::KEY_TLS_PKEY_PASS,
                 )),
             ));
         }
     };
     argck_duplicate_values::<CS>(&tls_cert, CS::KEY_TLS_CERT)?;
     argck_duplicate_values::<CS>(&tls_key, CS::KEY_TLS_KEY)?;
-    Ok(decode_tls_ep(&tls_cert[0], &tls_key[0], host, port)?)
+    argck_duplicate_values::<CS>(&tls_passphrase, CS::KEY_TLS_PKEY_PASS)?;
+    Ok(decode_tls_ep(
+        &tls_cert[0],
+        &tls_key[0],
+        &tls_passphrase[0],
+        host,
+        port,
+    )?)
 }
 
 /*
@@ -747,7 +768,7 @@ pub fn parse_cli_args<'a, T: 'a + AsRef<str>>(
 
 /// Parse environment variables
 pub fn parse_env_args() -> ConfigResult<Option<ParsedRawArgs>> {
-    const KEYS: [&str; 7] = [
+    const KEYS: [&str; 8] = [
         CSEnvArgs::KEY_AUTH_DRIVER,
         CSEnvArgs::KEY_AUTH_ROOT_PASSWORD,
         CSEnvArgs::KEY_ENDPOINTS,
@@ -755,6 +776,7 @@ pub fn parse_env_args() -> ConfigResult<Option<ParsedRawArgs>> {
         CSEnvArgs::KEY_SERVICE_WINDOW,
         CSEnvArgs::KEY_TLS_CERT,
         CSEnvArgs::KEY_TLS_KEY,
+        CSEnvArgs::KEY_TLS_PKEY_PASS,
     ];
     let mut ret = HashMap::new();
     for key in KEYS {
@@ -853,6 +875,7 @@ impl ConfigurationSource for CSCommandLine {
     const KEY_AUTH_ROOT_PASSWORD: &'static str = "--auth-root-password";
     const KEY_TLS_CERT: &'static str = "--tlscert";
     const KEY_TLS_KEY: &'static str = "--tlskey";
+    const KEY_TLS_PKEY_PASS: &'static str = "--tls-passphrase";
     const KEY_ENDPOINTS: &'static str = "--endpoint";
     const KEY_RUN_MODE: &'static str = "--mode";
     const KEY_SERVICE_WINDOW: &'static str = "--service-window";
@@ -865,6 +888,7 @@ impl ConfigurationSource for CSEnvArgs {
     const KEY_AUTH_ROOT_PASSWORD: &'static str = "SKYDB_AUTH_ROOT_PASSWORD";
     const KEY_TLS_CERT: &'static str = "SKYDB_TLS_CERT";
     const KEY_TLS_KEY: &'static str = "SKYDB_TLS_KEY";
+    const KEY_TLS_PKEY_PASS: &'static str = "SKYDB_TLS_PRIVATE_KEY_PASSWORD";
     const KEY_ENDPOINTS: &'static str = "SKYDB_ENDPOINTS";
     const KEY_RUN_MODE: &'static str = "SKYDB_RUN_MODE";
     const KEY_SERVICE_WINDOW: &'static str = "SKYDB_SERVICE_WINDOW";
@@ -877,6 +901,7 @@ impl ConfigurationSource for CSConfigFile {
     const KEY_AUTH_ROOT_PASSWORD: &'static str = "auth.root_password";
     const KEY_TLS_CERT: &'static str = "endpoints.secure.cert";
     const KEY_TLS_KEY: &'static str = "endpoints.secure.key";
+    const KEY_TLS_PKEY_PASS: &'static str = "endpoints.secure.pkey_passphrase";
     const KEY_ENDPOINTS: &'static str = "endpoints";
     const KEY_RUN_MODE: &'static str = "system.mode";
     const KEY_SERVICE_WINDOW: &'static str = "system.service_window";
@@ -937,10 +962,11 @@ fn validate_configuration<CS: ConfigurationSource>(
                 let secure_ep = ConfigEndpointTls {
                     tcp: ConfigEndpointTcp {
                         host: secure.host,
-                        port: secure.port
+                        port: secure.port,
                     },
                     cert: secure.cert,
-                    private_key: secure.private_key
+                    private_key: secure.private_key,
+                    pkey_pass: secure.pkey_passphrase,
                 };
                 match &config.endpoints {
                     ConfigEndpoint::Insecure(is) => if has_insecure {
@@ -961,7 +987,7 @@ fn validate_configuration<CS: ConfigurationSource>(
             CS::SOURCE,
             ConfigErrorKind::ErrorString("invalid value for service window. must be nonzero".into()),
         ),
-        if config.auth.root_key.len() <= ROOT_PASSWORD_MIN_LEN => ConfigError::with_src(
+        if config.auth.root_key.len() < ROOT_PASSWORD_MIN_LEN => ConfigError::with_src(
             CS::SOURCE,
             ConfigErrorKind::ErrorString("the root password must have at least 16 characters".into()),
         ),
@@ -1175,8 +1201,10 @@ fn check_config_file(
                 Some(secure_ep) => {
                     let cert = fs::read_to_string(&secure_ep.cert)?;
                     let private_key = fs::read_to_string(&secure_ep.private_key)?;
+                    let private_key_passphrase = fs::read_to_string(&secure_ep.pkey_passphrase)?;
                     secure_ep.cert = cert;
                     secure_ep.private_key = private_key;
+                    secure_ep.pkey_passphrase = private_key_passphrase;
                 }
                 None => {}
             },
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index ecf1c60e..72ebe463 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -24,8 +24,20 @@
  *
 */
 
-use super::{storage::v1::SDSSError, txn::TransactionError};
+use {
+    super::{
+        storage::v1::{SDSSError, SDSSErrorKind},
+        txn::TransactionError,
+    },
+    crate::util::os::SysIOError,
+    std::fmt,
+};
+
 pub type QueryResult<T> = Result<T, Error>;
+// stack
+pub type CtxResult<T, E> = Result<T, CtxError<E>>;
+pub type RuntimeResult<T> = CtxResult<T, RuntimeErrorKind>;
+pub type RuntimeError = CtxError<RuntimeErrorKind>;
 
 /// an enumeration of 'flat' errors that the server actually responds to the client with, since we do not want to send specific information
 /// about anything (as that will be a security hole). The variants correspond with their actual response codes
@@ -102,3 +114,119 @@ direct_from! {
         TransactionError as TransactionalError,
     }
 }
+
+/*
+    contextual errors
+*/
+
+/// An error context
+pub enum CtxErrorDescription {
+    A(&'static str),
+    B(Box<str>),
+}
+
+impl CtxErrorDescription {
+    fn inner(&self) -> &str {
+        match self {
+            Self::A(a) => a,
+            Self::B(b) => &b,
+        }
+    }
+}
+
+impl PartialEq for CtxErrorDescription {
+    fn eq(&self, other: &Self) -> bool {
+        self.inner() == other.inner()
+    }
+}
+
+impl fmt::Display for CtxErrorDescription {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(self.inner())
+    }
+}
+
+impl fmt::Debug for CtxErrorDescription {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(self.inner())
+    }
+}
+
+direct_from! {
+    CtxErrorDescription => {
+        &'static str as A,
+        String as B,
+        Box<str> as B,
+    }
+}
+
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+/// A contextual error
+pub struct CtxError<E> {
+    kind: E,
+    ctx: Option<CtxErrorDescription>,
+}
+
+impl<E> CtxError<E> {
+    fn _new(kind: E, ctx: Option<CtxErrorDescription>) -> Self {
+        Self { kind, ctx }
+    }
+    pub fn new(kind: E) -> Self {
+        Self::_new(kind, None)
+    }
+    pub fn with_ctx(kind: E, ctx: impl Into<CtxErrorDescription>) -> Self {
+        Self::_new(kind, Some(ctx.into()))
+    }
+    pub fn add_ctx(self, ctx: impl Into<CtxErrorDescription>) -> Self {
+        Self::with_ctx(self.kind, ctx)
+    }
+    pub fn into_result<T>(self) -> CtxResult<T, E> {
+        Err(self)
+    }
+    pub fn result<T, F>(result: Result<T, F>) -> CtxResult<T, E>
+    where
+        E: From<F>,
+    {
+        result.map_err(|e| CtxError::new(e.into()))
+    }
+    pub fn result_ctx<T, F>(
+        result: Result<T, F>,
+        ctx: impl Into<CtxErrorDescription>,
+    ) -> CtxResult<T, E>
+    where
+        E: From<F>,
+    {
+        result.map_err(|e| CtxError::with_ctx(e.into(), ctx))
+    }
+}
+
+macro_rules! impl_from_hack {
+    ($($ty:ty),*) => {
+        $(impl<E> From<E> for CtxError<$ty> where E: Into<$ty> {fn from(e: E) -> Self { CtxError::new(e.into()) }})*
+    }
+}
+
+/*
+    Contextual error impls
+*/
+
+impl_from_hack!(RuntimeErrorKind, SDSSErrorKind);
+
+#[derive(Debug)]
+pub enum RuntimeErrorKind {
+    StorageSubsytem(SDSSError),
+    IoError(SysIOError),
+    OSSLErrorMulti(openssl::error::ErrorStack),
+    OSSLError(openssl::ssl::Error),
+}
+
+direct_from! {
+    RuntimeErrorKind => {
+        SDSSError as StorageSubsytem,
+        std::io::Error as IoError,
+        SysIOError as IoError,
+        openssl::error::ErrorStack as OSSLErrorMulti,
+        openssl::ssl::Error as OSSLError,
+    }
+}
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 72f46d29..72ea4638 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -174,8 +174,8 @@ impl FractalMgr {
         hp_receiver: UnboundedReceiver<Task<CriticalTask>>,
     ) -> FractalServiceHandles {
         let fractal_mgr = global.get_state().fractal_mgr();
-        let global_1 = global.__global_clone();
-        let global_2 = global.__global_clone();
+        let global_1 = global.clone();
+        let global_2 = global.clone();
         let hp_handle = tokio::spawn(async move {
             FractalMgr::hp_executor_svc(fractal_mgr, global_1, hp_receiver).await
         });
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index 46159de8..25fa5866 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -53,8 +53,6 @@ pub use {
 
 pub type ModelDrivers<Fs> = HashMap<ModelUniqueID, drivers::FractalModelDriver<Fs>>;
 
-static mut GLOBAL: MaybeUninit<GlobalState> = MaybeUninit::uninit();
-
 /*
     global state init
 */
@@ -89,10 +87,10 @@ pub unsafe fn enable_and_start_all(
         mgr::FractalMgr::new(hp_sender, lp_sender, model_cnt_on_boot),
         config,
     );
-    GLOBAL = MaybeUninit::new(global_state);
+    *Global::__gref_raw() = MaybeUninit::new(global_state);
     let token = Global::new();
     GlobalStateStart {
-        global: token.__global_clone(),
+        global: token.clone(),
         mgr_handles: mgr::FractalMgr::start_all(token, lp_recv, hp_recv),
     }
 }
@@ -187,7 +185,7 @@ impl GlobalInstanceLike for Global {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone, Copy)]
 /// A handle to the global state
 pub struct Global(());
 
@@ -195,18 +193,12 @@ impl Global {
     unsafe fn new() -> Self {
         Self(())
     }
-    fn __global_clone(&self) -> Self {
-        unsafe {
-            // UNSAFE(@ohsayan): safe to call within this module
-            Self::new()
-        }
-    }
     fn get_state(&self) -> &'static GlobalState {
-        unsafe { GLOBAL.assume_init_ref() }
+        unsafe { self.__gref() }
     }
     /// Returns a handle to the [`GlobalNS`]
     fn _namespace(&self) -> &'static GlobalNS {
-        &unsafe { GLOBAL.assume_init_ref() }.gns
+        &unsafe { self.__gref() }.gns
     }
     /// Post an urgent task
     fn _post_high_priority_task(&self, task: Task<CriticalTask>) {
@@ -227,6 +219,16 @@ impl Global {
             .get_rt_stat()
             .per_mdl_delta_max_size()
     }
+    unsafe fn __gref_raw() -> &'static mut MaybeUninit<GlobalState> {
+        static mut G: MaybeUninit<GlobalState> = MaybeUninit::uninit();
+        &mut G
+    }
+    unsafe fn __gref(&self) -> &'static GlobalState {
+        Self::__gref_raw().assume_init_ref()
+    }
+    pub unsafe fn unload_all(self) {
+        core::ptr::drop_in_place(Self::__gref_raw().as_mut_ptr())
+    }
 }
 
 /*
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index db600b35..a7f43885 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -43,3 +43,9 @@ mod txn;
 // test
 #[cfg(test)]
 mod tests;
+
+use error::RuntimeResult;
+
+pub fn load_all() -> RuntimeResult<fractal::Global> {
+    todo!()
+}
diff --git a/server/src/engine/net/mod.rs b/server/src/engine/net/mod.rs
index 213135b9..c90db058 100644
--- a/server/src/engine/net/mod.rs
+++ b/server/src/engine/net/mod.rs
@@ -24,13 +24,247 @@
  *
 */
 
-use tokio::io::{AsyncRead, AsyncWrite};
 mod protocol;
 
+use {
+    crate::engine::{
+        error::{RuntimeError, RuntimeResult},
+        fractal::Global,
+    },
+    bytes::BytesMut,
+    openssl::{
+        pkey::PKey,
+        ssl::Ssl,
+        ssl::{SslAcceptor, SslMethod},
+        x509::X509,
+    },
+    std::{cell::Cell, net::SocketAddr, pin::Pin, time::Duration},
+    tokio::{
+        io::{AsyncRead, AsyncWrite, BufWriter},
+        net::{TcpListener, TcpStream},
+        sync::{broadcast, mpsc, Semaphore},
+    },
+    tokio_openssl::SslStream,
+};
+
 pub trait Socket: AsyncWrite + AsyncRead + Unpin {}
 pub type IoResult<T> = Result<T, std::io::Error>;
 
+const BUF_WRITE_CAP: usize = 16384;
+const BUF_READ_CAP: usize = 16384;
+const CLIMIT: usize = 50000;
+
+static CLIM: Semaphore = Semaphore::const_new(CLIMIT);
+
+/*
+    socket definitions
+*/
+
+impl Socket for TcpStream {}
+impl Socket for SslStream<TcpStream> {}
+
 pub enum QLoopReturn {
     Fin,
     ConnectionRst,
 }
+
+struct NetBackoff {
+    at: Cell<u8>,
+}
+
+impl NetBackoff {
+    const BACKOFF_MAX: u8 = 64;
+    fn new() -> Self {
+        Self { at: Cell::new(1) }
+    }
+    async fn spin(&self) {
+        let current = self.at.get();
+        self.at.set(current << 1);
+        tokio::time::sleep(Duration::from_secs(current as _)).await
+    }
+    fn should_disconnect(&self) -> bool {
+        self.at.get() >= Self::BACKOFF_MAX
+    }
+}
+
+/*
+    listener
+*/
+
+/// Connection handler for a remote connection
+pub struct ConnectionHandler<S> {
+    socket: BufWriter<S>,
+    buffer: BytesMut,
+    global: Global,
+    sig_terminate: broadcast::Receiver<()>,
+    _sig_inflight_complete: mpsc::Sender<()>,
+}
+
+impl<S: Socket> ConnectionHandler<S> {
+    pub fn new(
+        socket: S,
+        global: Global,
+        term_sig: broadcast::Receiver<()>,
+        _inflight_complete: mpsc::Sender<()>,
+    ) -> Self {
+        Self {
+            socket: BufWriter::with_capacity(BUF_WRITE_CAP, socket),
+            buffer: BytesMut::with_capacity(BUF_READ_CAP),
+            global,
+            sig_terminate: term_sig,
+            _sig_inflight_complete: _inflight_complete,
+        }
+    }
+    pub async fn run(&mut self) -> IoResult<()> {
+        let Self {
+            socket,
+            buffer,
+            global,
+            ..
+        } = self;
+        loop {
+            tokio::select! {
+                _ = protocol::query_loop(socket, buffer, global) => {},
+                _ = self.sig_terminate.recv() => {
+                    return Ok(())
+                }
+            }
+        }
+    }
+}
+
+/// A TCP listener bound to a socket
+pub struct Listener {
+    global: Global,
+    listener: TcpListener,
+    sig_shutdown: broadcast::Sender<()>,
+    sig_inflight: mpsc::Sender<()>,
+    sig_inflight_wait: mpsc::Receiver<()>,
+}
+
+impl Listener {
+    pub async fn new(
+        binaddr: &str,
+        global: Global,
+        sig_shutdown: broadcast::Sender<()>,
+    ) -> RuntimeResult<Self> {
+        let (sig_inflight, sig_inflight_wait) = mpsc::channel(1);
+        let listener = RuntimeError::result_ctx(
+            TcpListener::bind(binaddr).await,
+            format!("failed to bind to port `{binaddr}`"),
+        )?;
+        Ok(Self {
+            global,
+            listener,
+            sig_shutdown,
+            sig_inflight,
+            sig_inflight_wait,
+        })
+    }
+    pub async fn terminate(self) {
+        let Self {
+            mut sig_inflight_wait,
+            sig_inflight,
+            sig_shutdown,
+            ..
+        } = self;
+        drop(sig_shutdown);
+        drop(sig_inflight); // could be that we are the only ones holding this lol
+        let _ = sig_inflight_wait.recv().await; // wait
+    }
+    async fn accept(&mut self) -> IoResult<(TcpStream, SocketAddr)> {
+        let backoff = NetBackoff::new();
+        loop {
+            match self.listener.accept().await {
+                Ok(s) => return Ok(s),
+                Err(e) => {
+                    if backoff.should_disconnect() {
+                        // that's enough of your crappy connection dear sir
+                        return Err(e.into());
+                    }
+                }
+            }
+            backoff.spin().await;
+        }
+    }
+    async fn listen_tcp(&mut self) -> IoResult<()> {
+        loop {
+            // acquire a permit
+            let permit = CLIM.acquire().await.unwrap();
+            let (stream, _) = match self.accept().await {
+                Ok(s) => s,
+                Err(e) => {
+                    /*
+                        SECURITY: IGNORE THIS ERROR
+                    */
+                    log::error!("failed to accept connection on TCP socket: `{e}`");
+                    continue;
+                }
+            };
+            let mut handler = ConnectionHandler::new(
+                stream,
+                self.global,
+                self.sig_shutdown.subscribe(),
+                self.sig_inflight.clone(),
+            );
+            tokio::spawn(async move {
+                if let Err(e) = handler.run().await {
+                    log::error!("error handling client connection: `{e}`");
+                }
+            });
+            // return the permit
+            drop(permit);
+        }
+    }
+    async fn listen_tls(
+        self: &mut Self,
+        tls_cert: String,
+        tls_priv_key: String,
+        tls_key_password: String,
+    ) -> RuntimeResult<()> {
+        let build_acceptor = || {
+            let cert = X509::from_pem(tls_cert.as_bytes())?;
+            let priv_key = PKey::private_key_from_pem_passphrase(
+                tls_priv_key.as_bytes(),
+                tls_key_password.as_bytes(),
+            )?;
+            let mut builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls())?;
+            builder.set_certificate(&cert)?;
+            builder.set_private_key(&priv_key)?;
+            builder.check_private_key()?;
+            Ok::<_, openssl::error::ErrorStack>(builder.build())
+        };
+        let acceptor =
+            RuntimeError::result_ctx(build_acceptor(), "failed to initialize TLS socket")?;
+        loop {
+            let stream = async {
+                let (stream, _) = self.accept().await?;
+                let ssl = Ssl::new(acceptor.context())?;
+                let mut stream = SslStream::new(ssl, stream)?;
+                Pin::new(&mut stream).accept().await?;
+                RuntimeResult::Ok(stream)
+            };
+            let stream = match stream.await {
+                Ok(s) => s,
+                Err(e) => {
+                    /*
+                        SECURITY: Once again, ignore this error
+                    */
+                    log::error!("failed to accept connection on TLS socket: `{e:#?}`");
+                    continue;
+                }
+            };
+            let mut handler = ConnectionHandler::new(
+                stream,
+                self.global,
+                self.sig_shutdown.subscribe(),
+                self.sig_inflight.clone(),
+            );
+            tokio::spawn(async move {
+                if let Err(e) = handler.run().await {
+                    log::error!("error handling client TLS connection: `{e}`");
+                }
+            });
+        }
+    }
+}
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index 81bc3f2f..538ab007 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -32,7 +32,7 @@ mod tests;
 use {
     self::handshake::{CHandshake, HandshakeResult, HandshakeState},
     super::{IoResult, QLoopReturn, Socket},
-    crate::engine::mem::BufferedScanner,
+    crate::engine::{fractal::Global, mem::BufferedScanner},
     bytes::{Buf, BytesMut},
     tokio::io::{AsyncReadExt, BufWriter},
 };
@@ -40,6 +40,7 @@ use {
 pub async fn query_loop<S: Socket>(
     con: &mut BufWriter<S>,
     buf: &mut BytesMut,
+    _global: &Global,
 ) -> IoResult<QLoopReturn> {
     // handshake
     match do_handshake(con, buf).await? {
@@ -56,6 +57,7 @@ pub async fn query_loop<S: Socket>(
     }
 }
 
+#[inline(always)]
 fn see_if_connection_terminates(read_many: usize, buf: &[u8]) -> Option<QLoopReturn> {
     if read_many == 0 {
         // that's a connection termination
diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs
index f47ee07f..06ba5ad8 100644
--- a/server/src/engine/storage/v1/batch_jrnl/persist.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs
@@ -46,7 +46,7 @@ use {
             storage::v1::{
                 inf::PersistTypeDscr,
                 rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedWriter},
-                SDSSError, SDSSResult,
+                SDSSErrorKind, SDSSResult,
             },
         },
         util::EndianQW,
@@ -76,7 +76,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         {
             return Ok(());
         } else {
-            return Err(SDSSError::DataBatchCloseError);
+            return Err(SDSSErrorKind::DataBatchCloseError.into());
         }
     }
     pub fn write_new_batch(&mut self, model: &Model, observed_len: usize) -> SDSSResult<()> {
@@ -154,7 +154,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         schema_version: DeltaVersion,
         pk_tag: TagUnique,
         col_cnt: usize,
-    ) -> Result<(), SDSSError> {
+    ) -> SDSSResult<()> {
         self.f
             .unfsynced_write(&[MARKER_ACTUAL_BATCH_EVENT, pk_tag.value_u8()])?;
         let observed_len_bytes = observed_len.u64_bytes_le();
@@ -169,7 +169,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         &mut self,
         observed_len: usize,
         inconsistent_reads: usize,
-    ) -> Result<(), SDSSError> {
+    ) -> SDSSResult<()> {
         // [0xFD][actual_commit][checksum]
         self.f.unfsynced_write(&[MARKER_END_OF_BATCH])?;
         let actual_commit = (observed_len - inconsistent_reads).u64_bytes_le();
@@ -189,7 +189,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         if f.fsynced_write(&[MARKER_RECOVERY_EVENT]).is_ok() {
             return Ok(());
         }
-        Err(SDSSError::DataBatchRecoveryFailStageOne)
+        Err(SDSSErrorKind::DataBatchRecoveryFailStageOne.into())
     }
 }
 
@@ -288,7 +288,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         Ok(())
     }
     /// Write the change type and txnid
-    fn write_batch_item_common_row_data(&mut self, delta: &DataDelta) -> Result<(), SDSSError> {
+    fn write_batch_item_common_row_data(&mut self, delta: &DataDelta) -> SDSSResult<()> {
         let change_type = [delta.change().value_u8()];
         self.f.unfsynced_write(&change_type)?;
         let txn_id = delta.data_version().value_u64().to_le_bytes();
diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs
index 3475cd0b..d6bafbf6 100644
--- a/server/src/engine/storage/v1/batch_jrnl/restore.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs
@@ -42,7 +42,7 @@ use {
         storage::v1::{
             inf::PersistTypeDscr,
             rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedReader},
-            SDSSError, SDSSResult,
+            SDSSErrorKind, SDSSResult,
         },
     },
     crossbeam_epoch::pin,
@@ -187,7 +187,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
             }
         }
         // nope, this is a corrupted file
-        Err(SDSSError::DataBatchRestoreCorruptedBatchFile)
+        Err(SDSSErrorKind::DataBatchRestoreCorruptedBatchFile.into())
     }
     fn handle_reopen_is_actual_close(&mut self) -> SDSSResult<bool> {
         if self.f.is_eof() {
@@ -200,7 +200,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                 Ok(false)
             } else {
                 // that's just a nice bug
-                Err(SDSSError::DataBatchRestoreCorruptedBatchFile)
+                Err(SDSSErrorKind::DataBatchRestoreCorruptedBatchFile.into())
             }
         }
     }
@@ -303,7 +303,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
             // we must read the batch termination signature
             let b = self.f.read_byte()?;
             if b != MARKER_END_OF_BATCH {
-                return Err(SDSSError::DataBatchRestoreCorruptedBatch);
+                return Err(SDSSErrorKind::DataBatchRestoreCorruptedBatch.into());
             }
         }
         // read actual commit
@@ -320,7 +320,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
         if actual_checksum == u64::from_le_bytes(hardcoded_checksum) {
             Ok(actual_commit)
         } else {
-            Err(SDSSError::DataBatchRestoreCorruptedBatch)
+            Err(SDSSErrorKind::DataBatchRestoreCorruptedBatch.into())
         }
     }
     fn read_batch(&mut self) -> SDSSResult<Batch> {
@@ -340,7 +340,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
             }
             _ => {
                 // this is the only singular byte that is expected to be intact. If this isn't intact either, I'm sorry
-                return Err(SDSSError::DataBatchRestoreCorruptedBatch);
+                return Err(SDSSErrorKind::DataBatchRestoreCorruptedBatch.into());
             }
         }
         // decode batch start block
@@ -384,7 +384,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                                 this_col_cnt -= 1;
                             }
                             if this_col_cnt != 0 {
-                                return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+                                return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
                             }
                             if change_type == 1 {
                                 this_batch.push(DecodedBatchEvent::new(
@@ -402,7 +402,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                             processed_in_this_batch += 1;
                         }
                         _ => {
-                            return Err(SDSSError::DataBatchRestoreCorruptedBatch);
+                            return Err(SDSSErrorKind::DataBatchRestoreCorruptedBatch.into());
                         }
                     }
                 }
@@ -417,7 +417,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
         if let Ok(MARKER_RECOVERY_EVENT) = self.f.inner_file().read_byte() {
             return Ok(());
         }
-        Err(SDSSError::DataBatchRestoreCorruptedBatch)
+        Err(SDSSErrorKind::DataBatchRestoreCorruptedBatch.into())
     }
     fn read_start_batch_block(&mut self) -> SDSSResult<BatchStartBlock> {
         let pk_tag = self.f.read_byte()?;
@@ -467,7 +467,7 @@ impl BatchStartBlock {
 impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
     fn decode_primary_key(&mut self, pk_type: u8) -> SDSSResult<PrimaryIndexKey> {
         let Some(pk_type) = TagUnique::try_from_raw(pk_type) else {
-            return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+            return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
         };
         Ok(match pk_type {
             TagUnique::SignedInt | TagUnique::UnsignedInt => {
@@ -483,7 +483,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                 self.f.read_into_buffer(&mut data)?;
                 if pk_type == TagUnique::Str {
                     if core::str::from_utf8(&data).is_err() {
-                        return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+                        return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
                     }
                 }
                 unsafe {
@@ -501,14 +501,14 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
     fn decode_cell(&mut self) -> SDSSResult<Datacell> {
         let cell_type_sig = self.f.read_byte()?;
         let Some(cell_type) = PersistTypeDscr::try_from_raw(cell_type_sig) else {
-            return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+            return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
         };
         Ok(match cell_type {
             PersistTypeDscr::Null => Datacell::null(),
             PersistTypeDscr::Bool => {
                 let bool = self.f.read_byte()?;
                 if bool > 1 {
-                    return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+                    return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
                 }
                 Datacell::new_bool(bool == 1)
             }
@@ -528,7 +528,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                     // UNSAFE(@ohsayan): +tagck
                     if cell_type == PersistTypeDscr::Str {
                         if core::str::from_utf8(&data).is_err() {
-                            return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+                            return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
                         }
                         Datacell::new_str(String::from_utf8_unchecked(data).into_boxed_str())
                     } else {
@@ -543,13 +543,13 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                     list.push(self.decode_cell()?);
                 }
                 if len != list.len() as u64 {
-                    return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+                    return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
                 }
                 Datacell::new_list(list)
             }
             PersistTypeDscr::Dict => {
                 // we don't support dicts just yet
-                return Err(SDSSError::DataBatchRestoreCorruptedEntry);
+                return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
             }
         })
     }
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index 380d5ca2..57a2699d 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -40,7 +40,7 @@ use {
             },
             idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
             mem::BufferedScanner,
-            storage::v1::{inf, SDSSError, SDSSResult},
+            storage::v1::{inf, SDSSError, SDSSErrorKind, SDSSResult},
         },
         util::{copy_slice_to_array as memcpy, EndianQW},
     },
@@ -92,11 +92,12 @@ where
         while M::pretest_entry_metadata(scanner) & (dict.st_len() != dict_size) {
             let md = unsafe {
                 // UNSAFE(@ohsayan): +pretest
-                M::entry_md_dec(scanner)
-                    .ok_or(SDSSError::InternalDecodeStructureCorruptedPayload)?
+                M::entry_md_dec(scanner).ok_or::<SDSSError>(
+                    SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into(),
+                )?
             };
             if !M::pretest_entry_data(scanner, &md) {
-                return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+                return Err(SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into());
             }
             let key;
             let val;
@@ -107,7 +108,11 @@ where
                             key = _k;
                             val = _v;
                         }
-                        None => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
+                        None => {
+                            return Err(
+                                SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into()
+                            )
+                        }
                     }
                 } else {
                     let _k = M::dec_key(scanner, &md);
@@ -117,18 +122,22 @@ where
                             key = _k;
                             val = _v;
                         }
-                        _ => return Err(SDSSError::InternalDecodeStructureCorruptedPayload),
+                        _ => {
+                            return Err(
+                                SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into()
+                            )
+                        }
                     }
                 }
             }
             if !dict.st_insert(key, val) {
-                return Err(SDSSError::InternalDecodeStructureIllegalData);
+                return Err(SDSSErrorKind::InternalDecodeStructureIllegalData.into());
             }
         }
         if dict.st_len() == dict_size {
             Ok(dict)
         } else {
-            Err(SDSSError::InternalDecodeStructureIllegalData)
+            Err(SDSSErrorKind::InternalDecodeStructureIllegalData.into())
         }
     }
 }
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index afa17003..75db030e 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -42,7 +42,7 @@ use {
         },
         idx::{AsKey, AsValue},
         mem::BufferedScanner,
-        storage::v1::{SDSSError, SDSSResult},
+        storage::v1::{SDSSErrorKind, SDSSResult},
     },
     std::mem,
 };
@@ -157,14 +157,14 @@ pub trait PersistObject {
     /// Default routine to decode an object + its metadata (however, the metadata is used and not returned)
     fn default_full_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::OutputType> {
         if !Self::pretest_can_dec_metadata(scanner) {
-            return Err(SDSSError::InternalDecodeStructureCorrupted);
+            return Err(SDSSErrorKind::InternalDecodeStructureCorrupted.into());
         }
         let md = unsafe {
             // UNSAFE(@ohsayan): +pretest
             Self::meta_dec(scanner)?
         };
         if !Self::pretest_can_dec_object(scanner, &md) {
-            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+            return Err(SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into());
         }
         unsafe {
             // UNSAFE(@ohsayan): +obj pretest
@@ -290,11 +290,11 @@ pub mod dec {
     pub mod utils {
         use crate::engine::{
             mem::BufferedScanner,
-            storage::v1::{SDSSError, SDSSResult},
+            storage::v1::{SDSSErrorKind, SDSSResult},
         };
         pub unsafe fn decode_string(s: &mut BufferedScanner, len: usize) -> SDSSResult<String> {
             String::from_utf8(s.next_chunk_variable(len).to_owned())
-                .map_err(|_| SDSSError::InternalDecodeStructureCorruptedPayload)
+                .map_err(|_| SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into())
         }
     }
 }
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 01afe7fd..51a12e5e 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -39,7 +39,7 @@ use {
                 DictGeneric,
             },
             mem::{BufferedScanner, VInline},
-            storage::v1::{inf, SDSSError, SDSSResult},
+            storage::v1::{inf, SDSSErrorKind, SDSSResult},
         },
         util::EndianQW,
     },
@@ -119,7 +119,7 @@ impl<'a> PersistObject for LayerRef<'a> {
     fn obj_enc(_: &mut VecU8, _: Self::InputType) {}
     unsafe fn obj_dec(_: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
         if (md.type_selector > TagSelector::List.value_qword()) | (md.prop_set_arity != 0) {
-            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+            return Err(SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into());
         }
         Ok(Layer::new_empty_props(
             TagSelector::from_raw(md.type_selector as u8).into_full(),
@@ -202,7 +202,7 @@ impl<'a> PersistObject for FieldRef<'a> {
         if (field.layers().len() as u64 == md.layer_c) & (md.null <= 1) & (md.prop_c == 0) & fin {
             Ok(field)
         } else {
-            Err(SDSSError::InternalDecodeStructureCorrupted)
+            Err(SDSSErrorKind::InternalDecodeStructureCorrupted.into())
         }
     }
 }
@@ -281,7 +281,7 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
                 super::map::MapIndexSizeMD(md.field_c as usize),
             )?;
         let ptag = if md.p_key_tag > TagSelector::MAX as u64 {
-            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+            return Err(SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into());
         } else {
             TagSelector::from_raw(md.p_key_tag as u8)
         };
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index 4dbaaadb..6cdfcebf 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -44,7 +44,7 @@
 use {
     super::{
         rw::{FileOpen, RawFSInterface, SDSSFileIO},
-        spec, SDSSError, SDSSResult,
+        spec, SDSSErrorKind, SDSSResult,
     },
     crate::util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy},
     std::marker::PhantomData,
@@ -222,7 +222,7 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
         }
         match entry_metadata
             .event_source_marker()
-            .ok_or(SDSSError::JournalLogEntryCorrupted)?
+            .ok_or(SDSSErrorKind::JournalLogEntryCorrupted)?
         {
             EventSourceMarker::ServerStandard => {}
             EventSourceMarker::DriverClosed => {
@@ -237,7 +237,7 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
             EventSourceMarker::DriverReopened | EventSourceMarker::RecoveryReverseLastJournal => {
                 // these two are only taken in close and error paths (respectively) so we shouldn't see them here; this is bad
                 // two special directives in the middle of nowhere? incredible
-                return Err(SDSSError::JournalCorrupted);
+                return Err(SDSSErrorKind::JournalCorrupted.into());
             }
         }
         // read payload
@@ -270,10 +270,10 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
                 Ok(())
             } else {
                 // FIXME(@ohsayan): tolerate loss in this directive too
-                Err(SDSSError::JournalCorrupted)
+                Err(SDSSErrorKind::JournalCorrupted.into())
             }
         } else {
-            Err(SDSSError::JournalCorrupted)
+            Err(SDSSErrorKind::JournalCorrupted.into())
         }
     }
     #[cold] // FIXME(@ohsayan): how bad can prod systems be? (clue: pretty bad, so look for possible changes)
@@ -286,7 +286,7 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
         self.__record_read_bytes(JournalEntryMetadata::SIZE); // FIXME(@ohsayan): don't assume read length?
         let mut entry_buf = [0u8; JournalEntryMetadata::SIZE];
         if self.log_file.read_to_buffer(&mut entry_buf).is_err() {
-            return Err(SDSSError::JournalCorrupted);
+            return Err(SDSSErrorKind::JournalCorrupted.into());
         }
         let entry = JournalEntryMetadata::decode(entry_buf);
         let okay = (entry.event_id == self.evid as u128)
@@ -297,7 +297,7 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
         if okay {
             return Ok(());
         } else {
-            Err(SDSSError::JournalCorrupted)
+            Err(SDSSErrorKind::JournalCorrupted.into())
         }
     }
     /// Read and apply all events in the given log file to the global state, returning the (open file, last event ID)
@@ -309,7 +309,7 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
         if slf.closed {
             Ok((slf.log_file, slf.evid))
         } else {
-            Err(SDSSError::JournalCorrupted)
+            Err(SDSSErrorKind::JournalCorrupted.into())
         }
     }
 }
@@ -397,7 +397,7 @@ impl<Fs: RawFSInterface, TA> JournalWriter<Fs, TA> {
         if self.log_file.fsynced_write(&entry.encoded()).is_ok() {
             return Ok(());
         }
-        Err(SDSSError::JournalWRecoveryStageOneFailCritical)
+        Err(SDSSErrorKind::JournalWRecoveryStageOneFailCritical.into())
     }
     pub fn append_journal_reopen(&mut self) -> SDSSResult<()> {
         let id = self._incr_id() as u128;
diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs
index acf7f248..5cebafdb 100644
--- a/server/src/engine/storage/v1/loader.rs
+++ b/server/src/engine/storage/v1/loader.rs
@@ -32,7 +32,7 @@ use crate::engine::{
         batch_jrnl,
         journal::{self, JournalWriter},
         rw::{FileOpen, RawFSInterface},
-        spec, LocalFS, SDSSErrorContext, SDSSResult,
+        spec, LocalFS, SDSSResult,
     },
     txn::gns::{GNSAdapter, GNSTransactionDriverAnyFS},
 };
@@ -73,14 +73,11 @@ impl SEInitState {
                 for (model_name, model) in space.models().read().iter() {
                     let path =
                         Self::model_path(space_name, space_uuid, model_name, model.get_uuid());
-                    let persist_driver = match batch_jrnl::reinit(&path, model) {
-                        Ok(j) => j,
-                        Err(e) => {
-                            return Err(e.with_extra(format!(
-                                "failed to restore model data from journal in `{path}`"
-                            )))
-                        }
-                    };
+                    let persist_driver = batch_jrnl::reinit(&path, model).map_err(|e| {
+                        e.add_ctx(format!(
+                            "failed to restore model data from journal in `{path}`"
+                        ))
+                    })?;
                     let _ = model_drivers.insert(
                         ModelUniqueID::new(space_name, model_name, model.get_uuid()),
                         FractalModelDriver::init(persist_driver),
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index 9f11804d..6132a515 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -33,7 +33,6 @@ pub mod spec;
 mod sysdb;
 // hl
 pub mod inf;
-mod start_stop;
 // test
 pub mod memfs;
 #[cfg(test)]
@@ -49,49 +48,25 @@ pub mod data_batch {
     pub use super::batch_jrnl::{create, reinit, DataBatchPersistDriver, DataBatchRestoreDriver};
 }
 
-use crate::{engine::txn::TransactionError, util::os::SysIOError as IoError};
-
-pub type SDSSResult<T> = Result<T, SDSSError>;
-
-pub trait SDSSErrorContext {
-    type ExtraData;
-    fn with_extra(self, extra: Self::ExtraData) -> SDSSError;
-}
-
-impl SDSSErrorContext for IoError {
-    type ExtraData = &'static str;
-    fn with_extra(self, extra: Self::ExtraData) -> SDSSError {
-        SDSSError::IoErrorExtra(self, extra)
-    }
-}
-
-impl SDSSErrorContext for std::io::Error {
-    type ExtraData = &'static str;
-    fn with_extra(self, extra: Self::ExtraData) -> SDSSError {
-        SDSSError::IoErrorExtra(self.into(), extra)
-    }
-}
+use crate::{
+    engine::{
+        error::{CtxError, CtxResult},
+        txn::TransactionError,
+    },
+    util::os::SysIOError as IoError,
+};
 
-impl SDSSErrorContext for SDSSError {
-    type ExtraData = String;
-
-    fn with_extra(self, extra: Self::ExtraData) -> SDSSError {
-        SDSSError::Extra(Box::new(self), extra)
-    }
-}
+pub type SDSSResult<T> = CtxResult<T, SDSSErrorKind>;
+pub type SDSSError = CtxError<SDSSErrorKind>;
 
 #[derive(Debug)]
 #[cfg_attr(test, derive(PartialEq))]
-pub enum SDSSError {
+pub enum SDSSErrorKind {
     // IO errors
     /// An IO err
     IoError(IoError),
-    /// An IO err with extra ctx
-    IoErrorExtra(IoError, &'static str),
-    /// A corrupted file
-    CorruptedFile(&'static str),
-    // process errors
     OtherError(&'static str),
+    CorruptedFile(&'static str),
     // header
     /// version mismatch
     HeaderDecodeVersionMismatch,
@@ -127,41 +102,18 @@ pub enum SDSSError {
     DataBatchCloseError,
     DataBatchRestoreCorruptedBatchFile,
     JournalRestoreTxnError,
-    /// An error with more context
-    // TODO(@ohsayan): avoid the box; we'll clean this up soon
-    Extra(Box<Self>, String),
     SysDBCorrupted,
 }
 
-impl From<TransactionError> for SDSSError {
+impl From<TransactionError> for SDSSErrorKind {
     fn from(_: TransactionError) -> Self {
         Self::JournalRestoreTxnError
     }
 }
 
-impl SDSSError {
-    pub const fn corrupted_file(fname: &'static str) -> Self {
-        Self::CorruptedFile(fname)
-    }
-    pub const fn ioerror_extra(error: IoError, extra: &'static str) -> Self {
-        Self::IoErrorExtra(error, extra)
-    }
-    pub fn with_ioerror_extra(self, extra: &'static str) -> Self {
-        match self {
-            Self::IoError(ioe) => Self::IoErrorExtra(ioe, extra),
-            x => x,
-        }
-    }
-}
-
-impl From<IoError> for SDSSError {
-    fn from(e: IoError) -> Self {
-        Self::IoError(e)
-    }
-}
-
-impl From<std::io::Error> for SDSSError {
-    fn from(e: std::io::Error) -> Self {
-        Self::IoError(e.into())
+direct_from! {
+    SDSSErrorKind => {
+        std::io::Error as IoError,
+        IoError as IoError,
     }
 }
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index be4233d8..266b5371 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -29,10 +29,7 @@ use {
         spec::{FileSpec, Header},
         SDSSResult,
     },
-    crate::{
-        engine::storage::{v1::SDSSError, SCrc},
-        util::os::SysIOError,
-    },
+    crate::{engine::storage::SCrc, util::os::SysIOError},
     std::{
         fs::{self, File},
         io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write},
@@ -348,9 +345,7 @@ impl<Fs: RawFSInterface> SDSSFileTrackedReader<Fs> {
                 Err(e) => return Err(e),
             }
         } else {
-            Err(SDSSError::IoError(SysIOError::from(
-                std::io::ErrorKind::InvalidInput,
-            )))
+            Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into())
         }
     }
     pub fn read_byte(&mut self) -> SDSSResult<u8> {
@@ -373,9 +368,7 @@ impl<Fs: RawFSInterface> SDSSFileTrackedReader<Fs> {
     }
     pub fn read_block<const N: usize>(&mut self) -> SDSSResult<[u8; N]> {
         if !self.has_left(N as _) {
-            return Err(SDSSError::IoError(SysIOError::from(
-                std::io::ErrorKind::InvalidInput,
-            )));
+            return Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into());
         }
         let mut buf = [0; N];
         self.read_into_buffer(&mut buf)?;
diff --git a/server/src/engine/storage/v1/spec.rs b/server/src/engine/storage/v1/spec.rs
index 6a448f3b..c470badc 100644
--- a/server/src/engine/storage/v1/spec.rs
+++ b/server/src/engine/storage/v1/spec.rs
@@ -40,7 +40,7 @@ use {
     crate::{
         engine::storage::{
             header::{HostArch, HostEndian, HostOS, HostPointerWidth},
-            v1::SDSSError,
+            v1::SDSSErrorKind,
             versions::{self, DriverVersion, HeaderVersion, ServerVersion},
         },
         util::os,
@@ -375,12 +375,12 @@ impl SDSSStaticHeaderV1Compact {
         } else {
             let version_okay = okay_header_version & okay_server_version & okay_driver_version;
             let md = ManuallyDrop::new([
-                SDSSError::HeaderDecodeCorruptedHeader,
-                SDSSError::HeaderDecodeVersionMismatch,
+                SDSSErrorKind::HeaderDecodeCorruptedHeader,
+                SDSSErrorKind::HeaderDecodeVersionMismatch,
             ]);
             Err(unsafe {
                 // UNSAFE(@ohsayan): while not needed, md for drop safety + correct index
-                md.as_ptr().add(!version_okay as usize).read()
+                md.as_ptr().add(!version_okay as usize).read().into()
             })
         }
     }
@@ -511,7 +511,7 @@ impl Header for SDSSStaticHeaderV1Compact {
         {
             Ok(())
         } else {
-            Err(SDSSError::HeaderDecodeDataMismatch)
+            Err(SDSSErrorKind::HeaderDecodeDataMismatch.into())
         }
     }
 }
diff --git a/server/src/engine/storage/v1/start_stop.rs b/server/src/engine/storage/v1/start_stop.rs
deleted file mode 100644
index 11121523..00000000
--- a/server/src/engine/storage/v1/start_stop.rs
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Created on Mon May 29 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{SDSSError, SDSSErrorContext, SDSSResult},
-    crate::util::os,
-    std::{
-        fs::File,
-        io::{ErrorKind, Read, Write},
-    },
-};
-
-#[cfg(not(test))]
-const START_FILE: &'static str = ".start";
-#[cfg(test)]
-const START_FILE: &'static str = ".start_testmode";
-#[cfg(not(test))]
-const STOP_FILE: &'static str = ".stop";
-#[cfg(test)]
-const STOP_FILE: &'static str = ".stop_testmode";
-
-const EMSG_FAILED_WRITE_START_FILE: &str =
-    concat_str_to_str!("failed to write to `", START_FILE, "` file");
-const EMSG_FAILED_WRITE_STOP_FILE: &str =
-    concat_str_to_str!("failed to write to `", STOP_FILE, "` file");
-const EMSG_FAILED_OPEN_START_FILE: &str =
-    concat_str_to_str!("failed to open `", START_FILE, "` file");
-const EMSG_FAILED_OPEN_STOP_FILE: &str =
-    concat_str_to_str!("failed to open `", STOP_FILE, "` file");
-const EMSG_FAILED_VERIFY: &str = concat_str_to_str!(
-    "failed to verify `",
-    START_FILE,
-    concat_str_to_str!("` and `", STOP_FILE, "` timestamps")
-);
-
-#[derive(Debug)]
-pub struct StartStop {
-    begin: u128,
-    stop_file: File,
-}
-
-#[derive(Debug)]
-enum ReadNX {
-    Created(File),
-    Read(File, u128),
-}
-
-impl ReadNX {
-    const fn created(&self) -> bool {
-        matches!(self, Self::Created(_))
-    }
-    fn file_mut(&mut self) -> &mut File {
-        match self {
-            Self::Created(ref mut f) => f,
-            Self::Read(ref mut f, _) => f,
-        }
-    }
-    fn into_file(self) -> File {
-        match self {
-            Self::Created(f) => f,
-            Self::Read(f, _) => f,
-        }
-    }
-}
-
-impl StartStop {
-    fn read_time_file(f: &str, create_new_if_nx: bool) -> SDSSResult<ReadNX> {
-        let mut f = match File::options().write(true).read(true).open(f) {
-            Ok(f) => f,
-            Err(e) if e.kind() == ErrorKind::NotFound && create_new_if_nx => {
-                let f = File::create(f)?;
-                return Ok(ReadNX::Created(f));
-            }
-            Err(e) => return Err(e.into()),
-        };
-        let len = f.metadata().map(|m| m.len())?;
-        if len != sizeof!(u128) as u64 {
-            return Err(SDSSError::corrupted_file(START_FILE));
-        }
-        let mut buf = [0u8; sizeof!(u128)];
-        f.read_exact(&mut buf)?;
-        Ok(ReadNX::Read(f, u128::from_le_bytes(buf)))
-    }
-    pub fn terminate(mut self) -> SDSSResult<()> {
-        self.stop_file
-            .write_all(self.begin.to_le_bytes().as_ref())
-            .map_err(|e| e.with_extra(EMSG_FAILED_WRITE_STOP_FILE))
-    }
-    pub fn verify_and_start() -> SDSSResult<Self> {
-        // read start file
-        let mut start_file = Self::read_time_file(START_FILE, true)
-            .map_err(|e| e.with_ioerror_extra(EMSG_FAILED_OPEN_START_FILE))?;
-        // read stop file
-        let stop_file = Self::read_time_file(STOP_FILE, start_file.created())
-            .map_err(|e| e.with_ioerror_extra(EMSG_FAILED_OPEN_STOP_FILE))?;
-        // read current time
-        let ctime = os::get_epoch_time();
-        match (&start_file, &stop_file) {
-            (ReadNX::Read(_, time_start), ReadNX::Read(_, time_stop))
-                if time_start == time_stop => {}
-            (ReadNX::Created(_), ReadNX::Created(_)) => {}
-            _ => return Err(SDSSError::OtherError(EMSG_FAILED_VERIFY)),
-        }
-        start_file
-            .file_mut()
-            .write_all(&ctime.to_le_bytes())
-            .map_err(|e| e.with_extra(EMSG_FAILED_WRITE_START_FILE))?;
-        Ok(Self {
-            stop_file: stop_file.into_file(),
-            begin: ctime,
-        })
-    }
-}
-
-#[test]
-fn verify_test() {
-    let x = || -> SDSSResult<()> {
-        let ss = StartStop::verify_and_start()?;
-        ss.terminate()?;
-        let ss = StartStop::verify_and_start()?;
-        ss.terminate()?;
-        std::fs::remove_file(START_FILE)?;
-        std::fs::remove_file(STOP_FILE)?;
-        Ok(())
-    };
-    x().unwrap();
-}
diff --git a/server/src/engine/storage/v1/sysdb.rs b/server/src/engine/storage/v1/sysdb.rs
index 7bb70c55..b76e0218 100644
--- a/server/src/engine/storage/v1/sysdb.rs
+++ b/server/src/engine/storage/v1/sysdb.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    super::{rw::FileOpen, SDSSError},
+    super::{rw::FileOpen, SDSSErrorKind},
     crate::engine::{
         config::ConfigAuth,
         data::{cell::Datacell, DictEntryGeneric, DictGeneric},
@@ -175,7 +175,7 @@ fn rkey<T>(
 ) -> SDSSResult<T> {
     match d.remove(key).map(transform) {
         Some(Some(k)) => Ok(k),
-        _ => Err(SDSSError::SysDBCorrupted),
+        _ => Err(SDSSErrorKind::SysDBCorrupted.into()),
     }
 }
 
@@ -201,14 +201,14 @@ pub fn decode_system_database<Fs: RawFSInterface>(mut f: SDSSFileIO<Fs>) -> SDSS
         let mut userdata = userdata
             .into_data()
             .and_then(Datacell::into_list)
-            .ok_or(SDSSError::SysDBCorrupted)?;
+            .ok_or(SDSSErrorKind::SysDBCorrupted)?;
         if userdata.len() != 1 {
-            return Err(SDSSError::SysDBCorrupted);
+            return Err(SDSSErrorKind::SysDBCorrupted.into());
         }
         let user_password = userdata
             .remove(0)
             .into_bin()
-            .ok_or(SDSSError::SysDBCorrupted)?;
+            .ok_or(SDSSErrorKind::SysDBCorrupted)?;
         loaded_users.insert(username, SysAuthUser::new(user_password.into_boxed_slice()));
     }
     let sys_auth = SysAuth::new(root_key.into_boxed_slice(), loaded_users);
@@ -220,7 +220,7 @@ pub fn decode_system_database<Fs: RawFSInterface>(mut f: SDSSFileIO<Fs>) -> SDSS
         d.into_data()?.into_uint()
     })?;
     if !(sysdb_data.is_empty() & auth_store.is_empty() & sys_store.is_empty()) {
-        return Err(SDSSError::SysDBCorrupted);
+        return Err(SDSSErrorKind::SysDBCorrupted.into());
     }
     Ok(SysConfig::new(
         RwLock::new(sys_auth),
diff --git a/server/src/engine/storage/v1/tests/tx.rs b/server/src/engine/storage/v1/tests/tx.rs
index 45140c4d..5e39b737 100644
--- a/server/src/engine/storage/v1/tests/tx.rs
+++ b/server/src/engine/storage/v1/tests/tx.rs
@@ -28,7 +28,7 @@ use {
     crate::{
         engine::storage::v1::{
             journal::{self, JournalAdapter, JournalWriter},
-            spec, SDSSError, SDSSResult,
+            spec, SDSSError, SDSSErrorKind, SDSSResult,
         },
         util,
     },
@@ -115,7 +115,9 @@ impl JournalAdapter for DatabaseTxnAdapter {
 
     fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> Result<(), TxError> {
         if payload.len() != 10 {
-            return Err(SDSSError::CorruptedFile("testtxn.log").into());
+            return Err(TxError::SDSS(
+                SDSSErrorKind::CorruptedFile("testtxn.log").into(),
+            ));
         }
         let opcode = payload[0];
         let index = u64::from_le_bytes(util::copy_slice_to_array(&payload[1..9]));
@@ -123,7 +125,11 @@ impl JournalAdapter for DatabaseTxnAdapter {
         match opcode {
             0 if index == 0 && new_value == 0 => gs.reset(),
             1 if index < 10 && index < isize::MAX as u64 => gs.set(index as usize, new_value),
-            _ => return Err(SDSSError::JournalLogEntryCorrupted.into()),
+            _ => {
+                return Err(TxError::SDSS(
+                    SDSSErrorKind::JournalLogEntryCorrupted.into(),
+                ))
+            }
         }
         Ok(())
     }
diff --git a/server/src/engine/tests/mod.rs b/server/src/engine/tests/mod.rs
index 092acf3a..1ac07cb3 100644
--- a/server/src/engine/tests/mod.rs
+++ b/server/src/engine/tests/mod.rs
@@ -90,8 +90,12 @@ mod cfg {
     #[test]
     fn parse_validate_cli_args() {
         with_files(
-            ["__cli_args_test_private.key", "__cli_args_test_cert.pem"],
-            |[pkey, cert]| {
+            [
+                "__cli_args_test_private.key",
+                "__cli_args_test_cert.pem",
+                "__cli_args_test_passphrase.key",
+            ],
+            |[pkey, cert, pass]| {
                 let payload = format!(
                     "skyd --mode=dev \
                 --endpoint tcp@127.0.0.1:2003 \
@@ -99,6 +103,7 @@ mod cfg {
                 --service-window=600 \
                 --tlskey {pkey} \
                 --tlscert {cert} \
+                --tls-passphrase {pass} \
                 --auth-plugin pwd \
                 --auth-root-password password12345678
                 "
@@ -115,6 +120,7 @@ mod cfg {
                             ConfigEndpointTls::new(
                                 ConfigEndpointTcp::new("127.0.0.2".into(), 2004),
                                 "".into(),
+                                "".into(),
                                 "".into()
                             )
                         ),
@@ -205,13 +211,18 @@ mod cfg {
     #[test]
     fn parse_validate_env_args() {
         with_files(
-            ["__env_args_test_cert.pem", "__env_args_test_private.key"],
-            |[cert, key]| {
+            [
+                "__env_args_test_cert.pem",
+                "__env_args_test_private.key",
+                "__env_args_test_private.passphrase.txt",
+            ],
+            |[cert, key, pass]| {
                 let variables = [
                     format!("SKYDB_AUTH_PLUGIN=pwd"),
                     format!("SKYDB_AUTH_ROOT_PASSWORD=password12345678"),
                     format!("SKYDB_TLS_CERT={cert}"),
                     format!("SKYDB_TLS_KEY={key}"),
+                    format!("SKYDB_TLS_PRIVATE_KEY_PASSWORD={pass}"),
                     format!("SKYDB_ENDPOINTS=tcp@localhost:8080,tls@localhost:8081"),
                     format!("SKYDB_RUN_MODE=dev"),
                     format!("SKYDB_SERVICE_WINDOW=600"),
@@ -226,6 +237,7 @@ mod cfg {
                             ConfigEndpointTls::new(
                                 ConfigEndpointTcp::new("localhost".into(), 8081),
                                 "".into(),
+                                "".into(),
                                 "".into()
                             )
                         ),
@@ -252,6 +264,7 @@ endpoints:
     port: 2004
     cert: ._test_sample_cert.pem
     private_key: ._test_sample_private.key
+    pkey_passphrase: ._test_sample_private.pass.txt
   insecure:
     host: 127.0.0.1
     port: 2003
@@ -259,7 +272,11 @@ endpoints:
     #[test]
     fn test_config_file() {
         with_files(
-            ["._test_sample_cert.pem", "._test_sample_private.key"],
+            [
+                "._test_sample_cert.pem",
+                "._test_sample_private.key",
+                "._test_sample_private.pass.txt",
+            ],
             |_| {
                 config::set_cli_src(vec!["skyd".into(), "--config=config.yml".into()]);
                 config::set_file_src(CONFIG_FILE);
@@ -272,6 +289,7 @@ endpoints:
                             ConfigEndpointTls::new(
                                 ConfigEndpointTcp::new("127.0.0.1".into(), 2004),
                                 "".into(),
+                                "".into(),
                                 "".into()
                             )
                         ),
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index a31b28ac..f8097f15 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -39,7 +39,7 @@ use {
             ql::lex::Ident,
             storage::v1::{
                 inf::{self, map, obj, PersistObject},
-                SDSSError, SDSSResult,
+                SDSSErrorKind, SDSSResult,
             },
             txn::TransactionError,
         },
@@ -498,7 +498,7 @@ impl<'a> PersistObject for AlterModelRemoveTxn<'a> {
             removed_fields.push(inf::dec::utils::decode_string(s, len)?.into_boxed_str());
         }
         if removed_fields.len() as u64 != md.remove_field_c {
-            return Err(SDSSError::InternalDecodeStructureCorruptedPayload);
+            return Err(SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into());
         }
         Ok(AlterModelRemoveTxnRestorePL {
             model_id,

From a69093aa964a0af2a4f4c52c7b763d11886d8c9c Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 2 Oct 2023 07:18:17 +0000
Subject: [PATCH 270/310] Implement new error tracing system

[ci] Update rust installation
---
 .github/workflows/test-pr.yml                 |   1 +
 .github/workflows/test-push.yml               |   2 +
 server/src/engine/config.rs                   | 111 +++---
 server/src/engine/core/dml/del.rs             |   4 +-
 server/src/engine/core/dml/ins.rs             |   6 +-
 server/src/engine/core/dml/mod.rs             |   4 +-
 server/src/engine/core/dml/sel.rs             |   6 +-
 server/src/engine/core/dml/upd.rs             |  12 +-
 server/src/engine/core/mod.rs                 |   4 +-
 server/src/engine/core/model/alt.rs           |  24 +-
 server/src/engine/core/model/mod.rs           |  12 +-
 server/src/engine/core/space.rs               |  18 +-
 server/src/engine/core/tests/ddl_model/alt.rs |  20 +-
 server/src/engine/core/tests/ddl_model/crt.rs |  12 +-
 .../src/engine/core/tests/ddl_model/layer.rs  |   6 +-
 .../src/engine/core/tests/ddl_space/alter.rs  |   4 +-
 .../src/engine/core/tests/ddl_space/create.rs |   6 +-
 server/src/engine/core/tests/dml/delete.rs    |   4 +-
 server/src/engine/core/tests/dml/insert.rs    |   4 +-
 server/src/engine/core/tests/dml/select.rs    |   4 +-
 server/src/engine/core/tests/dml/update.rs    |   8 +-
 server/src/engine/core/util.rs                |   4 +-
 server/src/engine/error.rs                    | 224 ++++++------
 server/src/engine/fractal/config.rs           |   8 +-
 server/src/engine/fractal/context.rs          | 195 ++++++++++
 server/src/engine/fractal/error.rs            | 340 ++++++++++++++++++
 server/src/engine/fractal/mod.rs              |   9 +-
 server/src/engine/fractal/test_utils.rs       |   2 +-
 server/src/engine/net/mod.rs                  |  15 +-
 server/src/engine/ql/ast/mod.rs               |  14 +-
 server/src/engine/ql/ast/traits.rs            |   8 +-
 server/src/engine/ql/dcl.rs                   |  10 +-
 server/src/engine/ql/ddl/alt.rs               |  18 +-
 server/src/engine/ql/ddl/crt.rs               |  10 +-
 server/src/engine/ql/ddl/drop.rs              |   8 +-
 server/src/engine/ql/ddl/ins.rs               |   6 +-
 server/src/engine/ql/ddl/syn.rs               |  20 +-
 server/src/engine/ql/dml/del.rs               |   6 +-
 server/src/engine/ql/dml/ins.rs               |   6 +-
 server/src/engine/ql/dml/mod.rs               |   4 +-
 server/src/engine/ql/dml/sel.rs               |   8 +-
 server/src/engine/ql/dml/upd.rs               |   6 +-
 server/src/engine/ql/lex/mod.rs               |  42 +--
 server/src/engine/ql/tests/lexer_tests.rs     |  17 +-
 .../src/engine/storage/v1/batch_jrnl/mod.rs   |   8 +-
 .../engine/storage/v1/batch_jrnl/persist.rs   |  28 +-
 .../engine/storage/v1/batch_jrnl/restore.rs   |  58 +--
 server/src/engine/storage/v1/inf/map.rs       |  25 +-
 server/src/engine/storage/v1/inf/mod.rs       |  33 +-
 server/src/engine/storage/v1/inf/obj.rs       |  28 +-
 server/src/engine/storage/v1/journal.rs       |  57 +--
 server/src/engine/storage/v1/loader.rs        |  16 +-
 server/src/engine/storage/v1/memfs.rs         | 111 +++---
 server/src/engine/storage/v1/mod.rs           |  70 ----
 server/src/engine/storage/v1/rw.rs            | 122 +++----
 server/src/engine/storage/v1/spec.rs          |  44 +--
 server/src/engine/storage/v1/sysdb.rs         |  29 +-
 server/src/engine/storage/v1/tests/tx.rs      |  38 +-
 server/src/engine/txn/gns/mod.rs              |  31 +-
 server/src/engine/txn/gns/model.rs            |  96 ++---
 server/src/engine/txn/gns/space.rs            |  44 ++-
 server/src/engine/txn/gns/tests/full_chain.rs |   4 +-
 server/src/engine/txn/mod.rs                  |  29 --
 63 files changed, 1298 insertions(+), 825 deletions(-)
 create mode 100644 server/src/engine/fractal/context.rs
 create mode 100644 server/src/engine/fractal/error.rs

diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml
index b4390181..6aebd0b6 100644
--- a/.github/workflows/test-pr.yml
+++ b/.github/workflows/test-pr.yml
@@ -36,6 +36,7 @@ jobs:
       - name: Install Rust
         run: |
           rustup update ${{ matrix.rust }} --no-self-update
+          rustup update
           rustup default ${{ matrix.rust }}
         if: env.BUILD == 'true'
       - name: Install perl modules
diff --git a/.github/workflows/test-push.yml b/.github/workflows/test-push.yml
index 7ddac1f9..e794039d 100644
--- a/.github/workflows/test-push.yml
+++ b/.github/workflows/test-push.yml
@@ -109,6 +109,7 @@ jobs:
 
       - name: Install Rust
         run: |
+          rustup self update
           rustup default stable
           rustup target add ${{ matrix.rust }}
 
@@ -153,6 +154,7 @@ jobs:
 
       - name: Install Rust
         run: |
+          rustup self update
           rustup default stable
           rustup target add ${{ matrix.rust }}
 
diff --git a/server/src/engine/config.rs b/server/src/engine/config.rs
index 4c75f009..c981af5e 100644
--- a/server/src/engine/config.rs
+++ b/server/src/engine/config.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    crate::util::os::SysIOError,
+    crate::engine::error::RuntimeResult,
     core::fmt,
     serde::Deserialize,
     std::{collections::HashMap, fs},
@@ -294,9 +294,6 @@ impl DecodedEPInsecureConfig {
     errors and misc
 */
 
-/// Configuration result
-pub type ConfigResult<T> = Result<T, ConfigError>;
-
 #[derive(Debug)]
 #[cfg_attr(test, derive(PartialEq))]
 /// A configuration error (with an optional error origin source)
@@ -332,10 +329,6 @@ impl fmt::Display for ConfigError {
                 "conflicting settings. please choose either CLI or ENV or configuration file"
             ),
             ConfigErrorKind::ErrorString(e) => write!(f, "{e}"),
-            ConfigErrorKind::IoError(e) => write!(
-                f,
-                "an I/O error occurred while reading a configuration related file: `{e}`",
-            ),
         }
     }
 }
@@ -369,20 +362,6 @@ pub enum ConfigErrorKind {
     Conflict,
     /// A custom error output
     ErrorString(String),
-    /// An I/O error related to configuration
-    IoError(SysIOError),
-}
-
-direct_from! {
-    ConfigErrorKind => {
-        SysIOError as IoError,
-    }
-}
-
-impl From<std::io::Error> for ConfigError {
-    fn from(value: std::io::Error) -> Self {
-        Self::new(ConfigErrorKind::IoError(value.into()))
-    }
 }
 
 /// A configuration source implementation
@@ -420,9 +399,9 @@ pub(super) trait ConfigurationSource {
 fn argck_duplicate_values<CS: ConfigurationSource>(
     v: &[String],
     key: &'static str,
-) -> ConfigResult<()> {
+) -> RuntimeResult<()> {
     if v.len() != 1 {
-        return Err(CS::err_too_many_values_for(key));
+        return Err(CS::err_too_many_values_for(key).into());
     }
     Ok(())
 }
@@ -438,14 +417,15 @@ enum ConnectionProtocol {
 }
 
 /// Parse an endpoint (`protocol@host:port`)
-fn parse_endpoint(source: ConfigSource, s: &str) -> ConfigResult<(ConnectionProtocol, &str, u16)> {
+fn parse_endpoint(source: ConfigSource, s: &str) -> RuntimeResult<(ConnectionProtocol, &str, u16)> {
     let err = || {
         Err(ConfigError::with_src(
             source,
             ConfigErrorKind::ErrorString(format!(
                 "invalid endpoint syntax. should be `protocol@hostname:port`"
             )),
-        ))
+        )
+        .into())
     };
     let x = s.split("@").collect::<Vec<&str>>();
     if x.len() != 2 {
@@ -475,7 +455,7 @@ fn decode_tls_ep(
     pkey_pass: &str,
     host: &str,
     port: u16,
-) -> ConfigResult<DecodedEPSecureConfig> {
+) -> RuntimeResult<DecodedEPSecureConfig> {
     let tls_key = fs::read_to_string(key_path)?;
     let tls_cert = fs::read_to_string(cert_path)?;
     let tls_priv_key_passphrase = fs::read_to_string(pkey_pass)?;
@@ -493,7 +473,7 @@ fn arg_decode_tls_endpoint<CS: ConfigurationSource>(
     args: &mut ParsedRawArgs,
     host: &str,
     port: u16,
-) -> ConfigResult<DecodedEPSecureConfig> {
+) -> RuntimeResult<DecodedEPSecureConfig> {
     let _cert = args.remove(CS::KEY_TLS_CERT);
     let _key = args.remove(CS::KEY_TLS_KEY);
     let _passphrase = args.remove(CS::KEY_TLS_PKEY_PASS);
@@ -508,7 +488,8 @@ fn arg_decode_tls_endpoint<CS: ConfigurationSource>(
                     CS::KEY_TLS_KEY,
                     CS::KEY_TLS_PKEY_PASS,
                 )),
-            ));
+            )
+            .into());
         }
     };
     argck_duplicate_values::<CS>(&tls_cert, CS::KEY_TLS_CERT)?;
@@ -530,7 +511,7 @@ fn arg_decode_tls_endpoint<CS: ConfigurationSource>(
 fn arg_decode_auth<CS: ConfigurationSource>(
     src_args: &mut ParsedRawArgs,
     config: &mut ModifyGuard<DecodedConfiguration>,
-) -> ConfigResult<()> {
+) -> RuntimeResult<()> {
     let (Some(auth_driver), Some(mut root_key)) = (
         src_args.remove(CS::KEY_AUTH_DRIVER),
         src_args.remove(CS::KEY_AUTH_ROOT_PASSWORD),
@@ -542,13 +523,14 @@ fn arg_decode_auth<CS: ConfigurationSource>(
                 CS::KEY_AUTH_DRIVER,
                 CS::KEY_AUTH_ROOT_PASSWORD
             )),
-        ));
+        )
+        .into());
     };
     argck_duplicate_values::<CS>(&auth_driver, CS::KEY_AUTH_DRIVER)?;
     argck_duplicate_values::<CS>(&root_key, CS::KEY_AUTH_DRIVER)?;
     let auth_plugin = match auth_driver[0].as_str() {
         "pwd" => AuthDriver::Pwd,
-        _ => return Err(CS::err_invalid_value_for(CS::KEY_AUTH_DRIVER)),
+        _ => return Err(CS::err_invalid_value_for(CS::KEY_AUTH_DRIVER).into()),
     };
     config.auth = Some(DecodedAuth {
         plugin: auth_plugin,
@@ -561,14 +543,14 @@ fn arg_decode_auth<CS: ConfigurationSource>(
 fn arg_decode_endpoints<CS: ConfigurationSource>(
     args: &mut ParsedRawArgs,
     config: &mut ModifyGuard<DecodedConfiguration>,
-) -> ConfigResult<()> {
+) -> RuntimeResult<()> {
     let mut insecure = None;
     let mut secure = None;
     let Some(endpoints) = args.remove(CS::KEY_ENDPOINTS) else {
         return Ok(());
     };
     if endpoints.len() > 2 {
-        return Err(CS::err_too_many_values_for(CS::KEY_ENDPOINTS));
+        return Err(CS::err_too_many_values_for(CS::KEY_ENDPOINTS).into());
     }
     for ep in endpoints {
         let (proto, host, port) = parse_endpoint(CS::SOURCE, &ep)?;
@@ -583,7 +565,8 @@ fn arg_decode_endpoints<CS: ConfigurationSource>(
                 return Err(CS::custom_err(format!(
                     "duplicate endpoints specified in `{}`",
                     CS::KEY_ENDPOINTS
-                )));
+                ))
+                .into());
             }
         }
     }
@@ -599,12 +582,12 @@ fn arg_decode_endpoints<CS: ConfigurationSource>(
 fn arg_decode_mode<CS: ConfigurationSource>(
     mode: &[String],
     config: &mut ModifyGuard<DecodedConfiguration>,
-) -> ConfigResult<()> {
+) -> RuntimeResult<()> {
     argck_duplicate_values::<CS>(&mode, CS::KEY_RUN_MODE)?;
     let mode = match mode[0].as_str() {
         "dev" => ConfigMode::Dev,
         "prod" => ConfigMode::Prod,
-        _ => return Err(CS::err_invalid_value_for(CS::KEY_RUN_MODE)),
+        _ => return Err(CS::err_invalid_value_for(CS::KEY_RUN_MODE).into()),
     };
     match config.system.as_mut() {
         Some(s) => s.mode = Some(mode),
@@ -622,7 +605,7 @@ fn arg_decode_mode<CS: ConfigurationSource>(
 fn arg_decode_rs_window<CS: ConfigurationSource>(
     mode: &[String],
     config: &mut ModifyGuard<DecodedConfiguration>,
-) -> ConfigResult<()> {
+) -> RuntimeResult<()> {
     argck_duplicate_values::<CS>(&mode, CS::KEY_SERVICE_WINDOW)?;
     match mode[0].parse::<u64>() {
         Ok(n) => match config.system.as_mut() {
@@ -634,7 +617,7 @@ fn arg_decode_rs_window<CS: ConfigurationSource>(
                 })
             }
         },
-        Err(_) => return Err(CS::err_invalid_value_for(CS::KEY_SERVICE_WINDOW)),
+        Err(_) => return Err(CS::err_invalid_value_for(CS::KEY_SERVICE_WINDOW).into()),
     }
     Ok(())
 }
@@ -703,7 +686,7 @@ impl<T> CLIConfigParseReturn<T> {
 /// - `--{option}={value}`
 pub fn parse_cli_args<'a, T: 'a + AsRef<str>>(
     src: impl Iterator<Item = T>,
-) -> ConfigResult<CLIConfigParseReturn<ParsedRawArgs>> {
+) -> RuntimeResult<CLIConfigParseReturn<ParsedRawArgs>> {
     let mut args_iter = src.into_iter().skip(1);
     let mut cli_args: ParsedRawArgs = HashMap::new();
     while let Some(arg) = args_iter.next() {
@@ -718,7 +701,8 @@ pub fn parse_cli_args<'a, T: 'a + AsRef<str>>(
             return Err(ConfigError::with_src(
                 ConfigSource::Cli,
                 ConfigErrorKind::ErrorString(format!("unexpected argument `{arg}`")),
-            ));
+            )
+            .into());
         }
         // x=1
         let arg_key;
@@ -734,13 +718,15 @@ pub fn parse_cli_args<'a, T: 'a + AsRef<str>>(
             return Err(ConfigError::with_src(
                 ConfigSource::Cli,
                 ConfigErrorKind::ErrorString(format!("incorrectly formatted argument `{arg}`")),
-            ));
+            )
+            .into());
         } else {
             let Some(value) = args_iter.next() else {
                 return Err(ConfigError::with_src(
                     ConfigSource::Cli,
                     ConfigErrorKind::ErrorString(format!("missing value for option `{arg}`")),
-                ));
+                )
+                .into());
             };
             arg_key = arg;
             arg_val = value.as_ref().to_string();
@@ -767,7 +753,7 @@ pub fn parse_cli_args<'a, T: 'a + AsRef<str>>(
 */
 
 /// Parse environment variables
-pub fn parse_env_args() -> ConfigResult<Option<ParsedRawArgs>> {
+pub fn parse_env_args() -> RuntimeResult<Option<ParsedRawArgs>> {
     const KEYS: [&str; 8] = [
         CSEnvArgs::KEY_AUTH_DRIVER,
         CSEnvArgs::KEY_AUTH_ROOT_PASSWORD,
@@ -788,7 +774,8 @@ pub fn parse_env_args() -> ConfigResult<Option<ParsedRawArgs>> {
                     return Err(ConfigError::with_src(
                         ConfigSource::Env,
                         ConfigErrorKind::ErrorString(format!("invalid value for `{key}`")),
-                    ))
+                    )
+                    .into())
                 }
             },
         };
@@ -809,15 +796,15 @@ pub fn parse_env_args() -> ConfigResult<Option<ParsedRawArgs>> {
 /// Apply the configuration changes to the given mutable config
 fn apply_config_changes<CS: ConfigurationSource>(
     args: &mut ParsedRawArgs,
-) -> ConfigResult<ModifyGuard<DecodedConfiguration>> {
+) -> RuntimeResult<ModifyGuard<DecodedConfiguration>> {
     let mut config = ModifyGuard::new(DecodedConfiguration::default());
     enum DecodeKind {
         Simple {
             key: &'static str,
-            f: fn(&[String], &mut ModifyGuard<DecodedConfiguration>) -> ConfigResult<()>,
+            f: fn(&[String], &mut ModifyGuard<DecodedConfiguration>) -> RuntimeResult<()>,
         },
         Complex {
-            f: fn(&mut ParsedRawArgs, &mut ModifyGuard<DecodedConfiguration>) -> ConfigResult<()>,
+            f: fn(&mut ParsedRawArgs, &mut ModifyGuard<DecodedConfiguration>) -> RuntimeResult<()>,
         },
     }
     let decode_tasks = [
@@ -856,7 +843,8 @@ fn apply_config_changes<CS: ConfigurationSource>(
         Err(ConfigError::with_src(
             CS::SOURCE,
             ConfigErrorKind::ErrorString("found unknown arguments".into()),
-        ))
+        )
+        .into())
     } else {
         Ok(config)
     }
@@ -933,7 +921,7 @@ fn validate_configuration<CS: ConfigurationSource>(
         endpoints,
         auth,
     }: DecodedConfiguration,
-) -> ConfigResult<Configuration> {
+) -> RuntimeResult<Configuration> {
     let Some(auth) = auth else {
         return Err(ConfigError::with_src(
             CS::SOURCE,
@@ -941,7 +929,8 @@ fn validate_configuration<CS: ConfigurationSource>(
                 "root account must be configured with {}",
                 CS::KEY_AUTH_ROOT_PASSWORD
             )),
-        ));
+        )
+        .into());
     };
     // initialize our default configuration
     let mut config = Configuration::default_dev_mode(auth);
@@ -986,11 +975,11 @@ fn validate_configuration<CS: ConfigurationSource>(
         if config.system.reliability_system_window == 0 => ConfigError::with_src(
             CS::SOURCE,
             ConfigErrorKind::ErrorString("invalid value for service window. must be nonzero".into()),
-        ),
+        ).into(),
         if config.auth.root_key.len() < ROOT_PASSWORD_MIN_LEN => ConfigError::with_src(
             CS::SOURCE,
             ConfigErrorKind::ErrorString("the root password must have at least 16 characters".into()),
-        ),
+        ).into(),
     );
     Ok(config)
 }
@@ -1023,7 +1012,7 @@ impl ConfigReturn {
 /// Apply the changes and validate the configuration
 pub(super) fn apply_and_validate<CS: ConfigurationSource>(
     mut args: ParsedRawArgs,
-) -> ConfigResult<ConfigReturn> {
+) -> RuntimeResult<ConfigReturn> {
     let cfg = apply_config_changes::<CS>(&mut args)?;
     if ModifyGuard::modified(&cfg) {
         validate_configuration::<CS>(cfg.val).map(ConfigReturn::Config)
@@ -1067,7 +1056,7 @@ pub(super) fn set_file_src(src: &str) {
         s.borrow_mut().replace(src.to_string());
     })
 }
-fn get_file_from_store(filename: &str) -> ConfigResult<String> {
+fn get_file_from_store(filename: &str) -> RuntimeResult<String> {
     let _f = filename;
     let f;
     #[cfg(test)]
@@ -1122,7 +1111,7 @@ fn get_cli_from_store() -> Vec<String> {
 /// - CLI args
 /// - ENV variables
 /// - Config file (if any)
-pub fn check_configuration() -> ConfigResult<ConfigReturn> {
+pub fn check_configuration() -> RuntimeResult<ConfigReturn> {
     // read in our environment variables
     let env_args = parse_env_args()?;
     // read in our CLI args (since that can tell us whether we need a configuration file)
@@ -1154,7 +1143,8 @@ pub fn check_configuration() -> ConfigResult<ConfigReturn> {
                         return Err(ConfigError::with_src(
                             ConfigSource::Cli,
                             ConfigErrorKind::Conflict,
-                        ));
+                        )
+                        .into());
                     }
                     return apply_and_validate::<CSCommandLine>(cfg_from_cli);
                 }
@@ -1180,7 +1170,7 @@ fn check_config_file(
     cfg_from_cli: &ParsedRawArgs,
     env_args: &Option<ParsedRawArgs>,
     cfg_file: &Vec<String>,
-) -> ConfigResult<ConfigReturn> {
+) -> RuntimeResult<ConfigReturn> {
     if cfg_from_cli.len() == 1 && env_args.is_none() {
         // yes, we only have the config file
         argck_duplicate_values::<CSCommandLine>(&cfg_file, CSCommandLine::ARG_CONFIG_FILE)?;
@@ -1214,9 +1204,6 @@ fn check_config_file(
         return validate_configuration::<CSConfigFile>(config_from_file).map(ConfigReturn::Config);
     } else {
         // so there are more configuration options + a config file? (and maybe even env?)
-        return Err(ConfigError::with_src(
-            ConfigSource::Cli,
-            ConfigErrorKind::Conflict,
-        ));
+        return Err(ConfigError::with_src(ConfigSource::Cli, ConfigErrorKind::Conflict).into());
     }
 }
diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs
index 8256b9f3..72102737 100644
--- a/server/src/engine/core/dml/del.rs
+++ b/server/src/engine/core/dml/del.rs
@@ -26,7 +26,7 @@
 
 use crate::engine::{
     core::{self, model::delta::DataDeltaKind},
-    error::{Error, QueryResult},
+    error::{QueryError, QueryResult},
     fractal::GlobalInstanceLike,
     idx::MTIndex,
     ql::dml::del::DeleteStatement,
@@ -55,7 +55,7 @@ pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) ->
                 );
                 Ok(())
             }
-            None => Err(Error::QPDmlRowNotFound),
+            None => Err(QueryError::QPDmlRowNotFound),
         }
     })
 }
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index d64efef6..295bface 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -30,7 +30,7 @@ use crate::engine::{
         index::{DcFieldIndex, PrimaryIndexKey, Row},
         model::{delta::DataDeltaKind, Fields, Model},
     },
-    error::{Error, QueryResult},
+    error::{QueryError, QueryResult},
     fractal::GlobalInstanceLike,
     idx::{IndexBaseSpec, MTIndex, STIndex, STIndexSeq},
     ql::dml::ins::{InsertData, InsertStatement},
@@ -57,7 +57,7 @@ pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> Quer
             );
             Ok(())
         } else {
-            Err(Error::QPDmlDuplicate)
+            Err(QueryError::QPDmlDuplicate)
         }
     })
 }
@@ -113,6 +113,6 @@ fn prepare_insert(
         };
         Ok((primary_key, prepared_data))
     } else {
-        Err(Error::QPDmlValidationError)
+        Err(QueryError::QPDmlValidationError)
     }
 }
diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index 4cf41c44..a7414382 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -33,7 +33,7 @@ use crate::{
     engine::{
         core::model::Model,
         data::{lit::Lit, tag::DataTag},
-        error::{Error, QueryResult},
+        error::{QueryError, QueryResult},
         ql::dml::WhereClause,
     },
     util::compiler,
@@ -55,7 +55,7 @@ impl Model {
             {
                 Ok(clause.rhs())
             }
-            _ => compiler::cold_rerr(Error::QPDmlWhereHasUnindexedColumn),
+            _ => compiler::cold_rerr(QueryError::QPDmlWhereHasUnindexedColumn),
         }
     }
 }
diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
index a7af4fdc..5dd4d535 100644
--- a/server/src/engine/core/dml/sel.rs
+++ b/server/src/engine/core/dml/sel.rs
@@ -27,7 +27,7 @@
 use crate::engine::{
     core::index::DcFieldIndex,
     data::cell::{Datacell, VirtualDatacell},
-    error::{Error, QueryResult},
+    error::{QueryError, QueryResult},
     fractal::GlobalInstanceLike,
     idx::{STIndex, STIndexSeq},
     ql::dml::sel::SelectStatement,
@@ -51,7 +51,7 @@ where
             match fields.st_get(key) {
                 Some(dc) => cellfn(dc),
                 None if key == mdl.p_key() => cellfn(&pkdc),
-                None => return Err(Error::QPUnknownField),
+                None => return Err(QueryError::QPUnknownField),
             }
             Ok(())
         };
@@ -68,7 +68,7 @@ where
                     }
                 }
             }
-            None => return Err(Error::QPDmlRowNotFound),
+            None => return Err(QueryError::QPDmlRowNotFound),
         }
         Ok(())
     })
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index 95bd6089..5d37fa5a 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -36,7 +36,7 @@ use {
                 lit::Lit,
                 tag::{DataTag, TagClass},
             },
-            error::{Error, QueryResult},
+            error::{QueryError, QueryResult},
             fractal::GlobalInstanceLike,
             idx::STIndex,
             ql::dml::upd::{AssignmentExpression, UpdateStatement},
@@ -242,7 +242,7 @@ pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) ->
         // fetch row
         let g = sync::atm::cpin();
         let Some(row) = mdl.primary_index().select(key, &g) else {
-            return Err(Error::QPDmlRowNotFound);
+            return Err(QueryError::QPDmlRowNotFound);
         };
         // lock row
         let mut row_data_wl = row.d_data().write();
@@ -279,7 +279,7 @@ pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) ->
                 _ => {
                     input_trace("fieldnotfound");
                     rollback_now = true;
-                    ret = Err(Error::QPUnknownField);
+                    ret = Err(QueryError::QPUnknownField);
                     break;
                 }
             }
@@ -313,20 +313,20 @@ pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) ->
                                 list.push(rhs.into());
                             } else {
                                 rollback_now = true;
-                                ret = Err(Error::SysOutOfMemory);
+                                ret = Err(QueryError::SysOutOfMemory);
                                 break;
                             }
                         }
                     } else {
                         input_trace("list;badtag");
                         rollback_now = true;
-                        ret = Err(Error::QPDmlValidationError);
+                        ret = Err(QueryError::QPDmlValidationError);
                         break;
                     }
                 }
                 _ => {
                     input_trace("unknown_reason;exitmainloop");
-                    ret = Err(Error::QPDmlValidationError);
+                    ret = Err(QueryError::QPDmlValidationError);
                     rollback_now = true;
                     break;
                 }
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index ddcb599e..e9fcb39b 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -38,7 +38,7 @@ use {
     self::{model::Model, util::EntityLocator},
     crate::engine::{
         core::space::Space,
-        error::{Error, QueryResult},
+        error::{QueryError, QueryResult},
         fractal::GlobalInstanceLike,
         idx::{IndexST, STIndex},
     },
@@ -100,7 +100,7 @@ impl GlobalNS {
     ) -> QueryResult<T> {
         let sread = self.index_space.read();
         let Some(space) = sread.st_get(space) else {
-            return Err(Error::QPObjectNotFound);
+            return Err(QueryError::QPObjectNotFound);
         };
         f(space)
     }
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index d98f77cb..638f3211 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -33,7 +33,7 @@ use {
                 tag::{DataTag, TagClass},
                 DictEntryGeneric,
             },
-            error::{Error, QueryResult},
+            error::{QueryError, QueryResult},
             fractal::GlobalInstanceLike,
             idx::{IndexST, IndexSTSeqCns, STIndex, STIndexSeq},
             ql::{
@@ -84,7 +84,7 @@ fn no_field(mr: &IWModel, new: &str) -> bool {
 fn check_nullable(props: &mut HashMap<Box<str>, DictEntryGeneric>) -> QueryResult<bool> {
     match props.remove("nullable") {
         Some(DictEntryGeneric::Data(b)) if b.kind() == TagClass::Bool => Ok(b.bool()),
-        Some(_) => Err(Error::QPDdlInvalidProperties),
+        Some(_) => Err(QueryError::QPDdlInvalidProperties),
         None => Ok(false),
     }
 }
@@ -101,7 +101,7 @@ impl<'a> AlterPlan<'a> {
             AlterKind::Remove(r) => {
                 let mut x = HashSet::new();
                 if !r.iter().all(|id| x.insert(id.as_str())) {
-                    return Err(Error::QPDdlModelAlterIllegal);
+                    return Err(QueryError::QPDdlModelAlterIllegal);
                 }
                 let mut not_found = false;
                 if r.iter().all(|id| {
@@ -112,9 +112,9 @@ impl<'a> AlterPlan<'a> {
                 }) {
                     can_ignore!(AlterAction::Remove(r))
                 } else if not_found {
-                    return Err(Error::QPUnknownField);
+                    return Err(QueryError::QPUnknownField);
                 } else {
-                    return Err(Error::QPDdlModelAlterIllegal);
+                    return Err(QueryError::QPDdlModelAlterIllegal);
                 }
             }
             AlterKind::Add(new_fields) => {
@@ -148,7 +148,7 @@ impl<'a> AlterPlan<'a> {
                     mv.guard_pk(&field_name)?;
                     // get the current field
                     let Some(current_field) = wm.fields().st_get(field_name.as_str()) else {
-                        return Err(Error::QPUnknownField);
+                        return Err(QueryError::QPUnknownField);
                     };
                     // check props
                     let is_nullable = check_nullable(&mut props)?;
@@ -174,7 +174,7 @@ impl<'a> AlterPlan<'a> {
                 no_lock,
             })
         } else {
-            Err(Error::QPDdlModelAlterIllegal)
+            Err(QueryError::QPDdlModelAlterIllegal)
         }
     }
     fn ldeltas(
@@ -197,7 +197,7 @@ impl<'a> AlterPlan<'a> {
         }
         if layers.len() > current.layers().len() {
             // simply a dumb tomato; ELIMINATE THESE DUMB TOMATOES
-            return Err(Error::QPDdlModelAlterIllegal);
+            return Err(QueryError::QPDdlModelAlterIllegal);
         }
         let mut no_lock = !(current.is_nullable() & !nullable);
         let mut deltasize = (current.is_nullable() ^ nullable) as usize;
@@ -216,7 +216,7 @@ impl<'a> AlterPlan<'a> {
             // actually parse the new layer
             okay &= props.is_empty();
             let Some(new_parsed_layer) = Layer::get_layer(&ty) else {
-                return Err(Error::QPDdlInvalidTypeDefinition);
+                return Err(QueryError::QPDdlInvalidTypeDefinition);
             };
             match (
                 current_layer.tag.tag_selector(),
@@ -233,7 +233,7 @@ impl<'a> AlterPlan<'a> {
                 }
                 _ => {
                     // can't cast this directly
-                    return Err(Error::QPDdlInvalidTypeDefinition);
+                    return Err(QueryError::QPDdlInvalidTypeDefinition);
                 }
             }
             *new_layer = new_parsed_layer;
@@ -243,7 +243,7 @@ impl<'a> AlterPlan<'a> {
         if okay {
             Ok((deltasize != 0, new_field))
         } else {
-            Err(Error::QPDdlModelAlterIllegal)
+            Err(QueryError::QPDdlModelAlterIllegal)
         }
     }
 }
@@ -263,7 +263,7 @@ impl Model {
                 // we have a legal plan; acquire exclusive if we need it
                 if !plan.no_lock {
                     // TODO(@ohsayan): allow this later on, once we define the syntax
-                    return Err(Error::QPNeedLock);
+                    return Err(QueryError::QPNeedLock);
                 }
                 // fine, we're good
                 let mut iwm = iwm;
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 2c689e46..cc84d962 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -39,7 +39,7 @@ use {
             tag::{DataTag, FullTag, TagClass, TagSelector},
             uuid::Uuid,
         },
-        error::{Error, QueryResult},
+        error::{QueryError, QueryResult},
         fractal::{GenericTask, GlobalInstanceLike, Task},
         idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
         mem::VInline,
@@ -134,7 +134,7 @@ impl Model {
     }
     fn guard_pk(&self, new: &str) -> QueryResult<()> {
         if self.is_pk(new) {
-            Err(Error::QPDdlModelAlterIllegal)
+            Err(QueryError::QPDdlModelAlterIllegal)
         } else {
             Ok(())
         }
@@ -199,7 +199,7 @@ impl Model {
                 return Ok(Self::new_restore(Uuid::new(), last_pk.into(), tag, fields));
             }
         }
-        Err(Error::QPDdlModelBadDefinition)
+        Err(QueryError::QPDdlModelBadDefinition)
     }
 }
 
@@ -213,7 +213,7 @@ impl Model {
         global.namespace().with_space(space_name, |space| {
             let mut w_space = space.models().write();
             if w_space.st_contains(model_name) {
-                return Err(Error::QPDdlObjectAlreadyExists);
+                return Err(QueryError::QPDdlObjectAlreadyExists);
             }
             if G::FS_IS_NON_NULL {
                 let irm = model.intent_read_model();
@@ -262,7 +262,7 @@ impl Model {
         global.namespace().with_space(space_name, |space| {
             let mut w_space = space.models().write();
             let Some(model) = w_space.get(model_name) else {
-                return Err(Error::QPObjectNotFound);
+                return Err(QueryError::QPObjectNotFound);
             };
             if G::FS_IS_NON_NULL {
                 // prepare txn
@@ -361,7 +361,7 @@ impl Field {
                 nullable,
             })
         } else {
-            Err(Error::QPDdlInvalidTypeDefinition)
+            Err(QueryError::QPDdlInvalidTypeDefinition)
         }
     }
     #[inline(always)]
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 6d4b8496..78171379 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -28,7 +28,7 @@ use {
     crate::engine::{
         core::{model::Model, RWLIdx},
         data::{dict, uuid::Uuid, DictEntryGeneric, DictGeneric},
-        error::{Error, QueryResult},
+        error::{QueryError, QueryResult},
         fractal::{GenericTask, GlobalInstanceLike, Task},
         idx::{IndexST, STIndex},
         ql::ddl::{alt::AlterSpace, crt::CreateSpace, drop::DropSpace},
@@ -108,7 +108,7 @@ impl Space {
         {
             Ok(())
         } else {
-            Err(Error::QPDdlObjectAlreadyExists)
+            Err(QueryError::QPDdlObjectAlreadyExists)
         }
     }
     pub fn get_uuid(&self) -> Uuid {
@@ -127,7 +127,7 @@ impl Space {
     ) -> QueryResult<T> {
         let mread = self.mns.read();
         let Some(model) = mread.st_get(model) else {
-            return Err(Error::QPObjectNotFound);
+            return Err(QueryError::QPObjectNotFound);
         };
         f(model)
     }
@@ -174,7 +174,7 @@ impl Space {
             None if props.is_empty() => IndexST::default(),
             _ => {
                 // unknown properties
-                return Err(Error::QPDdlInvalidProperties);
+                return Err(QueryError::QPDdlInvalidProperties);
             }
         };
         Ok(ProcedureCreate {
@@ -200,7 +200,7 @@ impl Space {
         // acquire access
         let mut wl = global.namespace().spaces().write();
         if wl.st_contains(&space_name) {
-            return Err(Error::QPDdlObjectAlreadyExists);
+            return Err(QueryError::QPDdlObjectAlreadyExists);
         }
         // commit txn
         if G::FS_IS_NON_NULL {
@@ -240,13 +240,13 @@ impl Space {
                 Some(DictEntryGeneric::Map(_)) if updated_props.len() == 1 => {}
                 Some(DictEntryGeneric::Data(l)) if updated_props.len() == 1 && l.is_null() => {}
                 None if updated_props.is_empty() => return Ok(()),
-                _ => return Err(Error::QPDdlInvalidProperties),
+                _ => return Err(QueryError::QPDdlInvalidProperties),
             }
             let mut space_props = space.meta.dict().write();
             // create patch
             let patch = match dict::rprepare_metadata_patch(&space_props, updated_props) {
                 Some(patch) => patch,
-                None => return Err(Error::QPDdlInvalidProperties),
+                None => return Err(QueryError::QPDdlInvalidProperties),
             };
             if G::FS_IS_NON_NULL {
                 // prepare txn
@@ -276,11 +276,11 @@ impl Space {
         let mut wgns = global.namespace().spaces().write();
         let space = match wgns.get(space_name.as_str()) {
             Some(space) => space,
-            None => return Err(Error::QPObjectNotFound),
+            None => return Err(QueryError::QPObjectNotFound),
         };
         let space_w = space.mns.write();
         if space_w.st_len() != 0 {
-            return Err(Error::QPDdlNotEmpty);
+            return Err(QueryError::QPDdlNotEmpty);
         }
         // we can remove this
         if G::FS_IS_NON_NULL {
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index 70526fc9..09ac8fc7 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -77,7 +77,7 @@ mod plan {
     use crate::{
         engine::{
             core::model::{self, alt::AlterAction, Field, Layer},
-            error::Error,
+            error::QueryError,
         },
         vecfuse,
     };
@@ -164,7 +164,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            Error::QPUnknownField
+            QueryError::QPUnknownField
         );
     }
     #[test]
@@ -176,7 +176,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            Error::QPDdlModelAlterIllegal
+            QueryError::QPDdlModelAlterIllegal
         );
     }
     #[test]
@@ -188,7 +188,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            Error::QPDdlModelAlterIllegal
+            QueryError::QPDdlModelAlterIllegal
         );
     }
     #[test]
@@ -200,7 +200,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            Error::QPDdlModelAlterIllegal
+            QueryError::QPDdlModelAlterIllegal
         );
     }
     #[test]
@@ -212,7 +212,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            Error::QPDdlModelAlterIllegal
+            QueryError::QPDdlModelAlterIllegal
         );
     }
     #[test]
@@ -224,7 +224,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            Error::QPUnknownField
+            QueryError::QPUnknownField
         );
     }
     fn bad_type_cast(orig_ty: &str, new_ty: &str) {
@@ -235,7 +235,7 @@ mod plan {
             super::with_plan(&create, &alter, |_| {}).expect_err(&format!(
                 "found no error in transformation: {orig_ty} -> {new_ty}"
             )),
-            Error::QPDdlInvalidTypeDefinition,
+            QueryError::QPDdlInvalidTypeDefinition,
             "failed to match error in transformation: {orig_ty} -> {new_ty}",
         )
     }
@@ -353,7 +353,7 @@ mod plan {
 mod exec {
     use crate::engine::{
         core::model::{DeltaVersion, Field, Layer},
-        error::Error,
+        error::QueryError,
         fractal::test_utils::TestGlobal,
         idx::{STIndex, STIndexSeq},
     };
@@ -445,7 +445,7 @@ mod exec {
                 |_| {},
             )
             .unwrap_err(),
-            Error::QPNeedLock
+            QueryError::QPNeedLock
         );
     }
 }
diff --git a/server/src/engine/core/tests/ddl_model/crt.rs b/server/src/engine/core/tests/ddl_model/crt.rs
index a42c5904..7193dc14 100644
--- a/server/src/engine/core/tests/ddl_model/crt.rs
+++ b/server/src/engine/core/tests/ddl_model/crt.rs
@@ -30,7 +30,7 @@ mod validation {
         crate::engine::{
             core::model::{DeltaVersion, Field, Layer},
             data::tag::{DataTag, FullTag},
-            error::Error,
+            error::QueryError,
             idx::STIndexSeq,
         },
     };
@@ -89,7 +89,7 @@ mod validation {
                 "create model mymodel(primary username: string, primary contract_location: binary)"
             )
             .unwrap_err(),
-            Error::QPDdlModelBadDefinition
+            QueryError::QPDdlModelBadDefinition
         );
     }
 
@@ -97,7 +97,7 @@ mod validation {
     fn duplicate_fields() {
         assert_eq!(
             create("create model mymodel(primary username: string, username: binary)").unwrap_err(),
-            Error::QPDdlModelBadDefinition
+            QueryError::QPDdlModelBadDefinition
         );
     }
 
@@ -105,7 +105,7 @@ mod validation {
     fn illegal_props() {
         assert_eq!(
         create("create model mymodel(primary username: string, password: binary) with { lol_prop: false }").unwrap_err(),
-        Error::QPDdlModelBadDefinition
+        QueryError::QPDdlModelBadDefinition
     );
     }
 
@@ -116,12 +116,12 @@ mod validation {
             "create model mymodel(primary username_bytes: list { type: uint8 }, password: binary)"
         )
         .unwrap_err(),
-        Error::QPDdlModelBadDefinition
+        QueryError::QPDdlModelBadDefinition
     );
         assert_eq!(
             create("create model mymodel(primary username: float32, password: binary)")
                 .unwrap_err(),
-            Error::QPDdlModelBadDefinition
+            QueryError::QPDdlModelBadDefinition
         );
     }
 }
diff --git a/server/src/engine/core/tests/ddl_model/layer.rs b/server/src/engine/core/tests/ddl_model/layer.rs
index e5241483..0ee2626e 100644
--- a/server/src/engine/core/tests/ddl_model/layer.rs
+++ b/server/src/engine/core/tests/ddl_model/layer.rs
@@ -42,7 +42,7 @@ fn layerview(layer_def: &str) -> QueryResult<Field> {
 mod layer_spec_validation {
     use {
         super::layerview,
-        crate::engine::{core::model::Layer, error::Error},
+        crate::engine::{core::model::Layer, error::QueryError},
     };
 
     #[test]
@@ -64,7 +64,7 @@ mod layer_spec_validation {
     fn invalid_list() {
         assert_eq!(
             layerview("list").unwrap_err(),
-            Error::QPDdlInvalidTypeDefinition
+            QueryError::QPDdlInvalidTypeDefinition
         );
     }
 
@@ -72,7 +72,7 @@ mod layer_spec_validation {
     fn invalid_flat() {
         assert_eq!(
             layerview("string { type: string }").unwrap_err(),
-            Error::QPDdlInvalidTypeDefinition
+            QueryError::QPDdlInvalidTypeDefinition
         );
     }
 }
diff --git a/server/src/engine/core/tests/ddl_space/alter.rs b/server/src/engine/core/tests/ddl_space/alter.rs
index a37c9255..e71e7ec0 100644
--- a/server/src/engine/core/tests/ddl_space/alter.rs
+++ b/server/src/engine/core/tests/ddl_space/alter.rs
@@ -27,7 +27,7 @@
 use crate::engine::{
     core::space::{Space, SpaceMeta},
     data::cell::Datacell,
-    error::Error,
+    error::QueryError,
     fractal::test_utils::TestGlobal,
 };
 
@@ -122,7 +122,7 @@ fn alter_nx() {
             |_| {},
         )
         .unwrap_err(),
-        Error::QPObjectNotFound
+        QueryError::QPObjectNotFound
     );
 }
 
diff --git a/server/src/engine/core/tests/ddl_space/create.rs b/server/src/engine/core/tests/ddl_space/create.rs
index 669f9051..b6e70bb2 100644
--- a/server/src/engine/core/tests/ddl_space/create.rs
+++ b/server/src/engine/core/tests/ddl_space/create.rs
@@ -27,7 +27,7 @@
 use crate::engine::{
     core::space::{Space, SpaceMeta},
     data::cell::Datacell,
-    error::Error,
+    error::QueryError,
     fractal::test_utils::TestGlobal,
 };
 
@@ -73,7 +73,7 @@ fn exec_create_space_with_bad_env_type() {
     let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_create(&global, "create space myspace with { env: 100 }", |_| {}).unwrap_err(),
-        Error::QPDdlInvalidProperties
+        QueryError::QPDdlInvalidProperties
     );
 }
 
@@ -87,6 +87,6 @@ fn exec_create_space_with_random_property() {
             |_| {}
         )
         .unwrap_err(),
-        Error::QPDdlInvalidProperties
+        QueryError::QPDdlInvalidProperties
     );
 }
diff --git a/server/src/engine/core/tests/dml/delete.rs b/server/src/engine/core/tests/dml/delete.rs
index 8e7d375a..5a8a32c4 100644
--- a/server/src/engine/core/tests/dml/delete.rs
+++ b/server/src/engine/core/tests/dml/delete.rs
@@ -24,7 +24,7 @@
  *
 */
 
-use crate::engine::{error::Error, fractal::test_utils::TestGlobal};
+use crate::engine::{error::QueryError, fractal::test_utils::TestGlobal};
 
 #[test]
 fn simple_delete() {
@@ -51,6 +51,6 @@ fn delete_nonexisting() {
             "sayan",
         )
         .unwrap_err(),
-        Error::QPDmlRowNotFound
+        QueryError::QPDmlRowNotFound
     );
 }
diff --git a/server/src/engine/core/tests/dml/insert.rs b/server/src/engine/core/tests/dml/insert.rs
index 09d47806..109d41c8 100644
--- a/server/src/engine/core/tests/dml/insert.rs
+++ b/server/src/engine/core/tests/dml/insert.rs
@@ -24,7 +24,7 @@
  *
 */
 
-use crate::engine::{data::cell::Datacell, error::Error, fractal::test_utils::TestGlobal};
+use crate::engine::{data::cell::Datacell, error::QueryError, fractal::test_utils::TestGlobal};
 
 #[derive(sky_macros::Wrapper, Debug)]
 struct Tuple(Vec<(Box<str>, Datacell)>);
@@ -83,6 +83,6 @@ fn insert_duplicate() {
     assert_eq!(
         super::exec_insert_only(&global, "insert into myspace.mymodel('sayan', 'pass123')")
             .unwrap_err(),
-        Error::QPDmlDuplicate
+        QueryError::QPDmlDuplicate
     );
 }
diff --git a/server/src/engine/core/tests/dml/select.rs b/server/src/engine/core/tests/dml/select.rs
index 206b3adc..966b0411 100644
--- a/server/src/engine/core/tests/dml/select.rs
+++ b/server/src/engine/core/tests/dml/select.rs
@@ -24,7 +24,7 @@
  *
 */
 
-use crate::engine::{data::cell::Datacell, error::Error, fractal::test_utils::TestGlobal};
+use crate::engine::{data::cell::Datacell, error::QueryError, fractal::test_utils::TestGlobal};
 
 #[test]
 fn simple_select_wildcard() {
@@ -97,6 +97,6 @@ fn select_nonexisting() {
             "select username, password from myspace.mymodel where username = 'notsayan'",
         )
         .unwrap_err(),
-        Error::QPDmlRowNotFound
+        QueryError::QPDmlRowNotFound
     );
 }
diff --git a/server/src/engine/core/tests/dml/update.rs b/server/src/engine/core/tests/dml/update.rs
index f32b4ee0..b0cf57d3 100644
--- a/server/src/engine/core/tests/dml/update.rs
+++ b/server/src/engine/core/tests/dml/update.rs
@@ -25,7 +25,7 @@
 */
 
 use crate::engine::{
-    core::dml, data::cell::Datacell, error::Error, fractal::test_utils::TestGlobal,
+    core::dml, data::cell::Datacell, error::QueryError, fractal::test_utils::TestGlobal,
 };
 
 #[test]
@@ -96,7 +96,7 @@ fn fail_operation_on_null() {
             "select * from myspace.mymodel where username='sayan'"
         )
         .unwrap_err(),
-        Error::QPDmlValidationError
+        QueryError::QPDmlValidationError
     );
     assert_eq!(
         dml::update_flow_trace(),
@@ -116,7 +116,7 @@ fn fail_unknown_fields() {
             "select * from myspace.mymodel where username='sayan'"
         )
         .unwrap_err(),
-        Error::QPUnknownField
+        QueryError::QPUnknownField
     );
     assert_eq!(dml::update_flow_trace(), ["fieldnotfound", "rollback"]);
     // verify integrity
@@ -142,7 +142,7 @@ fn fail_typedef_violation() {
             "select * from myspace.mymodel where username = 'sayan'"
         )
         .unwrap_err(),
-        Error::QPDmlValidationError
+        QueryError::QPDmlValidationError
     );
     assert_eq!(
         dml::update_flow_trace(),
diff --git a/server/src/engine/core/util.rs b/server/src/engine/core/util.rs
index 4f74fb8c..15c07be6 100644
--- a/server/src/engine/core/util.rs
+++ b/server/src/engine/core/util.rs
@@ -25,7 +25,7 @@
 */
 
 use crate::engine::{
-    error::{Error, QueryResult},
+    error::{QueryError, QueryResult},
     ql::ast::Entity,
 };
 
@@ -46,6 +46,6 @@ impl<'a> EntityLocator<'a> for Entity<'a> {
     where
         Self: 'a,
     {
-        self.into_full_str().ok_or(Error::QPExpectedEntity)
+        self.into_full_str().ok_or(QueryError::QPExpectedEntity)
     }
 }
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 72ebe463..b0e48b8f 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -24,27 +24,17 @@
  *
 */
 
-use {
-    super::{
-        storage::v1::{SDSSError, SDSSErrorKind},
-        txn::TransactionError,
-    },
-    crate::util::os::SysIOError,
-    std::fmt,
-};
+use {super::config::ConfigError, crate::util::os::SysIOError, std::fmt};
 
-pub type QueryResult<T> = Result<T, Error>;
-// stack
-pub type CtxResult<T, E> = Result<T, CtxError<E>>;
-pub type RuntimeResult<T> = CtxResult<T, RuntimeErrorKind>;
-pub type RuntimeError = CtxError<RuntimeErrorKind>;
+pub type RuntimeResult<T> = Result<T, super::fractal::error::Error>;
+pub type QueryResult<T> = Result<T, QueryError>;
 
 /// an enumeration of 'flat' errors that the server actually responds to the client with, since we do not want to send specific information
 /// about anything (as that will be a security hole). The variants correspond with their actual response codes
 #[derive(Debug, Clone, Copy, PartialEq)]
-pub enum Error {
+pub enum QueryError {
     /// I/O error
-    SysIOError,
+    SysServerError,
     /// out of memory
     SysOutOfMemory,
     /// unknown server error
@@ -108,125 +98,131 @@ pub enum Error {
     SysAuthError,
 }
 
-direct_from! {
-    Error[_] => {
-        SDSSError as StorageSubsystemError,
-        TransactionError as TransactionalError,
+impl From<super::fractal::error::Error> for QueryError {
+    fn from(e: super::fractal::error::Error) -> Self {
+        match e.kind() {
+            ErrorKind::IoError(_) | ErrorKind::Storage(_) => QueryError::SysServerError,
+            ErrorKind::Txn(_) => QueryError::TransactionalError,
+            ErrorKind::Other(_) => QueryError::SysUnknownError,
+            ErrorKind::Config(_) => unreachable!("config error cannot propagate here"),
+        }
     }
 }
 
-/*
-    contextual errors
-*/
-
-/// An error context
-pub enum CtxErrorDescription {
-    A(&'static str),
-    B(Box<str>),
-}
-
-impl CtxErrorDescription {
-    fn inner(&self) -> &str {
-        match self {
-            Self::A(a) => a,
-            Self::B(b) => &b,
+macro_rules! enumerate_err {
+    ($(#[$attr:meta])* $vis:vis enum $errname:ident { $($(#[$varattr:meta])* $variant:ident = $errstring:expr),* $(,)? }) => {
+        $(#[$attr])*
+        $vis enum $errname { $($(#[$varattr])* $variant),* }
+        impl core::fmt::Display for $errname {
+            fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+                match self {$( Self::$variant => write!(f, "{}", $errstring),)*}
+            }
         }
+        impl std::error::Error for $errname {}
     }
 }
 
-impl PartialEq for CtxErrorDescription {
-    fn eq(&self, other: &Self) -> bool {
-        self.inner() == other.inner()
-    }
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+/// A "master" error kind enumeration for all kinds of runtime errors
+pub enum ErrorKind {
+    /// An I/O error
+    IoError(SysIOError),
+    /// An SDSS error
+    Storage(StorageError),
+    /// A transactional error
+    Txn(TransactionError),
+    /// other errors
+    Other(String),
+    /// configuration errors
+    Config(ConfigError),
 }
 
-impl fmt::Display for CtxErrorDescription {
+impl fmt::Display for ErrorKind {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str(self.inner())
+        match self {
+            Self::IoError(io) => write!(f, "io error: {io}"),
+            Self::Storage(se) => write!(f, "storage error: {se}"),
+            Self::Txn(txe) => write!(f, "txn error: {txe}"),
+            Self::Other(oe) => write!(f, "error: {oe}"),
+            Self::Config(cfg) => write!(f, "config error: {cfg}"),
+        }
     }
 }
 
-impl fmt::Debug for CtxErrorDescription {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str(self.inner())
-    }
-}
+impl std::error::Error for ErrorKind {}
 
 direct_from! {
-    CtxErrorDescription => {
-        &'static str as A,
-        String as B,
-        Box<str> as B,
-    }
-}
-
-#[derive(Debug)]
-#[cfg_attr(test, derive(PartialEq))]
-/// A contextual error
-pub struct CtxError<E> {
-    kind: E,
-    ctx: Option<CtxErrorDescription>,
-}
-
-impl<E> CtxError<E> {
-    fn _new(kind: E, ctx: Option<CtxErrorDescription>) -> Self {
-        Self { kind, ctx }
-    }
-    pub fn new(kind: E) -> Self {
-        Self::_new(kind, None)
-    }
-    pub fn with_ctx(kind: E, ctx: impl Into<CtxErrorDescription>) -> Self {
-        Self::_new(kind, Some(ctx.into()))
-    }
-    pub fn add_ctx(self, ctx: impl Into<CtxErrorDescription>) -> Self {
-        Self::with_ctx(self.kind, ctx)
-    }
-    pub fn into_result<T>(self) -> CtxResult<T, E> {
-        Err(self)
-    }
-    pub fn result<T, F>(result: Result<T, F>) -> CtxResult<T, E>
-    where
-        E: From<F>,
-    {
-        result.map_err(|e| CtxError::new(e.into()))
-    }
-    pub fn result_ctx<T, F>(
-        result: Result<T, F>,
-        ctx: impl Into<CtxErrorDescription>,
-    ) -> CtxResult<T, E>
-    where
-        E: From<F>,
-    {
-        result.map_err(|e| CtxError::with_ctx(e.into(), ctx))
+    ErrorKind => {
+        std::io::Error as IoError,
+        SysIOError as IoError,
     }
 }
 
-macro_rules! impl_from_hack {
-    ($($ty:ty),*) => {
-        $(impl<E> From<E> for CtxError<$ty> where E: Into<$ty> {fn from(e: E) -> Self { CtxError::new(e.into()) }})*
+enumerate_err! {
+    #[derive(Debug, PartialEq)]
+    /// Errors that occur when restoring transactional data
+    pub enum TransactionError {
+        /// corrupted txn payload. has more bytes than expected
+        DecodeCorruptedPayloadMoreBytes = "txn-payload-unexpected-content",
+        /// transaction payload is corrupted. has lesser bytes than expected
+        DecodedUnexpectedEof = "txn-payload-unexpected-eof",
+        /// unknown transaction operation. usually indicates a corrupted payload
+        DecodeUnknownTxnOp = "txn-payload-unknown-payload",
+        /// While restoring a certain item, a non-resolvable conflict was encountered in the global state, because the item was
+        /// already present (when it was expected to not be present)
+        OnRestoreDataConflictAlreadyExists = "txn-payload-conflict-already-exists",
+        /// On restore, a certain item that was expected to be present was missing in the global state
+        OnRestoreDataMissing = "txn-payload-conflict-missing",
+        /// On restore, a certain item that was expected to match a certain value, has a different value
+        OnRestoreDataConflictMismatch = "txn-payload-conflict-mismatch",
+        /// out of memory
+        OutOfMemory = "txn-error-oom",
     }
 }
 
-/*
-    Contextual error impls
-*/
-
-impl_from_hack!(RuntimeErrorKind, SDSSErrorKind);
-
-#[derive(Debug)]
-pub enum RuntimeErrorKind {
-    StorageSubsytem(SDSSError),
-    IoError(SysIOError),
-    OSSLErrorMulti(openssl::error::ErrorStack),
-    OSSLError(openssl::ssl::Error),
-}
-
-direct_from! {
-    RuntimeErrorKind => {
-        SDSSError as StorageSubsytem,
-        std::io::Error as IoError,
-        SysIOError as IoError,
-        openssl::error::ErrorStack as OSSLErrorMulti,
-        openssl::ssl::Error as OSSLError,
+enumerate_err! {
+    #[derive(Debug, PartialEq)]
+    /// SDSS based storage engine errors
+    pub enum StorageError {
+        // header
+        /// version mismatch
+        HeaderDecodeVersionMismatch = "header-version-mismatch",
+        /// The entire header is corrupted
+        HeaderDecodeCorruptedHeader = "header-corrupted",
+        /// Expected header values were not matched with the current header
+        HeaderDecodeDataMismatch = "header-data-mismatch",
+        /// The time in the [header/dynrec/rtsig] is in the future
+        HeaderTimeConflict = "header-invalid-time",
+        // journal
+        /// While attempting to handle a basic failure (such as adding a journal entry), the recovery engine ran into an exceptional
+        /// situation where it failed to make a necessary repair the log
+        JournalWRecoveryStageOneFailCritical = "journal-recovery-failure",
+        /// An entry in the journal is corrupted
+        JournalLogEntryCorrupted = "journal-entry-corrupted",
+        /// The structure of the journal is corrupted
+        JournalCorrupted = "journal-corrupted",
+        /// when restoring the journal, a transactional error (i.e constraint violation) occurred
+        JournalRestoreTxnError = "journal-illegal-data",
+        // internal file structures
+        /// While attempting to decode a structure in an internal segment of a file, the storage engine ran into a possibly irrecoverable error
+        InternalDecodeStructureCorrupted = "structure-decode-corrupted",
+        /// the payload (non-static) part of a structure in an internal segment of a file is corrupted
+        InternalDecodeStructureCorruptedPayload = "structure-decode-corrupted-payload",
+        /// the data for an internal structure was decoded but is logically invalid
+        InternalDecodeStructureIllegalData = "structure-decode-illegal-data",
+        /// when attempting to flush a data batch, the batch journal crashed and a recovery event was triggered. But even then,
+        /// the data batch journal could not be fixed
+        DataBatchRecoveryFailStageOne = "batch-recovery-failure",
+        /// when attempting to restore a data batch from disk, the batch journal crashed and had a corruption, but it is irrecoverable
+        DataBatchRestoreCorruptedBatch = "batch-corrupted-batch",
+        /// when attempting to restore a data batch from disk, the driver encountered a corrupted entry
+        DataBatchRestoreCorruptedEntry = "batch-corrupted-entry",
+        /// we failed to close the data batch
+        DataBatchCloseError = "batch-persist-close-failed",
+        /// the data batch file is corrupted
+        DataBatchRestoreCorruptedBatchFile = "batch-corrupted-file",
+        /// the system database is corrupted
+        SysDBCorrupted = "sysdb-corrupted",
     }
 }
diff --git a/server/src/engine/fractal/config.rs b/server/src/engine/fractal/config.rs
index 40188c47..c6dac042 100644
--- a/server/src/engine/fractal/config.rs
+++ b/server/src/engine/fractal/config.rs
@@ -27,7 +27,7 @@
 use crate::engine::config::ConfigAuth;
 
 use {
-    crate::engine::error::{Error, QueryResult},
+    crate::engine::error::{QueryError, QueryResult},
     parking_lot::RwLock,
     std::collections::{hash_map::Entry, HashMap},
 };
@@ -151,7 +151,7 @@ impl SysAuth {
                 ));
                 Ok(())
             }
-            Entry::Occupied(_) => Err(Error::SysAuthError),
+            Entry::Occupied(_) => Err(QueryError::SysAuthError),
         }
     }
     /// Verify the user with the given details
@@ -160,12 +160,12 @@ impl SysAuth {
             if rcrypt::verify(password, self.root_key()).unwrap() {
                 return Ok(());
             } else {
-                return Err(Error::SysAuthError);
+                return Err(QueryError::SysAuthError);
             }
         }
         match self.users.get(username) {
             Some(user) if rcrypt::verify(password, user.key()).unwrap() => Ok(()),
-            Some(_) | None => Err(Error::SysAuthError),
+            Some(_) | None => Err(QueryError::SysAuthError),
         }
     }
     pub fn root_key(&self) -> &[u8] {
diff --git a/server/src/engine/fractal/context.rs b/server/src/engine/fractal/context.rs
new file mode 100644
index 00000000..4d798268
--- /dev/null
+++ b/server/src/engine/fractal/context.rs
@@ -0,0 +1,195 @@
+/*
+ * Created on Sun Oct 01 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use core::fmt;
+use std::cell::RefCell;
+
+/// The current engine context
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum Subsystem {
+    Init,     // the init system
+    Storage,  // the storage engine
+    Database, // the database engine
+    Network,  // the network layer
+}
+
+impl Subsystem {
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::Init => "init system",
+            Self::Storage => "storage error",
+            Self::Database => "engine error",
+            Self::Network => "network error",
+        }
+    }
+}
+
+/*
+    diagnostics
+*/
+
+#[derive(Clone)]
+/// A dmsg
+pub enum Dmsg {
+    A(Box<str>),
+    B(&'static str),
+}
+
+impl PartialEq for Dmsg {
+    fn eq(&self, other: &Self) -> bool {
+        self.as_ref() == other.as_ref()
+    }
+}
+
+impl AsRef<str> for Dmsg {
+    fn as_ref(&self) -> &str {
+        match self {
+            Self::A(a) => a,
+            Self::B(b) => b,
+        }
+    }
+}
+
+direct_from! {
+    Dmsg => {
+        String as A,
+        Box<str> as A,
+        &'static str as B,
+    }
+}
+
+impl fmt::Display for Dmsg {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        <str as fmt::Display>::fmt(self.as_ref(), f)
+    }
+}
+
+impl fmt::Debug for Dmsg {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        <str as fmt::Debug>::fmt(self.as_ref(), f)
+    }
+}
+
+/*
+    context
+*/
+
+macro_rules! exported {
+    ($($vis:vis impl $ty:ty { $($(#[$attr:meta])* $fnvis:vis fn $fn:ident($($fnarg:ident: $fnarg_ty:ty),*) $(-> $fnret:ty)? $fnblock:block)*})*) => {
+        $(impl $ty { $( $(#[$attr])* $fnvis fn $fn($($fnarg: $fnarg_ty),*) $( -> $fnret)? $fnblock )*}
+        $($(#[$attr])* $vis fn $fn($($fnarg: $fnarg_ty),*) $( -> $fnret)? { <$ty>::$fn($($fnarg),*) })*)*
+    }
+}
+
+struct LocalContext {
+    origin: Option<Subsystem>,
+    dmsg: Option<Dmsg>,
+}
+
+fn if_test(f: impl FnOnce()) {
+    if cfg!(test) {
+        f()
+    }
+}
+
+/// A copy of the local context (might be either popped or cloned)
+#[derive(Debug, PartialEq, Clone)]
+pub struct LocalCtxInstance {
+    origin: Option<Subsystem>,
+    dmsg: Option<Dmsg>,
+}
+
+impl LocalCtxInstance {
+    fn new(origin: Option<Subsystem>, dmsg: Option<Dmsg>) -> Self {
+        Self { origin, dmsg }
+    }
+    pub fn origin(&self) -> Option<Subsystem> {
+        self.origin
+    }
+    pub fn dmsg(&self) -> Option<&Dmsg> {
+        self.dmsg.as_ref()
+    }
+}
+
+impl From<LocalContext> for LocalCtxInstance {
+    fn from(LocalContext { origin, dmsg }: LocalContext) -> Self {
+        Self { origin, dmsg }
+    }
+}
+
+exported! {
+    pub impl LocalContext {
+        // all
+        fn set(origin: Subsystem, msg: impl Into<Dmsg>) { Self::_ctx(|ctx| { ctx.origin = Some(origin); ctx.dmsg = Some(msg.into()) }) }
+        fn test_set(origin: Subsystem, msg: impl Into<Dmsg>) { if_test(|| Self::set(origin, msg)) }
+        // dmsg
+        /// set a local dmsg
+        fn set_dmsg(msg: impl Into<Dmsg>) { Self::_ctx(|ctx| ctx.dmsg = Some(msg.into())) }
+        /// (only in test) set a local dmsg
+        fn test_set_dmsg(msg: impl Into<Dmsg>) { if_test(|| Self::set_dmsg(msg)) }
+        /// Set a local dmsg iff not already set
+        fn set_dmsg_if_unset(msg: impl Into<Dmsg>) { Self::_ctx(|ctx| { ctx.dmsg.get_or_insert(msg.into()); }) }
+        /// (only in test) set a local dmsg iff not already set
+        fn test_set_dmsg_if_unset(msg: impl Into<Dmsg>) { if_test(|| Self::set_dmsg_if_unset(msg)) }
+        // origin
+        /// set a local origin
+        fn set_origin(origin: Subsystem) { Self::_ctx(|ctx| ctx.origin = Some(origin)) }
+        /// (only in test) set a local origin
+        fn test_set_origin(origin: Subsystem) { if_test(|| Self::set_origin(origin)) }
+        /// set origin iff unset
+        fn set_origin_if_unset(origin: Subsystem) { Self::_ctx(|ctx| { ctx.origin.get_or_insert(origin); }) }
+        /// (only in test) set a local origin iff not already set
+        fn test_set_origin_if_unset(origin: Subsystem) { if_test(|| Self::set_origin_if_unset(origin)) }
+    }
+    pub(super) impl LocalContext {
+        // alter context
+        /// pop the origin from the local context
+        fn pop_origin() -> Option<Subsystem> { Self::_ctx(|ctx| ctx.origin.take()) }
+        /// pop the dmsg from the local context
+        fn pop_dmsg() -> Option<Dmsg> { Self::_ctx(|ctx| ctx.dmsg.take()) }
+        /// pop the entire context
+        fn pop() -> LocalCtxInstance { Self::_ctx(|ctx| core::mem::replace(ctx, LocalContext::null()).into()) }
+        /// get the origin
+        fn get_origin() -> Option<Subsystem> { Self::_ctx(|ctx| ctx.origin.clone()) }
+        /// get the dmsg
+        fn get_dmsg() -> Option<Dmsg> { Self::_ctx(|ctx| ctx.dmsg.clone()) }
+        /// get a clone of the local context
+        fn cloned() -> LocalCtxInstance { Self::_ctx(|ctx| LocalCtxInstance::new(ctx.origin.clone(), ctx.dmsg.clone())) }
+    }
+}
+
+impl LocalContext {
+    fn _new(origin: Option<Subsystem>, dmsg: Option<Dmsg>) -> Self {
+        Self { origin, dmsg }
+    }
+    fn null() -> Self {
+        Self::_new(None, None)
+    }
+    fn _ctx<T>(f: impl FnOnce(&mut Self) -> T) -> T {
+        thread_local! { static CTX: RefCell<LocalContext> = RefCell::new(LocalContext::null()) }
+        CTX.with(|lctx| f(&mut lctx.borrow_mut()))
+    }
+}
diff --git a/server/src/engine/fractal/error.rs b/server/src/engine/fractal/error.rs
new file mode 100644
index 00000000..42e1d741
--- /dev/null
+++ b/server/src/engine/fractal/error.rs
@@ -0,0 +1,340 @@
+/*
+ * Created on Mon Oct 02 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::context::{self, Dmsg, Subsystem},
+    crate::engine::{
+        config::ConfigError,
+        error::{ErrorKind, StorageError, TransactionError},
+    },
+    core::fmt,
+};
+
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+/// An error implementation with context tracing and propagation
+///
+/// - All errors that are classified in [`ErrorKind`] will automatically inherit all local context, unless explicitly orphaned,
+/// or manually constructed (see [`IntoError::err_noinherit`])
+/// - All other errors will generally take the context from parent
+///
+/// Error propagation and tracing relies on the fact that the first error that occurs will end the routine in question, entering
+/// a new local context; if otherwise, it will fail. To manage such custom conditions, look at [`ErrorContext`] or manually
+/// constructing [`Error`]s.
+pub struct Error {
+    kind: ErrorKind,
+    origin: Option<Subsystem>,
+    dmsg: Option<Dmsg>,
+}
+
+impl Error {
+    /// Returns the diagnostic message
+    pub fn dmsg(&self) -> Option<&Dmsg> {
+        self.dmsg.as_ref()
+    }
+    /// Returns the origin
+    pub fn origin(&self) -> Option<Subsystem> {
+        self.origin
+    }
+    /// Returns the error kind
+    pub fn kind(&self) -> &ErrorKind {
+        &self.kind
+    }
+    /// Replace the origin in self
+    pub fn add_origin(self, origin: Subsystem) -> Self {
+        Self::_new(self.kind, Some(origin), self.dmsg)
+    }
+    /// Replace the dmsg in self
+    pub fn add_dmsg(self, dmsg: impl Into<Dmsg>) -> Self {
+        Self::_new(self.kind, self.origin, Some(dmsg.into()))
+    }
+}
+
+impl Error {
+    /// ctor
+    fn _new(kind: ErrorKind, origin: Option<Subsystem>, dmsg: Option<Dmsg>) -> Self {
+        Self { kind, origin, dmsg }
+    }
+    /// new full error
+    pub fn new(kind: ErrorKind, origin: Subsystem, dmsg: impl Into<Dmsg>) -> Self {
+        Self::_new(kind, Some(origin), Some(dmsg.into()))
+    }
+    /// new error with kind and no ctx
+    pub fn with_kind(kind: ErrorKind) -> Self {
+        Self::_new(kind, None, None)
+    }
+    /// new error with kind and origin
+    fn with_origin(kind: ErrorKind, origin: Subsystem) -> Self {
+        Self::_new(kind, Some(origin), None)
+    }
+    /// new error with kind and dmsg
+    fn with_dmsg(kind: ErrorKind, dmsg: impl Into<Dmsg>) -> Self {
+        Self::_new(kind, None, Some(dmsg.into()))
+    }
+    /// remove the dmsg from self
+    fn remove_dmsg(self) -> Self {
+        Self::_new(self.kind, self.origin, None)
+    }
+    /// remove the origin from self
+    fn remove_origin(self) -> Self {
+        Self::_new(self.kind, None, self.dmsg)
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self.origin {
+            Some(orig) => write!(f, "{} error: ", orig.as_str()),
+            None => write!(f, "runtime error: "),
+        }?;
+        match self.dmsg.as_ref() {
+            Some(dmsg) => write!(f, "{dmsg}; ")?,
+            None => {}
+        }
+        write!(f, "{}", self.kind)
+    }
+}
+
+impl std::error::Error for Error {}
+
+/*
+    generic error casts
+*/
+
+// for all other direct error casts, always inherit context
+impl<E: Into<ErrorKind>> From<E> for Error {
+    fn from(e: E) -> Self {
+        Self::_new(e.into(), context::get_origin(), context::get_dmsg())
+    }
+}
+
+/*
+    error casts used during result private context mutation
+*/
+
+// only used when you're modifying context
+pub trait IntoError {
+    fn err_noinherit(self) -> Error;
+    fn err_inherit_parent(self) -> Error;
+}
+
+// error kinds do not carry any context
+impl<E: Into<ErrorKind>> IntoError for E {
+    fn err_noinherit(self) -> Error {
+        Error::with_kind(self.into())
+    }
+    fn err_inherit_parent(self) -> Error {
+        Self::err_noinherit(self)
+    }
+}
+
+impl IntoError for Error {
+    fn err_noinherit(self) -> Error {
+        Error::with_kind(self.kind)
+    }
+    fn err_inherit_parent(self) -> Error {
+        self
+    }
+}
+
+/*
+    error context and tracing
+*/
+
+pub trait ErrorContext<T> {
+    // no inherit
+    /// set the origin (do not inherit parent or local)
+    fn set_origin(self, origin: Subsystem) -> Result<T, Error>;
+    /// set the dmsg (do not inherit parent or local)
+    fn set_dmsg(self, dmsg: impl Into<Dmsg>) -> Result<T, Error>;
+    /// set the origin and dmsg (do not inherit)
+    fn set_ctx(self, origin: Subsystem, dmsg: impl Into<Dmsg>) -> Result<T, Error>;
+    // inherit parent
+    /// set the origin (inherit rest from parent)
+    fn ip_set_origin(self, origin: Subsystem) -> Result<T, Error>;
+    /// set the dmsg (inherit rest from origin)
+    fn ip_set_dmsg(self, dmsg: impl Into<Dmsg>) -> Result<T, Error>;
+    // inherit local
+    /// set the origin (inherit rest from local)
+    fn il_set_origin(self, origin: Subsystem) -> Result<T, Error>;
+    /// set the dmsg (inherit rest from local)
+    fn il_set_dmsg(self, dmsg: impl Into<Dmsg>) -> Result<T, Error>;
+    /// inherit everything from local (assuming this has no context)
+    fn inherit_local(self) -> Result<T, Error>;
+    // inherit any
+    /// set the origin (inherit rest from either parent, then local)
+    fn inherit_set_origin(self, origin: Subsystem) -> Result<T, Error>;
+    /// set the dmsg (inherit rest from either parent, then local)
+    fn inherit_set_dmsg(self, dmsg: impl Into<Dmsg>) -> Result<T, Error>;
+    // orphan
+    /// orphan the entire context (if any)
+    fn orphan(self) -> Result<T, Error>;
+    /// orphan the origin (if any)
+    fn orphan_origin(self) -> Result<T, Error>;
+    /// orphan the dmsg (if any)
+    fn orphan_dmsg(self) -> Result<T, Error>;
+}
+
+impl<T, E> ErrorContext<T> for Result<T, E>
+where
+    E: IntoError,
+{
+    // no inherit
+    fn set_origin(self, origin: Subsystem) -> Result<T, Error> {
+        self.map_err(|e| e.err_noinherit().add_origin(origin))
+    }
+    fn set_dmsg(self, dmsg: impl Into<Dmsg>) -> Result<T, Error> {
+        self.map_err(|e| e.err_noinherit().add_dmsg(dmsg))
+    }
+    fn set_ctx(self, origin: Subsystem, dmsg: impl Into<Dmsg>) -> Result<T, Error> {
+        self.map_err(|e| Error::new(e.err_noinherit().kind, origin, dmsg))
+    }
+    // inherit local
+    fn il_set_origin(self, origin: Subsystem) -> Result<T, Error> {
+        self.map_err(|e| Error::_new(e.err_noinherit().kind, Some(origin), context::pop_dmsg()))
+    }
+    fn il_set_dmsg(self, dmsg: impl Into<Dmsg>) -> Result<T, Error> {
+        self.map_err(|e| {
+            Error::_new(
+                e.err_noinherit().kind,
+                context::pop_origin(),
+                Some(dmsg.into()),
+            )
+        })
+    }
+    fn inherit_local(self) -> Result<T, Error> {
+        self.map_err(|e| {
+            Error::_new(
+                e.err_noinherit().kind,
+                context::get_origin(),
+                context::get_dmsg(),
+            )
+        })
+    }
+    // inherit parent
+    fn ip_set_origin(self, origin: Subsystem) -> Result<T, Error> {
+        self.map_err(|e| e.err_inherit_parent().add_origin(origin))
+    }
+    fn ip_set_dmsg(self, dmsg: impl Into<Dmsg>) -> Result<T, Error> {
+        self.map_err(|e| e.err_inherit_parent().add_dmsg(dmsg))
+    }
+    // inherit any
+    fn inherit_set_dmsg(self, dmsg: impl Into<Dmsg>) -> Result<T, Error> {
+        self.map_err(|e| {
+            // inherit from parent
+            let mut e = e.err_inherit_parent();
+            // inherit from local if parent has no ctx
+            e.origin = e.origin.or_else(|| context::pop_origin());
+            e.add_dmsg(dmsg)
+        })
+    }
+    fn inherit_set_origin(self, origin: Subsystem) -> Result<T, Error> {
+        self.map_err(|e| {
+            // inherit from parent
+            let mut e = e.err_inherit_parent();
+            // inherit form local if parent has no ctx
+            e.dmsg = e.dmsg.or_else(|| context::pop_dmsg());
+            e.add_origin(origin)
+        })
+    }
+    fn orphan(self) -> Result<T, Error> {
+        self.map_err(|e| e.err_noinherit())
+    }
+    fn orphan_dmsg(self) -> Result<T, Error> {
+        self.map_err(|e| e.err_inherit_parent().remove_dmsg())
+    }
+    fn orphan_origin(self) -> Result<T, Error> {
+        self.map_err(|e| e.err_inherit_parent().remove_origin())
+    }
+}
+
+/*
+    foreign type casts
+*/
+
+macro_rules! impl_other_err_tostring {
+    ($($ty:ty => $origin:ident),* $(,)?) => {
+        $(
+            impl From<$ty> for Error {
+                fn from(e: $ty) -> Self { Self::_new(ErrorKind::Other(e.to_string()), Some(Subsystem::$origin), context::pop_dmsg()) }
+            }
+            impl IntoError for $ty {
+                fn err_noinherit(self) -> Error { Error::with_kind(ErrorKind::Other(self.to_string())) }
+                fn err_inherit_parent(self) -> Error { Self::err_noinherit(self) }
+            }
+        )*
+    }
+}
+
+impl_other_err_tostring! {
+    openssl::ssl::Error => Network,
+    openssl::error::Error => Network,
+    openssl::error::ErrorStack => Network,
+}
+
+impl From<StorageError> for Error {
+    fn from(value: StorageError) -> Self {
+        Self::_new(
+            ErrorKind::Storage(value),
+            context::pop_origin(),
+            context::pop_dmsg(),
+        )
+    }
+}
+
+impl From<TransactionError> for Error {
+    fn from(value: TransactionError) -> Self {
+        Self::_new(
+            ErrorKind::Txn(value),
+            context::pop_origin(),
+            context::pop_dmsg(),
+        )
+    }
+}
+
+impl From<ConfigError> for Error {
+    fn from(e: ConfigError) -> Self {
+        Self::with_origin(ErrorKind::Config(e), Subsystem::Init)
+    }
+}
+
+impl IntoError for StorageError {
+    fn err_noinherit(self) -> Error {
+        Error::with_kind(ErrorKind::Storage(self))
+    }
+    fn err_inherit_parent(self) -> Error {
+        self.into()
+    }
+}
+
+impl IntoError for TransactionError {
+    fn err_noinherit(self) -> Error {
+        Error::with_kind(ErrorKind::Txn(self))
+    }
+    fn err_inherit_parent(self) -> Error {
+        self.into()
+    }
+}
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index 25fa5866..b915a6db 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -30,17 +30,20 @@ use {
         data::uuid::Uuid,
         storage::{
             self,
-            v1::{LocalFS, RawFSInterface, SDSSResult},
+            v1::{LocalFS, RawFSInterface},
         },
         txn::gns::GNSTransactionDriverAnyFS,
     },
+    crate::engine::error::RuntimeResult,
     parking_lot::{Mutex, RwLock},
     std::{collections::HashMap, mem::MaybeUninit},
     tokio::sync::mpsc::unbounded_channel,
 };
 
 pub mod config;
+pub mod context;
 mod drivers;
+pub mod error;
 mod mgr;
 #[cfg(test)]
 pub mod test_utils;
@@ -115,7 +118,7 @@ pub trait GlobalInstanceLike {
         space_uuid: Uuid,
         model_name: &str,
         model_uuid: Uuid,
-    ) -> SDSSResult<()>;
+    ) -> RuntimeResult<()>;
     // taskmgr
     fn taskmgr_post_high_priority(&self, task: Task<CriticalTask>);
     fn taskmgr_post_standard_priority(&self, task: Task<GenericTask>);
@@ -167,7 +170,7 @@ impl GlobalInstanceLike for Global {
         space_uuid: Uuid,
         model_name: &str,
         model_uuid: Uuid,
-    ) -> SDSSResult<()> {
+    ) -> RuntimeResult<()> {
         // create dir
         LocalFS::fs_create_dir(&storage::v1::loader::SEInitState::model_dir(
             space_name, space_uuid, model_name, model_uuid,
diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
index b312db99..ce5b1e9b 100644
--- a/server/src/engine/fractal/test_utils.rs
+++ b/server/src/engine/fractal/test_utils.rs
@@ -125,7 +125,7 @@ impl<Fs: RawFSInterface> GlobalInstanceLike for TestGlobal<Fs> {
         space_uuid: Uuid,
         model_name: &str,
         model_uuid: Uuid,
-    ) -> storage::v1::SDSSResult<()> {
+    ) -> crate::engine::error::RuntimeResult<()> {
         // create model dir
         Fs::fs_create_dir(&storage::v1::loader::SEInitState::model_dir(
             space_name, space_uuid, model_name, model_uuid,
diff --git a/server/src/engine/net/mod.rs b/server/src/engine/net/mod.rs
index c90db058..eeebfe71 100644
--- a/server/src/engine/net/mod.rs
+++ b/server/src/engine/net/mod.rs
@@ -27,10 +27,7 @@
 mod protocol;
 
 use {
-    crate::engine::{
-        error::{RuntimeError, RuntimeResult},
-        fractal::Global,
-    },
+    crate::engine::{error::RuntimeResult, fractal::error::ErrorContext, fractal::Global},
     bytes::BytesMut,
     openssl::{
         pkey::PKey,
@@ -149,10 +146,9 @@ impl Listener {
         sig_shutdown: broadcast::Sender<()>,
     ) -> RuntimeResult<Self> {
         let (sig_inflight, sig_inflight_wait) = mpsc::channel(1);
-        let listener = RuntimeError::result_ctx(
-            TcpListener::bind(binaddr).await,
-            format!("failed to bind to port `{binaddr}`"),
-        )?;
+        let listener = TcpListener::bind(binaddr)
+            .await
+            .set_dmsg(format!("failed to bind to port `{binaddr}`"))?;
         Ok(Self {
             global,
             listener,
@@ -234,8 +230,7 @@ impl Listener {
             builder.check_private_key()?;
             Ok::<_, openssl::error::ErrorStack>(builder.build())
         };
-        let acceptor =
-            RuntimeError::result_ctx(build_acceptor(), "failed to initialize TLS socket")?;
+        let acceptor = build_acceptor().set_dmsg("failed to initialize TLS socket")?;
         loop {
             let stream = async {
                 let (stream, _) = self.accept().await?;
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 4fb2ebd6..940aabbc 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -37,7 +37,7 @@ use {
     crate::{
         engine::{
             data::{cell::Datacell, lit::Lit},
-            error::{Error, QueryResult},
+            error::{QueryError, QueryResult},
         },
         util::{compiler, MaybeInit},
     },
@@ -421,7 +421,7 @@ impl<'a> Entity<'a> {
                 *c += 1;
                 Self::parse_uck_tokens_single(tok)
             },
-            _ => return Err(Error::QPExpectedEntity),
+            _ => return Err(QueryError::QPExpectedEntity),
         };
         Ok(r)
     }
@@ -437,7 +437,7 @@ impl<'a> Entity<'a> {
                 Ok(e.assume_init())
             }
         } else {
-            Err(Error::QPExpectedEntity)
+            Err(QueryError::QPExpectedEntity)
         }
     }
     #[inline(always)]
@@ -531,7 +531,7 @@ pub fn compile_test<'a>(tok: &'a [Token<'a>]) -> QueryResult<Statement<'a>> {
 #[inline(always)]
 pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResult<Statement<'a>> {
     if compiler::unlikely(tok.len() < 2) {
-        return Err(Error::QLUnexpectedEndOfStatement);
+        return Err(QueryError::QLUnexpectedEndOfStatement);
     }
     let mut state = State::new(tok, d);
     match state.fw_read() {
@@ -540,12 +540,12 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResul
         Token![create] => match state.fw_read() {
             Token![model] => ASTNode::from_state(&mut state).map(Statement::CreateModel),
             Token![space] => ASTNode::from_state(&mut state).map(Statement::CreateSpace),
-            _ => compiler::cold_rerr(Error::QPUnknownStatement),
+            _ => compiler::cold_rerr(QueryError::QPUnknownStatement),
         },
         Token![alter] => match state.fw_read() {
             Token![model] => ASTNode::from_state(&mut state).map(Statement::AlterModel),
             Token![space] => ASTNode::from_state(&mut state).map(Statement::AlterSpace),
-            _ => compiler::cold_rerr(Error::QPUnknownStatement),
+            _ => compiler::cold_rerr(QueryError::QPUnknownStatement),
         },
         Token![drop] if state.remaining() >= 2 => ddl::drop::parse_drop(&mut state),
         Token::Ident(id) if id.eq_ignore_ascii_case("inspect") => {
@@ -556,6 +556,6 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResul
         Token![select] => ASTNode::from_state(&mut state).map(Statement::Select),
         Token![update] => ASTNode::from_state(&mut state).map(Statement::Update),
         Token![delete] => ASTNode::from_state(&mut state).map(Statement::Delete),
-        _ => compiler::cold_rerr(Error::QPUnknownStatement),
+        _ => compiler::cold_rerr(QueryError::QPUnknownStatement),
     }
 }
diff --git a/server/src/engine/ql/ast/traits.rs b/server/src/engine/ql/ast/traits.rs
index a8750a2c..79a919ff 100644
--- a/server/src/engine/ql/ast/traits.rs
+++ b/server/src/engine/ql/ast/traits.rs
@@ -27,7 +27,7 @@
 #[cfg(test)]
 use crate::engine::ql::{ast::InplaceData, lex::Token};
 use crate::engine::{
-    error::{Error, QueryResult},
+    error::{QueryError, QueryResult},
     ql::ast::{QueryData, State},
 };
 
@@ -47,7 +47,7 @@ pub trait ASTNode<'a>: Sized {
             return if state.okay() {
                 r
             } else {
-                Err(Error::QLInvalidSyntax)
+                Err(QueryError::QLInvalidSyntax)
             };
         }
         r
@@ -64,7 +64,7 @@ pub trait ASTNode<'a>: Sized {
             return if state.okay() {
                 r
             } else {
-                Err(Error::QLInvalidSyntax)
+                Err(QueryError::QLInvalidSyntax)
             };
         }
         r
@@ -86,7 +86,7 @@ pub trait ASTNode<'a>: Sized {
         if state.exhausted() && state.okay() {
             r
         } else {
-            Err(Error::QLInvalidSyntax)
+            Err(QueryError::QLInvalidSyntax)
         }
     }
 }
diff --git a/server/src/engine/ql/dcl.rs b/server/src/engine/ql/dcl.rs
index 0456dc1a..630e2f42 100644
--- a/server/src/engine/ql/dcl.rs
+++ b/server/src/engine/ql/dcl.rs
@@ -29,7 +29,7 @@ use crate::engine::{
         tag::{DataTag, TagClass},
         DictGeneric,
     },
-    error::{Error, QueryResult},
+    error::{QueryError, QueryResult},
     ql::{
         ast::{traits, QueryData, State},
         ddl::syn,
@@ -43,7 +43,7 @@ fn parse<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<UserMe
         7 tokens
     */
     if state.remaining() < 7 {
-        return Err(Error::QLInvalidSyntax);
+        return Err(QueryError::QLInvalidSyntax);
     }
     let token_buffer = state.current();
     // initial sig
@@ -54,7 +54,7 @@ fn parse<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<UserMe
     state.poison_if_not(signature_okay);
     state.cursor_ahead_by(2);
     let Some(dict) = syn::parse_dict(state) else {
-        return Err(Error::QLInvalidCollectionSyntax);
+        return Err(QueryError::QLInvalidCollectionSyntax);
     };
     let maybe_username = unsafe {
         // UNSAFE(@ohsayan): the dict parse ensures state correctness
@@ -63,7 +63,7 @@ fn parse<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<UserMe
     state.poison_if_not(maybe_username.kind().tag_class() == TagClass::Str);
     if state.not_exhausted() & !state.okay() {
         // we shouldn't have more tokens
-        return Err(Error::QLInvalidSyntax);
+        return Err(QueryError::QLInvalidSyntax);
     }
     Ok(UserMeta {
         username: unsafe {
@@ -129,7 +129,7 @@ impl<'a> UserDel<'a> {
                 }));
             }
         }
-        Err(Error::QLInvalidSyntax)
+        Err(QueryError::QLInvalidSyntax)
     }
 }
 
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 14ddea42..c0d577b3 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -29,7 +29,7 @@ use {
     crate::{
         engine::{
             data::DictGeneric,
-            error::{Error, QueryResult},
+            error::{QueryError, QueryResult},
             ql::{
                 ast::{Entity, QueryData, State},
                 lex::{Ident, Token},
@@ -57,7 +57,7 @@ impl<'a> AlterSpace<'a> {
     /// Parse alter space from tokens
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         if compiler::unlikely(state.remaining() <= 3) {
-            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
+            return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
         }
         let space_name = state.fw_read();
         state.poison_if_not(space_name.is_ident());
@@ -67,7 +67,7 @@ impl<'a> AlterSpace<'a> {
         state.cursor_ahead(); // ignore errors
 
         if compiler::unlikely(!state.okay()) {
-            return Err(Error::QLInvalidSyntax);
+            return Err(QueryError::QLInvalidSyntax);
         }
 
         let space_name = unsafe {
@@ -82,7 +82,7 @@ impl<'a> AlterSpace<'a> {
                 updated_props: d,
             })
         } else {
-            Err(Error::QLInvalidCollectionSyntax)
+            Err(QueryError::QLInvalidCollectionSyntax)
         }
     }
 }
@@ -114,7 +114,7 @@ impl<'a> AlterModel<'a> {
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         // alter model mymodel remove x
         if state.remaining() <= 2 || !state.cursor_has_ident_rounded() {
-            return compiler::cold_rerr(Error::QLInvalidSyntax);
+            return compiler::cold_rerr(QueryError::QLInvalidSyntax);
             // FIXME(@ohsayan): bad because no specificity
         }
         let model_name = Entity::parse_from_state_rounded_result(state)?;
@@ -122,7 +122,7 @@ impl<'a> AlterModel<'a> {
             Token![add] => AlterKind::alter_add(state),
             Token![remove] => AlterKind::alter_remove(state),
             Token![update] => AlterKind::alter_update(state),
-            _ => Err(Error::QPExpectedStatement),
+            _ => Err(QueryError::QPExpectedStatement),
         };
         kind.map(|kind| AlterModel::new(model_name, kind))
     }
@@ -148,7 +148,7 @@ impl<'a> AlterKind<'a> {
             <remove> ::= <ident> | <openparen> (<ident> <comma>)*<closeparen>
         */
         if compiler::unlikely(state.exhausted()) {
-            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
+            return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
         }
 
         let r = match state.fw_read() {
@@ -177,10 +177,10 @@ impl<'a> AlterKind<'a> {
                 if state.okay() {
                     cols.into_boxed_slice()
                 } else {
-                    return Err(Error::QLInvalidSyntax);
+                    return Err(QueryError::QLInvalidSyntax);
                 }
             }
-            _ => return Err(Error::QLInvalidSyntax),
+            _ => return Err(QueryError::QLInvalidSyntax),
         };
         Ok(Self::Remove(r))
     }
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index 39f1c9ac..d9436143 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -29,7 +29,7 @@ use {
     crate::{
         engine::{
             data::DictGeneric,
-            error::{Error, QueryResult},
+            error::{QueryError, QueryResult},
             ql::{
                 ast::{Entity, QueryData, State},
                 lex::Ident,
@@ -54,7 +54,7 @@ impl<'a> CreateSpace<'a> {
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         // smallest declaration: `create space myspace` -> >= 1 token
         if compiler::unlikely(state.remaining() < 1) {
-            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
+            return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
         }
         let space_name = state.fw_read();
         state.poison_if_not(space_name.is_ident());
@@ -76,7 +76,7 @@ impl<'a> CreateSpace<'a> {
                 props: d,
             })
         } else {
-            Err(Error::QLInvalidSyntax)
+            Err(QueryError::QLInvalidSyntax)
         }
     }
 }
@@ -110,7 +110,7 @@ impl<'a> CreateModel<'a> {
 
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         if compiler::unlikely(state.remaining() < 10) {
-            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
+            return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
         }
         // model name; ignore errors
         let mut model_uninit = MaybeInit::uninit();
@@ -147,7 +147,7 @@ impl<'a> CreateModel<'a> {
                 props,
             })
         } else {
-            Err(Error::QLInvalidSyntax)
+            Err(QueryError::QLInvalidSyntax)
         }
     }
 }
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index aedb11a9..6ac4b311 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -25,7 +25,7 @@
 */
 
 use crate::engine::{
-    error::{Error, QueryResult},
+    error::{QueryError, QueryResult},
     ql::{
         ast::{Entity, QueryData, State, Statement},
         lex::{Ident, Token},
@@ -62,7 +62,7 @@ impl<'a> DropSpace<'a> {
                 ));
             }
         }
-        Err(Error::QLInvalidSyntax)
+        Err(QueryError::QLInvalidSyntax)
     }
 }
 
@@ -84,7 +84,7 @@ impl<'a> DropModel<'a> {
         if state.exhausted() {
             return Ok(DropModel::new(e, force));
         } else {
-            Err(Error::QLInvalidSyntax)
+            Err(QueryError::QLInvalidSyntax)
         }
     }
 }
@@ -97,7 +97,7 @@ pub fn parse_drop<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResu
     match state.fw_read() {
         Token![model] => DropModel::parse(state).map(Statement::DropModel),
         Token![space] => return DropSpace::parse(state).map(Statement::DropSpace),
-        _ => Err(Error::QPUnknownStatement),
+        _ => Err(QueryError::QPUnknownStatement),
     }
 }
 
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
index db5f5842..827dd84e 100644
--- a/server/src/engine/ql/ddl/ins.rs
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -26,7 +26,7 @@
 
 use crate::{
     engine::{
-        error::{Error, QueryResult},
+        error::{QueryError, QueryResult},
         ql::{
             ast::{Entity, QueryData, State, Statement},
             lex::Token,
@@ -47,7 +47,7 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
     */
 
     if compiler::unlikely(state.remaining() < 1) {
-        return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
+        return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
     }
 
     match state.fw_read() {
@@ -65,7 +65,7 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
         }
         _ => {
             state.cursor_back();
-            Err(Error::QPExpectedStatement)
+            Err(QueryError::QPExpectedStatement)
         }
     }
 }
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index dd256224..7491ed1f 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -50,7 +50,7 @@ use crate::{
             cell::Datacell,
             dict::{DictEntryGeneric, DictGeneric},
         },
-        error::{Error, QueryResult},
+        error::{QueryError, QueryResult},
         ql::{
             ast::{QueryData, State},
             lex::{Ident, Token},
@@ -369,7 +369,7 @@ impl<'a> FieldSpec<'a> {
     pub fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         if compiler::unlikely(state.remaining() < 2) {
             // smallest field: `ident: type`
-            return Err(Error::QLUnexpectedEndOfStatement);
+            return Err(QueryError::QLUnexpectedEndOfStatement);
         }
         // check if primary or null
         let is_primary = state.cursor_eq(Token![primary]);
@@ -381,7 +381,7 @@ impl<'a> FieldSpec<'a> {
         // field name
         let field_name = match (state.fw_read(), state.fw_read()) {
             (Token::Ident(id), Token![:]) => id,
-            _ => return Err(Error::QLInvalidSyntax),
+            _ => return Err(QueryError::QLInvalidSyntax),
         };
         // layers
         let mut layers = Vec::new();
@@ -394,7 +394,7 @@ impl<'a> FieldSpec<'a> {
                 primary: is_primary,
             })
         } else {
-            Err(Error::QLInvalidTypeDefinitionSyntax)
+            Err(QueryError::QLInvalidTypeDefinitionSyntax)
         }
     }
 }
@@ -420,7 +420,7 @@ impl<'a> ExpandedField<'a> {
     pub(super) fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         if compiler::unlikely(state.remaining() < 6) {
             // smallest: fieldname { type: ident }
-            return Err(Error::QLUnexpectedEndOfStatement);
+            return Err(QueryError::QLUnexpectedEndOfStatement);
         }
         let field_name = state.fw_read();
         state.poison_if_not(field_name.is_ident());
@@ -433,7 +433,7 @@ impl<'a> ExpandedField<'a> {
             // this has layers. fold them; but don't forget the colon
             if compiler::unlikely(state.exhausted()) {
                 // we need more tokens
-                return Err(Error::QLUnexpectedEndOfStatement);
+                return Err(QueryError::QLUnexpectedEndOfStatement);
             }
             state.poison_if_not(state.cursor_eq(Token![:]));
             state.cursor_ahead();
@@ -460,7 +460,7 @@ impl<'a> ExpandedField<'a> {
                 layers,
             })
         } else {
-            Err(Error::QLInvalidSyntax)
+            Err(QueryError::QLInvalidSyntax)
         }
     }
     #[inline(always)]
@@ -478,7 +478,7 @@ impl<'a> ExpandedField<'a> {
             alter model add myfield { type string }
         */
         if compiler::unlikely(state.remaining() < 5) {
-            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
+            return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
         }
         match state.read() {
             Token::Ident(_) => {
@@ -510,10 +510,10 @@ impl<'a> ExpandedField<'a> {
                 if state.okay() {
                     Ok(cols.into_boxed_slice())
                 } else {
-                    Err(Error::QLInvalidSyntax)
+                    Err(QueryError::QLInvalidSyntax)
                 }
             }
-            _ => Err(Error::QPExpectedStatement),
+            _ => Err(QueryError::QPExpectedStatement),
         }
     }
 }
diff --git a/server/src/engine/ql/dml/del.rs b/server/src/engine/ql/dml/del.rs
index 3d3a283b..86b4661e 100644
--- a/server/src/engine/ql/dml/del.rs
+++ b/server/src/engine/ql/dml/del.rs
@@ -30,7 +30,7 @@ use {
     super::WhereClause,
     crate::{
         engine::{
-            error::{Error, QueryResult},
+            error::{QueryError, QueryResult},
             ql::ast::{Entity, QueryData, State},
         },
         util::{compiler, MaybeInit},
@@ -81,7 +81,7 @@ impl<'a> DeleteStatement<'a> {
                    ^1   ^2    ^3    ^4  ^5
         */
         if compiler::unlikely(state.remaining() < 5) {
-            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
+            return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
         }
         // from + entity
         state.poison_if_not(state.cursor_eq(Token![from]));
@@ -101,7 +101,7 @@ impl<'a> DeleteStatement<'a> {
                 wc,
             })
         } else {
-            compiler::cold_rerr(Error::QLInvalidSyntax)
+            compiler::cold_rerr(QueryError::QLInvalidSyntax)
         }
     }
 }
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index e3d9b117..02d8f8fb 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -28,7 +28,7 @@ use {
     crate::{
         engine::{
             data::cell::Datacell,
-            error::{Error, QueryResult},
+            error::{QueryError, QueryResult},
             ql::{
                 ast::{Entity, QueryData, State},
                 lex::{Ident, Token},
@@ -357,7 +357,7 @@ impl<'a> InsertStatement<'a> {
                    ^1    ^2   ^3      ^4 ^5
         */
         if compiler::unlikely(state.remaining() < 5) {
-            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
+            return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
         }
         state.poison_if_not(state.cursor_eq(Token![into]));
         state.cursor_ahead(); // ignore errors
@@ -392,7 +392,7 @@ impl<'a> InsertStatement<'a> {
                 data,
             })
         } else {
-            compiler::cold_rerr(Error::QLInvalidSyntax)
+            compiler::cold_rerr(QueryError::QLInvalidSyntax)
         }
     }
 }
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index 76c9087c..3fc5030b 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -168,7 +168,7 @@ mod impls {
     use {
         super::{RelationalExpr, WhereClause},
         crate::engine::{
-            error::{Error, QueryResult},
+            error::{QueryError, QueryResult},
             ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
@@ -182,7 +182,7 @@ mod impls {
     }
     impl<'a> ASTNode<'a> for RelationalExpr<'a> {
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
-            Self::try_parse(state).ok_or(Error::QLIllegalRelExp)
+            Self::try_parse(state).ok_or(QueryError::QLIllegalRelExp)
         }
     }
 }
diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs
index e387daab..63daa477 100644
--- a/server/src/engine/ql/dml/sel.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -28,7 +28,7 @@ use {
     super::{WhereClause, WhereClauseCollection},
     crate::{
         engine::{
-            error::{Error, QueryResult},
+            error::{QueryError, QueryResult},
             ql::{
                 ast::{Entity, QueryData, State},
                 lex::{Ident, Token},
@@ -104,7 +104,7 @@ impl<'a> SelectStatement<'a> {
                    1 2    3
         */
         if compiler::unlikely(state.remaining() < 3) {
-            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
+            return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
         }
         let mut select_fields = Vec::new();
         let is_wildcard = state.cursor_eq(Token![*]);
@@ -123,7 +123,7 @@ impl<'a> SelectStatement<'a> {
         state.poison_if_not(is_wildcard | !select_fields.is_empty());
         // we should have from + model
         if compiler::unlikely(state.remaining() < 2 || !state.okay()) {
-            return compiler::cold_rerr(Error::QLInvalidSyntax);
+            return compiler::cold_rerr(QueryError::QLInvalidSyntax);
         }
         state.poison_if_not(state.cursor_eq(Token![from]));
         state.cursor_ahead(); // ignore errors
@@ -146,7 +146,7 @@ impl<'a> SelectStatement<'a> {
                 clause: WhereClause::new(clauses),
             })
         } else {
-            compiler::cold_rerr(Error::QLInvalidSyntax)
+            compiler::cold_rerr(QueryError::QLInvalidSyntax)
         }
     }
 }
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index f6a59d4a..dcd2dd21 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -32,7 +32,7 @@ use {
         engine::{
             core::query_meta::AssignmentOperator,
             data::lit::Lit,
-            error::{Error, QueryResult},
+            error::{QueryError, QueryResult},
             ql::{
                 ast::{Entity, QueryData, State},
                 lex::Ident,
@@ -180,7 +180,7 @@ impl<'a> UpdateStatement<'a> {
                    ^1    ^2  ^3 ^4 ^5^6    ^7^8^9
         */
         if compiler::unlikely(state.remaining() < 9) {
-            return compiler::cold_rerr(Error::QLUnexpectedEndOfStatement);
+            return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
         }
         // parse entity
         let mut entity = MaybeInit::uninit();
@@ -218,7 +218,7 @@ impl<'a> UpdateStatement<'a> {
                 wc: WhereClause::new(clauses),
             })
         } else {
-            compiler::cold_rerr(Error::QLInvalidSyntax)
+            compiler::cold_rerr(QueryError::QLInvalidSyntax)
         }
     }
 }
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index 3d4f9363..154a6f85 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -31,7 +31,7 @@ use {
     crate::{
         engine::{
             data::lit::Lit,
-            error::{Error, QueryResult},
+            error::{QueryError, QueryResult},
             mem::BufferedScanner,
         },
         util::compiler,
@@ -51,7 +51,7 @@ type Slice<'a> = &'a [u8];
 pub struct Lexer<'a> {
     token_buffer: BufferedScanner<'a>,
     tokens: Vec<Token<'a>>,
-    last_error: Option<Error>,
+    last_error: Option<QueryError>,
 }
 
 impl<'a> Lexer<'a> {
@@ -66,7 +66,7 @@ impl<'a> Lexer<'a> {
     /// set an error
     #[inline(never)]
     #[cold]
-    fn set_error(&mut self, e: Error) {
+    fn set_error(&mut self, e: QueryError) {
         self.last_error = Some(e);
     }
     /// push in a new token
@@ -116,7 +116,7 @@ impl<'a> Lexer<'a> {
     fn scan_byte(&mut self, byte: u8) {
         match symof(byte) {
             Some(tok) => self.push_token(tok),
-            None => return self.set_error(Error::LexUnexpectedByte),
+            None => return self.set_error(QueryError::LexUnexpectedByte),
         }
         unsafe {
             // UNSAFE(@ohsayan): we are sent a byte, so fw cursor
@@ -210,13 +210,13 @@ impl<'a> InsecureLexer<'a> {
             .token_buffer
             .try_next_ascii_u64_lf_separated_or_restore_cursor()
         else {
-            self.l.set_error(Error::LexInvalidLiteral);
+            self.l.set_error(QueryError::LexInvalidLiteral);
             return;
         };
         let len = len as usize;
         match self.l.token_buffer.try_next_variable_block(len) {
             Some(block) => self.l.push_token(Lit::new_bin(block)),
-            None => self.l.set_error(Error::LexInvalidLiteral),
+            None => self.l.set_error(QueryError::LexInvalidLiteral),
         }
     }
     fn scan_quoted_string(&mut self, quote_style: u8) {
@@ -249,7 +249,7 @@ impl<'a> InsecureLexer<'a> {
                             // UNSAFE(@ohsayan): we move the cursor ahead, now we're moving it back
                             self.l.token_buffer.decr_cursor()
                         }
-                        self.l.set_error(Error::LexInvalidLiteral);
+                        self.l.set_error(QueryError::LexInvalidLiteral);
                         return;
                     }
                 }
@@ -267,7 +267,7 @@ impl<'a> InsecureLexer<'a> {
         }
         match String::from_utf8(buf) {
             Ok(s) if ended_with_quote => self.l.push_token(Lit::new_string(s)),
-            Err(_) | Ok(_) => self.l.set_error(Error::LexInvalidLiteral),
+            Err(_) | Ok(_) => self.l.set_error(QueryError::LexInvalidLiteral),
         }
     }
     fn scan_unsigned_integer(&mut self) {
@@ -288,7 +288,7 @@ impl<'a> InsecureLexer<'a> {
                     .token_buffer
                     .rounded_cursor_not_eof_matches(u8::is_ascii_alphanumeric),
         ) {
-            self.l.set_error(Error::LexInvalidLiteral);
+            self.l.set_error(QueryError::LexInvalidLiteral);
         } else {
             self.l.push_token(Lit::new_uint(int))
         }
@@ -312,7 +312,7 @@ impl<'a> InsecureLexer<'a> {
             {
                 self.l.push_token(Lit::new_sint(int))
             } else {
-                self.l.set_error(Error::LexInvalidLiteral)
+                self.l.set_error(QueryError::LexInvalidLiteral)
             }
         } else {
             self.l.push_token(Token![-]);
@@ -409,7 +409,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             let nb = slf.param_buffer.next_byte();
             slf.l.push_token(Token::Lit(Lit::new_bool(nb == 1)));
             if nb > 1 {
-                slf.l.set_error(Error::LexInvalidEscapedLiteral);
+                slf.l.set_error(QueryError::LexInvalidEscapedLiteral);
             }
         },
         // uint
@@ -418,7 +418,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             .try_next_ascii_u64_lf_separated_or_restore_cursor()
         {
             Some(int) => slf.l.push_token(Lit::new_uint(int)),
-            None => slf.l.set_error(Error::LexInvalidEscapedLiteral),
+            None => slf.l.set_error(QueryError::LexInvalidEscapedLiteral),
         },
         // sint
         |slf| {
@@ -426,7 +426,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             if okay {
                 slf.l.push_token(Lit::new_sint(int))
             } else {
-                slf.l.set_error(Error::LexInvalidLiteral)
+                slf.l.set_error(QueryError::LexInvalidLiteral)
             }
         },
         // float
@@ -435,7 +435,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
                 .param_buffer
                 .try_next_ascii_u64_lf_separated_or_restore_cursor()
             else {
-                slf.l.set_error(Error::LexInvalidEscapedLiteral);
+                slf.l.set_error(QueryError::LexInvalidEscapedLiteral);
                 return;
             };
             let body = match slf
@@ -444,13 +444,13 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             {
                 Some(body) => body,
                 None => {
-                    slf.l.set_error(Error::LexInvalidEscapedLiteral);
+                    slf.l.set_error(QueryError::LexInvalidEscapedLiteral);
                     return;
                 }
             };
             match core::str::from_utf8(body).map(core::str::FromStr::from_str) {
                 Ok(Ok(fp)) => slf.l.push_token(Lit::new_float(fp)),
-                _ => slf.l.set_error(Error::LexInvalidEscapedLiteral),
+                _ => slf.l.set_error(QueryError::LexInvalidEscapedLiteral),
             }
         },
         // binary
@@ -459,7 +459,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
                 .param_buffer
                 .try_next_ascii_u64_lf_separated_or_restore_cursor()
             else {
-                slf.l.set_error(Error::LexInvalidEscapedLiteral);
+                slf.l.set_error(QueryError::LexInvalidEscapedLiteral);
                 return;
             };
             match slf
@@ -467,7 +467,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
                 .try_next_variable_block(size_of_body as usize)
             {
                 Some(block) => slf.l.push_token(Lit::new_bin(block)),
-                None => slf.l.set_error(Error::LexInvalidEscapedLiteral),
+                None => slf.l.set_error(QueryError::LexInvalidEscapedLiteral),
             }
         },
         // string
@@ -476,7 +476,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
                 .param_buffer
                 .try_next_ascii_u64_lf_separated_or_restore_cursor()
             else {
-                slf.l.set_error(Error::LexInvalidEscapedLiteral);
+                slf.l.set_error(QueryError::LexInvalidEscapedLiteral);
                 return;
             };
             match slf
@@ -486,10 +486,10 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             {
                 // TODO(@ohsayan): obliterate this alloc
                 Some(Ok(s)) => slf.l.push_token(Lit::new_string(s.to_owned())),
-                _ => slf.l.set_error(Error::LexInvalidEscapedLiteral),
+                _ => slf.l.set_error(QueryError::LexInvalidEscapedLiteral),
             }
         },
         // ecc
-        |s| s.l.set_error(Error::LexInvalidEscapedLiteral),
+        |s| s.l.set_error(QueryError::LexInvalidEscapedLiteral),
     ]
 };
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index 0af6afc0..d5e3e121 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -29,7 +29,7 @@ use {
         super::lex::{Ident, Token},
         lex_insecure, lex_secure,
     },
-    crate::engine::{data::lit::Lit, error::Error},
+    crate::engine::{data::lit::Lit, error::QueryError},
 };
 
 macro_rules! v(
@@ -143,14 +143,23 @@ fn lex_string_escape_bs() {
 #[test]
 fn lex_string_bad_escape() {
     let wth = br#" '\a should be an alert on windows apparently' "#;
-    assert_eq!(lex_insecure(wth).unwrap_err(), Error::LexInvalidLiteral);
+    assert_eq!(
+        lex_insecure(wth).unwrap_err(),
+        QueryError::LexInvalidLiteral
+    );
 }
 #[test]
 fn lex_string_unclosed() {
     let wth = br#" 'omg where did the end go "#;
-    assert_eq!(lex_insecure(wth).unwrap_err(), Error::LexInvalidLiteral);
+    assert_eq!(
+        lex_insecure(wth).unwrap_err(),
+        QueryError::LexInvalidLiteral
+    );
     let wth = br#" 'see, we escaped the end\' "#;
-    assert_eq!(lex_insecure(wth).unwrap_err(), Error::LexInvalidLiteral);
+    assert_eq!(
+        lex_insecure(wth).unwrap_err(),
+        QueryError::LexInvalidLiteral
+    );
 }
 #[test]
 fn lex_unsafe_literal_mini() {
diff --git a/server/src/engine/storage/v1/batch_jrnl/mod.rs b/server/src/engine/storage/v1/batch_jrnl/mod.rs
index 0348c7a6..36f147b5 100644
--- a/server/src/engine/storage/v1/batch_jrnl/mod.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/mod.rs
@@ -43,15 +43,15 @@ pub(super) use restore::{DecodedBatchEvent, DecodedBatchEventKind, NormalBatch};
 pub use {persist::DataBatchPersistDriver, restore::DataBatchRestoreDriver};
 
 use {
-    super::{rw::SDSSFileIO, spec, RawFSInterface, SDSSResult},
-    crate::engine::core::model::Model,
+    super::{rw::SDSSFileIO, spec, RawFSInterface},
+    crate::engine::{core::model::Model, error::RuntimeResult},
 };
 
 /// Re-initialize an existing batch journal and read all its data into model
 pub fn reinit<Fs: RawFSInterface>(
     name: &str,
     model: &Model,
-) -> SDSSResult<DataBatchPersistDriver<Fs>> {
+) -> RuntimeResult<DataBatchPersistDriver<Fs>> {
     let (f, _header) = SDSSFileIO::<Fs>::open::<spec::DataBatchJournalV1>(name)?;
     // restore
     let mut restore_driver = DataBatchRestoreDriver::new(f)?;
@@ -60,7 +60,7 @@ pub fn reinit<Fs: RawFSInterface>(
 }
 
 /// Create a new batch journal
-pub fn create<Fs: RawFSInterface>(path: &str) -> SDSSResult<DataBatchPersistDriver<Fs>> {
+pub fn create<Fs: RawFSInterface>(path: &str) -> RuntimeResult<DataBatchPersistDriver<Fs>> {
     let f = SDSSFileIO::<Fs>::create::<spec::DataBatchJournalV1>(path)?;
     DataBatchPersistDriver::new(f, true)
 }
diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs
index 06ba5ad8..9a5bba1f 100644
--- a/server/src/engine/storage/v1/batch_jrnl/persist.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs
@@ -42,11 +42,11 @@ use {
                 cell::Datacell,
                 tag::{DataTag, TagClass, TagUnique},
             },
+            error::{RuntimeResult, StorageError},
             idx::STIndexSeq,
             storage::v1::{
                 inf::PersistTypeDscr,
                 rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedWriter},
-                SDSSErrorKind, SDSSResult,
             },
         },
         util::EndianQW,
@@ -59,7 +59,7 @@ pub struct DataBatchPersistDriver<Fs: RawFSInterface> {
 }
 
 impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
-    pub fn new(mut file: SDSSFileIO<Fs>, is_new: bool) -> SDSSResult<Self> {
+    pub fn new(mut file: SDSSFileIO<Fs>, is_new: bool) -> RuntimeResult<Self> {
         if !is_new {
             file.fsynced_write(&[MARKER_BATCH_REOPEN])?;
         }
@@ -67,7 +67,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
             f: SDSSFileTrackedWriter::new(file),
         })
     }
-    pub fn close(mut self) -> SDSSResult<()> {
+    pub fn close(mut self) -> RuntimeResult<()> {
         if self
             .f
             .inner_file()
@@ -76,10 +76,10 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         {
             return Ok(());
         } else {
-            return Err(SDSSErrorKind::DataBatchCloseError.into());
+            return Err(StorageError::DataBatchCloseError.into());
         }
     }
-    pub fn write_new_batch(&mut self, model: &Model, observed_len: usize) -> SDSSResult<()> {
+    pub fn write_new_batch(&mut self, model: &Model, observed_len: usize) -> RuntimeResult<()> {
         // pin model
         let irm = model.intent_read_model();
         let schema_version = model.delta_state().schema_current_version();
@@ -89,7 +89,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         // prepare computations
         let mut i = 0;
         let mut inconsistent_reads = 0;
-        let mut exec = || -> SDSSResult<()> {
+        let mut exec = || -> RuntimeResult<()> {
             // write batch start
             self.write_batch_start(
                 observed_len,
@@ -154,7 +154,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         schema_version: DeltaVersion,
         pk_tag: TagUnique,
         col_cnt: usize,
-    ) -> SDSSResult<()> {
+    ) -> RuntimeResult<()> {
         self.f
             .unfsynced_write(&[MARKER_ACTUAL_BATCH_EVENT, pk_tag.value_u8()])?;
         let observed_len_bytes = observed_len.u64_bytes_le();
@@ -169,7 +169,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         &mut self,
         observed_len: usize,
         inconsistent_reads: usize,
-    ) -> SDSSResult<()> {
+    ) -> RuntimeResult<()> {
         // [0xFD][actual_commit][checksum]
         self.f.unfsynced_write(&[MARKER_END_OF_BATCH])?;
         let actual_commit = (observed_len - inconsistent_reads).u64_bytes_le();
@@ -180,7 +180,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
     }
     /// Attempt to fix the batch journal
     // TODO(@ohsayan): declare an "international system disaster" when this happens
-    fn attempt_fix_data_batchfile(&mut self) -> SDSSResult<()> {
+    fn attempt_fix_data_batchfile(&mut self) -> RuntimeResult<()> {
         /*
             attempt to append 0xFF to the part of the file where a corruption likely occurred, marking
             it recoverable
@@ -189,13 +189,13 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         if f.fsynced_write(&[MARKER_RECOVERY_EVENT]).is_ok() {
             return Ok(());
         }
-        Err(SDSSErrorKind::DataBatchRecoveryFailStageOne.into())
+        Err(StorageError::DataBatchRecoveryFailStageOne.into())
     }
 }
 
 impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
     /// encode the primary key only. this means NO TAG is encoded.
-    fn encode_pk_only(&mut self, pk: &PrimaryIndexKey) -> SDSSResult<()> {
+    fn encode_pk_only(&mut self, pk: &PrimaryIndexKey) -> RuntimeResult<()> {
         let buf = &mut self.f;
         match pk.tag() {
             TagUnique::UnsignedInt | TagUnique::SignedInt => {
@@ -223,7 +223,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         Ok(())
     }
     /// Encode a single cell
-    fn encode_cell(&mut self, value: &Datacell) -> SDSSResult<()> {
+    fn encode_cell(&mut self, value: &Datacell) -> RuntimeResult<()> {
         let ref mut buf = self.f;
         buf.unfsynced_write(&[
             PersistTypeDscr::translate_from_class(value.tag().tag_class()).value_u8(),
@@ -275,7 +275,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         mdl: &Model,
         irm: &IRModel,
         row_data: &RowData,
-    ) -> SDSSResult<()> {
+    ) -> RuntimeResult<()> {
         for field_name in irm.fields().stseq_ord_key() {
             match row_data.fields().get(field_name) {
                 Some(cell) => {
@@ -288,7 +288,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         Ok(())
     }
     /// Write the change type and txnid
-    fn write_batch_item_common_row_data(&mut self, delta: &DataDelta) -> SDSSResult<()> {
+    fn write_batch_item_common_row_data(&mut self, delta: &DataDelta) -> RuntimeResult<()> {
         let change_type = [delta.change().value_u8()];
         self.f.unfsynced_write(&change_type)?;
         let txn_id = delta.data_version().value_u64().to_le_bytes();
diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs
index d6bafbf6..c8b24696 100644
--- a/server/src/engine/storage/v1/batch_jrnl/restore.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs
@@ -38,11 +38,11 @@ use {
             cell::Datacell,
             tag::{CUTag, TagClass, TagUnique},
         },
+        error::{RuntimeResult, StorageError},
         idx::{MTIndex, STIndex, STIndexSeq},
         storage::v1::{
             inf::PersistTypeDscr,
             rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedReader},
-            SDSSErrorKind, SDSSResult,
         },
     },
     crossbeam_epoch::pin,
@@ -110,7 +110,7 @@ pub struct DataBatchRestoreDriver<F: RawFSInterface> {
 }
 
 impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
-    pub fn new(f: SDSSFileIO<F>) -> SDSSResult<Self> {
+    pub fn new(f: SDSSFileIO<F>) -> RuntimeResult<Self> {
         Ok(Self {
             f: SDSSFileTrackedReader::new(f)?,
         })
@@ -121,7 +121,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
     pub(in crate::engine::storage::v1) fn read_data_batch_into_model(
         &mut self,
         model: &Model,
-    ) -> SDSSResult<()> {
+    ) -> RuntimeResult<()> {
         self.read_all_batches_and_for_each(|batch| {
             // apply the batch
             Self::apply_batch(model, batch)
@@ -129,7 +129,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
     }
     pub(in crate::engine::storage::v1) fn read_all_batches(
         &mut self,
-    ) -> SDSSResult<Vec<NormalBatch>> {
+    ) -> RuntimeResult<Vec<NormalBatch>> {
         let mut all_batches = vec![];
         self.read_all_batches_and_for_each(|batch| {
             all_batches.push(batch);
@@ -142,8 +142,8 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
 impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
     fn read_all_batches_and_for_each(
         &mut self,
-        mut f: impl FnMut(NormalBatch) -> SDSSResult<()>,
-    ) -> SDSSResult<()> {
+        mut f: impl FnMut(NormalBatch) -> RuntimeResult<()>,
+    ) -> RuntimeResult<()> {
         // begin
         let mut closed = false;
         while !self.f.is_eof() && !closed {
@@ -187,9 +187,9 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
             }
         }
         // nope, this is a corrupted file
-        Err(SDSSErrorKind::DataBatchRestoreCorruptedBatchFile.into())
+        Err(StorageError::DataBatchRestoreCorruptedBatchFile.into())
     }
-    fn handle_reopen_is_actual_close(&mut self) -> SDSSResult<bool> {
+    fn handle_reopen_is_actual_close(&mut self) -> RuntimeResult<bool> {
         if self.f.is_eof() {
             // yup, it was closed
             Ok(true)
@@ -200,7 +200,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                 Ok(false)
             } else {
                 // that's just a nice bug
-                Err(SDSSErrorKind::DataBatchRestoreCorruptedBatchFile.into())
+                Err(StorageError::DataBatchRestoreCorruptedBatchFile.into())
             }
         }
     }
@@ -213,7 +213,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
             events,
             schema_version,
         }: NormalBatch,
-    ) -> SDSSResult<()> {
+    ) -> RuntimeResult<()> {
         // NOTE(@ohsayan): current complexity is O(n) which is good enough (in the future I might revise this to a fancier impl)
         // pin model
         let irm = m.intent_read_model();
@@ -298,12 +298,12 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
 }
 
 impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
-    fn read_batch_summary(&mut self, finished_early: bool) -> SDSSResult<u64> {
+    fn read_batch_summary(&mut self, finished_early: bool) -> RuntimeResult<u64> {
         if !finished_early {
             // we must read the batch termination signature
             let b = self.f.read_byte()?;
             if b != MARKER_END_OF_BATCH {
-                return Err(SDSSErrorKind::DataBatchRestoreCorruptedBatch.into());
+                return Err(StorageError::DataBatchRestoreCorruptedBatch.into());
             }
         }
         // read actual commit
@@ -320,10 +320,10 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
         if actual_checksum == u64::from_le_bytes(hardcoded_checksum) {
             Ok(actual_commit)
         } else {
-            Err(SDSSErrorKind::DataBatchRestoreCorruptedBatch.into())
+            Err(StorageError::DataBatchRestoreCorruptedBatch.into())
         }
     }
-    fn read_batch(&mut self) -> SDSSResult<Batch> {
+    fn read_batch(&mut self) -> RuntimeResult<Batch> {
         let mut this_batch = vec![];
         // check batch type
         let batch_type = self.f.read_byte()?;
@@ -340,7 +340,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
             }
             _ => {
                 // this is the only singular byte that is expected to be intact. If this isn't intact either, I'm sorry
-                return Err(SDSSErrorKind::DataBatchRestoreCorruptedBatch.into());
+                return Err(StorageError::DataBatchRestoreCorruptedBatch.into());
             }
         }
         // decode batch start block
@@ -384,7 +384,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                                 this_col_cnt -= 1;
                             }
                             if this_col_cnt != 0 {
-                                return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
+                                return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
                             }
                             if change_type == 1 {
                                 this_batch.push(DecodedBatchEvent::new(
@@ -402,7 +402,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                             processed_in_this_batch += 1;
                         }
                         _ => {
-                            return Err(SDSSErrorKind::DataBatchRestoreCorruptedBatch.into());
+                            return Err(StorageError::DataBatchRestoreCorruptedBatch.into());
                         }
                     }
                 }
@@ -413,13 +413,13 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
             batch_start_block.schema_version(),
         )))
     }
-    fn attempt_recover_data_batch(&mut self) -> SDSSResult<()> {
+    fn attempt_recover_data_batch(&mut self) -> RuntimeResult<()> {
         if let Ok(MARKER_RECOVERY_EVENT) = self.f.inner_file().read_byte() {
             return Ok(());
         }
-        Err(SDSSErrorKind::DataBatchRestoreCorruptedBatch.into())
+        Err(StorageError::DataBatchRestoreCorruptedBatch.into())
     }
-    fn read_start_batch_block(&mut self) -> SDSSResult<BatchStartBlock> {
+    fn read_start_batch_block(&mut self) -> RuntimeResult<BatchStartBlock> {
         let pk_tag = self.f.read_byte()?;
         let expected_commit = self.f.read_u64_le()?;
         let schema_version = self.f.read_u64_le()?;
@@ -465,9 +465,9 @@ impl BatchStartBlock {
 }
 
 impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
-    fn decode_primary_key(&mut self, pk_type: u8) -> SDSSResult<PrimaryIndexKey> {
+    fn decode_primary_key(&mut self, pk_type: u8) -> RuntimeResult<PrimaryIndexKey> {
         let Some(pk_type) = TagUnique::try_from_raw(pk_type) else {
-            return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
+            return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
         };
         Ok(match pk_type {
             TagUnique::SignedInt | TagUnique::UnsignedInt => {
@@ -483,7 +483,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                 self.f.read_into_buffer(&mut data)?;
                 if pk_type == TagUnique::Str {
                     if core::str::from_utf8(&data).is_err() {
-                        return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
+                        return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
                     }
                 }
                 unsafe {
@@ -498,17 +498,17 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
             },
         })
     }
-    fn decode_cell(&mut self) -> SDSSResult<Datacell> {
+    fn decode_cell(&mut self) -> RuntimeResult<Datacell> {
         let cell_type_sig = self.f.read_byte()?;
         let Some(cell_type) = PersistTypeDscr::try_from_raw(cell_type_sig) else {
-            return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
+            return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
         };
         Ok(match cell_type {
             PersistTypeDscr::Null => Datacell::null(),
             PersistTypeDscr::Bool => {
                 let bool = self.f.read_byte()?;
                 if bool > 1 {
-                    return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
+                    return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
                 }
                 Datacell::new_bool(bool == 1)
             }
@@ -528,7 +528,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                     // UNSAFE(@ohsayan): +tagck
                     if cell_type == PersistTypeDscr::Str {
                         if core::str::from_utf8(&data).is_err() {
-                            return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
+                            return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
                         }
                         Datacell::new_str(String::from_utf8_unchecked(data).into_boxed_str())
                     } else {
@@ -543,13 +543,13 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                     list.push(self.decode_cell()?);
                 }
                 if len != list.len() as u64 {
-                    return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
+                    return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
                 }
                 Datacell::new_list(list)
             }
             PersistTypeDscr::Dict => {
                 // we don't support dicts just yet
-                return Err(SDSSErrorKind::DataBatchRestoreCorruptedEntry.into());
+                return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
             }
         })
     }
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index 57a2699d..1a92b96a 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -38,9 +38,10 @@ use {
                 tag::{CUTag, TagUnique},
                 DictGeneric,
             },
+            error::{RuntimeResult, StorageError},
             idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
             mem::BufferedScanner,
-            storage::v1::{inf, SDSSError, SDSSErrorKind, SDSSResult},
+            storage::v1::inf,
         },
         util::{copy_slice_to_array as memcpy, EndianQW},
     },
@@ -70,7 +71,7 @@ where
     fn meta_enc(buf: &mut VecU8, data: Self::InputType) {
         buf.extend(data.st_len().u64_bytes_le());
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         Ok(MapIndexSizeMD(scanner.next_u64_le() as usize))
     }
     fn obj_enc(buf: &mut VecU8, map: Self::InputType) {
@@ -87,17 +88,17 @@ where
     unsafe fn obj_dec(
         scanner: &mut BufferedScanner,
         MapIndexSizeMD(dict_size): Self::Metadata,
-    ) -> SDSSResult<Self::OutputType> {
+    ) -> RuntimeResult<Self::OutputType> {
         let mut dict = M::MapType::idx_init();
         while M::pretest_entry_metadata(scanner) & (dict.st_len() != dict_size) {
             let md = unsafe {
                 // UNSAFE(@ohsayan): +pretest
-                M::entry_md_dec(scanner).ok_or::<SDSSError>(
-                    SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into(),
+                M::entry_md_dec(scanner).ok_or::<StorageError>(
+                    StorageError::InternalDecodeStructureCorruptedPayload.into(),
                 )?
             };
             if !M::pretest_entry_data(scanner, &md) {
-                return Err(SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into());
+                return Err(StorageError::InternalDecodeStructureCorruptedPayload.into());
             }
             let key;
             let val;
@@ -109,9 +110,7 @@ where
                             val = _v;
                         }
                         None => {
-                            return Err(
-                                SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into()
-                            )
+                            return Err(StorageError::InternalDecodeStructureCorruptedPayload.into())
                         }
                     }
                 } else {
@@ -123,21 +122,19 @@ where
                             val = _v;
                         }
                         _ => {
-                            return Err(
-                                SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into()
-                            )
+                            return Err(StorageError::InternalDecodeStructureCorruptedPayload.into())
                         }
                     }
                 }
             }
             if !dict.st_insert(key, val) {
-                return Err(SDSSErrorKind::InternalDecodeStructureIllegalData.into());
+                return Err(StorageError::InternalDecodeStructureIllegalData.into());
             }
         }
         if dict.st_len() == dict_size {
             Ok(dict)
         } else {
-            Err(SDSSErrorKind::InternalDecodeStructureIllegalData.into())
+            Err(StorageError::InternalDecodeStructureIllegalData.into())
         }
     }
 }
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index 75db030e..c08d2fd3 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -40,9 +40,9 @@ use {
             dict::DictEntryGeneric,
             tag::{DataTag, TagClass},
         },
+        error::{RuntimeResult, StorageError},
         idx::{AsKey, AsValue},
         mem::BufferedScanner,
-        storage::v1::{SDSSErrorKind, SDSSResult},
     },
     std::mem,
 };
@@ -138,7 +138,7 @@ pub trait PersistObject {
     /// ## Safety
     ///
     /// Must pass the [`PersistObject::pretest_can_dec_metadata`] assertion
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata>;
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata>;
     // obj
     /// obj enc
     fn obj_enc(buf: &mut VecU8, data: Self::InputType);
@@ -147,7 +147,10 @@ pub trait PersistObject {
     /// ## Safety
     ///
     /// Must pass the [`PersistObject::pretest_can_dec_object`] assertion
-    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType>;
+    unsafe fn obj_dec(
+        s: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> RuntimeResult<Self::OutputType>;
     // default
     /// Default routine to encode an object + its metadata
     fn default_full_enc(buf: &mut VecU8, data: Self::InputType) {
@@ -155,16 +158,16 @@ pub trait PersistObject {
         Self::obj_enc(buf, data);
     }
     /// Default routine to decode an object + its metadata (however, the metadata is used and not returned)
-    fn default_full_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::OutputType> {
+    fn default_full_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::OutputType> {
         if !Self::pretest_can_dec_metadata(scanner) {
-            return Err(SDSSErrorKind::InternalDecodeStructureCorrupted.into());
+            return Err(StorageError::InternalDecodeStructureCorrupted.into());
         }
         let md = unsafe {
             // UNSAFE(@ohsayan): +pretest
             Self::meta_dec(scanner)?
         };
         if !Self::pretest_can_dec_object(scanner, &md) {
-            return Err(SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into());
+            return Err(StorageError::InternalDecodeStructureCorruptedPayload.into());
         }
         unsafe {
             // UNSAFE(@ohsayan): +obj pretest
@@ -262,39 +265,39 @@ pub mod enc {
 pub mod dec {
     use {
         super::{map, PersistMapSpec, PersistObject},
-        crate::engine::{mem::BufferedScanner, storage::v1::SDSSResult},
+        crate::engine::{error::RuntimeResult, mem::BufferedScanner},
     };
     // obj
-    pub fn dec_full<Obj: PersistObject>(data: &[u8]) -> SDSSResult<Obj::OutputType> {
+    pub fn dec_full<Obj: PersistObject>(data: &[u8]) -> RuntimeResult<Obj::OutputType> {
         let mut scanner = BufferedScanner::new(data);
         dec_full_from_scanner::<Obj>(&mut scanner)
     }
     pub fn dec_full_from_scanner<Obj: PersistObject>(
         scanner: &mut BufferedScanner,
-    ) -> SDSSResult<Obj::OutputType> {
+    ) -> RuntimeResult<Obj::OutputType> {
         Obj::default_full_dec(scanner)
     }
-    pub fn dec_full_self<Obj: PersistObject<OutputType = Obj>>(data: &[u8]) -> SDSSResult<Obj> {
+    pub fn dec_full_self<Obj: PersistObject<OutputType = Obj>>(data: &[u8]) -> RuntimeResult<Obj> {
         dec_full::<Obj>(data)
     }
     // dec
-    pub fn dec_dict_full<PM: PersistMapSpec>(data: &[u8]) -> SDSSResult<PM::MapType> {
+    pub fn dec_dict_full<PM: PersistMapSpec>(data: &[u8]) -> RuntimeResult<PM::MapType> {
         let mut scanner = BufferedScanner::new(data);
         dec_dict_full_from_scanner::<PM>(&mut scanner)
     }
     fn dec_dict_full_from_scanner<PM: PersistMapSpec>(
         scanner: &mut BufferedScanner,
-    ) -> SDSSResult<PM::MapType> {
+    ) -> RuntimeResult<PM::MapType> {
         <map::PersistMapImpl<PM> as PersistObject>::default_full_dec(scanner)
     }
     pub mod utils {
         use crate::engine::{
+            error::{RuntimeResult, StorageError},
             mem::BufferedScanner,
-            storage::v1::{SDSSErrorKind, SDSSResult},
         };
-        pub unsafe fn decode_string(s: &mut BufferedScanner, len: usize) -> SDSSResult<String> {
+        pub unsafe fn decode_string(s: &mut BufferedScanner, len: usize) -> RuntimeResult<String> {
             String::from_utf8(s.next_chunk_variable(len).to_owned())
-                .map_err(|_| SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into())
+                .map_err(|_| StorageError::InternalDecodeStructureCorruptedPayload.into())
         }
     }
 }
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 51a12e5e..8daace71 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -38,8 +38,9 @@ use {
                 uuid::Uuid,
                 DictGeneric,
             },
+            error::{RuntimeResult, StorageError},
             mem::{BufferedScanner, VInline},
-            storage::v1::{inf, SDSSErrorKind, SDSSResult},
+            storage::v1::inf,
         },
         util::EndianQW,
     },
@@ -113,13 +114,16 @@ impl<'a> PersistObject for LayerRef<'a> {
         buf.extend(layer.tag().tag_selector().value_qword().to_le_bytes());
         buf.extend(0u64.to_le_bytes());
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         Ok(LayerMD::new(scanner.next_u64_le(), scanner.next_u64_le()))
     }
     fn obj_enc(_: &mut VecU8, _: Self::InputType) {}
-    unsafe fn obj_dec(_: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(
+        _: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> RuntimeResult<Self::OutputType> {
         if (md.type_selector > TagSelector::List.value_qword()) | (md.prop_set_arity != 0) {
-            return Err(SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into());
+            return Err(StorageError::InternalDecodeStructureCorruptedPayload.into());
         }
         Ok(Layer::new_empty_props(
             TagSelector::from_raw(md.type_selector as u8).into_full(),
@@ -167,7 +171,7 @@ impl<'a> PersistObject for FieldRef<'a> {
         buf.extend(slf.layers().len().u64_bytes_le());
         buf.push(slf.is_nullable() as u8);
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         Ok(FieldMD::new(
             scanner.next_u64_le(),
             scanner.next_u64_le(),
@@ -182,7 +186,7 @@ impl<'a> PersistObject for FieldRef<'a> {
     unsafe fn obj_dec(
         scanner: &mut BufferedScanner,
         md: Self::Metadata,
-    ) -> SDSSResult<Self::OutputType> {
+    ) -> RuntimeResult<Self::OutputType> {
         let mut layers = VInline::new();
         let mut fin = false;
         while (!scanner.eof())
@@ -202,7 +206,7 @@ impl<'a> PersistObject for FieldRef<'a> {
         if (field.layers().len() as u64 == md.layer_c) & (md.null <= 1) & (md.prop_c == 0) & fin {
             Ok(field)
         } else {
-            Err(SDSSErrorKind::InternalDecodeStructureCorrupted.into())
+            Err(StorageError::InternalDecodeStructureCorrupted.into())
         }
     }
 }
@@ -255,7 +259,7 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
         buf.extend(v.p_tag().tag_selector().value_qword().to_le_bytes());
         buf.extend(irm.fields().len().u64_bytes_le());
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         Ok(ModelLayoutMD::new(
             Uuid::from_bytes(scanner.next_chunk()),
             scanner.next_u64_le(),
@@ -273,7 +277,7 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
     unsafe fn obj_dec(
         scanner: &mut BufferedScanner,
         md: Self::Metadata,
-    ) -> SDSSResult<Self::OutputType> {
+    ) -> RuntimeResult<Self::OutputType> {
         let key = inf::dec::utils::decode_string(scanner, md.p_key_len as usize)?;
         let fieldmap =
             <super::map::PersistMapImpl<super::map::FieldMapSpec> as PersistObject>::obj_dec(
@@ -281,7 +285,7 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
                 super::map::MapIndexSizeMD(md.field_c as usize),
             )?;
         let ptag = if md.p_key_tag > TagSelector::MAX as u64 {
-            return Err(SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into());
+            return Err(StorageError::InternalDecodeStructureCorruptedPayload.into());
         } else {
             TagSelector::from_raw(md.p_key_tag as u8)
         };
@@ -326,7 +330,7 @@ impl<'a> PersistObject for SpaceLayoutRef<'a> {
         buf.extend(space.get_uuid().to_le_bytes());
         buf.extend(space_meta.len().u64_bytes_le());
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         Ok(SpaceLayoutMD::new(
             Uuid::from_bytes(scanner.next_chunk()),
             scanner.next_u64_le() as usize,
@@ -340,7 +344,7 @@ impl<'a> PersistObject for SpaceLayoutRef<'a> {
     unsafe fn obj_dec(
         scanner: &mut BufferedScanner,
         md: Self::Metadata,
-    ) -> SDSSResult<Self::OutputType> {
+    ) -> RuntimeResult<Self::OutputType> {
         let space_meta =
             <super::map::PersistMapImpl<super::map::GenericDictSpec> as PersistObject>::obj_dec(
                 scanner,
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index 6cdfcebf..57f6e6cf 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -44,9 +44,12 @@
 use {
     super::{
         rw::{FileOpen, RawFSInterface, SDSSFileIO},
-        spec, SDSSErrorKind, SDSSResult,
+        spec,
+    },
+    crate::{
+        engine::error::{RuntimeResult, StorageError},
+        util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy},
     },
-    crate::util::{compiler, copy_a_into_b, copy_slice_to_array as memcpy},
     std::marker::PhantomData,
 };
 
@@ -55,7 +58,7 @@ const CRC: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
 pub fn open_journal<TA: JournalAdapter, Fs: RawFSInterface, F: spec::FileSpec>(
     log_file_name: &str,
     gs: &TA::GlobalState,
-) -> SDSSResult<FileOpen<JournalWriter<Fs, TA>>> {
+) -> RuntimeResult<FileOpen<JournalWriter<Fs, TA>>> {
     let file = match SDSSFileIO::<Fs>::open_or_create_perm_rw::<F>(log_file_name)? {
         FileOpen::Created(f) => return Ok(FileOpen::Created(JournalWriter::new(f, 0, true)?)),
         FileOpen::Existing((file, _header)) => file,
@@ -188,7 +191,7 @@ pub struct JournalReader<TA, Fs: RawFSInterface> {
 }
 
 impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
-    pub fn new(log_file: SDSSFileIO<Fs>) -> SDSSResult<Self> {
+    pub fn new(log_file: SDSSFileIO<Fs>) -> RuntimeResult<Self> {
         let log_size = log_file.file_length()? - spec::SDSSStaticHeaderV1Compact::SIZE as u64;
         Ok(Self {
             log_file,
@@ -200,7 +203,7 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
         })
     }
     /// Read the next event and apply it to the global state
-    pub fn rapply_next_event(&mut self, gs: &TA::GlobalState) -> SDSSResult<()> {
+    pub fn rapply_next_event(&mut self, gs: &TA::GlobalState) -> RuntimeResult<()> {
         // read metadata
         let mut en_jrnl_md = [0u8; JournalEntryMetadata::SIZE];
         self.logfile_read_into_buffer(&mut en_jrnl_md)?; // FIXME(@ohsayan): increase tolerance to not just payload
@@ -222,7 +225,7 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
         }
         match entry_metadata
             .event_source_marker()
-            .ok_or(SDSSErrorKind::JournalLogEntryCorrupted)?
+            .ok_or(StorageError::JournalLogEntryCorrupted)?
         {
             EventSourceMarker::ServerStandard => {}
             EventSourceMarker::DriverClosed => {
@@ -237,7 +240,7 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
             EventSourceMarker::DriverReopened | EventSourceMarker::RecoveryReverseLastJournal => {
                 // these two are only taken in close and error paths (respectively) so we shouldn't see them here; this is bad
                 // two special directives in the middle of nowhere? incredible
-                return Err(SDSSErrorKind::JournalCorrupted.into());
+                return Err(StorageError::JournalCorrupted.into());
             }
         }
         // read payload
@@ -256,7 +259,7 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
         Ok(())
     }
     /// handle a driver reopen (IMPORTANT: every event is unique so this must be called BEFORE the ID is incremented)
-    fn handle_driver_reopen(&mut self) -> SDSSResult<()> {
+    fn handle_driver_reopen(&mut self) -> RuntimeResult<()> {
         if self.has_remaining_bytes(JournalEntryMetadata::SIZE as _) {
             let mut reopen_block = [0u8; JournalEntryMetadata::SIZE];
             self.logfile_read_into_buffer(&mut reopen_block)?; // exit jump -> not our business since we have checked flen and if it changes due to user intervention, that's a you problem
@@ -270,10 +273,10 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
                 Ok(())
             } else {
                 // FIXME(@ohsayan): tolerate loss in this directive too
-                Err(SDSSErrorKind::JournalCorrupted.into())
+                Err(StorageError::JournalCorrupted.into())
             }
         } else {
-            Err(SDSSErrorKind::JournalCorrupted.into())
+            Err(StorageError::JournalCorrupted.into())
         }
     }
     #[cold] // FIXME(@ohsayan): how bad can prod systems be? (clue: pretty bad, so look for possible changes)
@@ -281,12 +284,12 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
     /// attempt to recover the journal using the reverse directive (simple strategy)
     /// IMPORTANT: every event is unique so this must be called BEFORE the ID is incremented (remember that we only increment
     /// once we **sucessfully** finish processing a normal (aka server event origin) event and not a non-normal branch)
-    fn try_recover_journal_strategy_simple_reverse(&mut self) -> SDSSResult<()> {
+    fn try_recover_journal_strategy_simple_reverse(&mut self) -> RuntimeResult<()> {
         debug_assert!(TA::RECOVERY_PLUGIN, "recovery plugin not enabled");
         self.__record_read_bytes(JournalEntryMetadata::SIZE); // FIXME(@ohsayan): don't assume read length?
         let mut entry_buf = [0u8; JournalEntryMetadata::SIZE];
         if self.log_file.read_to_buffer(&mut entry_buf).is_err() {
-            return Err(SDSSErrorKind::JournalCorrupted.into());
+            return Err(StorageError::JournalCorrupted.into());
         }
         let entry = JournalEntryMetadata::decode(entry_buf);
         let okay = (entry.event_id == self.evid as u128)
@@ -297,11 +300,14 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
         if okay {
             return Ok(());
         } else {
-            Err(SDSSErrorKind::JournalCorrupted.into())
+            Err(StorageError::JournalCorrupted.into())
         }
     }
     /// Read and apply all events in the given log file to the global state, returning the (open file, last event ID)
-    pub fn scroll(file: SDSSFileIO<Fs>, gs: &TA::GlobalState) -> SDSSResult<(SDSSFileIO<Fs>, u64)> {
+    pub fn scroll(
+        file: SDSSFileIO<Fs>,
+        gs: &TA::GlobalState,
+    ) -> RuntimeResult<(SDSSFileIO<Fs>, u64)> {
         let mut slf = Self::new(file)?;
         while !slf.end_of_file() {
             slf.rapply_next_event(gs)?;
@@ -309,7 +315,7 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
         if slf.closed {
             Ok((slf.log_file, slf.evid))
         } else {
-            Err(SDSSErrorKind::JournalCorrupted.into())
+            Err(StorageError::JournalCorrupted.into())
         }
     }
 }
@@ -330,7 +336,7 @@ impl<TA, Fs: RawFSInterface> JournalReader<TA, Fs> {
 }
 
 impl<TA, Fs: RawFSInterface> JournalReader<TA, Fs> {
-    fn logfile_read_into_buffer(&mut self, buf: &mut [u8]) -> SDSSResult<()> {
+    fn logfile_read_into_buffer(&mut self, buf: &mut [u8]) -> RuntimeResult<()> {
         if !self.has_remaining_bytes(buf.len() as _) {
             // do this right here to avoid another syscall
             return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof).into());
@@ -351,7 +357,7 @@ pub struct JournalWriter<Fs: RawFSInterface, TA> {
 }
 
 impl<Fs: RawFSInterface, TA: JournalAdapter> JournalWriter<Fs, TA> {
-    pub fn new(mut log_file: SDSSFileIO<Fs>, last_txn_id: u64, new: bool) -> SDSSResult<Self> {
+    pub fn new(mut log_file: SDSSFileIO<Fs>, last_txn_id: u64, new: bool) -> RuntimeResult<Self> {
         let log_size = log_file.file_length()?;
         log_file.seek_from_start(log_size)?; // avoid jumbling with headers
         let mut slf = Self {
@@ -366,7 +372,7 @@ impl<Fs: RawFSInterface, TA: JournalAdapter> JournalWriter<Fs, TA> {
         }
         Ok(slf)
     }
-    pub fn append_event(&mut self, event: TA::JournalEvent) -> SDSSResult<()> {
+    pub fn append_event(&mut self, event: TA::JournalEvent) -> RuntimeResult<()> {
         let encoded = TA::encode(event);
         let md = JournalEntryMetadata::new(
             self._incr_id() as u128,
@@ -380,7 +386,10 @@ impl<Fs: RawFSInterface, TA: JournalAdapter> JournalWriter<Fs, TA> {
         self.log_file.fsync_all()?;
         Ok(())
     }
-    pub fn append_event_with_recovery_plugin(&mut self, event: TA::JournalEvent) -> SDSSResult<()> {
+    pub fn append_event_with_recovery_plugin(
+        &mut self,
+        event: TA::JournalEvent,
+    ) -> RuntimeResult<()> {
         debug_assert!(TA::RECOVERY_PLUGIN);
         match self.append_event(event) {
             Ok(()) => Ok(()),
@@ -390,22 +399,22 @@ impl<Fs: RawFSInterface, TA: JournalAdapter> JournalWriter<Fs, TA> {
 }
 
 impl<Fs: RawFSInterface, TA> JournalWriter<Fs, TA> {
-    pub fn appendrec_journal_reverse_entry(&mut self) -> SDSSResult<()> {
+    pub fn appendrec_journal_reverse_entry(&mut self) -> RuntimeResult<()> {
         let mut entry =
             JournalEntryMetadata::new(0, EventSourceMarker::RECOVERY_REVERSE_LAST_JOURNAL, 0, 0);
         entry.event_id = self._incr_id() as u128;
         if self.log_file.fsynced_write(&entry.encoded()).is_ok() {
             return Ok(());
         }
-        Err(SDSSErrorKind::JournalWRecoveryStageOneFailCritical.into())
+        Err(StorageError::JournalWRecoveryStageOneFailCritical.into())
     }
-    pub fn append_journal_reopen(&mut self) -> SDSSResult<()> {
+    pub fn append_journal_reopen(&mut self) -> RuntimeResult<()> {
         let id = self._incr_id() as u128;
         self.log_file.fsynced_write(
             &JournalEntryMetadata::new(id, EventSourceMarker::DRIVER_REOPENED, 0, 0).encoded(),
         )
     }
-    pub fn __append_journal_close_and_close(&mut self) -> SDSSResult<()> {
+    pub fn __append_journal_close_and_close(&mut self) -> RuntimeResult<()> {
         self.closed = true;
         let id = self._incr_id() as u128;
         self.log_file.fsynced_write(
@@ -413,7 +422,7 @@ impl<Fs: RawFSInterface, TA> JournalWriter<Fs, TA> {
         )?;
         Ok(())
     }
-    pub fn append_journal_close_and_close(mut self) -> SDSSResult<()> {
+    pub fn append_journal_close_and_close(mut self) -> RuntimeResult<()> {
         self.__append_journal_close_and_close()
     }
 }
diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs
index 5cebafdb..b43e6cc1 100644
--- a/server/src/engine/storage/v1/loader.rs
+++ b/server/src/engine/storage/v1/loader.rs
@@ -27,12 +27,14 @@
 use crate::engine::{
     core::GlobalNS,
     data::uuid::Uuid,
+    error::RuntimeResult,
+    fractal::error::ErrorContext,
     fractal::{FractalModelDriver, ModelDrivers, ModelUniqueID},
     storage::v1::{
         batch_jrnl,
         journal::{self, JournalWriter},
         rw::{FileOpen, RawFSInterface},
-        spec, LocalFS, SDSSResult,
+        spec, LocalFS,
     },
     txn::gns::{GNSAdapter, GNSTransactionDriverAnyFS},
 };
@@ -61,7 +63,7 @@ impl SEInitState {
             gns,
         }
     }
-    pub fn try_init() -> SDSSResult<Self> {
+    pub fn try_init() -> RuntimeResult<Self> {
         let gns = GlobalNS::empty();
         let gns_txn_driver = open_gns_driver(GNS_FILE_PATH, &gns)?;
         let new_instance = gns_txn_driver.is_created();
@@ -73,11 +75,9 @@ impl SEInitState {
                 for (model_name, model) in space.models().read().iter() {
                     let path =
                         Self::model_path(space_name, space_uuid, model_name, model.get_uuid());
-                    let persist_driver = batch_jrnl::reinit(&path, model).map_err(|e| {
-                        e.add_ctx(format!(
-                            "failed to restore model data from journal in `{path}`"
-                        ))
-                    })?;
+                    let persist_driver = batch_jrnl::reinit(&path, model).inherit_set_dmsg(
+                        format!("failed to restore model data from journal in `{path}`"),
+                    )?;
                     let _ = model_drivers.insert(
                         ModelUniqueID::new(space_name, model_name, model.get_uuid()),
                         FractalModelDriver::init(persist_driver),
@@ -119,6 +119,6 @@ impl SEInitState {
 pub fn open_gns_driver<Fs: RawFSInterface>(
     path: &str,
     gns: &GlobalNS,
-) -> SDSSResult<FileOpen<JournalWriter<Fs, GNSAdapter>>> {
+) -> RuntimeResult<FileOpen<JournalWriter<Fs, GNSAdapter>>> {
     journal::open_journal::<GNSAdapter, Fs, spec::GNSTransactionLogV1>(path, gns)
 }
diff --git a/server/src/engine/storage/v1/memfs.rs b/server/src/engine/storage/v1/memfs.rs
index 9dd11fa9..2ae0ef43 100644
--- a/server/src/engine/storage/v1/memfs.rs
+++ b/server/src/engine/storage/v1/memfs.rs
@@ -26,12 +26,10 @@
 
 use {
     crate::engine::{
-        storage::v1::{
-            rw::{
-                FileOpen, RawFSInterface, RawFileInterface, RawFileInterfaceExt,
-                RawFileInterfaceRead, RawFileInterfaceWrite, RawFileInterfaceWriteExt,
-            },
-            SDSSResult,
+        error::RuntimeResult,
+        storage::v1::rw::{
+            FileOpen, RawFSInterface, RawFileInterface, RawFileInterfaceExt, RawFileInterfaceRead,
+            RawFileInterfaceWrite, RawFileInterfaceWriteExt,
         },
         sync::cell::Lazy,
     },
@@ -81,16 +79,16 @@ impl VNode {
     errors
 */
 
-fn err_item_is_not_file<T>() -> super::SDSSResult<T> {
+fn err_item_is_not_file<T>() -> RuntimeResult<T> {
     Err(Error::new(ErrorKind::InvalidInput, "found directory, not a file").into())
 }
-fn err_file_in_dir_path<T>() -> super::SDSSResult<T> {
+fn err_file_in_dir_path<T>() -> RuntimeResult<T> {
     Err(Error::new(ErrorKind::InvalidInput, "found file in directory path").into())
 }
-fn err_dir_missing_in_path<T>() -> super::SDSSResult<T> {
+fn err_dir_missing_in_path<T>() -> RuntimeResult<T> {
     Err(Error::new(ErrorKind::InvalidInput, "could not find directory in path").into())
 }
-fn err_could_not_find_item<T>() -> super::SDSSResult<T> {
+fn err_could_not_find_item<T>() -> RuntimeResult<T> {
     Err(Error::new(ErrorKind::NotFound, "could not find item").into())
 }
 
@@ -117,7 +115,7 @@ pub struct VirtualFS;
 
 impl RawFSInterface for VirtualFS {
     type File = VFileDescriptor;
-    fn fs_rename_file(from: &str, to: &str) -> SDSSResult<()> {
+    fn fs_rename_file(from: &str, to: &str) -> RuntimeResult<()> {
         // get file data
         let data = with_file(from, |f| Ok(f.data.clone()))?;
         // create new file
@@ -134,7 +132,7 @@ impl RawFSInterface for VirtualFS {
         // delete old file
         Self::fs_remove_file(from)
     }
-    fn fs_remove_file(fpath: &str) -> SDSSResult<()> {
+    fn fs_remove_file(fpath: &str) -> RuntimeResult<()> {
         handle_item_mut(fpath, |e| match e.get() {
             VNode::File(_) => {
                 e.remove();
@@ -143,7 +141,7 @@ impl RawFSInterface for VirtualFS {
             _ => return err_item_is_not_file(),
         })
     }
-    fn fs_create_dir(fpath: &str) -> super::SDSSResult<()> {
+    fn fs_create_dir(fpath: &str) -> RuntimeResult<()> {
         // get vfs
         let mut vfs = VFS.write();
         // get root dir
@@ -167,12 +165,12 @@ impl RawFSInterface for VirtualFS {
             }
         }
     }
-    fn fs_create_dir_all(fpath: &str) -> super::SDSSResult<()> {
+    fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()> {
         let mut vfs = VFS.write();
         fn create_ahead(
             mut ahead: &[&str],
             current: &mut HashMap<Box<str>, VNode>,
-        ) -> SDSSResult<()> {
+        ) -> RuntimeResult<()> {
             if ahead.is_empty() {
                 return Ok(());
             }
@@ -197,13 +195,13 @@ impl RawFSInterface for VirtualFS {
         let pieces = split_parts(fpath);
         create_ahead(&pieces, &mut *vfs)
     }
-    fn fs_delete_dir(fpath: &str) -> super::SDSSResult<()> {
+    fn fs_delete_dir(fpath: &str) -> RuntimeResult<()> {
         delete_dir(fpath, false)
     }
-    fn fs_delete_dir_all(fpath: &str) -> super::SDSSResult<()> {
+    fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()> {
         delete_dir(fpath, true)
     }
-    fn fs_fopen_or_create_rw(fpath: &str) -> super::SDSSResult<super::rw::FileOpen<Self::File>> {
+    fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult<super::rw::FileOpen<Self::File>> {
         let mut vfs = VFS.write();
         // components
         let (target_file, components) = split_target_and_components(fpath);
@@ -223,7 +221,7 @@ impl RawFSInterface for VirtualFS {
             }
         }
     }
-    fn fs_fcreate_rw(fpath: &str) -> SDSSResult<Self::File> {
+    fn fs_fcreate_rw(fpath: &str) -> RuntimeResult<Self::File> {
         let mut vfs = VFS.write();
         let (target_file, components) = split_target_and_components(fpath);
         let target_dir = find_target_dir_mut(components, &mut vfs)?;
@@ -254,7 +252,7 @@ impl RawFSInterface for VirtualFS {
             }
         }
     }
-    fn fs_fopen_rw(fpath: &str) -> SDSSResult<Self::File> {
+    fn fs_fopen_rw(fpath: &str) -> RuntimeResult<Self::File> {
         with_file_mut(fpath, |f| {
             f.read = true;
             f.write = true;
@@ -266,7 +264,7 @@ impl RawFSInterface for VirtualFS {
 fn find_target_dir_mut<'a>(
     components: ComponentIter,
     mut current: &'a mut HashMap<Box<str>, VNode>,
-) -> Result<&'a mut HashMap<Box<str>, VNode>, super::SDSSError> {
+) -> RuntimeResult<&'a mut HashMap<Box<str>, VNode>> {
     for component in components {
         match current.get_mut(component) {
             Some(VNode::Dir(d)) => current = d,
@@ -280,7 +278,7 @@ fn find_target_dir_mut<'a>(
 fn find_target_dir<'a>(
     components: ComponentIter,
     mut current: &'a HashMap<Box<str>, VNode>,
-) -> Result<&'a HashMap<Box<str>, VNode>, super::SDSSError> {
+) -> RuntimeResult<&'a HashMap<Box<str>, VNode>> {
     for component in components {
         match current.get(component) {
             Some(VNode::Dir(d)) => current = d,
@@ -293,8 +291,8 @@ fn find_target_dir<'a>(
 
 fn handle_item_mut<T>(
     fpath: &str,
-    f: impl Fn(OccupiedEntry<Box<str>, VNode>) -> super::SDSSResult<T>,
-) -> super::SDSSResult<T> {
+    f: impl Fn(OccupiedEntry<Box<str>, VNode>) -> RuntimeResult<T>,
+) -> RuntimeResult<T> {
     let mut vfs = VFS.write();
     let mut current = &mut *vfs;
     // process components
@@ -313,7 +311,7 @@ fn handle_item_mut<T>(
         Entry::Vacant(_) => return err_could_not_find_item(),
     }
 }
-fn handle_item<T>(fpath: &str, f: impl Fn(&VNode) -> super::SDSSResult<T>) -> super::SDSSResult<T> {
+fn handle_item<T>(fpath: &str, f: impl Fn(&VNode) -> RuntimeResult<T>) -> RuntimeResult<T> {
     let vfs = VFS.read();
     let mut current = &*vfs;
     // process components
@@ -332,7 +330,7 @@ fn handle_item<T>(fpath: &str, f: impl Fn(&VNode) -> super::SDSSResult<T>) -> su
         None => return err_could_not_find_item(),
     }
 }
-fn delete_dir(fpath: &str, allow_if_non_empty: bool) -> Result<(), super::SDSSError> {
+fn delete_dir(fpath: &str, allow_if_non_empty: bool) -> RuntimeResult<()> {
     handle_item_mut(fpath, |node| match node.get() {
         VNode::Dir(d) => {
             if allow_if_non_empty || d.is_empty() {
@@ -387,7 +385,10 @@ impl Drop for VFileDescriptor {
     }
 }
 
-fn with_file_mut<T>(fpath: &str, mut f: impl FnMut(&mut VFile) -> SDSSResult<T>) -> SDSSResult<T> {
+fn with_file_mut<T>(
+    fpath: &str,
+    mut f: impl FnMut(&mut VFile) -> RuntimeResult<T>,
+) -> RuntimeResult<T> {
     let mut vfs = VFS.write();
     let (target_file, components) = split_target_and_components(fpath);
     let target_dir = find_target_dir_mut(components, &mut vfs)?;
@@ -398,7 +399,7 @@ fn with_file_mut<T>(fpath: &str, mut f: impl FnMut(&mut VFile) -> SDSSResult<T>)
     }
 }
 
-fn with_file<T>(fpath: &str, mut f: impl FnMut(&VFile) -> SDSSResult<T>) -> SDSSResult<T> {
+fn with_file<T>(fpath: &str, mut f: impl FnMut(&VFile) -> RuntimeResult<T>) -> RuntimeResult<T> {
     let vfs = VFS.read();
     let (target_file, components) = split_target_and_components(fpath);
     let target_dir = find_target_dir(components, &vfs)?;
@@ -412,16 +413,16 @@ fn with_file<T>(fpath: &str, mut f: impl FnMut(&VFile) -> SDSSResult<T>) -> SDSS
 impl RawFileInterface for VFileDescriptor {
     type Reader = Self;
     type Writer = Self;
-    fn into_buffered_reader(self) -> super::SDSSResult<Self::Reader> {
+    fn into_buffered_reader(self) -> RuntimeResult<Self::Reader> {
         Ok(self)
     }
-    fn into_buffered_writer(self) -> super::SDSSResult<Self::Writer> {
+    fn into_buffered_writer(self) -> RuntimeResult<Self::Writer> {
         Ok(self)
     }
 }
 
 impl RawFileInterfaceRead for VFileDescriptor {
-    fn fr_read_exact(&mut self, buf: &mut [u8]) -> super::SDSSResult<()> {
+    fn fr_read_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()> {
         with_file_mut(&self.0, |file| {
             if !file.read {
                 return Err(
@@ -440,7 +441,7 @@ impl RawFileInterfaceRead for VFileDescriptor {
 }
 
 impl RawFileInterfaceWrite for VFileDescriptor {
-    fn fw_write_all(&mut self, bytes: &[u8]) -> super::SDSSResult<()> {
+    fn fw_write_all(&mut self, bytes: &[u8]) -> RuntimeResult<()> {
         with_file_mut(&self.0, |file| {
             if !file.write {
                 return Err(
@@ -458,10 +459,10 @@ impl RawFileInterfaceWrite for VFileDescriptor {
 }
 
 impl RawFileInterfaceWriteExt for VFileDescriptor {
-    fn fw_fsync_all(&mut self) -> super::SDSSResult<()> {
+    fn fw_fsync_all(&mut self) -> RuntimeResult<()> {
         with_file(&self.0, |_| Ok(()))
     }
-    fn fw_truncate_to(&mut self, to: u64) -> super::SDSSResult<()> {
+    fn fw_truncate_to(&mut self, to: u64) -> RuntimeResult<()> {
         with_file_mut(&self.0, |file| {
             if !file.write {
                 return Err(
@@ -482,13 +483,13 @@ impl RawFileInterfaceWriteExt for VFileDescriptor {
 }
 
 impl RawFileInterfaceExt for VFileDescriptor {
-    fn fext_file_length(&self) -> super::SDSSResult<u64> {
+    fn fext_file_length(&self) -> RuntimeResult<u64> {
         with_file(&self.0, |f| Ok(f.data.len() as u64))
     }
-    fn fext_cursor(&mut self) -> super::SDSSResult<u64> {
+    fn fext_cursor(&mut self) -> RuntimeResult<u64> {
         with_file(&self.0, |f| Ok(f.pos as u64))
     }
-    fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> super::SDSSResult<()> {
+    fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> RuntimeResult<()> {
         with_file_mut(&self.0, |file| {
             if by > file.data.len() as u64 {
                 return Err(
@@ -517,72 +518,72 @@ pub struct NullFile;
 impl RawFSInterface for NullFS {
     const NOT_NULL: bool = false;
     type File = NullFile;
-    fn fs_rename_file(_: &str, _: &str) -> SDSSResult<()> {
+    fn fs_rename_file(_: &str, _: &str) -> RuntimeResult<()> {
         Ok(())
     }
-    fn fs_remove_file(_: &str) -> SDSSResult<()> {
+    fn fs_remove_file(_: &str) -> RuntimeResult<()> {
         Ok(())
     }
-    fn fs_create_dir(_: &str) -> SDSSResult<()> {
+    fn fs_create_dir(_: &str) -> RuntimeResult<()> {
         Ok(())
     }
-    fn fs_create_dir_all(_: &str) -> SDSSResult<()> {
+    fn fs_create_dir_all(_: &str) -> RuntimeResult<()> {
         Ok(())
     }
-    fn fs_delete_dir(_: &str) -> SDSSResult<()> {
+    fn fs_delete_dir(_: &str) -> RuntimeResult<()> {
         Ok(())
     }
-    fn fs_delete_dir_all(_: &str) -> SDSSResult<()> {
+    fn fs_delete_dir_all(_: &str) -> RuntimeResult<()> {
         Ok(())
     }
-    fn fs_fopen_or_create_rw(_: &str) -> SDSSResult<FileOpen<Self::File>> {
+    fn fs_fopen_or_create_rw(_: &str) -> RuntimeResult<FileOpen<Self::File>> {
         Ok(FileOpen::Created(NullFile))
     }
-    fn fs_fopen_rw(_: &str) -> SDSSResult<Self::File> {
+    fn fs_fopen_rw(_: &str) -> RuntimeResult<Self::File> {
         Ok(NullFile)
     }
-    fn fs_fcreate_rw(_: &str) -> SDSSResult<Self::File> {
+    fn fs_fcreate_rw(_: &str) -> RuntimeResult<Self::File> {
         Ok(NullFile)
     }
 }
 impl RawFileInterfaceRead for NullFile {
-    fn fr_read_exact(&mut self, _: &mut [u8]) -> SDSSResult<()> {
+    fn fr_read_exact(&mut self, _: &mut [u8]) -> RuntimeResult<()> {
         Ok(())
     }
 }
 impl RawFileInterfaceWrite for NullFile {
-    fn fw_write_all(&mut self, _: &[u8]) -> SDSSResult<()> {
+    fn fw_write_all(&mut self, _: &[u8]) -> RuntimeResult<()> {
         Ok(())
     }
 }
 impl RawFileInterfaceWriteExt for NullFile {
-    fn fw_fsync_all(&mut self) -> SDSSResult<()> {
+    fn fw_fsync_all(&mut self) -> RuntimeResult<()> {
         Ok(())
     }
-    fn fw_truncate_to(&mut self, _: u64) -> SDSSResult<()> {
+    fn fw_truncate_to(&mut self, _: u64) -> RuntimeResult<()> {
         Ok(())
     }
 }
 impl RawFileInterfaceExt for NullFile {
-    fn fext_file_length(&self) -> SDSSResult<u64> {
+    fn fext_file_length(&self) -> RuntimeResult<u64> {
         Ok(0)
     }
 
-    fn fext_cursor(&mut self) -> SDSSResult<u64> {
+    fn fext_cursor(&mut self) -> RuntimeResult<u64> {
         Ok(0)
     }
 
-    fn fext_seek_ahead_from_start_by(&mut self, _: u64) -> SDSSResult<()> {
+    fn fext_seek_ahead_from_start_by(&mut self, _: u64) -> RuntimeResult<()> {
         Ok(())
     }
 }
 impl RawFileInterface for NullFile {
     type Reader = Self;
     type Writer = Self;
-    fn into_buffered_reader(self) -> SDSSResult<Self::Reader> {
+    fn into_buffered_reader(self) -> RuntimeResult<Self::Reader> {
         Ok(self)
     }
-    fn into_buffered_writer(self) -> SDSSResult<Self::Writer> {
+    fn into_buffered_writer(self) -> RuntimeResult<Self::Writer> {
         Ok(self)
     }
 }
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index 6132a515..eca3f968 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -47,73 +47,3 @@ pub use {
 pub mod data_batch {
     pub use super::batch_jrnl::{create, reinit, DataBatchPersistDriver, DataBatchRestoreDriver};
 }
-
-use crate::{
-    engine::{
-        error::{CtxError, CtxResult},
-        txn::TransactionError,
-    },
-    util::os::SysIOError as IoError,
-};
-
-pub type SDSSResult<T> = CtxResult<T, SDSSErrorKind>;
-pub type SDSSError = CtxError<SDSSErrorKind>;
-
-#[derive(Debug)]
-#[cfg_attr(test, derive(PartialEq))]
-pub enum SDSSErrorKind {
-    // IO errors
-    /// An IO err
-    IoError(IoError),
-    OtherError(&'static str),
-    CorruptedFile(&'static str),
-    // header
-    /// version mismatch
-    HeaderDecodeVersionMismatch,
-    /// The entire header is corrupted
-    HeaderDecodeCorruptedHeader,
-    /// Expected header values were not matched with the current header
-    HeaderDecodeDataMismatch,
-    /// The time in the [header/dynrec/rtsig] is in the future
-    HeaderTimeConflict,
-    // journal
-    /// While attempting to handle a basic failure (such as adding a journal entry), the recovery engine ran into an exceptional
-    /// situation where it failed to make a necessary repair the log
-    JournalWRecoveryStageOneFailCritical,
-    /// An entry in the journal is corrupted
-    JournalLogEntryCorrupted,
-    /// The structure of the journal is corrupted
-    JournalCorrupted,
-    // internal file structures
-    /// While attempting to decode a structure in an internal segment of a file, the storage engine ran into a possibly irrecoverable error
-    InternalDecodeStructureCorrupted,
-    /// the payload (non-static) part of a structure in an internal segment of a file is corrupted
-    InternalDecodeStructureCorruptedPayload,
-    /// the data for an internal structure was decoded but is logically invalid
-    InternalDecodeStructureIllegalData,
-    /// when attempting to flush a data batch, the batch journal crashed and a recovery event was triggered. But even then,
-    /// the data batch journal could not be fixed
-    DataBatchRecoveryFailStageOne,
-    /// when attempting to restore a data batch from disk, the batch journal crashed and had a corruption, but it is irrecoverable
-    DataBatchRestoreCorruptedBatch,
-    /// when attempting to restore a data batch from disk, the driver encountered a corrupted entry
-    DataBatchRestoreCorruptedEntry,
-    /// we failed to close the data batch
-    DataBatchCloseError,
-    DataBatchRestoreCorruptedBatchFile,
-    JournalRestoreTxnError,
-    SysDBCorrupted,
-}
-
-impl From<TransactionError> for SDSSErrorKind {
-    fn from(_: TransactionError) -> Self {
-        Self::JournalRestoreTxnError
-    }
-}
-
-direct_from! {
-    SDSSErrorKind => {
-        std::io::Error as IoError,
-        IoError as IoError,
-    }
-}
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 266b5371..f6cb0a78 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -25,11 +25,11 @@
 */
 
 use {
-    super::{
-        spec::{FileSpec, Header},
-        SDSSResult,
+    super::spec::{FileSpec, Header},
+    crate::{
+        engine::{error::RuntimeResult, storage::SCrc},
+        util::os::SysIOError,
     },
-    crate::{engine::storage::SCrc, util::os::SysIOError},
     std::{
         fs::{self, File},
         io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write},
@@ -81,27 +81,27 @@ pub trait RawFSInterface {
     /// the file descriptor that is returned by the file system when a file is opened
     type File: RawFileInterface;
     /// Remove a file
-    fn fs_remove_file(fpath: &str) -> SDSSResult<()>;
+    fn fs_remove_file(fpath: &str) -> RuntimeResult<()>;
     /// Rename a file
-    fn fs_rename_file(from: &str, to: &str) -> SDSSResult<()>;
+    fn fs_rename_file(from: &str, to: &str) -> RuntimeResult<()>;
     /// Create a directory
-    fn fs_create_dir(fpath: &str) -> SDSSResult<()>;
+    fn fs_create_dir(fpath: &str) -> RuntimeResult<()>;
     /// Create a directory and all corresponding path components
-    fn fs_create_dir_all(fpath: &str) -> SDSSResult<()>;
+    fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()>;
     /// Delete a directory
-    fn fs_delete_dir(fpath: &str) -> SDSSResult<()>;
+    fn fs_delete_dir(fpath: &str) -> RuntimeResult<()>;
     /// Delete a directory and recursively remove all (if any) children
-    fn fs_delete_dir_all(fpath: &str) -> SDSSResult<()>;
+    fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()>;
     /// Open or create a file in R/W mode
     ///
     /// This will:
     /// - Create a file if it doesn't exist
     /// - Open a file it it does exist
-    fn fs_fopen_or_create_rw(fpath: &str) -> SDSSResult<FileOpen<Self::File>>;
+    fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult<FileOpen<Self::File>>;
     /// Open an existing file
-    fn fs_fopen_rw(fpath: &str) -> SDSSResult<Self::File>;
+    fn fs_fopen_rw(fpath: &str) -> RuntimeResult<Self::File>;
     /// Create a new file
-    fn fs_fcreate_rw(fpath: &str) -> SDSSResult<Self::File>;
+    fn fs_fcreate_rw(fpath: &str) -> RuntimeResult<Self::File>;
 }
 
 /// A file (well, probably) that can be used for RW operations along with advanced write and extended operations (such as seeking)
@@ -114,46 +114,46 @@ where
 {
     type Reader: RawFileInterfaceRead + RawFileInterfaceExt;
     type Writer: RawFileInterfaceWrite + RawFileInterfaceExt;
-    fn into_buffered_reader(self) -> SDSSResult<Self::Reader>;
-    fn into_buffered_writer(self) -> SDSSResult<Self::Writer>;
+    fn into_buffered_reader(self) -> RuntimeResult<Self::Reader>;
+    fn into_buffered_writer(self) -> RuntimeResult<Self::Writer>;
 }
 
 /// A file interface that supports read operations
 pub trait RawFileInterfaceRead {
-    fn fr_read_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()>;
+    fn fr_read_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()>;
 }
 
 impl<R: Read> RawFileInterfaceRead for R {
-    fn fr_read_exact(&mut self, buf: &mut [u8]) -> SDSSResult<()> {
+    fn fr_read_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()> {
         self.read_exact(buf).map_err(From::from)
     }
 }
 
 /// A file interface that supports write operations
 pub trait RawFileInterfaceWrite {
-    fn fw_write_all(&mut self, buf: &[u8]) -> SDSSResult<()>;
+    fn fw_write_all(&mut self, buf: &[u8]) -> RuntimeResult<()>;
 }
 
 impl<W: Write> RawFileInterfaceWrite for W {
-    fn fw_write_all(&mut self, buf: &[u8]) -> SDSSResult<()> {
+    fn fw_write_all(&mut self, buf: &[u8]) -> RuntimeResult<()> {
         self.write_all(buf).map_err(From::from)
     }
 }
 
 /// A file interface that supports advanced write operations
 pub trait RawFileInterfaceWriteExt {
-    fn fw_fsync_all(&mut self) -> SDSSResult<()>;
-    fn fw_truncate_to(&mut self, to: u64) -> SDSSResult<()>;
+    fn fw_fsync_all(&mut self) -> RuntimeResult<()>;
+    fn fw_truncate_to(&mut self, to: u64) -> RuntimeResult<()>;
 }
 
 /// A file interface that supports advanced file operations
 pub trait RawFileInterfaceExt {
-    fn fext_file_length(&self) -> SDSSResult<u64>;
-    fn fext_cursor(&mut self) -> SDSSResult<u64>;
-    fn fext_seek_ahead_from_start_by(&mut self, ahead_by: u64) -> SDSSResult<()>;
+    fn fext_file_length(&self) -> RuntimeResult<u64>;
+    fn fext_cursor(&mut self) -> RuntimeResult<u64>;
+    fn fext_seek_ahead_from_start_by(&mut self, ahead_by: u64) -> RuntimeResult<()>;
 }
 
-fn cvt<T>(v: std::io::Result<T>) -> SDSSResult<T> {
+fn cvt<T>(v: std::io::Result<T>) -> RuntimeResult<T> {
     let r = v?;
     Ok(r)
 }
@@ -164,25 +164,25 @@ pub struct LocalFS;
 
 impl RawFSInterface for LocalFS {
     type File = File;
-    fn fs_remove_file(fpath: &str) -> SDSSResult<()> {
+    fn fs_remove_file(fpath: &str) -> RuntimeResult<()> {
         cvt(fs::remove_file(fpath))
     }
-    fn fs_rename_file(from: &str, to: &str) -> SDSSResult<()> {
+    fn fs_rename_file(from: &str, to: &str) -> RuntimeResult<()> {
         cvt(fs::rename(from, to))
     }
-    fn fs_create_dir(fpath: &str) -> SDSSResult<()> {
+    fn fs_create_dir(fpath: &str) -> RuntimeResult<()> {
         cvt(fs::create_dir(fpath))
     }
-    fn fs_create_dir_all(fpath: &str) -> SDSSResult<()> {
+    fn fs_create_dir_all(fpath: &str) -> RuntimeResult<()> {
         cvt(fs::create_dir_all(fpath))
     }
-    fn fs_delete_dir(fpath: &str) -> SDSSResult<()> {
+    fn fs_delete_dir(fpath: &str) -> RuntimeResult<()> {
         cvt(fs::remove_dir(fpath))
     }
-    fn fs_delete_dir_all(fpath: &str) -> SDSSResult<()> {
+    fn fs_delete_dir_all(fpath: &str) -> RuntimeResult<()> {
         cvt(fs::remove_dir_all(fpath))
     }
-    fn fs_fopen_or_create_rw(fpath: &str) -> SDSSResult<FileOpen<Self::File>> {
+    fn fs_fopen_or_create_rw(fpath: &str) -> RuntimeResult<FileOpen<Self::File>> {
         let f = File::options()
             .create(true)
             .read(true)
@@ -195,7 +195,7 @@ impl RawFSInterface for LocalFS {
             Ok(FileOpen::Existing(f))
         }
     }
-    fn fs_fcreate_rw(fpath: &str) -> SDSSResult<Self::File> {
+    fn fs_fcreate_rw(fpath: &str) -> RuntimeResult<Self::File> {
         let f = File::options()
             .create_new(true)
             .read(true)
@@ -203,7 +203,7 @@ impl RawFSInterface for LocalFS {
             .open(fpath)?;
         Ok(f)
     }
-    fn fs_fopen_rw(fpath: &str) -> SDSSResult<Self::File> {
+    fn fs_fopen_rw(fpath: &str) -> RuntimeResult<Self::File> {
         let f = File::options().read(true).write(true).open(fpath)?;
         Ok(f)
     }
@@ -212,19 +212,19 @@ impl RawFSInterface for LocalFS {
 impl RawFileInterface for File {
     type Reader = BufReader<Self>;
     type Writer = BufWriter<Self>;
-    fn into_buffered_reader(self) -> SDSSResult<Self::Reader> {
+    fn into_buffered_reader(self) -> RuntimeResult<Self::Reader> {
         Ok(BufReader::new(self))
     }
-    fn into_buffered_writer(self) -> SDSSResult<Self::Writer> {
+    fn into_buffered_writer(self) -> RuntimeResult<Self::Writer> {
         Ok(BufWriter::new(self))
     }
 }
 
 impl RawFileInterfaceWriteExt for File {
-    fn fw_fsync_all(&mut self) -> SDSSResult<()> {
+    fn fw_fsync_all(&mut self) -> RuntimeResult<()> {
         cvt(self.sync_all())
     }
-    fn fw_truncate_to(&mut self, to: u64) -> SDSSResult<()> {
+    fn fw_truncate_to(&mut self, to: u64) -> RuntimeResult<()> {
         cvt(self.set_len(to))
     }
 }
@@ -262,13 +262,13 @@ impl LocalFSFile for BufWriter<File> {
 }
 
 impl<F: LocalFSFile> RawFileInterfaceExt for F {
-    fn fext_file_length(&self) -> SDSSResult<u64> {
+    fn fext_file_length(&self) -> RuntimeResult<u64> {
         Ok(self.file().metadata()?.len())
     }
-    fn fext_cursor(&mut self) -> SDSSResult<u64> {
+    fn fext_cursor(&mut self) -> RuntimeResult<u64> {
         cvt(self.file_mut().stream_position())
     }
-    fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> SDSSResult<()> {
+    fn fext_seek_ahead_from_start_by(&mut self, by: u64) -> RuntimeResult<()> {
         cvt(self.file_mut().seek(SeekFrom::Start(by)).map(|_| ()))
     }
 }
@@ -282,7 +282,7 @@ impl<Fs: RawFSInterface> SDSSFileTrackedWriter<Fs> {
     pub fn new(f: SDSSFileIO<Fs>) -> Self {
         Self { f, cs: SCrc::new() }
     }
-    pub fn unfsynced_write(&mut self, block: &[u8]) -> SDSSResult<()> {
+    pub fn unfsynced_write(&mut self, block: &[u8]) -> RuntimeResult<()> {
         match self.f.unfsynced_write(block) {
             Ok(()) => {
                 self.cs.recompute_with_new_var_block(block);
@@ -291,7 +291,7 @@ impl<Fs: RawFSInterface> SDSSFileTrackedWriter<Fs> {
             e => e,
         }
     }
-    pub fn fsync_all(&mut self) -> SDSSResult<()> {
+    pub fn fsync_all(&mut self) -> RuntimeResult<()> {
         self.f.fsync_all()
     }
     pub fn reset_and_finish_checksum(&mut self) -> u64 {
@@ -315,7 +315,7 @@ pub struct SDSSFileTrackedReader<Fs: RawFSInterface> {
 
 impl<Fs: RawFSInterface> SDSSFileTrackedReader<Fs> {
     /// Important: this will only look at the data post the current cursor!
-    pub fn new(mut f: SDSSFileIO<Fs>) -> SDSSResult<Self> {
+    pub fn new(mut f: SDSSFileIO<Fs>) -> RuntimeResult<Self> {
         let len = f.file_length()?;
         let pos = f.retrieve_cursor()?;
         Ok(Self {
@@ -334,7 +334,7 @@ impl<Fs: RawFSInterface> SDSSFileTrackedReader<Fs> {
     pub fn has_left(&self, v: u64) -> bool {
         self.remaining() >= v
     }
-    pub fn read_into_buffer(&mut self, buf: &mut [u8]) -> SDSSResult<()> {
+    pub fn read_into_buffer(&mut self, buf: &mut [u8]) -> RuntimeResult<()> {
         if self.remaining() >= buf.len() as u64 {
             match self.f.read_to_buffer(buf) {
                 Ok(()) => {
@@ -348,7 +348,7 @@ impl<Fs: RawFSInterface> SDSSFileTrackedReader<Fs> {
             Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into())
         }
     }
-    pub fn read_byte(&mut self) -> SDSSResult<u8> {
+    pub fn read_byte(&mut self) -> RuntimeResult<u8> {
         let mut buf = [0u8; 1];
         self.read_into_buffer(&mut buf).map(|_| buf[0])
     }
@@ -366,7 +366,7 @@ impl<Fs: RawFSInterface> SDSSFileTrackedReader<Fs> {
     pub fn __cursor_ahead_by(&mut self, sizeof: usize) {
         self.pos += sizeof as u64;
     }
-    pub fn read_block<const N: usize>(&mut self) -> SDSSResult<[u8; N]> {
+    pub fn read_block<const N: usize>(&mut self) -> RuntimeResult<[u8; N]> {
         if !self.has_left(N as _) {
             return Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into());
         }
@@ -374,7 +374,7 @@ impl<Fs: RawFSInterface> SDSSFileTrackedReader<Fs> {
         self.read_into_buffer(&mut buf)?;
         Ok(buf)
     }
-    pub fn read_u64_le(&mut self) -> SDSSResult<u64> {
+    pub fn read_u64_le(&mut self) -> RuntimeResult<u64> {
         Ok(u64::from_le_bytes(self.read_block()?))
     }
 }
@@ -386,19 +386,19 @@ pub struct SDSSFileIO<Fs: RawFSInterface> {
 }
 
 impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
-    pub fn open<F: FileSpec>(fpath: &str) -> SDSSResult<(Self, F::Header)> {
+    pub fn open<F: FileSpec>(fpath: &str) -> RuntimeResult<(Self, F::Header)> {
         let mut f = Self::_new(Fs::fs_fopen_rw(fpath)?);
         let header = F::Header::decode_verify(&mut f, F::DECODE_DATA, F::VERIFY_DATA)?;
         Ok((f, header))
     }
-    pub fn create<F: FileSpec>(fpath: &str) -> SDSSResult<Self> {
+    pub fn create<F: FileSpec>(fpath: &str) -> RuntimeResult<Self> {
         let mut f = Self::_new(Fs::fs_fcreate_rw(fpath)?);
         F::Header::encode(&mut f, F::ENCODE_DATA)?;
         Ok(f)
     }
     pub fn open_or_create_perm_rw<F: FileSpec>(
         fpath: &str,
-    ) -> SDSSResult<FileOpen<Self, (Self, F::Header)>> {
+    ) -> RuntimeResult<FileOpen<Self, (Self, F::Header)>> {
         match Fs::fs_fopen_or_create_rw(fpath)? {
             FileOpen::Created(c) => {
                 let mut f = Self::_new(c);
@@ -421,37 +421,37 @@ impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
             _fs: PhantomData,
         }
     }
-    pub fn unfsynced_write(&mut self, data: &[u8]) -> SDSSResult<()> {
+    pub fn unfsynced_write(&mut self, data: &[u8]) -> RuntimeResult<()> {
         self.f.fw_write_all(data)
     }
-    pub fn fsync_all(&mut self) -> SDSSResult<()> {
+    pub fn fsync_all(&mut self) -> RuntimeResult<()> {
         self.f.fw_fsync_all()?;
         Ok(())
     }
-    pub fn fsynced_write(&mut self, data: &[u8]) -> SDSSResult<()> {
+    pub fn fsynced_write(&mut self, data: &[u8]) -> RuntimeResult<()> {
         self.f.fw_write_all(data)?;
         self.f.fw_fsync_all()
     }
-    pub fn read_to_buffer(&mut self, buffer: &mut [u8]) -> SDSSResult<()> {
+    pub fn read_to_buffer(&mut self, buffer: &mut [u8]) -> RuntimeResult<()> {
         self.f.fr_read_exact(buffer)
     }
-    pub fn file_length(&self) -> SDSSResult<u64> {
+    pub fn file_length(&self) -> RuntimeResult<u64> {
         self.f.fext_file_length()
     }
-    pub fn seek_from_start(&mut self, by: u64) -> SDSSResult<()> {
+    pub fn seek_from_start(&mut self, by: u64) -> RuntimeResult<()> {
         self.f.fext_seek_ahead_from_start_by(by)
     }
-    pub fn trim_file_to(&mut self, to: u64) -> SDSSResult<()> {
+    pub fn trim_file_to(&mut self, to: u64) -> RuntimeResult<()> {
         self.f.fw_truncate_to(to)
     }
-    pub fn retrieve_cursor(&mut self) -> SDSSResult<u64> {
+    pub fn retrieve_cursor(&mut self) -> RuntimeResult<u64> {
         self.f.fext_cursor()
     }
-    pub fn read_byte(&mut self) -> SDSSResult<u8> {
+    pub fn read_byte(&mut self) -> RuntimeResult<u8> {
         let mut r = [0; 1];
         self.read_to_buffer(&mut r).map(|_| r[0])
     }
-    pub fn load_remaining_into_buffer(&mut self) -> SDSSResult<Vec<u8>> {
+    pub fn load_remaining_into_buffer(&mut self) -> RuntimeResult<Vec<u8>> {
         let len = self.file_length()? - self.retrieve_cursor()?;
         let mut buf = vec![0; len as usize];
         self.read_to_buffer(&mut buf)?;
diff --git a/server/src/engine/storage/v1/spec.rs b/server/src/engine/storage/v1/spec.rs
index c470badc..fe230bdc 100644
--- a/server/src/engine/storage/v1/spec.rs
+++ b/server/src/engine/storage/v1/spec.rs
@@ -33,15 +33,14 @@
 */
 
 use {
-    super::{
-        rw::{RawFSInterface, SDSSFileIO},
-        SDSSResult,
-    },
+    super::rw::{RawFSInterface, SDSSFileIO},
     crate::{
-        engine::storage::{
-            header::{HostArch, HostEndian, HostOS, HostPointerWidth},
-            v1::SDSSErrorKind,
-            versions::{self, DriverVersion, HeaderVersion, ServerVersion},
+        engine::{
+            error::{RuntimeResult, StorageError},
+            storage::{
+                header::{HostArch, HostEndian, HostOS, HostPointerWidth},
+                versions::{self, DriverVersion, HeaderVersion, ServerVersion},
+            },
         },
         util::os,
     },
@@ -202,21 +201,23 @@ pub trait Header: Sized {
     /// Decode verify arguments
     type DecodeVerifyArgs;
     /// Encode the header
-    fn encode<Fs: RawFSInterface>(f: &mut SDSSFileIO<Fs>, args: Self::EncodeArgs)
-        -> SDSSResult<()>;
+    fn encode<Fs: RawFSInterface>(
+        f: &mut SDSSFileIO<Fs>,
+        args: Self::EncodeArgs,
+    ) -> RuntimeResult<()>;
     /// Decode the header
     fn decode<Fs: RawFSInterface>(
         f: &mut SDSSFileIO<Fs>,
         args: Self::DecodeArgs,
-    ) -> SDSSResult<Self>;
+    ) -> RuntimeResult<Self>;
     /// Verify the header
-    fn verify(&self, args: Self::DecodeVerifyArgs) -> SDSSResult<()>;
+    fn verify(&self, args: Self::DecodeVerifyArgs) -> RuntimeResult<()>;
     /// Decode and verify the header
     fn decode_verify<Fs: RawFSInterface>(
         f: &mut SDSSFileIO<Fs>,
         d_args: Self::DecodeArgs,
         v_args: Self::DecodeVerifyArgs,
-    ) -> SDSSResult<Self> {
+    ) -> RuntimeResult<Self> {
         let h = Self::decode(f, d_args)?;
         h.verify(v_args)?;
         Ok(h)
@@ -295,7 +296,7 @@ impl SDSSStaticHeaderV1Compact {
     /// - If padding block was not zeroed, handle
     /// - No file metadata and is verified. Check!
     ///
-    fn _decode(block: [u8; 64]) -> SDSSResult<Self> {
+    fn _decode(block: [u8; 64]) -> RuntimeResult<Self> {
         var!(let raw_magic, raw_header_version, raw_server_version, raw_driver_version, raw_host_os, raw_host_arch,
             raw_host_ptr_width, raw_host_endian, raw_file_class, raw_file_specifier, raw_file_specifier_version,
             raw_runtime_epoch_time, raw_paddding_block,
@@ -375,8 +376,8 @@ impl SDSSStaticHeaderV1Compact {
         } else {
             let version_okay = okay_header_version & okay_server_version & okay_driver_version;
             let md = ManuallyDrop::new([
-                SDSSErrorKind::HeaderDecodeCorruptedHeader,
-                SDSSErrorKind::HeaderDecodeVersionMismatch,
+                StorageError::HeaderDecodeCorruptedHeader,
+                StorageError::HeaderDecodeVersionMismatch,
             ]);
             Err(unsafe {
                 // UNSAFE(@ohsayan): while not needed, md for drop safety + correct index
@@ -495,23 +496,26 @@ impl Header for SDSSStaticHeaderV1Compact {
     fn encode<Fs: RawFSInterface>(
         f: &mut SDSSFileIO<Fs>,
         (scope, spec, spec_v): Self::EncodeArgs,
-    ) -> SDSSResult<()> {
+    ) -> RuntimeResult<()> {
         let b = Self::_encode_auto(scope, spec, spec_v);
         f.fsynced_write(&b)
     }
-    fn decode<Fs: RawFSInterface>(f: &mut SDSSFileIO<Fs>, _: Self::DecodeArgs) -> SDSSResult<Self> {
+    fn decode<Fs: RawFSInterface>(
+        f: &mut SDSSFileIO<Fs>,
+        _: Self::DecodeArgs,
+    ) -> RuntimeResult<Self> {
         let mut buf = [0u8; 64];
         f.read_to_buffer(&mut buf)?;
         Self::_decode(buf)
     }
-    fn verify(&self, (scope, spec, spec_v): Self::DecodeVerifyArgs) -> SDSSResult<()> {
+    fn verify(&self, (scope, spec, spec_v): Self::DecodeVerifyArgs) -> RuntimeResult<()> {
         if (self.file_class() == scope)
             & (self.file_specifier() == spec)
             & (self.file_specifier_version() == spec_v)
         {
             Ok(())
         } else {
-            Err(SDSSErrorKind::HeaderDecodeDataMismatch.into())
+            Err(StorageError::HeaderDecodeDataMismatch.into())
         }
     }
 }
diff --git a/server/src/engine/storage/v1/sysdb.rs b/server/src/engine/storage/v1/sysdb.rs
index b76e0218..f7d48181 100644
--- a/server/src/engine/storage/v1/sysdb.rs
+++ b/server/src/engine/storage/v1/sysdb.rs
@@ -25,12 +25,13 @@
 */
 
 use {
-    super::{rw::FileOpen, SDSSErrorKind},
+    super::rw::FileOpen,
     crate::engine::{
         config::ConfigAuth,
         data::{cell::Datacell, DictEntryGeneric, DictGeneric},
+        error::{RuntimeResult, StorageError},
         fractal::config::{SysAuth, SysAuthUser, SysConfig, SysHostData},
-        storage::v1::{inf, spec, RawFSInterface, SDSSFileIO, SDSSResult},
+        storage::v1::{inf, spec, RawFSInterface, SDSSFileIO},
     },
     parking_lot::RwLock,
     std::collections::HashMap,
@@ -73,7 +74,9 @@ impl SystemStoreInit {
 ///
 /// - If it doesn't exist, create it
 /// - If it exists, look for config changes and sync them
-pub fn open_system_database<Fs: RawFSInterface>(auth: ConfigAuth) -> SDSSResult<SystemStoreInit> {
+pub fn open_system_database<Fs: RawFSInterface>(
+    auth: ConfigAuth,
+) -> RuntimeResult<SystemStoreInit> {
     open_or_reinit_system_database::<Fs>(auth, SYSDB_PATH, SYSDB_COW_PATH)
 }
 
@@ -82,7 +85,7 @@ pub fn open_or_reinit_system_database<Fs: RawFSInterface>(
     auth: ConfigAuth,
     sysdb_path: &str,
     sysdb_path_cow: &str,
-) -> SDSSResult<SystemStoreInit> {
+) -> RuntimeResult<SystemStoreInit> {
     let sysdb_file = match SDSSFileIO::<Fs>::open_or_create_perm_rw::<spec::SysDBV1>(sysdb_path)? {
         FileOpen::Created(new) => {
             // init new syscfg
@@ -130,7 +133,7 @@ pub fn open_or_reinit_system_database<Fs: RawFSInterface>(
 pub fn sync_system_database_to<Fs: RawFSInterface>(
     cfg: &SysConfig,
     mut f: SDSSFileIO<Fs>,
-) -> SDSSResult<()> {
+) -> RuntimeResult<()> {
     // prepare our flat file
     let mut map: DictGeneric = into_dict!(
         SYS_KEY_SYS => DictEntryGeneric::Map(into_dict!(
@@ -172,15 +175,17 @@ fn rkey<T>(
     d: &mut DictGeneric,
     key: &str,
     transform: impl Fn(DictEntryGeneric) -> Option<T>,
-) -> SDSSResult<T> {
+) -> RuntimeResult<T> {
     match d.remove(key).map(transform) {
         Some(Some(k)) => Ok(k),
-        _ => Err(SDSSErrorKind::SysDBCorrupted.into()),
+        _ => Err(StorageError::SysDBCorrupted.into()),
     }
 }
 
 /// Decode the system database
-pub fn decode_system_database<Fs: RawFSInterface>(mut f: SDSSFileIO<Fs>) -> SDSSResult<SysConfig> {
+pub fn decode_system_database<Fs: RawFSInterface>(
+    mut f: SDSSFileIO<Fs>,
+) -> RuntimeResult<SysConfig> {
     let mut sysdb_data =
         inf::dec::dec_dict_full::<inf::map::GenericDictSpec>(&f.load_remaining_into_buffer()?)?;
     // get our auth and sys stores
@@ -201,14 +206,14 @@ pub fn decode_system_database<Fs: RawFSInterface>(mut f: SDSSFileIO<Fs>) -> SDSS
         let mut userdata = userdata
             .into_data()
             .and_then(Datacell::into_list)
-            .ok_or(SDSSErrorKind::SysDBCorrupted)?;
+            .ok_or(StorageError::SysDBCorrupted)?;
         if userdata.len() != 1 {
-            return Err(SDSSErrorKind::SysDBCorrupted.into());
+            return Err(StorageError::SysDBCorrupted.into());
         }
         let user_password = userdata
             .remove(0)
             .into_bin()
-            .ok_or(SDSSErrorKind::SysDBCorrupted)?;
+            .ok_or(StorageError::SysDBCorrupted)?;
         loaded_users.insert(username, SysAuthUser::new(user_password.into_boxed_slice()));
     }
     let sys_auth = SysAuth::new(root_key.into_boxed_slice(), loaded_users);
@@ -220,7 +225,7 @@ pub fn decode_system_database<Fs: RawFSInterface>(mut f: SDSSFileIO<Fs>) -> SDSS
         d.into_data()?.into_uint()
     })?;
     if !(sysdb_data.is_empty() & auth_store.is_empty() & sys_store.is_empty()) {
-        return Err(SDSSErrorKind::SysDBCorrupted.into());
+        return Err(StorageError::SysDBCorrupted.into());
     }
     Ok(SysConfig::new(
         RwLock::new(sys_auth),
diff --git a/server/src/engine/storage/v1/tests/tx.rs b/server/src/engine/storage/v1/tests/tx.rs
index 5e39b737..d1df0351 100644
--- a/server/src/engine/storage/v1/tests/tx.rs
+++ b/server/src/engine/storage/v1/tests/tx.rs
@@ -26,14 +26,18 @@
 
 use {
     crate::{
-        engine::storage::v1::{
-            journal::{self, JournalAdapter, JournalWriter},
-            spec, SDSSError, SDSSErrorKind, SDSSResult,
+        engine::{
+            error::{RuntimeResult, StorageError},
+            storage::v1::{
+                journal::{self, JournalAdapter, JournalWriter},
+                spec,
+            },
         },
         util,
     },
     std::cell::RefCell,
 };
+
 pub struct Database {
     data: RefCell<[u8; 10]>,
 }
@@ -52,7 +56,7 @@ impl Database {
     fn txn_reset(
         &self,
         txn_writer: &mut JournalWriter<super::VirtualFS, DatabaseTxnAdapter>,
-    ) -> SDSSResult<()> {
+    ) -> RuntimeResult<()> {
         self.reset();
         txn_writer.append_event(TxEvent::Reset)
     }
@@ -64,7 +68,7 @@ impl Database {
         pos: usize,
         val: u8,
         txn_writer: &mut JournalWriter<super::VirtualFS, DatabaseTxnAdapter>,
-    ) -> SDSSResult<()> {
+    ) -> RuntimeResult<()> {
         self.set(pos, val);
         txn_writer.append_event(TxEvent::Set(pos, val))
     }
@@ -75,11 +79,11 @@ pub enum TxEvent {
 }
 #[derive(Debug)]
 pub enum TxError {
-    SDSS(SDSSError),
+    SDSS(StorageError),
 }
 direct_from! {
     TxError => {
-        SDSSError as SDSS
+        StorageError as SDSS
     }
 }
 #[derive(Debug)]
@@ -114,22 +118,14 @@ impl JournalAdapter for DatabaseTxnAdapter {
     }
 
     fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> Result<(), TxError> {
-        if payload.len() != 10 {
-            return Err(TxError::SDSS(
-                SDSSErrorKind::CorruptedFile("testtxn.log").into(),
-            ));
-        }
+        assert!(payload.len() >= 10, "corrupt file");
         let opcode = payload[0];
         let index = u64::from_le_bytes(util::copy_slice_to_array(&payload[1..9]));
         let new_value = payload[9];
         match opcode {
             0 if index == 0 && new_value == 0 => gs.reset(),
             1 if index < 10 && index < isize::MAX as u64 => gs.set(index as usize, new_value),
-            _ => {
-                return Err(TxError::SDSS(
-                    SDSSErrorKind::JournalLogEntryCorrupted.into(),
-                ))
-            }
+            _ => return Err(TxError::SDSS(StorageError::JournalLogEntryCorrupted.into())),
         }
         Ok(())
     }
@@ -138,7 +134,7 @@ impl JournalAdapter for DatabaseTxnAdapter {
 fn open_log(
     log_name: &str,
     db: &Database,
-) -> SDSSResult<JournalWriter<super::VirtualFS, DatabaseTxnAdapter>> {
+) -> RuntimeResult<JournalWriter<super::VirtualFS, DatabaseTxnAdapter>> {
     journal::open_journal::<DatabaseTxnAdapter, super::VirtualFS, spec::TestFile>(log_name, db)
         .map(|v| v.into_inner())
 }
@@ -147,7 +143,7 @@ fn open_log(
 fn first_boot_second_readonly() {
     // create log
     let db1 = Database::new();
-    let x = || -> SDSSResult<()> {
+    let x = || -> RuntimeResult<()> {
         let mut log = open_log("testtxn.log", &db1)?;
         db1.txn_set(0, 20, &mut log)?;
         db1.txn_set(9, 21, &mut log)?;
@@ -168,7 +164,7 @@ fn first_boot_second_readonly() {
 fn oneboot_mod_twoboot_mod_thirdboot_read() {
     // first boot: set all to 1
     let db1 = Database::new();
-    let x = || -> SDSSResult<()> {
+    let x = || -> RuntimeResult<()> {
         let mut log = open_log("duatxn.db-tlog", &db1)?;
         for i in 0..10 {
             db1.txn_set(i, 1, &mut log)?;
@@ -180,7 +176,7 @@ fn oneboot_mod_twoboot_mod_thirdboot_read() {
     drop(db1);
     // second boot
     let db2 = Database::new();
-    let x = || -> SDSSResult<()> {
+    let x = || -> RuntimeResult<()> {
         let mut log = open_log("duatxn.db-tlog", &db2)?;
         assert_eq!(bkp_db1, db2.copy_data());
         for i in 0..10 {
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index 07479d46..cb0f85d6 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -27,15 +27,15 @@
 #[cfg(test)]
 use crate::engine::storage::v1::memfs::VirtualFS;
 use {
-    super::{TransactionError, TransactionResult},
     crate::{
         engine::{
             core::{space::Space, GlobalNS},
             data::uuid::Uuid,
+            error::{RuntimeResult, TransactionError},
             mem::BufferedScanner,
             storage::v1::{
                 inf::{self, PersistObject},
-                JournalAdapter, JournalWriter, LocalFS, RawFSInterface, SDSSResult,
+                JournalAdapter, JournalWriter, LocalFS, RawFSInterface,
             },
         },
         util::EndianQW,
@@ -77,7 +77,7 @@ impl<Fs: RawFSInterface> GNSTransactionDriverAnyFS<Fs> {
     }
     /// Attempts to commit the given event into the journal, handling any possible recovery triggers and returning
     /// errors (if any)
-    pub fn try_commit<GE: GNSEvent>(&mut self, gns_event: GE) -> TransactionResult<()> {
+    pub fn try_commit<GE: GNSEvent>(&mut self, gns_event: GE) -> RuntimeResult<()> {
         let mut buf = vec![];
         buf.extend(GE::OPC.to_le_bytes());
         GE::encode_super_event(gns_event, &mut buf);
@@ -99,20 +99,20 @@ impl JournalAdapter for GNSAdapter {
     const RECOVERY_PLUGIN: bool = true;
     type JournalEvent = GNSSuperEvent;
     type GlobalState = GlobalNS;
-    type Error = TransactionError;
+    type Error = crate::engine::fractal::error::Error;
     fn encode(GNSSuperEvent(b): Self::JournalEvent) -> Box<[u8]> {
         b
     }
-    fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> TransactionResult<()> {
+    fn decode_and_update_state(payload: &[u8], gs: &Self::GlobalState) -> RuntimeResult<()> {
         if payload.len() < 2 {
-            return Err(TransactionError::DecodedUnexpectedEof);
+            return Err(TransactionError::DecodedUnexpectedEof.into());
         }
         macro_rules! dispatch {
             ($($item:ty),* $(,)?) => {
-                [$(<$item as GNSEvent>::decode_and_update_global_state),*, |_, _| Err(TransactionError::DecodeUnknownTxnOp)]
+                [$(<$item as GNSEvent>::decode_and_update_global_state),*, |_, _| Err(TransactionError::DecodeUnknownTxnOp.into())]
             };
         }
-        static DISPATCH: [fn(&mut BufferedScanner, &GlobalNS) -> TransactionResult<()>; 9] = dispatch!(
+        static DISPATCH: [fn(&mut BufferedScanner, &GlobalNS) -> RuntimeResult<()>; 9] = dispatch!(
             CreateSpaceTxn,
             AlterSpaceTxn,
             DropSpaceTxn,
@@ -129,7 +129,7 @@ impl JournalAdapter for GNSAdapter {
         };
         match DISPATCH[(opc as usize).min(DISPATCH.len())](&mut scanner, gs) {
             Ok(()) if scanner.eof() => return Ok(()),
-            Ok(_) => Err(TransactionError::DecodeCorruptedPayloadMoreBytes),
+            Ok(_) => Err(TransactionError::DecodeCorruptedPayloadMoreBytes.into()),
             Err(e) => Err(e),
         }
     }
@@ -165,15 +165,15 @@ where
     fn decode_and_update_global_state(
         scanner: &mut BufferedScanner,
         gns: &GlobalNS,
-    ) -> TransactionResult<()> {
+    ) -> RuntimeResult<()> {
         Self::update_global_state(Self::decode(scanner)?, gns)
     }
     /// Attempts to decode the event using the given scanner
-    fn decode(scanner: &mut BufferedScanner) -> TransactionResult<Self::RestoreType> {
+    fn decode(scanner: &mut BufferedScanner) -> RuntimeResult<Self::RestoreType> {
         inf::dec::dec_full_from_scanner::<Self>(scanner).map_err(|e| e.into())
     }
     /// Update the global state from the restored event
-    fn update_global_state(restore: Self::RestoreType, gns: &GlobalNS) -> TransactionResult<()>;
+    fn update_global_state(restore: Self::RestoreType, gns: &GlobalNS) -> RuntimeResult<()>;
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -220,7 +220,7 @@ impl<'a> PersistObject for SpaceID<'a> {
         buf.extend(data.uuid.to_le_bytes());
         buf.extend(data.name.len().u64_bytes_le());
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         Ok(SpaceIDMD {
             uuid: Uuid::from_bytes(scanner.next_chunk()),
             space_name_l: scanner.next_u64_le(),
@@ -229,7 +229,10 @@ impl<'a> PersistObject for SpaceID<'a> {
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
         buf.extend(data.name.as_bytes());
     }
-    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(
+        s: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> RuntimeResult<Self::OutputType> {
         let str = inf::dec::utils::decode_string(s, md.space_name_l as usize)?;
         Ok(SpaceIDRes {
             uuid: md.uuid,
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index f8097f15..4db39bdf 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    super::{GNSEvent, TransactionResult},
+    super::GNSEvent,
     crate::{
         engine::{
             core::{
@@ -34,14 +34,12 @@ use {
                 GlobalNS,
             },
             data::uuid::Uuid,
+            error::TransactionError,
+            error::{RuntimeResult, StorageError},
             idx::{IndexST, IndexSTSeqCns, STIndex, STIndexSeq},
             mem::BufferedScanner,
             ql::lex::Ident,
-            storage::v1::{
-                inf::{self, map, obj, PersistObject},
-                SDSSErrorKind, SDSSResult,
-            },
-            txn::TransactionError,
+            storage::v1::inf::{self, map, obj, PersistObject},
         },
         util::EndianQW,
     },
@@ -130,7 +128,7 @@ impl<'a> PersistObject for ModelID<'a> {
         buf.extend(data.model_version.to_le_bytes());
         buf.extend(data.model_uuid.to_le_bytes());
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         Ok(ModelIDMD {
             space_id: <super::SpaceID as PersistObject>::meta_dec(scanner)?,
             model_name_l: scanner.next_u64_le(),
@@ -142,7 +140,10 @@ impl<'a> PersistObject for ModelID<'a> {
         <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id);
         buf.extend(data.model_name.as_bytes());
     }
-    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(
+        s: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> RuntimeResult<Self::OutputType> {
         Ok(ModelIDRes {
             space_id: <super::SpaceID as PersistObject>::obj_dec(s, md.space_id)?,
             model_name: inf::dec::utils::decode_string(s, md.model_name_l as usize)?
@@ -156,14 +157,14 @@ impl<'a> PersistObject for ModelID<'a> {
 fn with_space<T>(
     gns: &GlobalNS,
     space_id: &super::SpaceIDRes,
-    mut f: impl FnMut(&Space) -> TransactionResult<T>,
-) -> TransactionResult<T> {
+    mut f: impl FnMut(&Space) -> RuntimeResult<T>,
+) -> RuntimeResult<T> {
     let spaces = gns.spaces().read();
     let Some(space) = spaces.st_get(&space_id.name) else {
-        return Err(TransactionError::OnRestoreDataMissing);
+        return Err(TransactionError::OnRestoreDataMissing.into());
     };
     if space.get_uuid() != space_id.uuid {
-        return Err(TransactionError::OnRestoreDataConflictMismatch);
+        return Err(TransactionError::OnRestoreDataConflictMismatch.into());
     }
     f(space)
 }
@@ -172,16 +173,16 @@ fn with_model<T>(
     gns: &GlobalNS,
     space_id: &super::SpaceIDRes,
     model_id: &ModelIDRes,
-    mut f: impl FnMut(&Model) -> TransactionResult<T>,
-) -> TransactionResult<T> {
+    mut f: impl FnMut(&Model) -> RuntimeResult<T>,
+) -> RuntimeResult<T> {
     with_space(gns, space_id, |space| {
         let models = space.models().read();
         let Some(model) = models.st_get(&model_id.model_name) else {
-            return Err(TransactionError::OnRestoreDataMissing);
+            return Err(TransactionError::OnRestoreDataMissing.into());
         };
         if model.get_uuid() != model_id.model_uuid {
             // this should have been handled by an earlier transaction
-            return Err(TransactionError::OnRestoreDataConflictMismatch);
+            return Err(TransactionError::OnRestoreDataConflictMismatch.into());
         }
         f(model)
     })
@@ -251,7 +252,7 @@ impl<'a> PersistObject for CreateModelTxn<'a> {
             obj::ModelLayoutRef::from((data.model, data.model_read)),
         )
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         let space_id = <super::SpaceID as PersistObject>::meta_dec(scanner)?;
         let model_name_l = scanner.next_u64_le();
         let model_meta = <obj::ModelLayoutRef as PersistObject>::meta_dec(scanner)?;
@@ -272,7 +273,10 @@ impl<'a> PersistObject for CreateModelTxn<'a> {
             obj::ModelLayoutRef::from((data.model, data.model_read)),
         )
     }
-    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(
+        s: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> RuntimeResult<Self::OutputType> {
         let space_id = <super::SpaceID as PersistObject>::obj_dec(s, md.space_id_meta)?;
         let model_name =
             inf::dec::utils::decode_string(s, md.model_name_l as usize)?.into_boxed_str();
@@ -296,7 +300,7 @@ impl<'a> GNSEvent for CreateModelTxn<'a> {
             model,
         }: Self::RestoreType,
         gns: &GlobalNS,
-    ) -> crate::engine::txn::TransactionResult<()> {
+    ) -> RuntimeResult<()> {
         let rgns = gns.spaces().read();
         /*
             NOTE(@ohsayan):
@@ -311,11 +315,11 @@ impl<'a> GNSEvent for CreateModelTxn<'a> {
                 if space._create_model(&model_name, model).is_ok() {
                     Ok(())
                 } else {
-                    Err(TransactionError::OnRestoreDataConflictAlreadyExists)
+                    Err(TransactionError::OnRestoreDataConflictAlreadyExists.into())
                 }
             }
-            Some(_) => return Err(TransactionError::OnRestoreDataConflictMismatch),
-            None => return Err(TransactionError::OnRestoreDataMissing),
+            Some(_) => return Err(TransactionError::OnRestoreDataConflictMismatch.into()),
+            None => return Err(TransactionError::OnRestoreDataMissing.into()),
         }
     }
 }
@@ -366,7 +370,7 @@ impl<'a> PersistObject for AlterModelAddTxn<'a> {
         <ModelID as PersistObject>::meta_enc(buf, data.model_id);
         buf.extend(data.new_fields.st_len().u64_bytes_le());
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         let model_id_meta = <ModelID as PersistObject>::meta_dec(scanner)?;
         let new_field_c = scanner.next_u64_le();
         Ok(AlterModelAddTxnMD {
@@ -378,7 +382,10 @@ impl<'a> PersistObject for AlterModelAddTxn<'a> {
         <ModelID as PersistObject>::obj_enc(buf, data.model_id);
         <map::PersistMapImpl<map::FieldMapSpec> as PersistObject>::obj_enc(buf, data.new_fields);
     }
-    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(
+        s: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> RuntimeResult<Self::OutputType> {
         let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_meta)?;
         let new_fields = <map::PersistMapImpl<map::FieldMapSpec> as PersistObject>::obj_dec(
             s,
@@ -401,7 +408,7 @@ impl<'a> GNSEvent for AlterModelAddTxn<'a> {
             new_fields,
         }: Self::RestoreType,
         gns: &GlobalNS,
-    ) -> crate::engine::txn::TransactionResult<()> {
+    ) -> RuntimeResult<()> {
         with_model(gns, &model_id.space_id, &model_id, |model| {
             let mut wmodel = model.intent_write_model();
             for (i, (field_name, field)) in new_fields.stseq_ord_kv().enumerate() {
@@ -413,7 +420,7 @@ impl<'a> GNSEvent for AlterModelAddTxn<'a> {
                     new_fields.stseq_ord_key().take(i).for_each(|field_id| {
                         let _ = wmodel.fields_mut().st_delete(field_id);
                     });
-                    return Err(TransactionError::OnRestoreDataConflictMismatch);
+                    return Err(TransactionError::OnRestoreDataConflictMismatch.into());
                 }
             }
             // TODO(@ohsayan): avoid double iteration
@@ -470,7 +477,7 @@ impl<'a> PersistObject for AlterModelRemoveTxn<'a> {
         <ModelID as PersistObject>::meta_enc(buf, data.model_id);
         buf.extend(data.removed_fields.len().u64_bytes_le());
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         let model_id_meta = <ModelID as PersistObject>::meta_dec(scanner)?;
         Ok(AlterModelRemoveTxnMD {
             model_id_meta,
@@ -484,7 +491,10 @@ impl<'a> PersistObject for AlterModelRemoveTxn<'a> {
             buf.extend(field.as_bytes());
         }
     }
-    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(
+        s: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> RuntimeResult<Self::OutputType> {
         let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_meta)?;
         let mut removed_fields = Vec::with_capacity(md.remove_field_c as usize);
         while !s.eof()
@@ -498,7 +508,7 @@ impl<'a> PersistObject for AlterModelRemoveTxn<'a> {
             removed_fields.push(inf::dec::utils::decode_string(s, len)?.into_boxed_str());
         }
         if removed_fields.len() as u64 != md.remove_field_c {
-            return Err(SDSSErrorKind::InternalDecodeStructureCorruptedPayload.into());
+            return Err(StorageError::InternalDecodeStructureCorruptedPayload.into());
         }
         Ok(AlterModelRemoveTxnRestorePL {
             model_id,
@@ -517,7 +527,7 @@ impl<'a> GNSEvent for AlterModelRemoveTxn<'a> {
             removed_fields,
         }: Self::RestoreType,
         gns: &GlobalNS,
-    ) -> crate::engine::txn::TransactionResult<()> {
+    ) -> RuntimeResult<()> {
         with_model(gns, &model_id.space_id, &model_id, |model| {
             let mut iwm = model.intent_write_model();
             let mut removed_fields_rb = vec![];
@@ -531,7 +541,7 @@ impl<'a> GNSEvent for AlterModelRemoveTxn<'a> {
                         removed_fields_rb.into_iter().for_each(|(field_id, field)| {
                             let _ = iwm.fields_mut().st_insert(field_id.into(), field);
                         });
-                        return Err(TransactionError::OnRestoreDataConflictMismatch);
+                        return Err(TransactionError::OnRestoreDataConflictMismatch.into());
                     }
                 }
             }
@@ -593,7 +603,7 @@ impl<'a> PersistObject for AlterModelUpdateTxn<'a> {
         <ModelID as PersistObject>::meta_enc(buf, data.model_id);
         buf.extend(data.updated_fields.st_len().u64_bytes_le());
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         let model_id_md = <ModelID as PersistObject>::meta_dec(scanner)?;
         Ok(AlterModelUpdateTxnMD {
             model_id_md,
@@ -607,7 +617,10 @@ impl<'a> PersistObject for AlterModelUpdateTxn<'a> {
             data.updated_fields,
         );
     }
-    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(
+        s: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> RuntimeResult<Self::OutputType> {
         let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_md)?;
         let updated_fields = <map::PersistMapImpl<map::FieldMapSpecST> as PersistObject>::obj_dec(
             s,
@@ -630,7 +643,7 @@ impl<'a> GNSEvent for AlterModelUpdateTxn<'a> {
             updated_fields,
         }: Self::RestoreType,
         gns: &GlobalNS,
-    ) -> TransactionResult<()> {
+    ) -> RuntimeResult<()> {
         with_model(gns, &model_id.space_id, &model_id, |model| {
             let mut iwm = model.intent_write_model();
             let mut fields_rb = vec![];
@@ -642,7 +655,7 @@ impl<'a> GNSEvent for AlterModelUpdateTxn<'a> {
                         fields_rb.into_iter().for_each(|(field_id, field)| {
                             let _ = iwm.fields_mut().st_update(field_id, field);
                         });
-                        return Err(TransactionError::OnRestoreDataConflictMismatch);
+                        return Err(TransactionError::OnRestoreDataConflictMismatch.into());
                     }
                 }
             }
@@ -682,14 +695,17 @@ impl<'a> PersistObject for DropModelTxn<'a> {
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
         <ModelID as PersistObject>::meta_enc(buf, data.model_id);
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         let model_id_md = <ModelID as PersistObject>::meta_dec(scanner)?;
         Ok(DropModelTxnMD { model_id_md })
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
         <ModelID as PersistObject>::obj_enc(buf, data.model_id);
     }
-    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(
+        s: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> RuntimeResult<Self::OutputType> {
         <ModelID as PersistObject>::obj_dec(s, md.model_id_md)
     }
 }
@@ -706,13 +722,13 @@ impl<'a> GNSEvent for DropModelTxn<'a> {
             model_version: _,
         }: Self::RestoreType,
         gns: &GlobalNS,
-    ) -> TransactionResult<()> {
+    ) -> RuntimeResult<()> {
         with_space(gns, &space_id, |space| {
             let mut wgns = space.models().write();
             match wgns.st_delete_if(&model_name, |mdl| mdl.get_uuid() == model_uuid) {
                 Some(true) => Ok(()),
-                Some(false) => return Err(TransactionError::OnRestoreDataConflictMismatch),
-                None => Err(TransactionError::OnRestoreDataMissing),
+                Some(false) => return Err(TransactionError::OnRestoreDataConflictMismatch.into()),
+                None => Err(TransactionError::OnRestoreDataMissing.into()),
             }
         })
     }
diff --git a/server/src/engine/txn/gns/space.rs b/server/src/engine/txn/gns/space.rs
index 04bfe874..5f9032b8 100644
--- a/server/src/engine/txn/gns/space.rs
+++ b/server/src/engine/txn/gns/space.rs
@@ -30,13 +30,10 @@ use {
         engine::{
             core::{space::Space, GlobalNS},
             data::DictGeneric,
+            error::{RuntimeResult, TransactionError},
             idx::STIndex,
             mem::BufferedScanner,
-            storage::v1::{
-                inf::{self, map, obj, PersistObject},
-                SDSSResult,
-            },
-            txn::{TransactionError, TransactionResult},
+            storage::v1::inf::{self, map, obj, PersistObject},
         },
         util::EndianQW,
     },
@@ -92,7 +89,7 @@ impl<'a> PersistObject for CreateSpaceTxn<'a> {
             obj::SpaceLayoutRef::from((data.space, data.space_meta)),
         );
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         let space_name_l = scanner.next_u64_le();
         let space_meta = <obj::SpaceLayoutRef as PersistObject>::meta_dec(scanner)?;
         Ok(CreateSpaceTxnMD {
@@ -104,7 +101,10 @@ impl<'a> PersistObject for CreateSpaceTxn<'a> {
         buf.extend(data.space_name.as_bytes());
         <obj::SpaceLayoutRef as PersistObject>::obj_enc(buf, (data.space, data.space_meta).into());
     }
-    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(
+        s: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> RuntimeResult<Self::OutputType> {
         let space_name =
             inf::dec::utils::decode_string(s, md.space_name_l as usize)?.into_boxed_str();
         let space = <obj::SpaceLayoutRef as PersistObject>::obj_dec(s, md.space_meta)?;
@@ -119,12 +119,12 @@ impl<'a> GNSEvent for CreateSpaceTxn<'a> {
     fn update_global_state(
         CreateSpaceTxnRestorePL { space_name, space }: CreateSpaceTxnRestorePL,
         gns: &crate::engine::core::GlobalNS,
-    ) -> crate::engine::txn::TransactionResult<()> {
+    ) -> RuntimeResult<()> {
         let mut wgns = gns.spaces().write();
         if wgns.st_insert(space_name, space) {
             Ok(())
         } else {
-            Err(TransactionError::OnRestoreDataConflictAlreadyExists)
+            Err(TransactionError::OnRestoreDataConflictAlreadyExists.into())
         }
     }
 }
@@ -174,7 +174,7 @@ impl<'a> PersistObject for AlterSpaceTxn<'a> {
         <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
         buf.extend(data.updated_props.len().u64_bytes_le());
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         Ok(AlterSpaceTxnMD {
             space_id_meta: <super::SpaceID as PersistObject>::meta_dec(scanner)?,
             dict_len: scanner.next_u64_le(),
@@ -187,7 +187,10 @@ impl<'a> PersistObject for AlterSpaceTxn<'a> {
             data.updated_props,
         );
     }
-    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(
+        s: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> RuntimeResult<Self::OutputType> {
         let space_id = <super::SpaceID as PersistObject>::obj_dec(s, md.space_id_meta)?;
         let space_meta = <map::PersistMapImpl<map::GenericDictSpec> as PersistObject>::obj_dec(
             s,
@@ -211,16 +214,16 @@ impl<'a> GNSEvent for AlterSpaceTxn<'a> {
             space_meta,
         }: Self::RestoreType,
         gns: &crate::engine::core::GlobalNS,
-    ) -> TransactionResult<()> {
+    ) -> RuntimeResult<()> {
         let gns = gns.spaces().read();
         match gns.st_get(&space_id.name) {
             Some(space) => {
                 let mut wmeta = space.metadata().dict().write();
                 if !crate::engine::data::dict::rmerge_metadata(&mut wmeta, space_meta) {
-                    return Err(TransactionError::OnRestoreDataConflictMismatch);
+                    return Err(TransactionError::OnRestoreDataConflictMismatch.into());
                 }
             }
-            None => return Err(TransactionError::OnRestoreDataMissing),
+            None => return Err(TransactionError::OnRestoreDataMissing.into()),
         }
         Ok(())
     }
@@ -253,13 +256,16 @@ impl<'a> PersistObject for DropSpaceTxn<'a> {
     fn meta_enc(buf: &mut Vec<u8>, data: Self::InputType) {
         <super::SpaceID as PersistObject>::meta_enc(buf, data.space_id);
     }
-    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> SDSSResult<Self::Metadata> {
+    unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         <super::SpaceID as PersistObject>::meta_dec(scanner)
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
         <super::SpaceID as PersistObject>::obj_enc(buf, data.space_id)
     }
-    unsafe fn obj_dec(s: &mut BufferedScanner, md: Self::Metadata) -> SDSSResult<Self::OutputType> {
+    unsafe fn obj_dec(
+        s: &mut BufferedScanner,
+        md: Self::Metadata,
+    ) -> RuntimeResult<Self::OutputType> {
         <super::SpaceID as PersistObject>::obj_dec(s, md)
     }
 }
@@ -271,7 +277,7 @@ impl<'a> GNSEvent for DropSpaceTxn<'a> {
     fn update_global_state(
         super::SpaceIDRes { uuid, name }: Self::RestoreType,
         gns: &GlobalNS,
-    ) -> TransactionResult<()> {
+    ) -> RuntimeResult<()> {
         let mut wgns = gns.spaces().write();
         match wgns.entry(name) {
             std::collections::hash_map::Entry::Occupied(oe) => {
@@ -279,11 +285,11 @@ impl<'a> GNSEvent for DropSpaceTxn<'a> {
                     oe.remove_entry();
                     Ok(())
                 } else {
-                    return Err(TransactionError::OnRestoreDataConflictMismatch);
+                    return Err(TransactionError::OnRestoreDataConflictMismatch.into());
                 }
             }
             std::collections::hash_map::Entry::Vacant(_) => {
-                return Err(TransactionError::OnRestoreDataMissing)
+                return Err(TransactionError::OnRestoreDataMissing.into())
             }
         }
     }
diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
index 175d6311..4bac479e 100644
--- a/server/src/engine/txn/gns/tests/full_chain.rs
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -30,7 +30,7 @@ use crate::engine::{
         space::{Space, SpaceMeta},
     },
     data::{cell::Datacell, tag::TagSelector, uuid::Uuid, DictEntryGeneric},
-    error::Error,
+    error::QueryError,
     fractal::{test_utils::TestGlobal, GlobalInstanceLike},
     idx::STIndex,
     ql::{
@@ -318,7 +318,7 @@ fn drop_model() {
                     .namespace()
                     .with_model(("myspace", "mymodel"), |_| { Ok(()) })
                     .unwrap_err(),
-                Error::QPObjectNotFound
+                QueryError::QPObjectNotFound
             );
         })
     })
diff --git a/server/src/engine/txn/mod.rs b/server/src/engine/txn/mod.rs
index 733839dc..3c258b82 100644
--- a/server/src/engine/txn/mod.rs
+++ b/server/src/engine/txn/mod.rs
@@ -25,32 +25,3 @@
 */
 
 pub mod gns;
-
-use super::storage::v1::SDSSError;
-pub type TransactionResult<T> = Result<T, TransactionError>;
-
-#[derive(Debug)]
-#[cfg_attr(test, derive(PartialEq))]
-pub enum TransactionError {
-    /// corrupted txn payload. has more bytes than expected
-    DecodeCorruptedPayloadMoreBytes,
-    /// transaction payload is corrupted. has lesser bytes than expected
-    DecodedUnexpectedEof,
-    /// unknown transaction operation. usually indicates a corrupted payload
-    DecodeUnknownTxnOp,
-    /// While restoring a certain item, a non-resolvable conflict was encountered in the global state, because the item was
-    /// already present (when it was expected to not be present)
-    OnRestoreDataConflictAlreadyExists,
-    /// On restore, a certain item that was expected to be present was missing in the global state
-    OnRestoreDataMissing,
-    /// On restore, a certain item that was expected to match a certain value, has a different value
-    OnRestoreDataConflictMismatch,
-    SDSSError(SDSSError),
-    OutOfMemory,
-}
-
-direct_from! {
-    TransactionError => {
-        SDSSError as SDSSError
-    }
-}

From dfdb8a39ecbce549f257639b5377a6601063f8a7 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 5 Oct 2023 04:35:15 +0000
Subject: [PATCH 271/310] Migrate to use new database engine

---
 Cargo.lock                                    |  255 +---
 server/Cargo.toml                             |   14 -
 server/src/actions/dbsize.rs                  |   44 -
 server/src/actions/del.rs                     |   81 --
 server/src/actions/exists.rs                  |   62 -
 server/src/actions/flushdb.rs                 |   49 -
 server/src/actions/get.rs                     |   53 -
 server/src/actions/keylen.rs                  |   59 -
 server/src/actions/lists/lget.rs              |  207 ---
 server/src/actions/lists/lmod.rs              |  186 ---
 server/src/actions/lists/macros.rs            |   35 -
 server/src/actions/lists/mod.rs               |   57 -
 server/src/actions/lskeys.rs                  |   87 --
 server/src/actions/macros.rs                  |   82 --
 server/src/actions/mget.rs                    |   53 -
 server/src/actions/mod.rs                     |  148 --
 server/src/actions/mpop.rs                    |   57 -
 server/src/actions/mset.rs                    |   64 -
 server/src/actions/mupdate.rs                 |   65 -
 server/src/actions/pop.rs                     |   50 -
 server/src/actions/set.rs                     |   62 -
 server/src/actions/strong/mod.rs              |   65 -
 server/src/actions/strong/sdel.rs             |  139 --
 server/src/actions/strong/sset.rs             |  125 --
 server/src/actions/strong/supdate.rs          |  144 --
 server/src/actions/strong/tests.rs            |  168 ---
 server/src/actions/update.rs                  |   59 -
 server/src/actions/uset.rs                    |   59 -
 server/src/actions/whereami.rs                |   46 -
 server/src/admin/mksnap.rs                    |   97 --
 server/src/admin/mod.rs                       |   30 -
 server/src/admin/sys.rs                       |   81 --
 server/src/arbiter.rs                         |  211 ---
 server/src/auth/keys.rs                       |   64 -
 server/src/auth/mod.rs                        |  159 ---
 server/src/auth/provider.rs                   |  281 ----
 server/src/auth/tests.rs                      |  124 --
 server/src/blueql/ast.rs                      |  485 -------
 server/src/blueql/error.rs                    |   81 --
 server/src/blueql/executor.rs                 |  115 --
 server/src/blueql/lexer.rs                    |  394 ------
 server/src/blueql/mod.rs                      |   98 --
 server/src/blueql/tests.rs                    |  323 -----
 server/src/blueql/util.rs                     |   41 -
 server/src/config/cfgcli.rs                   |  137 --
 server/src/config/cfgenv.rs                   |   74 -
 server/src/config/cfgfile.rs                  |  239 ----
 server/src/config/definitions.rs              |  504 -------
 server/src/config/feedback.rs                 |  314 -----
 server/src/config/mod.rs                      |  703 ----------
 server/src/config/tests.rs                    |  836 ------------
 server/src/corestore/array.rs                 |  624 ---------
 server/src/corestore/backoff.rs               |   58 -
 server/src/corestore/booltable.rs             |  146 --
 server/src/corestore/buffers.rs               |  413 ------
 server/src/corestore/heap_array.rs            |  132 --
 server/src/corestore/htable.rs                |  217 ---
 server/src/corestore/iarray.rs                |  659 ---------
 server/src/corestore/lazy.rs                  |  308 -----
 server/src/corestore/lock.rs                  |  183 ---
 server/src/corestore/map/bref.rs              |  267 ----
 server/src/corestore/map/iter.rs              |  143 --
 server/src/corestore/map/mod.rs               |  439 ------
 server/src/corestore/memstore.rs              |  492 -------
 server/src/corestore/mod.rs                   |  381 ------
 server/src/corestore/rc.rs                    |  268 ----
 server/src/corestore/table.rs                 |  286 ----
 server/src/corestore/tests.rs                 |  170 ---
 server/src/dbnet/connection.rs                |  248 ----
 server/src/dbnet/listener.rs                  |  280 ----
 server/src/dbnet/mod.rs                       |  264 ----
 server/src/dbnet/prelude.rs                   |   46 -
 server/src/dbnet/tcp.rs                       |  105 --
 server/src/dbnet/tls.rs                       |  142 --
 server/src/diskstore/flock.rs                 |  398 ------
 server/src/diskstore/mod.rs                   |   29 -
 server/src/engine/config.rs                   |   53 +-
 server/src/engine/core/dml/del.rs             |    1 +
 server/src/engine/core/dml/ins.rs             |    1 +
 server/src/engine/core/dml/sel.rs             |    1 +
 server/src/engine/core/dml/upd.rs             |    1 +
 .../{dbnet/macros.rs => engine/core/exec.rs}  |   19 +-
 server/src/engine/core/index/key.rs           |   15 +-
 server/src/engine/core/index/mod.rs           |    3 -
 server/src/engine/core/index/row.rs           |   11 -
 server/src/engine/core/mod.rs                 |    1 +
 server/src/engine/core/model/alt.rs           |    1 +
 server/src/engine/core/model/delta.rs         |    2 +
 server/src/engine/core/model/mod.rs           |   18 +-
 server/src/engine/core/space.rs               |   22 +-
 server/src/engine/core/tests/ddl_model/mod.rs |    7 -
 server/src/engine/data/cell.rs                |   14 -
 server/src/engine/data/lit.rs                 |   12 +
 server/src/engine/data/tag.rs                 |    6 -
 server/src/engine/data/uuid.rs                |    3 -
 server/src/engine/error.rs                    |   15 +-
 server/src/engine/fractal/config.rs           |   28 +-
 server/src/engine/fractal/context.rs          |   14 +-
 server/src/engine/fractal/drivers.rs          |   14 +-
 server/src/engine/fractal/error.rs            |   12 -
 server/src/engine/fractal/mgr.rs              |   79 +-
 server/src/engine/fractal/mod.rs              |   21 +-
 server/src/engine/fractal/test_utils.rs       |    1 +
 server/src/engine/fractal/util.rs             |    2 +
 server/src/engine/idx/meta/hash.rs            |    2 -
 server/src/engine/idx/mod.rs                  |    3 +-
 server/src/engine/idx/mtchm/imp.rs            |    3 +-
 server/src/engine/idx/mtchm/mod.rs            |   20 +-
 server/src/engine/idx/mtchm/patch.rs          |    1 -
 server/src/engine/idx/mtchm/tests.rs          |    2 -
 server/src/engine/idx/stord/mod.rs            |   12 +-
 server/src/engine/idx/tests.rs                |   11 -
 server/src/engine/macros.rs                   |   40 +-
 server/src/engine/mem/scanner.rs              |   21 -
 server/src/engine/mem/stackop.rs              |    1 +
 server/src/engine/mem/uarray.rs               |    5 +-
 server/src/engine/mem/vinline.rs              |    5 +
 server/src/engine/mod.rs                      |  184 ++-
 server/src/engine/net/mod.rs                  |   67 +-
 server/src/engine/net/protocol/exchange.rs    |   27 +-
 server/src/engine/net/protocol/handshake.rs   |   48 +-
 server/src/engine/net/protocol/mod.rs         |  153 ++-
 server/src/engine/net/protocol/tests.rs       |   58 +-
 server/src/engine/ql/ast/mod.rs               |   46 +-
 server/src/engine/ql/ddl/alt.rs               |    1 +
 server/src/engine/ql/ddl/crt.rs               |    1 +
 server/src/engine/ql/ddl/syn.rs               |    4 +-
 server/src/engine/ql/dml/del.rs               |    4 +-
 server/src/engine/ql/dml/ins.rs               |   12 +-
 server/src/engine/ql/dml/mod.rs               |    5 +-
 server/src/engine/ql/dml/sel.rs               |    9 +-
 server/src/engine/ql/dml/upd.rs               |   13 -
 server/src/engine/ql/lex/mod.rs               |  375 ++---
 server/src/engine/ql/lex/raw.rs               |  382 ++++--
 server/src/engine/ql/macros.rs                |   91 +-
 server/src/engine/ql/tests.rs                 |    2 +-
 server/src/engine/storage/checksum.rs         |    6 -
 server/src/engine/storage/header.rs           |  441 ------
 server/src/engine/storage/mod.rs              |    1 -
 .../engine/storage/v1/batch_jrnl/persist.rs   |    2 +-
 .../engine/storage/v1/batch_jrnl/restore.rs   |    1 +
 server/src/engine/storage/v1/inf/map.rs       |    7 -
 server/src/engine/storage/v1/inf/mod.rs       |   19 +-
 server/src/engine/storage/v1/inf/obj.rs       |    2 -
 server/src/engine/storage/v1/journal.rs       |   32 +-
 server/src/engine/storage/v1/loader.rs        |   40 +-
 server/src/engine/storage/v1/memfs.rs         |   25 -
 server/src/engine/storage/v1/mod.rs           |    4 +-
 server/src/engine/storage/v1/rw.rs            |   11 +-
 server/src/engine/storage/v1/spec.rs          |  169 ++-
 server/src/engine/storage/v1/sysdb.rs         |   22 +-
 server/src/engine/storage/v1/tests.rs         |   11 +-
 server/src/engine/storage/v1/tests/batch.rs   |    2 +-
 server/src/engine/storage/v1/tests/tx.rs      |   11 +-
 server/src/engine/sync/atm.rs                 |   22 -
 server/src/engine/sync/cell.rs                |  147 +-
 server/src/engine/sync/queue.rs               |   12 +-
 server/src/engine/sync/smart.rs               |   11 +-
 server/src/engine/txn/gns/mod.rs              |    8 +-
 server/src/engine/txn/gns/model.rs            |    1 +
 server/src/kvengine/encoding.rs               |  342 -----
 server/src/kvengine/mod.rs                    |  295 ----
 server/src/kvengine/tests.rs                  |  111 --
 server/src/main.rs                            |  126 +-
 server/src/protocol/interface.rs              |  222 ---
 server/src/protocol/iter.rs                   |  218 ---
 server/src/protocol/mod.rs                    |  190 ---
 server/src/protocol/raw_parser.rs             |  174 ---
 server/src/protocol/v1/benches.rs             |   83 --
 server/src/protocol/v1/interface_impls.rs     |  130 --
 server/src/protocol/v1/mod.rs                 |  242 ----
 server/src/protocol/v1/tests.rs               |   93 --
 server/src/protocol/v2/benches.rs             |   82 --
 server/src/protocol/v2/interface_impls.rs     |  130 --
 server/src/protocol/v2/mod.rs                 |  187 ---
 server/src/protocol/v2/tests.rs               |  645 ---------
 server/src/queryengine/mod.rs                 |  182 ---
 server/src/registry/mod.rs                    |  114 --
 server/src/services/bgsave.rs                 |  101 --
 server/src/services/mod.rs                    |   55 -
 server/src/services/snapshot.rs               |   93 --
 server/src/storage/mod.rs                     |   75 -
 server/src/storage/v1/bytemarks.rs            |   69 -
 server/src/storage/v1/error.rs                |   90 --
 server/src/storage/v1/flush.rs                |  336 -----
 server/src/storage/v1/interface.rs            |  183 ---
 server/src/storage/v1/iter.rs                 |  237 ----
 server/src/storage/v1/macros.rs               |  121 --
 server/src/storage/v1/mod.rs                  |  660 ---------
 server/src/storage/v1/preload.rs              |   97 --
 server/src/storage/v1/sengine.rs              |  354 -----
 server/src/storage/v1/tests.rs                |  821 -----------
 server/src/storage/v1/unflush.rs              |  266 ----
 server/src/tests/auth.rs                      |  378 ------
 server/src/tests/ddl_tests.rs                 |  169 ---
 server/src/tests/inspect_tests.rs             |   95 --
 server/src/tests/issue_tests.rs               |   52 -
 server/src/tests/kvengine.rs                  | 1205 -----------------
 server/src/tests/kvengine_encoding.rs         |   68 -
 server/src/tests/kvengine_list.rs             |  451 ------
 server/src/tests/macros.rs                    |  135 --
 server/src/tests/mod.rs                       |  145 --
 server/src/tests/persist/auth.rs              |   52 -
 server/src/tests/persist/kv.rs                |  146 --
 server/src/tests/persist/kvlist.rs            |  168 ---
 server/src/tests/persist/mod.rs               |  238 ----
 server/src/tests/pipeline.rs                  |   85 --
 server/src/tests/snapshot.rs                  |   89 --
 server/src/util/error.rs                      |   92 --
 server/src/util/macros.rs                     |   32 -
 server/src/util/mod.rs                        |   19 -
 server/src/util/os.rs                         |    3 +-
 server/src/util/os/flock.rs                   |  117 ++
 213 files changed, 1486 insertions(+), 27033 deletions(-)
 delete mode 100644 server/src/actions/dbsize.rs
 delete mode 100644 server/src/actions/del.rs
 delete mode 100644 server/src/actions/exists.rs
 delete mode 100644 server/src/actions/flushdb.rs
 delete mode 100644 server/src/actions/get.rs
 delete mode 100644 server/src/actions/keylen.rs
 delete mode 100644 server/src/actions/lists/lget.rs
 delete mode 100644 server/src/actions/lists/lmod.rs
 delete mode 100644 server/src/actions/lists/macros.rs
 delete mode 100644 server/src/actions/lists/mod.rs
 delete mode 100644 server/src/actions/lskeys.rs
 delete mode 100644 server/src/actions/macros.rs
 delete mode 100644 server/src/actions/mget.rs
 delete mode 100644 server/src/actions/mod.rs
 delete mode 100644 server/src/actions/mpop.rs
 delete mode 100644 server/src/actions/mset.rs
 delete mode 100644 server/src/actions/mupdate.rs
 delete mode 100644 server/src/actions/pop.rs
 delete mode 100644 server/src/actions/set.rs
 delete mode 100644 server/src/actions/strong/mod.rs
 delete mode 100644 server/src/actions/strong/sdel.rs
 delete mode 100644 server/src/actions/strong/sset.rs
 delete mode 100644 server/src/actions/strong/supdate.rs
 delete mode 100644 server/src/actions/strong/tests.rs
 delete mode 100644 server/src/actions/update.rs
 delete mode 100644 server/src/actions/uset.rs
 delete mode 100644 server/src/actions/whereami.rs
 delete mode 100644 server/src/admin/mksnap.rs
 delete mode 100644 server/src/admin/mod.rs
 delete mode 100644 server/src/admin/sys.rs
 delete mode 100644 server/src/arbiter.rs
 delete mode 100644 server/src/auth/keys.rs
 delete mode 100644 server/src/auth/mod.rs
 delete mode 100644 server/src/auth/provider.rs
 delete mode 100644 server/src/auth/tests.rs
 delete mode 100644 server/src/blueql/ast.rs
 delete mode 100644 server/src/blueql/error.rs
 delete mode 100644 server/src/blueql/executor.rs
 delete mode 100644 server/src/blueql/lexer.rs
 delete mode 100644 server/src/blueql/mod.rs
 delete mode 100644 server/src/blueql/tests.rs
 delete mode 100644 server/src/blueql/util.rs
 delete mode 100644 server/src/config/cfgcli.rs
 delete mode 100644 server/src/config/cfgenv.rs
 delete mode 100644 server/src/config/cfgfile.rs
 delete mode 100644 server/src/config/definitions.rs
 delete mode 100644 server/src/config/feedback.rs
 delete mode 100644 server/src/config/mod.rs
 delete mode 100644 server/src/config/tests.rs
 delete mode 100644 server/src/corestore/array.rs
 delete mode 100644 server/src/corestore/backoff.rs
 delete mode 100644 server/src/corestore/booltable.rs
 delete mode 100644 server/src/corestore/buffers.rs
 delete mode 100644 server/src/corestore/heap_array.rs
 delete mode 100644 server/src/corestore/htable.rs
 delete mode 100644 server/src/corestore/iarray.rs
 delete mode 100644 server/src/corestore/lazy.rs
 delete mode 100644 server/src/corestore/lock.rs
 delete mode 100644 server/src/corestore/map/bref.rs
 delete mode 100644 server/src/corestore/map/iter.rs
 delete mode 100644 server/src/corestore/map/mod.rs
 delete mode 100644 server/src/corestore/memstore.rs
 delete mode 100644 server/src/corestore/mod.rs
 delete mode 100644 server/src/corestore/rc.rs
 delete mode 100644 server/src/corestore/table.rs
 delete mode 100644 server/src/corestore/tests.rs
 delete mode 100644 server/src/dbnet/connection.rs
 delete mode 100644 server/src/dbnet/listener.rs
 delete mode 100644 server/src/dbnet/mod.rs
 delete mode 100644 server/src/dbnet/prelude.rs
 delete mode 100644 server/src/dbnet/tcp.rs
 delete mode 100644 server/src/dbnet/tls.rs
 delete mode 100644 server/src/diskstore/flock.rs
 delete mode 100644 server/src/diskstore/mod.rs
 rename server/src/{dbnet/macros.rs => engine/core/exec.rs} (66%)
 delete mode 100644 server/src/kvengine/encoding.rs
 delete mode 100644 server/src/kvengine/mod.rs
 delete mode 100644 server/src/kvengine/tests.rs
 delete mode 100644 server/src/protocol/interface.rs
 delete mode 100644 server/src/protocol/iter.rs
 delete mode 100644 server/src/protocol/mod.rs
 delete mode 100644 server/src/protocol/raw_parser.rs
 delete mode 100644 server/src/protocol/v1/benches.rs
 delete mode 100644 server/src/protocol/v1/interface_impls.rs
 delete mode 100644 server/src/protocol/v1/mod.rs
 delete mode 100644 server/src/protocol/v1/tests.rs
 delete mode 100644 server/src/protocol/v2/benches.rs
 delete mode 100644 server/src/protocol/v2/interface_impls.rs
 delete mode 100644 server/src/protocol/v2/mod.rs
 delete mode 100644 server/src/protocol/v2/tests.rs
 delete mode 100644 server/src/queryengine/mod.rs
 delete mode 100644 server/src/registry/mod.rs
 delete mode 100644 server/src/services/bgsave.rs
 delete mode 100644 server/src/services/mod.rs
 delete mode 100644 server/src/services/snapshot.rs
 delete mode 100644 server/src/storage/mod.rs
 delete mode 100644 server/src/storage/v1/bytemarks.rs
 delete mode 100644 server/src/storage/v1/error.rs
 delete mode 100644 server/src/storage/v1/flush.rs
 delete mode 100644 server/src/storage/v1/interface.rs
 delete mode 100644 server/src/storage/v1/iter.rs
 delete mode 100644 server/src/storage/v1/macros.rs
 delete mode 100644 server/src/storage/v1/mod.rs
 delete mode 100644 server/src/storage/v1/preload.rs
 delete mode 100644 server/src/storage/v1/sengine.rs
 delete mode 100644 server/src/storage/v1/tests.rs
 delete mode 100644 server/src/storage/v1/unflush.rs
 delete mode 100644 server/src/tests/auth.rs
 delete mode 100644 server/src/tests/ddl_tests.rs
 delete mode 100644 server/src/tests/inspect_tests.rs
 delete mode 100644 server/src/tests/issue_tests.rs
 delete mode 100644 server/src/tests/kvengine.rs
 delete mode 100644 server/src/tests/kvengine_encoding.rs
 delete mode 100644 server/src/tests/kvengine_list.rs
 delete mode 100644 server/src/tests/macros.rs
 delete mode 100644 server/src/tests/mod.rs
 delete mode 100644 server/src/tests/persist/auth.rs
 delete mode 100644 server/src/tests/persist/kv.rs
 delete mode 100644 server/src/tests/persist/kvlist.rs
 delete mode 100644 server/src/tests/persist/mod.rs
 delete mode 100644 server/src/tests/pipeline.rs
 delete mode 100644 server/src/tests/snapshot.rs
 delete mode 100644 server/src/util/error.rs
 create mode 100644 server/src/util/os/flock.rs

diff --git a/Cargo.lock b/Cargo.lock
index 0ab36bfd..48518883 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -28,18 +28,6 @@ dependencies = [
  "cpufeatures",
 ]
 
-[[package]]
-name = "ahash"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
-dependencies = [
- "cfg-if",
- "getrandom",
- "once_cell",
- "version_check",
-]
-
 [[package]]
 name = "aho-corasick"
 version = "1.0.3"
@@ -49,27 +37,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "allocator-api2"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
-
-[[package]]
-name = "android-tzdata"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
-
-[[package]]
-name = "android_system_properties"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "ansi_term"
 version = "0.12.1"
@@ -128,12 +95,6 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
-[[package]]
-name = "base64"
-version = "0.21.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
-
 [[package]]
 name = "base64ct"
 version = "1.6.0"
@@ -193,12 +154,6 @@ dependencies = [
  "cipher",
 ]
 
-[[package]]
-name = "bumpalo"
-version = "3.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
-
 [[package]]
 name = "byteorder"
 version = "1.4.3"
@@ -248,21 +203,6 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
-[[package]]
-name = "chrono"
-version = "0.4.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5"
-dependencies = [
- "android-tzdata",
- "iana-time-zone",
- "js-sys",
- "num-traits",
- "time 0.1.45",
- "wasm-bindgen",
- "winapi",
-]
-
 [[package]]
 name = "cipher"
 version = "0.4.4"
@@ -597,7 +537,7 @@ checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
 dependencies = [
  "cfg-if",
  "libc",
- "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasi",
 ]
 
 [[package]]
@@ -624,10 +564,6 @@ name = "hashbrown"
 version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
-dependencies = [
- "ahash",
- "allocator-api2",
-]
 
 [[package]]
 name = "hermit-abi"
@@ -668,29 +604,6 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
-[[package]]
-name = "iana-time-zone"
-version = "0.1.57"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"
-dependencies = [
- "android_system_properties",
- "core-foundation-sys",
- "iana-time-zone-haiku",
- "js-sys",
- "wasm-bindgen",
- "windows",
-]
-
-[[package]]
-name = "iana-time-zone-haiku"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
-dependencies = [
- "cc",
-]
-
 [[package]]
 name = "indexmap"
 version = "2.0.0"
@@ -756,15 +669,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "js-sys"
-version = "0.3.64"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a"
-dependencies = [
- "wasm-bindgen",
-]
-
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -845,7 +749,7 @@ checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"
 dependencies = [
  "libc",
  "log",
- "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasi",
  "windows-sys",
 ]
 
@@ -879,15 +783,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "num-traits"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "num_cpus"
 version = "1.16.0"
@@ -1134,7 +1029,7 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c70c3bf2f0b523cc6f2420446e6fca7e14db888cbec907991df043a4d6d2e1e"
 dependencies = [
- "base64 0.13.1",
+ "base64",
  "blowfish",
  "getrandom",
 ]
@@ -1271,15 +1166,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "serde_spanned"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "serde_yaml"
 version = "0.9.25"
@@ -1385,34 +1271,24 @@ dependencies = [
 name = "skyd"
 version = "0.8.0"
 dependencies = [
- "ahash",
- "base64 0.21.2",
- "bincode",
  "bytes",
  "cc",
- "chrono",
- "clap",
  "crc",
  "crossbeam-epoch",
  "env_logger",
- "hashbrown",
  "jemallocator",
  "libc",
  "libsky",
- "libstress",
  "log",
  "openssl",
  "parking_lot",
  "rand",
  "rcrypt",
- "regex",
  "serde",
  "serde_yaml",
  "sky_macros",
- "skytable 0.8.0 (git+https://github.com/skytable/client-rust?branch=next)",
  "tokio",
  "tokio-openssl",
- "toml",
  "uuid",
  "winapi",
 ]
@@ -1570,17 +1446,6 @@ dependencies = [
  "unicode-width",
 ]
 
-[[package]]
-name = "time"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
-dependencies = [
- "libc",
- "wasi 0.10.0+wasi-snapshot-preview1",
- "winapi",
-]
-
 [[package]]
 name = "time"
 version = "0.3.25"
@@ -1640,40 +1505,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "toml"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542"
-dependencies = [
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_edit",
-]
-
-[[package]]
-name = "toml_datetime"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "toml_edit"
-version = "0.19.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a"
-dependencies = [
- "indexmap",
- "serde",
- "serde_spanned",
- "toml_datetime",
- "winnow",
-]
-
 [[package]]
 name = "typenum"
 version = "1.16.0"
@@ -1750,72 +1581,12 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
-[[package]]
-name = "wasi"
-version = "0.10.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
-
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
-[[package]]
-name = "wasm-bindgen"
-version = "0.2.87"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342"
-dependencies = [
- "cfg-if",
- "wasm-bindgen-macro",
-]
-
-[[package]]
-name = "wasm-bindgen-backend"
-version = "0.2.87"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd"
-dependencies = [
- "bumpalo",
- "log",
- "once_cell",
- "proc-macro2",
- "quote",
- "syn 2.0.28",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-macro"
-version = "0.2.87"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
-dependencies = [
- "quote",
- "wasm-bindgen-macro-support",
-]
-
-[[package]]
-name = "wasm-bindgen-macro-support"
-version = "0.2.87"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.28",
- "wasm-bindgen-backend",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-shared"
-version = "0.2.87"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
-
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -1847,15 +1618,6 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
-[[package]]
-name = "windows"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
-dependencies = [
- "windows-targets",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.48.0"
@@ -1922,15 +1684,6 @@ version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
 
-[[package]]
-name = "winnow"
-version = "0.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acaaa1190073b2b101e15083c38ee8ec891b5e05cbee516521e94ec008f61e64"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "yaml-rust"
 version = "0.3.5"
@@ -1953,7 +1706,7 @@ dependencies = [
  "hmac",
  "pbkdf2",
  "sha1",
- "time 0.3.25",
+ "time",
  "zstd",
 ]
 
diff --git a/server/Cargo.toml b/server/Cargo.toml
index c0eb8280..e218fcbc 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -11,22 +11,15 @@ libsky = { path = "../libsky" }
 sky_macros = { path = "../sky-macros" }
 rcrypt = "0.4.0"
 # external deps
-ahash = "0.8.3"
 bytes = "1.4.0"
-chrono = "0.4.26"
-clap = { version = "2", features = ["yaml"] }
 env_logger = "0.10.0"
-hashbrown = { version = "0.14.0", features = ["raw"] }
 log = "0.4.19"
 openssl = { version = "0.10.56", features = ["vendored"] }
 crossbeam-epoch = { version = "0.9.15" }
 parking_lot = "0.12.1"
-regex = "1.9.1"
 serde = { version = "1.0.183", features = ["derive"] }
 tokio = { version = "1.30.0", features = ["full"] }
 tokio-openssl = "0.6.3"
-toml = "0.7.6"
-base64 = "0.21.2"
 uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics"] }
 crc = "3.0.1"
 serde_yaml = "0.9"
@@ -47,14 +40,7 @@ libc = "0.2.147"
 cc = "1.0.82"
 
 [dev-dependencies]
-# internal deps
-libstress = { path = "../libstress" }
-skytable = { git = "https://github.com/skytable/client-rust", features = [
-  "aio",
-  "aio-ssl",
-], default-features = false, branch = "next" }
 # external deps
-bincode = "1.3.3"
 rand = "0.8.5"
 tokio = { version = "1.30.0", features = ["test-util"] }
 
diff --git a/server/src/actions/dbsize.rs b/server/src/actions/dbsize.rs
deleted file mode 100644
index 0519436d..00000000
--- a/server/src/actions/dbsize.rs
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Created on Thu Sep 24 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::dbnet::prelude::*;
-
-action!(
-    /// Returns the number of keys in the database
-    fn dbsize(handle: &Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
-        ensure_length::<P>(act.len(), |len| len < 2)?;
-        if act.is_empty() {
-            let len = get_tbl_ref!(handle, con).count();
-            con.write_usize(len).await?;
-        } else {
-            let raw_entity = unsafe { act.next().unsafe_unwrap() };
-            let entity = handle_entity!(con, raw_entity);
-            con.write_usize(get_tbl!(&entity, handle, con).count())
-                .await?;
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/del.rs b/server/src/actions/del.rs
deleted file mode 100644
index 7019944f..00000000
--- a/server/src/actions/del.rs
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Created on Wed Aug 19 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # `DEL` queries
-//! This module provides functions to work with `DEL` queries
-
-use crate::{
-    corestore::table::DataModel, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER,
-    util::compiler,
-};
-
-action!(
-    /// Run a `DEL` query
-    ///
-    /// Do note that this function is blocking since it acquires a write lock.
-    /// It will write an entire datagroup, for this `del` action
-    fn del(handle: &Corestore, con: &mut Connection<C, P>, act: ActionIter<'a>) {
-        ensure_length::<P>(act.len(), |size| size != 0)?;
-        let table = get_tbl_ref!(handle, con);
-        macro_rules! remove {
-            ($engine:expr) => {{
-                let encoding_is_okay = ENCODING_LUT_ITER[$engine.is_key_encoded()](act.as_ref());
-                if compiler::likely(encoding_is_okay) {
-                    let done_howmany: Option<usize>;
-                    {
-                        if registry::state_okay() {
-                            let mut many = 0;
-                            act.for_each(|key| {
-                                many += $engine.remove_unchecked(key) as usize;
-                            });
-                            done_howmany = Some(many);
-                        } else {
-                            done_howmany = None;
-                        }
-                    }
-                    if let Some(done_howmany) = done_howmany {
-                        con.write_usize(done_howmany).await?;
-                    } else {
-                        con._write_raw(P::RCODE_SERVER_ERR).await?;
-                    }
-                } else {
-                    return util::err(P::RCODE_ENCODING_ERROR);
-                }
-            }};
-        }
-        match table.get_model_ref() {
-            DataModel::KV(kve) => {
-                remove!(kve)
-            }
-            DataModel::KVExtListmap(kvlmap) => {
-                remove!(kvlmap)
-            }
-            #[allow(unreachable_patterns)]
-            _ => return util::err(P::RSTRING_WRONG_MODEL),
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/exists.rs b/server/src/actions/exists.rs
deleted file mode 100644
index 93238da3..00000000
--- a/server/src/actions/exists.rs
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Created on Wed Aug 19 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # `EXISTS` queries
-//! This module provides functions to work with `EXISTS` queries
-
-use crate::{
-    corestore::table::DataModel, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER,
-    queryengine::ActionIter, util::compiler,
-};
-
-action!(
-    /// Run an `EXISTS` query
-    fn exists(handle: &Corestore, con: &mut Connection<C, P>, act: ActionIter<'a>) {
-        ensure_length::<P>(act.len(), |len| len != 0)?;
-        let mut how_many_of_them_exist = 0usize;
-        macro_rules! exists {
-            ($engine:expr) => {{
-                let encoding_is_okay = ENCODING_LUT_ITER[$engine.is_key_encoded()](act.as_ref());
-                if compiler::likely(encoding_is_okay) {
-                    act.for_each(|key| {
-                        how_many_of_them_exist += $engine.exists_unchecked(key) as usize;
-                    });
-                    con.write_usize(how_many_of_them_exist).await?;
-                } else {
-                    return util::err(P::RCODE_ENCODING_ERROR);
-                }
-            }};
-        }
-        let tbl = get_tbl_ref!(handle, con);
-        match tbl.get_model_ref() {
-            DataModel::KV(kve) => exists!(kve),
-            DataModel::KVExtListmap(kve) => exists!(kve),
-            #[allow(unreachable_patterns)]
-            _ => return util::err(P::RSTRING_WRONG_MODEL),
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/flushdb.rs b/server/src/actions/flushdb.rs
deleted file mode 100644
index 7bb9e06c..00000000
--- a/server/src/actions/flushdb.rs
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Created on Thu Sep 24 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{dbnet::prelude::*, queryengine::ActionIter};
-
-action!(
-    /// Delete all the keys in the database
-    fn flushdb(handle: &Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
-        ensure_length::<P>(act.len(), |len| len < 2)?;
-        if registry::state_okay() {
-            if act.is_empty() {
-                // flush the current table
-                get_tbl_ref!(handle, con).truncate_table();
-            } else {
-                // flush the entity
-                let raw_entity = unsafe { act.next_unchecked() };
-                let entity = handle_entity!(con, raw_entity);
-                get_tbl!(&entity, handle, con).truncate_table();
-            }
-            con._write_raw(P::RCODE_OKAY).await?;
-        } else {
-            con._write_raw(P::RCODE_SERVER_ERR).await?;
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/get.rs b/server/src/actions/get.rs
deleted file mode 100644
index 5240ce12..00000000
--- a/server/src/actions/get.rs
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Created on Fri Aug 14 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # `GET` queries
-//! This module provides functions to work with `GET` queries
-
-use crate::{dbnet::prelude::*, util::compiler};
-
-action!(
-    /// Run a `GET` query
-    fn get(
-        handle: &crate::corestore::Corestore,
-        con: &mut Connection<C, P>,
-        mut act: ActionIter<'a>,
-    ) {
-        ensure_length::<P>(act.len(), |len| len == 1)?;
-        let kve = handle.get_table_with::<P, KVEBlob>()?;
-        unsafe {
-            match kve.get_cloned(act.next_unchecked()) {
-                Ok(Some(val)) => {
-                    con.write_mono_length_prefixed_with_tsymbol(&val, kve.get_value_tsymbol())
-                        .await?
-                }
-                Err(_) => compiler::cold_val(con._write_raw(P::RCODE_ENCODING_ERROR)).await?,
-                Ok(_) => con._write_raw(P::RCODE_NIL).await?,
-            }
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/keylen.rs b/server/src/actions/keylen.rs
deleted file mode 100644
index d32a5ad3..00000000
--- a/server/src/actions/keylen.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Created on Sun Sep 27 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::dbnet::prelude::*;
-
-action!(
-    /// Run a `KEYLEN` query
-    ///
-    /// At this moment, `keylen` only supports a single key
-    fn keylen(
-        handle: &crate::corestore::Corestore,
-        con: &mut Connection<C, P>,
-        mut act: ActionIter<'a>,
-    ) {
-        ensure_length::<P>(act.len(), |len| len == 1)?;
-        let res: Option<usize> = {
-            let reader = handle.get_table_with::<P, KVEBlob>()?;
-            unsafe {
-                // UNSAFE(@ohsayan): this is completely safe as we've already checked
-                // the number of arguments is one
-                match reader.get(act.next_unchecked()) {
-                    Ok(v) => v.map(|b| b.len()),
-                    Err(_) => None,
-                }
-            }
-        };
-        if let Some(value) = res {
-            // Good, we got the key's length, write it off to the stream
-            con.write_usize(value).await?;
-        } else {
-            // Ah, couldn't find that key
-            con._write_raw(P::RCODE_NIL).await?;
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/lists/lget.rs b/server/src/actions/lists/lget.rs
deleted file mode 100644
index 6c2fc5ee..00000000
--- a/server/src/actions/lists/lget.rs
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Created on Fri Sep 17 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{corestore::SharedSlice, dbnet::prelude::*};
-
-const LEN: &[u8] = "LEN".as_bytes();
-const LIMIT: &[u8] = "LIMIT".as_bytes();
-const VALUEAT: &[u8] = "VALUEAT".as_bytes();
-const LAST: &[u8] = "LAST".as_bytes();
-const FIRST: &[u8] = "FIRST".as_bytes();
-const RANGE: &[u8] = "RANGE".as_bytes();
-
-struct Range {
-    start: usize,
-    stop: Option<usize>,
-}
-
-impl Range {
-    pub fn new(start: usize) -> Self {
-        Self { start, stop: None }
-    }
-    pub fn set_stop(&mut self, stop: usize) {
-        self.stop = Some(stop);
-    }
-    pub fn into_vec(self, slice: &[SharedSlice]) -> Option<Vec<SharedSlice>> {
-        slice
-            .get(self.start..self.stop.unwrap_or(slice.len()))
-            .map(|slc| slc.to_vec())
-    }
-}
-
-action! {
-    /// Handle an `LGET` query for the list model (KVExt)
-    /// ## Syntax
-    /// - `LGET <mylist>` will return the full list
-    /// - `LGET <mylist> LEN` will return the length of the list
-    /// - `LGET <mylist> LIMIT <limit>` will return a maximum of `limit` elements
-    /// - `LGET <mylist> VALUEAT <index>` will return the value at the provided index
-    /// - `LGET <mylist> FIRST` will return the first item
-    /// - `LGET <mylist> LAST` will return the last item
-    /// if it exists
-    fn lget(handle: &Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
-        ensure_length::<P>(act.len(), |len| len != 0)?;
-        let listmap = handle.get_table_with::<P, KVEList>()?;
-        // get the list name
-        let listname = unsafe { act.next_unchecked() };
-        // now let us see what we need to do
-        macro_rules! get_numeric_count {
-            () => {
-                match unsafe { String::from_utf8_lossy(act.next_unchecked()) }.parse::<usize>() {
-                    Ok(int) => int,
-                    Err(_) => return util::err(P::RCODE_WRONGTYPE_ERR),
-                }
-            };
-        }
-        match act.next_uppercase().as_ref() {
-            None => {
-                // just return everything in the list
-                let items = match listmap.list_cloned_full(listname) {
-                    Ok(Some(list)) => list,
-                    Ok(None) => return Err(P::RCODE_NIL.into()),
-                    Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()),
-                };
-                writelist!(con, listmap, items);
-            }
-            Some(subaction) => {
-                match subaction.as_ref() {
-                    LEN => {
-                        ensure_length::<P>(act.len(), |len| len == 0)?;
-                        match listmap.list_len(listname) {
-                            Ok(Some(len)) => con.write_usize(len).await?,
-                            Ok(None) => return Err(P::RCODE_NIL.into()),
-                            Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()),
-                        }
-                    }
-                    LIMIT => {
-                        ensure_length::<P>(act.len(), |len| len == 1)?;
-                        let count = get_numeric_count!();
-                        match listmap.list_cloned(listname, count) {
-                            Ok(Some(items)) => writelist!(con, listmap, items),
-                            Ok(None) => return Err(P::RCODE_NIL.into()),
-                            Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()),
-                        }
-                    }
-                    VALUEAT => {
-                        ensure_length::<P>(act.len(), |len| len == 1)?;
-                        let idx = get_numeric_count!();
-                        let maybe_value = listmap.get(listname).map(|list| {
-                            list.map(|lst| lst.read().get(idx).cloned())
-                        });
-                        match maybe_value {
-                            Ok(v) => match v {
-                                Some(Some(value)) => {
-                                    con.write_mono_length_prefixed_with_tsymbol(
-                                        &value, listmap.get_value_tsymbol()
-                                    ).await?;
-                                }
-                                Some(None) => {
-                                    // bad index
-                                    return Err(P::RSTRING_LISTMAP_BAD_INDEX.into());
-                                }
-                                None => {
-                                    // not found
-                                    return Err(P::RCODE_NIL.into());
-                                }
-                            }
-                            Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()),
-                        }
-                    }
-                    LAST => {
-                        ensure_length::<P>(act.len(), |len| len == 0)?;
-                        let maybe_value = listmap.get(listname).map(|list| {
-                            list.map(|lst| lst.read().last().cloned())
-                        });
-                        match maybe_value {
-                            Ok(v) => match v {
-                                Some(Some(value)) => {
-                                    con.write_mono_length_prefixed_with_tsymbol(
-                                        &value, listmap.get_value_tsymbol()
-                                    ).await?;
-                                },
-                                Some(None) => return Err(P::RSTRING_LISTMAP_LIST_IS_EMPTY.into()),
-                                None => return Err(P::RCODE_NIL.into()),
-                            }
-                            Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()),
-                        }
-                    }
-                    FIRST => {
-                        ensure_length::<P>(act.len(), |len| len == 0)?;
-                        let maybe_value = listmap.get(listname).map(|list| {
-                            list.map(|lst| lst.read().first().cloned())
-                        });
-                        match maybe_value {
-                            Ok(v) => match v {
-                                Some(Some(value)) => {
-                                    con.write_mono_length_prefixed_with_tsymbol(
-                                        &value, listmap.get_value_tsymbol()
-                                    ).await?;
-                                },
-                                Some(None) => return Err(P::RSTRING_LISTMAP_LIST_IS_EMPTY.into()),
-                                None => return Err(P::RCODE_NIL.into()),
-                            }
-                            Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()),
-                        }
-                    }
-                    RANGE => {
-                        match act.next_string_owned() {
-                            Some(start) => {
-                                let start: usize = match start.parse() {
-                                    Ok(v) => v,
-                                    Err(_) => return util::err(P::RCODE_WRONGTYPE_ERR),
-                                };
-                                let mut range = Range::new(start);
-                                if let Some(stop) = act.next_string_owned() {
-                                    let stop: usize = match stop.parse() {
-                                        Ok(v) => v,
-                                        Err(_) => return util::err(P::RCODE_WRONGTYPE_ERR),
-                                    };
-                                    range.set_stop(stop);
-                                };
-                                match listmap.get(listname) {
-                                    Ok(Some(list)) => {
-                                        let ret = range.into_vec(&list.read());
-                                        match ret {
-                                            Some(ret) => {
-                                                writelist!(con, listmap, ret);
-                                            },
-                                            None => return Err(P::RSTRING_LISTMAP_BAD_INDEX.into()),
-                                        }
-                                    }
-                                    Ok(None) => return Err(P::RCODE_NIL.into()),
-                                    Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()),
-                                }
-                            }
-                            None => return Err(P::RCODE_ACTION_ERR.into()),
-                        }
-                    }
-                    _ => return Err(P::RCODE_UNKNOWN_ACTION.into()),
-                }
-            }
-        }
-        Ok(())
-    }
-}
diff --git a/server/src/actions/lists/lmod.rs b/server/src/actions/lists/lmod.rs
deleted file mode 100644
index cb728a85..00000000
--- a/server/src/actions/lists/lmod.rs
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Created on Wed Sep 15 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{corestore::SharedSlice, dbnet::prelude::*, util::compiler};
-
-const CLEAR: &[u8] = "CLEAR".as_bytes();
-const PUSH: &[u8] = "PUSH".as_bytes();
-const REMOVE: &[u8] = "REMOVE".as_bytes();
-const INSERT: &[u8] = "INSERT".as_bytes();
-const POP: &[u8] = "POP".as_bytes();
-
-action! {
-    /// Handle `LMOD` queries
-    /// ## Syntax
-    /// - `LMOD <mylist> push <value>`
-    /// - `LMOD <mylist> pop <optional idx>`
-    /// - `LMOD <mylist> insert <index> <value>`
-    /// - `LMOD <mylist> remove <index>`
-    /// - `LMOD <mylist> clear`
-    fn lmod(handle: &Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
-        ensure_length::<P>(act.len(), |len| len > 1)?;
-        let listmap = handle.get_table_with::<P, KVEList>()?;
-        // get the list name
-        let listname = unsafe { act.next_unchecked() };
-        macro_rules! get_numeric_count {
-            () => {
-                match unsafe { String::from_utf8_lossy(act.next_unchecked()) }.parse::<usize>() {
-                    Ok(int) => int,
-                    Err(_) => return Err(P::RCODE_WRONGTYPE_ERR.into()),
-                }
-            };
-        }
-        // now let us see what we need to do
-        match unsafe { act.next_uppercase_unchecked() }.as_ref() {
-            CLEAR => {
-                ensure_length::<P>(act.len(), |len| len == 0)?;
-                let list = match listmap.get_inner_ref().get(listname) {
-                    Some(l) => l,
-                    _ => return Err(P::RCODE_NIL.into()),
-                };
-                let okay = if registry::state_okay() {
-                    list.write().clear();
-                    P::RCODE_OKAY
-                } else {
-                    P::RCODE_SERVER_ERR
-                };
-                con._write_raw(okay).await?
-            }
-            PUSH => {
-                ensure_boolean_or_aerr::<P>(!act.is_empty())?;
-                let list = match listmap.get_inner_ref().get(listname) {
-                    Some(l) => l,
-                    _ => return Err(P::RCODE_NIL.into()),
-                };
-                let venc_ok = listmap.get_val_encoder();
-                let ret = if compiler::likely(act.as_ref().all(venc_ok)) {
-                    if registry::state_okay() {
-                        list.write().extend(act.map(SharedSlice::new));
-                        P::RCODE_OKAY
-                    } else {
-                        P::RCODE_SERVER_ERR
-                    }
-                } else {
-                    P::RCODE_ENCODING_ERROR
-                };
-                con._write_raw(ret).await?
-            }
-            REMOVE => {
-                ensure_length::<P>(act.len(), |len| len == 1)?;
-                let idx_to_remove = get_numeric_count!();
-                if registry::state_okay() {
-                    let maybe_value = listmap.get_inner_ref().get(listname).map(|list| {
-                        let mut wlock = list.write();
-                        if idx_to_remove < wlock.len() {
-                            wlock.remove(idx_to_remove);
-                            true
-                        } else {
-                            false
-                        }
-                    });
-                    con._write_raw(P::OKAY_BADIDX_NIL_NLUT[maybe_value]).await?
-                } else {
-                    return Err(P::RCODE_SERVER_ERR.into());
-                }
-            }
-            INSERT => {
-                ensure_length::<P>(act.len(), |len| len == 2)?;
-                let idx_to_insert_at = get_numeric_count!();
-                let bts = unsafe { act.next_unchecked() };
-                let ret = if compiler::likely(listmap.is_val_ok(bts)) {
-                    if registry::state_okay() {
-                        // okay state, good to insert
-                        let maybe_insert = match listmap.get(listname) {
-                            Ok(lst) => lst.map(|list| {
-                                let mut wlock = list.write();
-                                if idx_to_insert_at < wlock.len() {
-                                    // we can insert
-                                    wlock.insert(idx_to_insert_at, SharedSlice::new(bts));
-                                    true
-                                } else {
-                                    // oops, out of bounds
-                                    false
-                                }
-                            }),
-                            Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()),
-                        };
-                        P::OKAY_BADIDX_NIL_NLUT[maybe_insert]
-                    } else {
-                        // flush broken; server err
-                        P::RCODE_SERVER_ERR
-                    }
-                } else {
-                    // encoding failed, uh
-                    P::RCODE_ENCODING_ERROR
-                };
-                con._write_raw(ret).await?
-            }
-            POP => {
-                ensure_length::<P>(act.len(), |len| len < 2)?;
-                let idx = if act.len() == 1 {
-                    // we have an idx
-                    Some(get_numeric_count!())
-                } else {
-                    // no idx
-                    None
-                };
-                if registry::state_okay() {
-                    let maybe_pop = match listmap.get(listname) {
-                        Ok(lst) => lst.map(|list| {
-                            let mut wlock = list.write();
-                            if let Some(idx) = idx {
-                                if idx < wlock.len() {
-                                    // so we can pop
-                                    Some(wlock.remove(idx))
-                                } else {
-                                    None
-                                }
-                            } else {
-                                wlock.pop()
-                            }
-                        }),
-                        Err(()) => return Err(P::RCODE_ENCODING_ERROR.into()),
-                    };
-                    match maybe_pop {
-                        Some(Some(val)) => {
-                            con.write_mono_length_prefixed_with_tsymbol(
-                                &val, listmap.get_value_tsymbol()
-                            ).await?;
-                        }
-                        Some(None) => {
-                            con._write_raw(P::RSTRING_LISTMAP_BAD_INDEX).await?;
-                        }
-                        None => con._write_raw(P::RCODE_NIL).await?,
-                    }
-                } else {
-                    con._write_raw(P::RCODE_SERVER_ERR).await?
-                }
-            }
-            _ => con._write_raw(P::RCODE_UNKNOWN_ACTION).await?,
-        }
-        Ok(())
-    }
-}
diff --git a/server/src/actions/lists/macros.rs b/server/src/actions/lists/macros.rs
deleted file mode 100644
index fa428330..00000000
--- a/server/src/actions/lists/macros.rs
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Created on Wed Sep 15 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-macro_rules! writelist {
-    ($con:expr, $listmap:expr, $items:expr) => {{
-        $con.write_typed_non_null_array_header($items.len(), $listmap.get_value_tsymbol())
-            .await?;
-        for item in $items {
-            $con.write_typed_non_null_array_element(&item).await?;
-        }
-    }};
-}
diff --git a/server/src/actions/lists/mod.rs b/server/src/actions/lists/mod.rs
deleted file mode 100644
index c4526918..00000000
--- a/server/src/actions/lists/mod.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Created on Tue Sep 07 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#[macro_use]
-mod macros;
-// modules
-pub mod lget;
-pub mod lmod;
-
-use crate::{corestore::SharedSlice, dbnet::prelude::*, kvengine::LockedVec};
-
-action! {
-    /// Handle an `LSET` query for the list model
-    /// Syntax: `LSET <listname> <values ...>`
-    fn lset(handle: &Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
-        ensure_length::<P>(act.len(), |len| len > 0)?;
-        let listmap = handle.get_table_with::<P, KVEList>()?;
-        let listname = unsafe { act.next_unchecked_bytes() };
-        let list = listmap.get_inner_ref();
-        if registry::state_okay() {
-            let did = if let Some(entry) = list.fresh_entry(listname) {
-                let v: Vec<SharedSlice> = act.map(SharedSlice::new).collect();
-                entry.insert(LockedVec::new(v));
-                true
-            } else {
-                false
-            };
-            con._write_raw(P::OKAY_OVW_BLUT[did]).await?
-        } else {
-            con._write_raw(P::RCODE_SERVER_ERR).await?
-        }
-        Ok(())
-    }
-}
diff --git a/server/src/actions/lskeys.rs b/server/src/actions/lskeys.rs
deleted file mode 100644
index 647f5256..00000000
--- a/server/src/actions/lskeys.rs
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Created on Thu May 13 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{
-    corestore::{table::DataModel, SharedSlice},
-    dbnet::prelude::*,
-};
-
-const DEFAULT_COUNT: usize = 10;
-
-action!(
-    /// Run an `LSKEYS` query
-    fn lskeys(
-        handle: &crate::corestore::Corestore,
-        con: &mut Connection<C, P>,
-        mut act: ActionIter<'a>,
-    ) {
-        ensure_length::<P>(act.len(), |size| size < 4)?;
-        let (table, count) = if act.is_empty() {
-            (get_tbl!(handle, con), DEFAULT_COUNT)
-        } else if act.len() == 1 {
-            // two args, could either be count or an entity
-            let nextret = unsafe { act.next_unchecked() };
-            if unsafe { ucidx!(nextret, 0) }.is_ascii_digit() {
-                // noice, this is a number; let's try to parse it
-                let count = if let Ok(cnt) = String::from_utf8_lossy(nextret).parse::<usize>() {
-                    cnt
-                } else {
-                    return util::err(P::RCODE_WRONGTYPE_ERR);
-                };
-                (get_tbl!(handle, con), count)
-            } else {
-                // sigh, an entity
-                let entity = handle_entity!(con, nextret);
-                (get_tbl!(&entity, handle, con), DEFAULT_COUNT)
-            }
-        } else {
-            // an entity and a count, gosh this fella is really trying us
-            let entity_ret = unsafe { act.next().unsafe_unwrap() };
-            let count_ret = unsafe { act.next().unsafe_unwrap() };
-            let entity = handle_entity!(con, entity_ret);
-            let count = if let Ok(cnt) = String::from_utf8_lossy(count_ret).parse::<usize>() {
-                cnt
-            } else {
-                return util::err(P::RCODE_WRONGTYPE_ERR);
-            };
-            (get_tbl!(&entity, handle, con), count)
-        };
-        let tsymbol = match table.get_model_ref() {
-            DataModel::KV(kv) => kv.get_value_tsymbol(),
-            DataModel::KVExtListmap(kv) => kv.get_value_tsymbol(),
-        };
-        let items: Vec<SharedSlice> = match table.get_model_ref() {
-            DataModel::KV(kv) => kv.get_inner_ref().get_keys(count),
-            DataModel::KVExtListmap(kv) => kv.get_inner_ref().get_keys(count),
-        };
-        con.write_typed_non_null_array_header(items.len(), tsymbol)
-            .await?;
-        for key in items {
-            con.write_typed_non_null_array_element(&key).await?;
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/macros.rs b/server/src/actions/macros.rs
deleted file mode 100644
index a1998078..00000000
--- a/server/src/actions/macros.rs
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Created on Thu Nov 11 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
- Don't modulo because it's an L1 miss and an L2 hit. Use lowbit checks to check for parity
-*/
-
-#[macro_export]
-/// endian independent check to see if the lowbit is set or not. Returns true if the lowbit
-/// is set. this is undefined to be applied on signed values on one's complement targets
-macro_rules! is_lowbit_set {
-    ($v:expr) => {
-        $v & 1 == 1
-    };
-}
-
-#[macro_export]
-/// endian independent check to see if the lowbit is unset or not. Returns true if the lowbit
-/// is unset. this is undefined to be applied on signed values on one's complement targets
-macro_rules! is_lowbit_unset {
-    ($v:expr) => {
-        $v & 1 == 0
-    };
-}
-
-#[macro_export]
-macro_rules! get_tbl {
-    ($entity:expr, $store:expr, $con:expr) => {{
-        $crate::actions::translate_ddl_error::<P, ::std::sync::Arc<$crate::corestore::table::Table>>(
-            $store.get_table($entity),
-        )?
-    }};
-    ($store:expr, $con:expr) => {{
-        match $store.get_ctable() {
-            Some(tbl) => tbl,
-            None => return $crate::util::err(P::RSTRING_DEFAULT_UNSET),
-        }
-    }};
-}
-
-#[macro_export]
-macro_rules! get_tbl_ref {
-    ($store:expr, $con:expr) => {{
-        match $store.get_ctable_ref() {
-            Some(tbl) => tbl,
-            None => return $crate::util::err(P::RSTRING_DEFAULT_UNSET),
-        }
-    }};
-}
-
-#[macro_export]
-macro_rules! handle_entity {
-    ($con:expr, $ident:expr) => {{
-        match $crate::blueql::util::from_slice_action_result::<P>(&$ident) {
-            Ok(e) => e,
-            Err(e) => return Err(e.into()),
-        }
-    }};
-}
diff --git a/server/src/actions/mget.rs b/server/src/actions/mget.rs
deleted file mode 100644
index 73b0cc40..00000000
--- a/server/src/actions/mget.rs
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Created on Thu Aug 27 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{
-    dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER, queryengine::ActionIter,
-    util::compiler,
-};
-
-action!(
-    /// Run an `MGET` query
-    ///
-    fn mget(handle: &crate::corestore::Corestore, con: &mut Connection<C, P>, act: ActionIter<'a>) {
-        ensure_length::<P>(act.len(), |size| size != 0)?;
-        let kve = handle.get_table_with::<P, KVEBlob>()?;
-        let encoding_is_okay = ENCODING_LUT_ITER[kve.is_key_encoded()](act.as_ref());
-        if compiler::likely(encoding_is_okay) {
-            con.write_typed_array_header(act.len(), kve.get_value_tsymbol())
-                .await?;
-            for key in act {
-                match kve.get_cloned_unchecked(key) {
-                    Some(v) => con.write_typed_array_element(&v).await?,
-                    None => con.write_typed_array_element_null().await?,
-                }
-            }
-        } else {
-            return util::err(P::RCODE_ENCODING_ERROR);
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/mod.rs b/server/src/actions/mod.rs
deleted file mode 100644
index 7d7337df..00000000
--- a/server/src/actions/mod.rs
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Created on Wed Aug 19 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # Actions
-//!
-//! Actions are like shell commands, you provide arguments -- they return output. This module contains a collection
-//! of the actions supported by Skytable
-//!
-
-#[macro_use]
-mod macros;
-pub mod dbsize;
-pub mod del;
-pub mod exists;
-pub mod flushdb;
-pub mod get;
-pub mod keylen;
-pub mod lists;
-pub mod lskeys;
-pub mod mget;
-pub mod mpop;
-pub mod mset;
-pub mod mupdate;
-pub mod pop;
-pub mod set;
-pub mod strong;
-pub mod update;
-pub mod uset;
-pub mod whereami;
-use {
-    crate::{corestore::memstore::DdlError, protocol::interface::ProtocolSpec, util},
-    std::io::Error as IoError,
-};
-
-/// A generic result for actions
-pub type ActionResult<T> = Result<T, ActionError>;
-
-/// Errors that can occur while running actions
-#[derive(Debug)]
-pub enum ActionError {
-    ActionError(&'static [u8]),
-    IoError(std::io::Error),
-}
-
-impl PartialEq for ActionError {
-    fn eq(&self, other: &Self) -> bool {
-        match (self, other) {
-            (Self::ActionError(a1), Self::ActionError(a2)) => a1 == a2,
-            (Self::IoError(ioe1), Self::IoError(ioe2)) => ioe1.to_string() == ioe2.to_string(),
-            _ => false,
-        }
-    }
-}
-
-impl From<&'static [u8]> for ActionError {
-    fn from(e: &'static [u8]) -> Self {
-        Self::ActionError(e)
-    }
-}
-
-impl From<IoError> for ActionError {
-    fn from(e: IoError) -> Self {
-        Self::IoError(e)
-    }
-}
-
-#[cold]
-#[inline(never)]
-fn map_ddl_error_to_status<P: ProtocolSpec>(e: DdlError) -> ActionError {
-    let r = match e {
-        DdlError::AlreadyExists => P::RSTRING_ALREADY_EXISTS,
-        DdlError::DdlTransactionFailure => P::RSTRING_DDL_TRANSACTIONAL_FAILURE,
-        DdlError::DefaultNotFound => P::RSTRING_DEFAULT_UNSET,
-        DdlError::NotEmpty => P::RSTRING_KEYSPACE_NOT_EMPTY,
-        DdlError::NotReady => P::RSTRING_NOT_READY,
-        DdlError::ObjectNotFound => P::RSTRING_CONTAINER_NOT_FOUND,
-        DdlError::ProtectedObject => P::RSTRING_PROTECTED_OBJECT,
-        DdlError::StillInUse => P::RSTRING_STILL_IN_USE,
-        DdlError::WrongModel => P::RSTRING_WRONG_MODEL,
-    };
-    ActionError::ActionError(r)
-}
-
-#[inline(always)]
-pub fn translate_ddl_error<P: ProtocolSpec, T>(r: Result<T, DdlError>) -> Result<T, ActionError> {
-    match r {
-        Ok(r) => Ok(r),
-        Err(e) => Err(map_ddl_error_to_status::<P>(e)),
-    }
-}
-
-pub fn ensure_length<P: ProtocolSpec>(len: usize, is_valid: fn(usize) -> bool) -> ActionResult<()> {
-    if util::compiler::likely(is_valid(len)) {
-        Ok(())
-    } else {
-        util::err(P::RCODE_ACTION_ERR)
-    }
-}
-
-pub fn ensure_boolean_or_aerr<P: ProtocolSpec>(boolean: bool) -> ActionResult<()> {
-    if util::compiler::likely(boolean) {
-        Ok(())
-    } else {
-        util::err(P::RCODE_ACTION_ERR)
-    }
-}
-
-pub mod heya {
-    //! Respond to `HEYA` queries
-    use crate::dbnet::prelude::*;
-    action!(
-        /// Returns a `HEY!` `Response`
-        fn heya(_handle: &Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
-            ensure_length::<P>(act.len(), |len| len == 0 || len == 1)?;
-            if act.len() == 1 {
-                let raw_byte = unsafe { act.next_unchecked() };
-                con.write_mono_length_prefixed_with_tsymbol(raw_byte, b'+')
-                    .await?;
-            } else {
-                con._write_raw(P::ELEMRESP_HEYA).await?;
-            }
-            Ok(())
-        }
-    );
-}
diff --git a/server/src/actions/mpop.rs b/server/src/actions/mpop.rs
deleted file mode 100644
index 1f931905..00000000
--- a/server/src/actions/mpop.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Created on Wed Aug 11 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{
-    corestore, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER, queryengine::ActionIter,
-    util::compiler,
-};
-
-action!(
-    /// Run an MPOP action
-    fn mpop(handle: &corestore::Corestore, con: &mut Connection<C, P>, act: ActionIter<'a>) {
-        ensure_length::<P>(act.len(), |len| len != 0)?;
-        if registry::state_okay() {
-            let kve = handle.get_table_with::<P, KVEBlob>()?;
-            let encoding_is_okay = ENCODING_LUT_ITER[kve.is_key_encoded()](act.as_ref());
-            if compiler::likely(encoding_is_okay) {
-                con.write_typed_array_header(act.len(), kve.get_value_tsymbol())
-                    .await?;
-                for key in act {
-                    match kve.pop_unchecked(key) {
-                        Some(val) => con.write_typed_array_element(&val).await?,
-                        None => con.write_typed_array_element_null().await?,
-                    }
-                }
-            } else {
-                return util::err(P::RCODE_ENCODING_ERROR);
-            }
-        } else {
-            // don't begin the operation at all if the database is poisoned
-            return util::err(P::RCODE_SERVER_ERR);
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/mset.rs b/server/src/actions/mset.rs
deleted file mode 100644
index 36382969..00000000
--- a/server/src/actions/mset.rs
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Created on Thu Aug 27 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{
-    corestore::SharedSlice, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER_PAIR,
-    util::compiler,
-};
-
-action!(
-    /// Run an `MSET` query
-    fn mset(
-        handle: &crate::corestore::Corestore,
-        con: &mut Connection<C, P>,
-        mut act: ActionIter<'a>,
-    ) {
-        let howmany = act.len();
-        ensure_length::<P>(howmany, |size| size & 1 == 0 && size != 0)?;
-        let kve = handle.get_table_with::<P, KVEBlob>()?;
-        let encoding_is_okay = ENCODING_LUT_ITER_PAIR[kve.get_encoding_tuple()](&act);
-        if compiler::likely(encoding_is_okay) {
-            let done_howmany: Option<usize> = if registry::state_okay() {
-                let mut didmany = 0;
-                while let (Some(key), Some(val)) = (act.next(), act.next()) {
-                    didmany +=
-                        kve.set_unchecked(SharedSlice::new(key), SharedSlice::new(val)) as usize;
-                }
-                Some(didmany)
-            } else {
-                None
-            };
-            if let Some(done_howmany) = done_howmany {
-                con.write_usize(done_howmany).await?;
-            } else {
-                return util::err(P::RCODE_SERVER_ERR);
-            }
-        } else {
-            return util::err(P::RCODE_ENCODING_ERROR);
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/mupdate.rs b/server/src/actions/mupdate.rs
deleted file mode 100644
index 9e8472fb..00000000
--- a/server/src/actions/mupdate.rs
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Created on Thu Aug 27 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{
-    corestore::SharedSlice, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER_PAIR,
-    util::compiler,
-};
-
-action!(
-    /// Run an `MUPDATE` query
-    fn mupdate(
-        handle: &crate::corestore::Corestore,
-        con: &mut Connection<C, P>,
-        mut act: ActionIter<'a>,
-    ) {
-        let howmany = act.len();
-        ensure_length::<P>(howmany, |size| size & 1 == 0 && size != 0)?;
-        let kve = handle.get_table_with::<P, KVEBlob>()?;
-        let encoding_is_okay = ENCODING_LUT_ITER_PAIR[kve.get_encoding_tuple()](&act);
-        let done_howmany: Option<usize>;
-        if compiler::likely(encoding_is_okay) {
-            if registry::state_okay() {
-                let mut didmany = 0;
-                while let (Some(key), Some(val)) = (act.next(), act.next()) {
-                    didmany +=
-                        kve.update_unchecked(SharedSlice::new(key), SharedSlice::new(val)) as usize;
-                }
-                done_howmany = Some(didmany);
-            } else {
-                done_howmany = None;
-            }
-            if let Some(done_howmany) = done_howmany {
-                con.write_usize(done_howmany).await?;
-            } else {
-                return util::err(P::RCODE_SERVER_ERR);
-            }
-        } else {
-            return util::err(P::RCODE_ENCODING_ERROR);
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/pop.rs b/server/src/actions/pop.rs
deleted file mode 100644
index a162dd34..00000000
--- a/server/src/actions/pop.rs
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Created on Mon Jun 14 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::dbnet::prelude::*;
-
-action! {
-    fn pop(handle: &Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
-        ensure_length::<P>(act.len(), |len| len == 1)?;
-        let key = unsafe {
-            // SAFETY: We have checked for there to be one arg
-            act.next_unchecked()
-        };
-        if registry::state_okay() {
-            let kve = handle.get_table_with::<P, KVEBlob>()?;
-            match kve.pop(key) {
-                Ok(Some(val)) => con.write_mono_length_prefixed_with_tsymbol(
-                    &val, kve.get_value_tsymbol()
-                ).await?,
-                Ok(None) => return util::err(P::RCODE_NIL),
-                Err(()) => return util::err(P::RCODE_ENCODING_ERROR),
-            }
-        } else {
-            return util::err(P::RCODE_SERVER_ERR);
-        }
-        Ok(())
-    }
-}
diff --git a/server/src/actions/set.rs b/server/src/actions/set.rs
deleted file mode 100644
index 1c1c0c30..00000000
--- a/server/src/actions/set.rs
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Created on Fri Aug 14 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # `SET` queries
-//! This module provides functions to work with `SET` queries
-
-use crate::{corestore::SharedSlice, dbnet::prelude::*, queryengine::ActionIter};
-
-action!(
-    /// Run a `SET` query
-    fn set(
-        handle: &crate::corestore::Corestore,
-        con: &mut Connection<C, P>,
-        mut act: ActionIter<'a>,
-    ) {
-        ensure_length::<P>(act.len(), |len| len == 2)?;
-        if registry::state_okay() {
-            let did_we = {
-                let writer = handle.get_table_with::<P, KVEBlob>()?;
-                match unsafe {
-                    // UNSAFE(@ohsayan): This is completely safe as we've already checked
-                    // that there are exactly 2 arguments
-                    writer.set(
-                        SharedSlice::new(act.next().unsafe_unwrap()),
-                        SharedSlice::new(act.next().unsafe_unwrap()),
-                    )
-                } {
-                    Ok(true) => Some(true),
-                    Ok(false) => Some(false),
-                    Err(()) => None,
-                }
-            };
-            con._write_raw(P::SET_NLUT[did_we]).await?;
-        } else {
-            con._write_raw(P::RCODE_SERVER_ERR).await?;
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/strong/mod.rs b/server/src/actions/strong/mod.rs
deleted file mode 100644
index 2b7d73d6..00000000
--- a/server/src/actions/strong/mod.rs
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Created on Fri Jul 30 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # Strong Actions
-//! Strong actions are like "do all" or "fail all" actions, built specifically for
-//! multiple keys. So let's say you used `SSET` instead of `MSET` for setting keys:
-//! what'd be the difference?
-//! In this case, if all the keys are non-existing, which is a requirement for `MSET`,
-//! only then would the keys be set. That is, only if all the keys can be set, will the action
-//! run and return code `0` - otherwise the action won't do anything and return an overwrite error.
-//! There is no point of using _strong actions_ for a single key/value pair, since it will only
-//! slow things down due to the checks performed.
-//! Do note that this isn't the same as the gurantees provided by ACID transactions
-
-pub use self::{sdel::sdel, sset::sset, supdate::supdate};
-mod sdel;
-mod sset;
-mod supdate;
-
-#[cfg(test)]
-mod tests;
-
-#[derive(Debug)]
-enum StrongActionResult {
-    /// Internal server error
-    ServerError,
-    /// A single value was not found
-    Nil,
-    /// An overwrite error for a single value
-    OverwriteError,
-    /// An encoding error occurred
-    EncodingError,
-    /// Everything worked as expected
-    Okay,
-}
-
-#[cfg(test)]
-impl StrongActionResult {
-    pub const fn is_ok(&self) -> bool {
-        matches!(self, StrongActionResult::Okay)
-    }
-}
diff --git a/server/src/actions/strong/sdel.rs b/server/src/actions/strong/sdel.rs
deleted file mode 100644
index baff6aef..00000000
--- a/server/src/actions/strong/sdel.rs
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Created on Fri Jul 30 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{
-        actions::strong::StrongActionResult,
-        dbnet::prelude::*,
-        kvengine::{KVEStandard, SingleEncoder},
-        protocol::iter::DerefUnsafeSlice,
-        util::compiler,
-    },
-    core::slice::Iter,
-};
-
-action! {
-    /// Run an `SDEL` query
-    ///
-    /// This either returns `Okay` if all the keys were `del`eted, or it returns a
-    /// `Nil`, which is code `1`
-    fn sdel(handle: &crate::corestore::Corestore, con: &mut Connection<C, P>, act: ActionIter<'a>) {
-        ensure_length::<P>(act.len(), |len| len != 0)?;
-        let kve = handle.get_table_with::<P, KVEBlob>()?;
-        if registry::state_okay() {
-            // guarantee one check: consistency
-            let key_encoder = kve.get_key_encoder();
-            let outcome = unsafe {
-                // UNSAFE(@ohsayan): The lifetime of `act` ensures that the
-                // pointers are still valid
-                self::snapshot_and_del(kve, key_encoder, act.into_inner())
-            };
-            match outcome {
-                StrongActionResult::Okay => con._write_raw(P::RCODE_OKAY).await?,
-                StrongActionResult::Nil => {
-                    // good, it failed because some key didn't exist
-                    return util::err(P::RCODE_NIL);
-                },
-                StrongActionResult::ServerError => return util::err(P::RCODE_SERVER_ERR),
-                StrongActionResult::EncodingError => {
-                    // error we love to hate: encoding error, ugh
-                    return util::err(P::RCODE_ENCODING_ERROR);
-                },
-                StrongActionResult::OverwriteError => unsafe {
-                    // SAFETY check: never the case
-                    impossible!()
-                }
-            }
-        } else {
-            return util::err(P::RCODE_SERVER_ERR);
-        }
-        Ok(())
-    }
-}
-
-/// Snapshot the current status and then delete maintaining concurrency
-/// guarantees
-pub(super) fn snapshot_and_del<'a, T: 'a + DerefUnsafeSlice>(
-    kve: &'a KVEStandard,
-    key_encoder: SingleEncoder,
-    act: Iter<'a, T>,
-) -> StrongActionResult {
-    let mut snapshots = Vec::with_capacity(act.len());
-    let mut err_enc = false;
-    let iter_stat_ok;
-    {
-        iter_stat_ok = act.as_ref().iter().all(|key| {
-            let key = unsafe {
-                // UNSAFE(@ohsayan): The caller has passed a slice and they should
-                // ensure that it is valid
-                key.deref_slice()
-            };
-            if compiler::likely(key_encoder(key)) {
-                if let Some(snap) = kve.take_snapshot_unchecked(key) {
-                    snapshots.push(snap);
-                    true
-                } else {
-                    false
-                }
-            } else {
-                err_enc = true;
-                false
-            }
-        });
-    }
-    cfg_test!({
-        // give the caller 10 seconds to do some crap
-        do_sleep!(10 s);
-    });
-    if compiler::unlikely(err_enc) {
-        return compiler::cold_val(StrongActionResult::EncodingError);
-    }
-    if registry::state_okay() {
-        // guarantee upholded: consistency
-        if iter_stat_ok {
-            // nice, all keys exist; let's plonk 'em
-            let kve = kve;
-            let lowtable = kve.get_inner_ref();
-            act.zip(snapshots).for_each(|(key, snapshot)| {
-                let key = unsafe {
-                    // UNSAFE(@ohsayan): The caller has passed a slice and they should
-                    // ensure that it is valid
-                    key.deref_slice()
-                };
-                // the check is very important: some thread may have updated the
-                // value after we snapshotted it. In that case, let this key
-                // be whatever the "newer" value is. Since our snapshot is a "happens-before"
-                // thing, this is absolutely fine
-                let _ = lowtable.remove_if(key, |_, val| val.eq(&snapshot));
-            });
-            StrongActionResult::Okay
-        } else {
-            StrongActionResult::Nil
-        }
-    } else {
-        StrongActionResult::ServerError
-    }
-}
diff --git a/server/src/actions/strong/sset.rs b/server/src/actions/strong/sset.rs
deleted file mode 100644
index 825f5679..00000000
--- a/server/src/actions/strong/sset.rs
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Created on Fri Jul 30 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{
-        actions::strong::StrongActionResult,
-        corestore::SharedSlice,
-        dbnet::prelude::*,
-        kvengine::{DoubleEncoder, KVEStandard},
-        protocol::iter::DerefUnsafeSlice,
-        util::compiler,
-    },
-    core::slice::Iter,
-};
-
-action! {
-    /// Run an `SSET` query
-    ///
-    /// This either returns `Okay` if all the keys were set, or it returns an
-    /// `Overwrite Error` or code `2`
-    fn sset(handle: &crate::corestore::Corestore, con: &mut Connection<C, P>, act: ActionIter<'a>) {
-        let howmany = act.len();
-        ensure_length::<P>(howmany, |size| size & 1 == 0 && size != 0)?;
-        let kve = handle.get_table_with::<P, KVEBlob>()?;
-        if registry::state_okay() {
-            let encoder = kve.get_double_encoder();
-            let outcome = unsafe {
-                // UNSAFE(@ohsayan): The lifetime of `act` guarantees that the
-                // pointers remain valid
-                self::snapshot_and_insert(kve, encoder, act.into_inner())
-            };
-            match outcome {
-                StrongActionResult::Okay => con._write_raw(P::RCODE_OKAY).await?,
-                StrongActionResult::OverwriteError => return util::err(P::RCODE_OVERWRITE_ERR),
-                StrongActionResult::ServerError => return util::err(P::RCODE_SERVER_ERR),
-                StrongActionResult::EncodingError => {
-                    // error we love to hate: encoding error, ugh
-                    return util::err(P::RCODE_ENCODING_ERROR);
-                },
-                StrongActionResult::Nil => unsafe {
-                    // SAFETY check: never the case
-                    impossible!()
-                }
-            }
-        } else {
-            return util::err(P::RCODE_SERVER_ERR);
-        }
-        Ok(())
-    }
-}
-
-/// Take a consistent snapshot of the database at this current point in time
-/// and then mutate the entries, respecting concurrency guarantees
-pub(super) fn snapshot_and_insert<'a, T: 'a + DerefUnsafeSlice>(
-    kve: &'a KVEStandard,
-    encoder: DoubleEncoder,
-    mut act: Iter<'a, T>,
-) -> StrongActionResult {
-    let mut enc_err = false;
-    let lowtable = kve.get_inner_ref();
-    let key_iter_stat_ok;
-    {
-        key_iter_stat_ok = act.as_ref().chunks_exact(2).all(|kv| unsafe {
-            let key = ucidx!(kv, 0).deref_slice();
-            let value = ucidx!(kv, 1).deref_slice();
-            if compiler::likely(encoder(key, value)) {
-                lowtable.get(key).is_none()
-            } else {
-                enc_err = true;
-                false
-            }
-        });
-    }
-    cfg_test!({
-        // give the caller 10 seconds to do some crap
-        do_sleep!(10 s);
-    });
-    if compiler::unlikely(enc_err) {
-        return compiler::cold_val(StrongActionResult::EncodingError);
-    }
-    if registry::state_okay() {
-        if key_iter_stat_ok {
-            let _kve = kve;
-            let lowtable = lowtable;
-            // fine, the keys were non-existent when we looked at them
-            while let (Some(key), Some(value)) = (act.next(), act.next()) {
-                unsafe {
-                    if let Some(fresh) = lowtable.fresh_entry(SharedSlice::new(key.deref_slice())) {
-                        fresh.insert(SharedSlice::new(value.deref_slice()));
-                    }
-                    // we don't care if some other thread initialized the value we checked
-                    // it. We expected a fresh entry, so that's what we'll check and use
-                }
-            }
-            StrongActionResult::Okay
-        } else {
-            StrongActionResult::OverwriteError
-        }
-    } else {
-        StrongActionResult::ServerError
-    }
-}
diff --git a/server/src/actions/strong/supdate.rs b/server/src/actions/strong/supdate.rs
deleted file mode 100644
index 836b25c6..00000000
--- a/server/src/actions/strong/supdate.rs
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Created on Fri Jul 30 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{
-        actions::strong::StrongActionResult,
-        corestore::SharedSlice,
-        dbnet::prelude::*,
-        kvengine::{DoubleEncoder, KVEStandard},
-        protocol::iter::DerefUnsafeSlice,
-        util::compiler,
-    },
-    core::slice::Iter,
-};
-
-action! {
-    /// Run an `SUPDATE` query
-    ///
-    /// This either returns `Okay` if all the keys were updated, or it returns `Nil`
-    /// or code `1`
-    fn supdate(handle: &crate::corestore::Corestore, con: &mut Connection<C, P>, act: ActionIter<'a>) {
-        let howmany = act.len();
-        ensure_length::<P>(howmany, |size| size & 1 == 0 && size != 0)?;
-        let kve = handle.get_table_with::<P, KVEBlob>()?;
-        if registry::state_okay() {
-            let encoder = kve.get_double_encoder();
-            let outcome = unsafe {
-                // UNSAFE(@ohsayan): the lifetime of `act` ensure ptr validity
-                self::snapshot_and_update(kve, encoder, act.into_inner())
-            };
-            match outcome {
-                StrongActionResult::Okay => con._write_raw(P::RCODE_OKAY).await?,
-                StrongActionResult::Nil => {
-                    // good, it failed because some key didn't exist
-                    return util::err(P::RCODE_NIL);
-                },
-                StrongActionResult::ServerError => return util::err(P::RCODE_SERVER_ERR),
-                StrongActionResult::EncodingError => {
-                    // error we love to hate: encoding error, ugh
-                    return util::err(P::RCODE_ENCODING_ERROR);
-                },
-                StrongActionResult::OverwriteError => unsafe {
-                    // SAFETY check: never the case
-                    impossible!()
-                }
-            }
-        } else {
-            return util::err(P::RCODE_SERVER_ERR);
-        }
-        Ok(())
-    }
-}
-
-/// Take a consistent snapshot of the database at this point in time. Once snapshotting
-/// completes, mutate the entries in place while keeping up with isolation guarantees
-/// `(all_okay, enc_err)`
-pub(super) fn snapshot_and_update<'a, T: 'a + DerefUnsafeSlice>(
-    kve: &'a KVEStandard,
-    encoder: DoubleEncoder,
-    mut act: Iter<'a, T>,
-) -> StrongActionResult {
-    let mut enc_err = false;
-    let mut snapshots = Vec::with_capacity(act.len());
-    let iter_stat_ok;
-    {
-        // snapshot the values at this point in time
-        iter_stat_ok = act.as_ref().chunks_exact(2).all(|kv| unsafe {
-            let key = ucidx!(kv, 0).deref_slice();
-            let value = ucidx!(kv, 1).deref_slice();
-            if compiler::likely(encoder(key, value)) {
-                if let Some(snapshot) = kve.take_snapshot_unchecked(key) {
-                    snapshots.push(snapshot);
-                    true
-                } else {
-                    false
-                }
-            } else {
-                enc_err = true;
-                false
-            }
-        });
-    }
-    cfg_test!({
-        // give the caller 10 seconds to do some crap
-        do_sleep!(10 s);
-    });
-    if compiler::unlikely(enc_err) {
-        return compiler::cold_val(StrongActionResult::EncodingError);
-    }
-    if registry::state_okay() {
-        // uphold consistency
-        if iter_stat_ok {
-            let kve = kve;
-            // good, so all the values existed when we snapshotted them; let's update 'em
-            let mut snap_cc = snapshots.into_iter();
-            let lowtable = kve.get_inner_ref();
-            while let (Some(key), Some(value), Some(snapshot)) =
-                (act.next(), act.next(), snap_cc.next())
-            {
-                unsafe {
-                    // When we snapshotted, we looked at `snapshot`. If the value is still the
-                    // same, then we'll update it. Otherwise, let it be
-                    if let Some(mut mutable) =
-                        lowtable.mut_entry(SharedSlice::new(key.deref_slice()))
-                    {
-                        if mutable.value().eq(&snapshot) {
-                            mutable.insert(SharedSlice::new(value.deref_slice()));
-                        } else {
-                            drop(mutable);
-                        }
-                    }
-                }
-            }
-            StrongActionResult::Okay
-        } else {
-            StrongActionResult::Nil
-        }
-    } else {
-        StrongActionResult::ServerError
-    }
-}
diff --git a/server/src/actions/strong/tests.rs b/server/src/actions/strong/tests.rs
deleted file mode 100644
index 9208fa60..00000000
--- a/server/src/actions/strong/tests.rs
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Created on Fri Jul 30 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-mod sdel_concurrency_tests {
-    use {
-        super::super::sdel,
-        crate::{corestore::SharedSlice, kvengine::KVEStandard},
-        std::{sync::Arc, thread},
-    };
-
-    #[test]
-    fn test_snapshot_okay() {
-        let kve = KVEStandard::init(true, true);
-        kve.upsert(SharedSlice::from("k1"), SharedSlice::from("v1"))
-            .unwrap();
-        kve.upsert(SharedSlice::from("k2"), SharedSlice::from("v2"))
-            .unwrap();
-        let encoder = kve.get_key_encoder();
-        let it = bi!("k1", "k2");
-        let ret = sdel::snapshot_and_del(&kve, encoder, it.as_ref().iter().as_ref().iter());
-        assert!(ret.is_ok());
-    }
-    #[test]
-    fn test_sdel_snapshot_fail_with_t2() {
-        let kve = Arc::new(KVEStandard::init(true, true));
-        let kve1 = kve.clone();
-        let encoder = kve.get_key_encoder();
-        {
-            kve.upsert(SharedSlice::from("k1"), SharedSlice::from("v1"))
-                .unwrap();
-            kve.upsert(SharedSlice::from("k2"), SharedSlice::from("v2"))
-                .unwrap();
-        }
-        let it = bi!("k1", "k2");
-        // sdel will wait 10s for us
-        let t1handle = thread::spawn(move || {
-            sdel::snapshot_and_del(&kve1, encoder, it.as_ref().iter().as_ref().iter())
-        });
-        // we have 10s: we sleep 5 to let the snapshot complete (thread spawning takes time)
-        do_sleep!(5 s);
-        assert!(kve
-            .update(SharedSlice::from("k1"), SharedSlice::from("updated-v1"))
-            .unwrap());
-        // let us join t1
-        let ret = t1handle.join().unwrap();
-        assert!(ret.is_ok());
-        // although we told sdel to delete it, it.as_ref().iter() shouldn't because we externally
-        // updated the value
-        assert!(kve.exists(&SharedSlice::from("k1")).unwrap());
-    }
-}
-
-mod sset_concurrency_tests {
-    use {
-        super::super::sset,
-        crate::{corestore::SharedSlice, kvengine::KVEStandard},
-        std::{sync::Arc, thread},
-    };
-
-    #[test]
-    fn test_snapshot_okay() {
-        let kve = KVEStandard::init(true, true);
-        let encoder = kve.get_double_encoder();
-        let it = bi!("k1", "v1", "k2", "v2");
-        let ret = sset::snapshot_and_insert(&kve, encoder, it.as_ref().iter());
-        assert!(ret.is_ok());
-    }
-    #[test]
-    fn test_sset_snapshot_fail_with_t2() {
-        let kve = Arc::new(KVEStandard::init(true, true));
-        let kve1 = kve.clone();
-        let encoder = kve.get_double_encoder();
-        let it = bi!("k1", "v1", "k2", "v2");
-        // sset will wait 10s for us
-        let t1handle =
-            thread::spawn(move || sset::snapshot_and_insert(&kve1, encoder, it.as_ref().iter()));
-        // we have 10s: we sleep 5 to let the snapshot complete (thread spawning takes time)
-        do_sleep!(5 s);
-        // update the value externally
-        assert!(kve
-            .set(SharedSlice::from("k1"), SharedSlice::from("updated-v1"))
-            .unwrap());
-        // let us join t1
-        let ret = t1handle.join().unwrap();
-        // but set won't fail because someone set it before it did; this is totally
-        // acceptable because we only wanted to set it if it matches the status when
-        // we created a snapshot
-        assert!(ret.is_ok());
-        // although we told sset to set a key, but it shouldn't because we updated it
-        assert_eq!(
-            kve.get(&SharedSlice::from("k1")).unwrap().unwrap().clone(),
-            SharedSlice::from("updated-v1")
-        );
-    }
-}
-
-mod supdate_concurrency_tests {
-    use {
-        super::super::supdate,
-        crate::{corestore::SharedSlice, kvengine::KVEStandard},
-        std::{sync::Arc, thread},
-    };
-
-    #[test]
-    fn test_snapshot_okay() {
-        let kve = KVEStandard::init(true, true);
-        kve.upsert(SharedSlice::from("k1"), SharedSlice::from("v1"))
-            .unwrap();
-        kve.upsert(SharedSlice::from("k2"), SharedSlice::from("v2"))
-            .unwrap();
-        let encoder = kve.get_double_encoder();
-        let it = bi!("k1", "v1", "k2", "v2");
-        let ret = supdate::snapshot_and_update(&kve, encoder, it.as_ref().iter());
-        assert!(ret.is_ok());
-    }
-    #[test]
-    fn test_supdate_snapshot_fail_with_t2() {
-        let kve = Arc::new(KVEStandard::init(true, true));
-        kve.upsert(SharedSlice::from("k1"), SharedSlice::from("v1"))
-            .unwrap();
-        kve.upsert(SharedSlice::from("k2"), SharedSlice::from("v2"))
-            .unwrap();
-        let kve1 = kve.clone();
-        let encoder = kve.get_double_encoder();
-        let it = bi!("k1", "v1", "k2", "v2");
-        // supdate will wait 10s for us
-        let t1handle =
-            thread::spawn(move || supdate::snapshot_and_update(&kve1, encoder, it.as_ref().iter()));
-        // we have 10s: we sleep 5 to let the snapshot complete (thread spawning takes time)
-        do_sleep!(5 s);
-        // lets update the value externally
-        assert!(kve
-            .update(SharedSlice::from("k1"), SharedSlice::from("updated-v1"))
-            .unwrap());
-        // let us join t1
-        let ret = t1handle.join().unwrap();
-        assert!(ret.is_ok());
-        // although we told supdate to update the key, it.as_ref().iter() shouldn't because we updated it
-        // externally; hence our `updated-v1` value should persist
-        assert_eq!(
-            kve.get(&SharedSlice::from("k1")).unwrap().unwrap().clone(),
-            SharedSlice::from("updated-v1")
-        );
-    }
-}
diff --git a/server/src/actions/update.rs b/server/src/actions/update.rs
deleted file mode 100644
index dd08dc47..00000000
--- a/server/src/actions/update.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Created on Mon Aug 17 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # `UPDATE` queries
-//! This module provides functions to work with `UPDATE` queries
-//!
-
-use crate::{corestore::SharedSlice, dbnet::prelude::*};
-
-action!(
-    /// Run an `UPDATE` query
-    fn update(handle: &Corestore, con: &mut Connection<C, P>, mut act: ActionIter<'a>) {
-        ensure_length::<P>(act.len(), |len| len == 2)?;
-        if registry::state_okay() {
-            let did_we = {
-                let writer = handle.get_table_with::<P, KVEBlob>()?;
-                match unsafe {
-                    // UNSAFE(@ohsayan): This is completely safe as we've already checked
-                    // that there are exactly 2 arguments
-                    writer.update(
-                        SharedSlice::new(act.next_unchecked()),
-                        SharedSlice::new(act.next_unchecked()),
-                    )
-                } {
-                    Ok(true) => Some(true),
-                    Ok(false) => Some(false),
-                    Err(()) => None,
-                }
-            };
-            con._write_raw(P::UPDATE_NLUT[did_we]).await?;
-        } else {
-            return util::err(P::RCODE_SERVER_ERR);
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/uset.rs b/server/src/actions/uset.rs
deleted file mode 100644
index 87893f7f..00000000
--- a/server/src/actions/uset.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Created on Fri Sep 25 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{
-    corestore::SharedSlice, dbnet::prelude::*, kvengine::encoding::ENCODING_LUT_ITER_PAIR,
-    queryengine::ActionIter, util::compiler,
-};
-
-action!(
-    /// Run an `USET` query
-    ///
-    /// This is like "INSERT or UPDATE"
-    fn uset(
-        handle: &crate::corestore::Corestore,
-        con: &mut Connection<C, P>,
-        mut act: ActionIter<'a>,
-    ) {
-        let howmany = act.len();
-        ensure_length::<P>(howmany, |size| size & 1 == 0 && size != 0)?;
-        let kve = handle.get_table_with::<P, KVEBlob>()?;
-        let encoding_is_okay = ENCODING_LUT_ITER_PAIR[kve.get_encoding_tuple()](&act);
-        if compiler::likely(encoding_is_okay) {
-            if registry::state_okay() {
-                while let (Some(key), Some(val)) = (act.next(), act.next()) {
-                    kve.upsert_unchecked(SharedSlice::new(key), SharedSlice::new(val));
-                }
-                con.write_usize(howmany / 2).await?;
-            } else {
-                return util::err(P::RCODE_SERVER_ERR);
-            }
-        } else {
-            return util::err(P::RCODE_ENCODING_ERROR);
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/actions/whereami.rs b/server/src/actions/whereami.rs
deleted file mode 100644
index a9655c98..00000000
--- a/server/src/actions/whereami.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Created on Fri Nov 12 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::dbnet::prelude::*;
-
-action! {
-    fn whereami(store: &Corestore, con: &mut Connection<C, P>, act: ActionIter<'a>) {
-        ensure_length::<P>(act.len(), |len| len == 0)?;
-        match store.get_ids() {
-            (Some(ks), Some(tbl)) =>  {
-                con.write_typed_non_null_array_header(2, b'+').await?;
-                con.write_typed_non_null_array_element(ks).await?;
-                con.write_typed_non_null_array_element(tbl).await?;
-            },
-            (Some(ks), None) => {
-                con.write_typed_non_null_array_header(1, b'+').await?;
-                con.write_typed_non_null_array_element(ks).await?;
-            },
-            _ => unsafe { impossible!() }
-        }
-        Ok(())
-    }
-}
diff --git a/server/src/admin/mksnap.rs b/server/src/admin/mksnap.rs
deleted file mode 100644
index b0031ad6..00000000
--- a/server/src/admin/mksnap.rs
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Created on Tue Oct 13 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{dbnet::prelude::*, kvengine::encoding, storage::v1::sengine::SnapshotActionResult},
-    core::str,
-    std::path::{Component, PathBuf},
-};
-
-action!(
-    /// Create a snapshot
-    ///
-    fn mksnap(
-        handle: &crate::corestore::Corestore,
-        con: &mut Connection<C, P>,
-        mut act: ActionIter<'a>,
-    ) {
-        let engine = handle.get_engine();
-        if act.is_empty() {
-            // traditional mksnap
-            match engine.mksnap(handle.clone_store()).await {
-                SnapshotActionResult::Ok => con._write_raw(P::RCODE_OKAY).await?,
-                SnapshotActionResult::Failure => return util::err(P::RCODE_SERVER_ERR),
-                SnapshotActionResult::Disabled => return util::err(P::RSTRING_SNAPSHOT_DISABLED),
-                SnapshotActionResult::Busy => return util::err(P::RSTRING_SNAPSHOT_BUSY),
-                _ => unsafe { impossible!() },
-            }
-        } else if act.len() == 1 {
-            // remote snapshot, let's see what we've got
-            let name = unsafe {
-                // SAFETY: We have already checked that there is one item
-                act.next_unchecked_bytes()
-            };
-            if !encoding::is_utf8(&name) {
-                return util::err(P::RCODE_ENCODING_ERROR);
-            }
-
-            // SECURITY: Check for directory traversal syntax
-            let st = unsafe {
-                // SAFETY: We have already checked for UTF-8 validity
-                str::from_utf8_unchecked(&name)
-            };
-            let path = PathBuf::from(st);
-            let illegal_snapshot = path
-                .components()
-                .filter(|dir| {
-                    // Sanitize snapshot name, to avoid directory traversal attacks
-                    // If the snapshot name has any root directory or parent directory, then
-                    // we'll allow it to pass through this adaptor.
-                    // As a result, this iterator will give us a count of the 'bad' components
-                    dir == &Component::RootDir || dir == &Component::ParentDir
-                })
-                .count()
-                != 0;
-            if illegal_snapshot {
-                return util::err(P::RSTRING_SNAPSHOT_ILLEGAL_NAME);
-            }
-
-            // now make the snapshot
-            match engine.mkrsnap(&name, handle.clone_store()).await {
-                SnapshotActionResult::Ok => con._write_raw(P::RCODE_OKAY).await?,
-                SnapshotActionResult::Failure => return util::err(P::RCODE_SERVER_ERR),
-                SnapshotActionResult::Busy => return util::err(P::RSTRING_SNAPSHOT_BUSY),
-                SnapshotActionResult::AlreadyExists => {
-                    return util::err(P::RSTRING_SNAPSHOT_DUPLICATE)
-                }
-                _ => unsafe { impossible!() },
-            }
-        } else {
-            return util::err(P::RCODE_ACTION_ERR);
-        }
-        Ok(())
-    }
-);
diff --git a/server/src/admin/mod.rs b/server/src/admin/mod.rs
deleted file mode 100644
index 1791b211..00000000
--- a/server/src/admin/mod.rs
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Created on Tue Nov 03 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! Modules for administration of Skytable
-
-pub mod mksnap;
-pub mod sys;
diff --git a/server/src/admin/sys.rs b/server/src/admin/sys.rs
deleted file mode 100644
index d517a120..00000000
--- a/server/src/admin/sys.rs
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Created on Tue Mar 29 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{corestore::booltable::BoolTable, dbnet::prelude::*, storage::v1::interface::DIR_ROOT},
-    libsky::VERSION,
-};
-
-const INFO: &[u8] = b"info";
-const METRIC: &[u8] = b"metric";
-const INFO_PROTOCOL: &[u8] = b"protocol";
-const INFO_PROTOVER: &[u8] = b"protover";
-const INFO_VERSION: &[u8] = b"version";
-const METRIC_HEALTH: &[u8] = b"health";
-const METRIC_STORAGE_USAGE: &[u8] = b"storage";
-const ERR_UNKNOWN_PROPERTY: &[u8] = b"!16\nunknown-property\n";
-const ERR_UNKNOWN_METRIC: &[u8] = b"!14\nunknown-metric\n";
-
-const HEALTH_TABLE: BoolTable<&str> = BoolTable::new("good", "critical");
-
-action! {
-    fn sys(_handle: &Corestore, con: &mut Connection<C, P>, iter: ActionIter<'_>) {
-        let mut iter = iter;
-        ensure_boolean_or_aerr::<P>(iter.len() == 2)?;
-        match unsafe { iter.next_lowercase_unchecked() }.as_ref() {
-            INFO => sys_info(con, &mut iter).await,
-            METRIC => sys_metric(con, &mut iter).await,
-            _ => util::err(P::RCODE_UNKNOWN_ACTION),
-        }
-    }
-    fn sys_info(con: &mut Connection<C, P>, iter: &mut ActionIter<'_>) {
-        match unsafe { iter.next_lowercase_unchecked() }.as_ref() {
-            INFO_PROTOCOL => con.write_string(P::PROTOCOL_VERSIONSTRING).await?,
-            INFO_PROTOVER => con.write_float(P::PROTOCOL_VERSION).await?,
-            INFO_VERSION => con.write_string(VERSION).await?,
-            _ => return util::err(ERR_UNKNOWN_PROPERTY),
-        }
-        Ok(())
-    }
-    fn sys_metric(con: &mut Connection<C, P>, iter: &mut ActionIter<'_>) {
-        match unsafe { iter.next_lowercase_unchecked() }.as_ref() {
-            METRIC_HEALTH => {
-                con.write_string(HEALTH_TABLE[registry::state_okay()]).await?
-            }
-            METRIC_STORAGE_USAGE => {
-                match util::os::dirsize(DIR_ROOT) {
-                    Ok(size) => con.write_int64(size).await?,
-                    Err(e) => {
-                        log::error!("Failed to get storage usage with: {e}");
-                        return util::err(P::RCODE_SERVER_ERR);
-                    },
-                }
-            }
-            _ => return util::err(ERR_UNKNOWN_METRIC),
-        }
-        Ok(())
-    }
-}
diff --git a/server/src/arbiter.rs b/server/src/arbiter.rs
deleted file mode 100644
index 53ab967b..00000000
--- a/server/src/arbiter.rs
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Created on Sat Jun 26 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{
-        auth::AuthProvider,
-        config::{ConfigurationSet, SnapshotConfig, SnapshotPref},
-        corestore::Corestore,
-        dbnet,
-        diskstore::flock::FileLock,
-        services,
-        storage::v1::sengine::SnapshotEngine,
-        util::{
-            error::{Error, SkyResult},
-            os::TerminationSignal,
-        },
-    },
-    std::{sync::Arc, thread::sleep},
-    tokio::{
-        sync::{
-            broadcast,
-            mpsc::{self, Sender},
-        },
-        task::{self, JoinHandle},
-        time::Duration,
-    },
-};
-
-const TERMSIG_THRESHOLD: usize = 3;
-
-/// Start the server waiting for incoming connections or a termsig
-pub async fn run(
-    ConfigurationSet {
-        ports,
-        bgsave,
-        snapshot,
-        maxcon,
-        auth,
-        protocol,
-        ..
-    }: ConfigurationSet,
-    restore_filepath: Option<String>,
-) -> SkyResult<Corestore> {
-    // Intialize the broadcast channel
-    let (signal, _) = broadcast::channel(1);
-    let engine = match &snapshot {
-        SnapshotConfig::Enabled(SnapshotPref { atmost, .. }) => SnapshotEngine::new(*atmost),
-        SnapshotConfig::Disabled => SnapshotEngine::new_disabled(),
-    };
-    let engine = Arc::new(engine);
-    // restore data
-    services::restore_data(restore_filepath)
-        .map_err(|e| Error::ioerror_extra(e, "restoring data from backup"))?;
-    // init the store
-    let db = Corestore::init_with_snapcfg(engine.clone())?;
-    // refresh the snapshotengine state
-    engine.parse_dir()?;
-    let auth_provider = match auth.origin_key {
-        Some(key) => {
-            let authref = db.get_store().setup_auth();
-            AuthProvider::new(authref, Some(key.into_inner()))
-        }
-        None => AuthProvider::new_disabled(),
-    };
-
-    // initialize the background services
-    let bgsave_handle = tokio::spawn(services::bgsave::bgsave_scheduler(
-        db.clone(),
-        bgsave,
-        signal.subscribe(),
-    ));
-    let snapshot_handle = tokio::spawn(services::snapshot::snapshot_service(
-        engine,
-        db.clone(),
-        snapshot,
-        signal.subscribe(),
-    ));
-
-    // bind to signals
-    let termsig =
-        TerminationSignal::init().map_err(|e| Error::ioerror_extra(e, "binding to signals"))?;
-    // start the server (single or multiple listeners)
-    let mut server = dbnet::connect(
-        ports,
-        protocol,
-        maxcon,
-        db.clone(),
-        auth_provider,
-        signal.clone(),
-    )
-    .await?;
-
-    tokio::select! {
-        _ = server.run_server() => {},
-        _ = termsig => {}
-    }
-
-    log::info!("Signalling all workers to shut down");
-    // drop the signal and let others exit
-    drop(signal);
-    server.finish_with_termsig().await;
-
-    // wait for the background services to terminate
-    let _ = snapshot_handle.await;
-    let _ = bgsave_handle.await;
-    Ok(db)
-}
-
-fn spawn_task(tx: Sender<bool>, db: Corestore, do_sleep: bool) -> JoinHandle<()> {
-    task::spawn_blocking(move || {
-        if do_sleep {
-            log::info!("Waiting for 10 seconds before retrying ...");
-            sleep(Duration::from_secs(10));
-        }
-        let ret = match crate::services::bgsave::run_bgsave(&db) {
-            Ok(()) => {
-                log::info!("Save before termination successful");
-                true
-            }
-            Err(e) => {
-                log::error!("Failed to run save on termination: {e}");
-                false
-            }
-        };
-        tx.blocking_send(ret).expect("Receiver dropped");
-    })
-}
-
-pub fn finalize_shutdown(corestore: Corestore, pid_file: FileLock) {
-    assert_eq!(
-        corestore.strong_count(),
-        1,
-        "Correctness error. finalize_shutdown called before dropping server runtime"
-    );
-    let rt = tokio::runtime::Builder::new_multi_thread()
-        .thread_name("server-final")
-        .enable_all()
-        .build()
-        .unwrap();
-    let dbc = corestore.clone();
-    let mut okay: bool = rt.block_on(async move {
-        let db = dbc;
-        let (tx, mut rx) = mpsc::channel::<bool>(1);
-        spawn_task(tx.clone(), db.clone(), false);
-        let spawn_again = || {
-            // we failed, so we better sleep
-            // now spawn it again to see the state
-            spawn_task(tx.clone(), db.clone(), true)
-        };
-        let mut threshold = TERMSIG_THRESHOLD;
-        loop {
-            let termsig = match TerminationSignal::init().map_err(|e| e.to_string()) {
-                Ok(sig) => sig,
-                Err(e) => {
-                    log::error!("Failed to bind to signal with error: {e}");
-                    crate::exit_error();
-                }
-            };
-            tokio::select! {
-                ret = rx.recv() => {
-                    if ret.unwrap() {
-                        // that's good to go
-                        break true;
-                    } else {
-                        spawn_again();
-                    }
-                }
-                _ = termsig => {
-                    threshold -= 1;
-                    if threshold == 0 {
-                        log::error!("Termination signal received but failed to flush data. Quitting because threshold exceeded");
-                        break false;
-                    } else {
-                        log::error!("Termination signal received but failed to flush data. Threshold is at {threshold}");
-                        continue;
-                    }
-                },
-            }
-        }
-    });
-    okay &= services::pre_shutdown_cleanup(pid_file, Some(corestore.get_store()));
-    if okay {
-        log::info!("Goodbye :)");
-    } else {
-        log::error!("Didn't terminate successfully");
-        crate::exit_error();
-    }
-}
diff --git a/server/src/auth/keys.rs b/server/src/auth/keys.rs
deleted file mode 100644
index 9eccd89c..00000000
--- a/server/src/auth/keys.rs
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Created on Mon Feb 21 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::provider::{Authkey, AUTHKEY_SIZE},
-    crate::corestore::array::Array,
-    base64::{
-        alphabet::BCRYPT,
-        engine::{GeneralPurpose, GeneralPurposeConfig},
-        Engine,
-    },
-};
-
-type AuthkeyArray = Array<u8, { AUTHKEY_SIZE }>;
-const RAN_BYTES_SIZE: usize = 40;
-const BASE64: GeneralPurpose = GeneralPurpose::new(&BCRYPT, GeneralPurposeConfig::new());
-
-/// Return a "human readable key" and the "authbytes" that can be stored
-/// safely. To do this:
-/// - Generate 64 random bytes
-/// - Encode that into base64. This is the client key
-/// - Hash the key using rcrypt. This is the server key that
-/// will be stored
-pub fn generate_full() -> (String, Authkey) {
-    let mut bytes: [u8; RAN_BYTES_SIZE] = [0u8; RAN_BYTES_SIZE];
-    openssl::rand::rand_bytes(&mut bytes).unwrap();
-    let ret = BASE64.encode(&bytes);
-    let hash = rcrypt::hash(&ret, rcrypt::DEFAULT_COST).unwrap();
-    let store_in_db = unsafe {
-        let mut array = AuthkeyArray::new();
-        // we guarantee that the size is equal to 40
-        array.extend_from_slice_unchecked(&hash);
-        array.into_array_unchecked()
-    };
-    (ret, store_in_db)
-}
-
-/// Verify a "human readable key" against the provided "authbytes"
-pub fn verify_key(input: &[u8], hash: &[u8]) -> Option<bool> {
-    rcrypt::verify(input, hash).ok()
-}
diff --git a/server/src/auth/mod.rs b/server/src/auth/mod.rs
deleted file mode 100644
index c60cdc8c..00000000
--- a/server/src/auth/mod.rs
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Created on Mon Feb 21 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
- * For our authn/authz, we have two important keys:
- * - The origin key: This is the key saved in the configuration file that can also be
- * used as the "recovery key" in the event the "root key" is lost. To claim the root
- * account, one needs this key. This is a fixed width key with 40 characters
- * - The root key: This is the superuser key that can be used to create/deny other
- * accounts. On claiming the root account, this key is issued
- *
- * When the root account is claimed, it can be used to create "standard users". standard
- * users have access to everything but the ability to create/revoke other users
-*/
-
-mod keys;
-pub mod provider;
-pub use provider::{AuthProvider, Authmap};
-
-#[cfg(test)]
-mod tests;
-
-use crate::dbnet::prelude::*;
-
-const AUTH_CLAIM: &[u8] = b"claim";
-const AUTH_LOGIN: &[u8] = b"login";
-const AUTH_LOGOUT: &[u8] = b"logout";
-const AUTH_ADDUSER: &[u8] = b"adduser";
-const AUTH_DELUSER: &[u8] = b"deluser";
-const AUTH_RESTORE: &[u8] = b"restore";
-const AUTH_LISTUSER: &[u8] = b"listuser";
-const AUTH_WHOAMI: &[u8] = b"whoami";
-
-action! {
-    /// Handle auth. Should have passed the `auth` token
-    fn auth(
-        con: &mut Connection<C, P>,
-        auth: &mut AuthProviderHandle,
-        iter: ActionIter<'_>
-    ) {
-        let mut iter = iter;
-        match iter.next_lowercase().unwrap_or_aerr::<P>()?.as_ref() {
-            AUTH_LOGIN => self::_auth_login(con, auth, &mut iter).await,
-            AUTH_CLAIM => self::_auth_claim(con, auth, &mut iter).await,
-            AUTH_ADDUSER => {
-                ensure_boolean_or_aerr::<P>(iter.len() == 1)?; // just the username
-                let username = unsafe { iter.next_unchecked() };
-                let key = auth.provider_mut().claim_user::<P>(username)?;
-                con.write_string(&key).await?;
-                Ok(())
-            }
-            AUTH_LOGOUT => {
-                ensure_boolean_or_aerr::<P>(iter.is_empty())?; // nothing else
-                auth.provider_mut().logout::<P>()?;
-                auth.set_unauth();
-                con._write_raw(P::RCODE_OKAY).await?;
-                Ok(())
-            }
-            AUTH_DELUSER => {
-                ensure_boolean_or_aerr::<P>(iter.len() == 1)?; // just the username
-                auth.provider_mut().delete_user::<P>(unsafe { iter.next_unchecked() })?;
-                con._write_raw(P::RCODE_OKAY).await?;
-                Ok(())
-            }
-            AUTH_RESTORE => self::auth_restore(con, auth, &mut iter).await,
-            AUTH_LISTUSER => self::auth_listuser(con, auth, &mut iter).await,
-            AUTH_WHOAMI => self::auth_whoami(con, auth, &mut iter).await,
-            _ => util::err(P::RCODE_UNKNOWN_ACTION),
-        }
-    }
-    fn auth_whoami(con: &mut Connection<C, P>, auth: &mut AuthProviderHandle, iter: &mut ActionIter<'_>) {
-        ensure_boolean_or_aerr::<P>(ActionIter::is_empty(iter))?;
-        con.write_string(&auth.provider().whoami::<P>()?).await?;
-        Ok(())
-    }
-    fn auth_listuser(con: &mut Connection<C, P>, auth: &mut AuthProviderHandle, iter: &mut ActionIter<'_>) {
-        ensure_boolean_or_aerr::<P>(ActionIter::is_empty(iter))?;
-        let usernames = auth.provider().collect_usernames::<P>()?;
-        con.write_typed_non_null_array_header(usernames.len(), b'+').await?;
-        for username in usernames {
-            con.write_typed_non_null_array_element(username.as_bytes()).await?;
-        }
-        Ok(())
-    }
-    fn auth_restore(con: &mut Connection<C, P>, auth: &mut AuthProviderHandle, iter: &mut ActionIter<'_>) {
-        let newkey = match iter.len() {
-            1 => {
-                // so this fella thinks they're root
-                auth.provider().regenerate::<P>(
-                    unsafe { iter.next_unchecked() }
-                )?
-            }
-            2 => {
-                // so this fella is giving us the origin key
-                let origin = unsafe { iter.next_unchecked() };
-                let id = unsafe { iter.next_unchecked() };
-                auth.provider().regenerate_using_origin::<P>(origin, id)?
-            }
-            _ => return util::err(P::RCODE_ACTION_ERR),
-        };
-        con.write_string(&newkey).await?;
-        Ok(())
-    }
-    fn _auth_claim(con: &mut Connection<C, P>, auth: &mut AuthProviderHandle, iter: &mut ActionIter<'_>) {
-        ensure_boolean_or_aerr::<P>(iter.len() == 1)?; // just the origin key
-        let origin_key = unsafe { iter.next_unchecked() };
-        let key = auth.provider_mut().claim_root::<P>(origin_key)?;
-        auth.set_auth();
-        con.write_string(&key).await?;
-        Ok(())
-    }
-    /// Handle a login operation only. The **`login` token is expected to be present**
-    fn auth_login_only(
-        con: &mut Connection<C, P>,
-        auth: &mut AuthProviderHandle,
-        iter: ActionIter<'_>
-    ) {
-        let mut iter = iter;
-        match iter.next_lowercase().unwrap_or_aerr::<P>()?.as_ref() {
-            AUTH_LOGIN => self::_auth_login(con, auth, &mut iter).await,
-            AUTH_CLAIM => self::_auth_claim(con, auth, &mut iter).await,
-            AUTH_RESTORE => self::auth_restore(con, auth, &mut iter).await,
-            AUTH_WHOAMI => self::auth_whoami(con, auth, &mut iter).await,
-            _ => util::err(P::AUTH_CODE_PERMS),
-        }
-    }
-    fn _auth_login(con: &mut Connection<C, P>, auth: &mut AuthProviderHandle, iter: &mut ActionIter<'_>) {
-        // sweet, where's our username and password
-        ensure_boolean_or_aerr::<P>(iter.len() == 2)?; // just the uname and pass
-        let (username, password) = unsafe { (iter.next_unchecked(), iter.next_unchecked()) };
-        auth.provider_mut().login::<P>(username, password)?;
-        auth.set_auth();
-        con._write_raw(P::RCODE_OKAY).await?;
-        Ok(())
-    }
-}
diff --git a/server/src/auth/provider.rs b/server/src/auth/provider.rs
deleted file mode 100644
index 19495138..00000000
--- a/server/src/auth/provider.rs
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Created on Sun Mar 06 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::keys,
-    crate::{
-        actions::{ActionError, ActionResult},
-        corestore::{array::Array, htable::Coremap},
-        protocol::interface::ProtocolSpec,
-        util::err,
-    },
-    std::sync::Arc,
-};
-
-// constants
-/// Size of an authn key in bytes
-pub const AUTHKEY_SIZE: usize = 40;
-/// Size of an authn ID in bytes
-pub const AUTHID_SIZE: usize = 40;
-
-pub mod testsuite_data {
-    #![allow(unused)]
-    //! Temporary users created by the testsuite in debug mode
-    pub const TESTSUITE_ROOT_USER: &str = "root";
-    pub const TESTSUITE_TEST_USER: &str = "testuser";
-    pub const TESTSUITE_ROOT_TOKEN: &str = "XUOdVKhEONnnGwNwT7WeLqbspDgVtKex0/nwFwBSW7XJxioHwpg6H.";
-    pub const TESTSUITE_TEST_TOKEN: &str = "mpobAB7EY8vnBs70d/..h1VvfinKIeEJgt1rg4wUkwF6aWCvGGR9le";
-}
-
-uninit_array! {
-    const USER_ROOT_ARRAY: [u8; 40] = [b'r', b'o', b'o', b't'];
-}
-/// The root user
-const USER_ROOT: AuthID = unsafe { AuthID::from_const(USER_ROOT_ARRAY, 4) };
-
-/// An authn ID
-type AuthID = Array<u8, AUTHID_SIZE>;
-/// An authn key
-pub type Authkey = [u8; AUTHKEY_SIZE];
-/// Authmap
-pub type Authmap = Arc<Coremap<AuthID, Authkey>>;
-
-/// The authn/authz provider
-///
-pub struct AuthProvider {
-    origin: Option<Authkey>,
-    /// the current user
-    whoami: Option<AuthID>,
-    /// a map of users
-    authmap: Authmap,
-}
-
-impl AuthProvider {
-    fn _new(authmap: Authmap, whoami: Option<AuthID>, origin: Option<Authkey>) -> Self {
-        Self {
-            authmap,
-            whoami,
-            origin,
-        }
-    }
-    /// New provider with no origin-key
-    pub fn new_disabled() -> Self {
-        Self::_new(Default::default(), None, None)
-    }
-    /// New provider with zero users
-    #[cfg(test)]
-    pub fn new_blank(origin: Option<Authkey>) -> Self {
-        Self::_new(Default::default(), None, origin)
-    }
-    /// New provider with users from the provided map
-    ///
-    /// ## Test suite
-    /// The testsuite creates users `root` and `testuser`; this **does not** apply to
-    /// release mode
-    pub fn new(authmap: Arc<Coremap<AuthID, Authkey>>, origin: Option<Authkey>) -> Self {
-        let slf = Self::_new(authmap, None, origin);
-        #[cfg(debug_assertions)]
-        {
-            // 'root' user in test mode
-            slf.authmap.true_if_insert(
-                AuthID::try_from_slice(testsuite_data::TESTSUITE_ROOT_USER).unwrap(),
-                [
-                    172, 143, 117, 169, 158, 156, 33, 106, 139, 107, 20, 106, 91, 219, 34, 157, 98,
-                    147, 142, 91, 222, 238, 205, 120, 72, 171, 90, 218, 147, 2, 75, 67, 44, 108,
-                    185, 124, 55, 40, 156, 252,
-                ],
-            );
-            // 'testuser' user in test mode
-            slf.authmap.true_if_insert(
-                AuthID::try_from_slice(testsuite_data::TESTSUITE_TEST_USER).unwrap(),
-                [
-                    172, 183, 60, 221, 53, 240, 231, 217, 113, 112, 98, 16, 109, 62, 235, 95, 184,
-                    107, 130, 139, 43, 197, 40, 31, 176, 127, 185, 22, 172, 124, 39, 225, 124, 71,
-                    193, 115, 176, 162, 239, 93,
-                ],
-            );
-        }
-        slf
-    }
-    pub const fn is_enabled(&self) -> bool {
-        matches!(self.origin, Some(_))
-    }
-    pub fn claim_root<P: ProtocolSpec>(&mut self, origin_key: &[u8]) -> ActionResult<String> {
-        self.verify_origin::<P>(origin_key)?;
-        // the origin key was good, let's try claiming root
-        let (key, store) = keys::generate_full();
-        if self.authmap.true_if_insert(USER_ROOT, store) {
-            // claimed, sweet, log them in
-            self.whoami = Some(USER_ROOT);
-            Ok(key)
-        } else {
-            err(P::AUTH_ERROR_ALREADYCLAIMED)
-        }
-    }
-    fn are_you_root<P: ProtocolSpec>(&self) -> ActionResult<bool> {
-        self.ensure_enabled::<P>()?;
-        match self.whoami.as_ref().map(|v| v.eq(&USER_ROOT)) {
-            Some(v) => Ok(v),
-            None => err(P::AUTH_CODE_PERMS),
-        }
-    }
-    pub fn claim_user<P: ProtocolSpec>(&self, claimant: &[u8]) -> ActionResult<String> {
-        self.ensure_root::<P>()?;
-        self._claim_user::<P>(claimant)
-    }
-    pub fn _claim_user<P: ProtocolSpec>(&self, claimant: &[u8]) -> ActionResult<String> {
-        let (key, store) = keys::generate_full();
-        if self
-            .authmap
-            .true_if_insert(Self::try_auth_id::<P>(claimant)?, store)
-        {
-            Ok(key)
-        } else {
-            err(P::AUTH_ERROR_ALREADYCLAIMED)
-        }
-    }
-    pub fn login<P: ProtocolSpec>(&mut self, account: &[u8], token: &[u8]) -> ActionResult<()> {
-        self.ensure_enabled::<P>()?;
-        match self
-            .authmap
-            .get(account)
-            .map(|token_hash| keys::verify_key(token, token_hash.as_slice()))
-        {
-            Some(Some(true)) => {
-                // great, authenticated
-                self.whoami = Some(Self::try_auth_id::<P>(account)?);
-                Ok(())
-            }
-            _ => {
-                // either the password was wrong, or the username was wrong
-                err(P::AUTH_CODE_BAD_CREDENTIALS)
-            }
-        }
-    }
-    pub fn regenerate_using_origin<P: ProtocolSpec>(
-        &self,
-        origin: &[u8],
-        account: &[u8],
-    ) -> ActionResult<String> {
-        self.verify_origin::<P>(origin)?;
-        self._regenerate::<P>(account)
-    }
-    pub fn regenerate<P: ProtocolSpec>(&self, account: &[u8]) -> ActionResult<String> {
-        self.ensure_root::<P>()?;
-        self._regenerate::<P>(account)
-    }
-    /// Regenerate the token for the given user. This returns a new token
-    fn _regenerate<P: ProtocolSpec>(&self, account: &[u8]) -> ActionResult<String> {
-        let id = Self::try_auth_id::<P>(account)?;
-        let (key, store) = keys::generate_full();
-        if self.authmap.true_if_update(id, store) {
-            Ok(key)
-        } else {
-            err(P::AUTH_CODE_BAD_CREDENTIALS)
-        }
-    }
-    fn try_auth_id<P: ProtocolSpec>(authid: &[u8]) -> ActionResult<AuthID> {
-        if authid.is_ascii() && authid.len() <= AUTHID_SIZE {
-            Ok(unsafe {
-                // We just verified the length
-                AuthID::from_slice(authid)
-            })
-        } else {
-            err(P::AUTH_ERROR_ILLEGAL_USERNAME)
-        }
-    }
-    pub fn logout<P: ProtocolSpec>(&mut self) -> ActionResult<()> {
-        self.ensure_enabled::<P>()?;
-        self.whoami
-            .take()
-            .map(|_| ())
-            .ok_or(ActionError::ActionError(P::AUTH_CODE_PERMS))
-    }
-    fn ensure_enabled<P: ProtocolSpec>(&self) -> ActionResult<()> {
-        self.origin
-            .as_ref()
-            .map(|_| ())
-            .ok_or(ActionError::ActionError(P::AUTH_ERROR_DISABLED))
-    }
-    pub fn verify_origin<P: ProtocolSpec>(&self, origin: &[u8]) -> ActionResult<()> {
-        if self.get_origin::<P>()?.eq(origin) {
-            Ok(())
-        } else {
-            err(P::AUTH_CODE_BAD_CREDENTIALS)
-        }
-    }
-    fn get_origin<P: ProtocolSpec>(&self) -> ActionResult<&Authkey> {
-        match self.origin.as_ref() {
-            Some(key) => Ok(key),
-            None => err(P::AUTH_ERROR_DISABLED),
-        }
-    }
-    fn ensure_root<P: ProtocolSpec>(&self) -> ActionResult<()> {
-        if self.are_you_root::<P>()? {
-            Ok(())
-        } else {
-            err(P::AUTH_CODE_PERMS)
-        }
-    }
-    pub fn delete_user<P: ProtocolSpec>(&self, user: &[u8]) -> ActionResult<()> {
-        self.ensure_root::<P>()?;
-        if user.eq(&USER_ROOT) {
-            // can't delete root!
-            err(P::AUTH_ERROR_FAILED_TO_DELETE_USER)
-        } else if self.authmap.true_if_removed(user) {
-            Ok(())
-        } else {
-            err(P::AUTH_CODE_BAD_CREDENTIALS)
-        }
-    }
-    /// List all the users
-    pub fn collect_usernames<P: ProtocolSpec>(&self) -> ActionResult<Vec<String>> {
-        self.ensure_root::<P>()?;
-        Ok(self
-            .authmap
-            .iter()
-            .map(|kv| String::from_utf8_lossy(kv.key()).to_string())
-            .collect())
-    }
-    /// Return the AuthID of the current user
-    pub fn whoami<P: ProtocolSpec>(&self) -> ActionResult<String> {
-        self.ensure_enabled::<P>()?;
-        self.whoami
-            .as_ref()
-            .map(|v| String::from_utf8_lossy(v).to_string())
-            .ok_or(ActionError::ActionError(P::AUTH_CODE_PERMS))
-    }
-}
-
-impl Clone for AuthProvider {
-    fn clone(&self) -> Self {
-        Self {
-            authmap: self.authmap.clone(),
-            whoami: None,
-            origin: self.origin,
-        }
-    }
-}
diff --git a/server/src/auth/tests.rs b/server/src/auth/tests.rs
deleted file mode 100644
index 1c773e4d..00000000
--- a/server/src/auth/tests.rs
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Created on Tue Feb 22 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-mod keys {
-    use super::super::keys::{generate_full, verify_key};
-
-    #[test]
-    fn test_verify_key() {
-        let (key, store) = generate_full();
-        assert!(verify_key(key.as_bytes(), &store).unwrap());
-    }
-}
-
-mod authn {
-    use crate::{
-        actions::ActionError,
-        auth::AuthProvider,
-        protocol::{interface::ProtocolSpec, Skyhash2},
-    };
-
-    const ORIG: &[u8; 40] = b"c4299d190fb9a00626797fcc138c56eae9971664";
-
-    #[test]
-    fn claim_root_okay() {
-        let mut provider = AuthProvider::new_blank(Some(*ORIG));
-        let _ = provider.claim_root::<Skyhash2>(ORIG).unwrap();
-    }
-    #[test]
-    fn claim_root_wrongkey() {
-        let mut provider = AuthProvider::new_blank(Some(*ORIG));
-        let claim_err = provider.claim_root::<Skyhash2>(&ORIG[1..]).unwrap_err();
-        assert_eq!(
-            claim_err,
-            ActionError::ActionError(Skyhash2::AUTH_CODE_BAD_CREDENTIALS)
-        );
-    }
-    #[test]
-    fn claim_root_disabled() {
-        let mut provider = AuthProvider::new_disabled();
-        assert_eq!(
-            provider.claim_root::<Skyhash2>(b"abcd").unwrap_err(),
-            ActionError::ActionError(Skyhash2::AUTH_ERROR_DISABLED)
-        );
-    }
-    #[test]
-    fn claim_root_already_claimed() {
-        let mut provider = AuthProvider::new_blank(Some(*ORIG));
-        let _ = provider.claim_root::<Skyhash2>(ORIG).unwrap();
-        assert_eq!(
-            provider.claim_root::<Skyhash2>(ORIG).unwrap_err(),
-            ActionError::ActionError(Skyhash2::AUTH_ERROR_ALREADYCLAIMED)
-        );
-    }
-    #[test]
-    fn claim_user_okay_with_login() {
-        let mut provider = AuthProvider::new_blank(Some(*ORIG));
-        // claim root
-        let rootkey = provider.claim_root::<Skyhash2>(ORIG).unwrap();
-        // login as root
-        provider
-            .login::<Skyhash2>(b"root", rootkey.as_bytes())
-            .unwrap();
-        // claim user
-        let _ = provider.claim_user::<Skyhash2>(b"sayan").unwrap();
-    }
-
-    #[test]
-    fn claim_user_fail_not_root_with_login() {
-        let mut provider = AuthProvider::new_blank(Some(*ORIG));
-        // claim root
-        let rootkey = provider.claim_root::<Skyhash2>(ORIG).unwrap();
-        // login as root
-        provider
-            .login::<Skyhash2>(b"root", rootkey.as_bytes())
-            .unwrap();
-        // claim user
-        let userkey = provider.claim_user::<Skyhash2>(b"user").unwrap();
-        // login as user
-        provider
-            .login::<Skyhash2>(b"user", userkey.as_bytes())
-            .unwrap();
-        // now try to claim an user being a non-root account
-        assert_eq!(
-            provider.claim_user::<Skyhash2>(b"otheruser").unwrap_err(),
-            ActionError::ActionError(Skyhash2::AUTH_CODE_PERMS)
-        );
-    }
-    #[test]
-    fn claim_user_fail_anonymous() {
-        let mut provider = AuthProvider::new_blank(Some(*ORIG));
-        // claim root
-        let _ = provider.claim_root::<Skyhash2>(ORIG).unwrap();
-        // logout
-        provider.logout::<Skyhash2>().unwrap();
-        // try to claim as an anonymous user
-        assert_eq!(
-            provider.claim_user::<Skyhash2>(b"newuser").unwrap_err(),
-            ActionError::ActionError(Skyhash2::AUTH_CODE_PERMS)
-        );
-    }
-}
diff --git a/server/src/blueql/ast.rs b/server/src/blueql/ast.rs
deleted file mode 100644
index 27d3a78d..00000000
--- a/server/src/blueql/ast.rs
+++ /dev/null
@@ -1,485 +0,0 @@
-/*
- * Created on Tue Jun 14 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{
-        error::{LangError, LangResult},
-        lexer::{Keyword, Lexer, Token, Type, TypeExpression},
-        RawSlice,
-    },
-    crate::util::{compiler, Life},
-    core::{marker::PhantomData, mem::transmute, ptr},
-};
-
-#[derive(Debug)]
-#[cfg_attr(test, derive(PartialEq))]
-#[repr(u8)]
-/// A statement that can be executed
-pub enum Statement {
-    /// Create a new space with the provided ID
-    CreateSpace(RawSlice),
-    /// Create a new model with the provided configuration
-    CreateModel {
-        entity: Entity,
-        model: FieldConfig,
-        volatile: bool,
-    },
-    /// Drop the given model
-    DropModel { entity: Entity, force: bool },
-    /// Drop the given space
-    DropSpace { entity: RawSlice, force: bool },
-    /// Inspect the given space
-    InspectSpace(Option<RawSlice>),
-    /// Inspect the given model
-    InspectModel(Option<Entity>),
-    /// Inspect all the spaces in the database
-    InspectSpaces,
-    /// Switch to the given entity
-    Use(Entity),
-}
-
-pub type StatementLT<'a> = Life<'a, Statement>;
-
-#[derive(Debug)]
-#[cfg_attr(test, derive(PartialEq))]
-pub enum Entity {
-    Current(RawSlice),
-    Full(RawSlice, RawSlice),
-}
-
-impl Entity {
-    const MAX_LENGTH_EX: usize = 65;
-    pub fn from_slice(slice: &[u8]) -> LangResult<Self> {
-        Compiler::new(&Lexer::lex(slice)?).parse_entity_name()
-    }
-}
-
-#[derive(Debug)]
-#[cfg_attr(test, derive(PartialEq))]
-/// The field configuration used when declaring the fields for a model
-pub struct FieldConfig {
-    /// the types of the fields
-    pub types: Vec<TypeExpression>,
-    /// the names of the fields
-    pub names: Vec<RawSlice>,
-}
-
-impl FieldConfig {
-    /// Create an empty field configuration
-    pub const fn new() -> Self {
-        Self {
-            types: Vec::new(),
-            names: Vec::new(),
-        }
-    }
-    // TODO(@ohsayan): Completely deprecate the model-code based API
-    pub fn get_model_code(&self) -> LangResult<u8> {
-        let Self { types, names } = self;
-        let invalid_expr = {
-            // the model API doesn't support named fields (it's super limited; we need to drop it)
-            !names.is_empty()
-            || types.len() != 2
-            // the key type cannot be compound
-            || types[0].0.len() != 1
-            // the key type cannot be a list
-            || types[0].0[0] == Type::List
-            // the value cannot have a depth more than two
-            || types[1].0.len() > 2
-            // if the value is a string or binary, it cannot have a depth more than 1
-            || ((types[1].0[0] == Type::Binary || types[1].0[0] == Type::String) && types[1].0.len() != 1)
-            // if the value is a list, it must have a depth of two
-            || (types[1].0[0] == Type::List && types[1].0.len() != 2)
-            // if the value is a list, the type argument cannot be a list (it's stupid, I know; that's exactly
-            // why I'll be ditching this API in the next two PRs)
-            || (types[1].0[0] == Type::List && types[1].0[1] == Type::List)
-        };
-        if compiler::unlikely(invalid_expr) {
-            // the value type cannot have a depth more than 2
-            return Err(LangError::UnsupportedModelDeclaration);
-        }
-        let key_expr = &types[0].0;
-        let value_expr = &types[1].0;
-        if value_expr[0] == Type::List {
-            let k_enc = key_expr[0] == Type::String;
-            let v_enc = value_expr[1] == Type::String;
-            Ok(((k_enc as u8) << 1) + (v_enc as u8) + 4)
-        } else {
-            let k_enc = key_expr[0] == Type::String;
-            let v_enc = value_expr[0] == Type::String;
-            let ret = k_enc as u8 + v_enc as u8;
-            Ok((ret & 1) + ((k_enc as u8) << 1))
-        }
-    }
-}
-
-// expect state
-#[derive(Debug)]
-#[repr(u8)]
-#[derive(PartialEq)]
-/// What to expect next
-enum Expect {
-    /// Expect a type
-    Type = 0,
-    /// Expect a [`Token::CloseAngular`]
-    Close = 1,
-}
-
-/// A compiler for BlueQL queries
-///
-/// This compiler takes an input stream and evaluates the query using a traditional
-/// lexer-parser pipeline
-pub struct Compiler<'a> {
-    cursor: *const Token,
-    end_ptr: *const Token,
-    _lt: PhantomData<&'a [u8]>,
-}
-
-impl<'a> Compiler<'a> {
-    #[inline(always)]
-    /// Check if we have not exhausted the token stream
-    fn not_exhausted(&self) -> bool {
-        self.cursor < self.end_ptr
-    }
-    #[inline(always)]
-    /// Deref the cursor
-    unsafe fn deref_cursor(&self) -> &Token {
-        &*self.cursor
-    }
-    #[inline(always)]
-    fn peek_neq(&self, token: &Token) -> bool {
-        self.not_exhausted() && unsafe { self.deref_cursor() != token }
-    }
-    #[inline(always)]
-    /// Check if the token ahead equals the given token
-    fn peek_eq(&self, tok: &Token) -> bool {
-        self.not_exhausted() && unsafe { self.deref_cursor() == tok }
-    }
-    #[inline(always)]
-    /// Check if the token ahead equals the given token, moving the cursor ahead if so
-    fn next_eq(&mut self, tok: &Token) -> bool {
-        let next_is_eq = self.not_exhausted() && unsafe { self.deref_cursor() == tok };
-        unsafe { self.incr_cursor_if(next_is_eq) };
-        next_is_eq
-    }
-    #[inline(always)]
-    /// Increment the cursor if the condition is true
-    unsafe fn incr_cursor_if(&mut self, cond: bool) {
-        self.incr_cursor_by(cond as usize)
-    }
-    #[inline(always)]
-    /// Move the cursor ahead by `by` positions
-    unsafe fn incr_cursor_by(&mut self, by: usize) {
-        self.cursor = self.cursor.add(by)
-    }
-    #[inline(always)]
-    /// Move the cursor ahead by one
-    unsafe fn incr_cursor(&mut self) {
-        self.incr_cursor_by(1)
-    }
-    #[inline(always)]
-    unsafe fn decr_cursor(&mut self) {
-        self.decr_cursor_by(1)
-    }
-    #[inline(always)]
-    unsafe fn decr_cursor_by(&mut self, by: usize) {
-        self.cursor = self.cursor.sub(by)
-    }
-    #[inline(always)]
-    /// Read the element ahead if we have not exhausted the token stream. This
-    /// will forward the cursor
-    fn next(&mut self) -> Option<Token> {
-        if self.not_exhausted() {
-            let r = Some(unsafe { ptr::read(self.cursor) });
-            unsafe { self.incr_cursor() };
-            r
-        } else {
-            None
-        }
-    }
-    #[inline(always)]
-    fn next_result(&mut self) -> LangResult<Token> {
-        if compiler::likely(self.not_exhausted()) {
-            let r = unsafe { ptr::read(self.cursor) };
-            unsafe { self.incr_cursor() };
-            Ok(r)
-        } else {
-            Err(LangError::UnexpectedEOF)
-        }
-    }
-    #[inline(always)]
-    fn next_ident(&mut self) -> LangResult<RawSlice> {
-        match self.next() {
-            Some(Token::Identifier(rws)) => Ok(rws),
-            Some(_) => Err(LangError::InvalidSyntax),
-            None => Err(LangError::UnexpectedEOF),
-        }
-    }
-    #[inline(always)]
-    /// Returns the remaining number of tokens
-    fn remaining(&self) -> usize {
-        self.end_ptr as usize - self.cursor as usize
-    }
-}
-
-impl<'a> Compiler<'a> {
-    #[inline(always)]
-    #[cfg(test)]
-    /// Compile the given BlueQL source
-    pub fn compile(src: &'a [u8]) -> LangResult<Life<'a, Statement>> {
-        Self::compile_with_extra(src, 0)
-    }
-    #[inline(always)]
-    /// Compile the given BlueQL source with optionally supplied extra arguments
-    /// HACK: Just helps us omit an additional check
-    pub fn compile_with_extra(src: &'a [u8], len: usize) -> LangResult<Life<'a, Statement>> {
-        let tokens = Lexer::lex(src)?;
-        Self::new(&tokens).eval(len).map(Life::new)
-    }
-    #[inline(always)]
-    pub const fn new(tokens: &[Token]) -> Self {
-        unsafe {
-            Self {
-                cursor: tokens.as_ptr(),
-                end_ptr: tokens.as_ptr().add(tokens.len()),
-                _lt: PhantomData,
-            }
-        }
-    }
-    #[inline(always)]
-    /// The inner eval method
-    fn eval(&mut self, extra_len: usize) -> LangResult<Statement> {
-        let stmt = match self.next() {
-            Some(tok) => match tok {
-                Token::Keyword(Keyword::Create) => self.parse_create0(),
-                Token::Keyword(Keyword::Drop) => self.parse_drop0(),
-                Token::Keyword(Keyword::Inspect) => self.parse_inspect0(),
-                Token::Keyword(Keyword::Use) => self.parse_use0(),
-                _ => Err(LangError::ExpectedStatement),
-            },
-            None => Err(LangError::UnexpectedEOF),
-        };
-        if compiler::likely(self.remaining() == 0 && extra_len == 0) {
-            stmt
-        } else {
-            Err(LangError::InvalidSyntax)
-        }
-    }
-    #[inline(always)]
-    fn parse_use0(&mut self) -> LangResult<Statement> {
-        Ok(Statement::Use(self.parse_entity_name()?))
-    }
-    #[inline(always)]
-    /// Parse an inspect statement
-    fn parse_inspect0(&mut self) -> LangResult<Statement> {
-        match self.next_result()? {
-            Token::Keyword(Keyword::Model) => self.parse_inspect_model0(),
-            Token::Keyword(Keyword::Space) => self.parse_inspect_space0(),
-            Token::Identifier(spaces)
-                if unsafe { spaces.as_slice() }.eq_ignore_ascii_case(b"spaces") =>
-            {
-                Ok(Statement::InspectSpaces)
-            }
-            _ => Err(LangError::InvalidSyntax),
-        }
-    }
-    #[inline(always)]
-    /// Parse `inspect model <model>`
-    fn parse_inspect_model0(&mut self) -> LangResult<Statement> {
-        match self.next() {
-            Some(Token::Identifier(ident)) => Ok(Statement::InspectModel(Some(
-                self.parse_entity_name_with_start(ident)?,
-            ))),
-            Some(_) => Err(LangError::InvalidSyntax),
-            None => Ok(Statement::InspectModel(None)),
-        }
-    }
-    #[inline(always)]
-    /// Parse `inspect space <space>`
-    fn parse_inspect_space0(&mut self) -> LangResult<Statement> {
-        match self.next() {
-            Some(Token::Identifier(ident)) => Ok(Statement::InspectSpace(Some(ident))),
-            Some(_) => Err(LangError::InvalidSyntax),
-            None => Ok(Statement::InspectSpace(None)),
-        }
-    }
-    #[inline(always)]
-    /// Parse a drop statement
-    fn parse_drop0(&mut self) -> LangResult<Statement> {
-        let (drop_container, drop_id) = (self.next(), self.next());
-        match (drop_container, drop_id) {
-            (Some(Token::Keyword(Keyword::Model)), Some(Token::Identifier(model_name))) => {
-                Ok(Statement::DropModel {
-                    entity: self.parse_entity_name_with_start(model_name)?,
-                    force: self.next_eq(&Token::Keyword(Keyword::Force)),
-                })
-            }
-            (Some(Token::Keyword(Keyword::Space)), Some(Token::Identifier(space_name))) => {
-                Ok(Statement::DropSpace {
-                    entity: space_name,
-                    force: self.next_eq(&Token::Keyword(Keyword::Force)),
-                })
-            }
-            _ => Err(LangError::InvalidSyntax),
-        }
-    }
-    #[inline(always)]
-    /// Parse a create statement
-    fn parse_create0(&mut self) -> LangResult<Statement> {
-        match self.next() {
-            Some(Token::Keyword(Keyword::Model)) => self.parse_create_model0(),
-            Some(Token::Keyword(Keyword::Space)) => self.parse_create_space0(),
-            Some(_) => Err(LangError::UnknownCreateQuery),
-            None => Err(LangError::UnexpectedEOF),
-        }
-    }
-    #[inline(always)]
-    /// Parse a `create model` statement
-    fn parse_create_model0(&mut self) -> LangResult<Statement> {
-        let entity = self.parse_entity_name()?;
-        self.parse_create_model1(entity)
-    }
-    #[inline(always)]
-    /// Parse a field expression and return a `Statement::CreateModel`
-    pub(super) fn parse_create_model1(&mut self, entity: Entity) -> LangResult<Statement> {
-        let mut fc = FieldConfig::new();
-        let mut is_good_expr = self.next_eq(&Token::OpenParen);
-        while is_good_expr && self.peek_neq(&Token::CloseParen) {
-            match self.next() {
-                Some(Token::Identifier(field_name)) => {
-                    // we have a field name
-                    is_good_expr &= self.next_eq(&Token::Colon);
-                    if let Some(Token::Keyword(Keyword::Type(ty))) = self.next() {
-                        fc.names.push(field_name);
-                        fc.types.push(self.parse_type_expression(ty)?);
-                    } else {
-                        is_good_expr = false;
-                    }
-                    is_good_expr &= self.peek_eq(&Token::CloseParen) || self.next_eq(&Token::Comma);
-                }
-                Some(Token::Keyword(Keyword::Type(ty))) => {
-                    // we have a type name
-                    fc.types.push(self.parse_type_expression(ty)?);
-                    is_good_expr &= self.peek_eq(&Token::CloseParen) || self.next_eq(&Token::Comma);
-                }
-                _ => is_good_expr = false,
-            }
-        }
-        is_good_expr &= self.next_eq(&Token::CloseParen);
-        is_good_expr &= fc.types.len() >= 2;
-        // important; we either have all unnamed fields or all named fields; having some unnamed
-        // and some named is ambiguous because there's not "straightforward" way to query them
-        // without introducing some funky naming conventions ($<field_number> if you don't have the
-        // right name sounds like an outrageous idea)
-        is_good_expr &= fc.names.is_empty() || fc.names.len() == fc.types.len();
-        let volatile = self.next_eq(&Token::Keyword(Keyword::Volatile));
-        if compiler::likely(is_good_expr) {
-            Ok(Statement::CreateModel {
-                entity,
-                model: fc,
-                volatile,
-            })
-        } else {
-            Err(LangError::BadExpression)
-        }
-    }
-    #[inline(always)]
-    /// Parse a type expression return a `TypeExpression`
-    fn parse_type_expression(&mut self, first_type: Type) -> LangResult<TypeExpression> {
-        let mut expr = Vec::with_capacity(2);
-        expr.push(first_type);
-
-        // count of open and close brackets
-        let mut p_open = 0;
-        let mut p_close = 0;
-        let mut valid_expr = true;
-
-        // we already have the starting type; next is either nothing or open angular
-        let mut has_more_args = self.peek_eq(&Token::OpenAngular);
-
-        let mut expect = Expect::Type;
-        while valid_expr && has_more_args && self.peek_neq(&Token::CloseParen) {
-            match self.next() {
-                Some(Token::OpenAngular) => p_open += 1,
-                Some(Token::Keyword(Keyword::Type(ty))) if expect == Expect::Type => {
-                    expr.push(ty);
-                    let next = self.next();
-                    let next_is_open = next == Some(Token::OpenAngular);
-                    let next_is_close = next == Some(Token::CloseAngular);
-                    p_open += next_is_open as usize;
-                    p_close += next_is_close as usize;
-                    expect = unsafe { transmute(next_is_close) };
-                }
-                Some(Token::CloseAngular) if expect == Expect::Close => {
-                    p_close += 1;
-                    expect = Expect::Close;
-                }
-                Some(Token::Comma) => {
-                    unsafe { self.decr_cursor() }
-                    has_more_args = false
-                }
-                _ => valid_expr = false,
-            }
-        }
-        valid_expr &= p_open == p_close;
-        if compiler::likely(valid_expr) {
-            Ok(TypeExpression(expr))
-        } else {
-            Err(LangError::InvalidSyntax)
-        }
-    }
-    #[inline(always)]
-    /// Parse a `create space` statement
-    fn parse_create_space0(&mut self) -> LangResult<Statement> {
-        match self.next() {
-            Some(Token::Identifier(model_name)) => Ok(Statement::CreateSpace(model_name)),
-            Some(_) => Err(LangError::InvalidSyntax),
-            None => Err(LangError::UnexpectedEOF),
-        }
-    }
-    #[inline(always)]
-    fn parse_entity_name_with_start(&mut self, start: RawSlice) -> LangResult<Entity> {
-        if self.peek_eq(&Token::Period) {
-            unsafe { self.incr_cursor() };
-            Ok(Entity::Full(start, self.next_ident()?))
-        } else {
-            Ok(Entity::Current(start))
-        }
-    }
-    #[inline(always)]
-    pub(super) fn parse_entity_name(&mut self) -> LangResult<Entity> {
-        // let's peek the next token
-        match self.next_ident()? {
-            id if self.peek_eq(&Token::Period)
-                && compiler::likely(id.len() < Entity::MAX_LENGTH_EX) =>
-            {
-                unsafe { self.incr_cursor() };
-                Ok(Entity::Full(id, self.next_ident()?))
-            }
-            id if compiler::likely(id.len() < Entity::MAX_LENGTH_EX) => Ok(Entity::Current(id)),
-            _ => Err(LangError::InvalidSyntax),
-        }
-    }
-}
diff --git a/server/src/blueql/error.rs b/server/src/blueql/error.rs
deleted file mode 100644
index 6295aa94..00000000
--- a/server/src/blueql/error.rs
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Created on Tue Jun 14 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{
-    actions::{ActionError, ActionResult},
-    protocol::interface::ProtocolSpec,
-};
-
-#[derive(Debug, PartialEq)]
-#[repr(u8)]
-/// BlueQL errors
-pub enum LangError {
-    /// Invalid syntax
-    InvalidSyntax,
-    /// Invalid numeric literal
-    InvalidNumericLiteral,
-    /// Unexpected end-of-statement
-    UnexpectedEOF,
-    /// Expected a statement but found some other token
-    ExpectedStatement,
-    /// Got an unknown create query
-    UnknownCreateQuery,
-    /// Bad expression
-    BadExpression,
-    /// An invalid string literal
-    InvalidStringLiteral,
-    /// Unsupported model declaration
-    UnsupportedModelDeclaration,
-    /// Unexpected character
-    UnexpectedChar,
-}
-
-/// Results for BlueQL
-pub type LangResult<T> = Result<T, LangError>;
-
-#[inline(never)]
-#[cold]
-pub(super) const fn cold_err<P: ProtocolSpec>(e: LangError) -> &'static [u8] {
-    match e {
-        LangError::BadExpression => P::BQL_BAD_EXPRESSION,
-        LangError::ExpectedStatement => P::BQL_EXPECTED_STMT,
-        LangError::InvalidNumericLiteral => P::BQL_INVALID_NUMERIC_LITERAL,
-        LangError::InvalidStringLiteral => P::BQL_INVALID_STRING_LITERAL,
-        LangError::InvalidSyntax => P::BQL_INVALID_SYNTAX,
-        LangError::UnexpectedEOF => P::BQL_UNEXPECTED_EOF,
-        LangError::UnknownCreateQuery => P::BQL_UNKNOWN_CREATE_QUERY,
-        LangError::UnsupportedModelDeclaration => P::BQL_UNSUPPORTED_MODEL_DECL,
-        LangError::UnexpectedChar => P::BQL_UNEXPECTED_CHAR,
-    }
-}
-
-#[inline(always)]
-pub fn map_ql_err_to_resp<T, P: ProtocolSpec>(e: LangResult<T>) -> ActionResult<T> {
-    match e {
-        Ok(v) => Ok(v),
-        Err(e) => Err(ActionError::ActionError(cold_err::<P>(e))),
-    }
-}
diff --git a/server/src/blueql/executor.rs b/server/src/blueql/executor.rs
deleted file mode 100644
index 9dac3aa6..00000000
--- a/server/src/blueql/executor.rs
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Created on Wed Jun 15 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::dbnet::BufferedSocketStream;
-
-use {
-    super::{
-        ast::{Statement, StatementLT},
-        error,
-    },
-    crate::{
-        actions::{self, ActionError, ActionResult},
-        blueql,
-        corestore::memstore::ObjectID,
-        dbnet::prelude::*,
-    },
-};
-
-pub async fn execute<'a, P, C>(
-    handle: &'a mut Corestore,
-    con: &mut Connection<C, P>,
-    maybe_statement: &[u8],
-    extra: usize,
-) -> ActionResult<()>
-where
-    P: ProtocolSpec,
-    C: BufferedSocketStream,
-{
-    let statement =
-        error::map_ql_err_to_resp::<StatementLT, P>(blueql::compile(maybe_statement, extra))?;
-    let system_health_okay = registry::state_okay();
-    let result = match statement.as_ref() {
-        Statement::Use(entity) => handle.swap_entity(entity),
-        Statement::CreateSpace(space_name) if system_health_okay => {
-            // ret okay
-            handle.create_keyspace(unsafe { ObjectID::from_slice(space_name.as_slice()) })
-        }
-        Statement::DropSpace { entity, force } if system_health_okay => {
-            // ret okay
-            let entity = unsafe { ObjectID::from_slice(entity.as_slice()) };
-            if *force {
-                handle.force_drop_keyspace(entity)
-            } else {
-                handle.drop_keyspace(entity)
-            }
-        }
-        Statement::DropModel { entity, force } if system_health_okay => {
-            // ret okay
-            handle.drop_table(entity, *force)
-        }
-        Statement::CreateModel {
-            entity,
-            model,
-            volatile,
-        } if system_health_okay => {
-            match model.get_model_code() {
-                // ret okay
-                Ok(code) => handle.create_table(entity, code, *volatile),
-                Err(e) => return Err(ActionError::ActionError(error::cold_err::<P>(e))),
-            }
-        }
-        Statement::InspectSpaces => {
-            // ret directly
-            con.write_typed_non_null_array(&handle.get_store().list_keyspaces(), b'+')
-                .await?;
-            return Ok(());
-        }
-        Statement::InspectSpace(space) => {
-            // ret directly
-            con.write_typed_non_null_array(
-                handle.list_tables::<P>(space.as_ref().map(|v| unsafe { v.as_slice() }))?,
-                b'+',
-            )
-            .await?;
-            return Ok(());
-        }
-        Statement::InspectModel(model) => {
-            // ret directly
-            con.write_string(&handle.describe_table::<P>(model)?)
-                .await?;
-            return Ok(());
-        }
-        _ => {
-            // the server is broken
-            con._write_raw(P::RCODE_SERVER_ERR).await?;
-            return Ok(());
-        }
-    };
-    actions::translate_ddl_error::<P, ()>(result)?;
-    con._write_raw(P::RCODE_OKAY).await?;
-    Ok(())
-}
diff --git a/server/src/blueql/lexer.rs b/server/src/blueql/lexer.rs
deleted file mode 100644
index e2f23249..00000000
--- a/server/src/blueql/lexer.rs
+++ /dev/null
@@ -1,394 +0,0 @@
-/*
- * Created on Tue Jun 14 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{
-        error::{LangError, LangResult},
-        RawSlice,
-    },
-    crate::util::compiler,
-    core::{marker::PhantomData, slice, str},
-};
-
-#[derive(Debug, PartialEq)]
-#[repr(u8)]
-/// BQL tokens
-pub enum Token {
-    OpenParen,    // (
-    CloseParen,   // )
-    OpenAngular,  // <
-    CloseAngular, // >
-    Comma,        // ,
-    Colon,        // :
-    Period,       // .
-    QuotedString(String),
-    Identifier(RawSlice),
-    Number(u64),
-    Keyword(Keyword),
-}
-
-impl From<Keyword> for Token {
-    fn from(kw: Keyword) -> Self {
-        Self::Keyword(kw)
-    }
-}
-
-#[cfg(test)]
-impl From<&'static str> for Token {
-    fn from(sl: &'static str) -> Self {
-        Self::Identifier(sl.into())
-    }
-}
-
-impl From<u64> for Token {
-    fn from(num: u64) -> Self {
-        Self::Number(num)
-    }
-}
-
-impl From<Type> for Token {
-    fn from(ty: Type) -> Self {
-        Self::Keyword(Keyword::Type(ty))
-    }
-}
-
-#[derive(Debug, PartialEq, Clone, Copy)]
-#[repr(u8)]
-/// BlueQL keywords
-pub enum Keyword {
-    Create,
-    Use,
-    Drop,
-    Inspect,
-    Model,
-    Space,
-    Volatile,
-    Force,
-    Type(Type),
-}
-
-#[derive(Debug, Clone, Copy, PartialEq)]
-#[repr(u8)]
-/// BlueQL types
-pub enum Type {
-    String,
-    Binary,
-    List,
-}
-
-#[derive(Debug, PartialEq)]
-/// Type expression (ty<ty<...>>)
-pub struct TypeExpression(pub Vec<Type>);
-
-impl Keyword {
-    /// Attempt to parse a keyword from the given slice
-    #[inline(always)]
-    pub fn try_from_slice(slice: &[u8]) -> Option<Self> {
-        let r = match slice.to_ascii_lowercase().as_slice() {
-            b"create" => Keyword::Create,
-            b"drop" => Keyword::Drop,
-            b"inspect" => Keyword::Inspect,
-            b"model" => Keyword::Model,
-            b"space" => Keyword::Space,
-            b"volatile" => Keyword::Volatile,
-            b"string" => Keyword::Type(Type::String),
-            b"binary" => Keyword::Type(Type::Binary),
-            b"list" => Keyword::Type(Type::List),
-            b"force" => Keyword::Force,
-            b"use" => Keyword::Use,
-            _ => return None,
-        };
-        Some(r)
-    }
-}
-
-#[inline(always)]
-/// Find the distance between two pointers
-fn find_ptr_distance(start: *const u8, stop: *const u8) -> usize {
-    stop as usize - start as usize
-}
-
-/// A `Lexer` for BlueQL tokens
-pub struct Lexer<'a> {
-    cursor: *const u8,
-    end_ptr: *const u8,
-    _lt: PhantomData<&'a [u8]>,
-    last_error: Option<LangError>,
-    tokens: Vec<Token>,
-}
-
-const _ENSURE_EQ_SIZE: () =
-    assert!(std::mem::size_of::<Option<LangError>>() == std::mem::size_of::<LangError>());
-
-impl<'a> Lexer<'a> {
-    #[inline(always)]
-    /// Create a new `Lexer`
-    pub const fn new(buf: &'a [u8]) -> Self {
-        unsafe {
-            Self {
-                cursor: buf.as_ptr(),
-                end_ptr: buf.as_ptr().add(buf.len()),
-                last_error: None,
-                tokens: Vec::new(),
-                _lt: PhantomData,
-            }
-        }
-    }
-}
-
-impl<'a> Lexer<'a> {
-    #[inline(always)]
-    /// Returns the cursor
-    const fn cursor(&self) -> *const u8 {
-        self.cursor
-    }
-    #[inline(always)]
-    /// Returns the end ptr
-    const fn end_ptr(&self) -> *const u8 {
-        self.end_ptr
-    }
-    #[inline(always)]
-    /// Increments the cursor by 1
-    unsafe fn incr_cursor(&mut self) {
-        self.incr_cursor_by(1)
-    }
-    /// Increments the cursor by 1 if `cond` is true
-    #[inline(always)]
-    unsafe fn incr_cursor_if(&mut self, cond: bool) {
-        self.incr_cursor_by(cond as usize)
-    }
-    #[inline(always)]
-    /// Increments the cursor by `by` positions
-    unsafe fn incr_cursor_by(&mut self, by: usize) {
-        self.cursor = self.cursor.add(by)
-    }
-    #[inline(always)]
-    /// Derefs the cursor
-    unsafe fn deref_cursor(&self) -> u8 {
-        *self.cursor()
-    }
-    #[inline(always)]
-    /// Checks if we have reached EOA
-    fn not_exhausted(&self) -> bool {
-        self.cursor() < self.end_ptr()
-    }
-    #[inline(always)]
-    /// Returns true if we have reached EOA
-    fn exhausted(&self) -> bool {
-        self.cursor() >= self.end_ptr()
-    }
-    #[inline(always)]
-    /// Check if the peeked value matches the predicate. Returns false if EOA
-    fn peek_is(&self, predicate: impl Fn(u8) -> bool) -> bool {
-        self.not_exhausted() && unsafe { predicate(self.deref_cursor()) }
-    }
-    #[inline(always)]
-    /// Check if the byte ahead is equal to the provided byte. Returns false
-    /// if reached EOA
-    fn peek_eq(&self, eq: u8) -> bool {
-        self.not_exhausted() && unsafe { self.deref_cursor() == eq }
-    }
-    #[inline(always)]
-    /// Check if the byte ahead is not equal to the provided byte. Returns false
-    /// if reached EOA
-    fn peek_neq(&self, neq: u8) -> bool {
-        self.not_exhausted() && unsafe { self.deref_cursor() != neq }
-    }
-    #[inline(always)]
-    /// Same as `peek_eq`, but forwards the cursor if the byte is matched
-    fn peek_eq_and_forward(&mut self, eq: u8) -> bool {
-        let did_peek = self.peek_eq(eq);
-        unsafe { self.incr_cursor_if(did_peek) };
-        did_peek
-    }
-    #[inline(always)]
-    /// Same as `peek_eq_or_eof` but forwards the cursor on match
-    fn peek_eq_or_eof_and_forward(&mut self, eq: u8) -> bool {
-        let did_forward = self.peek_eq_and_forward(eq);
-        unsafe { self.incr_cursor_if(did_forward) };
-        did_forward | self.exhausted()
-    }
-    #[inline(always)]
-    /// Trim the whitespace ahead
-    fn trim_ahead(&mut self) {
-        while self.peek_eq_and_forward(b' ') {}
-    }
-    #[inline(always)]
-    unsafe fn check_escaped(&mut self, escape_what: u8) -> bool {
-        debug_assert!(self.not_exhausted());
-        self.deref_cursor() == b'\\' && {
-            self.not_exhausted() && self.deref_cursor() == escape_what
-        }
-    }
-    #[inline(always)]
-    fn push_token(&mut self, token: impl Into<Token>) {
-        self.tokens.push(token.into())
-    }
-}
-
-impl<'a> Lexer<'a> {
-    #[inline(always)]
-    /// Attempt to scan a number
-    fn scan_number(&mut self) {
-        let start = self.cursor();
-        while self.peek_is(|byte| byte.is_ascii_digit()) {
-            unsafe { self.incr_cursor() }
-        }
-        let slice = unsafe {
-            str::from_utf8_unchecked(slice::from_raw_parts(
-                start,
-                find_ptr_distance(start, self.cursor()),
-            ))
-        };
-        let next_is_ws_or_eof = self.peek_eq_or_eof_and_forward(b' ');
-        match slice.parse() {
-            Ok(num) if compiler::likely(next_is_ws_or_eof) => {
-                // this is a good number; push it in
-                self.push_token(Token::Number(num));
-            }
-            _ => {
-                // that breaks the state
-                self.last_error = Some(LangError::InvalidNumericLiteral);
-            }
-        }
-    }
-    #[inline(always)]
-    /// Attempt to scan an ident
-    fn scan_ident(&mut self) -> RawSlice {
-        let start = self.cursor();
-        while self.peek_is(|byte| (byte.is_ascii_alphanumeric() || byte == b'_')) {
-            unsafe { self.incr_cursor() }
-        }
-        let len = find_ptr_distance(start, self.cursor());
-        unsafe { RawSlice::new(start, len) }
-    }
-    #[inline(always)]
-    fn scan_ident_or_keyword(&mut self) {
-        let ident = self.scan_ident();
-        match Keyword::try_from_slice(unsafe {
-            // UNSAFE(@ohsayan): The source buffer's presence guarantees that this is correct
-            ident.as_slice()
-        }) {
-            Some(kw) => self.push_token(kw),
-            None => self.push_token(Token::Identifier(ident)),
-        }
-    }
-    #[inline(always)]
-    /// Scan a quoted string
-    fn scan_quoted_string(&mut self, quote_style: u8) {
-        unsafe { self.incr_cursor() }
-        // a quoted string with the given quote style
-        let mut stringbuf = Vec::new();
-        // should start with  '"'
-        let mut is_okay = true;
-        while is_okay && self.peek_neq(quote_style) {
-            let is_escaped_backslash = unsafe {
-                // UNSAFE(@ohsayan): The qp is not exhausted, so this is fine
-                self.check_escaped(b'\\')
-            };
-            let is_escaped_quote = unsafe {
-                // UNSAFE(@ohsayan): The qp is not exhausted, so this is fine
-                self.check_escaped(b'"')
-            };
-            unsafe {
-                // UNSAFE(@ohsayan): If either is true, then it is correct to do this
-                self.incr_cursor_if(is_escaped_backslash | is_escaped_quote)
-            };
-            unsafe {
-                // UNSAFE(@ohsayan): if not escaped, this is fine. if escaped, this is still
-                // fine because the escaped byte was checked
-                stringbuf.push(self.deref_cursor());
-            }
-            unsafe {
-                // UNSAFE(@ohsayan): if escaped we have moved ahead by one but the escaped char
-                // is still one more so we go ahead. if not, then business as usual
-                self.incr_cursor()
-            };
-        }
-        // should be terminated by a '"'
-        is_okay &= self.peek_eq_and_forward(quote_style);
-        match String::from_utf8(stringbuf) {
-            Ok(s) if compiler::likely(is_okay) => {
-                // valid string literal
-                self.push_token(Token::QuotedString(s));
-            }
-            _ => {
-                // state broken
-                self.last_error = Some(LangError::InvalidStringLiteral)
-            }
-        }
-    }
-    #[inline(always)]
-    fn scan_arbitrary_byte(&mut self, byte: u8) {
-        let r = match byte {
-            b'<' => Token::OpenAngular,
-            b'>' => Token::CloseAngular,
-            b'(' => Token::OpenParen,
-            b')' => Token::CloseParen,
-            b',' => Token::Comma,
-            b':' => Token::Colon,
-            b'.' => Token::Period,
-            _ => {
-                self.last_error = Some(LangError::UnexpectedChar);
-                return;
-            }
-        };
-        unsafe { self.incr_cursor() };
-        self.push_token(r);
-    }
-}
-
-impl<'a> Lexer<'a> {
-    #[inline(always)]
-    /// Lex the input stream into tokens
-    pub fn lex(src: &'a [u8]) -> LangResult<Vec<Token>> {
-        Self::new(src)._lex()
-    }
-    #[inline(always)]
-    /// The inner lex method
-    fn _lex(mut self) -> LangResult<Vec<Token>> {
-        while self.not_exhausted() && self.last_error.is_none() {
-            match unsafe { self.deref_cursor() } {
-                byte if byte.is_ascii_alphabetic() => self.scan_ident_or_keyword(),
-                byte if byte.is_ascii_digit() => self.scan_number(),
-                b' ' => self.trim_ahead(),
-                b'\n' | b'\t' => {
-                    // simply ignore
-                    unsafe {
-                        // UNSAFE(@ohsayan): This is totally fine. We just looked at the byte
-                        self.incr_cursor()
-                    }
-                }
-                quote_style @ (b'"' | b'\'') => self.scan_quoted_string(quote_style),
-                byte => self.scan_arbitrary_byte(byte),
-            }
-        }
-        match self.last_error {
-            None => Ok(self.tokens),
-            Some(e) => Err(e),
-        }
-    }
-}
diff --git a/server/src/blueql/mod.rs b/server/src/blueql/mod.rs
deleted file mode 100644
index bfbeda12..00000000
--- a/server/src/blueql/mod.rs
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Created on Tue Jun 14 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-mod ast;
-mod error;
-mod executor;
-mod lexer;
-pub mod util;
-// test modules
-#[cfg(test)]
-mod tests;
-// re-export
-use {
-    self::{ast::Statement, error::LangResult},
-    crate::util::Life,
-};
-pub use {ast::Compiler, ast::Entity, executor::execute};
-
-#[cfg(test)]
-use core::fmt;
-use core::{mem, slice};
-
-#[allow(clippy::needless_lifetimes)]
-#[inline(always)]
-pub fn compile<'a>(src: &'a [u8], extra: usize) -> LangResult<Life<'a, Statement>> {
-    Compiler::compile_with_extra(src, extra)
-}
-
-#[cfg_attr(not(test), derive(Debug))]
-#[cfg_attr(not(test), derive(PartialEq))]
-pub struct RawSlice {
-    ptr: *const u8,
-    len: usize,
-}
-
-unsafe impl Send for RawSlice {}
-unsafe impl Sync for RawSlice {}
-
-#[cfg(test)]
-impl fmt::Debug for RawSlice {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", String::from_utf8_lossy(unsafe { self.as_slice() }))
-    }
-}
-
-#[cfg(test)]
-impl PartialEq for RawSlice {
-    fn eq(&self, other: &Self) -> bool {
-        unsafe { self.as_slice() == other.as_slice() }
-    }
-}
-
-impl RawSlice {
-    const _ENSURE_ALIGN: () = assert!(mem::align_of::<RawSlice>() == mem::align_of::<&[u8]>());
-    pub const unsafe fn new(ptr: *const u8, len: usize) -> Self {
-        Self { ptr, len }
-    }
-    pub unsafe fn as_slice(&self) -> &[u8] {
-        slice::from_raw_parts(self.ptr, self.len)
-    }
-    pub const fn len(&self) -> usize {
-        self.len
-    }
-}
-
-#[cfg(test)]
-impl<T> From<T> for RawSlice
-where
-    T: AsRef<[u8]>,
-{
-    fn from(t: T) -> Self {
-        let t = t.as_ref();
-        unsafe { Self::new(t.as_ptr(), t.len()) }
-    }
-}
diff --git a/server/src/blueql/tests.rs b/server/src/blueql/tests.rs
deleted file mode 100644
index aa187e80..00000000
--- a/server/src/blueql/tests.rs
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Created on Tue Jun 14 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use super::{
-    ast::{Compiler, Entity, FieldConfig, Statement},
-    error::LangError,
-    lexer::{Keyword, Lexer, Token, Type, TypeExpression},
-};
-
-macro_rules! src {
-    ($name:ident, $($src:expr),* $(,)?) => {
-        const $name: &[&[u8]] = &[$($src.as_bytes()),*];
-    };
-}
-
-mod lexer {
-    //! Lexer tests
-
-    use super::*;
-
-    #[test]
-    fn lex_ident() {
-        let src = b"mytbl";
-        assert_eq!(
-            Lexer::lex(src).unwrap(),
-            vec![Token::Identifier("mytbl".into())]
-        )
-    }
-
-    #[test]
-    fn lex_keyword() {
-        let src = b"create";
-        assert_eq!(
-            Lexer::lex(src).unwrap(),
-            vec![Token::Keyword(Keyword::Create)]
-        )
-    }
-
-    #[test]
-    fn lex_number() {
-        let src = b"123456";
-        assert_eq!(Lexer::lex(src).unwrap(), vec![Token::Number(123456)])
-    }
-
-    #[test]
-    fn lex_full() {
-        let src = b"create model tweet";
-        assert_eq!(
-            Lexer::lex(src).unwrap(),
-            vec![
-                Token::Keyword(Keyword::Create),
-                Token::Keyword(Keyword::Model),
-                Token::Identifier("tweet".into())
-            ]
-        );
-    }
-
-    #[test]
-    fn lex_combined_tokens() {
-        let src = b"create model tweet(name: string, pic: binary, posts: list<string>)";
-        assert_eq!(
-            Lexer::lex(src).unwrap(),
-            vec![
-                Keyword::Create.into(),
-                Keyword::Model.into(),
-                "tweet".into(),
-                Token::OpenParen,
-                Token::Identifier("name".into()),
-                Token::Colon,
-                Type::String.into(),
-                Token::Comma,
-                Token::Identifier("pic".into()),
-                Token::Colon,
-                Type::Binary.into(),
-                Token::Comma,
-                Token::Identifier("posts".into()),
-                Token::Colon,
-                Type::List.into(),
-                Token::OpenAngular,
-                Type::String.into(),
-                Token::CloseAngular,
-                Token::CloseParen
-            ]
-        );
-    }
-
-    #[test]
-    fn lex_quoted_string() {
-        let src_a = "'hello, world🦀!'".as_bytes();
-        let src_b = r#" "hello, world🦀!" "#.as_bytes();
-        let src_c = r#" "\"hello, world🦀!\"" "#.as_bytes();
-        assert_eq!(
-            Lexer::lex(src_a).unwrap(),
-            vec![Token::QuotedString("hello, world🦀!".into())]
-        );
-        assert_eq!(
-            Lexer::lex(src_b).unwrap(),
-            vec![Token::QuotedString("hello, world🦀!".into())]
-        );
-        assert_eq!(
-            Lexer::lex(src_c).unwrap(),
-            vec![Token::QuotedString("\"hello, world🦀!\"".into())]
-        )
-    }
-
-    #[test]
-    fn lex_fail_unknown_chars() {
-        const SOURCES: &[&[u8]] = &[
-            b"!", b"@", b"#", b"$", b"%", b"^", b"&", b"*", b"[", b"]", b"{", b"}", b"|", b"\\",
-            b"/", b"~", b"`", b";", b"hello?",
-        ];
-        for source in SOURCES {
-            assert_eq!(Lexer::lex(source).unwrap_err(), LangError::UnexpectedChar);
-        }
-    }
-
-    #[test]
-    fn lex_fail_unclosed_litstring() {
-        const SOURCES: &[&[u8]] = &[b"'hello, world", br#""hello, world"#];
-        for source in SOURCES {
-            assert_eq!(
-                Lexer::lex(source).unwrap_err(),
-                LangError::InvalidStringLiteral
-            );
-        }
-    }
-
-    #[test]
-    fn lex_fail_litnum() {
-        src!(SOURCES, "12345f", "123!", "123'");
-        for source in SOURCES {
-            assert_eq!(
-                Lexer::lex(source).unwrap_err(),
-                LangError::InvalidNumericLiteral
-            );
-        }
-    }
-
-    #[test]
-    fn lex_ignore_lf() {
-        let test_slice = b"create\n";
-        assert_eq!(
-            Lexer::lex(test_slice).unwrap(),
-            vec![Token::Keyword(Keyword::Create)]
-        )
-    }
-
-    #[test]
-    fn lex_ignore_tab() {
-        let test_slice = b"create\t";
-        assert_eq!(
-            Lexer::lex(test_slice).unwrap(),
-            vec![Token::Keyword(Keyword::Create)]
-        )
-    }
-}
-
-mod ast {
-    //! AST tests
-
-    #[test]
-    fn parse_entity_name_test() {
-        assert_eq!(
-            Compiler::new(&Lexer::lex(b"hello").unwrap())
-                .parse_entity_name()
-                .unwrap(),
-            Entity::Current("hello".into())
-        );
-        assert_eq!(
-            Compiler::new(&Lexer::lex(b"hello.world").unwrap())
-                .parse_entity_name()
-                .unwrap(),
-            Entity::Full("hello".into(), "world".into())
-        );
-    }
-
-    use super::*;
-    #[cfg(test)]
-    fn setup_src_stmt() -> (Vec<u8>, Statement) {
-        let src =
-            b"create model twitter.tweet(username: string, password: binary, posts: list<string>) volatile"
-                .to_vec();
-        let stmt = Statement::CreateModel {
-            entity: Entity::Full("twitter".into(), "tweet".into()),
-            model: FieldConfig {
-                types: vec![
-                    TypeExpression(vec![Type::String]),
-                    TypeExpression(vec![Type::Binary]),
-                    TypeExpression(vec![Type::List, Type::String]),
-                ],
-                names: vec!["username".into(), "password".into(), "posts".into()],
-            },
-            volatile: true,
-        };
-        (src, stmt)
-    }
-
-    #[test]
-    fn stmt_create_named_unnamed_mixed() {
-        let src = b"create model twitter.tweet(username: string, binary,)".to_vec();
-        assert_eq!(
-            Compiler::compile(&src).unwrap_err(),
-            LangError::BadExpression
-        );
-    }
-    #[test]
-    fn stmt_create_unnamed() {
-        let src = b"create model twitter.passwords(string, binary)".to_vec();
-        let expected = Statement::CreateModel {
-            entity: Entity::Full("twitter".into(), "passwords".into()),
-            model: FieldConfig {
-                names: vec![],
-                types: vec![
-                    TypeExpression(vec![Type::String]),
-                    TypeExpression(vec![Type::Binary]),
-                ],
-            },
-            volatile: false,
-        };
-        assert_eq!(Compiler::compile(&src).unwrap(), expected);
-    }
-    #[test]
-    fn stmt_drop_space() {
-        assert_eq!(
-            Compiler::compile(b"drop space twitter force").unwrap(),
-            Statement::DropSpace {
-                entity: "twitter".into(),
-                force: true
-            }
-        );
-    }
-    #[test]
-    fn stmt_drop_model() {
-        assert_eq!(
-            Compiler::compile(b"drop model twitter.tweet force").unwrap(),
-            Statement::DropModel {
-                entity: Entity::Full("twitter".into(), "tweet".into()),
-                force: true
-            }
-        );
-    }
-    #[test]
-    fn stmt_inspect_space() {
-        assert_eq!(
-            Compiler::compile(b"inspect space twitter").unwrap(),
-            Statement::InspectSpace(Some("twitter".into()))
-        );
-    }
-    #[test]
-    fn stmt_inspect_model() {
-        assert_eq!(
-            Compiler::compile(b"inspect model twitter.tweet").unwrap(),
-            Statement::InspectModel(Some(Entity::Full("twitter".into(), "tweet".into())))
-        );
-    }
-    #[test]
-    fn compile_full() {
-        let (src, stmt) = setup_src_stmt();
-        assert_eq!(Compiler::compile(&src).unwrap(), stmt)
-    }
-    #[test]
-    fn bad_model_code() {
-        let get_model_code = |src| {
-            let l = Lexer::lex(src).unwrap();
-            let stmt = Compiler::new(&l)
-                .parse_create_model1(Entity::Current("jotsy".into()))
-                .unwrap_or_else(|_| panic!("Failed for payload: {}", String::from_utf8_lossy(src)));
-            match stmt {
-                Statement::CreateModel { model, .. } => model.get_model_code(),
-                x => panic!("Expected model found {:?}", x),
-            }
-        };
-        // check rules
-        // first type cannot be list
-        src!(
-            SRC,
-            // rule: first cannot be list
-            "(list, string)",
-            "(list, binary)",
-            "(list<string>, string)",
-            "(list<binary>, string)",
-            // rule: cannot have more than two args
-            "(list<string>, string, string)",
-            // rule: non-compound types cannot have type arguments
-            "(string<binary>, binary<string>)",
-            // rule: fields can't be named
-            "(id: string, posts: list<string>)",
-            // rule: nested lists are disallowed
-            "(string, list<list<string>>)"
-        );
-        for src in SRC {
-            assert_eq!(
-                get_model_code(src).unwrap_err(),
-                LangError::UnsupportedModelDeclaration,
-                "{}",
-                String::from_utf8_lossy(src)
-            );
-        }
-    }
-}
diff --git a/server/src/blueql/util.rs b/server/src/blueql/util.rs
deleted file mode 100644
index f0501c31..00000000
--- a/server/src/blueql/util.rs
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Created on Mon Jun 20 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{ast::Entity, error},
-    crate::{
-        actions::{ActionError, ActionResult},
-        protocol::interface::ProtocolSpec,
-        util::Life,
-    },
-};
-
-pub fn from_slice_action_result<P: ProtocolSpec>(slice: &[u8]) -> ActionResult<Life<'_, Entity>> {
-    match Entity::from_slice(slice) {
-        Ok(slc) => Ok(Life::new(slc)),
-        Err(e) => Err(ActionError::ActionError(error::cold_err::<P>(e))),
-    }
-}
diff --git a/server/src/config/cfgcli.rs b/server/src/config/cfgcli.rs
deleted file mode 100644
index fd03da7e..00000000
--- a/server/src/config/cfgcli.rs
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Created on Fri Jan 28 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{ConfigSourceParseResult, Configset, TryFromConfigSource},
-    clap::ArgMatches,
-};
-
-/// A flag. The flag is said to be set if `self.set` is true and unset if `self.set` is false. However,
-/// if the flag is set, the value of SWITCH determines what value it is set to
-#[derive(Copy, Clone)]
-pub(super) struct Flag<const SWITCH: bool> {
-    set: bool,
-}
-
-impl<const SWITCH: bool> Flag<SWITCH> {
-    pub(super) fn new(set: bool) -> Self {
-        Self { set }
-    }
-}
-
-impl<const SWITCH: bool> TryFromConfigSource<bool> for Flag<SWITCH> {
-    fn is_present(&self) -> bool {
-        self.set
-    }
-    fn mutate_failed(self, target: &mut bool, trip: &mut bool) -> bool {
-        if self.set {
-            *trip = true;
-            *target = SWITCH;
-        }
-        false
-    }
-    fn try_parse(self) -> ConfigSourceParseResult<bool> {
-        if self.set {
-            ConfigSourceParseResult::Okay(SWITCH)
-        } else {
-            ConfigSourceParseResult::Absent
-        }
-    }
-}
-
-pub(super) fn parse_cli_args(matches: ArgMatches) -> Configset {
-    let mut defset = Configset::new_cli();
-    macro_rules! fcli {
-        ($fn:ident, $($source:expr, $key:literal),*) => {
-            defset.$fn(
-                $(
-                    $source,
-                    $key,
-                )*
-            )
-        };
-    }
-    // protocol settings
-    fcli! {
-        protocol_settings,
-        matches.value_of("protover"),
-        "--protover"
-    };
-    // server settings
-    fcli!(
-        server_tcp,
-        matches.value_of("host"),
-        "--host",
-        matches.value_of("port"),
-        "--port"
-    );
-    fcli!(
-        server_noart,
-        Flag::<true>::new(matches.is_present("noart")),
-        "--noart"
-    );
-    fcli!(server_mode, matches.value_of("mode"), "--mode");
-    fcli!(server_maxcon, matches.value_of("maxcon"), "--maxcon");
-    // bgsave settings
-    fcli!(
-        bgsave_settings,
-        Flag::<false>::new(matches.is_present("nosave")),
-        "--nosave",
-        matches.value_of("saveduration"),
-        "--saveduration"
-    );
-    // snapshot settings
-    fcli!(
-        snapshot_settings,
-        matches.value_of("snapevery"),
-        "--snapevery",
-        matches.value_of("snapkeep"),
-        "--snapkeep",
-        matches.value_of("stop-write-on-fail"),
-        "--stop-write-on-fail"
-    );
-    // TLS settings
-    fcli!(
-        tls_settings,
-        matches.value_of("sslkey"),
-        "--sslkey",
-        matches.value_of("sslchain"),
-        "--sslchain",
-        matches.value_of("sslport"),
-        "--sslport",
-        Flag::<true>::new(matches.is_present("sslonly")),
-        "--sslonly",
-        matches.value_of("tlspass"),
-        "--tlspassin"
-    );
-    // auth settings
-    fcli!(
-        auth_settings,
-        matches.value_of("authkey"),
-        "--auth-origin-key"
-    );
-    defset
-}
diff --git a/server/src/config/cfgenv.rs b/server/src/config/cfgenv.rs
deleted file mode 100644
index 23ead026..00000000
--- a/server/src/config/cfgenv.rs
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Created on Thu Jan 27 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use super::Configset;
-
-/// Returns the environment configuration
-pub(super) fn parse_env_config() -> Configset {
-    let mut defset = Configset::new_env();
-    macro_rules! fenv {
-        (
-            $fn:ident,
-            $(
-                $field:ident
-            ),*
-        ) => {
-            defset.$fn(
-                $(
-                    ::std::env::var(stringify!($field)),
-                    stringify!($field),
-                )*
-            );
-        };
-    }
-    // protocol settings
-    fenv!(protocol_settings, SKY_PROTOCOL_VERSION);
-    // server settings
-    fenv!(server_tcp, SKY_SYSTEM_HOST, SKY_SYSTEM_PORT);
-    fenv!(server_noart, SKY_SYSTEM_NOART);
-    fenv!(server_maxcon, SKY_SYSTEM_MAXCON);
-    fenv!(server_mode, SKY_DEPLOY_MODE);
-    // bgsave settings
-    fenv!(bgsave_settings, SKY_BGSAVE_ENABLED, SKY_BGSAVE_DURATION);
-    // snapshot settings
-    fenv!(
-        snapshot_settings,
-        SKY_SNAPSHOT_DURATION,
-        SKY_SNAPSHOT_KEEP,
-        SKY_SNAPSHOT_FAILSAFE
-    );
-    // TLS settings
-    fenv!(
-        tls_settings,
-        SKY_TLS_KEY,
-        SKY_TLS_CERT,
-        SKY_TLS_PORT,
-        SKY_TLS_ONLY,
-        SKY_TLS_PASSIN
-    );
-    fenv!(auth_settings, SKY_AUTH_ORIGIN_KEY);
-    defset
-}
diff --git a/server/src/config/cfgfile.rs b/server/src/config/cfgfile.rs
deleted file mode 100644
index d3283768..00000000
--- a/server/src/config/cfgfile.rs
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Created on Sat Oct 02 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{
-        AuthSettings, ConfigSourceParseResult, Configset, Modeset, OptString, ProtocolVersion,
-        TryFromConfigSource,
-    },
-    serde::Deserialize,
-    std::net::IpAddr,
-};
-
-/// This struct is an _object representation_ used for parsing the TOML file
-#[derive(Deserialize, Debug, PartialEq)]
-pub struct Config {
-    /// The `server` key
-    pub(super) server: ConfigKeyServer,
-    /// The `bgsave` key
-    pub(super) bgsave: Option<ConfigKeyBGSAVE>,
-    /// The snapshot key
-    pub(super) snapshot: Option<ConfigKeySnapshot>,
-    /// SSL configuration
-    pub(super) ssl: Option<KeySslOpts>,
-    /// auth settings
-    pub(super) auth: Option<AuthSettings>,
-}
-
-/// This struct represents the `server` key in the TOML file
-#[derive(Deserialize, Debug, PartialEq)]
-pub struct ConfigKeyServer {
-    /// The host key is any valid IPv4/IPv6 address
-    pub(super) host: IpAddr,
-    /// The port key is any valid port
-    pub(super) port: u16,
-    /// The noart key is an `Option`al boolean value which is set to true
-    /// for secure environments to disable terminal artwork
-    pub(super) noart: Option<bool>,
-    /// The maximum number of clients
-    pub(super) maxclient: Option<usize>,
-    /// The deployment mode
-    pub(super) mode: Option<Modeset>,
-    pub(super) protocol: Option<ProtocolVersion>,
-}
-
-/// The BGSAVE section in the config file
-#[derive(Deserialize, Debug, PartialEq)]
-pub struct ConfigKeyBGSAVE {
-    /// Whether BGSAVE is enabled or not
-    ///
-    /// If this key is missing, then we can assume that BGSAVE is enabled
-    pub(super) enabled: Option<bool>,
-    /// The duration after which BGSAVE should start
-    ///
-    /// If this is the only key specified, then it is clear that BGSAVE is enabled
-    /// and the duration is `every`
-    pub(super) every: Option<u64>,
-}
-
-/// The snapshot section in the TOML file
-#[derive(Deserialize, Debug, PartialEq)]
-pub struct ConfigKeySnapshot {
-    /// After how many seconds should the snapshot be created
-    pub(super) every: u64,
-    /// The maximum number of snapshots to keep
-    ///
-    /// If atmost is set to `0`, then all the snapshots will be kept
-    pub(super) atmost: usize,
-    /// Prevent writes to the database if snapshotting fails
-    pub(super) failsafe: Option<bool>,
-}
-
-#[derive(Deserialize, Debug, PartialEq)]
-pub struct KeySslOpts {
-    pub(super) key: String,
-    pub(super) chain: String,
-    pub(super) port: u16,
-    pub(super) only: Option<bool>,
-    pub(super) passin: Option<String>,
-}
-
-/// A custom non-null type for config files
-pub struct NonNull<T> {
-    val: T,
-}
-
-impl<T> From<T> for NonNull<T> {
-    fn from(val: T) -> Self {
-        Self { val }
-    }
-}
-
-impl<T> TryFromConfigSource<T> for NonNull<T> {
-    fn is_present(&self) -> bool {
-        true
-    }
-    fn mutate_failed(self, target: &mut T, trip: &mut bool) -> bool {
-        *target = self.val;
-        *trip = true;
-        false
-    }
-    fn try_parse(self) -> ConfigSourceParseResult<T> {
-        ConfigSourceParseResult::Okay(self.val)
-    }
-}
-
-pub struct Optional<T> {
-    base: Option<T>,
-}
-
-impl<T> Optional<T> {
-    pub const fn some(val: T) -> Self {
-        Self { base: Some(val) }
-    }
-}
-
-impl<T> From<Option<T>> for Optional<T> {
-    fn from(base: Option<T>) -> Self {
-        Self { base }
-    }
-}
-
-impl<T> TryFromConfigSource<T> for Optional<T> {
-    fn is_present(&self) -> bool {
-        self.base.is_some()
-    }
-    fn mutate_failed(self, target: &mut T, trip: &mut bool) -> bool {
-        if let Some(v) = self.base {
-            *trip = true;
-            *target = v;
-        }
-        false
-    }
-    fn try_parse(self) -> ConfigSourceParseResult<T> {
-        match self.base {
-            Some(v) => ConfigSourceParseResult::Okay(v),
-            None => ConfigSourceParseResult::Absent,
-        }
-    }
-}
-
-type ConfigFile = Config;
-
-pub fn from_file(file: ConfigFile) -> Configset {
-    let mut set = Configset::new_file();
-    let ConfigFile {
-        server,
-        bgsave,
-        snapshot,
-        ssl,
-        auth,
-    } = file;
-    // server settings
-    set.server_tcp(
-        Optional::some(server.host),
-        "server.host",
-        Optional::some(server.port),
-        "server.port",
-    );
-    set.protocol_settings(server.protocol, "server.protocol");
-    set.server_maxcon(Optional::from(server.maxclient), "server.maxcon");
-    set.server_noart(Optional::from(server.noart), "server.noart");
-    set.server_mode(Optional::from(server.mode), "server.mode");
-    // bgsave settings
-    if let Some(bgsave) = bgsave {
-        let ConfigKeyBGSAVE { enabled, every } = bgsave;
-        set.bgsave_settings(
-            Optional::from(enabled),
-            "bgsave.enabled",
-            Optional::from(every),
-            "bgsave.every",
-        );
-    }
-    // snapshot settings
-    if let Some(snapshot) = snapshot {
-        let ConfigKeySnapshot {
-            every,
-            atmost,
-            failsafe,
-        } = snapshot;
-        set.snapshot_settings(
-            NonNull::from(every),
-            "snapshot.every",
-            NonNull::from(atmost),
-            "snapshot.atmost",
-            Optional::from(failsafe),
-            "snapshot.failsafe",
-        );
-    }
-    // TLS settings
-    if let Some(tls) = ssl {
-        let KeySslOpts {
-            key,
-            chain,
-            port,
-            only,
-            passin,
-        } = tls;
-        set.tls_settings(
-            NonNull::from(key),
-            "ssl.key",
-            NonNull::from(chain),
-            "ssl.chain",
-            NonNull::from(port),
-            "ssl.port",
-            Optional::from(only),
-            "ssl.only",
-            OptString::from(passin),
-            "ssl.passin",
-        );
-    }
-    if let Some(auth) = auth {
-        let AuthSettings { origin_key } = auth;
-        set.auth_settings(Optional::from(origin_key), "auth.origin")
-    }
-    set
-}
diff --git a/server/src/config/definitions.rs b/server/src/config/definitions.rs
deleted file mode 100644
index fab2c0c7..00000000
--- a/server/src/config/definitions.rs
+++ /dev/null
@@ -1,504 +0,0 @@
-/*
- * Created on Fri Jan 28 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{feedback::WarningStack, DEFAULT_IPV4, DEFAULT_PORT},
-    crate::{config::AuthkeyWrapper, dbnet::MAXIMUM_CONNECTION_LIMIT},
-    core::{fmt, str::FromStr},
-    serde::{
-        de::{self, Deserializer, Visitor},
-        Deserialize,
-    },
-    std::net::IpAddr,
-};
-
-/// The BGSAVE configuration
-///
-/// If BGSAVE is enabled, then the duration (corresponding to `every`) is wrapped in the `Enabled`
-/// variant. Otherwise, the `Disabled` variant is to be used
-#[derive(PartialEq, Debug)]
-pub enum BGSave {
-    Enabled(u64),
-    Disabled,
-}
-
-impl BGSave {
-    /// Create a new BGSAVE configuration with all the fields
-    pub const fn new(enabled: bool, every: u64) -> Self {
-        if enabled {
-            BGSave::Enabled(every)
-        } else {
-            BGSave::Disabled
-        }
-    }
-    /// The default BGSAVE configuration
-    ///
-    /// Defaults:
-    /// - `enabled`: true
-    /// - `every`: 120
-    pub const fn default() -> Self {
-        BGSave::new(true, 120)
-    }
-    /// Check if BGSAVE is disabled
-    pub const fn is_disabled(&self) -> bool {
-        matches!(self, Self::Disabled)
-    }
-}
-
-#[repr(u8)]
-#[derive(Debug, PartialEq)]
-pub enum ProtocolVersion {
-    V1,
-    V2,
-}
-
-impl Default for ProtocolVersion {
-    fn default() -> Self {
-        Self::V2
-    }
-}
-
-impl ToString for ProtocolVersion {
-    fn to_string(&self) -> String {
-        match self {
-            Self::V1 => "Skyhash 1.0".to_owned(),
-            Self::V2 => "Skyhash 2.0".to_owned(),
-        }
-    }
-}
-
-struct ProtocolVersionVisitor;
-
-impl<'de> Visitor<'de> for ProtocolVersionVisitor {
-    type Value = ProtocolVersion;
-    fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "a 40 character ASCII string")
-    }
-    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
-    where
-        E: de::Error,
-    {
-        value.parse().map_err(|_| {
-            E::custom("Invalid value for protocol version. Valid inputs: 1.0, 1.1, 1.2, 2.0")
-        })
-    }
-}
-
-impl<'de> Deserialize<'de> for ProtocolVersion {
-    fn deserialize<D>(deserializer: D) -> Result<ProtocolVersion, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        deserializer.deserialize_str(ProtocolVersionVisitor)
-    }
-}
-
-/// A `ConfigurationSet` which can be used by main::check_args_or_connect() to bind
-/// to a `TcpListener` and show the corresponding terminal output for the given
-/// configuration
-#[derive(Debug, PartialEq)]
-pub struct ConfigurationSet {
-    /// If `noart` is set to true, no terminal artwork should be displayed
-    pub noart: bool,
-    /// The BGSAVE configuration
-    pub bgsave: BGSave,
-    /// The snapshot configuration
-    pub snapshot: SnapshotConfig,
-    /// Port configuration
-    pub ports: PortConfig,
-    /// The maximum number of connections
-    pub maxcon: usize,
-    /// The deployment mode
-    pub mode: Modeset,
-    /// The auth settings
-    pub auth: AuthSettings,
-    /// The protocol version
-    pub protocol: ProtocolVersion,
-}
-
-impl ConfigurationSet {
-    #[allow(clippy::too_many_arguments)]
-    pub const fn new(
-        noart: bool,
-        bgsave: BGSave,
-        snapshot: SnapshotConfig,
-        ports: PortConfig,
-        maxcon: usize,
-        mode: Modeset,
-        auth: AuthSettings,
-        protocol: ProtocolVersion,
-    ) -> Self {
-        Self {
-            noart,
-            bgsave,
-            snapshot,
-            ports,
-            maxcon,
-            mode,
-            auth,
-            protocol,
-        }
-    }
-    /// Create a default `ConfigurationSet` with the following setup defaults:
-    /// - `host`: 127.0.0.1
-    /// - `port` : 2003
-    /// - `noart` : false
-    /// - `bgsave_enabled` : true
-    /// - `bgsave_duration` : 120
-    /// - `ssl` : disabled
-    pub const fn default() -> Self {
-        Self::new(
-            false,
-            BGSave::default(),
-            SnapshotConfig::default(),
-            PortConfig::new_insecure_only(DEFAULT_IPV4, 2003),
-            MAXIMUM_CONNECTION_LIMIT,
-            Modeset::Dev,
-            AuthSettings::default(),
-            ProtocolVersion::V2,
-        )
-    }
-    /// Returns `false` if `noart` is enabled. Otherwise it returns `true`
-    pub const fn is_artful(&self) -> bool {
-        !self.noart
-    }
-}
-
-/// Port configuration
-///
-/// This enumeration determines whether the ports are:
-/// - `Multi`: This means that the database server will be listening to both
-/// SSL **and** non-SSL requests
-/// - `SecureOnly` : This means that the database server will only accept SSL requests
-/// and will not even activate the non-SSL socket
-/// - `InsecureOnly` : This indicates that the server would only accept non-SSL connections
-/// and will not even activate the SSL socket
-#[derive(Debug, PartialEq)]
-pub enum PortConfig {
-    SecureOnly {
-        host: IpAddr,
-        ssl: SslOpts,
-    },
-    Multi {
-        host: IpAddr,
-        port: u16,
-        ssl: SslOpts,
-    },
-    InsecureOnly {
-        host: IpAddr,
-        port: u16,
-    },
-}
-
-impl Default for PortConfig {
-    fn default() -> PortConfig {
-        PortConfig::InsecureOnly {
-            host: DEFAULT_IPV4,
-            port: DEFAULT_PORT,
-        }
-    }
-}
-
-impl PortConfig {
-    pub const fn new_secure_only(host: IpAddr, ssl: SslOpts) -> Self {
-        PortConfig::SecureOnly { host, ssl }
-    }
-    pub const fn new_insecure_only(host: IpAddr, port: u16) -> Self {
-        PortConfig::InsecureOnly { host, port }
-    }
-    pub fn get_host(&self) -> IpAddr {
-        match self {
-            Self::InsecureOnly { host, .. }
-            | Self::SecureOnly { host, .. }
-            | Self::Multi { host, .. } => *host,
-        }
-    }
-    pub fn upgrade_to_tls(&mut self, ssl: SslOpts) {
-        match self {
-            Self::InsecureOnly { host, port } => {
-                *self = Self::Multi {
-                    host: *host,
-                    port: *port,
-                    ssl,
-                }
-            }
-            Self::SecureOnly { .. } | Self::Multi { .. } => {
-                panic!("Port config is already upgraded to TLS")
-            }
-        }
-    }
-    pub const fn insecure_only(&self) -> bool {
-        matches!(self, Self::InsecureOnly { .. })
-    }
-    pub const fn secure_only(&self) -> bool {
-        matches!(self, Self::SecureOnly { .. })
-    }
-    pub fn get_description(&self) -> String {
-        match self {
-            Self::Multi { host, port, ssl } => {
-                format!(
-                    "skyhash://{host}:{port} and skyhash-secure://{host}:{tlsport}",
-                    tlsport = ssl.get_port(),
-                )
-            }
-            Self::SecureOnly {
-                host,
-                ssl: SslOpts { port, .. },
-            } => format!("skyhash-secure://{host}:{port}"),
-            Self::InsecureOnly { host, port } => format!("skyhash://{host}:{port}",),
-        }
-    }
-}
-
-#[derive(Deserialize, Debug, PartialEq)]
-pub struct SslOpts {
-    pub key: String,
-    pub chain: String,
-    pub port: u16,
-    pub passfile: Option<String>,
-}
-
-impl SslOpts {
-    pub const fn new(key: String, chain: String, port: u16, passfile: Option<String>) -> Self {
-        SslOpts {
-            key,
-            chain,
-            port,
-            passfile,
-        }
-    }
-    pub const fn get_port(&self) -> u16 {
-        self.port
-    }
-}
-
-#[derive(Debug, PartialEq)]
-/// The snapshot configuration
-///
-pub struct SnapshotPref {
-    /// Capture a snapshot `every` seconds
-    pub every: u64,
-    /// The maximum numeber of snapshots to be kept
-    pub atmost: usize,
-    /// Lock writes if snapshotting fails
-    pub poison: bool,
-}
-
-impl SnapshotPref {
-    /// Create a new a new `SnapshotPref` instance
-    pub const fn new(every: u64, atmost: usize, poison: bool) -> Self {
-        SnapshotPref {
-            every,
-            atmost,
-            poison,
-        }
-    }
-    /// Returns `every,almost` as a tuple for pattern matching
-    pub const fn decompose(self) -> (u64, usize, bool) {
-        (self.every, self.atmost, self.poison)
-    }
-}
-
-#[derive(Debug, PartialEq)]
-/// Snapshotting configuration
-///
-/// The variant `Enabled` directly carries a `ConfigKeySnapshot` object that
-/// is parsed from the configuration file, The variant `Disabled` is a ZST, and doesn't
-/// hold any data
-pub enum SnapshotConfig {
-    /// Snapshotting is enabled: this variant wraps around a `SnapshotPref`
-    /// object
-    Enabled(SnapshotPref),
-    /// Snapshotting is disabled
-    Disabled,
-}
-
-impl SnapshotConfig {
-    /// Snapshots are disabled by default, so `SnapshotConfig::Disabled` is the
-    /// default configuration
-    pub const fn default() -> Self {
-        SnapshotConfig::Disabled
-    }
-}
-
-type RestoreFile = Option<String>;
-
-#[derive(Debug, PartialEq)]
-/// The type of configuration:
-/// - The default configuration
-/// - A custom supplied configuration
-pub struct ConfigType {
-    pub(super) config: ConfigurationSet,
-    restore: RestoreFile,
-    is_custom: bool,
-    warnings: Option<WarningStack>,
-}
-
-impl ConfigType {
-    fn _new(
-        config: ConfigurationSet,
-        restore: RestoreFile,
-        is_custom: bool,
-        warnings: Option<WarningStack>,
-    ) -> Self {
-        Self {
-            config,
-            restore,
-            is_custom,
-            warnings,
-        }
-    }
-    pub fn print_warnings(&self) {
-        if let Some(warnings) = self.warnings.as_ref() {
-            warnings.print_warnings()
-        }
-    }
-    pub fn finish(self) -> (ConfigurationSet, Option<String>) {
-        (self.config, self.restore)
-    }
-    pub fn is_custom(&self) -> bool {
-        self.is_custom
-    }
-    pub fn is_artful(&self) -> bool {
-        self.config.is_artful()
-    }
-    pub fn new_custom(
-        config: ConfigurationSet,
-        restore: RestoreFile,
-        warnings: WarningStack,
-    ) -> Self {
-        Self::_new(config, restore, true, Some(warnings))
-    }
-    pub fn new_default(restore: RestoreFile) -> Self {
-        Self::_new(ConfigurationSet::default(), restore, false, None)
-    }
-    /// Check if the current deploy mode is prod
-    pub const fn is_prod_mode(&self) -> bool {
-        matches!(self.config.mode, Modeset::Prod)
-    }
-    pub fn wpush(&mut self, w: impl ToString) {
-        match self.warnings.as_mut() {
-            Some(stack) => stack.push(w),
-            None => {
-                self.warnings = {
-                    let mut wstack = WarningStack::new("");
-                    wstack.push(w);
-                    Some(wstack)
-                };
-            }
-        }
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub enum Modeset {
-    Dev,
-    Prod,
-}
-
-impl FromStr for Modeset {
-    type Err = ();
-    fn from_str(st: &str) -> Result<Modeset, Self::Err> {
-        match st {
-            "dev" => Ok(Modeset::Dev),
-            "prod" => Ok(Modeset::Prod),
-            _ => Err(()),
-        }
-    }
-}
-
-struct ModesetVisitor;
-
-impl<'de> Visitor<'de> for ModesetVisitor {
-    type Value = Modeset;
-    fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "Expecting a string with the deployment mode")
-    }
-    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
-    where
-        E: de::Error,
-    {
-        match value {
-            "dev" => Ok(Modeset::Dev),
-            "prod" => Ok(Modeset::Prod),
-            _ => Err(E::custom(format!("Bad value `{value}` for modeset"))),
-        }
-    }
-}
-
-impl<'de> Deserialize<'de> for Modeset {
-    fn deserialize<D>(deserializer: D) -> Result<Modeset, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        deserializer.deserialize_str(ModesetVisitor)
-    }
-}
-
-#[derive(Debug, PartialEq, Deserialize)]
-pub struct AuthSettings {
-    pub origin_key: Option<AuthkeyWrapper>,
-}
-
-impl AuthSettings {
-    pub const fn default() -> Self {
-        Self { origin_key: None }
-    }
-    #[cfg(test)]
-    pub fn new(origin: AuthkeyWrapper) -> Self {
-        Self {
-            origin_key: Some(origin),
-        }
-    }
-}
-
-struct AuthSettingsVisitor;
-
-impl<'de> Visitor<'de> for AuthSettingsVisitor {
-    type Value = AuthkeyWrapper;
-    fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "a 40 character ASCII string")
-    }
-    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
-    where
-        E: de::Error,
-    {
-        AuthkeyWrapper::try_new(value).ok_or_else(|| {
-            E::custom(
-                "Invalid value for authkey. must be 40 ASCII characters with nonzero first char",
-            )
-        })
-    }
-}
-
-impl<'de> Deserialize<'de> for AuthkeyWrapper {
-    fn deserialize<D>(deserializer: D) -> Result<AuthkeyWrapper, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        deserializer.deserialize_str(AuthSettingsVisitor)
-    }
-}
diff --git a/server/src/config/feedback.rs b/server/src/config/feedback.rs
deleted file mode 100644
index eb797ec3..00000000
--- a/server/src/config/feedback.rs
+++ /dev/null
@@ -1,314 +0,0 @@
-/*
- * Created on Tue Jan 25 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#[cfg(unix)]
-use crate::util::os::ResourceLimit;
-use {
-    super::{ConfigurationSet, SnapshotConfig, SnapshotPref},
-    core::{fmt, ops},
-    std::io::Error as IoError,
-    toml::de::Error as TomlError,
-};
-
-#[cfg(test)]
-const EMSG_ENV: &str = "Environment";
-const EMSG_PROD: &str = "Production mode";
-const TAB: &str = "    ";
-
-#[derive(Debug, PartialEq)]
-pub struct FeedbackStack {
-    stack: Vec<String>,
-    feedback_type: &'static str,
-    feedback_source: &'static str,
-}
-
-impl FeedbackStack {
-    fn new(feedback_source: &'static str, feedback_type: &'static str) -> Self {
-        Self {
-            stack: Vec::new(),
-            feedback_type,
-            feedback_source,
-        }
-    }
-    pub fn source(&self) -> &'static str {
-        self.feedback_source
-    }
-    pub fn push(&mut self, f: impl ToString) {
-        self.stack.push(f.to_string())
-    }
-}
-
-impl fmt::Display for FeedbackStack {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if !self.is_empty() {
-            if self.len() == 1 {
-                write!(
-                    f,
-                    "{} {}: {}",
-                    self.feedback_source, self.feedback_type, self.stack[0]
-                )?;
-            } else {
-                write!(f, "{} {}:", self.feedback_source, self.feedback_type)?;
-                for err in self.stack.iter() {
-                    write!(f, "\n{}- {}", TAB, err)?;
-                }
-            }
-        }
-        Ok(())
-    }
-}
-
-impl ops::Deref for FeedbackStack {
-    type Target = Vec<String>;
-    fn deref(&self) -> &Self::Target {
-        &self.stack
-    }
-}
-impl ops::DerefMut for FeedbackStack {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.stack
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub struct ErrorStack {
-    feedback: FeedbackStack,
-}
-
-impl ErrorStack {
-    pub fn new(err_source: &'static str) -> Self {
-        Self {
-            feedback: FeedbackStack::new(err_source, "errors"),
-        }
-    }
-}
-
-impl fmt::Display for ErrorStack {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", self.feedback)
-    }
-}
-
-impl ops::Deref for ErrorStack {
-    type Target = FeedbackStack;
-    fn deref(&self) -> &Self::Target {
-        &self.feedback
-    }
-}
-
-impl ops::DerefMut for ErrorStack {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.feedback
-    }
-}
-
-#[derive(Debug, PartialEq)]
-pub struct WarningStack {
-    feedback: FeedbackStack,
-}
-
-impl WarningStack {
-    pub fn new(warning_source: &'static str) -> Self {
-        Self {
-            feedback: FeedbackStack::new(warning_source, "warnings"),
-        }
-    }
-    pub fn print_warnings(&self) {
-        if !self.feedback.is_empty() {
-            log::warn!("{}", self);
-        }
-    }
-}
-
-impl ops::Deref for WarningStack {
-    type Target = FeedbackStack;
-    fn deref(&self) -> &Self::Target {
-        &self.feedback
-    }
-}
-
-impl ops::DerefMut for WarningStack {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.feedback
-    }
-}
-
-impl fmt::Display for WarningStack {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", self.feedback)
-    }
-}
-
-#[derive(Debug)]
-pub enum ConfigError {
-    OSError(IoError),
-    CfgError(ErrorStack),
-    ConfigFileParseError(TomlError),
-    Conflict,
-    ProdError(ErrorStack),
-}
-
-impl PartialEq for ConfigError {
-    fn eq(&self, oth: &Self) -> bool {
-        match (self, oth) {
-            (Self::OSError(lhs), Self::OSError(rhs)) => lhs.to_string() == rhs.to_string(),
-            (Self::CfgError(lhs), Self::CfgError(rhs)) => lhs == rhs,
-            (Self::ConfigFileParseError(lhs), Self::ConfigFileParseError(rhs)) => lhs == rhs,
-            (Self::Conflict, Self::Conflict) => true,
-            (Self::ProdError(lhs), Self::ProdError(rhs)) => lhs == rhs,
-            _ => false,
-        }
-    }
-}
-
-impl From<IoError> for ConfigError {
-    fn from(e: IoError) -> Self {
-        Self::OSError(e)
-    }
-}
-
-impl From<toml::de::Error> for ConfigError {
-    fn from(e: toml::de::Error) -> Self {
-        Self::ConfigFileParseError(e)
-    }
-}
-
-impl fmt::Display for ConfigError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::ConfigFileParseError(e) => write!(f, "Configuration file parse failed: {}", e),
-            Self::OSError(e) => write!(f, "OS Error: {}", e),
-            Self::CfgError(e) => write!(f, "{}", e),
-            Self::Conflict => write!(
-                f,
-                "Conflict error: Either provide CLI args, environment variables or a config file for configuration"
-            ),
-            Self::ProdError(e) => write!(f, "You have invalid configuration for production mode. {}", e)
-        }
-    }
-}
-
-#[cfg(unix)]
-fn check_rlimit_or_err(current: usize, estack: &mut ErrorStack) -> Result<(), ConfigError> {
-    let rlim = ResourceLimit::get()?;
-    if rlim.is_over_limit(current) {
-        estack.push(
-            "The value for maximum connections exceeds available resources to the server process",
-        );
-        estack.push(
-                format!(
-                "The current process is set to a resource limit of {current} and can be set to a maximum limit of {max} in the OS",
-                current=rlim.current(),max=rlim.max()
-            ));
-    }
-    Ok(())
-}
-
-#[cfg(not(unix))]
-fn check_rlimit_or_err(_: usize, _: &mut ErrorStack) -> Result<(), ConfigError> {
-    Ok(())
-}
-
-/// Check if the settings are suitable for use in production mode
-pub(super) fn evaluate_prod_settings(cfg: &ConfigurationSet) -> Result<(), ConfigError> {
-    let mut estack = ErrorStack::new(EMSG_PROD);
-    // check `noart`
-    if cfg.is_artful() {
-        estack.push("Terminal artwork should be disabled");
-    }
-    // first check BGSAVE
-    if cfg.bgsave.is_disabled() {
-        estack.push("BGSAVE must be enabled");
-    }
-    // now check snapshot settings (failsafe)
-    if let SnapshotConfig::Enabled(SnapshotPref { poison, .. }) = cfg.snapshot {
-        if !poison {
-            estack.push("Snapshots must be failsafe");
-        }
-    }
-    // now check TLS settings
-    if cfg.ports.insecure_only() {
-        estack.push("Either multi-socket (TCP and TLS) or TLS only must be enabled");
-    }
-    if cfg.auth.origin_key.is_some() && !cfg.ports.secure_only() {
-        estack.push("When authn+authz is enabled, TLS-only mode must be enabled");
-    }
-    check_rlimit_or_err(cfg.maxcon, &mut estack)?;
-    if estack.is_empty() {
-        Ok(())
-    } else {
-        Err(ConfigError::ProdError(estack))
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::{ConfigError, ErrorStack, WarningStack, EMSG_ENV, EMSG_PROD};
-
-    #[test]
-    fn errorstack_fmt() {
-        const EXPECTED: &str = "\
-Environment errors: Invalid value for `SKY_SYSTEM_PORT`. Expected a 16-bit integer\
-";
-        let mut estk = ErrorStack::new(EMSG_ENV);
-        estk.push("Invalid value for `SKY_SYSTEM_PORT`. Expected a 16-bit integer");
-        let fmt = format!("{}", estk);
-        assert_eq!(fmt, EXPECTED);
-    }
-
-    #[test]
-    fn warningstack_fmt() {
-        const EXPECTED: &str = "\
-Environment warnings:
-    - BGSAVE is disabled. You may lose data if the host crashes
-    - The setting for `maxcon` is too high\
-    ";
-        let mut wstk = WarningStack::new(EMSG_ENV);
-        wstk.push("BGSAVE is disabled. You may lose data if the host crashes");
-        wstk.push("The setting for `maxcon` is too high");
-        let fmt = format!("{}", wstk);
-        assert_eq!(fmt, EXPECTED);
-    }
-    #[test]
-    fn prod_mode_error_fmt() {
-        let mut estack = ErrorStack::new(EMSG_PROD);
-        estack.push("BGSAVE must be enabled");
-        estack.push("Snapshots must be failsafe");
-        estack.push("Either multi-socket (TCP and TLS) or TLS-only mode must be enabled");
-        estack.push(
-            "The value for maximum connections exceeds available resources to the server process",
-        );
-        let e = ConfigError::ProdError(estack);
-        const EXPECTED: &str = "\
-You have invalid configuration for production mode. Production mode errors:
-    - BGSAVE must be enabled
-    - Snapshots must be failsafe
-    - Either multi-socket (TCP and TLS) or TLS-only mode must be enabled
-    - The value for maximum connections exceeds available resources to the server process\
-";
-        assert_eq!(format!("{}", e), EXPECTED);
-    }
-}
diff --git a/server/src/config/mod.rs b/server/src/config/mod.rs
deleted file mode 100644
index 7fe2634c..00000000
--- a/server/src/config/mod.rs
+++ /dev/null
@@ -1,703 +0,0 @@
-/*
- * Created on Thu Jan 27 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::auth::provider::Authkey,
-    clap::{load_yaml, App},
-    core::str::FromStr,
-    std::{
-        env::VarError,
-        fs,
-        net::{IpAddr, Ipv4Addr},
-    },
-};
-
-// internal modules
-mod cfgcli;
-mod cfgenv;
-mod cfgfile;
-mod definitions;
-mod feedback;
-#[cfg(test)]
-mod tests;
-
-// internal imports
-use self::cfgfile::Config as ConfigFile;
-pub use self::definitions::*;
-use self::feedback::{ConfigError, ErrorStack, WarningStack};
-use crate::dbnet::MAXIMUM_CONNECTION_LIMIT;
-
-// server defaults
-const DEFAULT_IPV4: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1));
-const DEFAULT_PORT: u16 = 2003;
-// bgsave defaults
-const DEFAULT_BGSAVE_DURATION: u64 = 120;
-// snapshot defaults
-const DEFAULT_SNAPSHOT_FAILSAFE: bool = true;
-// TLS defaults
-const DEFAULT_SSL_PORT: u16 = 2004;
-
-type StaticStr = &'static str;
-
-#[derive(Debug, PartialEq)]
-pub struct AuthkeyWrapper(pub Authkey);
-
-impl AuthkeyWrapper {
-    pub const fn empty() -> Self {
-        Self([0u8; 40])
-    }
-    pub fn try_new(slf: &str) -> Option<Self> {
-        let valid = slf.len() == 40 // must have 40 chars
-        && slf.chars().all(|ch| char::is_ascii_alphanumeric(&ch)); // must have ascii alpha
-        if valid {
-            let mut ret = Self::empty();
-            slf.bytes().enumerate().for_each(|(idx, byte)| {
-                ret.0[idx] = byte;
-            });
-            Some(ret)
-        } else {
-            None
-        }
-    }
-    pub fn into_inner(self) -> Authkey {
-        self.0
-    }
-}
-
-#[derive(Debug)]
-/// An enum representing the outcome of a parse operation for a specific configuration item from a
-/// specific configuration source
-pub enum ConfigSourceParseResult<T> {
-    Okay(T),
-    Absent,
-    ParseFailure,
-}
-
-/// A trait for configuration sources. Any type implementing this trait is considered to be a valid
-/// source for configuration
-pub trait TryFromConfigSource<T: Sized>: Sized {
-    /// Check if the value is present
-    fn is_present(&self) -> bool;
-    /// Attempt to mutate the value if present. If any error occurs
-    /// while parsing the value, return true. Else return false if all went well.
-    /// Here:
-    /// - `target_value`: is a mutable reference to the target var
-    /// - `trip`: is a mutable ref to a bool that will be set to true if a value is present
-    /// (whether parseable or not)
-    fn mutate_failed(self, target_value: &mut T, trip: &mut bool) -> bool;
-    /// Attempt to parse the value into the target type
-    fn try_parse(self) -> ConfigSourceParseResult<T>;
-}
-
-impl<'a, T: FromStr + 'a> TryFromConfigSource<T> for Option<&'a str> {
-    fn is_present(&self) -> bool {
-        self.is_some()
-    }
-    fn mutate_failed(self, target_value: &mut T, trip: &mut bool) -> bool {
-        self.map(|slf| {
-            *trip = true;
-            match slf.parse() {
-                Ok(p) => {
-                    *target_value = p;
-                    false
-                }
-                Err(_) => true,
-            }
-        })
-        .unwrap_or(false)
-    }
-    fn try_parse(self) -> ConfigSourceParseResult<T> {
-        self.map(|s| {
-            s.parse()
-                .map(|ret| ConfigSourceParseResult::Okay(ret))
-                .unwrap_or(ConfigSourceParseResult::ParseFailure)
-        })
-        .unwrap_or(ConfigSourceParseResult::Absent)
-    }
-}
-
-impl FromStr for AuthkeyWrapper {
-    type Err = ();
-    fn from_str(slf: &str) -> Result<Self, Self::Err> {
-        Self::try_new(slf).ok_or(())
-    }
-}
-
-impl<'a, T: FromStr + 'a> TryFromConfigSource<T> for Result<String, VarError> {
-    fn is_present(&self) -> bool {
-        !matches!(self, Err(VarError::NotPresent))
-    }
-    fn mutate_failed(self, target_value: &mut T, trip: &mut bool) -> bool {
-        match self {
-            Ok(s) => {
-                *trip = true;
-                s.parse()
-                    .map(|v| {
-                        *target_value = v;
-                        false
-                    })
-                    .unwrap_or(true)
-            }
-            Err(e) => {
-                if matches!(e, VarError::NotPresent) {
-                    false
-                } else {
-                    // yes, we got the var but failed to parse it into unicode; so trip
-                    *trip = true;
-                    true
-                }
-            }
-        }
-    }
-    fn try_parse(self) -> ConfigSourceParseResult<T> {
-        match self {
-            Ok(s) => s
-                .parse()
-                .map(|v| ConfigSourceParseResult::Okay(v))
-                .unwrap_or(ConfigSourceParseResult::ParseFailure),
-            Err(e) => match e {
-                VarError::NotPresent => ConfigSourceParseResult::Absent,
-                VarError::NotUnicode(_) => ConfigSourceParseResult::ParseFailure,
-            },
-        }
-    }
-}
-
-#[derive(Debug, PartialEq, Default)]
-/// Since we have conflicting trait implementations, we define a custom `Option<String>` type
-pub struct OptString {
-    base: Option<String>,
-}
-
-impl OptString {
-    pub const fn new_null() -> Self {
-        Self { base: None }
-    }
-}
-
-impl From<Option<String>> for OptString {
-    fn from(base: Option<String>) -> Self {
-        Self { base }
-    }
-}
-
-impl FromStr for OptString {
-    type Err = ();
-    fn from_str(st: &str) -> Result<Self, Self::Err> {
-        Ok(Self {
-            base: Some(st.to_string()),
-        })
-    }
-}
-
-impl FromStr for ProtocolVersion {
-    type Err = ();
-    fn from_str(st: &str) -> Result<Self, Self::Err> {
-        match st {
-            "1" | "1.0" | "1.1" | "1.2" => Ok(Self::V1),
-            "2" | "2.0" => Ok(Self::V2),
-            _ => Err(()),
-        }
-    }
-}
-
-impl TryFromConfigSource<ProtocolVersion> for Option<ProtocolVersion> {
-    fn is_present(&self) -> bool {
-        self.is_some()
-    }
-    fn mutate_failed(self, target: &mut ProtocolVersion, trip: &mut bool) -> bool {
-        if let Some(v) = self {
-            *target = v;
-            *trip = true;
-        }
-        false
-    }
-    fn try_parse(self) -> ConfigSourceParseResult<ProtocolVersion> {
-        self.map(ConfigSourceParseResult::Okay)
-            .unwrap_or(ConfigSourceParseResult::Absent)
-    }
-}
-
-impl TryFromConfigSource<OptString> for OptString {
-    fn is_present(&self) -> bool {
-        self.base.is_some()
-    }
-    fn mutate_failed(self, target: &mut OptString, trip: &mut bool) -> bool {
-        if let Some(v) = self.base {
-            target.base = Some(v);
-            *trip = true;
-        }
-        false
-    }
-    fn try_parse(self) -> ConfigSourceParseResult<OptString> {
-        self.base
-            .map(|v| ConfigSourceParseResult::Okay(OptString { base: Some(v) }))
-            .unwrap_or(ConfigSourceParseResult::Absent)
-    }
-}
-
-#[derive(Debug)]
-/// A high-level configuration set that automatically handles errors, warnings and provides a convenient [`Result`]
-/// type that can be used
-pub struct Configset {
-    did_mutate: bool,
-    cfg: ConfigurationSet,
-    estack: ErrorStack,
-    wstack: WarningStack,
-}
-
-impl Configset {
-    const EMSG_ENV: StaticStr = "Environment";
-    const EMSG_CLI: StaticStr = "CLI";
-    const EMSG_FILE: StaticStr = "Configuration file";
-
-    /// Internal ctor for a given feedback source. We do not want to expose this to avoid
-    /// erroneous feedback source names
-    fn _new(feedback_source: StaticStr) -> Self {
-        Self {
-            did_mutate: false,
-            cfg: ConfigurationSet::default(),
-            estack: ErrorStack::new(feedback_source),
-            wstack: WarningStack::new(feedback_source),
-        }
-    }
-    /// Create a new configset for environment variables
-    pub fn new_env() -> Self {
-        Self::_new(Self::EMSG_ENV)
-    }
-    /// Create a new configset for CLI args
-    pub fn new_cli() -> Self {
-        Self::_new(Self::EMSG_CLI)
-    }
-    /// Create a new configset for config files
-    pub fn new_file() -> Self {
-        Self {
-            did_mutate: true,
-            cfg: ConfigurationSet::default(),
-            estack: ErrorStack::new(Self::EMSG_FILE),
-            wstack: WarningStack::new(Self::EMSG_FILE),
-        }
-    }
-    /// Mark the configset mutated
-    fn mutated(&mut self) {
-        self.did_mutate = true;
-    }
-    /// Push an error onto the error stack
-    fn epush(&mut self, field_key: StaticStr, expected: StaticStr) {
-        self.estack
-            .push(format!("Bad value for `{field_key}`. Expected {expected}",))
-    }
-    /// Check if no errors have occurred
-    pub fn is_okay(&self) -> bool {
-        self.estack.is_empty()
-    }
-    /// Check if the configset was mutated
-    pub fn is_mutated(&self) -> bool {
-        self.did_mutate
-    }
-    /// Attempt to mutate with a target `TryFromConfigSource` type, and push in any error that occurs
-    /// using the given diagnostic info
-    fn try_mutate<T>(
-        &mut self,
-        new: impl TryFromConfigSource<T>,
-        target: &mut T,
-        field_key: StaticStr,
-        expected: StaticStr,
-    ) {
-        if new.mutate_failed(target, &mut self.did_mutate) {
-            self.epush(field_key, expected)
-        }
-    }
-    /// Attempt to mutate with a target `TryFromConfigSource` type, and push in any error that occurs
-    /// using the given diagnostic info while checking the correctly parsed type using the provided validation
-    /// closure for any additional validation check that goes beyond type correctness
-    fn try_mutate_with_condcheck<T, F>(
-        &mut self,
-        new: impl TryFromConfigSource<T>,
-        target: &mut T,
-        field_key: StaticStr,
-        expected: StaticStr,
-        validation_fn: F,
-    ) where
-        F: Fn(&T) -> bool,
-    {
-        let mut needs_error = false;
-        match new.try_parse() {
-            ConfigSourceParseResult::Okay(ok) => {
-                self.mutated();
-                needs_error = !validation_fn(&ok);
-                *target = ok;
-            }
-            ConfigSourceParseResult::ParseFailure => {
-                self.mutated();
-                needs_error = true
-            }
-            ConfigSourceParseResult::Absent => {}
-        }
-        if needs_error {
-            self.epush(field_key, expected)
-        }
-    }
-    /// This method can be used to chain configurations to ultimately return the first modified configuration
-    /// that occurs. For example: `cfg_file.and_then(cfg_cli).and_then(cfg_env)`; it will return the first
-    /// modified Configset
-    ///
-    /// ## Panics
-    /// This method will panic if both the provided sets are mutated. Hence, you need to check beforehand that
-    /// there is no conflict
-    pub fn and_then(self, other: Self) -> Self {
-        if self.is_mutated() {
-            if other.is_mutated() {
-                panic!(
-                    "Double mutation: {env_a} and {env_b}",
-                    env_a = self.estack.source(),
-                    env_b = other.estack.source()
-                );
-            }
-            self
-        } else {
-            other
-        }
-    }
-    /// Turns self into a Result that can be used by config::get_config()
-    pub fn into_result(self, restore_file: Option<String>) -> Result<ConfigType, ConfigError> {
-        let mut target = if self.is_okay() {
-            // no errors, sweet
-            if self.is_mutated() {
-                let Self { cfg, wstack, .. } = self;
-                ConfigType::new_custom(cfg, restore_file, wstack)
-            } else {
-                ConfigType::new_default(restore_file)
-            }
-        } else {
-            return Err(ConfigError::CfgError(self.estack));
-        };
-        if target.config.protocol != ProtocolVersion::default() {
-            target.wpush(format!(
-                "{} is deprecated. Switch to {}",
-                target.config.protocol.to_string(),
-                ProtocolVersion::default().to_string()
-            ));
-        }
-        if target.is_prod_mode() {
-            self::feedback::evaluate_prod_settings(&target.config).map(|_| target)
-        } else {
-            target.wpush("Running in `user` mode. Set mode to `prod` in production");
-            Ok(target)
-        }
-    }
-}
-
-// protocol settings
-impl Configset {
-    pub fn protocol_settings(
-        &mut self,
-        nproto: impl TryFromConfigSource<ProtocolVersion>,
-        nproto_key: StaticStr,
-    ) {
-        let mut proto = ProtocolVersion::default();
-        self.try_mutate(
-            nproto,
-            &mut proto,
-            nproto_key,
-            "a protocol version like 2.0 or 1.0",
-        );
-        self.cfg.protocol = proto;
-    }
-}
-
-// server settings
-impl Configset {
-    pub fn server_tcp(
-        &mut self,
-        nhost: impl TryFromConfigSource<IpAddr>,
-        nhost_key: StaticStr,
-        nport: impl TryFromConfigSource<u16>,
-        nport_key: StaticStr,
-    ) {
-        let mut host = DEFAULT_IPV4;
-        let mut port = DEFAULT_PORT;
-        self.try_mutate(nhost, &mut host, nhost_key, "an IPv4/IPv6 address");
-        self.try_mutate(nport, &mut port, nport_key, "a 16-bit positive integer");
-        self.cfg.ports = PortConfig::new_insecure_only(host, port);
-    }
-    pub fn server_noart(&mut self, nart: impl TryFromConfigSource<bool>, nart_key: StaticStr) {
-        let mut noart = false;
-        self.try_mutate(nart, &mut noart, nart_key, "true/false");
-        self.cfg.noart = noart;
-    }
-    pub fn server_maxcon(
-        &mut self,
-        nmaxcon: impl TryFromConfigSource<usize>,
-        nmaxcon_key: StaticStr,
-    ) {
-        let mut maxcon = MAXIMUM_CONNECTION_LIMIT;
-        self.try_mutate_with_condcheck(
-            nmaxcon,
-            &mut maxcon,
-            nmaxcon_key,
-            "a positive integer greater than zero",
-            |max| *max > 0,
-        );
-        self.cfg.maxcon = maxcon;
-    }
-    pub fn server_mode(&mut self, nmode: impl TryFromConfigSource<Modeset>, nmode_key: StaticStr) {
-        let mut modeset = Modeset::Dev;
-        self.try_mutate(
-            nmode,
-            &mut modeset,
-            nmode_key,
-            "a string with 'user' or 'prod'",
-        );
-        self.cfg.mode = modeset;
-    }
-}
-
-// bgsave settings
-impl Configset {
-    pub fn bgsave_settings(
-        &mut self,
-        nenabled: impl TryFromConfigSource<bool>,
-        nenabled_key: StaticStr,
-        nduration: impl TryFromConfigSource<u64>,
-        nduration_key: StaticStr,
-    ) {
-        let mut enabled = true;
-        let mut duration = DEFAULT_BGSAVE_DURATION;
-        let has_custom_duration = nduration.is_present();
-        self.try_mutate(nenabled, &mut enabled, nenabled_key, "true/false");
-        self.try_mutate_with_condcheck(
-            nduration,
-            &mut duration,
-            nduration_key,
-            "a positive integer greater than zero",
-            |dur| *dur > 0,
-        );
-        if enabled {
-            self.cfg.bgsave = BGSave::Enabled(duration);
-        } else {
-            if has_custom_duration {
-                self.wstack.push(format!(
-                    "Specifying `{nduration_key}` is useless when BGSAVE is disabled"
-                ));
-            }
-            self.wstack
-                .push("BGSAVE is disabled. You may lose data if the host crashes");
-        }
-    }
-}
-
-// snapshot settings
-impl Configset {
-    pub fn snapshot_settings(
-        &mut self,
-        nevery: impl TryFromConfigSource<u64>,
-        nevery_key: StaticStr,
-        natmost: impl TryFromConfigSource<usize>,
-        natmost_key: StaticStr,
-        nfailsafe: impl TryFromConfigSource<bool>,
-        nfailsafe_key: StaticStr,
-    ) {
-        match (nevery.is_present(), natmost.is_present()) {
-            (false, false) => {
-                // noice, disabled
-                if nfailsafe.is_present() {
-                    // this mutation is pointless, but it is just for the sake of making sure
-                    // that the `failsafe` key has a proper boolean, no matter if it is pointless
-                    let mut _failsafe = DEFAULT_SNAPSHOT_FAILSAFE;
-                    self.try_mutate(nfailsafe, &mut _failsafe, nfailsafe_key, "true/false");
-                    self.wstack.push(format!(
-                        "Specifying `{nfailsafe_key}` is usless when snapshots are disabled"
-                    ));
-                }
-            }
-            (true, true) => {
-                let mut every = 0;
-                let mut atmost = 0;
-                let mut failsafe = DEFAULT_SNAPSHOT_FAILSAFE;
-                self.try_mutate_with_condcheck(
-                    nevery,
-                    &mut every,
-                    nevery_key,
-                    "an integer greater than 0",
-                    |dur| *dur > 0,
-                );
-                self.try_mutate(
-                    natmost,
-                    &mut atmost,
-                    natmost_key,
-                    "a positive integer. 0 indicates that all snapshots will be kept",
-                );
-                self.try_mutate(nfailsafe, &mut failsafe, nfailsafe_key, "true/false");
-                self.cfg.snapshot =
-                    SnapshotConfig::Enabled(SnapshotPref::new(every, atmost, failsafe));
-            }
-            (false, true) | (true, false) => {
-                // no changes, but still attempted to change
-                self.mutated();
-                self.estack.push(format!(
-                    "To use snapshots, pass values for both `{nevery_key}` and `{natmost_key}`"
-                ))
-            }
-        }
-    }
-}
-
-// TLS settings
-#[allow(clippy::too_many_arguments)]
-impl Configset {
-    pub fn tls_settings(
-        &mut self,
-        nkey: impl TryFromConfigSource<String>,
-        nkey_key: StaticStr,
-        ncert: impl TryFromConfigSource<String>,
-        ncert_key: StaticStr,
-        nport: impl TryFromConfigSource<u16>,
-        nport_key: StaticStr,
-        nonly: impl TryFromConfigSource<bool>,
-        nonly_key: StaticStr,
-        npass: impl TryFromConfigSource<OptString>,
-        npass_key: StaticStr,
-    ) {
-        match (nkey.is_present(), ncert.is_present()) {
-            (true, true) => {
-                // get the cert details
-                let mut key = String::new();
-                let mut cert = String::new();
-                self.try_mutate(nkey, &mut key, nkey_key, "path to private key file");
-                self.try_mutate(ncert, &mut cert, ncert_key, "path to TLS certificate file");
-
-                // now get port info
-                let mut port = DEFAULT_SSL_PORT;
-                self.try_mutate(nport, &mut port, nport_key, "a positive 16-bit integer");
-
-                // now check if TLS only
-                let mut tls_only = false;
-                self.try_mutate(nonly, &mut tls_only, nonly_key, "true/false");
-
-                // check if we have a TLS cert
-                let mut tls_pass = OptString::new_null();
-                self.try_mutate(
-                    npass,
-                    &mut tls_pass,
-                    npass_key,
-                    "path to TLS cert passphrase",
-                );
-
-                let sslopts = SslOpts::new(key, cert, port, tls_pass.base);
-                // now check if TLS only
-                if tls_only {
-                    let host = self.cfg.ports.get_host();
-                    self.cfg.ports = PortConfig::new_secure_only(host, sslopts)
-                } else {
-                    // multi. go and upgrade existing
-                    self.cfg.ports.upgrade_to_tls(sslopts);
-                }
-            }
-            (true, false) | (false, true) => {
-                self.mutated();
-                self.estack.push(format!(
-                    "To use TLS, pass values for both `{nkey_key}` and `{ncert_key}`"
-                ));
-            }
-            (false, false) => {
-                if nport.is_present() {
-                    self.mutated();
-                    self.wstack.push(format!(
-                        "Specifying `{nport_key}` is pointless when TLS is disabled"
-                    ));
-                }
-                if nonly.is_present() {
-                    self.mutated();
-                    self.wstack.push(format!(
-                        "Specifying `{nonly_key}` is pointless when TLS is disabled"
-                    ));
-                }
-                if npass.is_present() {
-                    self.mutated();
-                    self.wstack.push(format!(
-                        "Specifying `{npass_key}` is pointless when TLS is disabled"
-                    ));
-                }
-            }
-        }
-    }
-}
-
-// Auth settings
-impl Configset {
-    pub fn auth_settings(
-        &mut self,
-        nauth: impl TryFromConfigSource<AuthkeyWrapper>,
-        nauth_key: StaticStr,
-    ) {
-        let mut def = AuthkeyWrapper::empty();
-        self.try_mutate(nauth, &mut def, nauth_key, "A 40-byte long ASCII string");
-        if def != AuthkeyWrapper::empty() {
-            self.cfg.auth = AuthSettings {
-                origin_key: Some(def),
-            };
-        }
-    }
-}
-
-pub fn get_config() -> Result<ConfigType, ConfigError> {
-    // initialize clap because that will let us check for CLI/file configs
-    let cfg_layout = load_yaml!("../cli.yml");
-    let matches = App::from_yaml(cfg_layout).get_matches();
-    let restore_file = matches.value_of("restore").map(|v| v.to_string());
-
-    // get config from file
-    let cfg_from_file = if let Some(file) = matches.value_of("config") {
-        let file = fs::read_to_string(file)?;
-        let cfg_file: ConfigFile = toml::from_str(&file)?;
-        Some(cfgfile::from_file(cfg_file))
-    } else {
-        None
-    };
-
-    // get config from CLI
-    let cfg_from_cli = cfgcli::parse_cli_args(matches);
-    // get config from env
-    let cfg_from_env = cfgenv::parse_env_config();
-    // calculate the number of config sources
-    let cfg_degree = cfg_from_cli.is_mutated() as u8
-        + cfg_from_env.is_mutated() as u8
-        + cfg_from_file.is_some() as u8;
-    // if degree is more than 1, there is a conflict
-    let has_conflict = cfg_degree > 1;
-    if has_conflict {
-        return Err(ConfigError::Conflict);
-    }
-    if cfg_degree == 0 {
-        // no configuration, use default
-        Ok(ConfigType::new_default(restore_file))
-    } else {
-        cfg_from_file
-            .unwrap_or_else(|| cfg_from_env.and_then(cfg_from_cli))
-            .into_result(restore_file)
-    }
-}
diff --git a/server/src/config/tests.rs b/server/src/config/tests.rs
deleted file mode 100644
index ca06e29e..00000000
--- a/server/src/config/tests.rs
+++ /dev/null
@@ -1,836 +0,0 @@
-/*
- * Created on Thu Sep 23 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{BGSave, Configset, PortConfig, SnapshotConfig, SnapshotPref, SslOpts, DEFAULT_IPV4},
-    crate::ROOT_DIR,
-    std::fs,
-};
-
-// server tests
-// TCP
-#[test]
-fn server_tcp() {
-    let mut cfgset = Configset::new_env();
-    cfgset.server_tcp(
-        Some("127.0.0.1"),
-        "SKY_SERVER_HOST",
-        Some("2004"),
-        "SKY_SERVER_PORT",
-    );
-    assert_eq!(
-        cfgset.cfg.ports,
-        PortConfig::new_insecure_only(DEFAULT_IPV4, 2004)
-    );
-    assert!(cfgset.is_mutated());
-    assert!(cfgset.is_okay());
-}
-
-#[test]
-fn server_tcp_fail_host() {
-    let mut cfgset = Configset::new_env();
-    cfgset.server_tcp(
-        Some("?127.0.0.1"),
-        "SKY_SERVER_HOST",
-        Some("2004"),
-        "SKY_SERVER_PORT",
-    );
-    assert_eq!(
-        cfgset.cfg.ports,
-        PortConfig::new_insecure_only(DEFAULT_IPV4, 2004)
-    );
-    assert!(cfgset.is_mutated());
-    assert!(!cfgset.is_okay());
-    assert_eq!(
-        cfgset.estack[0],
-        "Bad value for `SKY_SERVER_HOST`. Expected an IPv4/IPv6 address"
-    );
-}
-
-#[test]
-fn server_tcp_fail_port() {
-    let mut cfgset = Configset::new_env();
-    cfgset.server_tcp(
-        Some("127.0.0.1"),
-        "SKY_SERVER_HOST",
-        Some("65537"),
-        "SKY_SERVER_PORT",
-    );
-    assert_eq!(
-        cfgset.cfg.ports,
-        PortConfig::new_insecure_only(DEFAULT_IPV4, 2003)
-    );
-    assert!(cfgset.is_mutated());
-    assert!(!cfgset.is_okay());
-    assert_eq!(
-        cfgset.estack[0],
-        "Bad value for `SKY_SERVER_PORT`. Expected a 16-bit positive integer"
-    );
-}
-
-#[test]
-fn server_tcp_fail_both() {
-    let mut cfgset = Configset::new_env();
-    cfgset.server_tcp(
-        Some("?127.0.0.1"),
-        "SKY_SERVER_HOST",
-        Some("65537"),
-        "SKY_SERVER_PORT",
-    );
-    assert_eq!(
-        cfgset.cfg.ports,
-        PortConfig::new_insecure_only(DEFAULT_IPV4, 2003)
-    );
-    assert!(cfgset.is_mutated());
-    assert!(!cfgset.is_okay());
-    assert_eq!(
-        cfgset.estack[0],
-        "Bad value for `SKY_SERVER_HOST`. Expected an IPv4/IPv6 address"
-    );
-    assert_eq!(
-        cfgset.estack[1],
-        "Bad value for `SKY_SERVER_PORT`. Expected a 16-bit positive integer"
-    );
-}
-
-// noart
-#[test]
-fn server_noart_okay() {
-    let mut cfgset = Configset::new_env();
-    cfgset.server_noart(Some("true"), "SKY_SYSTEM_NOART");
-    assert!(!cfgset.cfg.is_artful());
-    assert!(cfgset.is_okay());
-    assert!(cfgset.is_mutated());
-}
-
-#[test]
-fn server_noart_fail() {
-    let mut cfgset = Configset::new_env();
-    cfgset.server_noart(Some("truee"), "SKY_SYSTEM_NOART");
-    assert!(cfgset.cfg.is_artful());
-    assert!(!cfgset.is_okay());
-    assert_eq!(
-        cfgset.estack[0],
-        "Bad value for `SKY_SYSTEM_NOART`. Expected true/false"
-    );
-    assert!(cfgset.is_mutated());
-}
-
-#[test]
-fn server_maxcon_okay() {
-    let mut cfgset = Configset::new_env();
-    cfgset.server_maxcon(Some("12345"), "SKY_SYSTEM_MAXCON");
-    assert!(cfgset.is_mutated());
-    assert!(cfgset.is_okay());
-    assert_eq!(cfgset.cfg.maxcon, 12345);
-}
-
-#[test]
-fn server_maxcon_fail() {
-    let mut cfgset = Configset::new_env();
-    cfgset.server_maxcon(Some("12345A"), "SKY_SYSTEM_MAXCON");
-    assert!(cfgset.is_mutated());
-    assert!(!cfgset.is_okay());
-    assert_eq!(
-        cfgset.estack[0],
-        "Bad value for `SKY_SYSTEM_MAXCON`. Expected a positive integer greater than zero"
-    );
-    assert_eq!(cfgset.cfg.maxcon, 50000);
-}
-
-// bgsave settings
-#[test]
-fn bgsave_okay() {
-    let mut cfgset = Configset::new_env();
-    cfgset.bgsave_settings(
-        Some("true"),
-        "SKY_BGSAVE_ENABLED",
-        Some("128"),
-        "SKY_BGSAVE_DURATION",
-    );
-    assert!(cfgset.is_mutated());
-    assert!(cfgset.is_okay());
-    assert_eq!(cfgset.cfg.bgsave, BGSave::Enabled(128));
-}
-
-#[test]
-fn bgsave_fail() {
-    let mut cfgset = Configset::new_env();
-    cfgset.bgsave_settings(
-        Some("truee"),
-        "SKY_BGSAVE_ENABLED",
-        Some("128"),
-        "SKY_BGSAVE_DURATION",
-    );
-    assert!(cfgset.is_mutated());
-    assert!(!cfgset.is_okay());
-    assert_eq!(
-        cfgset.estack[0],
-        "Bad value for `SKY_BGSAVE_ENABLED`. Expected true/false"
-    );
-    assert_eq!(cfgset.cfg.bgsave, BGSave::Enabled(128));
-}
-
-// snapshot settings
-#[test]
-fn snapshot_okay() {
-    let mut cfgset = Configset::new_env();
-    cfgset.snapshot_settings(
-        Some("3600"),
-        "SKY_SNAPSHOT_EVERY",
-        Some("0"),
-        "SKY_SNAPSHOT_ATMOST",
-        Some("false"),
-        "SKY_SNAPSHOT_FAILSAFE",
-    );
-    assert!(cfgset.is_mutated());
-    assert!(cfgset.is_okay());
-    assert_eq!(
-        cfgset.cfg.snapshot,
-        SnapshotConfig::Enabled(SnapshotPref::new(3600, 0, false))
-    );
-}
-
-#[test]
-fn snapshot_fail() {
-    let mut cfgset = Configset::new_env();
-    cfgset.snapshot_settings(
-        Some("3600"),
-        "SKY_SNAPSHOT_EVERY",
-        Some("0"),
-        "SKY_SNAPSHOT_ATMOST",
-        Some("falsee"),
-        "SKY_SNAPSHOT_FAILSAFE",
-    );
-    assert!(cfgset.is_mutated());
-    assert!(!cfgset.is_okay());
-    assert_eq!(
-        cfgset.estack[0],
-        "Bad value for `SKY_SNAPSHOT_FAILSAFE`. Expected true/false"
-    );
-    assert_eq!(
-        cfgset.cfg.snapshot,
-        SnapshotConfig::Enabled(SnapshotPref::new(3600, 0, true))
-    );
-}
-
-#[test]
-fn snapshot_fail_with_missing_required_values() {
-    let mut cfgset = Configset::new_env();
-    cfgset.snapshot_settings(
-        Some("3600"),
-        "SKY_SNAPSHOT_EVERY",
-        None,
-        "SKY_SNAPSHOT_ATMOST",
-        None,
-        "SKY_SNAPSHOT_FAILSAFE",
-    );
-    assert!(cfgset.is_mutated());
-    assert!(!cfgset.is_okay());
-    assert_eq!(
-        cfgset.estack[0],
-        "To use snapshots, pass values for both `SKY_SNAPSHOT_EVERY` and `SKY_SNAPSHOT_ATMOST`"
-    );
-    assert_eq!(cfgset.cfg.snapshot, SnapshotConfig::Disabled);
-}
-
-// TLS settings
-#[test]
-fn tls_settings_okay() {
-    let mut cfg = Configset::new_env();
-    cfg.tls_settings(
-        Some("key.pem"),
-        "SKY_TLS_KEY",
-        Some("cert.pem"),
-        "SKY_TLS_CERT",
-        Some("2005"),
-        "SKY_TLS_PORT",
-        Some("false"),
-        "SKY_TLS_ONLY",
-        None,
-        "SKY_TLS_PASSIN",
-    );
-    assert!(cfg.is_mutated());
-    assert!(cfg.is_okay());
-    assert_eq!(cfg.cfg.ports, {
-        let mut pf = PortConfig::default();
-        pf.upgrade_to_tls(SslOpts::new(
-            "key.pem".to_owned(),
-            "cert.pem".to_owned(),
-            2005,
-            None,
-        ));
-        pf
-    });
-}
-
-#[test]
-fn tls_settings_fail() {
-    let mut cfg = Configset::new_env();
-    cfg.tls_settings(
-        Some("key.pem"),
-        "SKY_TLS_KEY",
-        Some("cert.pem"),
-        "SKY_TLS_CERT",
-        Some("A2005"),
-        "SKY_TLS_PORT",
-        Some("false"),
-        "SKY_TLS_ONLY",
-        None,
-        "SKY_TLS_PASSIN",
-    );
-    assert!(cfg.is_mutated());
-    assert!(!cfg.is_okay());
-    assert_eq!(cfg.cfg.ports, {
-        let mut pf = PortConfig::default();
-        pf.upgrade_to_tls(SslOpts::new(
-            "key.pem".to_owned(),
-            "cert.pem".to_owned(),
-            2004,
-            None,
-        ));
-        pf
-    });
-}
-
-#[test]
-fn tls_settings_fail_with_missing_required_values() {
-    let mut cfg = Configset::new_env();
-    cfg.tls_settings(
-        Some("key.pem"),
-        "SKY_TLS_KEY",
-        None,
-        "SKY_TLS_CERT",
-        Some("2005"),
-        "SKY_TLS_PORT",
-        Some("false"),
-        "SKY_TLS_ONLY",
-        None,
-        "SKY_TLS_PASSIN",
-    );
-    assert!(cfg.is_mutated());
-    assert!(!cfg.is_okay());
-    assert_eq!(cfg.cfg.ports, PortConfig::default());
-}
-
-/// Gets a `toml` file from `WORKSPACEROOT/examples/config-files`
-fn get_toml_from_examples_dir(filename: &str) -> String {
-    let path = format!("{ROOT_DIR}examples/config-files/{filename}");
-    fs::read_to_string(path).unwrap()
-}
-
-mod cfg_file_tests {
-    use super::get_toml_from_examples_dir;
-    use crate::config::AuthkeyWrapper;
-    use crate::config::{
-        cfgfile, AuthSettings, BGSave, Configset, ConfigurationSet, Modeset, PortConfig,
-        ProtocolVersion, SnapshotConfig, SnapshotPref, SslOpts, DEFAULT_IPV4, DEFAULT_PORT,
-    };
-    use crate::dbnet::MAXIMUM_CONNECTION_LIMIT;
-    use std::net::{IpAddr, Ipv6Addr};
-
-    fn cfgset_from_toml_str(file: String) -> Result<Configset, toml::de::Error> {
-        let toml = toml::from_str(&file)?;
-        Ok(cfgfile::from_file(toml))
-    }
-
-    #[test]
-    fn config_file_okay() {
-        let file = get_toml_from_examples_dir("template.toml");
-        let toml = toml::from_str(&file).unwrap();
-        let cfg_from_file = cfgfile::from_file(toml);
-        assert!(cfg_from_file.is_mutated());
-        assert!(cfg_from_file.is_okay());
-        // expected
-        let mut expected = ConfigurationSet::default();
-        expected.snapshot = SnapshotConfig::Enabled(SnapshotPref::new(3600, 4, true));
-        expected.ports = PortConfig::new_secure_only(
-            crate::config::DEFAULT_IPV4,
-            SslOpts::new(
-                "/path/to/keyfile.pem".to_owned(),
-                "/path/to/chain.pem".to_owned(),
-                2004,
-                Some("/path/to/cert/passphrase.txt".to_owned()),
-            ),
-        );
-        expected.auth.origin_key =
-            Some(AuthkeyWrapper::try_new(crate::TEST_AUTH_ORIGIN_KEY).unwrap());
-        // check
-        assert_eq!(cfg_from_file.cfg, expected);
-    }
-
-    #[test]
-    fn test_config_file_ok() {
-        let file = get_toml_from_examples_dir("skyd.toml");
-        let cfg = cfgset_from_toml_str(file).unwrap();
-        assert_eq!(cfg.cfg, ConfigurationSet::default());
-    }
-
-    #[test]
-    fn test_config_file_noart() {
-        let file = get_toml_from_examples_dir("secure-noart.toml");
-        let cfg = cfgset_from_toml_str(file).unwrap();
-        assert_eq!(
-            cfg.cfg,
-            ConfigurationSet {
-                noart: true,
-                bgsave: BGSave::default(),
-                snapshot: SnapshotConfig::default(),
-                ports: PortConfig::default(),
-                maxcon: MAXIMUM_CONNECTION_LIMIT,
-                mode: Modeset::Dev,
-                auth: AuthSettings::default(),
-                protocol: ProtocolVersion::default(),
-            }
-        );
-    }
-
-    #[test]
-    fn test_config_file_ipv6() {
-        let file = get_toml_from_examples_dir("ipv6.toml");
-        let cfg = cfgset_from_toml_str(file).unwrap();
-        assert_eq!(
-            cfg.cfg,
-            ConfigurationSet {
-                noart: false,
-                bgsave: BGSave::default(),
-                snapshot: SnapshotConfig::default(),
-                ports: PortConfig::new_insecure_only(
-                    IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0x1)),
-                    DEFAULT_PORT
-                ),
-                maxcon: MAXIMUM_CONNECTION_LIMIT,
-                mode: Modeset::Dev,
-                auth: AuthSettings::default(),
-                protocol: ProtocolVersion::default(),
-            }
-        );
-    }
-
-    #[test]
-    fn test_config_file_template() {
-        let file = get_toml_from_examples_dir("template.toml");
-        let cfg = cfgset_from_toml_str(file).unwrap();
-        assert_eq!(
-            cfg.cfg,
-            ConfigurationSet::new(
-                false,
-                BGSave::default(),
-                SnapshotConfig::Enabled(SnapshotPref::new(3600, 4, true)),
-                PortConfig::new_secure_only(
-                    DEFAULT_IPV4,
-                    SslOpts::new(
-                        "/path/to/keyfile.pem".into(),
-                        "/path/to/chain.pem".into(),
-                        2004,
-                        Some("/path/to/cert/passphrase.txt".to_owned())
-                    )
-                ),
-                MAXIMUM_CONNECTION_LIMIT,
-                Modeset::Dev,
-                AuthSettings::new(AuthkeyWrapper::try_new(crate::TEST_AUTH_ORIGIN_KEY).unwrap()),
-                ProtocolVersion::default()
-            )
-        );
-    }
-
-    #[test]
-    fn test_config_file_bad_bgsave_section() {
-        let file = get_toml_from_examples_dir("badcfg2.toml");
-        let cfg = cfgset_from_toml_str(file);
-        assert!(cfg.is_err());
-    }
-
-    #[test]
-    fn test_config_file_custom_bgsave() {
-        let file = get_toml_from_examples_dir("withcustombgsave.toml");
-        let cfg = cfgset_from_toml_str(file).unwrap();
-        assert_eq!(
-            cfg.cfg,
-            ConfigurationSet {
-                noart: false,
-                bgsave: BGSave::new(true, 600),
-                snapshot: SnapshotConfig::default(),
-                ports: PortConfig::default(),
-                maxcon: MAXIMUM_CONNECTION_LIMIT,
-                mode: Modeset::Dev,
-                auth: AuthSettings::default(),
-                protocol: ProtocolVersion::default(),
-            }
-        );
-    }
-
-    #[test]
-    fn test_config_file_bgsave_enabled_only() {
-        /*
-         * This test demonstrates a case where the user just said that BGSAVE is enabled.
-         * In that case, we will default to the 120 second duration
-         */
-        let file = get_toml_from_examples_dir("bgsave-justenabled.toml");
-        let cfg = cfgset_from_toml_str(file).unwrap();
-        assert_eq!(
-            cfg.cfg,
-            ConfigurationSet {
-                noart: false,
-                bgsave: BGSave::default(),
-                snapshot: SnapshotConfig::default(),
-                ports: PortConfig::default(),
-                maxcon: MAXIMUM_CONNECTION_LIMIT,
-                mode: Modeset::Dev,
-                auth: AuthSettings::default(),
-                protocol: ProtocolVersion::default(),
-            }
-        )
-    }
-
-    #[test]
-    fn test_config_file_bgsave_every_only() {
-        /*
-         * This test demonstrates a case where the user just gave the value for every
-         * In that case, it means BGSAVE is enabled and set to `every` seconds
-         */
-        let file = get_toml_from_examples_dir("bgsave-justevery.toml");
-        let cfg = cfgset_from_toml_str(file).unwrap();
-        assert_eq!(
-            cfg.cfg,
-            ConfigurationSet {
-                noart: false,
-                bgsave: BGSave::new(true, 600),
-                snapshot: SnapshotConfig::default(),
-                ports: PortConfig::default(),
-                maxcon: MAXIMUM_CONNECTION_LIMIT,
-                mode: Modeset::Dev,
-                auth: AuthSettings::default(),
-                protocol: ProtocolVersion::default(),
-            }
-        )
-    }
-
-    #[test]
-    fn test_config_file_snapshot() {
-        let file = get_toml_from_examples_dir("snapshot.toml");
-        let cfg = cfgset_from_toml_str(file).unwrap();
-        assert_eq!(
-            cfg.cfg,
-            ConfigurationSet {
-                snapshot: SnapshotConfig::Enabled(SnapshotPref::new(3600, 4, true)),
-                bgsave: BGSave::default(),
-                noart: false,
-                ports: PortConfig::default(),
-                maxcon: MAXIMUM_CONNECTION_LIMIT,
-                mode: Modeset::Dev,
-                auth: AuthSettings::default(),
-                protocol: ProtocolVersion::default(),
-            }
-        );
-    }
-}
-
-mod cli_arg_tests {
-    use crate::config::{cfgcli, PortConfig};
-    use clap::{load_yaml, App};
-    #[test]
-    fn cli_args_okay() {
-        let cfg_layout = load_yaml!("../cli.yml");
-        let cli_args = ["skyd", "--host", "127.0.0.2"];
-        let matches = App::from_yaml(cfg_layout).get_matches_from(&cli_args);
-        let ret = cfgcli::parse_cli_args(matches);
-        assert_eq!(
-            ret.cfg.ports,
-            PortConfig::new_insecure_only("127.0.0.2".parse().unwrap(), 2003)
-        );
-        assert!(ret.is_mutated());
-        assert!(ret.is_okay());
-    }
-    #[test]
-    fn cli_args_okay_no_mut() {
-        let cfg_layout = load_yaml!("../cli.yml");
-        let cli_args = ["skyd", "--restore", "/some/restore/path"];
-        let matches = App::from_yaml(cfg_layout).get_matches_from(&cli_args);
-        let ret = cfgcli::parse_cli_args(matches);
-        assert!(!ret.is_mutated());
-        assert!(ret.is_okay());
-    }
-    #[test]
-    fn cli_args_fail() {
-        let cfg_layout = load_yaml!("../cli.yml");
-        let cli_args = ["skyd", "--port", "port2003"];
-        let matches = App::from_yaml(cfg_layout).get_matches_from(&cli_args);
-        let ret = cfgcli::parse_cli_args(matches);
-        assert!(ret.is_mutated());
-        assert!(!ret.is_okay());
-        assert_eq!(
-            ret.estack[0],
-            "Bad value for `--port`. Expected a 16-bit positive integer"
-        );
-    }
-}
-
-mod try_from_config_source_impls {
-    use crate::config::{cfgcli::Flag, cfgfile::Optional, TryFromConfigSource, DEFAULT_IPV4};
-    use std::env::{set_var, var};
-    use std::fmt::Debug;
-
-    const EXPECT_TRUE: bool = true;
-    const EXPECT_FALSE: bool = false;
-    const MUTATED: bool = true;
-    const NOT_MUTATED: bool = false;
-    const IS_PRESENT: bool = true;
-    const IS_ABSENT: bool = false;
-    const MUTATION_FAILURE: bool = true;
-    const NO_MUTATION_FAILURE: bool = false;
-
-    fn _mut_base_test_expected<T: Default + PartialEq + Debug>(
-        new: impl TryFromConfigSource<T>,
-        expected: T,
-        is_present: bool,
-        mutate_failed: bool,
-        has_mutated: bool,
-    ) {
-        let mut default = Default::default();
-        let mut mutated = false;
-        assert_eq!(new.is_present(), is_present);
-        assert_eq!(new.mutate_failed(&mut default, &mut mutated), mutate_failed);
-        assert_eq!(mutated, has_mutated);
-        assert_eq!(default, expected);
-    }
-
-    fn _mut_base_test<T>(
-        new: impl TryFromConfigSource<T>,
-        mut default: T,
-        is_present: bool,
-        mutate_failed: bool,
-        has_mutated: bool,
-    ) {
-        let mut mutated = false;
-        assert_eq!(new.is_present(), is_present);
-        assert_eq!(new.mutate_failed(&mut default, &mut mutated), mutate_failed);
-        assert_eq!(mutated, has_mutated);
-    }
-
-    fn mut_test_pass<T>(new: impl TryFromConfigSource<T>, default: T) {
-        _mut_base_test(new, default, IS_PRESENT, NO_MUTATION_FAILURE, MUTATED)
-    }
-
-    fn mut_test_fail<T>(new: impl TryFromConfigSource<T>, default: T) {
-        _mut_base_test(new, default, IS_PRESENT, MUTATION_FAILURE, MUTATED)
-    }
-
-    mod env_var {
-        use super::*;
-
-        // test for Result<String, VarError>
-        #[test]
-        fn env_okay_ipv4() {
-            set_var("TEST_SKY_SYSTEM_HOST", "127.0.0.1");
-            mut_test_pass(var("TEST_SKY_SYSTEM_HOST"), DEFAULT_IPV4);
-        }
-
-        #[test]
-        fn env_fail_ipv4() {
-            set_var("TEST_SKY_SYSTEM_HOST2", "127.0.0.1A");
-            mut_test_fail(var("TEST_SKY_SYSTEM_HOST2"), DEFAULT_IPV4);
-        }
-    }
-
-    mod option_str {
-        use super::*;
-
-        // test for Option<&str> (as in CLI)
-        #[test]
-        fn option_str_okay_ipv4() {
-            let ip = Some("127.0.0.1");
-            mut_test_pass(ip, DEFAULT_IPV4);
-        }
-
-        #[test]
-        fn option_str_fail_ipv4() {
-            let ip = Some("127.0.0.1A");
-            mut_test_fail(ip, DEFAULT_IPV4);
-        }
-
-        #[test]
-        fn option_str_nomut() {
-            let ip = None;
-            _mut_base_test(
-                ip,
-                DEFAULT_IPV4,
-                IS_ABSENT,
-                NO_MUTATION_FAILURE,
-                NOT_MUTATED,
-            );
-        }
-    }
-
-    mod cfgcli_flag {
-        use super::*;
-
-        #[test]
-        fn flag_true_if_set_okay_set() {
-            // this is true if flag is present
-            let flag = Flag::<true>::new(true);
-            // we expect true
-            _mut_base_test_expected(flag, EXPECT_TRUE, IS_PRESENT, NO_MUTATION_FAILURE, MUTATED);
-        }
-
-        #[test]
-        fn flag_true_if_set_okay_unset() {
-            // this is true if flag is present, but the flag here is not present
-            let flag = Flag::<true>::new(false);
-            // we expect no mutation because the flag was not set
-            _mut_base_test(
-                flag,
-                EXPECT_FALSE,
-                IS_ABSENT,
-                NO_MUTATION_FAILURE,
-                NOT_MUTATED,
-            );
-        }
-
-        #[test]
-        fn flag_false_if_set_okay_set() {
-            // this is false if flag is present
-            let flag = Flag::<false>::new(true);
-            // expect mutation to have happened
-            _mut_base_test_expected(flag, EXPECT_FALSE, IS_PRESENT, NO_MUTATION_FAILURE, MUTATED);
-        }
-
-        #[test]
-        fn flag_false_if_set_okay_unset() {
-            // this is false if flag is present, but the flag is absent
-            let flag = Flag::<true>::new(false);
-            // expect no mutation
-            _mut_base_test(
-                flag,
-                EXPECT_FALSE,
-                IS_ABSENT,
-                NO_MUTATION_FAILURE,
-                NOT_MUTATED,
-            );
-        }
-    }
-
-    mod optional {
-        use super::*;
-
-        // test for cfg file scenario
-        #[test]
-        fn optional_okay_ipv4() {
-            let ip = Optional::some(DEFAULT_IPV4);
-            mut_test_pass(ip, DEFAULT_IPV4);
-        }
-
-        #[test]
-        fn optional_okay_ipv4_none() {
-            let ip = Optional::from(None);
-            _mut_base_test(
-                ip,
-                DEFAULT_IPV4,
-                IS_ABSENT,
-                NO_MUTATION_FAILURE,
-                NOT_MUTATED,
-            );
-        }
-    }
-
-    mod cfgfile_nonull {
-        use super::*;
-        use crate::config::cfgfile::NonNull;
-
-        #[test]
-        fn nonnull_okay() {
-            let port = NonNull::from(2100);
-            _mut_base_test_expected(port, 2100, IS_PRESENT, NO_MUTATION_FAILURE, MUTATED);
-        }
-    }
-
-    mod optstring {
-        use super::*;
-        use crate::config::OptString;
-
-        #[test]
-        fn optstring_okay() {
-            let pass = OptString::from(Some("tlspass.txt".to_owned()));
-            _mut_base_test_expected(
-                pass,
-                OptString::from(Some("tlspass.txt".to_owned())),
-                IS_PRESENT,
-                NO_MUTATION_FAILURE,
-                MUTATED,
-            );
-        }
-
-        #[test]
-        fn optstring_null_okay() {
-            let pass = OptString::from(None);
-            _mut_base_test_expected(
-                pass,
-                OptString::new_null(),
-                IS_ABSENT,
-                NO_MUTATION_FAILURE,
-                NOT_MUTATED,
-            );
-        }
-    }
-}
-
-mod modeset_de {
-    use crate::config::Modeset;
-    use serde::Deserialize;
-
-    #[derive(Deserialize, Debug)]
-    struct Example {
-        mode: Modeset,
-    }
-
-    #[test]
-    fn deserialize_modeset_prod_okay() {
-        #[derive(Deserialize, Debug)]
-        struct Example {
-            mode: Modeset,
-        }
-        let toml = r#"mode="prod""#;
-        let x: Example = toml::from_str(toml).unwrap();
-        assert_eq!(x.mode, Modeset::Prod);
-    }
-
-    #[test]
-    fn deserialize_modeset_user_okay() {
-        let toml = r#"mode="dev""#;
-        let x: Example = toml::from_str(toml).unwrap();
-        assert_eq!(x.mode, Modeset::Dev);
-    }
-
-    #[test]
-    fn deserialize_modeset_fail() {
-        let toml = r#"mode="superuser""#;
-        let e = toml::from_str::<Example>(toml).unwrap_err();
-        assert_eq!(
-            e.to_string(),
-            "TOML parse error at line 1, column 6\n  |\n1 | mode=\"superuser\"\n  |      ^^^^^^^^^^^\nBad value `superuser` for modeset\n"
-        );
-    }
-}
diff --git a/server/src/corestore/array.rs b/server/src/corestore/array.rs
deleted file mode 100644
index 199be981..00000000
--- a/server/src/corestore/array.rs
+++ /dev/null
@@ -1,624 +0,0 @@
-/*
- * Created on Tue Jul 06 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use core::{
-    any,
-    borrow::{Borrow, BorrowMut},
-    cmp::Ordering,
-    fmt,
-    hash::{Hash, Hasher},
-    iter::FromIterator,
-    mem::{ManuallyDrop, MaybeUninit},
-    ops, ptr, slice, str,
-};
-
-/// A compile-time, fixed size array that can have unintialized memory. This array is as
-/// efficient as you'd expect a normal array to be, but with the added benefit that you
-/// don't have to initialize all the elements. This was inspired by the arrayvec crate.
-/// Safe abstractions are made available enabling us to not enter uninitialized space and
-/// read the _available_ elements. The array size is limited to 16 bits or 2 bytes to
-/// prevent stack overflows.
-///
-/// ## Panics
-/// To avoid stack corruption among other crazy things, several implementations like [`Extend`]
-/// can panic. There are _silently corrupting_ methods too which can be used if you can uphold
-/// the guarantees
-pub struct Array<T, const N: usize> {
-    /// the maybe bad stack
-    stack: [MaybeUninit<T>; N],
-    /// the initialized length
-    /// no stack should be more than 16 bytes
-    init_len: u16,
-}
-
-/// The len scopeguard is like a scopeguard that provides panic safety incase an append-like
-/// operation involving iterators causes the iterator to panic. This makes sure that we still
-/// set the len on panic
-pub struct LenScopeGuard<'a, T: Copy> {
-    real_ref: &'a mut T,
-    temp: T,
-}
-
-impl<'a, T: ops::AddAssign + Copy> LenScopeGuard<'a, T> {
-    pub fn new(real_ref: &'a mut T) -> Self {
-        let ret = *real_ref;
-        Self {
-            real_ref,
-            temp: ret,
-        }
-    }
-    pub fn incr(&mut self, val: T) {
-        self.temp += val;
-    }
-    pub fn get_temp(&self) -> T {
-        self.temp
-    }
-}
-
-impl<'a, T: Copy> Drop for LenScopeGuard<'a, T> {
-    fn drop(&mut self) {
-        *self.real_ref = self.temp;
-    }
-}
-
-macro_rules! impl_zeroed_nm {
-    ($($ty:ty),* $(,)?) => {
-        $(
-            impl<const N: usize> Array<$ty, N> {
-                pub const fn new_zeroed() -> Self {
-                    Self {
-                        stack: [MaybeUninit::new(0); N],
-                        init_len: N as u16,
-                    }
-                }
-            }
-        )*
-    };
-}
-
-impl_zeroed_nm! {
-    u8, i8, u16, i16, u32, i32, u64, i64, u128, i128, usize, isize
-}
-
-impl<T, const N: usize> Array<T, N> {
-    // just some silly hackery here because uninit_array isn't stabilized -- move on
-    const VALUE: MaybeUninit<T> = MaybeUninit::uninit();
-    const ARRAY: [MaybeUninit<T>; N] = [Self::VALUE; N];
-    /// Create a new array
-    pub const fn new() -> Self {
-        Array {
-            stack: Self::ARRAY,
-            init_len: 0,
-        }
-    }
-    /// This is very safe from the ctor point of view, but the correctness of `init_len`
-    /// may be a bad assumption and might make us read garbage
-    pub const unsafe fn from_const(array: [MaybeUninit<T>; N], init_len: u16) -> Self {
-        Self {
-            stack: array,
-            init_len,
-        }
-    }
-    pub unsafe fn bump_init_len(&mut self, bump: u16) {
-        self.init_len += bump
-    }
-    /// This literally turns [T; M] into [T; N]. How can you expect it to be safe?
-    /// This function is extremely unsafe. I mean, I don't even know how to call it safe.
-    /// There's one way though: make M == N. This will panic in debug mode if M > N. In
-    /// release mode, good luck
-    unsafe fn from_const_array<const M: usize>(arr: [T; M]) -> Self {
-        debug_assert!(
-            N >= M,
-            "Provided const array exceeds size limit of initialized array"
-        );
-        // do not double-free or destroy the elements
-        let array = ManuallyDrop::new(arr);
-        let mut arr = Array::<T, N>::new();
-        // copy it over
-        let ptr = &*array as *const [T; M] as *const [MaybeUninit<T>; N];
-        ptr.copy_to_nonoverlapping(&mut arr.stack as *mut [MaybeUninit<T>; N], 1);
-        arr.set_len(N);
-        arr
-    }
-    /// Get the apparent length of the array
-    pub const fn len(&self) -> usize {
-        self.init_len as usize
-    }
-    /// Get the capacity of the array
-    pub const fn capacity(&self) -> usize {
-        N
-    }
-    /// Check if the array is full
-    pub const fn is_full(&self) -> bool {
-        N == self.len()
-    }
-    /// Get the remaining capacity of the array
-    pub const fn remaining_cap(&self) -> usize {
-        self.capacity() - self.len()
-    }
-    /// Set the length of the array
-    ///
-    /// ## Safety
-    /// This is one of those, use to leak memory functions. If you change the length,
-    /// you'll be reading random garbage from the memory and doing a double-free on drop
-    pub unsafe fn set_len(&mut self, len: usize) {
-        self.init_len = len as u16; // lossy cast, we maintain all invariants
-    }
-    /// Get the array as a mut ptr
-    unsafe fn as_mut_ptr(&mut self) -> *mut T {
-        self.stack.as_mut_ptr() as *mut _
-    }
-    /// Get the array as a const ptr
-    unsafe fn as_ptr(&self) -> *const T {
-        self.stack.as_ptr() as *const _
-    }
-    /// Push an element into the array **without any bounds checking**.
-    ///
-    /// ## Safety
-    /// This function is **so unsafe** that you possibly don't want to call it, or
-    /// even think about calling it. You can end up corrupting your own stack or
-    /// other's valuable data
-    pub unsafe fn push_unchecked(&mut self, element: T) {
-        let len = self.len();
-        ptr::write(self.as_mut_ptr().add(len), element);
-        self.set_len(len + 1);
-    }
-    /// This is a nice version of a push that does bound checks
-    pub fn push_panic(&mut self, element: T) -> Result<(), ()> {
-        if self.len() < N {
-            // so we can push it in
-            unsafe { self.push_unchecked(element) };
-            Ok(())
-        } else {
-            Err(())
-        }
-    }
-    /// This is a _panicky_ but safer alternative to `push_unchecked` that panics on
-    /// incorrect lengths
-    pub fn push(&mut self, element: T) {
-        self.push_panic(element).unwrap();
-    }
-    /// Pop an item off the array
-    pub fn pop(&mut self) -> Option<T> {
-        if self.len() == 0 {
-            // nothing here
-            None
-        } else {
-            unsafe {
-                let new_len = self.len() - 1;
-                self.set_len(new_len);
-                // len - 1 == offset
-                Some(ptr::read(self.as_ptr().add(new_len)))
-            }
-        }
-    }
-    /// Truncate the array to a given size. This is super safe and doesn't even panic
-    /// if you provide a silly `new_len`.
-    pub fn truncate(&mut self, new_len: usize) {
-        let len = self.len();
-        if new_len < len {
-            // we need to drop off a part of the array
-            unsafe {
-                // drop_in_place will handle the ZST invariant for us
-                ptr::drop_in_place(slice::from_raw_parts_mut(
-                    self.as_mut_ptr().add(new_len),
-                    len - new_len,
-                ))
-            }
-        }
-    }
-    /// Empty the internal array
-    pub fn clear(&mut self) {
-        self.truncate(0)
-    }
-    /// Extend self from a slice
-    pub fn extend_from_slice(&mut self, slice: &[T]) -> Result<(), ()>
-    where
-        T: Copy,
-    {
-        if self.remaining_cap() < slice.len() {
-            // no more space here
-            return Err(());
-        }
-        unsafe {
-            self.extend_from_slice_unchecked(slice);
-        }
-        Ok(())
-    }
-    /// Extend self from a slice without doing a single check
-    ///
-    /// ## Safety
-    /// This function is just very very and. You can write giant things into your own
-    /// stack corrupting it, corrupting other people's things and creating undefined
-    /// behavior like no one else.
-    pub unsafe fn extend_from_slice_unchecked(&mut self, slice: &[T]) {
-        let self_len = self.len();
-        let other_len = slice.len();
-        ptr::copy_nonoverlapping(slice.as_ptr(), self.as_mut_ptr().add(self_len), other_len);
-        self.set_len(self_len + other_len);
-    }
-    /// Returns self as a `[T; N]` array if it is fully initialized. Else it will again return
-    /// itself
-    pub fn into_array(self) -> Result<[T; N], Self> {
-        if self.len() < self.capacity() {
-            // not fully initialized
-            Err(self)
-        } else {
-            unsafe { Ok(self.into_array_unchecked()) }
-        }
-    }
-    pub unsafe fn into_array_unchecked(self) -> [T; N] {
-        // make sure we don't do a double free or end up deleting the elements
-        let _self = ManuallyDrop::new(self);
-        ptr::read(_self.as_ptr() as *const [T; N])
-    }
-    pub fn try_from_slice(slice: impl AsRef<[T]>) -> Option<Self> {
-        let slice = slice.as_ref();
-        if slice.len() > N {
-            None
-        } else {
-            Some(unsafe { Self::from_slice(slice) })
-        }
-    }
-    /// Extend self from a slice
-    ///
-    /// ## Safety
-    /// The same danger as in from_slice_unchecked
-    pub unsafe fn from_slice(slice_ref: impl AsRef<[T]>) -> Self {
-        let mut slf = Self::new();
-        slf.extend_from_slice_unchecked(slice_ref.as_ref());
-        slf
-    }
-    // these operations are incredibly safe because we only pass the initialized part
-    // of the array
-    /// Get self as a slice. Super safe because we guarantee that all the other invarians
-    /// are upheld
-    pub fn as_slice(&self) -> &[T] {
-        unsafe { slice::from_raw_parts(self.as_ptr(), self.len()) }
-    }
-    /// Get self as a mutable slice. Super safe (see comment above)
-    fn as_slice_mut(&mut self) -> &mut [T] {
-        unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len()) }
-    }
-}
-
-impl<const N: usize> Array<u8, N> {
-    /// This isn't _unsafe_ but it can cause functions expecting pure unicode to
-    /// crash if the array contains invalid unicode
-    pub unsafe fn as_str(&self) -> &str {
-        str::from_utf8_unchecked(self)
-    }
-}
-
-impl<T, const N: usize> ops::Deref for Array<T, N> {
-    type Target = [T];
-    fn deref(&self) -> &Self::Target {
-        self.as_slice()
-    }
-}
-
-impl<T, const N: usize> ops::DerefMut for Array<T, N> {
-    fn deref_mut(&mut self) -> &mut [T] {
-        self.as_slice_mut()
-    }
-}
-
-impl<T, const N: usize> From<[T; N]> for Array<T, N> {
-    fn from(array: [T; N]) -> Self {
-        unsafe { Array::from_const_array::<N>(array) }
-    }
-}
-
-impl<T, const N: usize> Drop for Array<T, N> {
-    fn drop(&mut self) {
-        self.clear()
-    }
-}
-
-pub struct ArrayIntoIter<T, const N: usize> {
-    state: usize,
-    a: Array<T, N>,
-}
-
-impl<T, const N: usize> Iterator for ArrayIntoIter<T, N> {
-    type Item = T;
-    fn next(&mut self) -> Option<Self::Item> {
-        if self.state == self.a.len() {
-            // reached end
-            None
-        } else {
-            let idx = self.state;
-            self.state += 1;
-            Some(unsafe { ptr::read(self.a.as_ptr().add(idx)) })
-        }
-    }
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        let l = self.a.len() - self.state;
-        (l, Some(l))
-    }
-}
-
-impl<T, const N: usize> IntoIterator for Array<T, N> {
-    type Item = T;
-    type IntoIter = ArrayIntoIter<T, N>;
-    fn into_iter(self) -> Self::IntoIter {
-        ArrayIntoIter { state: 0, a: self }
-    }
-}
-
-impl<T, const N: usize> Array<T, N> {
-    /// Extend self using an iterator.
-    ///
-    /// ## Safety
-    /// This function can cause undefined damage to your application's stack and/or other's
-    /// data. Only use if you know what you're doing. If you don't use `extend_from_iter`
-    /// instead
-    pub unsafe fn extend_from_iter_unchecked<I>(&mut self, iterable: I)
-    where
-        I: IntoIterator<Item = T>,
-    {
-        // the ptr to start writing from
-        let mut ptr = Self::as_mut_ptr(self).add(self.len());
-        let mut guard = LenScopeGuard::new(&mut self.init_len);
-        let mut iter = iterable.into_iter();
-        loop {
-            if let Some(element) = iter.next() {
-                // write the element
-                ptr.write(element);
-                // move to the next location
-                ptr = ptr.add(1);
-                // tell the guard to increment
-                guard.incr(1);
-            } else {
-                return;
-            }
-        }
-    }
-    pub fn extend_from_iter<I>(&mut self, iterable: I)
-    where
-        I: IntoIterator<Item = T>,
-    {
-        unsafe {
-            // the ptr to start writing from
-            let mut ptr = Self::as_mut_ptr(self).add(self.len());
-            let end_ptr = Self::as_ptr(self).add(self.capacity());
-            let mut guard = LenScopeGuard::new(&mut self.init_len);
-            let mut iter = iterable.into_iter();
-            loop {
-                if let Some(element) = iter.next() {
-                    // write the element
-                    ptr.write(element);
-                    // move to the next location
-                    ptr = ptr.add(1);
-                    // tell the guard to increment
-                    guard.incr(1);
-                    if end_ptr < ptr {
-                        // our current ptr points to the end of the allocation
-                        // oh no, time for corruption, if the user says so
-                        panic!("Overflowed stack area.")
-                    }
-                } else {
-                    return;
-                }
-            }
-        }
-    }
-}
-
-impl<T, const N: usize> Extend<T> for Array<T, N> {
-    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
-        {
-            self.extend_from_iter::<_>(iter)
-        }
-    }
-}
-
-impl<T, const N: usize> FromIterator<T> for Array<T, N> {
-    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
-        let mut arr = Array::new();
-        arr.extend(iter);
-        arr
-    }
-}
-
-impl<T, const N: usize> Clone for Array<T, N>
-where
-    T: Clone,
-{
-    fn clone(&self) -> Self {
-        self.iter().cloned().collect()
-    }
-}
-
-impl<T, const N: usize> Hash for Array<T, N>
-where
-    T: Hash,
-{
-    fn hash<H>(&self, hasher: &mut H)
-    where
-        H: Hasher,
-    {
-        Hash::hash(&**self, hasher)
-    }
-}
-
-impl<const N: usize> PartialEq<[u8]> for Array<u8, N> {
-    fn eq(&self, oth: &[u8]) -> bool {
-        **self == *oth
-    }
-}
-
-impl<const N: usize> PartialEq<Array<u8, N>> for [u8] {
-    fn eq(&self, oth: &Array<u8, N>) -> bool {
-        oth.as_slice() == self
-    }
-}
-
-impl<T, const N: usize> PartialEq for Array<T, N>
-where
-    T: PartialEq,
-{
-    fn eq(&self, other: &Self) -> bool {
-        **self == **other
-    }
-}
-
-impl<T, const N: usize> Eq for Array<T, N> where T: Eq {}
-
-impl<T, const N: usize> PartialOrd for Array<T, N>
-where
-    T: PartialOrd,
-{
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        (**self).partial_cmp(&**other)
-    }
-}
-
-impl<T, const N: usize> Ord for Array<T, N>
-where
-    T: Ord,
-{
-    fn cmp(&self, other: &Self) -> Ordering {
-        (**self).cmp(&**other)
-    }
-}
-
-impl<T, const CAP: usize> Borrow<[T]> for Array<T, CAP> {
-    fn borrow(&self) -> &[T] {
-        self
-    }
-}
-
-impl<T, const CAP: usize> BorrowMut<[T]> for Array<T, CAP> {
-    fn borrow_mut(&mut self) -> &mut [T] {
-        self
-    }
-}
-
-impl<T, const CAP: usize> AsRef<[T]> for Array<T, CAP> {
-    fn as_ref(&self) -> &[T] {
-        self
-    }
-}
-
-impl<T, const CAP: usize> AsMut<[T]> for Array<T, CAP> {
-    fn as_mut(&mut self) -> &mut [T] {
-        self
-    }
-}
-
-impl<T, const CAP: usize> fmt::Debug for Array<T, CAP>
-where
-    T: fmt::Debug,
-{
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        if any::type_name::<T>().eq(any::type_name::<u8>()) {
-            let slf = unsafe {
-                // UNSAFE(@ohsayan): Guaranteed by the above invariant
-                &*(self as *const Array<T, CAP> as *const Array<u8, CAP>)
-            };
-            match String::from_utf8(slf.to_vec()) {
-                Ok(st) => write!(f, "{:#?}", st),
-                Err(_) => (**self).fmt(f),
-            }
-        } else {
-            (**self).fmt(f)
-        }
-    }
-}
-
-impl<const N: usize> Borrow<str> for Array<u8, N> {
-    fn borrow(&self) -> &str {
-        unsafe { self.as_str() }
-    }
-}
-
-unsafe impl<T, const N: usize> Send for Array<T, N> where T: Send {}
-unsafe impl<T, const N: usize> Sync for Array<T, N> where T: Sync {}
-
-#[test]
-fn test_basic() {
-    let mut b: Array<u8, 11> = Array::new();
-    b.extend_from_slice("Hello World".as_bytes()).unwrap();
-    assert_eq!(
-        b,
-        Array::from([b'H', b'e', b'l', b'l', b'o', b' ', b'W', b'o', b'r', b'l', b'd'])
-    );
-}
-
-#[test]
-fn test_uninitialized() {
-    let mut b: Array<u8, 16> = Array::new();
-    b.push(b'S');
-    assert_eq!(b.iter().count(), 1);
-}
-
-#[test]
-#[should_panic]
-fn test_array_overflow() {
-    let mut arr: Array<u8, 5> = Array::new();
-    arr.extend_from_slice("123456".as_bytes()).unwrap();
-}
-
-#[test]
-#[should_panic]
-fn test_array_overflow_iter() {
-    let mut arr: Array<char, 5> = Array::new();
-    arr.extend("123456".chars());
-}
-
-#[test]
-fn test_array_clone() {
-    let mut arr: Array<u8, 64> = Array::new();
-    arr.extend(
-        "qHwRsmyBYHbqyHfdShOfVSayVUmeKlEagvJoGuTyvaCqpsfFkZabeuqmVeiKbJxV"
-            .as_bytes()
-            .to_owned(),
-    );
-    let myclone = arr.clone();
-    assert_eq!(arr, myclone);
-}
-
-#[test]
-fn test_array_extend_okay() {
-    let mut arr: Array<u8, 64> = Array::new();
-    arr.extend(
-        "qHwRsmyBYHbqyHfdShOfVSayVUmeKlEagvJoGuTyvaCqpsfFkZabeuqmVeiKbJxV"
-            .as_bytes()
-            .to_owned(),
-    );
-}
-
-#[test]
-#[should_panic]
-fn test_array_extend_fail() {
-    let mut arr: Array<u8, 64> = Array::new();
-    arr.extend(
-        "qHwRsmyBYHbqyHfdShOfVSayVUmeKlEagvJoGuTyvaCqpsfFkZabeuqmVeiKbJxV_"
-            .as_bytes()
-            .to_owned(),
-    );
-}
diff --git a/server/src/corestore/backoff.rs b/server/src/corestore/backoff.rs
deleted file mode 100644
index adebc101..00000000
--- a/server/src/corestore/backoff.rs
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Created on Wed Feb 16 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use std::{cell::Cell, hint::spin_loop, thread};
-
-/// Type to perform exponential backoff
-pub struct Backoff {
-    cur: Cell<u8>,
-}
-
-impl Backoff {
-    const MAX_SPIN: u8 = 6;
-    const MAX_YIELD: u8 = 8;
-    pub fn new() -> Self {
-        Self { cur: Cell::new(0) }
-    }
-    /// Spin a few times, giving way to the CPU but if we have spun too many times,
-    /// then block by yielding to the OS scheduler. This will **eventually block**
-    /// if we spin more than the set `MAX_SPIN`
-    pub fn snooze(&self) {
-        if self.cur.get() <= Self::MAX_SPIN {
-            // we can still spin (exp)
-            for _ in 0..1 << self.cur.get() {
-                spin_loop();
-            }
-        } else {
-            // nope, yield to scheduler
-            thread::yield_now();
-        }
-        if self.cur.get() <= Self::MAX_YIELD {
-            // bump current step
-            self.cur.set(self.cur.get() + 1)
-        }
-    }
-}
diff --git a/server/src/corestore/booltable.rs b/server/src/corestore/booltable.rs
deleted file mode 100644
index e605b0c6..00000000
--- a/server/src/corestore/booltable.rs
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Created on Fri Sep 10 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
-    ⚠⚠⚠⚠ A WORD OF WARNING ⚠⚠⚠⚠
-    This module contains some dark stuff (and asumptions) about type layouts and/or representations,
-    things which can change from time to time. Do not rely on any of this!
-*/
-
-use core::ops::Index;
-
-pub type BytesBoolTable = BoolTable<&'static [u8]>;
-pub type BytesNicheLUT = NicheLUT<&'static [u8]>;
-
-/// A two-value boolean LUT
-pub struct BoolTable<T> {
-    base: [T; 2],
-}
-
-impl<T> BoolTable<T> {
-    /// Supply values in the order: `if_true` and `if_false`
-    pub const fn new(if_true: T, if_false: T) -> Self {
-        Self {
-            base: [if_false, if_true],
-        }
-    }
-}
-
-impl<T> Index<bool> for BoolTable<T> {
-    type Output = T;
-    fn index(&self, index: bool) -> &Self::Output {
-        unsafe { &*self.base.as_ptr().add(index as usize) }
-    }
-}
-
-/// A LUT based on niche values, especially built to support the `Option<bool>` optimized
-/// structure
-///
-/// **Warning:** This is a terrible opt and only works on the Rust ABI
-pub struct NicheLUT<T> {
-    base: [T; 3],
-}
-
-impl<T> NicheLUT<T> {
-    /// Supply values in the following order: [`if_none`, `if_true`, `if_false`]
-    pub const fn new(if_none: T, if_true: T, if_false: T) -> Self {
-        Self {
-            // 0 == S(F); 1 == S(T); 2 == NULL
-            base: [if_false, if_true, if_none],
-        }
-    }
-}
-
-impl<T> Index<Option<bool>> for NicheLUT<T> {
-    type Output = T;
-    fn index(&self, idx: Option<bool>) -> &Self::Output {
-        unsafe {
-            &*self
-                .base
-                .as_ptr()
-                .add(*(&idx as *const _ as *const u8) as usize)
-        }
-    }
-}
-
-#[test]
-fn niche_optim_sanity_test() {
-    let none: Option<bool> = None;
-    let some_t: Option<bool> = Some(true);
-    let some_f: Option<bool> = Some(false);
-    unsafe {
-        let r_some_f = &some_f as *const _ as *const u8;
-        let r_some_t = &some_t as *const _ as *const u8;
-        let r_none = &none as *const _ as *const u8;
-        assert_eq!(*r_some_f, 0);
-        assert_eq!(*r_some_t, 1);
-        assert_eq!(*r_none, 2);
-    }
-}
-
-/// A 2-bit indexed boolean LUT
-pub struct TwoBitLUT<T> {
-    base: [T; 4],
-}
-
-type Bit = bool;
-type TwoBitIndex = (Bit, Bit);
-
-impl<T> TwoBitLUT<T> {
-    /// Supply values in the following order:
-    /// - 1st unset, 2nd unset
-    /// - 1st unset, 2nd set
-    /// - 1st set, 2nd unset
-    /// - 1st set, 2nd set
-    pub const fn new(ff: T, ft: T, tf: T, tt: T) -> Self {
-        Self {
-            base: [ff, ft, tf, tt],
-        }
-    }
-}
-
-impl<T> Index<TwoBitIndex> for TwoBitLUT<T> {
-    type Output = T;
-    fn index(&self, (bit_a, bit_b): TwoBitIndex) -> &Self::Output {
-        unsafe {
-            &*self
-                .base
-                .as_ptr()
-                .add((((bit_a as u8) << 1) + (bit_b as u8)) as usize)
-        }
-    }
-}
-
-#[test]
-fn test_two_bit_indexed_lut() {
-    let (bit_a, bit_b) = unsafe { tmut_bool!(0, 0) };
-    let twobitlut = TwoBitLUT::new('a', 'b', 'c', 'd');
-    // the operators, are just for sanity
-    assert_eq!('d', twobitlut[(!bit_a, !bit_b)]);
-    assert_eq!('c', twobitlut[(!bit_a, bit_b)]);
-    assert_eq!('b', twobitlut[(bit_a, !bit_b)]);
-    assert_eq!('a', twobitlut[(bit_a, bit_b)]);
-}
diff --git a/server/src/corestore/buffers.rs b/server/src/corestore/buffers.rs
deleted file mode 100644
index 63c1e605..00000000
--- a/server/src/corestore/buffers.rs
+++ /dev/null
@@ -1,413 +0,0 @@
-/*
- * Created on Mon Jul 12 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::array::Array,
-    core::{ops::Deref, str},
-};
-
-macro_rules! push_self {
-    ($self:expr, $what:expr) => {
-        $self.inner_stack.push_unchecked($what)
-    };
-}
-
-macro_rules! lut {
-    ($e:expr) => {
-        ucidx!(PAIR_MAP_LUT, $e)
-    };
-}
-
-static PAIR_MAP_LUT: [u8; 200] = [
-    0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, 0x30, 0x36, 0x30, 0x37,
-    0x30, 0x38, 0x30, 0x39, // 0x30
-    0x31, 0x30, 0x31, 0x31, 0x31, 0x32, 0x31, 0x33, 0x31, 0x34, 0x31, 0x35, 0x31, 0x36, 0x31, 0x37,
-    0x31, 0x38, 0x31, 0x39, // 0x31
-    0x32, 0x30, 0x32, 0x31, 0x32, 0x32, 0x32, 0x33, 0x32, 0x34, 0x32, 0x35, 0x32, 0x36, 0x32, 0x37,
-    0x32, 0x38, 0x32, 0x39, // 0x32
-    0x33, 0x30, 0x33, 0x31, 0x33, 0x32, 0x33, 0x33, 0x33, 0x34, 0x33, 0x35, 0x33, 0x36, 0x33, 0x37,
-    0x33, 0x38, 0x33, 0x39, // 0x33
-    0x34, 0x30, 0x34, 0x31, 0x34, 0x32, 0x34, 0x33, 0x34, 0x34, 0x34, 0x35, 0x34, 0x36, 0x34, 0x37,
-    0x34, 0x38, 0x34, 0x39, // 0x34
-    0x35, 0x30, 0x35, 0x31, 0x35, 0x32, 0x35, 0x33, 0x35, 0x34, 0x35, 0x35, 0x35, 0x36, 0x35, 0x37,
-    0x35, 0x38, 0x35, 0x39, // 0x35
-    0x36, 0x30, 0x36, 0x31, 0x36, 0x32, 0x36, 0x33, 0x36, 0x34, 0x36, 0x35, 0x36, 0x36, 0x36, 0x37,
-    0x36, 0x38, 0x36, 0x39, // 0x36
-    0x37, 0x30, 0x37, 0x31, 0x37, 0x32, 0x37, 0x33, 0x37, 0x34, 0x37, 0x35, 0x37, 0x36, 0x37, 0x37,
-    0x37, 0x38, 0x37, 0x39, // 0x37
-    0x38, 0x30, 0x38, 0x31, 0x38, 0x32, 0x38, 0x33, 0x38, 0x34, 0x38, 0x35, 0x38, 0x36, 0x38, 0x37,
-    0x38, 0x38, 0x38, 0x39, // 0x38
-    0x39, 0x30, 0x39, 0x31, 0x39, 0x32, 0x39, 0x33, 0x39, 0x34, 0x39, 0x35, 0x39, 0x36, 0x39, 0x37,
-    0x39, 0x38, 0x39, 0x39, // 0x39
-];
-
-#[allow(dead_code)]
-/// A 32-bit integer buffer with one extra byte
-pub type Integer32Buffer = Integer32BufferRaw<11>;
-
-#[derive(Debug)]
-/// A buffer for unsigned 32-bit integers with one _extra byte_ of memory reserved for
-/// adding characters. On initialization (through [`Self::init`]), your integer will be
-/// encoded and stored into the _unsafe array_
-pub struct Integer32BufferRaw<const N: usize> {
-    inner_stack: Array<u8, 11>,
-}
-
-#[allow(dead_code)]
-impl<const N: usize> Integer32BufferRaw<N> {
-    /// Initialize a buffer
-    pub fn init(integer: u32) -> Self {
-        let mut slf = Self {
-            inner_stack: Array::new(),
-        };
-        unsafe {
-            slf._init_integer(integer);
-        }
-        slf
-    }
-    /// Initialize an integer. This is unsafe to be called outside because you'll be
-    /// pushing in another integer and might end up corrupting your own stack as all
-    /// pushes are unchecked!
-    unsafe fn _init_integer(&mut self, mut val: u32) {
-        if val < 10_000 {
-            let d1 = (val / 100) << 1;
-            let d2 = (val % 100) << 1;
-            if val >= 1000 {
-                push_self!(self, lut!(d1));
-            }
-            if val >= 100 {
-                push_self!(self, lut!(d1 + 1));
-            }
-            if val >= 10 {
-                push_self!(self, lut!(d2));
-            }
-            push_self!(self, lut!(d2 + 1));
-        } else if val < 100_000_000 {
-            let b = val / 10000;
-            let c = val % 10000;
-            let d1 = (b / 100) << 1;
-            let d2 = (b % 100) << 1;
-            let d3 = (c / 100) << 1;
-            let d4 = (c % 100) << 1;
-
-            if val > 10_000_000 {
-                push_self!(self, lut!(d1));
-            }
-            if val > 1_000_000 {
-                push_self!(self, lut!(d1 + 1));
-            }
-            if val > 100_000 {
-                push_self!(self, lut!(d2));
-            }
-            push_self!(self, lut!(d2 + 1));
-            push_self!(self, lut!(d3));
-            push_self!(self, lut!(d3 + 1));
-            push_self!(self, lut!(d4));
-            push_self!(self, lut!(d4 + 1));
-        } else {
-            // worst, 1B or more
-            let a = val / 100000000;
-            val %= 100000000;
-
-            if a >= 10 {
-                let i = a << 1;
-                push_self!(self, lut!(i));
-                push_self!(self, lut!(i + 1));
-            } else {
-                push_self!(self, 0x30);
-            }
-            let b = val / 10000;
-            let c = val % 10000;
-            let d1 = (b / 100) << 1;
-            let d2 = (b % 100) << 1;
-            let d3 = (c / 100) << 1;
-            let d4 = (c % 100) << 1;
-            // write back
-            push_self!(self, lut!(d1));
-            push_self!(self, lut!(d1 + 1));
-            push_self!(self, lut!(d2));
-            push_self!(self, lut!(d2 + 1));
-            push_self!(self, lut!(d3));
-            push_self!(self, lut!(d3 + 1));
-            push_self!(self, lut!(d4));
-            push_self!(self, lut!(d4 + 1));
-        }
-    }
-    /// **This is very unsafe** Only push something when you know that the capacity won't overflow
-    /// your allowance of 11 bytes. Oh no, there's no panic for you because you'll silently
-    /// corrupt your own memory (or others' :/)
-    pub unsafe fn push(&mut self, val: u8) {
-        push_self!(self, val)
-    }
-}
-
-impl<const N: usize> Deref for Integer32BufferRaw<N> {
-    type Target = str;
-    fn deref(&self) -> &Self::Target {
-        unsafe { str::from_utf8_unchecked(&self.inner_stack) }
-    }
-}
-
-impl<const N: usize> AsRef<str> for Integer32BufferRaw<N> {
-    fn as_ref(&self) -> &str {
-        self
-    }
-}
-
-impl<T, const N: usize> PartialEq<T> for Integer32BufferRaw<N>
-where
-    T: AsRef<str>,
-{
-    fn eq(&self, other_str: &T) -> bool {
-        self.as_ref() == other_str.as_ref()
-    }
-}
-
-#[test]
-fn test_int32_buffer() {
-    let buffer = Integer32Buffer::init(256);
-    assert_eq!(buffer, 256.to_string());
-}
-
-#[test]
-fn test_int32_buffer_push() {
-    let mut buffer = Integer32Buffer::init(278);
-    unsafe {
-        buffer.push(b'?');
-    }
-    assert_eq!(buffer, "278?");
-}
-
-/// A 64-bit integer buffer with **no extra byte**
-pub type Integer64 = Integer64BufferRaw<20>;
-
-#[derive(Debug)]
-pub struct Integer64BufferRaw<const N: usize> {
-    inner_stack: Array<u8, N>,
-}
-
-const Z_8: u64 = 100_000_000;
-const Z_9: u64 = Z_8 * 10;
-const Z_10: u64 = Z_9 * 10;
-const Z_11: u64 = Z_10 * 10;
-const Z_12: u64 = Z_11 * 10;
-const Z_13: u64 = Z_12 * 10;
-const Z_14: u64 = Z_13 * 10;
-const Z_15: u64 = Z_14 * 10;
-const Z_16: u64 = Z_15 * 10;
-
-impl<const N: usize> Integer64BufferRaw<N> {
-    pub fn init(integer: u64) -> Self {
-        let mut slf = Self {
-            inner_stack: Array::new(),
-        };
-        unsafe {
-            slf._init_integer(integer);
-        }
-        slf
-    }
-    unsafe fn _init_integer(&mut self, mut int: u64) {
-        if int < Z_8 {
-            if int < 10_000 {
-                let d1 = (int / 100) << 1;
-                let d2 = (int % 100) << 1;
-                if int >= 1_000 {
-                    push_self!(self, lut!(d1));
-                }
-                if int >= 100 {
-                    push_self!(self, lut!(d1 + 1));
-                }
-                if int >= 10 {
-                    push_self!(self, lut!(d2));
-                }
-                push_self!(self, lut!(d2 + 1));
-            } else {
-                let b = int / 10000;
-                let c = int % 10000;
-                let d1 = (b / 100) << 1;
-                let d2 = (b % 100) << 1;
-                let d3 = (c / 100) << 1;
-                let d4 = (c % 100) << 1;
-                if int >= 10_000_000 {
-                    push_self!(self, lut!(d1));
-                }
-                if int >= 1_000_000 {
-                    push_self!(self, lut!(d1 + 1));
-                }
-                if int >= 100_000 {
-                    push_self!(self, lut!(d2));
-                }
-                push_self!(self, lut!(d2 + 1));
-                push_self!(self, lut!(d3));
-                push_self!(self, lut!(d3 + 1));
-                push_self!(self, lut!(d4));
-                push_self!(self, lut!(d4 + 1));
-            }
-        } else if int < Z_16 {
-            // lets do 8 at a time
-            let v0 = int / Z_8;
-            let v1 = int & Z_8;
-            let b0 = v0 / 10000;
-            let c0 = v0 % 10000;
-            let d1 = (b0 / 100) << 1;
-            let d2 = (b0 % 100) << 1;
-            let d3 = (c0 / 100) << 1;
-            let d4 = (c0 % 100) << 1;
-            let b1 = v1 / 10000;
-            let c1 = v1 % 10000;
-            let d5 = (b1 / 100) << 1;
-            let d6 = (b1 % 100) << 1;
-            let d7 = (c1 / 100) << 1;
-            let d8 = (c1 % 100) << 1;
-            if int >= Z_15 {
-                push_self!(self, lut!(d1));
-            }
-            if int >= Z_14 {
-                push_self!(self, lut!(d1 + 1));
-            }
-            if int >= Z_13 {
-                push_self!(self, lut!(d2));
-            }
-            if int >= Z_12 {
-                push_self!(self, lut!(d2 + 1));
-            }
-            if int >= Z_11 {
-                push_self!(self, lut!(d3));
-            }
-            if int >= Z_10 {
-                push_self!(self, lut!(d3 + 1));
-            }
-            if int >= Z_9 {
-                push_self!(self, lut!(d4));
-            }
-            push_self!(self, lut!(d4 + 1));
-            push_self!(self, lut!(d5));
-            push_self!(self, lut!(d5 + 1));
-            push_self!(self, lut!(d6));
-            push_self!(self, lut!(d6 + 1));
-            push_self!(self, lut!(d7));
-            push_self!(self, lut!(d7 + 1));
-            push_self!(self, lut!(d8));
-            push_self!(self, lut!(d8 + 1));
-        } else {
-            let a = int / Z_16;
-            int %= Z_16;
-            if a < 10 {
-                push_self!(self, 0x30 + a as u8);
-            } else if a < 100 {
-                let i = a << 1;
-                push_self!(self, lut!(i));
-                push_self!(self, lut!(i + 1));
-            } else if a < 1000 {
-                push_self!(self, 0x30 + (a / 100) as u8);
-                let i = (a % 100) << 1;
-                push_self!(self, lut!(i));
-                push_self!(self, lut!(i + 1));
-            } else {
-                let i = (a / 100) << 1;
-                let j = (a % 100) << 1;
-                push_self!(self, lut!(i));
-                push_self!(self, lut!(i + 1));
-                push_self!(self, lut!(j));
-                push_self!(self, lut!(j + 1));
-            }
-
-            let v0 = int / Z_8;
-            let v1 = int % Z_8;
-            let b0 = v0 / 10000;
-            let c0 = v0 % 10000;
-            let d1 = (b0 / 100) << 1;
-            let d2 = (b0 % 100) << 1;
-            let d3 = (c0 / 100) << 1;
-            let d4 = (c0 % 100) << 1;
-            let b1 = v1 / 10000;
-            let c1 = v1 % 10000;
-            let d5 = (b1 / 100) << 1;
-            let d6 = (b1 % 100) << 1;
-            let d7 = (c1 / 100) << 1;
-            let d8 = (c1 % 100) << 1;
-            push_self!(self, lut!(d1));
-            push_self!(self, lut!(d1 + 1));
-            push_self!(self, lut!(d2));
-            push_self!(self, lut!(d2 + 1));
-            push_self!(self, lut!(d3));
-            push_self!(self, lut!(d3 + 1));
-            push_self!(self, lut!(d4));
-            push_self!(self, lut!(d4 + 1));
-            push_self!(self, lut!(d5));
-            push_self!(self, lut!(d5 + 1));
-            push_self!(self, lut!(d6));
-            push_self!(self, lut!(d6 + 1));
-            push_self!(self, lut!(d7));
-            push_self!(self, lut!(d7 + 1));
-            push_self!(self, lut!(d8));
-            push_self!(self, lut!(d8 + 1));
-        }
-    }
-}
-
-impl<const N: usize> From<usize> for Integer64BufferRaw<N> {
-    fn from(val: usize) -> Self {
-        Self::init(val as u64)
-    }
-}
-
-impl<const N: usize> From<u64> for Integer64BufferRaw<N> {
-    fn from(val: u64) -> Self {
-        Self::init(val)
-    }
-}
-
-impl<const N: usize> Deref for Integer64BufferRaw<N> {
-    type Target = [u8];
-    fn deref(&self) -> &Self::Target {
-        &self.inner_stack
-    }
-}
-
-impl<const N: usize> AsRef<str> for Integer64BufferRaw<N> {
-    fn as_ref(&self) -> &str {
-        unsafe { str::from_utf8_unchecked(&self.inner_stack) }
-    }
-}
-
-impl<T, const N: usize> PartialEq<T> for Integer64BufferRaw<N>
-where
-    T: AsRef<str>,
-{
-    fn eq(&self, other_str: &T) -> bool {
-        self.as_ref() == other_str.as_ref()
-    }
-}
-
-#[test]
-fn test_int64_buffer() {
-    assert_eq!(
-        9348910481349849081_u64.to_string(),
-        Integer64::init(9348910481349849081_u64).as_ref()
-    );
-    assert_eq!(u64::MAX.to_string(), Integer64::init(u64::MAX).as_ref());
-}
diff --git a/server/src/corestore/heap_array.rs b/server/src/corestore/heap_array.rs
deleted file mode 100644
index cf5f72e3..00000000
--- a/server/src/corestore/heap_array.rs
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Created on Mon Feb 21 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    core::{alloc::Layout, fmt, marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr, slice},
-    std::alloc::dealloc,
-};
-
-/// A heap-allocated array
-pub struct HeapArray<T> {
-    ptr: *const T,
-    len: usize,
-    _marker: PhantomData<T>,
-}
-
-pub struct HeapArrayWriter<T> {
-    base: Vec<T>,
-}
-
-impl<T> HeapArrayWriter<T> {
-    pub fn with_capacity(cap: usize) -> Self {
-        Self {
-            base: Vec::with_capacity(cap),
-        }
-    }
-    /// ## Safety
-    /// Caller must ensure that `idx <= cap`. If not, you'll corrupt your
-    /// memory
-    pub unsafe fn write_to_index(&mut self, idx: usize, element: T) {
-        debug_assert!(idx <= self.base.capacity());
-        ptr::write(self.base.as_mut_ptr().add(idx), element);
-        self.base.set_len(self.base.len() + 1);
-    }
-    /// ## Safety
-    /// This function can lead to memory unsafety in two ways:
-    /// - Excess capacity: In that case, it will leak memory
-    /// - Uninitialized elements: In that case, it will segfault while attempting to call
-    /// `T`'s dtor
-    pub unsafe fn finish(self) -> HeapArray<T> {
-        let base = ManuallyDrop::new(self.base);
-        HeapArray::new(base.as_ptr(), base.len())
-    }
-}
-
-impl<T> HeapArray<T> {
-    #[cfg(test)]
-    pub fn new_from_vec(mut v: Vec<T>) -> Self {
-        v.shrink_to_fit();
-        let v = ManuallyDrop::new(v);
-        unsafe { Self::new(v.as_ptr(), v.len()) }
-    }
-    pub unsafe fn new(ptr: *const T, len: usize) -> Self {
-        Self {
-            ptr,
-            len,
-            _marker: PhantomData,
-        }
-    }
-    pub fn new_writer(cap: usize) -> HeapArrayWriter<T> {
-        HeapArrayWriter::with_capacity(cap)
-    }
-    #[cfg(test)]
-    pub fn as_slice(&self) -> &[T] {
-        self
-    }
-}
-
-impl<T> Drop for HeapArray<T> {
-    fn drop(&mut self) {
-        unsafe {
-            // run dtor
-            ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.ptr as *mut T, self.len));
-            // deallocate
-            let layout = Layout::array::<T>(self.len).unwrap();
-            dealloc(self.ptr as *mut T as *mut u8, layout);
-        }
-    }
-}
-
-// totally fine because `u8`s can be safely shared across threads
-unsafe impl<T: Send> Send for HeapArray<T> {}
-unsafe impl<T: Sync> Sync for HeapArray<T> {}
-
-impl<T> Deref for HeapArray<T> {
-    type Target = [T];
-    fn deref(&self) -> &Self::Target {
-        unsafe { slice::from_raw_parts(self.ptr, self.len) }
-    }
-}
-
-impl<T: fmt::Debug> fmt::Debug for HeapArray<T> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_list().entries(self.iter()).finish()
-    }
-}
-
-impl<T: PartialEq> PartialEq for HeapArray<T> {
-    fn eq(&self, other: &Self) -> bool {
-        self == other
-    }
-}
-
-#[test]
-fn heaparray_impl() {
-    // basically, this shouldn't segfault
-    let heap_array = b"notasuperuser".to_vec();
-    let heap_array = HeapArray::new_from_vec(heap_array);
-    assert_eq!(heap_array.as_slice(), b"notasuperuser");
-}
diff --git a/server/src/corestore/htable.rs b/server/src/corestore/htable.rs
deleted file mode 100644
index b7a6349b..00000000
--- a/server/src/corestore/htable.rs
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Created on Sun May 09 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#![allow(unused)] // TODO(@ohsayan): Plonk this
-
-use {
-    crate::corestore::map::{
-        bref::{Entry, OccupiedEntry, Ref, VacantEntry},
-        iter::{BorrowedIter, OwnedIter},
-        Skymap,
-    },
-    ahash::RandomState,
-    std::{borrow::Borrow, hash::Hash, iter::FromIterator, ops::Deref},
-};
-
-type HashTable<K, V> = Skymap<K, V, RandomState>;
-
-#[derive(Debug)]
-/// The Coremap contains the actual key/value pairs along with additional fields for data safety
-/// and protection
-pub struct Coremap<K, V> {
-    pub(crate) inner: HashTable<K, V>,
-}
-
-impl<K, V> Default for Coremap<K, V> {
-    fn default() -> Self {
-        Coremap {
-            inner: HashTable::new_ahash(),
-        }
-    }
-}
-
-impl<K: Eq + Hash, V> Coremap<K, V> {
-    /// Create an empty coremap
-    pub fn new() -> Self {
-        Self::default()
-    }
-    pub fn with_capacity(cap: usize) -> Self {
-        Coremap {
-            inner: HashTable::with_capacity(cap),
-        }
-    }
-    pub fn try_with_capacity(cap: usize) -> Result<Self, ()> {
-        if cap > (isize::MAX as usize) {
-            Err(())
-        } else {
-            Ok(Self::with_capacity(cap))
-        }
-    }
-    /// Returns the total number of key value pairs
-    pub fn len(&self) -> usize {
-        self.inner.len()
-    }
-    /// Clears the inner table!
-    pub fn clear(&self) {
-        self.inner.clear()
-    }
-}
-
-impl<K, V> Coremap<K, V>
-where
-    K: Eq + Hash,
-{
-    /// Returns the removed value for key, it it existed
-    pub fn remove<Q>(&self, key: &Q) -> Option<(K, V)>
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        self.inner.remove(key)
-    }
-    /// Returns true if an existent key was removed
-    pub fn true_if_removed<Q>(&self, key: &Q) -> bool
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        self.inner.remove(key).is_some()
-    }
-    /// Check if a table contains a key
-    pub fn contains_key<Q>(&self, key: &Q) -> bool
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        self.inner.contains_key(key)
-    }
-    /// Return a non-consuming iterator
-    pub fn iter(&self) -> BorrowedIter<'_, K, V, RandomState> {
-        self.inner.get_iter()
-    }
-    /// Get a reference to the value of a key, if it exists
-    pub fn get<Q>(&self, key: &Q) -> Option<Ref<'_, K, V>>
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        self.inner.get(key)
-    }
-    /// Returns true if the non-existent key was assigned to a value
-    pub fn true_if_insert(&self, k: K, v: V) -> bool {
-        if let Entry::Vacant(ve) = self.inner.entry(k) {
-            ve.insert(v);
-            true
-        } else {
-            false
-        }
-    }
-    pub fn true_remove_if<Q>(&self, key: &Q, exec: impl FnOnce(&K, &V) -> bool) -> bool
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        self.remove_if(key, exec).is_some()
-    }
-    pub fn remove_if<Q>(&self, key: &Q, exec: impl FnOnce(&K, &V) -> bool) -> Option<(K, V)>
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        self.inner.remove_if(key, exec)
-    }
-    /// Update or insert
-    pub fn upsert(&self, k: K, v: V) {
-        let _ = self.inner.insert(k, v);
-    }
-    /// Returns true if the value was updated
-    pub fn true_if_update(&self, k: K, v: V) -> bool {
-        if let Entry::Occupied(mut oe) = self.inner.entry(k) {
-            oe.insert(v);
-            true
-        } else {
-            false
-        }
-    }
-    pub fn mut_entry(&self, key: K) -> Option<OccupiedEntry<K, V, RandomState>> {
-        if let Entry::Occupied(oe) = self.inner.entry(key) {
-            Some(oe)
-        } else {
-            None
-        }
-    }
-    pub fn fresh_entry(&self, key: K) -> Option<VacantEntry<K, V, RandomState>> {
-        if let Entry::Vacant(ve) = self.inner.entry(key) {
-            Some(ve)
-        } else {
-            None
-        }
-    }
-}
-
-impl<K: Eq + Hash, V: Clone> Coremap<K, V> {
-    pub fn get_cloned<Q>(&self, key: &Q) -> Option<V>
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        self.inner.get_cloned(key)
-    }
-}
-
-impl<K: Eq + Hash + Clone, V> Coremap<K, V> {
-    /// Returns atleast `count` number of keys from the hashtable
-    pub fn get_keys(&self, count: usize) -> Vec<K> {
-        let mut v = Vec::with_capacity(count);
-        self.iter()
-            .take(count)
-            .map(|kv| kv.key().clone())
-            .for_each(|key| v.push(key));
-        v
-    }
-}
-
-impl<K: Eq + Hash, V> IntoIterator for Coremap<K, V> {
-    type Item = (K, V);
-    type IntoIter = OwnedIter<K, V, RandomState>;
-    fn into_iter(self) -> Self::IntoIter {
-        self.inner.get_owned_iter()
-    }
-}
-
-impl<K, V> FromIterator<(K, V)> for Coremap<K, V>
-where
-    K: Eq + Hash,
-{
-    fn from_iter<T>(iter: T) -> Self
-    where
-        T: IntoIterator<Item = (K, V)>,
-    {
-        Coremap {
-            inner: Skymap::from_iter(iter),
-        }
-    }
-}
diff --git a/server/src/corestore/iarray.rs b/server/src/corestore/iarray.rs
deleted file mode 100644
index f081b497..00000000
--- a/server/src/corestore/iarray.rs
+++ /dev/null
@@ -1,659 +0,0 @@
-/*
- * Created on Sun Jul 04 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#![allow(dead_code)] // TODO(@ohsayan): Remove this lint or remove offending methods
-
-use {
-    crate::corestore::array::LenScopeGuard,
-    core::{
-        alloc::Layout,
-        borrow::{Borrow, BorrowMut},
-        cmp, fmt,
-        hash::{self, Hash},
-        iter::FromIterator,
-        mem::{self, ManuallyDrop, MaybeUninit},
-        ops,
-        ptr::{self, NonNull},
-        slice,
-    },
-    std::alloc as std_alloc,
-};
-
-pub const fn new_const_iarray<T, const N: usize>() -> IArray<[T; N]> {
-    IArray {
-        cap: 0,
-        store: InlineArray {
-            stack: ManuallyDrop::new(MaybeUninit::uninit()),
-        },
-    }
-}
-
-/// An arbitrary trait used for identifying something as a contiguous block of memory
-pub trait MemoryBlock {
-    /// The type that will be used for the memory layout
-    type LayoutItem;
-    /// The number of _units_ this memory block has
-    fn size() -> usize;
-}
-
-impl<T, const N: usize> MemoryBlock for [T; N] {
-    type LayoutItem = T;
-    fn size() -> usize {
-        N
-    }
-}
-
-/// An union that either holds a stack (ptr) or a heap
-///
-/// ## Safety
-/// If you're trying to access a field without knowing the most recently created one,
-/// behavior is undefined.
-pub union InlineArray<A: MemoryBlock> {
-    /// the stack
-    stack: ManuallyDrop<MaybeUninit<A>>,
-    /// a pointer to the heap allocation and the allocation size
-    heap_ptr_len: (*mut A::LayoutItem, usize),
-}
-
-impl<A: MemoryBlock> InlineArray<A> {
-    /// Get's the stack pointer. This is unsafe because it is not guranteed that the
-    /// stack pointer field is valid and the caller has to uphold this gurantee
-    unsafe fn stack_ptr(&self) -> *const A::LayoutItem {
-        self.stack.as_ptr() as *const _
-    }
-    /// Safe as `stack_ptr`, but returns a mutable pointer
-    unsafe fn stack_ptr_mut(&mut self) -> *mut A::LayoutItem {
-        self.stack.as_mut_ptr() as *mut _
-    }
-    /// Create a new union from a stack
-    fn from_stack(stack: MaybeUninit<A>) -> Self {
-        Self {
-            stack: ManuallyDrop::new(stack),
-        }
-    }
-    /// Create a new union from a heap (allocated).
-    fn from_heap_ptr(start_ptr: *mut A::LayoutItem, len: usize) -> Self {
-        Self {
-            heap_ptr_len: (start_ptr, len),
-        }
-    }
-    /// Returns the allocation size of the heap
-    unsafe fn heap_size(&self) -> usize {
-        self.heap_ptr_len.1
-    }
-    /// Returns a raw ptr to the heap
-    unsafe fn heap_ptr(&self) -> *const A::LayoutItem {
-        self.heap_ptr_len.0
-    }
-    /// Returns a mut ptr to the heap
-    unsafe fn heap_ptr_mut(&mut self) -> *mut A::LayoutItem {
-        self.heap_ptr_len.0 as *mut _
-    }
-    /// Returns a mut ref to the heap allocation size
-    unsafe fn heap_size_mut(&mut self) -> &mut usize {
-        &mut self.heap_ptr_len.1
-    }
-    /// Returns the entire heap field
-    unsafe fn heap(&self) -> (*mut A::LayoutItem, usize) {
-        self.heap_ptr_len
-    }
-    /// Returns a mutable reference to the entire heap field
-    unsafe fn heap_mut(&mut self) -> (*mut A::LayoutItem, &mut usize) {
-        (self.heap_ptr_mut(), &mut self.heap_ptr_len.1)
-    }
-}
-
-/// An utility tool for calculating the memory layout for a given `T`. Handles
-/// any possible overflows
-pub fn calculate_memory_layout<T>(count: usize) -> Result<Layout, ()> {
-    let size = mem::size_of::<T>().checked_mul(count).ok_or(())?;
-    // err is cap overflow
-    let alignment = mem::align_of::<T>();
-    Layout::from_size_align(size, alignment).map_err(|_| ())
-}
-
-/// Use the global allocator to deallocate the memory block for the given starting ptr
-/// upto the given capacity
-unsafe fn dealloc<T>(start_ptr: *mut T, capacity: usize) {
-    std_alloc::dealloc(
-        start_ptr as *mut u8,
-        calculate_memory_layout::<T>(capacity).expect("Memory capacity overflow"),
-    )
-}
-
-// Break free from Rust's aliasing rules with these typedefs
-type DataptrLenptrCapacity<T> = (*const T, usize, usize);
-type DataptrLenptrCapacityMut<'a, T> = (*mut T, &'a mut usize, usize);
-
-/// A stack optimized backing store
-///
-/// An [`IArray`] is heavily optimized for storing items on the stack and will
-/// not perform very well (but of course will) when the object overflows its
-/// stack and is moved to the heap. Optimizations are made to mark overflows
-/// as branches that are unlikely to be called. The IArray is like a smallvec,
-/// but with extremely aggressive optimizations for items stored on the stack,
-/// for example to avoid the maneuvers with speculative execution.
-/// This makes the [`IArray`] extremely performant for operations on the stack,
-/// but a little expensive when operations are done on the heap
-pub struct IArray<A: MemoryBlock> {
-    cap: usize,
-    store: InlineArray<A>,
-}
-
-#[cfg(test)]
-impl IArray<[u8; 48]> {
-    /// Returns a new 48-bit, stack allocated array of bytes
-    fn new_bytearray() -> Self {
-        Self::new()
-    }
-}
-
-impl<A: MemoryBlock> IArray<A> {
-    pub fn new() -> IArray<A> {
-        Self {
-            cap: 0,
-            store: InlineArray::from_stack(MaybeUninit::uninit()),
-        }
-    }
-    pub fn from_vec(mut vec: Vec<A::LayoutItem>) -> Self {
-        if vec.capacity() <= Self::stack_capacity() {
-            let mut store = InlineArray::<A>::from_stack(MaybeUninit::uninit());
-            let len = vec.len();
-            unsafe {
-                ptr::copy_nonoverlapping(vec.as_ptr(), store.stack_ptr_mut(), len);
-            }
-            // done with the copy
-            Self { cap: len, store }
-        } else {
-            // off to the heap
-            let (start_ptr, cap, len) = (vec.as_mut_ptr(), vec.capacity(), vec.len());
-            // leak the vec
-            mem::forget(vec);
-            IArray {
-                cap,
-                store: InlineArray::from_heap_ptr(start_ptr, len),
-            }
-        }
-    }
-    /// Returns the total capacity of the inline stack
-    fn stack_capacity() -> usize {
-        if mem::size_of::<A::LayoutItem>() > 0 {
-            // not a ZST, so cap of array
-            A::size()
-        } else {
-            // ZST. Just pile up some garbage and say that we have infinity
-            usize::MAX
-        }
-    }
-    /// Helper function that returns a ptr to the data, the len and the capacity
-    fn meta_triple(&self) -> DataptrLenptrCapacity<A::LayoutItem> {
-        unsafe {
-            if self.went_off_stack() {
-                let (data_ptr, len_ref) = self.store.heap();
-                (data_ptr, len_ref, self.cap)
-            } else {
-                // still on stack
-                (self.store.stack_ptr(), self.cap, Self::stack_capacity())
-            }
-        }
-    }
-    /// Mutable version of `meta_triple`
-    fn meta_triple_mut(&mut self) -> DataptrLenptrCapacityMut<A::LayoutItem> {
-        unsafe {
-            if self.went_off_stack() {
-                // get heap
-                let (data_ptr, len_ref) = self.store.heap_mut();
-                (data_ptr, len_ref, self.cap)
-            } else {
-                // still on stack
-                (
-                    self.store.stack_ptr_mut(),
-                    &mut self.cap,
-                    Self::stack_capacity(),
-                )
-            }
-        }
-    }
-    /// Returns a raw ptr to the data
-    fn get_data_ptr_mut(&mut self) -> *mut A::LayoutItem {
-        if self.went_off_stack() {
-            // get the heap ptr
-            unsafe { self.store.heap_ptr_mut() }
-        } else {
-            // get the stack ptr
-            unsafe { self.store.stack_ptr_mut() }
-        }
-    }
-    /// Returns true if the allocation is now on the heap
-    fn went_off_stack(&self) -> bool {
-        self.cap > Self::stack_capacity()
-    }
-    /// Returns the length
-    pub fn len(&self) -> usize {
-        if self.went_off_stack() {
-            // so we're off the stack
-            unsafe { self.store.heap_size() }
-        } else {
-            // still on the stack
-            self.cap
-        }
-    }
-    /// Returns true if the IArray is empty
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-    /// Returns the capacity
-    fn get_capacity(&self) -> usize {
-        if self.went_off_stack() {
-            self.cap
-        } else {
-            Self::stack_capacity()
-        }
-    }
-    /// Grow the allocation, if required, to make space for a total of `new_cap`
-    /// elements
-    fn grow_block(&mut self, new_cap: usize) {
-        // infallible
-        unsafe {
-            let (data_ptr, &mut len, cap) = self.meta_triple_mut();
-            let still_on_stack = !self.went_off_stack();
-            assert!(new_cap > len);
-            if new_cap <= Self::stack_capacity() {
-                if still_on_stack {
-                    return;
-                }
-                self.store = InlineArray::from_stack(MaybeUninit::uninit());
-                ptr::copy_nonoverlapping(data_ptr, self.store.stack_ptr_mut(), len);
-                self.cap = len;
-                dealloc(data_ptr, cap);
-            } else if new_cap != cap {
-                let layout =
-                    calculate_memory_layout::<A::LayoutItem>(new_cap).expect("Capacity overflow");
-                assert!(layout.size() > 0);
-                let new_alloc;
-                if still_on_stack {
-                    new_alloc = NonNull::new(std_alloc::alloc(layout).cast())
-                        .expect("Allocation error")
-                        .as_ptr();
-                    ptr::copy_nonoverlapping(data_ptr, new_alloc, len);
-                } else {
-                    // not on stack
-                    let old_layout =
-                        calculate_memory_layout::<A::LayoutItem>(cap).expect("Capacity overflow");
-                    // realloc the earlier buffer
-                    let new_memory_block_ptr =
-                        std_alloc::realloc(data_ptr as *mut _, old_layout, layout.size());
-                    new_alloc = NonNull::new(new_memory_block_ptr.cast())
-                        .expect("Allocation error")
-                        .as_ptr();
-                }
-                self.store = InlineArray::from_heap_ptr(new_alloc, len);
-                self.cap = new_cap;
-            }
-        }
-    }
-    /// Reserve space for `additional` elements
-    fn reserve(&mut self, additional: usize) {
-        let (_, &mut len, cap) = self.meta_triple_mut();
-        if cap - len >= additional {
-            // already have enough space
-            return;
-        }
-        let new_cap = len
-            .checked_add(additional)
-            .map(usize::next_power_of_two)
-            .expect("Capacity overflow");
-        self.grow_block(new_cap)
-    }
-    /// Push an element into this IArray
-    pub fn push(&mut self, val: A::LayoutItem) {
-        unsafe {
-            let (mut data_ptr, mut len, cap) = self.meta_triple_mut();
-            if (*len).eq(&cap) {
-                self.reserve(1);
-                let (heap_ptr, heap_len) = self.store.heap_mut();
-                data_ptr = heap_ptr;
-                len = heap_len;
-            }
-            ptr::write(data_ptr.add(*len), val);
-            *len += 1;
-        }
-    }
-    /// Pop an element off this IArray
-    pub fn pop(&mut self) -> Option<A::LayoutItem> {
-        unsafe {
-            let (data_ptr, len_mut, _cap) = self.meta_triple_mut();
-            if *len_mut == 0 {
-                // empty man, what do you want?
-                None
-            } else {
-                // there's something
-                let last_index = *len_mut - 1;
-                // we'll say that it's gone
-                *len_mut = last_index;
-                // but only read it now from the offset
-                Some(ptr::read(data_ptr.add(last_index)))
-            }
-        }
-    }
-    /// This is amazingly dangerous if `idx` doesn't exist. You can potentially
-    /// corrupt a bunch of things
-    pub unsafe fn remove(&mut self, idx: usize) -> A::LayoutItem {
-        let (mut ptr, len_ref, _) = self.meta_triple_mut();
-        let len = *len_ref;
-        *len_ref = len - 1;
-        ptr = ptr.add(idx);
-        let item = ptr::read(ptr);
-        ptr::copy(ptr.add(1), ptr, len - idx - 1);
-        item
-    }
-    /// Shrink this IArray so that it only occupies the required space and not anything
-    /// more
-    pub fn shrink(&mut self) {
-        if self.went_off_stack() {
-            // it's off the stack, so no chance of moving back to the stack
-            return;
-        }
-        let current_len = self.len();
-        if Self::stack_capacity() >= current_len {
-            // we have a chance of copying this over to our stack
-            unsafe {
-                let (data_ptr, len) = self.store.heap();
-                self.store = InlineArray::from_stack(MaybeUninit::uninit());
-                // copy to stack
-                ptr::copy_nonoverlapping(data_ptr, self.store.stack_ptr_mut(), len);
-                // now deallocate the heap
-                dealloc(data_ptr, self.cap);
-                self.cap = len;
-            }
-        } else if self.get_capacity() > current_len {
-            // more capacity than current len? so we're on the heap
-            // grow the block to place it on stack (this will dealloc the heap)
-            self.grow_block(current_len);
-        }
-    }
-    /// Truncate the IArray to a given length. This **will** call the destructors
-    pub fn truncate(&mut self, target_len: usize) {
-        unsafe {
-            let (data_ptr, len_mut, _cap) = self.meta_triple_mut();
-            while target_len < *len_mut {
-                // get the last index
-                let last_index = *len_mut - 1;
-                // drop it
-                ptr::drop_in_place(data_ptr.add(last_index));
-                // update the length
-                *len_mut = last_index;
-            }
-        }
-    }
-    /// Clear the internal store
-    pub fn clear(&mut self) {
-        // chop off the whole place
-        self.truncate(0);
-    }
-    /// Set the len, **without calling the destructor**. This is the ultimate function
-    /// to make valgrind unhappy, that is, **you can create memory leaks** if you don't
-    /// destroy the elements yourself
-    unsafe fn set_len(&mut self, new_len: usize) {
-        let (_dataptr, len_mut, _cap) = self.meta_triple_mut();
-        *len_mut = new_len;
-    }
-}
-
-impl<A: MemoryBlock> IArray<A>
-where
-    A::LayoutItem: Copy,
-{
-    /// Create an IArray from a slice by copying the elements of the slice into
-    /// the IArray
-    pub fn from_slice(slice: &[A::LayoutItem]) -> Self {
-        // FIXME(@ohsayan): Could we have had this as a From::from() method?
-        let slice_len = slice.len();
-        if slice_len <= Self::stack_capacity() {
-            // so we can place this thing on the stack
-            let mut new_stack = MaybeUninit::uninit();
-            unsafe {
-                ptr::copy_nonoverlapping(
-                    slice.as_ptr(),
-                    new_stack.as_mut_ptr() as *mut A::LayoutItem,
-                    slice_len,
-                );
-            }
-            Self {
-                cap: slice_len,
-                store: InlineArray::from_stack(new_stack),
-            }
-        } else {
-            // argggh, on the heap
-            let mut v = slice.to_vec();
-            let (ptr, cap) = (v.as_mut_ptr(), v.capacity());
-            // leak it
-            mem::forget(v);
-            Self {
-                cap,
-                store: InlineArray::from_heap_ptr(ptr, slice_len),
-            }
-        }
-    }
-    /// Insert a slice at the given index
-    pub fn insert_slice_at_index(&mut self, slice: &[A::LayoutItem], index: usize) {
-        self.reserve(slice.len());
-        let len = self.len();
-        // only catch during tests
-        debug_assert!(index <= len);
-        unsafe {
-            let slice_ptr = slice.as_ptr();
-            // we need to add it from the end of the current item
-            let data_ptr_start = self.get_data_ptr_mut().add(len);
-            // copy the slice over
-            ptr::copy(data_ptr_start, data_ptr_start.add(slice.len()), len - index);
-            ptr::copy_nonoverlapping(slice_ptr, data_ptr_start, slice.len());
-            self.set_len(len + slice.len());
-        }
-    }
-    /// Extend the IArray by using a slice
-    pub fn extend_from_slice(&mut self, slice: &[A::LayoutItem]) {
-        // at our len because we're appending it to the end
-        self.insert_slice_at_index(slice, self.len())
-    }
-    /// Create a new IArray from a pre-defined stack
-    pub fn from_stack(stack: A) -> Self {
-        Self {
-            cap: A::size(),
-            store: InlineArray::from_stack(MaybeUninit::new(stack)),
-        }
-    }
-}
-
-impl<A: MemoryBlock> ops::Deref for IArray<A> {
-    type Target = [A::LayoutItem];
-    fn deref(&self) -> &Self::Target {
-        unsafe {
-            let (start_ptr, len, _) = self.meta_triple();
-            slice::from_raw_parts(start_ptr, len)
-        }
-    }
-}
-
-impl<A: MemoryBlock> ops::DerefMut for IArray<A> {
-    fn deref_mut(&mut self) -> &mut [A::LayoutItem] {
-        unsafe {
-            let (start_ptr, &mut len, _) = self.meta_triple_mut();
-            slice::from_raw_parts_mut(start_ptr, len)
-        }
-    }
-}
-
-impl<A: MemoryBlock> AsRef<[A::LayoutItem]> for IArray<A> {
-    fn as_ref(&self) -> &[A::LayoutItem] {
-        self
-    }
-}
-
-impl<A: MemoryBlock> AsMut<[A::LayoutItem]> for IArray<A> {
-    fn as_mut(&mut self) -> &mut [A::LayoutItem] {
-        self
-    }
-}
-
-// we need these for our coremap
-
-impl<A: MemoryBlock> Borrow<[A::LayoutItem]> for IArray<A> {
-    fn borrow(&self) -> &[A::LayoutItem] {
-        self
-    }
-}
-
-impl<A: MemoryBlock> BorrowMut<[A::LayoutItem]> for IArray<A> {
-    fn borrow_mut(&mut self) -> &mut [A::LayoutItem] {
-        self
-    }
-}
-
-impl<A: MemoryBlock> Drop for IArray<A> {
-    fn drop(&mut self) {
-        unsafe {
-            if self.went_off_stack() {
-                // free the heap
-                let (ptr, len) = self.store.heap();
-                // let vec's destructor do the work
-                mem::drop(Vec::from_raw_parts(ptr, len, self.cap));
-            } else {
-                // on stack? get self as a slice and destruct it
-                ptr::drop_in_place(&mut self[..]);
-            }
-        }
-    }
-}
-
-impl<A: MemoryBlock> Extend<A::LayoutItem> for IArray<A> {
-    fn extend<I: IntoIterator<Item = A::LayoutItem>>(&mut self, iterable: I) {
-        let mut iter = iterable.into_iter();
-        let (lower_bound, _upper_bound) = iter.size_hint();
-        // reserve the lower bound; we really want it on the stack
-        self.reserve(lower_bound);
-
-        unsafe {
-            let (data_ptr, len_ref, cap) = self.meta_triple_mut();
-            let mut len = LenScopeGuard::new(len_ref);
-            while len.get_temp() < cap {
-                if let Some(out) = iter.next() {
-                    ptr::write(data_ptr.add(len.get_temp()), out);
-                    len.incr(1);
-                } else {
-                    return;
-                }
-            }
-        }
-        // still have something left, probably a heap alloc :(
-        for elem in iter {
-            self.push(elem);
-        }
-    }
-}
-
-impl<A: MemoryBlock> fmt::Debug for IArray<A>
-where
-    A::LayoutItem: fmt::Debug,
-{
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
-        f.debug_list().entries(self.iter()).finish()
-    }
-}
-
-impl<A: MemoryBlock, B: MemoryBlock> PartialEq<IArray<B>> for IArray<A>
-where
-    A::LayoutItem: PartialEq<B::LayoutItem>,
-{
-    fn eq(&self, rhs: &IArray<B>) -> bool {
-        self[..] == rhs[..]
-    }
-}
-
-impl<A: MemoryBlock> Eq for IArray<A> where A::LayoutItem: Eq {}
-
-impl<A: MemoryBlock> PartialOrd for IArray<A>
-where
-    A::LayoutItem: PartialOrd,
-{
-    fn partial_cmp(&self, rhs: &IArray<A>) -> Option<cmp::Ordering> {
-        PartialOrd::partial_cmp(&**self, &**rhs)
-    }
-}
-
-impl<A: MemoryBlock> Ord for IArray<A>
-where
-    A::LayoutItem: Ord,
-{
-    fn cmp(&self, rhs: &IArray<A>) -> cmp::Ordering {
-        Ord::cmp(&**self, &**rhs)
-    }
-}
-
-impl<A: MemoryBlock> Hash for IArray<A>
-where
-    A::LayoutItem: Hash,
-{
-    fn hash<H>(&self, hasher: &mut H)
-    where
-        H: hash::Hasher,
-    {
-        (**self).hash(hasher)
-    }
-}
-
-impl<A: MemoryBlock> FromIterator<A::LayoutItem> for IArray<A> {
-    fn from_iter<I: IntoIterator<Item = A::LayoutItem>>(iter: I) -> Self {
-        let mut iarray = IArray::new();
-        iarray.extend(iter);
-        iarray
-    }
-}
-
-impl<'a, A: MemoryBlock> From<&'a [A::LayoutItem]> for IArray<A>
-where
-    A::LayoutItem: Clone,
-{
-    fn from(slice: &'a [A::LayoutItem]) -> Self {
-        slice.iter().cloned().collect()
-    }
-}
-
-unsafe impl<A: MemoryBlock> Send for IArray<A> where A::LayoutItem: Send {}
-unsafe impl<A: MemoryBlock> Sync for IArray<A> where A::LayoutItem: Sync {}
-
-#[test]
-fn test_equality() {
-    let mut x = IArray::new_bytearray();
-    x.extend_from_slice("AVeryGoodKeyspaceName".as_bytes());
-    assert_eq!(x, {
-        let mut i = IArray::<[u8; 64]>::new();
-        "AVeryGoodKeyspaceName"
-            .chars()
-            .for_each(|char| i.push(char as u8));
-        i
-    })
-}
diff --git a/server/src/corestore/lazy.rs b/server/src/corestore/lazy.rs
deleted file mode 100644
index b3f80a05..00000000
--- a/server/src/corestore/lazy.rs
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Created on Sat Jul 03 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::backoff::Backoff,
-    core::{
-        mem,
-        ops::Deref,
-        ptr,
-        sync::atomic::{AtomicBool, AtomicPtr, Ordering},
-    },
-};
-
-const ORD_ACQ: Ordering = Ordering::Acquire;
-const ORD_SEQ: Ordering = Ordering::SeqCst;
-const ORD_REL: Ordering = Ordering::Release;
-const ORD_RLX: Ordering = Ordering::Relaxed;
-
-/// A lazily intialized, or _call by need_ value
-#[derive(Debug)]
-pub struct Lazy<T, F> {
-    /// the value (null at first)
-    value: AtomicPtr<T>,
-    /// the function that will init the value
-    init_func: F,
-    /// is some thread trying to initialize the value
-    init_state: AtomicBool,
-}
-
-impl<T, F> Lazy<T, F> {
-    pub const fn new(init_func: F) -> Self {
-        Self {
-            value: AtomicPtr::new(ptr::null_mut()),
-            init_func,
-            init_state: AtomicBool::new(false),
-        }
-    }
-}
-
-impl<T, F> Deref for Lazy<T, F>
-where
-    F: Fn() -> T,
-{
-    type Target = T;
-    fn deref(&self) -> &Self::Target {
-        let value_ptr = self.value.load(ORD_ACQ);
-        if !value_ptr.is_null() {
-            // the value has already been initialized, return
-            unsafe {
-                // UNSAFE(@ohsayan): We've just asserted that the value is not null
-                return &*value_ptr;
-            }
-        }
-        // it's null, so it's useless
-
-        // hold on until someone is trying to init
-        let backoff = Backoff::new();
-        while self
-            .init_state
-            .compare_exchange(false, true, ORD_SEQ, ORD_SEQ)
-            .is_err()
-        {
-            // wait until the other thread finishes
-            backoff.snooze();
-        }
-        /*
-         see the value before the last store. while we were one the loop,
-         some other thread could have initialized it already
-        */
-        let value_ptr = self.value.load(ORD_ACQ);
-        if !value_ptr.is_null() {
-            // no more init, someone initialized it already
-            assert!(self.init_state.swap(false, ORD_SEQ));
-            unsafe {
-                // UNSAFE(@ohsayan): We've already loaded the value checked
-                // that it isn't null
-                &*value_ptr
-            }
-        } else {
-            // so no one cared to initialize the value in between
-            // fine, we'll init it
-            let value = (self.init_func)();
-            let value_ptr = Box::into_raw(Box::new(value));
-            // now swap out the older value and check it for sanity
-            assert!(self.value.swap(value_ptr, ORD_SEQ).is_null());
-            // set trying to init flag to false
-            assert!(self.init_state.swap(false, ORD_SEQ));
-            unsafe {
-                // UNSAFE(@ohsayan): We just initialized the value ourselves
-                // so it is not null!
-                &*value_ptr
-            }
-        }
-    }
-}
-
-/*
- Note on memory leaks:
- Suddenly found a possible leak with a static created with the Lazy type? Well, we'll have to
- ignore it. That's because destructors aren't called on statics. A thunk leak. So what's to be
- done here? Well, the best we can do is implement a destructor but it is never guranteed that
- it will be called when used in global scope
-*/
-
-impl<T, F> Drop for Lazy<T, F> {
-    fn drop(&mut self) {
-        if mem::needs_drop::<T>() {
-            // this needs drop
-            let value_ptr = self.value.load(ORD_ACQ);
-            if !value_ptr.is_null() {
-                unsafe {
-                    // UNSAFE(@ohsayan): We've just checked if the value is null or not
-                    mem::drop(Box::from_raw(value_ptr))
-                }
-            }
-        }
-    }
-}
-
-cfg_test!(
-    use crate::corestore::SharedSlice;
-    use crate::corestore::lazy;
-    use std::collections::HashMap;
-    use std::thread;
-
-    #[allow(clippy::type_complexity)]
-    static LAZY_VALUE: lazy::Lazy<HashMap<SharedSlice, SharedSlice>, fn() -> HashMap<SharedSlice, SharedSlice>> = lazy::Lazy::new(|| {
-        #[allow(clippy::mutable_key_type)]
-        let mut ht = HashMap::new();
-        ht.insert("sayan".into(), "is doing something".into());
-        ht
-    });
-
-    #[test]
-    fn test_lazy() {
-        assert_eq!(
-            LAZY_VALUE.get("sayan".as_bytes()).unwrap().clone(),
-            SharedSlice::from("is doing something")
-        );
-    }
-
-    #[test]
-    fn test_two_threads_trying_to_get_at_once() {
-        let (t1, t2) = (
-            thread::spawn(|| {
-                assert_eq!(
-                    LAZY_VALUE.get("sayan".as_bytes()).unwrap().clone(),
-                    SharedSlice::from("is doing something")
-                );}),
-            thread::spawn(|| {
-                assert_eq!(
-                    LAZY_VALUE.get("sayan".as_bytes()).unwrap().clone(),
-                    SharedSlice::from("is doing something")
-                );
-            })
-        );
-        {
-            t1.join().unwrap();
-            t2.join().unwrap();
-        }
-    }
-
-    struct WeirdTestStruct(u8);
-    impl Drop for WeirdTestStruct {
-        fn drop(&mut self) {
-            panic!("PANIC ON DROP! THIS IS OKAY!");
-        }
-    }
-    #[test]
-    #[should_panic]
-    fn test_drop() {
-        // this is only when the lazy is initialized in local scope and not global scope
-        let x: Lazy<WeirdTestStruct, fn() -> WeirdTestStruct> = Lazy::new(|| {
-            WeirdTestStruct(0)
-        });
-        // just do an useless deref to make the pointer non null
-        let _deref = &*x;
-        drop(x); // we should panic right here
-    }
-    #[test]
-    fn test_no_drop_null() {
-        let x: Lazy<WeirdTestStruct, fn() -> WeirdTestStruct> = Lazy::new(|| {
-            WeirdTestStruct(0)
-        });
-        drop(x); // no panic because it is null
-    }
-);
-
-/// A "cell" that can be initialized once using a single atomic
-pub struct Once<T> {
-    value: AtomicPtr<T>,
-}
-
-#[allow(dead_code)] // TODO: Remove this
-impl<T> Once<T> {
-    pub const fn new() -> Self {
-        Self {
-            value: AtomicPtr::new(ptr::null_mut()),
-        }
-    }
-    pub fn with_value(val: T) -> Self {
-        Self {
-            value: AtomicPtr::new(Box::into_raw(Box::new(val))),
-        }
-    }
-    pub fn get(&self) -> Option<&T> {
-        // synchronizes with the store for value ptr
-        let ptr = self.value.load(ORD_ACQ);
-        if ptr.is_null() {
-            None
-        } else {
-            unsafe { Some(&*self.value.load(ORD_ACQ)) }
-        }
-    }
-    pub fn set(&self, val: T) -> bool {
-        // synchronizes with the store for set
-        let snapshot = self.value.load(ORD_ACQ);
-        if snapshot.is_null() {
-            // let's try to init this
-            let vptr = Box::into_raw(Box::new(val));
-            // if malloc fails, that's fine because there will be no cas
-            let r = self.value.compare_exchange(
-                snapshot, vptr, // we must use release ordering to sync with the acq
-                ORD_REL,
-                // on failure simply use relaxed because we don't use the value anyways
-                // -- so why bother stressing out the processor?
-                ORD_RLX,
-            );
-            r.is_ok()
-        } else {
-            false
-        }
-    }
-}
-
-impl<T> Drop for Once<T> {
-    fn drop(&mut self) {
-        let snapshot = self.value.load(ORD_ACQ);
-        if !snapshot.is_null() {
-            unsafe { mem::drop(Box::from_raw(snapshot)) }
-        }
-    }
-}
-
-impl<T> From<Option<T>> for Once<T> {
-    fn from(v: Option<T>) -> Self {
-        match v {
-            Some(v) => Self::with_value(v),
-            None => Self::new(),
-        }
-    }
-}
-
-#[test]
-fn once_get_none() {
-    let once: Once<u8> = Once::new();
-    assert_eq!(once.get(), None);
-}
-
-cfg_test! {
-    use std::sync::Arc;
-    use std::time::Duration;
-}
-
-#[test]
-fn once_set_get_some() {
-    let once: Arc<Once<u8>> = Arc::new(Once::new());
-    let t1 = once.clone();
-    let t2 = once.clone();
-    let t3 = once.clone();
-    let hdl1 = thread::spawn(move || {
-        assert!(t1.set(10));
-    });
-    thread::sleep(Duration::from_secs(3));
-    let hdl2 = thread::spawn(move || {
-        assert!(!t2.set(10));
-    });
-    let hdl3 = thread::spawn(move || {
-        assert_eq!(*t3.get().unwrap(), 10);
-    });
-    assert_eq!(*once.get().unwrap(), 10);
-    hdl1.join().unwrap();
-    hdl2.join().unwrap();
-    hdl3.join().unwrap();
-}
diff --git a/server/src/corestore/lock.rs b/server/src/corestore/lock.rs
deleted file mode 100644
index dbcaf7f5..00000000
--- a/server/src/corestore/lock.rs
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Created on Tue Jun 29 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # Locks
-//!
-//! In several scenarios, we may find `std`'s or other crates' implementations of synchronization
-//! primitives to be either _too sophisticated_ or _not what we want_. For these cases, we use
-//! the primitives that are defined here
-//!
-
-use {
-    super::backoff::Backoff,
-    std::{
-        cell::UnsafeCell,
-        ops::{Deref, DerefMut},
-        sync::atomic::{AtomicBool, Ordering},
-    },
-};
-
-const ORD_ACQUIRE: Ordering = Ordering::Acquire;
-const ORD_RELEASE: Ordering = Ordering::Release;
-
-/// An extremely simple lock without the extra fuss: just the raw data and an atomic bool
-#[derive(Debug)]
-pub struct QuickLock<T> {
-    rawdata: UnsafeCell<T>,
-    lock_state: AtomicBool,
-}
-
-unsafe impl<T: Send> Sync for QuickLock<T> {}
-unsafe impl<T: Send> Send for QuickLock<T> {}
-
-/// A lock guard created by [`QuickLock`]
-pub struct QLGuard<'a, T> {
-    lck: &'a QuickLock<T>,
-}
-
-impl<'a, T> QLGuard<'a, T> {
-    const fn init(lck: &'a QuickLock<T>) -> Self {
-        Self { lck }
-    }
-}
-
-/*
- * Acq/Rel semantics don't emit fences on Intel platforms, but on weakly ordered targets
- * things may look different.
-*/
-
-impl<T> QuickLock<T> {
-    pub const fn new(rawdata: T) -> Self {
-        Self {
-            lock_state: AtomicBool::new(false),
-            rawdata: UnsafeCell::new(rawdata),
-        }
-    }
-    /// Try to acquire a lock
-    pub fn try_lock(&self) -> Option<QLGuard<'_, T>> {
-        let ret = self
-            .lock_state
-            .compare_exchange(false, true, ORD_ACQUIRE, ORD_ACQUIRE);
-        if ret.is_ok() {
-            Some(QLGuard::init(self))
-        } else {
-            None
-        }
-    }
-    /// Enter a _busy loop_ waiting to get an unlock. Behold, this is blocking!
-    pub fn lock(&self) -> QLGuard<'_, T> {
-        let backoff = Backoff::new();
-        loop {
-            let ret = self.lock_state.compare_exchange_weak(
-                false,
-                true,
-                Ordering::SeqCst,
-                Ordering::Relaxed,
-            );
-            match ret {
-                Ok(_) => break QLGuard::init(self),
-                Err(is_locked) => {
-                    if !is_locked {
-                        break QLGuard::init(self);
-                    }
-                }
-            }
-            backoff.snooze();
-        }
-    }
-}
-
-impl<'a, T> Drop for QLGuard<'a, T> {
-    fn drop(&mut self) {
-        #[cfg(test)]
-        assert!(self.lck.lock_state.swap(false, ORD_RELEASE));
-        #[cfg(not(test))]
-        let _ = self.lck.lock_state.swap(false, ORD_RELEASE);
-    }
-}
-
-impl<'a, T> Deref for QLGuard<'a, T> {
-    type Target = T;
-    fn deref(&self) -> &Self::Target {
-        unsafe {
-            // UNSAFE(@ohsayan): Who doesn't like raw pointers anyway? (rustc: sigh)
-            &*self.lck.rawdata.get()
-        }
-    }
-}
-
-impl<'a, T> DerefMut for QLGuard<'a, T> {
-    fn deref_mut(&mut self) -> &mut T {
-        unsafe {
-            // UNSAFE(@ohsayan): Who doesn't like raw pointers anyway? (rustc: sigh)
-            &mut *self.lck.rawdata.get()
-        }
-    }
-}
-
-#[test]
-fn test_already_locked() {
-    let lck = QuickLock::new(200);
-    let _our_lock = lck.lock();
-    assert!(lck.try_lock().is_none());
-}
-
-#[cfg(test)]
-use std::{
-    sync::{mpsc, Arc},
-    thread,
-    time::Duration,
-};
-
-#[cfg(test)]
-fn panic_timeout<T, F>(dur: Duration, f: F) -> T
-where
-    T: Send + 'static,
-    F: (FnOnce() -> T) + Send + 'static,
-{
-    let (tx, rx) = mpsc::channel::<()>();
-    let handle = thread::spawn(move || {
-        let val = f();
-        let _ = tx.send(());
-        val
-    });
-    match rx.recv_timeout(dur) {
-        Ok(_) => handle.join().expect("Thread panicked"),
-        Err(_) => panic!("Thread passed timeout"),
-    }
-}
-
-#[test]
-#[should_panic]
-fn test_two_lock_timeout() {
-    let lck = Arc::new(QuickLock::new(1u8));
-    let child_lock = lck.clone();
-    let _lock = lck.lock();
-    panic_timeout(Duration::from_secs(1), move || {
-        let lck = child_lock;
-        let _ret = lck.lock();
-    });
-}
diff --git a/server/src/corestore/map/bref.rs b/server/src/corestore/map/bref.rs
deleted file mode 100644
index 7dbd3fcb..00000000
--- a/server/src/corestore/map/bref.rs
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Created on Mon Aug 09 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::LowMap,
-    crate::util::{compiler, Unwrappable},
-    core::{
-        hash::{BuildHasher, Hash},
-        mem,
-        ops::{Deref, DerefMut},
-    },
-    parking_lot::{RwLockReadGuard, RwLockWriteGuard},
-    std::{collections::hash_map::RandomState, sync::Arc},
-};
-
-/// A read-only reference to a bucket
-pub struct Ref<'a, K, V> {
-    _g: RwLockReadGuard<'a, LowMap<K, V>>,
-    k: &'a K,
-    v: &'a V,
-}
-
-impl<'a, K, V> Ref<'a, K, V> {
-    /// Create a new reference
-    pub(super) const fn new(_g: RwLockReadGuard<'a, LowMap<K, V>>, k: &'a K, v: &'a V) -> Self {
-        Self { _g, k, v }
-    }
-    /// Get a ref to the key
-    pub const fn key(&self) -> &K {
-        self.k
-    }
-    /// Get a ref to the value
-    pub const fn value(&self) -> &V {
-        self.v
-    }
-}
-
-impl<'a, K, V> Deref for Ref<'a, K, V> {
-    type Target = V;
-    fn deref(&self) -> &Self::Target {
-        self.value()
-    }
-}
-
-unsafe impl<'a, K: Send, V: Send> Send for Ref<'a, K, V> {}
-unsafe impl<'a, K: Sync, V: Sync> Sync for Ref<'a, K, V> {}
-
-/// A r/w ref to a bucket
-pub struct RefMut<'a, K, V> {
-    _g: RwLockWriteGuard<'a, LowMap<K, V>>,
-    _k: &'a K,
-    v: &'a mut V,
-}
-
-impl<'a, K, V> RefMut<'a, K, V> {
-    /// Create a new ref
-    pub(super) fn new(_g: RwLockWriteGuard<'a, LowMap<K, V>>, k: &'a K, v: &'a mut V) -> Self {
-        Self { _g, _k: k, v }
-    }
-    /// Get a ref to the value
-    pub const fn value(&self) -> &V {
-        self.v
-    }
-    /// Get a mutable ref to the value
-    pub fn value_mut(&mut self) -> &mut V {
-        self.v
-    }
-}
-
-impl<'a, K, V> Deref for RefMut<'a, K, V> {
-    type Target = V;
-    fn deref(&self) -> &Self::Target {
-        self.value()
-    }
-}
-
-impl<'a, K, V> DerefMut for RefMut<'a, K, V> {
-    fn deref_mut(&mut self) -> &mut V {
-        self.value_mut()
-    }
-}
-
-unsafe impl<'a, K: Send, V: Send> Send for RefMut<'a, K, V> {}
-unsafe impl<'a, K: Sync, V: Sync> Sync for RefMut<'a, K, V> {}
-
-/// A reference to an occupied entry
-pub struct OccupiedEntry<'a, K, V, S> {
-    guard: RwLockWriteGuard<'a, LowMap<K, V>>,
-    elem: (&'a K, &'a mut V),
-    key: K,
-    hasher: S,
-}
-
-impl<'a, K: Hash + Eq, V, S: BuildHasher> OccupiedEntry<'a, K, V, S> {
-    /// Create a new occupied entry ref
-    pub(super) fn new(
-        guard: RwLockWriteGuard<'a, LowMap<K, V>>,
-        key: K,
-        elem: (&'a K, &'a mut V),
-        hasher: S,
-    ) -> Self {
-        Self {
-            guard,
-            elem,
-            key,
-            hasher,
-        }
-    }
-    /// Get a ref to the value
-    pub fn value(&self) -> &V {
-        self.elem.1
-    }
-    /// Insert a value into this bucket
-    pub fn insert(&mut self, other: V) -> V {
-        mem::replace(self.elem.1, other)
-    }
-    /// Remove this element from the map
-    pub fn remove(mut self) -> V {
-        let hash = super::make_hash::<K, _, S>(&self.hasher, &self.key);
-        unsafe {
-            self.guard
-                .remove_entry(hash, super::ceq(self.elem.0))
-                .unsafe_unwrap()
-        }
-        .1
-    }
-}
-
-unsafe impl<'a, K: Send, V: Send, S> Send for OccupiedEntry<'a, K, V, S> {}
-unsafe impl<'a, K: Sync, V: Sync, S> Sync for OccupiedEntry<'a, K, V, S> {}
-
-/// A ref to a vacant entry
-pub struct VacantEntry<'a, K, V, S> {
-    guard: RwLockWriteGuard<'a, LowMap<K, V>>,
-    key: K,
-    hasher: S,
-}
-
-impl<'a, K: Hash + Eq, V, S: BuildHasher> VacantEntry<'a, K, V, S> {
-    /// Create a vacant entry ref
-    pub(super) fn new(guard: RwLockWriteGuard<'a, LowMap<K, V>>, key: K, hasher: S) -> Self {
-        Self { guard, key, hasher }
-    }
-    /// Insert a value into this bucket
-    pub fn insert(mut self, value: V) -> RefMut<'a, K, V> {
-        unsafe {
-            let hash = super::make_insert_hash::<K, S>(&self.hasher, &self.key);
-            let &mut (ref mut k, ref mut v) = self.guard.insert_entry(
-                hash,
-                (self.key, value),
-                super::make_hasher::<K, _, V, S>(&self.hasher),
-            );
-            let kptr = compiler::extend_lifetime(k);
-            let vptr = compiler::extend_lifetime_mut(v);
-            RefMut::new(self.guard, kptr, vptr)
-        }
-    }
-}
-
-/// An entry, either occupied or vacant
-pub enum Entry<'a, K, V, S = RandomState> {
-    Occupied(OccupiedEntry<'a, K, V, S>),
-    Vacant(VacantEntry<'a, K, V, S>),
-}
-
-#[cfg(test)]
-impl<'a, K, V, S> Entry<'a, K, V, S> {
-    pub fn is_occupied(&self) -> bool {
-        matches!(self, Self::Occupied(_))
-    }
-    pub fn is_vacant(&self) -> bool {
-        matches!(self, Self::Vacant(_))
-    }
-}
-
-/// A shared ref to a key
-pub struct RefMulti<'a, K, V> {
-    _g: Arc<RwLockReadGuard<'a, LowMap<K, V>>>,
-    k: &'a K,
-    v: &'a V,
-}
-
-impl<'a, K, V> RefMulti<'a, K, V> {
-    /// Create a new shared ref
-    pub const fn new(_g: Arc<RwLockReadGuard<'a, LowMap<K, V>>>, k: &'a K, v: &'a V) -> Self {
-        Self { _g, k, v }
-    }
-    /// Get a ref to the key
-    pub const fn key(&self) -> &K {
-        self.k
-    }
-    /// Get a ref to the value
-    pub const fn value(&self) -> &V {
-        self.v
-    }
-}
-
-impl<'a, K, V> Deref for RefMulti<'a, K, V> {
-    type Target = V;
-    fn deref(&self) -> &Self::Target {
-        self.value()
-    }
-}
-
-unsafe impl<'a, K: Sync, V: Sync> Sync for RefMulti<'a, K, V> {}
-unsafe impl<'a, K: Send, V: Send> Send for RefMulti<'a, K, V> {}
-
-/// A shared r/w ref to a bucket
-pub struct RefMultiMut<'a, K, V> {
-    _g: Arc<RwLockWriteGuard<'a, LowMap<K, V>>>,
-    _k: &'a K,
-    v: &'a mut V,
-}
-
-impl<'a, K, V> RefMultiMut<'a, K, V> {
-    /// Create a new shared r/w ref
-    pub fn new(_g: Arc<RwLockWriteGuard<'a, LowMap<K, V>>>, k: &'a K, v: &'a mut V) -> Self {
-        Self { _g, _k: k, v }
-    }
-    /// Get a ref to the value
-    pub const fn value(&self) -> &V {
-        self.v
-    }
-    /// Get a mutable ref to the value
-    pub fn value_mut(&mut self) -> &mut V {
-        self.v
-    }
-}
-
-impl<'a, K, V> Deref for RefMultiMut<'a, K, V> {
-    type Target = V;
-    fn deref(&self) -> &Self::Target {
-        self.value()
-    }
-}
-
-impl<'a, K, V> DerefMut for RefMultiMut<'a, K, V> {
-    fn deref_mut(&mut self) -> &mut V {
-        self.value_mut()
-    }
-}
-
-unsafe impl<'a, K: Sync, V: Sync> Sync for RefMultiMut<'a, K, V> {}
-unsafe impl<'a, K: Send, V: Send> Send for RefMultiMut<'a, K, V> {}
diff --git a/server/src/corestore/map/iter.rs b/server/src/corestore/map/iter.rs
deleted file mode 100644
index ca013d4f..00000000
--- a/server/src/corestore/map/iter.rs
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Created on Tue Aug 10 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{bref::RefMulti, LowMap, Skymap},
-    core::mem,
-    hashbrown::raw::{RawIntoIter, RawIter},
-    parking_lot::RwLockReadGuard,
-    std::{collections::hash_map::RandomState, sync::Arc},
-};
-
-/// An owned iterator for a [`Skymap`]
-pub struct OwnedIter<K, V, S = RandomState> {
-    map: Skymap<K, V, S>,
-    cs: usize,
-    current: Option<RawIntoIter<(K, V)>>,
-}
-
-impl<K, V, S> OwnedIter<K, V, S> {
-    pub fn new(map: Skymap<K, V, S>) -> Self {
-        Self {
-            map,
-            cs: 0usize,
-            current: None,
-        }
-    }
-}
-
-impl<K, V, S> Iterator for OwnedIter<K, V, S> {
-    type Item = (K, V);
-    fn next(&mut self) -> Option<Self::Item> {
-        loop {
-            if let Some(current) = self.current.as_mut() {
-                if let Some(bucket) = current.next() {
-                    return Some(bucket);
-                }
-            }
-            if self.cs == self.map.shards().len() {
-                return None;
-            }
-            let mut wshard = unsafe { self.map.get_wshard_unchecked(self.cs) };
-            // get the next map's iterator
-            let current_map = mem::replace(&mut *wshard, LowMap::new());
-            drop(wshard);
-            let iter = current_map.into_iter();
-            self.current = Some(iter);
-            self.cs += 1;
-        }
-    }
-}
-
-unsafe impl<K: Send, V: Send, S> Send for OwnedIter<K, V, S> {}
-unsafe impl<K: Sync, V: Sync, S> Sync for OwnedIter<K, V, S> {}
-
-type BorrowedIterGroup<'a, K, V> = (RawIter<(K, V)>, Arc<RwLockReadGuard<'a, LowMap<K, V>>>);
-
-/// A borrowed iterator for a [`Skymap`]
-pub struct BorrowedIter<'a, K, V, S = ahash::RandomState> {
-    map: &'a Skymap<K, V, S>,
-    cs: usize,
-    citer: Option<BorrowedIterGroup<'a, K, V>>,
-}
-
-impl<'a, K, V, S> BorrowedIter<'a, K, V, S> {
-    pub const fn new(map: &'a Skymap<K, V, S>) -> Self {
-        Self {
-            map,
-            cs: 0usize,
-            citer: None,
-        }
-    }
-}
-
-impl<'a, K, V, S> Iterator for BorrowedIter<'a, K, V, S> {
-    type Item = RefMulti<'a, K, V>;
-    fn next(&mut self) -> Option<Self::Item> {
-        loop {
-            if let Some(current) = self.citer.as_mut() {
-                if let Some(bucket) = current.0.next() {
-                    let (kptr, vptr) = unsafe {
-                        // we know that this is valid, and this guarantee is
-                        // provided to us by the lifetime
-                        bucket.as_ref()
-                    };
-                    let guard = current.1.clone();
-                    return Some(RefMulti::new(guard, kptr, vptr));
-                }
-            }
-            if self.cs == self.map.shards().len() {
-                // end of shards
-                return None;
-            }
-            // warning: the rawiter allows us to terribly violate conditions
-            // you can mutate!
-            let rshard = unsafe { self.map.get_rshard_unchecked(self.cs) };
-            let iter = unsafe {
-                // same thing: our lt params ensure validity
-                rshard.iter()
-            };
-            self.citer = Some((iter, Arc::new(rshard)));
-            self.cs += 1;
-        }
-    }
-}
-
-unsafe impl<'a, K: Send, V: Send, S> Send for BorrowedIter<'a, K, V, S> {}
-unsafe impl<'a, K: Sync, V: Sync, S> Sync for BorrowedIter<'a, K, V, S> {}
-
-#[test]
-fn test_iter() {
-    let map = Skymap::default();
-    map.insert("hello1", "world");
-    map.insert("hello2", "world");
-    map.insert("hello3", "world");
-    let collected: Vec<(&str, &str)> = map.get_owned_iter().collect();
-    assert!(collected.len() == 3);
-    assert!(collected.contains(&("hello1", "world")));
-    assert!(collected.contains(&("hello2", "world")));
-    assert!(collected.contains(&("hello3", "world")));
-}
diff --git a/server/src/corestore/map/mod.rs b/server/src/corestore/map/mod.rs
deleted file mode 100644
index 8427b2bb..00000000
--- a/server/src/corestore/map/mod.rs
+++ /dev/null
@@ -1,439 +0,0 @@
-/*
- * Created on Mon Aug 09 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#![allow(clippy::manual_map)] // avoid LLVM bloat
-#![allow(unused)] // TODO(@ohsayan): Plonk this
-
-use {
-    self::{
-        bref::{Entry, OccupiedEntry, Ref, RefMut, VacantEntry},
-        iter::{BorrowedIter, OwnedIter},
-    },
-    crate::util::compiler,
-    core::{
-        borrow::Borrow,
-        fmt,
-        hash::{BuildHasher, Hash, Hasher},
-        iter::FromIterator,
-        mem,
-        num::NonZeroUsize,
-    },
-    parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard},
-    std::{collections::hash_map::RandomState, thread::available_parallelism},
-};
-
-pub mod bref;
-pub mod iter;
-
-type LowMap<K, V> = hashbrown::raw::RawTable<(K, V)>;
-type ShardSlice<K, V> = [RwLock<LowMap<K, V>>];
-type SRlock<'a, K, V> = RwLockReadGuard<'a, hashbrown::raw::RawTable<(K, V)>>;
-type SWlock<'a, K, V> = RwLockWriteGuard<'a, hashbrown::raw::RawTable<(K, V)>>;
-const BITS_IN_USIZE: usize = mem::size_of::<usize>() * 8;
-const DEFAULT_CAP: usize = 128;
-
-fn make_hash<K, Q, S>(hash_builder: &S, val: &Q) -> u64
-where
-    K: Borrow<Q>,
-    Q: Hash + ?Sized,
-    S: BuildHasher,
-{
-    let mut state = hash_builder.build_hasher();
-    val.hash(&mut state);
-    state.finish()
-}
-
-fn make_insert_hash<K, S>(hash_builder: &S, val: &K) -> u64
-where
-    K: Hash,
-    S: BuildHasher,
-{
-    let mut state = hash_builder.build_hasher();
-    val.hash(&mut state);
-    state.finish()
-}
-
-fn make_hasher<K, Q, V, S>(hash_builder: &S) -> impl Fn(&(Q, V)) -> u64 + '_
-where
-    K: Borrow<Q>,
-    Q: Hash,
-    S: BuildHasher,
-{
-    move |val| make_hash::<K, Q, S>(hash_builder, &val.0)
-}
-
-fn ceq<Q, K, V>(k: &Q) -> impl Fn(&(K, V)) -> bool + '_
-where
-    K: Borrow<Q>,
-    Q: ?Sized + Eq,
-{
-    move |x| k.eq(x.0.borrow())
-}
-
-fn get_shard_count() -> usize {
-    (available_parallelism().map_or(1, usize::from) * 16).next_power_of_two()
-}
-
-const fn cttz(amount: usize) -> usize {
-    amount.trailing_zeros() as usize
-}
-
-/// A striped in-memory map
-pub struct Skymap<K, V, S = RandomState> {
-    shards: Box<ShardSlice<K, V>>,
-    hasher: S,
-    shift: usize,
-}
-
-impl<K, V> Default for Skymap<K, V, RandomState> {
-    fn default() -> Self {
-        Self::with_hasher(RandomState::default())
-    }
-}
-
-impl<K: fmt::Debug, V: fmt::Debug, S: BuildHasher + Default> fmt::Debug for Skymap<K, V, S> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let mut map = f.debug_map();
-        for s in self.get_iter() {
-            map.entry(s.key(), s.value());
-        }
-        map.finish()
-    }
-}
-
-impl<K, V, S> FromIterator<(K, V)> for Skymap<K, V, S>
-where
-    K: Eq + Hash,
-    S: BuildHasher + Default + Clone,
-{
-    fn from_iter<T>(iter: T) -> Self
-    where
-        T: IntoIterator<Item = (K, V)>,
-    {
-        let map = Skymap::new();
-        iter.into_iter().for_each(|(k, v)| {
-            let _ = map.insert(k, v);
-        });
-        map
-    }
-}
-
-impl<K, V> Skymap<K, V, ahash::RandomState> {
-    /// Get a Skymap with the ahash hasher
-    pub fn new_ahash() -> Self {
-        Skymap::new()
-    }
-}
-
-// basic impls
-impl<K, V, S> Skymap<K, V, S>
-where
-    S: BuildHasher + Default,
-{
-    /// Create a new Skymap with the default state (or seed) of the hasher
-    pub fn new() -> Self {
-        Self::with_hasher(S::default())
-    }
-    /// Create a new Skymap with the provided capacity
-    pub fn with_capacity(cap: usize) -> Self {
-        Self::with_capacity_and_hasher(cap, S::default())
-    }
-    /// Create a new Skymap with the provided cap and hasher
-    pub fn with_capacity_and_hasher(mut cap: usize, hasher: S) -> Self {
-        let shard_count = get_shard_count();
-        let shift = BITS_IN_USIZE - cttz(shard_count);
-        if cap != 0 {
-            cap = (cap + (shard_count - 1)) & !(shard_count - 1);
-        }
-
-        let cap_per_shard = cap / shard_count;
-        Self {
-            shards: (0..shard_count)
-                .map(|_| RwLock::new(LowMap::with_capacity(cap_per_shard)))
-                .collect(),
-            hasher,
-            shift,
-        }
-    }
-    /// Create a new Skymap with the provided hasher
-    pub fn with_hasher(hasher: S) -> Self {
-        Self::with_capacity_and_hasher(DEFAULT_CAP, hasher)
-    }
-    /// Get the len of the Skymap
-    pub fn len(&self) -> usize {
-        self.shards.iter().map(|s| s.read().len()).sum()
-    }
-    /// Get the capacity of the Skymap
-    pub fn capacity(&self) -> usize {
-        self.shards.iter().map(|s| s.read().capacity()).sum()
-    }
-    /// Check if the Skymap is empty
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-    /// Get a borrowed iterator for the Skymap. Bound to the lifetime
-    pub fn get_iter(&self) -> BorrowedIter<K, V, S> {
-        BorrowedIter::new(self)
-    }
-    /// Get an owned iterator to the Skymap
-    pub fn get_owned_iter(self) -> OwnedIter<K, V, S> {
-        OwnedIter::new(self)
-    }
-}
-
-// const impls
-impl<K, V, S> Skymap<K, V, S> {
-    /// Get a ref to the stripes
-    const fn shards(&self) -> &ShardSlice<K, V> {
-        &self.shards
-    }
-    /// Determine the shard
-    const fn determine_shard(&self, hash: usize) -> usize {
-        // the idea of the shift was inspired by Joel's idea
-        (hash << 7) >> self.shift
-    }
-    /// Get a ref to the underlying hasher
-    const fn h(&self) -> &S {
-        &self.hasher
-    }
-}
-
-// insert/get/remove impls
-
-impl<K, V, S> Skymap<K, V, S>
-where
-    K: Eq + Hash,
-    S: BuildHasher + Clone,
-{
-    /// Insert a key/value into the Skymap
-    pub fn insert(&self, k: K, v: V) -> Option<V> {
-        let hash = make_insert_hash::<K, S>(&self.hasher, &k);
-        let idx = self.determine_shard(hash as usize);
-        unsafe {
-            // begin critical section
-            let mut lowtable = self.get_wshard_unchecked(idx);
-            if let Some((_, item)) = lowtable.get_mut(hash, ceq(&k)) {
-                Some(mem::replace(item, v))
-            } else {
-                lowtable.insert(hash, (k, v), make_hasher::<K, _, V, S>(self.h()));
-                None
-            }
-            // end critical section
-        }
-    }
-    /// Remove a key/value from the Skymap
-    pub fn remove<Q>(&self, k: &Q) -> Option<(K, V)>
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        let hash = make_hash::<K, Q, S>(self.h(), k);
-        let idx = self.determine_shard(hash as usize);
-        unsafe {
-            // begin critical section
-            let mut lowtable = self.get_wshard_unchecked(idx);
-            match lowtable.remove_entry(hash, ceq(k)) {
-                Some(kv) => Some(kv),
-                None => None,
-            }
-            // end critical section
-        }
-    }
-    /// Remove a key/value from the Skymap if it satisfies a certain condition
-    pub fn remove_if<Q>(&self, k: &Q, f: impl FnOnce(&K, &V) -> bool) -> Option<(K, V)>
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        let hash = make_hash::<K, Q, S>(self.h(), k);
-        let idx = self.determine_shard(hash as usize);
-        unsafe {
-            // begin critical section
-            let mut lowtable = self.get_wshard_unchecked(idx);
-            match lowtable.find(hash, ceq(k)) {
-                Some(bucket) => {
-                    let (kptr, vptr) = bucket.as_ref();
-                    if f(kptr, vptr) {
-                        Some(lowtable.remove(bucket).0)
-                    } else {
-                        None
-                    }
-                }
-                None => None,
-            }
-            // end critical section
-        }
-    }
-}
-
-// lt impls
-impl<'a, K: 'a + Hash + Eq, V: 'a, S: BuildHasher + Clone> Skymap<K, V, S> {
-    /// Get a ref to an entry in the Skymap
-    pub fn get<Q>(&'a self, k: &Q) -> Option<Ref<'a, K, V>>
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        let hash = make_hash::<K, Q, S>(self.h(), k);
-        let idx = self.determine_shard(hash as usize);
-        unsafe {
-            // begin critical section
-            let lowtable = self.get_rshard_unchecked(idx);
-            match lowtable.get(hash, ceq(k)) {
-                Some((ref kptr, ref vptr)) => {
-                    let kptr = compiler::extend_lifetime(kptr);
-                    let vptr = compiler::extend_lifetime(vptr);
-                    Some(Ref::new(lowtable, kptr, vptr))
-                }
-                None => None,
-            }
-            // end critical section
-        }
-    }
-
-    /// Get a mutable ref to an entry in the Skymap
-    pub fn get_mut<Q>(&'a self, k: &Q) -> Option<RefMut<'a, K, V>>
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        let hash = make_hash::<K, Q, S>(self.h(), k);
-        let idx = self.determine_shard(hash as usize);
-        unsafe {
-            // begin critical section
-            let mut lowtable = self.get_wshard_unchecked(idx);
-            match lowtable.get_mut(hash, ceq(k)) {
-                Some(&mut (ref kptr, ref mut vptr)) => {
-                    let kptr = compiler::extend_lifetime(kptr);
-                    let vptr = compiler::extend_lifetime_mut(vptr);
-                    Some(RefMut::new(lowtable, kptr, vptr))
-                }
-                None => None,
-            }
-            // end critical section
-        }
-    }
-    /// Get an entry for in-place mutation
-    pub fn entry(&'a self, key: K) -> Entry<'a, K, V, S> {
-        let hash = make_insert_hash::<K, S>(self.h(), &key);
-        let idx = self.determine_shard(hash as usize);
-        unsafe {
-            // begin critical section
-            let lowtable = self.get_wshard_unchecked(idx);
-            if let Some(elem) = lowtable.find(hash, ceq(&key)) {
-                let (kptr, vptr) = elem.as_mut();
-                let kptr = compiler::extend_lifetime(kptr);
-                let vptr = compiler::extend_lifetime_mut(vptr);
-                Entry::Occupied(OccupiedEntry::new(
-                    lowtable,
-                    key,
-                    (kptr, vptr),
-                    self.hasher.clone(),
-                ))
-            } else {
-                Entry::Vacant(VacantEntry::new(lowtable, key, self.hasher.clone()))
-            }
-            // end critical section
-        }
-    }
-    /// Check if the Skymap contains the provided key
-    pub fn contains_key<Q>(&self, key: &Q) -> bool
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        self.get(key).is_some()
-    }
-    /// Clear out all the entries in the Skymap
-    pub fn clear(&self) {
-        self.shards().iter().for_each(|shard| shard.write().clear())
-    }
-}
-
-// cloned impls
-impl<'a, K, V: Clone, S: BuildHasher> Skymap<K, V, S> {
-    pub fn get_cloned<Q>(&'a self, k: &Q) -> Option<V>
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        let hash = make_hash::<K, Q, S>(self.h(), k);
-        let idx = self.determine_shard(hash as usize);
-        unsafe {
-            // begin critical section
-            let lowtable = self.get_rshard_unchecked(idx);
-            match lowtable.get(hash, ceq(k)) {
-                Some((_kptr, ref vptr)) => Some(vptr.clone()),
-                None => None,
-            }
-            // end critical section
-        }
-    }
-}
-
-// inner impls
-impl<'a, K: 'a, V: 'a, S> Skymap<K, V, S> {
-    /// Get a rlock to a certain stripe
-    unsafe fn get_rshard_unchecked(&'a self, shard: usize) -> SRlock<'a, K, V> {
-        ucidx!(self.shards, shard).read()
-    }
-    /// Get a wlock to a certain stripe
-    unsafe fn get_wshard_unchecked(&'a self, shard: usize) -> SWlock<'a, K, V> {
-        ucidx!(self.shards, shard).write()
-    }
-}
-
-#[test]
-fn test_insert_remove() {
-    let map = Skymap::default();
-    map.insert("hello", "world");
-    assert_eq!(map.remove("hello").unwrap().1, "world");
-}
-
-#[test]
-fn test_remove_if() {
-    let map = Skymap::default();
-    map.insert("hello", "world");
-    assert!(map
-        .remove_if("hello", |_k, v| { (*v).eq("notworld") })
-        .is_none());
-}
-
-#[test]
-fn test_insert_get() {
-    let map = Skymap::default();
-    map.insert("sayan", "likes computational dark arts");
-    let _ref = map.get("sayan").unwrap();
-    assert_eq!(*_ref, "likes computational dark arts")
-}
-
-#[test]
-fn test_entry() {
-    let map = Skymap::default();
-    map.insert("hello", "world");
-    assert!(map.entry("hello").is_occupied());
-    assert!(map.entry("world").is_vacant());
-}
diff --git a/server/src/corestore/memstore.rs b/server/src/corestore/memstore.rs
deleted file mode 100644
index 21050a84..00000000
--- a/server/src/corestore/memstore.rs
+++ /dev/null
@@ -1,492 +0,0 @@
-/*
- * Created on Fri Jul 02 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # In-memory store
-//!
-//! This is what things look like:
-//! ```text
-//! -----------------------------------------------------
-//! |                                                   |
-//! |  |-------------------|     |-------------------|  |
-//! |  |-------------------|     |-------------------|  |
-//! |  | | TABLE | TABLE | |     | | TABLE | TABLE | |  |
-//! |  | |-------|-------| |     | |-------|-------| |  |
-//! |  |      Keyspace     |     |      Keyspace     |  |
-//! |  |-------------------|     |-------------------|  |
-//!                                                     |
-//! |  |-------------------|     |-------------------|  |
-//! |  | |-------|-------| |     | |-------|-------| |  |
-//! |  | | TABLE | TABLE | |     | | TABLE | TABLE | |  |
-//! |  | |-------|-------| |     | |-------|-------| |  |
-//! |  |      Keyspace     |     |      Keyspace     |  |
-//! |  |-------------------|     |-------------------|  |
-//! |                                                   |
-//! |                                                   |
-//! |                                                   |
-//! -----------------------------------------------------
-//! |                         NODE                      |
-//! |---------------------------------------------------|
-//! ```
-//!
-//! So, all your data is at the mercy of [`Memstore`]'s constructor
-//! and destructor.
-
-use {
-    super::KeyspaceResult,
-    crate::{
-        auth::Authmap,
-        corestore::{
-            array::Array,
-            htable::Coremap,
-            table::{SystemDataModel, SystemTable, Table},
-        },
-        registry,
-        util::Wrapper,
-    },
-    core::{borrow::Borrow, hash::Hash},
-    std::sync::Arc,
-};
-
-uninit_array! {
-    const DEFAULT_ARRAY: [u8; 64] = [b'd', b'e', b'f', b'a', b'u', b'l', b't'];
-    const SYSTEM_ARRAY: [u8; 64] = [b's', b'y', b's', b't', b'e', b'm'];
-    const SYSTEM_AUTH_ARRAY: [u8; 64] = [b'a', b'u', b't', b'h'];
-}
-
-/// typedef for the keyspace/table IDs. We don't need too much fancy here,
-/// no atomic pointers and all. Just a nice array. With amazing gurantees
-pub type ObjectID = Array<u8, 64>;
-
-/// The `DEFAULT` array (with the rest uninit)
-pub const DEFAULT: ObjectID = unsafe {
-    // SAFETY: known init len
-    Array::from_const(DEFAULT_ARRAY, 7)
-};
-pub const SYSTEM: ObjectID = unsafe {
-    // SAFETY: known init len
-    Array::from_const(SYSTEM_ARRAY, 6)
-};
-pub const AUTH: ObjectID = unsafe {
-    // SAFETY: known init len
-    Array::from_const(SYSTEM_AUTH_ARRAY, 4)
-};
-
-#[test]
-fn test_def_macro_sanity() {
-    // just make sure our macro is working as expected
-    let mut def = DEFAULT;
-    def.push(b'?');
-    unsafe {
-        assert_eq!(def.as_str(), "default?");
-        let mut sys = SYSTEM;
-        sys.push(b'?');
-        assert_eq!(sys.as_str(), "system?");
-    }
-}
-
-mod cluster {
-    /// This is for the future where every node will be allocated a shard
-    #[derive(Debug)]
-    pub enum ClusterShardRange {
-        SingleNode,
-    }
-
-    impl Default for ClusterShardRange {
-        fn default() -> Self {
-            Self::SingleNode
-        }
-    }
-
-    /// This is for the future for determining the replication strategy
-    #[derive(Debug)]
-    pub enum ReplicationStrategy {
-        /// Single node, no replica sets
-        Default,
-    }
-
-    impl Default for ReplicationStrategy {
-        fn default() -> Self {
-            Self::Default
-        }
-    }
-}
-
-#[derive(Debug, PartialEq)]
-/// Errors arising from trying to modify/access containers
-#[allow(dead_code)]
-pub enum DdlError {
-    /// The object is still in use
-    StillInUse,
-    /// The object couldn't be found
-    ObjectNotFound,
-    /// The object is not user-accessible
-    ProtectedObject,
-    /// The default object wasn't found
-    DefaultNotFound,
-    /// Incorrect data model semantics were used on a data model
-    WrongModel,
-    /// The object already exists
-    AlreadyExists,
-    /// The target object is not ready
-    NotReady,
-    /// The target object is not empty
-    NotEmpty,
-    /// The DDL transaction failed
-    DdlTransactionFailure,
-}
-
-#[derive(Debug)]
-/// The core in-memory table
-///
-/// This in-memory table that houses all keyspaces along with other node properties.
-/// This is the structure that you should clone in an atomic RC wrapper. This object
-/// handles no sort of persistence
-pub struct Memstore {
-    /// the keyspaces
-    pub keyspaces: Coremap<ObjectID, Arc<Keyspace>>,
-    /// the system keyspace with the system tables
-    pub system: SystemKeyspace,
-}
-
-impl Memstore {
-    /// Create a new empty in-memory table with literally nothing in it
-    #[cfg(test)]
-    pub fn new_empty() -> Self {
-        Self {
-            keyspaces: Coremap::new(),
-            system: SystemKeyspace::new(Coremap::new()),
-        }
-    }
-    pub fn init_with_all(
-        keyspaces: Coremap<ObjectID, Arc<Keyspace>>,
-        system: SystemKeyspace,
-    ) -> Self {
-        Self { keyspaces, system }
-    }
-    /// Create a new in-memory table with the default keyspace and the default
-    /// tables. So, whenever you're calling this, this is what you get:
-    /// ```json
-    /// "YOURNODE": {
-    ///     "KEYSPACES": [
-    ///         "default" : {
-    ///             "TABLES": ["default"]
-    ///         },
-    ///         "system": {
-    ///             "TABLES": []
-    ///         }
-    ///     ]
-    /// }
-    /// ```
-    ///
-    /// When you connect a client without any information about the keyspace you're planning to
-    /// use, you'll be connected to `ks:default/table:default`. The `ks:default/table:_system` is not
-    /// for you. It's for the system
-    pub fn new_default() -> Self {
-        Self {
-            keyspaces: {
-                let n = Coremap::new();
-                n.true_if_insert(DEFAULT, Arc::new(Keyspace::empty_default()));
-                // HACK(@ohsayan): This just ensures that the record is in the preload
-                n.true_if_insert(SYSTEM, Arc::new(Keyspace::empty()));
-                n
-            },
-            system: SystemKeyspace::new(Coremap::new()),
-        }
-    }
-    pub fn setup_auth(&self) -> Authmap {
-        match self.system.tables.fresh_entry(AUTH) {
-            Some(fresh) => {
-                // created afresh, fine
-                let r = Authmap::default();
-                fresh.insert(Wrapper::new(SystemTable::new_auth(r.clone())));
-                r
-            }
-            None => match self.system.tables.get(&AUTH).unwrap().data {
-                SystemDataModel::Auth(ref am) => am.clone(),
-                #[allow(unreachable_patterns)]
-                _ => unsafe { impossible!() },
-            },
-        }
-    }
-    /// Get an atomic reference to a keyspace
-    pub fn get_keyspace_atomic_ref<Q>(&self, keyspace_identifier: &Q) -> Option<Arc<Keyspace>>
-    where
-        ObjectID: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        self.keyspaces.get(keyspace_identifier).map(|ns| ns.clone())
-    }
-    /// Returns true if a new keyspace was created
-    pub fn create_keyspace(&self, keyspace_identifier: ObjectID) -> bool {
-        self.keyspaces
-            .true_if_insert(keyspace_identifier, Arc::new(Keyspace::empty()))
-    }
-    /// Drop a keyspace only if it is empty and has no clients connected to it
-    ///
-    /// The invariants maintained here are:
-    /// 1. The keyspace is not referenced to
-    /// 2. There are no tables in the keyspace
-    ///
-    /// **Trip switch handled:** Yes
-    pub fn drop_keyspace(&self, ksid: ObjectID) -> KeyspaceResult<()> {
-        if ksid.eq(&SYSTEM) || ksid.eq(&DEFAULT) {
-            Err(DdlError::ProtectedObject)
-        } else if !self.keyspaces.contains_key(&ksid) {
-            Err(DdlError::ObjectNotFound)
-        } else {
-            let removed_keyspace = self.keyspaces.mut_entry(ksid);
-            match removed_keyspace {
-                Some(ks) => {
-                    let no_one_is_using_keyspace = Arc::strong_count(ks.value()) == 1;
-                    let no_tables_are_in_keyspace = ks.value().table_count() == 0;
-                    if no_one_is_using_keyspace && no_tables_are_in_keyspace {
-                        // we are free to drop this
-                        ks.remove();
-                        // trip the preload switch
-                        registry::get_preload_tripswitch().trip();
-                        // trip the cleanup switch
-                        registry::get_cleanup_tripswitch().trip();
-                        Ok(())
-                    } else if !no_tables_are_in_keyspace {
-                        // not empty; may be referenced to or not referenced to
-                        Err(DdlError::NotEmpty)
-                    } else {
-                        // still referenced to; but is empty
-                        Err(DdlError::StillInUse)
-                    }
-                }
-                None => Err(DdlError::ObjectNotFound),
-            }
-        }
-    }
-    /// Force remove a keyspace along with all its tables. This force however only
-    /// removes tables if they aren't in use and iff the keyspace is not currently
-    /// in use to avoid the problem of having "ghost tables"
-    ///
-    /// The invariants maintained here are:
-    /// 1. The keyspace is not referenced to
-    /// 2. The tables in the keyspace are not referenced to
-    ///
-    /// **Trip switch handled:** Yes
-    pub fn force_drop_keyspace(&self, ksid: ObjectID) -> KeyspaceResult<()> {
-        if ksid.eq(&SYSTEM) || ksid.eq(&DEFAULT) {
-            Err(DdlError::ProtectedObject)
-        } else if !self.keyspaces.contains_key(&ksid) {
-            Err(DdlError::ObjectNotFound)
-        } else {
-            let removed_keyspace = self.keyspaces.mut_entry(ksid);
-            match removed_keyspace {
-                Some(keyspace) => {
-                    // force remove drops tables, but only if they aren't in use
-                    // we can potentially have "ghost" tables if we don't do this
-                    // check. Similarly, if we don't check the strong count of the
-                    // keyspace, we might end up with "ghost" keyspaces. A good
-                    // question would be: why not just check the number of tables --
-                    // well, the force drop is specifically built to address the problem
-                    // of having not empty keyspaces. The invariant that `drop keyspace force`
-                    // maintains is that the keyspace or any of its objects are never
-                    // referenced to -- only then it is safe to delete the keyspace
-                    let no_tables_in_use = Arc::strong_count(keyspace.value()) == 1
-                        && keyspace
-                            .value()
-                            .tables
-                            .iter()
-                            .all(|table| Arc::strong_count(table.value()) == 1);
-                    if no_tables_in_use {
-                        keyspace.remove();
-                        // trip the preload switch
-                        registry::get_preload_tripswitch().trip();
-                        // trip the cleanup switch
-                        registry::get_cleanup_tripswitch().trip();
-                        Ok(())
-                    } else {
-                        Err(DdlError::StillInUse)
-                    }
-                }
-                None => Err(DdlError::ObjectNotFound),
-            }
-        }
-    }
-    pub fn list_keyspaces(&self) -> Vec<ObjectID> {
-        self.keyspaces.iter().map(|kv| kv.key().clone()).collect()
-    }
-}
-
-/// System keyspace
-#[derive(Debug)]
-pub struct SystemKeyspace {
-    pub tables: Coremap<ObjectID, Wrapper<SystemTable>>,
-}
-
-impl SystemKeyspace {
-    pub const fn new(tables: Coremap<ObjectID, Wrapper<SystemTable>>) -> Self {
-        Self { tables }
-    }
-}
-
-#[derive(Debug)]
-/// A keyspace houses all the other tables
-pub struct Keyspace {
-    /// the tables
-    pub tables: Coremap<ObjectID, Arc<Table>>,
-    /// the replication strategy for this keyspace
-    #[allow(dead_code)] // TODO: Remove this once we're ready with replication
-    replication_strategy: cluster::ReplicationStrategy,
-}
-
-#[cfg(test)]
-macro_rules! unsafe_objectid_from_slice {
-    ($slice:expr) => {{
-        unsafe { ObjectID::from_slice($slice) }
-    }};
-}
-
-impl Keyspace {
-    /// Create a new empty keyspace with the default tables: a `default` table
-    pub fn empty_default() -> Self {
-        Self {
-            tables: {
-                let ht = Coremap::new();
-                // add the default table
-                ht.true_if_insert(DEFAULT, Arc::new(Table::new_default_kve()));
-                ht
-            },
-            replication_strategy: cluster::ReplicationStrategy::default(),
-        }
-    }
-    pub fn init_with_all_def_strategy(tables: Coremap<ObjectID, Arc<Table>>) -> Self {
-        Self {
-            tables,
-            replication_strategy: cluster::ReplicationStrategy::default(),
-        }
-    }
-    /// Create a new empty keyspace with zero tables
-    pub fn empty() -> Self {
-        Self {
-            tables: Coremap::new(),
-            replication_strategy: cluster::ReplicationStrategy::default(),
-        }
-    }
-    pub fn table_count(&self) -> usize {
-        self.tables.len()
-    }
-    /// Get an atomic reference to a table in this keyspace if it exists
-    pub fn get_table_atomic_ref<Q>(&self, table_identifier: &Q) -> Option<Arc<Table>>
-    where
-        ObjectID: Borrow<Q>,
-        Q: Hash + Eq + PartialEq<ObjectID> + ?Sized,
-    {
-        self.tables.get(table_identifier).map(|v| v.clone())
-    }
-    /// Create a new table
-    pub fn create_table(&self, tableid: ObjectID, table: Table) -> bool {
-        self.tables.true_if_insert(tableid, Arc::new(table))
-    }
-    /// Drop a table if it exists, if it is not forbidden and if no one references
-    /// back to it. We don't want any looming table references i.e table gets deleted
-    /// for the current connection and newer connections, but older instances still
-    /// refer to the table.
-    // FIXME(@ohsayan): Should we actually care?
-    ///
-    /// **Trip switch handled:** Yes
-    fn drop_table_inner<Q>(&self, table_identifier: &Q, should_force: bool) -> KeyspaceResult<()>
-    where
-        ObjectID: Borrow<Q>,
-        Q: Hash + Eq + PartialEq<ObjectID> + ?Sized,
-    {
-        if table_identifier.eq(&DEFAULT) {
-            Err(DdlError::ProtectedObject)
-        } else if !self.tables.contains_key(table_identifier) {
-            Err(DdlError::ObjectNotFound)
-        } else {
-            // has table
-            let did_remove =
-                self.tables
-                    .true_remove_if(table_identifier, |_table_id, table_atomic_ref| {
-                        // 1 because this should just be us, the one instance
-                        Arc::strong_count(table_atomic_ref) == 1
-                            && (table_atomic_ref.is_empty() || should_force)
-                    });
-            if did_remove {
-                // we need to re-init tree; so trip
-                registry::get_preload_tripswitch().trip();
-                // we need to cleanup tree; so trip
-                registry::get_cleanup_tripswitch().trip();
-                Ok(())
-            } else {
-                Err(DdlError::StillInUse)
-            }
-        }
-    }
-    pub fn drop_table<Q>(&self, tblid: &Q, force: bool) -> KeyspaceResult<()>
-    where
-        ObjectID: Borrow<Q>,
-        Q: Hash + Eq + PartialEq<ObjectID> + ?Sized,
-    {
-        self.drop_table_inner(tblid, force)
-    }
-}
-
-#[test]
-fn test_keyspace_drop_no_atomic_ref() {
-    let our_keyspace = Keyspace::empty_default();
-    assert!(our_keyspace.create_table(
-        unsafe_objectid_from_slice!("apps"),
-        Table::new_default_kve()
-    ));
-    assert!(our_keyspace
-        .drop_table(&unsafe_objectid_from_slice!("apps"), false)
-        .is_ok());
-}
-
-#[test]
-fn test_keyspace_drop_fail_with_atomic_ref() {
-    let our_keyspace = Keyspace::empty_default();
-    assert!(our_keyspace.create_table(
-        unsafe_objectid_from_slice!("apps"),
-        Table::new_default_kve()
-    ));
-    let _atomic_tbl_ref = our_keyspace
-        .get_table_atomic_ref(&unsafe_objectid_from_slice!("apps"))
-        .unwrap();
-    assert_eq!(
-        our_keyspace
-            .drop_table(&unsafe_objectid_from_slice!("apps"), false)
-            .unwrap_err(),
-        DdlError::StillInUse
-    );
-}
-
-#[test]
-fn test_keyspace_try_delete_protected_table() {
-    let our_keyspace = Keyspace::empty_default();
-    assert_eq!(
-        our_keyspace
-            .drop_table(&unsafe_objectid_from_slice!("default"), false)
-            .unwrap_err(),
-        DdlError::ProtectedObject
-    );
-}
diff --git a/server/src/corestore/mod.rs b/server/src/corestore/mod.rs
deleted file mode 100644
index 18e7dd59..00000000
--- a/server/src/corestore/mod.rs
+++ /dev/null
@@ -1,381 +0,0 @@
-/*
- * Created on Tue Jul 20 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{
-        actions::{translate_ddl_error, ActionResult},
-        blueql::Entity,
-        corestore::{
-            memstore::{DdlError, Keyspace, Memstore, ObjectID, DEFAULT},
-            table::{DescribeTable, Table},
-        },
-        protocol::interface::ProtocolSpec,
-        registry,
-        storage::{
-            self,
-            v1::{error::StorageEngineResult, sengine::SnapshotEngine},
-        },
-        util::{self, Unwrappable},
-    },
-    core::{borrow::Borrow, hash::Hash},
-    std::sync::Arc,
-};
-
-pub mod array;
-pub mod backoff;
-pub mod booltable;
-pub mod buffers;
-pub mod heap_array;
-pub mod htable;
-pub mod iarray;
-pub mod lazy;
-pub mod lock;
-pub mod map;
-pub mod memstore;
-pub mod rc;
-pub mod table;
-#[cfg(test)]
-mod tests;
-
-pub use self::rc::SharedSlice;
-
-pub(super) type KeyspaceResult<T> = Result<T, DdlError>;
-
-#[derive(Debug, Clone)]
-struct ConnectionEntityState {
-    /// the current table for a connection
-    table: Option<(ObjectID, Arc<Table>)>,
-    /// the current keyspace for a connection
-    ks: Option<(ObjectID, Arc<Keyspace>)>,
-}
-
-impl ConnectionEntityState {
-    fn default(ks: Arc<Keyspace>, tbl: Arc<Table>) -> Self {
-        Self {
-            table: Some((DEFAULT, tbl)),
-            ks: Some((DEFAULT, ks)),
-        }
-    }
-    fn set_ks(&mut self, ks: Arc<Keyspace>, ksid: ObjectID) {
-        self.ks = Some((ksid, ks));
-        self.table = None;
-    }
-    fn set_table(&mut self, ks: Arc<Keyspace>, ksid: ObjectID, tbl: Arc<Table>, tblid: ObjectID) {
-        self.ks = Some((ksid, ks));
-        self.table = Some((tblid, tbl));
-    }
-    fn get_id_pack(&self) -> (Option<&ObjectID>, Option<&ObjectID>) {
-        (
-            self.ks.as_ref().map(|(id, _)| id),
-            self.table.as_ref().map(|(id, _)| id),
-        )
-    }
-}
-
-/// The top level abstraction for the in-memory store. This is free to be shared across
-/// threads, cloned and well, whatever. Most importantly, clones have an independent container
-/// state that is the state of one connection and its container state preferences are never
-/// synced across instances. This is important (see the impl for more info)
-#[derive(Debug, Clone)]
-pub struct Corestore {
-    estate: ConnectionEntityState,
-    /// an atomic reference to the actual backing storage
-    store: Arc<Memstore>,
-    /// the snapshot engine
-    sengine: Arc<SnapshotEngine>,
-}
-
-impl Corestore {
-    /// This is the only function you'll ever need to either create a new database instance
-    /// or restore from an earlier instance
-    pub fn init_with_snapcfg(sengine: Arc<SnapshotEngine>) -> StorageEngineResult<Self> {
-        let store = storage::unflush::read_full()?;
-        Ok(Self::default_with_store(store, sengine))
-    }
-    pub fn clone_store(&self) -> Arc<Memstore> {
-        self.store.clone()
-    }
-    pub fn default_with_store(store: Memstore, sengine: Arc<SnapshotEngine>) -> Self {
-        let cks = unsafe { store.get_keyspace_atomic_ref(&DEFAULT).unsafe_unwrap() };
-        let ctable = unsafe { cks.get_table_atomic_ref(&DEFAULT).unsafe_unwrap() };
-        Self {
-            estate: ConnectionEntityState::default(cks, ctable),
-            store: Arc::new(store),
-            sengine,
-        }
-    }
-    pub fn get_engine(&self) -> &SnapshotEngine {
-        &self.sengine
-    }
-    pub fn get_store(&self) -> &Memstore {
-        &self.store
-    }
-    /// Swap out the current table with a different one
-    ///
-    /// If the table is non-existent or the default keyspace was unset, then
-    /// false is returned. Else true is returned
-    pub fn swap_entity(&mut self, entity: &Entity) -> KeyspaceResult<()> {
-        match entity {
-            // Switch to the provided keyspace
-            Entity::Current(ks) => {
-                match self.store.get_keyspace_atomic_ref(unsafe { ks.as_slice() }) {
-                    Some(ksref) => self
-                        .estate
-                        .set_ks(ksref, unsafe { ObjectID::from_slice(ks.as_slice()) }),
-                    None => return Err(DdlError::ObjectNotFound),
-                }
-            }
-            // Switch to the provided table in the given keyspace
-            Entity::Full(ks, tbl) => {
-                match self.store.get_keyspace_atomic_ref(unsafe { ks.as_slice() }) {
-                    Some(kspace) => match kspace.get_table_atomic_ref(unsafe { tbl.as_slice() }) {
-                        Some(tblref) => unsafe {
-                            self.estate.set_table(
-                                kspace,
-                                ObjectID::from_slice(ks.as_slice()),
-                                tblref,
-                                ObjectID::from_slice(tbl.as_slice()),
-                            )
-                        },
-                        None => return Err(DdlError::ObjectNotFound),
-                    },
-                    None => return Err(DdlError::ObjectNotFound),
-                }
-            }
-        }
-        Ok(())
-    }
-    /// Returns the current keyspace, if set
-    pub fn get_cks(&self) -> KeyspaceResult<&Keyspace> {
-        match self.estate.ks {
-            Some((_, ref cks)) => Ok(cks),
-            _ => Err(DdlError::DefaultNotFound),
-        }
-    }
-    /// Returns the current table, if set
-    pub fn get_ctable_result(&self) -> KeyspaceResult<&Table> {
-        match self.estate.table {
-            Some((_, ref tbl)) => Ok(tbl),
-            _ => Err(DdlError::DefaultNotFound),
-        }
-    }
-    pub fn get_keyspace<Q>(&self, ksid: &Q) -> Option<Arc<Keyspace>>
-    where
-        ObjectID: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        self.store.get_keyspace_atomic_ref(ksid)
-    }
-    /// Get an atomic reference to a table
-    pub fn get_table(&self, entity: &Entity) -> KeyspaceResult<Arc<Table>> {
-        match entity {
-            Entity::Full(ksid, table) => {
-                match self
-                    .store
-                    .get_keyspace_atomic_ref(unsafe { ksid.as_slice() })
-                {
-                    Some(ks) => match ks.get_table_atomic_ref(unsafe { table.as_slice() }) {
-                        Some(tbl) => Ok(tbl),
-                        None => Err(DdlError::ObjectNotFound),
-                    },
-                    None => Err(DdlError::ObjectNotFound),
-                }
-            }
-            Entity::Current(tbl) => match &self.estate.ks {
-                Some((_, ks)) => match ks.get_table_atomic_ref(unsafe { tbl.as_slice() }) {
-                    Some(tbl) => Ok(tbl),
-                    None => Err(DdlError::ObjectNotFound),
-                },
-                None => Err(DdlError::DefaultNotFound),
-            },
-        }
-    }
-    pub fn get_ctable(&self) -> Option<Arc<Table>> {
-        self.estate.table.as_ref().map(|(_, tbl)| tbl.clone())
-    }
-    pub fn get_ctable_ref(&self) -> Option<&Table> {
-        self.estate.table.as_ref().map(|(_, tbl)| tbl.as_ref())
-    }
-    /// Returns a table with the provided specification
-    pub fn get_table_with<P: ProtocolSpec, T: DescribeTable>(&self) -> ActionResult<&T::Table> {
-        T::get::<P>(self)
-    }
-    /// Create a table: in-memory; **no transactional guarantees**. Two tables can be created
-    /// simultaneously, but are never flushed unless we are very lucky. If the global flush
-    /// system is close to a flush cycle -- then we are in luck: we pause the flush cycle
-    /// through a global flush lock and then allow it to resume once we're done adding the table.
-    /// This enables the flush routine to permanently write the table to disk. But it's all about
-    /// luck -- the next mutual access may be yielded to the next `create table` command
-    ///
-    /// **Trip switch handled:** Yes
-    pub fn create_table(
-        &self,
-        entity: &Entity,
-        modelcode: u8,
-        volatile: bool,
-    ) -> KeyspaceResult<()> {
-        // first lock the global flush state
-        let flush_lock = registry::lock_flush_state();
-        let ret = match entity {
-            // Important: create table <tblname> is only ks
-            Entity::Current(tblid) => {
-                match &self.estate.ks {
-                    Some((_, ks)) => {
-                        let tbl = Table::from_model_code(modelcode, volatile);
-                        if let Some(tbl) = tbl {
-                            if ks.create_table(
-                                unsafe { ObjectID::from_slice(tblid.as_slice()) },
-                                tbl,
-                            ) {
-                                // we need to re-init tree; so trip
-                                registry::get_preload_tripswitch().trip();
-                                Ok(())
-                            } else {
-                                Err(DdlError::AlreadyExists)
-                            }
-                        } else {
-                            Err(DdlError::WrongModel)
-                        }
-                    }
-                    None => Err(DdlError::DefaultNotFound),
-                }
-            }
-            Entity::Full(ksid, tblid) => {
-                match self
-                    .store
-                    .get_keyspace_atomic_ref(unsafe { ksid.as_slice() })
-                {
-                    Some(kspace) => {
-                        let tbl = Table::from_model_code(modelcode, volatile);
-                        if let Some(tbl) = tbl {
-                            if kspace.create_table(
-                                unsafe { ObjectID::from_slice(tblid.as_slice()) },
-                                tbl,
-                            ) {
-                                // trip the preload switch
-                                registry::get_preload_tripswitch().trip();
-                                Ok(())
-                            } else {
-                                Err(DdlError::AlreadyExists)
-                            }
-                        } else {
-                            Err(DdlError::WrongModel)
-                        }
-                    }
-                    None => Err(DdlError::ObjectNotFound),
-                }
-            }
-        };
-        // free the global flush lock
-        drop(flush_lock);
-        ret
-    }
-
-    /// Drop a table
-    pub fn drop_table(&self, entity: &Entity, force: bool) -> KeyspaceResult<()> {
-        match entity {
-            Entity::Current(tblid) => match &self.estate.ks {
-                Some((_, ks)) => ks.drop_table(unsafe { tblid.as_slice() }, force),
-                None => Err(DdlError::DefaultNotFound),
-            },
-            Entity::Full(ksid, tblid) => {
-                match self
-                    .store
-                    .get_keyspace_atomic_ref(unsafe { ksid.as_slice() })
-                {
-                    Some(ks) => ks.drop_table(unsafe { tblid.as_slice() }, force),
-                    None => Err(DdlError::ObjectNotFound),
-                }
-            }
-        }
-    }
-
-    /// Create a keyspace **without any transactional guarantees**
-    ///
-    /// **Trip switch handled:** Yes
-    pub fn create_keyspace(&self, ksid: ObjectID) -> KeyspaceResult<()> {
-        // lock the global flush lock (see comment in create_table to know why)
-        let flush_lock = registry::lock_flush_state();
-        let ret = if self.store.create_keyspace(ksid) {
-            // woo, created
-            // trip the preload switch
-            registry::get_preload_tripswitch().trip();
-            Ok(())
-        } else {
-            // ugh, already exists
-            Err(DdlError::AlreadyExists)
-        };
-        drop(flush_lock);
-        ret
-    }
-
-    /// Drop a keyspace
-    pub fn drop_keyspace(&self, ksid: ObjectID) -> KeyspaceResult<()> {
-        // trip switch is handled by memstore here
-        self.store.drop_keyspace(ksid)
-    }
-
-    /// Force drop a keyspace
-    pub fn force_drop_keyspace(&self, ksid: ObjectID) -> KeyspaceResult<()> {
-        // trip switch is handled by memstore here
-        self.store.force_drop_keyspace(ksid)
-    }
-    pub fn strong_count(&self) -> usize {
-        Arc::strong_count(&self.store)
-    }
-    pub fn get_ids(&self) -> (Option<&ObjectID>, Option<&ObjectID>) {
-        self.estate.get_id_pack()
-    }
-    pub fn list_tables<P: ProtocolSpec>(&self, ksid: Option<&[u8]>) -> ActionResult<Vec<ObjectID>> {
-        Ok(match ksid {
-            Some(keyspace_name) => {
-                // inspect the provided keyspace
-                let ksid = if keyspace_name.len() > 64 {
-                    return util::err(P::RSTRING_BAD_CONTAINER_NAME);
-                } else {
-                    keyspace_name
-                };
-                let ks = match self.get_keyspace(ksid) {
-                    Some(kspace) => kspace,
-                    None => return util::err(P::RSTRING_CONTAINER_NOT_FOUND),
-                };
-                ks.tables.iter().map(|kv| kv.key().clone()).collect()
-            }
-            None => {
-                // inspect the current keyspace
-                let cks = translate_ddl_error::<P, &Keyspace>(self.get_cks())?;
-                cks.tables.iter().map(|kv| kv.key().clone()).collect()
-            }
-        })
-    }
-    pub fn describe_table<P: ProtocolSpec>(&self, table: &Option<Entity>) -> ActionResult<String> {
-        let r = match table {
-            Some(tbl) => translate_ddl_error::<P, Arc<Table>>(self.get_table(tbl))?.describe_self(),
-            None => translate_ddl_error::<P, &Table>(self.get_ctable_result())?.describe_self(),
-        };
-        Ok(r.to_owned())
-    }
-}
diff --git a/server/src/corestore/rc.rs b/server/src/corestore/rc.rs
deleted file mode 100644
index 08439fbb..00000000
--- a/server/src/corestore/rc.rs
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Created on Mon Aug 15 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use std::{
-    alloc::{alloc, dealloc, Layout},
-    borrow::Borrow,
-    fmt::Debug,
-    hash::Hash,
-    ops::Deref,
-    ptr::{self, NonNull},
-    slice,
-    sync::atomic::{self, AtomicUsize, Ordering},
-};
-
-/// A [`SharedSlice`] is a dynamically sized, heap allocated slice that can be safely shared across threads. This
-/// type can be cheaply cloned and the only major cost is initialization that does a memcpy from the source into
-/// a new heap allocation. Once init is complete, cloning only increments an atomic counter and when no more owners
-/// of this data exists, i.e the object is orphaned, it will call its destructor and clean up the heap allocation.
-/// Do note that two heap allocations are made:
-/// - One for the actual data
-/// - One for the shared state
-pub struct SharedSlice {
-    inner: NonNull<SharedSliceInner>,
-}
-
-impl Debug for SharedSlice {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("SharedSlice")
-            .field("data", &self.as_slice())
-            .finish()
-    }
-}
-
-// UNSAFE(@ohsayan): This is completely safe because our impl guarantees this
-unsafe impl Send for SharedSlice {}
-unsafe impl Sync for SharedSlice {}
-
-impl SharedSlice {
-    #[inline(always)]
-    /// Create a new [`SharedSlice`] using the given local slice
-    pub fn new(slice: &[u8]) -> Self {
-        Self {
-            inner: unsafe {
-                NonNull::new_unchecked(Box::leak(Box::new(SharedSliceInner::new(slice))))
-            },
-        }
-    }
-    #[inline(always)]
-    /// Returns a reference to te inner heap allocation for shared state
-    fn inner(&self) -> &SharedSliceInner {
-        unsafe { &*self.inner.as_ptr() }
-    }
-    #[inline(never)]
-    /// A slow-path to deallocating all the heap allocations
-    unsafe fn slow_drop(&self) {
-        if self.len() != 0 {
-            // IMPORTANT: Do not use the aligned pointer as a sentinel
-            let inner = self.inner();
-            // heap array dtor
-            ptr::drop_in_place(slice::from_raw_parts_mut(inner.data as *mut u8, inner.len));
-            // dealloc heap array
-            dealloc(
-                inner.data as *mut u8,
-                Layout::array::<u8>(inner.len).unwrap(),
-            )
-        }
-        // destroy shared state alloc
-        drop(Box::from_raw(self.inner.as_ptr()))
-    }
-    /// Returns a local slice for the shared slice
-    #[inline(always)]
-    pub fn as_slice(&self) -> &[u8] {
-        unsafe {
-            /*
-                UNSAFE(@ohsayan): The dtor guarantees that:
-                1. we will never end up shooting ourselves in the foot
-                2. the ptr is either valid, or invalid but well aligned. this upholds the raw_parts contract
-                3. the len is either valid, or zero
-            */
-            let inner = self.inner();
-            slice::from_raw_parts(inner.data, inner.len)
-        }
-    }
-}
-
-impl Clone for SharedSlice {
-    #[inline(always)]
-    fn clone(&self) -> Self {
-        // relaxed is fine. the fencing in the dtor decr ensures we don't mess things up
-        let _new_refcount = self.inner().rc.fetch_add(1, Ordering::Relaxed);
-        Self { inner: self.inner }
-    }
-}
-
-impl Drop for SharedSlice {
-    #[inline(always)]
-    fn drop(&mut self) {
-        if self.inner().rc.fetch_sub(1, Ordering::Release) != 1 {
-            // not the last owner; return
-            return;
-        }
-        // use fence for sync with stores
-        atomic::fence(Ordering::Acquire);
-        unsafe {
-            // UNSAFE(@ohsayan): At this point, we can be sure that no one else is using the data
-            self.slow_drop();
-        }
-    }
-}
-
-// trait impls
-impl Hash for SharedSlice {
-    #[inline(always)]
-    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
-        self.as_slice().hash(state);
-    }
-}
-
-impl<T: AsRef<[u8]>> PartialEq<T> for SharedSlice {
-    #[inline(always)]
-    fn eq(&self, other: &T) -> bool {
-        self.as_slice() == other.as_ref()
-    }
-}
-
-impl PartialEq<str> for SharedSlice {
-    #[inline(always)]
-    fn eq(&self, other: &str) -> bool {
-        self.as_slice() == other.as_bytes()
-    }
-}
-
-impl AsRef<[u8]> for SharedSlice {
-    #[inline(always)]
-    fn as_ref(&self) -> &[u8] {
-        self.as_slice()
-    }
-}
-
-impl Borrow<[u8]> for SharedSlice {
-    #[inline(always)]
-    fn borrow(&self) -> &[u8] {
-        self.as_slice()
-    }
-}
-
-impl Deref for SharedSlice {
-    type Target = [u8];
-    #[inline(always)]
-    fn deref(&self) -> &Self::Target {
-        self.as_slice()
-    }
-}
-
-impl<'a> From<&'a [u8]> for SharedSlice {
-    #[inline(always)]
-    fn from(s: &'a [u8]) -> Self {
-        Self::new(s)
-    }
-}
-
-impl<'a> From<&'a str> for SharedSlice {
-    #[inline(always)]
-    fn from(s: &'a str) -> Self {
-        Self::new(s.as_bytes())
-    }
-}
-
-impl From<String> for SharedSlice {
-    #[inline(always)]
-    fn from(s: String) -> Self {
-        Self::new(s.as_bytes())
-    }
-}
-
-impl From<Vec<u8>> for SharedSlice {
-    #[inline(always)]
-    fn from(v: Vec<u8>) -> Self {
-        Self::new(v.as_slice())
-    }
-}
-
-impl Eq for SharedSlice {}
-
-/// The shared state structure
-struct SharedSliceInner {
-    /// data ptr
-    data: *const u8,
-    /// data len
-    len: usize,
-    /// ref count
-    rc: AtomicUsize,
-}
-
-impl SharedSliceInner {
-    #[inline(always)]
-    fn new(slice: &[u8]) -> Self {
-        let layout = Layout::array::<u8>(slice.len()).unwrap();
-        let data = unsafe {
-            if slice.is_empty() {
-                // HACK(@ohsayan): Just ensure that the address is aligned for this
-                layout.align() as *mut u8
-            } else {
-                // UNSAFE(@ohsayan): Come on, just a malloc and memcpy
-                let array_ptr = alloc(layout);
-                ptr::copy_nonoverlapping(slice.as_ptr(), array_ptr, slice.len());
-                array_ptr
-            }
-        };
-        Self {
-            data,
-            len: slice.len(),
-            rc: AtomicUsize::new(1),
-        }
-    }
-}
-
-#[test]
-fn basic() {
-    let slice = SharedSlice::from("hello");
-    assert_eq!(slice, b"hello");
-}
-
-#[test]
-fn basic_cloned() {
-    let slice_a = SharedSlice::from("hello");
-    let slice_a_clone = slice_a.clone();
-    drop(slice_a);
-    assert_eq!(slice_a_clone, b"hello");
-}
-
-#[test]
-fn basic_cloned_across_threads() {
-    use std::thread;
-    const ST: &str = "world";
-    const THREADS: usize = 8;
-    let slice = SharedSlice::from(ST);
-    let mut handles = Vec::with_capacity(THREADS);
-    for _ in 0..THREADS {
-        let clone = slice.clone();
-        handles.push(thread::spawn(move || assert_eq!(clone, ST)))
-    }
-    handles.into_iter().for_each(|h| h.join().unwrap());
-    assert_eq!(slice, ST);
-}
diff --git a/server/src/corestore/table.rs b/server/src/corestore/table.rs
deleted file mode 100644
index 9b068925..00000000
--- a/server/src/corestore/table.rs
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Created on Sat Jul 17 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#[cfg(test)]
-use crate::corestore::{memstore::DdlError, KeyspaceResult};
-use crate::{
-    actions::ActionResult,
-    auth::Authmap,
-    corestore::{htable::Coremap, SharedSlice},
-    dbnet::prelude::Corestore,
-    kvengine::{KVEListmap, KVEStandard, LockedVec},
-    protocol::interface::ProtocolSpec,
-    util,
-};
-
-pub trait DescribeTable {
-    type Table;
-    fn try_get(table: &Table) -> Option<&Self::Table>;
-    fn get<P: ProtocolSpec>(store: &Corestore) -> ActionResult<&Self::Table> {
-        match store.estate.table {
-            Some((_, ref table)) => {
-                // so we do have a table
-                match Self::try_get(table) {
-                    Some(tbl) => Ok(tbl),
-                    None => util::err(P::RSTRING_WRONG_MODEL),
-                }
-            }
-            None => util::err(P::RSTRING_DEFAULT_UNSET),
-        }
-    }
-}
-
-pub struct KVEBlob;
-
-impl DescribeTable for KVEBlob {
-    type Table = KVEStandard;
-    fn try_get(table: &Table) -> Option<&Self::Table> {
-        if let DataModel::KV(ref kve) = table.model_store {
-            Some(kve)
-        } else {
-            None
-        }
-    }
-}
-
-pub struct KVEList;
-
-impl DescribeTable for KVEList {
-    type Table = KVEListmap;
-    fn try_get(table: &Table) -> Option<&Self::Table> {
-        if let DataModel::KVExtListmap(ref kvl) = table.model_store {
-            Some(kvl)
-        } else {
-            None
-        }
-    }
-}
-
-#[derive(Debug)]
-pub enum SystemDataModel {
-    Auth(Authmap),
-}
-
-#[derive(Debug)]
-pub struct SystemTable {
-    /// data storage
-    pub data: SystemDataModel,
-}
-
-impl SystemTable {
-    pub const fn get_model_ref(&self) -> &SystemDataModel {
-        &self.data
-    }
-    pub fn new(data: SystemDataModel) -> Self {
-        Self { data }
-    }
-    pub fn new_auth(authmap: Authmap) -> Self {
-        Self::new(SystemDataModel::Auth(authmap))
-    }
-}
-
-#[derive(Debug)]
-pub enum DataModel {
-    KV(KVEStandard),
-    KVExtListmap(KVEListmap),
-}
-
-// same 8 byte ptrs; any chance of optimizations?
-
-#[derive(Debug)]
-/// The underlying table type. This is the place for the other data models (soon!)
-pub struct Table {
-    /// a key/value store
-    model_store: DataModel,
-    /// is the table volatile
-    volatile: bool,
-}
-
-impl Table {
-    #[cfg(test)]
-    pub const fn from_kve(kve: KVEStandard, volatile: bool) -> Self {
-        Self {
-            model_store: DataModel::KV(kve),
-            volatile,
-        }
-    }
-    #[cfg(test)]
-    pub const fn from_kve_listmap(kve: KVEListmap, volatile: bool) -> Self {
-        Self {
-            model_store: DataModel::KVExtListmap(kve),
-            volatile,
-        }
-    }
-    /// Get the key/value store if the table is a key/value store
-    #[cfg(test)]
-    pub const fn get_kvstore(&self) -> KeyspaceResult<&KVEStandard> {
-        #[allow(irrefutable_let_patterns)]
-        if let DataModel::KV(kvs) = &self.model_store {
-            Ok(kvs)
-        } else {
-            Err(DdlError::WrongModel)
-        }
-    }
-    pub fn count(&self) -> usize {
-        match &self.model_store {
-            DataModel::KV(kv) => kv.len(),
-            DataModel::KVExtListmap(kv) => kv.len(),
-        }
-    }
-    /// Returns this table's _description_
-    pub fn describe_self(&self) -> &'static str {
-        match self.get_model_code() {
-            // pure KV
-            0 if self.is_volatile() => "Keymap { data:(binstr,binstr), volatile:true }",
-            0 if !self.is_volatile() => "Keymap { data:(binstr,binstr), volatile:false }",
-            1 if self.is_volatile() => "Keymap { data:(binstr,str), volatile:true }",
-            1 if !self.is_volatile() => "Keymap { data:(binstr,str), volatile:false }",
-            2 if self.is_volatile() => "Keymap { data:(str,str), volatile:true }",
-            2 if !self.is_volatile() => "Keymap { data:(str,str), volatile:false }",
-            3 if self.is_volatile() => "Keymap { data:(str,binstr), volatile:true }",
-            3 if !self.is_volatile() => "Keymap { data:(str,binstr), volatile:false }",
-            // KVext => list
-            4 if self.is_volatile() => "Keymap { data:(binstr,list<binstr>), volatile:true }",
-            4 if !self.is_volatile() => "Keymap { data:(binstr,list<binstr>), volatile:false }",
-            5 if self.is_volatile() => "Keymap { data:(binstr,list<str>), volatile:true }",
-            5 if !self.is_volatile() => "Keymap { data:(binstr,list<str>), volatile:false }",
-            6 if self.is_volatile() => "Keymap { data:(str,list<binstr>), volatile:true }",
-            6 if !self.is_volatile() => "Keymap { data:(str,list<binstr>), volatile:false }",
-            7 if self.is_volatile() => "Keymap { data:(str,list<str>), volatile:true }",
-            7 if !self.is_volatile() => "Keymap { data:(str,list<str>), volatile:false }",
-            _ => unsafe { impossible!() },
-        }
-    }
-    pub fn truncate_table(&self) {
-        match self.model_store {
-            DataModel::KV(ref kv) => kv.truncate_table(),
-            DataModel::KVExtListmap(ref kv) => kv.truncate_table(),
-        }
-    }
-    pub fn is_empty(&self) -> bool {
-        self.count() == 0
-    }
-    /// Returns the storage type as an 8-bit uint
-    pub const fn storage_type(&self) -> u8 {
-        self.volatile as u8
-    }
-    /// Returns the volatility of the table
-    pub const fn is_volatile(&self) -> bool {
-        self.volatile
-    }
-    /// Create a new KVEBlob Table with the provided settings
-    pub fn new_pure_kve_with_data(
-        data: Coremap<SharedSlice, SharedSlice>,
-        volatile: bool,
-        k_enc: bool,
-        v_enc: bool,
-    ) -> Self {
-        Self {
-            volatile,
-            model_store: DataModel::KV(KVEStandard::new(k_enc, v_enc, data)),
-        }
-    }
-    pub fn new_kve_listmap_with_data(
-        data: Coremap<SharedSlice, LockedVec>,
-        volatile: bool,
-        k_enc: bool,
-        payload_enc: bool,
-    ) -> Self {
-        Self {
-            volatile,
-            model_store: DataModel::KVExtListmap(KVEListmap::new(k_enc, payload_enc, data)),
-        }
-    }
-    pub fn from_model_code(code: u8, volatile: bool) -> Option<Self> {
-        macro_rules! pkve {
-            ($kenc:expr, $venc:expr) => {
-                Self::new_pure_kve_with_data(Coremap::new(), volatile, $kenc, $venc)
-            };
-        }
-        macro_rules! listmap {
-            ($kenc:expr, $penc:expr) => {
-                Self::new_kve_listmap_with_data(Coremap::new(), volatile, $kenc, $penc)
-            };
-        }
-        let ret = match code {
-            // pure kve
-            0 => pkve!(false, false),
-            1 => pkve!(false, true),
-            2 => pkve!(true, true),
-            3 => pkve!(true, false),
-            // kvext: listmap
-            4 => listmap!(false, false),
-            5 => listmap!(false, true),
-            6 => listmap!(true, false),
-            7 => listmap!(true, true),
-            _ => return None,
-        };
-        Some(ret)
-    }
-    /// Create a new kve with default settings but with provided volatile configuration
-    #[cfg(test)]
-    pub fn new_kve_with_volatile(volatile: bool) -> Self {
-        Self::new_pure_kve_with_data(Coremap::new(), volatile, false, false)
-    }
-    /// Returns the default kve:
-    /// - `k_enc`: `false`
-    /// - `v_enc`: `false`
-    /// - `volatile`: `false`
-    pub fn new_default_kve() -> Self {
-        Self::new_pure_kve_with_data(Coremap::new(), false, false, false)
-    }
-    /// Returns the model code. See [`bytemarks`] for more info
-    pub fn get_model_code(&self) -> u8 {
-        match self.model_store {
-            DataModel::KV(ref kvs) => {
-                /*
-                bin,bin => 0
-                bin,str => 1
-                str,str => 2
-                str,bin => 3
-                */
-                let (kenc, venc) = kvs.get_encoding_tuple();
-                let ret = kenc as u8 + venc as u8;
-                // a little bitmagic goes a long way
-                (ret & 1) + ((kenc as u8) << 1)
-            }
-            DataModel::KVExtListmap(ref kvlistmap) => {
-                /*
-                bin,list<bin> => 4,
-                bin,list<str> => 5,
-                str,list<bin> => 6,
-                str,list<str> => 7
-                */
-                let (kenc, venc) = kvlistmap.get_encoding_tuple();
-                ((kenc as u8) << 1) + (venc as u8) + 4
-            }
-        }
-    }
-    /// Returns the inner data model
-    pub fn get_model_ref(&self) -> &DataModel {
-        &self.model_store
-    }
-}
diff --git a/server/src/corestore/tests.rs b/server/src/corestore/tests.rs
deleted file mode 100644
index 2b334482..00000000
--- a/server/src/corestore/tests.rs
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Created on Fri Jul 30 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-mod memstore_keyspace_tests {
-    use super::super::{memstore::*, table::Table};
-
-    #[test]
-    fn test_drop_keyspace_empty() {
-        let ms = Memstore::new_empty();
-        let obj = unsafe { ObjectID::from_slice("myks") };
-        ms.create_keyspace(obj.clone());
-        assert!(ms.drop_keyspace(obj).is_ok());
-    }
-
-    #[test]
-    fn test_drop_keyspace_still_accessed() {
-        let ms = Memstore::new_empty();
-        let obj = unsafe { ObjectID::from_slice("myks") };
-        ms.create_keyspace(obj.clone());
-        let _ks_ref = ms.get_keyspace_atomic_ref(&obj);
-        assert_eq!(ms.drop_keyspace(obj).unwrap_err(), DdlError::StillInUse);
-    }
-
-    #[test]
-    fn test_drop_keyspace_not_empty() {
-        let ms = Memstore::new_empty();
-        let obj = unsafe { ObjectID::from_slice("myks") };
-        ms.create_keyspace(obj.clone());
-        let ks_ref = ms.get_keyspace_atomic_ref(&obj).unwrap();
-        ks_ref.create_table(
-            unsafe { ObjectID::from_slice("mytbl") },
-            Table::new_default_kve(),
-        );
-        assert_eq!(ms.drop_keyspace(obj).unwrap_err(), DdlError::NotEmpty);
-    }
-
-    #[test]
-    fn test_force_drop_keyspace_empty() {
-        let ms = Memstore::new_empty();
-        let obj = unsafe { ObjectID::from_slice("myks") };
-        ms.create_keyspace(obj.clone());
-        assert!(ms.force_drop_keyspace(obj).is_ok());
-    }
-
-    #[test]
-    fn test_force_drop_keyspace_still_accessed() {
-        let ms = Memstore::new_empty();
-        let obj = unsafe { ObjectID::from_slice("myks") };
-        ms.create_keyspace(obj.clone());
-        let _ks_ref = ms.get_keyspace_atomic_ref(&obj);
-        assert_eq!(
-            ms.force_drop_keyspace(obj).unwrap_err(),
-            DdlError::StillInUse
-        );
-    }
-
-    #[test]
-    fn test_force_drop_keyspace_table_referenced() {
-        // the check here is to see if all the tables are not in active use
-        let ms = Memstore::new_empty();
-        let obj = unsafe { ObjectID::from_slice("myks") };
-        let tblid = unsafe { ObjectID::from_slice("mytbl") };
-        // create the ks
-        ms.create_keyspace(obj.clone());
-        // get an atomic ref to the keyspace
-        let ks_ref = ms.get_keyspace_atomic_ref(&obj).unwrap();
-        // create a table
-        ks_ref.create_table(tblid.clone(), Table::new_default_kve());
-        // ref to the table
-        let _tbl_ref = ks_ref.get_table_atomic_ref(&tblid).unwrap();
-        // drop ks ref
-        drop(ks_ref);
-        assert_eq!(
-            ms.force_drop_keyspace(obj).unwrap_err(),
-            DdlError::StillInUse
-        );
-    }
-
-    #[test]
-    fn test_force_drop_keyspace_nonempty_okay() {
-        // the check here is to see if drop succeeds, provided that no
-        // tables are in active use
-        let ms = Memstore::new_empty();
-        let obj = unsafe { ObjectID::from_slice("myks") };
-        let tblid = unsafe { ObjectID::from_slice("mytbl") };
-        // create the ks
-        ms.create_keyspace(obj.clone());
-        // get an atomic ref to the keyspace
-        let ks_ref = ms.get_keyspace_atomic_ref(&obj).unwrap();
-        // create a table
-        ks_ref.create_table(tblid, Table::new_default_kve());
-        // drop ks ref
-        drop(ks_ref);
-        // should succeed because the keyspace is non-empty, but no table is referenced to
-        assert!(ms.force_drop_keyspace(obj).is_ok());
-    }
-}
-
-mod modelcode_tests {
-    use {
-        super::super::table::Table,
-        crate::kvengine::{KVEListmap, KVEngine},
-    };
-
-    #[test]
-    fn test_model_code_pure_kve() {
-        // binstr, binstr
-        let kv1 = KVEngine::init(false, false);
-        // binstr, str
-        let kv2 = KVEngine::init(false, true);
-        // str, str
-        let kv3 = KVEngine::init(true, true);
-        // str, binstr
-        let kv4 = KVEngine::init(true, false);
-
-        // now check
-        let tbl1 = Table::from_kve(kv1, false);
-        assert_eq!(tbl1.get_model_code(), 0);
-        let tbl2 = Table::from_kve(kv2, false);
-        assert_eq!(tbl2.get_model_code(), 1);
-        let tbl3 = Table::from_kve(kv3, false);
-        assert_eq!(tbl3.get_model_code(), 2);
-        let tbl4 = Table::from_kve(kv4, false);
-        assert_eq!(tbl4.get_model_code(), 3);
-    }
-    #[test]
-    fn test_model_code_kvext_listmap() {
-        // binstr, list<binstr>
-        let l1 = KVEListmap::init(false, false);
-        // binstr, list<str>
-        let l2 = KVEListmap::init(false, true);
-        // str, list<binstr>
-        let l3 = KVEListmap::init(true, false);
-        // str, list<str>
-        let l4 = KVEListmap::init(true, true);
-
-        // now check
-        let tbl1 = Table::from_kve_listmap(l1, false);
-        assert_eq!(tbl1.get_model_code(), 4);
-        let tbl2 = Table::from_kve_listmap(l2, false);
-        assert_eq!(tbl2.get_model_code(), 5);
-        let tbl3 = Table::from_kve_listmap(l3, false);
-        assert_eq!(tbl3.get_model_code(), 6);
-        let tbl4 = Table::from_kve_listmap(l4, false);
-        assert_eq!(tbl4.get_model_code(), 7);
-    }
-}
diff --git a/server/src/dbnet/connection.rs b/server/src/dbnet/connection.rs
deleted file mode 100644
index 2ddcd7da..00000000
--- a/server/src/dbnet/connection.rs
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Created on Sun Aug 21 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{BufferedSocketStream, QueryResult},
-    crate::{
-        corestore::buffers::Integer64,
-        protocol::{interface::ProtocolSpec, ParseError},
-        IoResult,
-    },
-    bytes::BytesMut,
-    std::{
-        io::{Error as IoError, ErrorKind},
-        marker::PhantomData,
-    },
-    tokio::io::{AsyncReadExt, AsyncWriteExt, BufWriter},
-};
-
-const BUF_WRITE_CAP: usize = 8192;
-const BUF_READ_CAP: usize = 8192;
-
-/// A generic connection type
-///
-/// The generic connection type allows you to choose:
-/// 1. A stream (TCP, TLS(TCP), UDS, ...)
-/// 2. A protocol (one that implements [`ProtocolSpec`])
-pub struct Connection<T, P> {
-    pub(super) stream: BufWriter<T>,
-    pub(super) buffer: BytesMut,
-    _marker: PhantomData<P>,
-}
-
-impl<T: BufferedSocketStream, P: ProtocolSpec> Connection<T, P> {
-    pub fn new(stream: T) -> Self {
-        Connection {
-            stream: BufWriter::with_capacity(BUF_WRITE_CAP, stream),
-            buffer: BytesMut::with_capacity(BUF_READ_CAP),
-            _marker: PhantomData,
-        }
-    }
-}
-
-// protocol read
-impl<T: BufferedSocketStream, P: ProtocolSpec> Connection<T, P> {
-    /// Attempt to read a query
-    pub(super) async fn read_query(&mut self) -> IoResult<QueryResult> {
-        loop {
-            match self.stream.read_buf(&mut self.buffer).await {
-                Ok(0) => {
-                    if self.buffer.is_empty() {
-                        // buffer is empty, and the remote pulled off (simple disconnection)
-                        return Ok(QueryResult::Disconnected);
-                    } else {
-                        // wrote something, and then died. nope, that's an error
-                        return Err(IoError::from(ErrorKind::ConnectionReset));
-                    }
-                }
-                Ok(_) => {}
-                Err(e) => return Err(e),
-            }
-            // see if we have buffered enough data to run anything
-            match P::decode_packet(self.buffer.as_ref()) {
-                Ok(query_with_advance) => return Ok(QueryResult::Q(query_with_advance)),
-                Err(ParseError::NotEnough) => {}
-                Err(e) => {
-                    self.write_error(P::SKYHASH_PARSE_ERROR_LUT[e as usize - 1])
-                        .await?;
-                    return Ok(QueryResult::NextLoop);
-                }
-            }
-        }
-    }
-}
-
-// protocol write (metaframe)
-impl<T: BufferedSocketStream, P: ProtocolSpec> Connection<T, P> {
-    /// Write a simple query header to the stream
-    pub(super) async fn write_simple_query_header(&mut self) -> IoResult<()> {
-        self.stream.write_all(P::SIMPLE_QUERY_HEADER).await
-    }
-
-    /// Write the pipeline query header
-    pub(super) async fn write_pipelined_query_header(&mut self, count: usize) -> IoResult<()> {
-        // write pipeline first byte
-        self.stream.write_u8(P::PIPELINED_QUERY_FIRST_BYTE).await?;
-        // write pipeline query count
-        self.stream.write_all(&Integer64::from(count)).await?;
-        // write the LF
-        self.stream.write_u8(P::LF).await
-    }
-}
-
-// protocol write (helpers)
-impl<T: BufferedSocketStream, P: ProtocolSpec> Connection<T, P> {
-    /// Write an error to the stream (just used to differentiate between "normal" and "errored" writes)
-    pub(super) async fn write_error(&mut self, error: &[u8]) -> IoResult<()> {
-        self.stream.write_all(error).await?;
-        self.stream.flush().await
-    }
-    /// Write something "raw" to the stream (intentional underscore to avoid misuse)
-    pub async fn _write_raw(&mut self, raw: &[u8]) -> IoResult<()> {
-        self.stream.write_all(raw).await
-    }
-}
-
-// protocol write (dataframe)
-impl<T: BufferedSocketStream, P: ProtocolSpec> Connection<T, P> {
-    // monoelements
-    /// Encode and write a length-prefixed monoelement
-    pub async fn write_mono_length_prefixed_with_tsymbol(
-        &mut self,
-        data: &[u8],
-        tsymbol: u8,
-    ) -> IoResult<()> {
-        // first write the tsymbol
-        self.stream.write_u8(tsymbol).await?;
-        // now write length
-        self.stream.write_all(&Integer64::from(data.len())).await?;
-        // now write LF
-        self.stream.write_u8(P::LF).await?;
-        // now write the actual body
-        self.stream.write_all(data).await?;
-        if P::NEEDS_TERMINAL_LF {
-            self.stream.write_u8(P::LF).await
-        } else {
-            Ok(())
-        }
-    }
-    /// Encode and write a mon element (**without** length-prefixing)
-    pub async fn write_mono_with_tsymbol(&mut self, data: &[u8], tsymbol: u8) -> IoResult<()> {
-        // first write the tsymbol
-        self.stream.write_u8(tsymbol).await?;
-        // now write the actual body
-        self.stream.write_all(data).await?;
-        self.stream.write_u8(P::LF).await
-    }
-    /// Encode and write an unicode string
-    pub async fn write_string(&mut self, string: &str) -> IoResult<()> {
-        self.write_mono_length_prefixed_with_tsymbol(string.as_bytes(), P::TSYMBOL_STRING)
-            .await
-    }
-    /// Encode and write a blob
-    #[allow(unused)]
-    pub async fn write_binary(&mut self, binary: &[u8]) -> IoResult<()> {
-        self.write_mono_length_prefixed_with_tsymbol(binary, P::TSYMBOL_BINARY)
-            .await
-    }
-    /// Encode and write an `usize`
-    pub async fn write_usize(&mut self, size: usize) -> IoResult<()> {
-        self.write_mono_with_tsymbol(&Integer64::from(size), P::TSYMBOL_INT64)
-            .await
-    }
-    /// Encode and write an `u64`
-    pub async fn write_int64(&mut self, int: u64) -> IoResult<()> {
-        self.write_mono_with_tsymbol(&Integer64::from(int), P::TSYMBOL_INT64)
-            .await
-    }
-    /// Encode and write an `f32`
-    pub async fn write_float(&mut self, float: f32) -> IoResult<()> {
-        self.write_mono_with_tsymbol(float.to_string().as_bytes(), P::TSYMBOL_FLOAT)
-            .await
-    }
-
-    // typed array
-    /// Write a typed array header (including type information and size)
-    pub async fn write_typed_array_header(&mut self, len: usize, tsymbol: u8) -> IoResult<()> {
-        self.stream
-            .write_all(&[P::TSYMBOL_TYPED_ARRAY, tsymbol])
-            .await?;
-        self.stream.write_all(&Integer64::from(len)).await?;
-        self.stream.write_u8(P::LF).await
-    }
-    /// Encode and write a null element for a typed array
-    pub async fn write_typed_array_element_null(&mut self) -> IoResult<()> {
-        self.stream
-            .write_all(P::TYPE_TYPED_ARRAY_ELEMENT_NULL)
-            .await
-    }
-    /// Encode and write a typed array element
-    pub async fn write_typed_array_element(&mut self, element: &[u8]) -> IoResult<()> {
-        self.stream
-            .write_all(&Integer64::from(element.len()))
-            .await?;
-        self.stream.write_u8(P::LF).await?;
-        self.stream.write_all(element).await?;
-        if P::NEEDS_TERMINAL_LF {
-            self.stream.write_u8(P::LF).await
-        } else {
-            Ok(())
-        }
-    }
-
-    // typed non-null array
-    /// write typed non-null array header
-    pub async fn write_typed_non_null_array_header(
-        &mut self,
-        len: usize,
-        tsymbol: u8,
-    ) -> IoResult<()> {
-        self.stream
-            .write_all(&[P::TSYMBOL_TYPED_NON_NULL_ARRAY, tsymbol])
-            .await?;
-        self.stream.write_all(&Integer64::from(len)).await?;
-        self.stream.write_all(&[P::LF]).await
-    }
-    /// Encode and write typed non-null array element
-    pub async fn write_typed_non_null_array_element(&mut self, element: &[u8]) -> IoResult<()> {
-        self.write_typed_array_element(element).await
-    }
-    /// Encode and write a typed non-null array
-    pub async fn write_typed_non_null_array<A, B>(&mut self, body: B, tsymbol: u8) -> IoResult<()>
-    where
-        B: AsRef<[A]>,
-        A: AsRef<[u8]>,
-    {
-        let body = body.as_ref();
-        self.write_typed_non_null_array_header(body.len(), tsymbol)
-            .await?;
-        for element in body {
-            self.write_typed_non_null_array_element(element.as_ref())
-                .await?;
-        }
-        Ok(())
-    }
-}
diff --git a/server/src/dbnet/listener.rs b/server/src/dbnet/listener.rs
deleted file mode 100644
index ef96e50e..00000000
--- a/server/src/dbnet/listener.rs
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Created on Sun Aug 21 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{
-        tcp::{Listener, ListenerV1},
-        tls::{SslListener, SslListenerV1},
-    },
-    crate::{
-        auth::AuthProvider,
-        config::{PortConfig, ProtocolVersion, SslOpts},
-        corestore::Corestore,
-        util::error::{Error, SkyResult},
-        IoResult,
-    },
-    core::future::Future,
-    std::{net::IpAddr, sync::Arc},
-    tokio::{
-        net::TcpListener,
-        sync::{broadcast, mpsc, Semaphore},
-    },
-};
-
-/// The base TCP listener
-pub struct BaseListener {
-    /// An atomic reference to the coretable
-    pub db: Corestore,
-    /// The auth provider
-    pub auth: AuthProvider,
-    /// The incoming connection listener (binding)
-    pub listener: TcpListener,
-    /// The maximum number of connections
-    pub climit: Arc<Semaphore>,
-    /// The shutdown broadcaster
-    pub signal: broadcast::Sender<()>,
-    // When all `Sender`s are dropped - the `Receiver` gets a `None` value
-    // We send a clone of `terminate_tx` to each `CHandler`
-    pub terminate_tx: mpsc::Sender<()>,
-    pub terminate_rx: mpsc::Receiver<()>,
-}
-
-impl BaseListener {
-    pub async fn init(
-        db: &Corestore,
-        auth: AuthProvider,
-        host: IpAddr,
-        port: u16,
-        semaphore: Arc<Semaphore>,
-        signal: broadcast::Sender<()>,
-    ) -> SkyResult<Self> {
-        let (terminate_tx, terminate_rx) = mpsc::channel(1);
-        let listener = TcpListener::bind((host, port))
-            .await
-            .map_err(|e| Error::ioerror_extra(e, format!("binding to port {port}")))?;
-        Ok(Self {
-            db: db.clone(),
-            auth,
-            listener,
-            climit: semaphore,
-            signal,
-            terminate_tx,
-            terminate_rx,
-        })
-    }
-    pub async fn release_self(self) {
-        let Self {
-            mut terminate_rx,
-            terminate_tx,
-            signal,
-            ..
-        } = self;
-        drop(signal);
-        drop(terminate_tx);
-        let _ = terminate_rx.recv().await;
-    }
-}
-
-/// Multiple Listener Interface
-///
-/// A `MultiListener` is an abstraction over an `SslListener` or a `Listener` to facilitate
-/// easier asynchronous listening on multiple ports.
-///
-/// - The `SecureOnly` variant holds an `SslListener`
-/// - The `InsecureOnly` variant holds a `Listener`
-/// - The `Multi` variant holds both an `SslListener` and a `Listener`
-///     This variant enables listening to both secure and insecure sockets at the same time
-///     asynchronously
-#[allow(clippy::large_enum_variant)]
-pub enum MultiListener {
-    SecureOnly(SslListener),
-    SecureOnlyV1(SslListenerV1),
-    InsecureOnly(Listener),
-    InsecureOnlyV1(ListenerV1),
-    Multi(Listener, SslListener),
-    MultiV1(ListenerV1, SslListenerV1),
-}
-
-async fn wait_on_port_futures(
-    a: impl Future<Output = IoResult<()>>,
-    b: impl Future<Output = IoResult<()>>,
-) -> IoResult<()> {
-    let (e1, e2) = tokio::join!(a, b);
-    if let Err(e) = e1 {
-        log::error!("Insecure listener failed with: {}", e);
-    }
-    if let Err(e) = e2 {
-        log::error!("Secure listener failed with: {}", e);
-    }
-    Ok(())
-}
-
-impl MultiListener {
-    /// Create a new `InsecureOnly` listener
-    pub fn new_insecure_only(base: BaseListener, protocol: ProtocolVersion) -> Self {
-        match protocol {
-            ProtocolVersion::V2 => MultiListener::InsecureOnly(Listener::new(base)),
-            ProtocolVersion::V1 => MultiListener::InsecureOnlyV1(ListenerV1::new(base)),
-        }
-    }
-    /// Create a new `SecureOnly` listener
-    pub fn new_secure_only(
-        base: BaseListener,
-        ssl: SslOpts,
-        protocol: ProtocolVersion,
-    ) -> SkyResult<Self> {
-        let listener = match protocol {
-            ProtocolVersion::V2 => {
-                let listener = SslListener::new_pem_based_ssl_connection(
-                    ssl.key,
-                    ssl.chain,
-                    base,
-                    ssl.passfile,
-                )?;
-                MultiListener::SecureOnly(listener)
-            }
-            ProtocolVersion::V1 => {
-                let listener = SslListenerV1::new_pem_based_ssl_connection(
-                    ssl.key,
-                    ssl.chain,
-                    base,
-                    ssl.passfile,
-                )?;
-                MultiListener::SecureOnlyV1(listener)
-            }
-        };
-        Ok(listener)
-    }
-    /// Create a new `Multi` listener that has both a secure and an insecure listener
-    pub async fn new_multi(
-        ssl_base_listener: BaseListener,
-        tcp_base_listener: BaseListener,
-        ssl: SslOpts,
-        protocol: ProtocolVersion,
-    ) -> SkyResult<Self> {
-        let mls = match protocol {
-            ProtocolVersion::V2 => {
-                let secure_listener = SslListener::new_pem_based_ssl_connection(
-                    ssl.key,
-                    ssl.chain,
-                    ssl_base_listener,
-                    ssl.passfile,
-                )?;
-                let insecure_listener = Listener::new(tcp_base_listener);
-                MultiListener::Multi(insecure_listener, secure_listener)
-            }
-            ProtocolVersion::V1 => {
-                let secure_listener = SslListenerV1::new_pem_based_ssl_connection(
-                    ssl.key,
-                    ssl.chain,
-                    ssl_base_listener,
-                    ssl.passfile,
-                )?;
-                let insecure_listener = ListenerV1::new(tcp_base_listener);
-                MultiListener::MultiV1(insecure_listener, secure_listener)
-            }
-        };
-        Ok(mls)
-    }
-    /// Start the server
-    ///
-    /// The running of single and/or parallel listeners is handled by this function by
-    /// exploiting the working of async functions
-    pub async fn run_server(&mut self) -> IoResult<()> {
-        match self {
-            MultiListener::SecureOnly(secure_listener) => secure_listener.run().await,
-            MultiListener::SecureOnlyV1(secure_listener) => secure_listener.run().await,
-            MultiListener::InsecureOnly(insecure_listener) => insecure_listener.run().await,
-            MultiListener::InsecureOnlyV1(insecure_listener) => insecure_listener.run().await,
-            MultiListener::Multi(insecure_listener, secure_listener) => {
-                wait_on_port_futures(insecure_listener.run(), secure_listener.run()).await
-            }
-            MultiListener::MultiV1(insecure_listener, secure_listener) => {
-                wait_on_port_futures(insecure_listener.run(), secure_listener.run()).await
-            }
-        }
-    }
-    /// Signal the ports to shut down and only return after they have shut down
-    ///
-    /// **Do note:** This function doesn't flush the `Corestore` object! The **caller has to
-    /// make sure that the data is saved!**
-    pub async fn finish_with_termsig(self) {
-        match self {
-            MultiListener::InsecureOnly(Listener { base, .. })
-            | MultiListener::SecureOnly(SslListener { base, .. })
-            | MultiListener::InsecureOnlyV1(ListenerV1 { base, .. })
-            | MultiListener::SecureOnlyV1(SslListenerV1 { base, .. }) => base.release_self().await,
-            MultiListener::Multi(insecure, secure) => {
-                insecure.base.release_self().await;
-                secure.base.release_self().await;
-            }
-            MultiListener::MultiV1(insecure, secure) => {
-                insecure.base.release_self().await;
-                secure.base.release_self().await;
-            }
-        }
-    }
-}
-
-/// Initialize the database networking
-pub async fn connect(
-    ports: PortConfig,
-    protocol: ProtocolVersion,
-    maxcon: usize,
-    db: Corestore,
-    auth: AuthProvider,
-    signal: broadcast::Sender<()>,
-) -> SkyResult<MultiListener> {
-    let climit = Arc::new(Semaphore::new(maxcon));
-    let base_listener_init = |host, port| {
-        BaseListener::init(
-            &db,
-            auth.clone(),
-            host,
-            port,
-            climit.clone(),
-            signal.clone(),
-        )
-    };
-    let description = ports.get_description();
-    let server = match ports {
-        PortConfig::InsecureOnly { host, port } => {
-            MultiListener::new_insecure_only(base_listener_init(host, port).await?, protocol)
-        }
-        PortConfig::SecureOnly { host, ssl } => MultiListener::new_secure_only(
-            base_listener_init(host, ssl.port).await?,
-            ssl,
-            protocol,
-        )?,
-        PortConfig::Multi { host, port, ssl } => {
-            let secure_listener = base_listener_init(host, ssl.port).await?;
-            let insecure_listener = base_listener_init(host, port).await?;
-            MultiListener::new_multi(secure_listener, insecure_listener, ssl, protocol).await?
-        }
-    };
-    log::info!("Server started on {description}");
-    Ok(server)
-}
diff --git a/server/src/dbnet/mod.rs b/server/src/dbnet/mod.rs
deleted file mode 100644
index f54dc432..00000000
--- a/server/src/dbnet/mod.rs
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Created on Sun Aug 21 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    self::connection::Connection,
-    crate::{
-        actions::{ActionError, ActionResult},
-        auth::AuthProvider,
-        corestore::Corestore,
-        protocol::{interface::ProtocolSpec, Query},
-        util::compiler,
-        IoResult,
-    },
-    bytes::Buf,
-    std::{cell::Cell, sync::Arc, time::Duration},
-    tokio::{
-        io::{AsyncReadExt, AsyncWriteExt},
-        sync::{
-            broadcast::{self},
-            mpsc::{self},
-            Semaphore,
-        },
-        time,
-    },
-};
-
-pub type QueryWithAdvance = (Query, usize);
-pub const MAXIMUM_CONNECTION_LIMIT: usize = 50000;
-use crate::queryengine;
-
-pub use self::listener::connect;
-
-mod connection;
-#[macro_use]
-mod macros;
-mod listener;
-pub mod prelude;
-mod tcp;
-mod tls;
-
-/// This is a "marker trait" that ensures that no silly types are
-/// passed into the [`Connection`] type
-pub trait BufferedSocketStream: AsyncWriteExt + AsyncReadExt + Unpin {}
-
-/// Result of [`Connection::read_query`]
-enum QueryResult {
-    /// A [`Query`] read to be run
-    Q(QueryWithAdvance),
-    /// Simply proceed to the next run loop iter
-    NextLoop,
-    /// The client disconnected
-    Disconnected,
-}
-
-/// A backoff implementation that is meant to be used in connection loops
-pub(self) struct NetBackoff {
-    c: Cell<u8>,
-}
-
-impl NetBackoff {
-    /// The maximum backoff duration
-    const MAX_BACKOFF: u8 = 64;
-    /// Create a new [`NetBackoff`] instance
-    pub const fn new() -> Self {
-        Self { c: Cell::new(1) }
-    }
-    /// Wait for the current backoff duration
-    pub async fn spin(&self) {
-        time::sleep(Duration::from_secs(self.c.get() as _)).await;
-        self.c.set(self.c.get() << 1);
-    }
-    /// Should we disconnect the stream?
-    pub fn should_disconnect(&self) -> bool {
-        self.c.get() > Self::MAX_BACKOFF
-    }
-}
-
-pub struct AuthProviderHandle {
-    /// the source authentication provider
-    provider: AuthProvider,
-    /// authenticated
-    auth_good: bool,
-}
-
-impl AuthProviderHandle {
-    pub fn new(provider: AuthProvider) -> Self {
-        let auth_good = !provider.is_enabled();
-        Self {
-            provider,
-            auth_good,
-        }
-    }
-    /// This returns `true` if:
-    /// 1. Authn is disabled
-    /// 2. The connection is authenticated
-    pub const fn authenticated(&self) -> bool {
-        self.auth_good
-    }
-    pub fn set_auth(&mut self) {
-        self.auth_good = true;
-    }
-    pub fn set_unauth(&mut self) {
-        self.auth_good = false;
-    }
-    pub fn provider_mut(&mut self) -> &mut AuthProvider {
-        &mut self.provider
-    }
-    pub fn provider(&self) -> &AuthProvider {
-        &self.provider
-    }
-}
-
-/// A generic connection handler. You have two choices:
-/// 1. Choose the connection kind
-/// 2. Choose the protocol implementation
-pub struct ConnectionHandler<C, P> {
-    /// an atomic reference to the shared in-memory engine
-    db: Corestore,
-    /// the connection
-    con: Connection<C, P>,
-    /// the semaphore used to impose limits on number of connections
-    climit: Arc<Semaphore>,
-    /// the authentication handle
-    auth: AuthProviderHandle,
-    /// check for termination signals
-    termination_signal: broadcast::Receiver<()>,
-    /// the sender that we drop when we're done with handling a connection (used for gracefule exit)
-    _term_sig_tx: mpsc::Sender<()>,
-}
-
-impl<C, P> ConnectionHandler<C, P>
-where
-    C: BufferedSocketStream,
-    P: ProtocolSpec,
-{
-    /// Create a new connection handler
-    pub fn new(
-        db: Corestore,
-        con: Connection<C, P>,
-        auth_data: AuthProvider,
-        climit: Arc<Semaphore>,
-        termination_signal: broadcast::Receiver<()>,
-        _term_sig_tx: mpsc::Sender<()>,
-    ) -> Self {
-        Self {
-            db,
-            con,
-            climit,
-            auth: AuthProviderHandle::new(auth_data),
-            termination_signal,
-            _term_sig_tx,
-        }
-    }
-    pub async fn run(&mut self) -> IoResult<()> {
-        loop {
-            let packet = tokio::select! {
-                pkt = self.con.read_query() => pkt,
-                _ = self.termination_signal.recv() => {
-                    return Ok(());
-                }
-            };
-            match packet {
-                Ok(QueryResult::Q((query, advance))) => {
-                    // the mutable reference to self ensures that the buffer is not modified
-                    // hence ensuring that the pointers will remain valid
-                    #[cfg(debug_assertions)]
-                    let len_at_start = self.con.buffer.len();
-                    #[cfg(debug_assertions)]
-                    let sptr_at_start = self.con.buffer.as_ptr() as usize;
-                    #[cfg(debug_assertions)]
-                    let eptr_at_start = sptr_at_start + len_at_start;
-                    {
-                        // The actual execution (the assertions are just debug build sanity checks)
-                        match self.execute_query(query).await {
-                            Ok(()) => {}
-                            Err(ActionError::ActionError(e)) => self.con.write_error(e).await?,
-                            Err(ActionError::IoError(e)) => return Err(e),
-                        }
-                    }
-                    {
-                        // do these assertions to ensure memory safety (this is just for sanity sake)
-                        #[cfg(debug_assertions)]
-                        // len should be unchanged. no functions should **ever** touch the buffer
-                        debug_assert_eq!(self.con.buffer.len(), len_at_start);
-                        #[cfg(debug_assertions)]
-                        // start of allocation should be unchanged
-                        debug_assert_eq!(self.con.buffer.as_ptr() as usize, sptr_at_start);
-                        #[cfg(debug_assertions)]
-                        // end of allocation should be unchanged. else we're entirely violating
-                        // memory safety guarantees
-                        debug_assert_eq!(
-                            unsafe {
-                                // UNSAFE(@ohsayan): THis is always okay
-                                self.con.buffer.as_ptr().add(len_at_start)
-                            } as usize,
-                            eptr_at_start
-                        );
-                        // this is only when we clear the buffer. since execute_query is not called
-                        // at this point, it's totally fine (so invalidating ptrs is totally cool)
-                        self.con.buffer.advance(advance);
-                    }
-                }
-                Ok(QueryResult::Disconnected) => return Ok(()),
-                Ok(QueryResult::NextLoop) => {}
-                Err(e) => return Err(e),
-            }
-        }
-    }
-    async fn execute_query(&mut self, query: Query) -> ActionResult<()> {
-        let Self { db, con, auth, .. } = self;
-        match query {
-            Query::Simple(q) => {
-                con.write_simple_query_header().await?;
-                if compiler::likely(auth.authenticated()) {
-                    queryengine::execute_simple(db, con, auth, q).await?;
-                } else {
-                    queryengine::execute_simple_noauth(db, con, auth, q).await?;
-                }
-            }
-            Query::Pipelined(p) => {
-                if compiler::likely(auth.authenticated()) {
-                    con.write_pipelined_query_header(p.len()).await?;
-                    queryengine::execute_pipeline(db, con, auth, p).await?;
-                } else {
-                    con.write_simple_query_header().await?;
-                    con.write_error(P::AUTH_CODE_BAD_CREDENTIALS).await?;
-                }
-            }
-        }
-        con.stream.flush().await?;
-        Ok(())
-    }
-}
-
-impl<C, T> Drop for ConnectionHandler<C, T> {
-    fn drop(&mut self) {
-        // Make sure that the permit is returned to the semaphore
-        // in the case that there is a panic inside
-        self.climit.add_permits(1);
-    }
-}
diff --git a/server/src/dbnet/prelude.rs b/server/src/dbnet/prelude.rs
deleted file mode 100644
index 225aecad..00000000
--- a/server/src/dbnet/prelude.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Created on Sun Aug 21 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! A 'prelude' for imports to interface with the database and the client
-//!
-//! This module is hollow itself, it only re-exports from `dbnet::con` and `tokio::io`
-
-pub use {
-    super::{connection::Connection, AuthProviderHandle},
-    crate::{
-        actions::{ensure_boolean_or_aerr, ensure_length, translate_ddl_error},
-        corestore::{
-            table::{KVEBlob, KVEList},
-            Corestore,
-        },
-        get_tbl, handle_entity, is_lowbit_set,
-        protocol::interface::ProtocolSpec,
-        queryengine::ActionIter,
-        registry,
-        util::{self, UnwrapActionError, Unwrappable},
-    },
-    tokio::io::{AsyncReadExt, AsyncWriteExt},
-};
diff --git a/server/src/dbnet/tcp.rs b/server/src/dbnet/tcp.rs
deleted file mode 100644
index 0de3d224..00000000
--- a/server/src/dbnet/tcp.rs
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Created on Mon Apr 26 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-pub use protocol::{ParseResult, Query};
-use {
-    super::NetBackoff,
-    crate::{
-        dbnet::{listener::BaseListener, BufferedSocketStream, Connection, ConnectionHandler},
-        protocol::{self, interface::ProtocolSpec, Skyhash1, Skyhash2},
-        IoResult,
-    },
-    std::marker::PhantomData,
-    tokio::net::TcpStream,
-};
-
-impl BufferedSocketStream for TcpStream {}
-
-pub type Listener = RawListener<Skyhash2>;
-pub type ListenerV1 = RawListener<Skyhash1>;
-
-/// A listener
-pub struct RawListener<P> {
-    pub base: BaseListener,
-    _marker: PhantomData<P>,
-}
-
-impl<P: ProtocolSpec + 'static> RawListener<P> {
-    pub fn new(base: BaseListener) -> Self {
-        Self {
-            base,
-            _marker: PhantomData,
-        }
-    }
-    /// Accept an incoming connection
-    async fn accept(&mut self) -> IoResult<TcpStream> {
-        let backoff = NetBackoff::new();
-        loop {
-            match self.base.listener.accept().await {
-                // We don't need the bindaddr
-                Ok((stream, _)) => return Ok(stream),
-                Err(e) => {
-                    if backoff.should_disconnect() {
-                        // Too many retries, goodbye user
-                        return Err(e);
-                    }
-                }
-            }
-            // spin to wait for the backoff duration
-            backoff.spin().await;
-        }
-    }
-    /// Run the server
-    pub async fn run(&mut self) -> IoResult<()> {
-        loop {
-            // Take the permit first, but we won't use it right now
-            // that's why we will forget it
-            self.base.climit.acquire().await.unwrap().forget();
-            /*
-             SECURITY: Ignore any errors that may arise in the accept
-             loop. If we apply the try operator here, we will immediately
-             terminate the run loop causing the entire server to go down.
-             Also, do not log any errors because many connection errors
-             can arise and it will flood the log and might also result
-             in a crash
-            */
-            let stream = skip_loop_err!(self.accept().await);
-            let mut chandle = ConnectionHandler::<TcpStream, P>::new(
-                self.base.db.clone(),
-                Connection::new(stream),
-                self.base.auth.clone(),
-                self.base.climit.clone(),
-                self.base.signal.subscribe(),
-                self.base.terminate_tx.clone(),
-            );
-            tokio::spawn(async move {
-                if let Err(e) = chandle.run().await {
-                    log::error!("Error: {}", e);
-                }
-            });
-        }
-    }
-}
diff --git a/server/src/dbnet/tls.rs b/server/src/dbnet/tls.rs
deleted file mode 100644
index 13e512e5..00000000
--- a/server/src/dbnet/tls.rs
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Created on Fri Dec 18 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{
-        dbnet::{
-            listener::BaseListener, BufferedSocketStream, Connection, ConnectionHandler, NetBackoff,
-        },
-        protocol::{interface::ProtocolSpec, Skyhash1, Skyhash2},
-        util::error::{Error, SkyResult},
-        IoResult,
-    },
-    openssl::{
-        pkey::PKey,
-        rsa::Rsa,
-        ssl::{Ssl, SslAcceptor, SslFiletype, SslMethod},
-    },
-    std::{fs, marker::PhantomData, pin::Pin},
-    tokio::net::TcpStream,
-    tokio_openssl::SslStream,
-};
-
-impl BufferedSocketStream for SslStream<TcpStream> {}
-
-pub type SslListener = SslListenerRaw<Skyhash2>;
-pub type SslListenerV1 = SslListenerRaw<Skyhash1>;
-
-pub struct SslListenerRaw<P> {
-    pub base: BaseListener,
-    acceptor: SslAcceptor,
-    _marker: PhantomData<P>,
-}
-
-impl<P: ProtocolSpec + 'static> SslListenerRaw<P> {
-    pub fn new_pem_based_ssl_connection(
-        key_file: String,
-        chain_file: String,
-        base: BaseListener,
-        tls_passfile: Option<String>,
-    ) -> SkyResult<SslListenerRaw<P>> {
-        let mut acceptor_builder = SslAcceptor::mozilla_intermediate(SslMethod::tls())?;
-        // cert is the same for both
-        acceptor_builder.set_certificate_chain_file(chain_file)?;
-        if let Some(tls_passfile) = tls_passfile {
-            // first read in the private key
-            let tls_private_key = fs::read(key_file)
-                .map_err(|e| Error::ioerror_extra(e, "reading TLS private key"))?;
-            // read the passphrase because the passphrase file stream was provided
-            let tls_keyfile_stream = fs::read(tls_passfile)
-                .map_err(|e| Error::ioerror_extra(e, "reading TLS password file"))?;
-            // decrypt the private key
-            let pkey = Rsa::private_key_from_pem_passphrase(&tls_private_key, &tls_keyfile_stream)?;
-            let pkey = PKey::from_rsa(pkey)?;
-            // set the private key for the acceptor
-            acceptor_builder.set_private_key(&pkey)?;
-        } else {
-            // no passphrase, needs interactive
-            acceptor_builder.set_private_key_file(key_file, SslFiletype::PEM)?;
-        }
-        Ok(Self {
-            acceptor: acceptor_builder.build(),
-            base,
-            _marker: PhantomData,
-        })
-    }
-    async fn accept(&mut self) -> SkyResult<SslStream<TcpStream>> {
-        let backoff = NetBackoff::new();
-        loop {
-            match self.base.listener.accept().await {
-                // We don't need the bindaddr
-                // We get the encrypted stream which we need to decrypt
-                // by using the acceptor
-                Ok((stream, _)) => {
-                    let ssl = Ssl::new(self.acceptor.context())?;
-                    let mut stream = SslStream::new(ssl, stream)?;
-                    Pin::new(&mut stream).accept().await?;
-                    return Ok(stream);
-                }
-                Err(e) => {
-                    if backoff.should_disconnect() {
-                        // Too many retries, goodbye user
-                        return Err(e.into());
-                    }
-                }
-            }
-            // Wait for the `backoff` duration
-            backoff.spin().await;
-        }
-    }
-    pub async fn run(&mut self) -> IoResult<()> {
-        loop {
-            // Take the permit first, but we won't use it right now
-            // that's why we will forget it
-            self.base.climit.acquire().await.unwrap().forget();
-            /*
-             SECURITY: Ignore any errors that may arise in the accept
-             loop. If we apply the try operator here, we will immediately
-             terminate the run loop causing the entire server to go down.
-             Also, do not log any errors because many connection errors
-             can arise and it will flood the log and might also result
-             in a crash
-            */
-            let stream = skip_loop_err!(self.accept().await);
-            let mut sslhandle = ConnectionHandler::<SslStream<TcpStream>, P>::new(
-                self.base.db.clone(),
-                Connection::new(stream),
-                self.base.auth.clone(),
-                self.base.climit.clone(),
-                self.base.signal.subscribe(),
-                self.base.terminate_tx.clone(),
-            );
-            tokio::spawn(async move {
-                if let Err(e) = sslhandle.run().await {
-                    log::error!("Error: {}", e);
-                }
-            });
-        }
-    }
-}
diff --git a/server/src/diskstore/flock.rs b/server/src/diskstore/flock.rs
deleted file mode 100644
index 3fbc32dd..00000000
--- a/server/src/diskstore/flock.rs
+++ /dev/null
@@ -1,398 +0,0 @@
-/*
- * Created on Fri Apr 16 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # File Locking
-//!
-//! This module provides the `FileLock` struct that can be used for locking and/or unlocking files on
-//! unix-based systems and Windows systems
-
-#![allow(dead_code)] // TODO(@ohsayan): Remove lint or remove offending methods
-
-use std::{
-    fs::{File, OpenOptions},
-    io::{Result, Seek, SeekFrom, Write},
-    path::Path,
-};
-
-#[derive(Debug)]
-/// # File Lock
-/// A file lock object holds a `std::fs::File` that is used to `lock()` and `unlock()` a file with a given
-/// `filename` passed into the `lock()` method. The file lock is **not configured** to drop the file lock when the
-/// object is dropped. The `file` field is essentially used to get the raw file descriptor for passing to
-/// the platform-specific lock/unlock methods.
-///
-/// **Note:** You need to lock a file first using this object before unlocking it!
-///
-/// ## Suggestions
-///
-/// It is always a good idea to attempt a lock release (unlock) explicitly than leaving it to the operating
-/// system. If you manually run unlock, another unlock won't be called to avoid an extra costly (is it?)
-/// syscall; this is achieved with the `unlocked` flag (field) which is set to true when the `unlock()` function
-/// is called.
-///
-pub struct FileLock {
-    file: File,
-    unlocked: bool,
-}
-
-impl FileLock {
-    /// Initialize a new `FileLock` by locking a file
-    ///
-    /// This function will create and lock a file if it doesn't exist or it
-    /// will lock the existing file
-    /// **This will immediately fail if locking fails, i.e it is non-blocking**
-    pub fn lock(filename: impl AsRef<Path>) -> Result<Self> {
-        let file = OpenOptions::new()
-            .create(true)
-            .read(true)
-            .write(true)
-            .open(filename.as_ref())?;
-        Self::_lock(&file)?;
-        Ok(Self {
-            file,
-            unlocked: false,
-        })
-    }
-    /// The internal lock function
-    ///
-    /// This is the function that actually locks the file and is kept separate only for purposes
-    /// of maintainability
-    fn _lock(file: &File) -> Result<()> {
-        __sys::try_lock_ex(file)
-    }
-    /// Unlock the file
-    ///
-    /// This sets the `unlocked` flag to true
-    pub fn unlock(&mut self) -> Result<()> {
-        if !self.unlocked {
-            __sys::unlock_file(&self.file)?;
-            self.unlocked = true;
-            Ok(())
-        } else {
-            Ok(())
-        }
-    }
-    /// Write something to this file
-    pub fn write(&mut self, bytes: &[u8]) -> Result<()> {
-        // empty the file
-        self.file.set_len(0)?;
-        // set the cursor to start
-        self.file.seek(SeekFrom::Start(0))?;
-        // Now write to the file
-        self.file.write_all(bytes)
-    }
-    /// Sync all metadata and flush buffers before returning
-    pub fn fsync(&self) -> Result<()> {
-        self.file.sync_all()
-    }
-    #[cfg(test)]
-    pub fn try_clone(&self) -> Result<Self> {
-        Ok(FileLock {
-            file: __sys::duplicate(&self.file)?,
-            unlocked: self.unlocked,
-        })
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    #[test]
-    fn test_basic_file_lock() {
-        let mut file = FileLock::lock("datalock.bin").unwrap();
-        file.write(&[1, 2, 3]).unwrap();
-        file.unlock().unwrap();
-    }
-    #[test]
-    #[should_panic]
-    fn test_fail_with_two_flocks() {
-        let _file = FileLock::lock("data2.bin").unwrap();
-        let _file2 = FileLock::lock("data2.bin").unwrap();
-        std::fs::remove_file("data2.bin").unwrap();
-    }
-    #[cfg(windows)]
-    #[test]
-    fn test_windows_lock_and_then_unlock() {
-        let mut file = FileLock::lock("data4.bin").unwrap();
-        file.unlock().unwrap();
-        drop(file);
-        let mut file2 = FileLock::lock("data4.bin").unwrap();
-        file2.unlock().unwrap();
-        drop(file2);
-    }
-    #[test]
-    fn test_cloned_lock_writes() {
-        let mut file = FileLock::lock("data5.bin").unwrap();
-        let mut cloned = file.try_clone().unwrap();
-        // this writes 1, 2, 3
-        file.write(&[1, 2, 3]).unwrap();
-        // this will truncate the entire previous file and write 4, 5, 6
-        cloned.write(&[4, 5, 6]).unwrap();
-        drop(cloned);
-        // this will again truncate the entire previous file and write 7, 8
-        file.write(&[7, 8]).unwrap();
-        drop(file);
-        let res = std::fs::read("data5.bin").unwrap();
-        // hence ultimately we'll have 7, 8
-        assert_eq!(res, vec![7, 8]);
-    }
-}
-
-#[cfg(windows)]
-mod __sys {
-    //! # Windows platform-specific file locking
-    //! This module contains methods used by the `FileLock` object in this module to lock and/or
-    //! unlock files.
-
-    use {
-        std::{
-            fs::File,
-            io::{Error, Result},
-            mem,
-            os::windows::io::{AsRawHandle, FromRawHandle},
-            ptr,
-        },
-        winapi::{
-            shared::minwindef::{BOOL, DWORD},
-            um::{
-                fileapi::{LockFileEx, UnlockFile},
-                handleapi::DuplicateHandle,
-                minwinbase::{LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY},
-                processthreadsapi::GetCurrentProcess,
-                winnt::{DUPLICATE_SAME_ACCESS, MAXDWORD},
-            },
-        },
-    };
-
-    /// Obtain an exclusive lock and **block** until we acquire it
-    pub fn lock_ex(file: &File) -> Result<()> {
-        lock_file(file, LOCKFILE_EXCLUSIVE_LOCK)
-    }
-    /// Try to obtain an exclusive lock and **immediately return an error if this is blocking**
-    pub fn try_lock_ex(file: &File) -> Result<()> {
-        lock_file(file, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY)
-    }
-    /// Use the LockFileEx method from Windows fileapi.h to set flags on a file
-    ///
-    /// This is the internal function that is used by `lock_ex` and `try_lock_ex` to lock and/or
-    /// unlock files on Windows platforms.
-    fn lock_file(file: &File, flags: DWORD) -> Result<()> {
-        unsafe {
-            // UNSAFE(@ohsayan): Interfacing with low-level winapi stuff, and we know what's happening here :D
-            let mut overlapped = mem::zeroed();
-            let ret = LockFileEx(
-                file.as_raw_handle(), // handle
-                flags,                // flags
-                0,                    // reserved DWORD, has to be 0
-                MAXDWORD, // nNumberOfBytesToLockLow; low-order (LOWORD) 32-bits of file range to lock
-                MAXDWORD, // nNumberOfBytesToLockHigh; high-order (HIWORD) 32-bits of file range to lock
-                &mut overlapped,
-            );
-            if ret == 0 {
-                Err(Error::last_os_error())
-            } else {
-                Ok(())
-            }
-        }
-    }
-    /// Attempt to unlock a file
-    pub fn unlock_file(file: &File) -> Result<()> {
-        let ret = unsafe {
-            // UNSAFE(@ohsayan): Interfacing with low-level winapi stuff, and we know what's happening here :D
-            UnlockFile(
-                file.as_raw_handle(), // handle
-                0,                    // LOWORD of starting byte offset
-                0,                    // HIWORD of starting byte offset
-                MAXDWORD,             // LOWORD of file range to unlock
-                MAXDWORD,             // HIWORD of file range to unlock
-            )
-        };
-        if ret == 0 {
-            Err(Error::last_os_error())
-        } else {
-            Ok(())
-        }
-    }
-    /// Duplicate a file
-    ///
-    /// The most important part is the `DUPLICATE_SAME_ACCESS` DWORD. It ensures that the cloned file
-    /// has the same permissions as the original file
-    pub fn duplicate(file: &File) -> Result<File> {
-        unsafe {
-            // UNSAFE(@ohsayan): Interfacing with low-level winapi stuff, and we know what's happening here :D
-            let mut handle = ptr::null_mut();
-            let current_process = GetCurrentProcess();
-            let ret = DuplicateHandle(
-                current_process,
-                file.as_raw_handle(),
-                current_process,
-                &mut handle,
-                0,
-                true as BOOL,
-                DUPLICATE_SAME_ACCESS,
-            );
-            if ret == 0 {
-                Err(Error::last_os_error())
-            } else {
-                Ok(File::from_raw_handle(handle))
-            }
-        }
-    }
-}
-
-#[cfg(all(not(target_os = "solaris"), unix))]
-mod __sys {
-    //! # Unix platform-specific file locking
-    //! This module contains methods used by the `FileLock` object in this module to lock and/or
-    //! unlock files.
-    use {
-        libc::c_int,
-        std::{
-            fs::File,
-            io::{Error, Result},
-            os::unix::io::{AsRawFd, FromRawFd},
-        },
-    };
-
-    extern "C" {
-        /// Block and acquire an exclusive lock with `libc`'s `flock`
-        fn lock_exclusive(fd: i32) -> c_int;
-        /// Attempt to acquire an exclusive lock in a non-blocking manner with `libc`'s `flock`
-        fn try_lock_exclusive(fd: i32) -> c_int;
-        /// Attempt to unlock a file with `libc`'s flock
-        fn unlock(fd: i32) -> c_int;
-    }
-    /// Obtain an exclusive lock and **block** until we acquire it
-    pub fn lock_ex(file: &File) -> Result<()> {
-        let errno = unsafe {
-            // UNSAFE(@ohsayan): This is completely fine to do as we've already written the function
-            // ourselves and are very much aware that it is safe
-            lock_exclusive(file.as_raw_fd())
-        };
-        match errno {
-            0 => Ok(()),
-            x => Err(Error::from_raw_os_error(x)),
-        }
-    }
-    /// Try to obtain an exclusive lock and **immediately return an error if this is blocking**
-    pub fn try_lock_ex(file: &File) -> Result<()> {
-        let errno = unsafe {
-            // UNSAFE(@ohsayan): Again, we've written the function ourselves and know what is going on!
-            try_lock_exclusive(file.as_raw_fd())
-        };
-        match errno {
-            0 => Ok(()),
-            x => Err(Error::from_raw_os_error(x)),
-        }
-    }
-    /// Attempt to unlock a file
-    pub fn unlock_file(file: &File) -> Result<()> {
-        let errno = unsafe {
-            // UNSAFE(@ohsayan): Again, we know what's going on here. Good ol' C stuff
-            unlock(file.as_raw_fd())
-        };
-        match errno {
-            0 => Ok(()),
-            x => Err(Error::from_raw_os_error(x)),
-        }
-    }
-    /// Duplicate a file
-    ///
-    /// Good ol' libc dup() calls
-    pub fn duplicate(file: &File) -> Result<File> {
-        unsafe {
-            // UNSAFE(@ohsayan): Completely safe, just that this is FFI
-            let fd = libc::dup(file.as_raw_fd());
-            if fd < 0 {
-                Err(Error::last_os_error())
-            } else {
-                Ok(File::from_raw_fd(fd))
-            }
-        }
-    }
-}
-
-#[cfg(all(target_os = "solaris", unix))]
-mod __sys {
-    //! Solaris doesn't have flock so we'll have to simulate that using fcntl
-    use std::{
-        fs::File,
-        io::{Error, Result},
-        os::unix::io::{AsRawFd, FromRawFd},
-    };
-
-    fn simulate_flock(file: &File, flag: libc::c_int) -> Result<()> {
-        let mut fle = libc::flock {
-            l_whence: 0,
-            l_start: 0,
-            l_len: 0,
-            l_type: 0,
-            l_pad: [0; 4],
-            l_pid: 0,
-            l_sysid: 0,
-        };
-        let (cmd, op) = match flag & libc::LOCK_NB {
-            0 => (libc::F_SETLKW, flag),
-            _ => (libc::F_SETLK, flag & !libc::LOCK_NB),
-        };
-        match op {
-            libc::LOCK_SH => fle.l_type |= libc::F_RDLCK,
-            libc::LOCK_EX => fle.l_type |= libc::F_WRLCK,
-            libc::LOCK_UN => fle.l_type |= libc::F_UNLCK,
-            _ => return Err(Error::from_raw_os_error(libc::EINVAL)),
-        }
-        let ret = unsafe { libc::fcntl(file.as_raw_fd(), cmd, &fle) };
-        match ret {
-            -1 => match Error::last_os_error().raw_os_error() {
-                Some(libc::EACCES) => {
-                    // this is the 'sort of' solaris equivalent to EWOULDBLOCK
-                    Err(Error::from_raw_os_error(libc::EWOULDBLOCK))
-                }
-                _ => return Err(Error::last_os_error()),
-            },
-            _ => Ok(()),
-        }
-    }
-    pub fn lock_ex(file: &File) -> Result<()> {
-        simulate_flock(file, libc::LOCK_EX)
-    }
-    pub fn try_lock_ex(file: &File) -> Result<()> {
-        simulate_flock(file, libc::LOCK_EX | libc::LOCK_NB)
-    }
-    pub fn unlock_file(file: &File) -> Result<()> {
-        simulate_flock(file, libc::LOCK_UN)
-    }
-    pub fn duplicate(file: &File) -> Result<File> {
-        unsafe {
-            let fd = libc::dup(file.as_raw_fd());
-            if fd < 0 {
-                Err(Error::last_os_error())
-            } else {
-                Ok(File::from_raw_fd(fd))
-            }
-        }
-    }
-}
diff --git a/server/src/diskstore/mod.rs b/server/src/diskstore/mod.rs
deleted file mode 100644
index 587a8f26..00000000
--- a/server/src/diskstore/mod.rs
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Created on Wed Aug 05 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! This module provides tools for handling persistently stored data
-
-pub mod flock;
diff --git a/server/src/engine/config.rs b/server/src/engine/config.rs
index c981af5e..93c44647 100644
--- a/server/src/engine/config.rs
+++ b/server/src/engine/config.rs
@@ -51,12 +51,6 @@ impl<T> ModifyGuard<T> {
             modified: false,
         }
     }
-    pub const fn modified(me: &Self) -> bool {
-        me.modified
-    }
-    pub const fn same(me: &Self) -> bool {
-        !me.modified
-    }
 }
 
 impl<T> core::ops::Deref for ModifyGuard<T> {
@@ -80,13 +74,14 @@ impl<T> core::ops::DerefMut for ModifyGuard<T> {
 #[derive(Debug, PartialEq)]
 /// The final configuration that can be used to start up all services
 pub struct Configuration {
-    endpoints: ConfigEndpoint,
-    mode: ConfigMode,
-    system: ConfigSystem,
-    auth: ConfigAuth,
+    pub endpoints: ConfigEndpoint,
+    pub mode: ConfigMode,
+    pub system: ConfigSystem,
+    pub auth: ConfigAuth,
 }
 
 impl Configuration {
+    #[cfg(test)]
     pub fn new(
         endpoints: ConfigEndpoint,
         mode: ConfigMode,
@@ -136,21 +131,29 @@ pub struct ConfigEndpointTcp {
 }
 
 impl ConfigEndpointTcp {
+    #[cfg(test)]
     pub fn new(host: String, port: u16) -> Self {
         Self { host, port }
     }
+    pub fn host(&self) -> &str {
+        self.host.as_ref()
+    }
+    pub fn port(&self) -> u16 {
+        self.port
+    }
 }
 
 #[derive(Debug, PartialEq)]
 /// TLS endpoint configuration
 pub struct ConfigEndpointTls {
-    tcp: ConfigEndpointTcp,
+    pub tcp: ConfigEndpointTcp,
     cert: String,
     private_key: String,
     pkey_pass: String,
 }
 
 impl ConfigEndpointTls {
+    #[cfg(test)]
     pub fn new(
         tcp: ConfigEndpointTcp,
         cert: String,
@@ -164,13 +167,25 @@ impl ConfigEndpointTls {
             pkey_pass,
         }
     }
+    pub fn tcp(&self) -> &ConfigEndpointTcp {
+        &self.tcp
+    }
+    pub fn cert(&self) -> &str {
+        self.cert.as_ref()
+    }
+    pub fn private_key(&self) -> &str {
+        self.private_key.as_ref()
+    }
+    pub fn pkey_pass(&self) -> &str {
+        self.pkey_pass.as_ref()
+    }
 }
 
 /*
     config mode
 */
 
-#[derive(Debug, PartialEq, Deserialize)]
+#[derive(Debug, PartialEq, Deserialize, Clone, Copy)]
 /// The configuration mode
 pub enum ConfigMode {
     /// In [`ConfigMode::Dev`] we're allowed to be more relaxed with settings
@@ -193,6 +208,7 @@ pub struct ConfigSystem {
 }
 
 impl ConfigSystem {
+    #[cfg(test)]
     pub fn new(reliability_system_window: u64) -> Self {
         Self {
             reliability_system_window,
@@ -991,8 +1007,6 @@ fn validate_configuration<CS: ConfigurationSource>(
 /// The return from parsing a configuration file
 #[derive(Debug, PartialEq)]
 pub enum ConfigReturn {
-    /// No configuration was provided. Need to use default
-    Default,
     /// Don't need to do anything. We've output a message and we're good to exit
     HelpMessage(String),
     /// A configuration that we have fully validated was provided
@@ -1014,11 +1028,7 @@ pub(super) fn apply_and_validate<CS: ConfigurationSource>(
     mut args: ParsedRawArgs,
 ) -> RuntimeResult<ConfigReturn> {
     let cfg = apply_config_changes::<CS>(&mut args)?;
-    if ModifyGuard::modified(&cfg) {
-        validate_configuration::<CS>(cfg.val).map(ConfigReturn::Config)
-    } else {
-        Ok(ConfigReturn::Default)
-    }
+    validate_configuration::<CS>(cfg.val).map(ConfigReturn::Config)
 }
 
 /*
@@ -1158,7 +1168,10 @@ pub fn check_configuration() -> RuntimeResult<ConfigReturn> {
                 }
                 None => {
                     // no env args or cli args; we're running on default
-                    return Ok(ConfigReturn::Default);
+                    return Err(ConfigError::new(ConfigErrorKind::ErrorString(
+                        "no configuration provided".into(),
+                    ))
+                    .into());
                 }
             }
         }
diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs
index 72102737..1ba3b8e5 100644
--- a/server/src/engine/core/dml/del.rs
+++ b/server/src/engine/core/dml/del.rs
@@ -33,6 +33,7 @@ use crate::engine::{
     sync,
 };
 
+#[allow(unused)]
 pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) -> QueryResult<()> {
     core::with_model_for_data_update(global, delete.entity(), |model| {
         let g = sync::atm::cpin();
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index 295bface..8b5d7992 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -37,6 +37,7 @@ use crate::engine::{
     sync::atm::cpin,
 };
 
+#[allow(unused)]
 pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> QueryResult<()> {
     core::with_model_for_data_update(global, insert.entity(), |mdl| {
         let irmwd = mdl.intent_write_new_data();
diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
index 5dd4d535..4ee9698c 100644
--- a/server/src/engine/core/dml/sel.rs
+++ b/server/src/engine/core/dml/sel.rs
@@ -34,6 +34,7 @@ use crate::engine::{
     sync,
 };
 
+#[allow(unused)]
 pub fn select_custom<F>(
     global: &impl GlobalInstanceLike,
     mut select: SelectStatement,
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index 5d37fa5a..3be58b52 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -232,6 +232,7 @@ pub fn collect_trace_path() -> Vec<&'static str> {
     ROUTE_TRACE.with(|v| v.borrow().iter().cloned().collect())
 }
 
+#[allow(unused)]
 pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) -> QueryResult<()> {
     core::with_model_for_data_update(global, update.entity(), |mdl| {
         let mut ret = Ok(());
diff --git a/server/src/dbnet/macros.rs b/server/src/engine/core/exec.rs
similarity index 66%
rename from server/src/dbnet/macros.rs
rename to server/src/engine/core/exec.rs
index 64faab26..642960df 100644
--- a/server/src/dbnet/macros.rs
+++ b/server/src/engine/core/exec.rs
@@ -1,5 +1,5 @@
 /*
- * Created on Thu Aug 05 2021
+ * Created on Thu Oct 05 2023
  *
  * This file is a part of Skytable
  * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@@ -7,7 +7,7 @@
  * vision to provide flexibility in data modelling without compromising
  * on performance, queryability or scalability.
  *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published by
@@ -24,11 +24,12 @@
  *
 */
 
-macro_rules! skip_loop_err {
-    ($expr:expr) => {
-        match $expr {
-            Ok(ret) => ret,
-            Err(_) => continue,
-        }
-    };
+use crate::engine::{error::QueryResult, fractal::Global, net::protocol::SQuery};
+
+pub async fn execute_query<'a>(_global: &Global, query: SQuery<'a>) -> QueryResult<()> {
+    let tokens =
+        crate::engine::ql::lex::SecureLexer::new_with_segments(query.query(), query.params())
+            .lex()?;
+    let _ = crate::engine::ql::ast::compile(&tokens, crate::engine::ql::ast::InplaceData::new());
+    todo!()
 }
diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
index f8d8280e..02137ec6 100644
--- a/server/src/engine/core/index/key.rs
+++ b/server/src/engine/core/index/key.rs
@@ -119,6 +119,7 @@ impl PrimaryIndexKey {
 }
 
 impl PrimaryIndexKey {
+    #[cfg(test)]
     pub fn try_from_dc(dc: Datacell) -> Option<Self> {
         Self::check(&dc).then(|| unsafe { Self::new_from_dc(dc) })
     }
@@ -172,20 +173,14 @@ impl PrimaryIndexKey {
         }
     }
     pub unsafe fn raw_clone(&self) -> Self {
-        Self {
-            tag: self.tag,
-            data: {
-                let (qw, nw) = self.data.dwordqn_load_qw_nw();
-                SpecialPaddedWord::new(qw, nw)
-            },
-        }
+        Self::new(self.tag, {
+            let (qw, nw) = self.data.dwordqn_load_qw_nw();
+            SpecialPaddedWord::new(qw, nw)
+        })
     }
     pub fn check(dc: &Datacell) -> bool {
         dc.tag().tag_unique().is_unique()
     }
-    pub fn check_opt(dc: &Option<Datacell>) -> bool {
-        dc.as_ref().map(Self::check).unwrap_or(false)
-    }
     /// ## Safety
     /// If you mess up construction, everything will fall apart
     pub unsafe fn new(tag: TagUnique, data: SpecialPaddedWord) -> Self {
diff --git a/server/src/engine/core/index/mod.rs b/server/src/engine/core/index/mod.rs
index a92b9741..348ee017 100644
--- a/server/src/engine/core/index/mod.rs
+++ b/server/src/engine/core/index/mod.rs
@@ -49,9 +49,6 @@ impl PrimaryIndex {
             data: IndexMTRaw::idx_init(),
         }
     }
-    pub fn remove<'a>(&self, key: Lit<'a>, g: &Guard) -> bool {
-        self.data.mt_delete(&key, g)
-    }
     pub fn select<'a, 'v, 't: 'v, 'g: 't>(&'t self, key: Lit<'a>, g: &'g Guard) -> Option<&'v Row> {
         self.data.mt_get_element(&key, g)
     }
diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
index 5729bfaf..c2e409bf 100644
--- a/server/src/engine/core/index/row.rs
+++ b/server/src/engine/core/index/row.rs
@@ -70,9 +70,6 @@ impl RowData {
     pub fn get_txn_revised(&self) -> DeltaVersion {
         self.txn_revised_data
     }
-    pub fn set_restored_txn_revised(&mut self, new: DeltaVersion) {
-        self.restore_txn_id = new;
-    }
     pub fn get_restored_txn_revised(&self) -> DeltaVersion {
         self.restore_txn_id
     }
@@ -138,14 +135,6 @@ impl Row {
             },
         }
     }
-    pub fn with_data_read<T>(&self, f: impl Fn(&DcFieldIndex) -> T) -> T {
-        let data = self.__rc.data().read();
-        f(&data.fields)
-    }
-    pub fn with_data_write<T>(&self, f: impl Fn(&mut DcFieldIndex) -> T) -> T {
-        let mut data = self.__rc.data().write();
-        f(&mut data.fields)
-    }
     pub fn d_key(&self) -> &PrimaryIndexKey {
         &self.__pk
     }
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index e9fcb39b..7385cf4c 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -25,6 +25,7 @@
 */
 
 pub(in crate::engine) mod dml;
+pub mod exec;
 pub(in crate::engine) mod index;
 pub(in crate::engine) mod model;
 pub(in crate::engine) mod query_meta;
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index 638f3211..d9716a70 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -249,6 +249,7 @@ impl<'a> AlterPlan<'a> {
 }
 
 impl Model {
+    #[allow(unused)]
     pub fn transactional_exec_alter<G: GlobalInstanceLike>(
         global: &G,
         alter: AlterModel,
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index dd5b13b3..0565f481 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -24,6 +24,8 @@
  *
 */
 
+#![allow(unused)]
+
 use {
     super::{Fields, Model},
     crate::{
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index cc84d962..82acd88c 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -139,10 +139,6 @@ impl Model {
             Ok(())
         }
     }
-    pub fn is_empty_atomic(&self) -> bool {
-        // TODO(@ohsayan): change this!
-        true
-    }
     pub fn primary_index(&self) -> &PrimaryIndex {
         &self.data
     }
@@ -204,6 +200,7 @@ impl Model {
 }
 
 impl Model {
+    #[allow(unused)]
     pub fn transactional_exec_create<G: GlobalInstanceLike>(
         global: &G,
         stmt: CreateModel,
@@ -254,6 +251,7 @@ impl Model {
             Ok(())
         })
     }
+    #[allow(unused)]
     pub fn transactional_exec_drop<G: GlobalInstanceLike>(
         global: &G,
         stmt: DropModel,
@@ -314,12 +312,19 @@ static LUT: [(&str, FullTag); 14] = [
     ("list", FullTag::LIST),
 ];
 
+#[cfg(test)]
 pub static TY_BOOL: &str = LUT[0].0;
+#[cfg(test)]
 pub static TY_UINT: [&str; 4] = [LUT[1].0, LUT[2].0, LUT[3].0, LUT[4].0];
+#[cfg(test)]
 pub static TY_SINT: [&str; 4] = [LUT[5].0, LUT[6].0, LUT[7].0, LUT[8].0];
+#[cfg(test)]
 pub static TY_FLOAT: [&str; 2] = [LUT[9].0, LUT[10].0];
+#[cfg(test)]
 pub static TY_BINARY: &str = LUT[11].0;
+#[cfg(test)]
 pub static TY_STRING: &str = LUT[12].0;
+#[cfg(test)]
 pub static TY_LIST: &str = LUT[13].0;
 
 #[derive(Debug, PartialEq, Clone)]
@@ -426,6 +431,7 @@ pub struct Layer {
     tag: FullTag,
 }
 
+#[allow(unused)]
 impl Layer {
     pub const fn bool() -> Self {
         Self::empty(FullTag::BOOL)
@@ -478,10 +484,6 @@ impl Layer {
     pub fn new_empty_props(tag: FullTag) -> Self {
         Self::new(tag)
     }
-    #[inline(always)]
-    fn compute_index(&self, dc: &Datacell) -> usize {
-        self.tag.tag_class().value_word() * (dc.is_null() as usize)
-    }
     const fn new(tag: FullTag) -> Self {
         Self { tag }
     }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 78171379..52013191 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -67,20 +67,13 @@ impl SpaceMeta {
     pub fn dict(&self) -> &RwLock<DictGeneric> {
         &self.props
     }
+    #[cfg(test)]
     pub fn get_env<'a>(rwl: &'a parking_lot::RwLockReadGuard<'a, DictGeneric>) -> &'a DictGeneric {
         match rwl.get(Self::KEY_ENV).unwrap() {
             DictEntryGeneric::Data(_) => unreachable!(),
             DictEntryGeneric::Map(m) => m,
         }
     }
-    pub fn get_env_mut<'a>(
-        rwl: &'a mut parking_lot::RwLockWriteGuard<'a, DictGeneric>,
-    ) -> &'a mut DictGeneric {
-        match rwl.get_mut(Self::KEY_ENV).unwrap() {
-            DictEntryGeneric::Data(_) => unreachable!(),
-            DictEntryGeneric::Map(m) => m,
-        }
-    }
 }
 
 #[derive(Debug)]
@@ -91,14 +84,6 @@ struct ProcedureCreate {
     space: Space,
 }
 
-impl ProcedureCreate {
-    #[inline(always)]
-    /// Define the procedure
-    fn new(space_name: Box<str>, space: Space) -> Self {
-        Self { space_name, space }
-    }
-}
-
 impl Space {
     pub fn _create_model(&self, name: &str, model: Model) -> QueryResult<()> {
         if self
@@ -137,9 +122,11 @@ impl Space {
 }
 
 impl Space {
+    #[cfg(test)]
     pub fn empty() -> Self {
         Space::new_auto(Default::default(), SpaceMeta::with_env(into_dict! {}))
     }
+    #[cfg(test)]
     pub fn empty_with_uuid(uuid: Uuid) -> Self {
         Space::new_with_uuid(Default::default(), SpaceMeta::with_env(into_dict!()), uuid)
     }
@@ -191,6 +178,7 @@ impl Space {
 }
 
 impl Space {
+    #[allow(unused)]
     pub fn transactional_exec_create<G: GlobalInstanceLike>(
         global: &G,
         space: CreateSpace,
@@ -228,6 +216,7 @@ impl Space {
         let _ = wl.st_insert(space_name, space);
         Ok(())
     }
+    #[allow(unused)]
     pub fn transactional_exec_alter<G: GlobalInstanceLike>(
         global: &G,
         AlterSpace {
@@ -266,6 +255,7 @@ impl Space {
             Ok(())
         })
     }
+    #[allow(unused)]
     pub fn transactional_exec_drop<G: GlobalInstanceLike>(
         global: &G,
         DropSpace { space, force: _ }: DropSpace,
diff --git a/server/src/engine/core/tests/ddl_model/mod.rs b/server/src/engine/core/tests/ddl_model/mod.rs
index a40710d7..94df1dda 100644
--- a/server/src/engine/core/tests/ddl_model/mod.rs
+++ b/server/src/engine/core/tests/ddl_model/mod.rs
@@ -71,13 +71,6 @@ pub fn exec_create_new_space(
     exec_create(global, create_stmt, true).map(|_| ())
 }
 
-pub fn exec_create_no_create(
-    global: &impl GlobalInstanceLike,
-    create_stmt: &str,
-) -> QueryResult<()> {
-    exec_create(global, create_stmt, false).map(|_| ())
-}
-
 fn with_space(global: &impl GlobalInstanceLike, space_name: &str, f: impl Fn(&Space)) {
     let rl = global.namespace().spaces().read();
     let space = rl.st_get(space_name).unwrap();
diff --git a/server/src/engine/data/cell.rs b/server/src/engine/data/cell.rs
index 842df8a2..e5b54bd0 100644
--- a/server/src/engine/data/cell.rs
+++ b/server/src/engine/data/cell.rs
@@ -333,13 +333,6 @@ impl Datacell {
     pub fn is_init(&self) -> bool {
         self.init
     }
-    pub fn as_option(&self) -> Option<&Datacell> {
-        if self.init {
-            Some(self)
-        } else {
-            None
-        }
-    }
     unsafe fn load_word<'a, T>(&'a self) -> T
     where
         NativeQword: WordIO<T>,
@@ -349,13 +342,6 @@ impl Datacell {
     unsafe fn _new(tag: CUTag, data: DataRaw, init: bool) -> Self {
         Self { init, tag, data }
     }
-    pub unsafe fn upgrade_from(a: u64, b: usize, tag: CUTag) -> Self {
-        Self {
-            init: true,
-            tag,
-            data: DataRaw::word(DwordQN::dwordqn_store_qw_nw(a, b)),
-        }
-    }
     unsafe fn new(tag: CUTag, data: DataRaw) -> Self {
         Self::_new(tag, data, true)
     }
diff --git a/server/src/engine/data/lit.rs b/server/src/engine/data/lit.rs
index e48e3d6d..ed9cb223 100644
--- a/server/src/engine/data/lit.rs
+++ b/server/src/engine/data/lit.rs
@@ -80,6 +80,7 @@ impl<'a> Lit<'a> {
     }
 }
 
+#[allow(unused)]
 impl<'a> Lit<'a> {
     /// Attempt to read a bool
     pub fn try_bool(&self) -> Option<bool> {
@@ -130,6 +131,7 @@ impl<'a> Lit<'a> {
     }
 }
 
+#[allow(unused)]
 impl<'a> Lit<'a> {
     /// Attempt to read a binary value
     pub fn try_bin(&self) -> Option<&'a [u8]> {
@@ -399,6 +401,16 @@ fn token_array_lt_test() {
         primary_key: tokens[0].as_ir(),
         shorthand: tokens[1].as_ir(),
     };
+    {
+        {
+            let SelectStatement {
+                primary_key,
+                shorthand,
+            } = &select_stmt;
+            let _ = primary_key.as_ir();
+            let _ = shorthand.as_ir();
+        }
+    }
     drop(select_stmt);
     drop(tokens);
 }
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
index d8694b5f..837610dc 100644
--- a/server/src/engine/data/tag.rs
+++ b/server/src/engine/data/tag.rs
@@ -37,12 +37,6 @@ pub enum TagClass {
 }
 
 impl TagClass {
-    pub const fn try_from_raw(v: u8) -> Option<Self> {
-        if v > Self::MAX {
-            return None;
-        }
-        Some(unsafe { Self::from_raw(v) })
-    }
     pub const unsafe fn from_raw(v: u8) -> Self {
         core::mem::transmute(v)
     }
diff --git a/server/src/engine/data/uuid.rs b/server/src/engine/data/uuid.rs
index 819c536c..18207b1a 100644
--- a/server/src/engine/data/uuid.rs
+++ b/server/src/engine/data/uuid.rs
@@ -37,9 +37,6 @@ impl Uuid {
             data: uuid::Uuid::new_v4(),
         }
     }
-    pub fn as_slice(&self) -> &[u8] {
-        self.data.as_bytes()
-    }
     pub fn from_bytes(b: [u8; 16]) -> Self {
         Self {
             data: uuid::Uuid::from_u128_le(u128::from_le_bytes(b)),
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index b0e48b8f..1d0ef66f 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -32,6 +32,7 @@ pub type QueryResult<T> = Result<T, QueryError>;
 /// an enumeration of 'flat' errors that the server actually responds to the client with, since we do not want to send specific information
 /// about anything (as that will be a security hole). The variants correspond with their actual response codes
 #[derive(Debug, Clone, Copy, PartialEq)]
+#[repr(u8)]
 pub enum QueryError {
     /// I/O error
     SysServerError,
@@ -39,8 +40,6 @@ pub enum QueryError {
     SysOutOfMemory,
     /// unknown server error
     SysUnknownError,
-    /// invalid protocol packet
-    NetProtocolIllegalPacket,
     /// something like an integer that randomly has a character to attached to it like `1234q`
     LexInvalidLiteral,
     /// something like an invalid 'string" or a safe string with a bad length etc
@@ -51,14 +50,10 @@ pub enum QueryError {
     QLUnexpectedEndOfStatement,
     /// incorrect syntax for "something"
     QLInvalidSyntax,
-    /// expected a statement keyword found something else
-    QLExpectedStatement,
     /// invalid collection definition definition
     QLInvalidCollectionSyntax,
     /// invalid type definition syntax
     QLInvalidTypeDefinitionSyntax,
-    /// invalid relational expression
-    QLIllegalRelExp,
     /// expected a full entity definition
     QPExpectedEntity,
     /// expected a statement, found something else
@@ -93,8 +88,6 @@ pub enum QueryError {
     QPDmlRowNotFound,
     /// transactional error
     TransactionalError,
-    /// storage subsystem error
-    StorageSubsystemError,
     SysAuthError,
 }
 
@@ -176,8 +169,6 @@ enumerate_err! {
         OnRestoreDataMissing = "txn-payload-conflict-missing",
         /// On restore, a certain item that was expected to match a certain value, has a different value
         OnRestoreDataConflictMismatch = "txn-payload-conflict-mismatch",
-        /// out of memory
-        OutOfMemory = "txn-error-oom",
     }
 }
 
@@ -192,8 +183,6 @@ enumerate_err! {
         HeaderDecodeCorruptedHeader = "header-corrupted",
         /// Expected header values were not matched with the current header
         HeaderDecodeDataMismatch = "header-data-mismatch",
-        /// The time in the [header/dynrec/rtsig] is in the future
-        HeaderTimeConflict = "header-invalid-time",
         // journal
         /// While attempting to handle a basic failure (such as adding a journal entry), the recovery engine ran into an exceptional
         /// situation where it failed to make a necessary repair the log
@@ -202,8 +191,6 @@ enumerate_err! {
         JournalLogEntryCorrupted = "journal-entry-corrupted",
         /// The structure of the journal is corrupted
         JournalCorrupted = "journal-corrupted",
-        /// when restoring the journal, a transactional error (i.e constraint violation) occurred
-        JournalRestoreTxnError = "journal-illegal-data",
         // internal file structures
         /// While attempting to decode a structure in an internal segment of a file, the storage engine ran into a possibly irrecoverable error
         InternalDecodeStructureCorrupted = "structure-decode-corrupted",
diff --git a/server/src/engine/fractal/config.rs b/server/src/engine/fractal/config.rs
index c6dac042..699b401a 100644
--- a/server/src/engine/fractal/config.rs
+++ b/server/src/engine/fractal/config.rs
@@ -27,7 +27,10 @@
 use crate::engine::config::ConfigAuth;
 
 use {
-    crate::engine::error::{QueryError, QueryResult},
+    crate::engine::{
+        config::ConfigMode,
+        error::{QueryError, QueryResult},
+    },
     parking_lot::RwLock,
     std::collections::{hash_map::Entry, HashMap},
 };
@@ -37,23 +40,27 @@ use {
 pub struct SysConfig {
     auth_data: RwLock<SysAuth>,
     host_data: SysHostData,
+    run_mode: ConfigMode,
 }
 
 impl PartialEq for SysConfig {
     fn eq(&self, other: &Self) -> bool {
-        self.host_data == other.host_data && self.auth_data.read().eq(&other.auth_data.read())
+        self.run_mode == other.run_mode
+            && self.host_data == other.host_data
+            && self.auth_data.read().eq(&other.auth_data.read())
     }
 }
 
 impl SysConfig {
     /// Initialize a new system config
-    pub fn new(auth_data: RwLock<SysAuth>, host_data: SysHostData) -> Self {
+    pub fn new(auth_data: RwLock<SysAuth>, host_data: SysHostData, run_mode: ConfigMode) -> Self {
         Self {
             auth_data,
             host_data,
+            run_mode,
         }
     }
-    pub fn new_full(new_auth: ConfigAuth, host_data: SysHostData) -> Self {
+    pub fn new_full(new_auth: ConfigAuth, host_data: SysHostData, run_mode: ConfigMode) -> Self {
         Self::new(
             RwLock::new(SysAuth::new(
                 rcrypt::hash(new_auth.root_key, rcrypt::DEFAULT_COST)
@@ -62,10 +69,11 @@ impl SysConfig {
                 Default::default(),
             )),
             host_data,
+            run_mode,
         )
     }
-    pub fn new_auth(new_auth: ConfigAuth) -> Self {
-        Self::new_full(new_auth, SysHostData::new(0, 0))
+    pub fn new_auth(new_auth: ConfigAuth, run_mode: ConfigMode) -> Self {
+        Self::new_full(new_auth, SysHostData::new(0, 0), run_mode)
     }
     #[cfg(test)]
     /// A test-mode default setting with the root password set to `password12345678`
@@ -78,6 +86,7 @@ impl SysConfig {
                 Default::default(),
             )),
             host_data: SysHostData::new(0, 0),
+            run_mode: ConfigMode::Dev,
         }
     }
     /// Returns a handle to the authentication data
@@ -141,6 +150,7 @@ impl SysAuth {
         Self { root_key, users }
     }
     /// Create a new user with the given details
+    #[allow(unused)]
     pub fn create_new_user(&mut self, username: &str, password: &str) -> QueryResult<()> {
         match self.users.entry(username.into()) {
             Entry::Vacant(ve) => {
@@ -155,7 +165,11 @@ impl SysAuth {
         }
     }
     /// Verify the user with the given details
-    pub fn verify_user(&self, username: &str, password: &str) -> QueryResult<()> {
+    pub fn verify_user<T: AsRef<[u8]> + ?Sized>(
+        &self,
+        username: &str,
+        password: &T,
+    ) -> QueryResult<()> {
         if username == "root" {
             if rcrypt::verify(password, self.root_key()).unwrap() {
                 return Ok(());
diff --git a/server/src/engine/fractal/context.rs b/server/src/engine/fractal/context.rs
index 4d798268..5b46c4f8 100644
--- a/server/src/engine/fractal/context.rs
+++ b/server/src/engine/fractal/context.rs
@@ -24,6 +24,8 @@
  *
 */
 
+#![allow(unused)]
+
 use core::fmt;
 use std::cell::RefCell;
 
@@ -143,17 +145,17 @@ impl From<LocalContext> for LocalCtxInstance {
 exported! {
     pub impl LocalContext {
         // all
-        fn set(origin: Subsystem, msg: impl Into<Dmsg>) { Self::_ctx(|ctx| { ctx.origin = Some(origin); ctx.dmsg = Some(msg.into()) }) }
-        fn test_set(origin: Subsystem, msg: impl Into<Dmsg>) { if_test(|| Self::set(origin, msg)) }
+        fn set(origin: Subsystem, dmsg: impl Into<Dmsg>) { Self::_ctx(|ctx| { ctx.origin = Some(origin); ctx.dmsg = Some(dmsg.into()) }) }
+        fn test_set(origin: Subsystem, dmsg: impl Into<Dmsg>) { if_test(|| Self::set(origin, dmsg)) }
         // dmsg
         /// set a local dmsg
-        fn set_dmsg(msg: impl Into<Dmsg>) { Self::_ctx(|ctx| ctx.dmsg = Some(msg.into())) }
+        fn set_dmsg(dmsg: impl Into<Dmsg>) { Self::_ctx(|ctx| ctx.dmsg = Some(dmsg.into())) }
         /// (only in test) set a local dmsg
-        fn test_set_dmsg(msg: impl Into<Dmsg>) { if_test(|| Self::set_dmsg(msg)) }
+        fn test_set_dmsg(dmsg: impl Into<Dmsg>) { if_test(|| Self::set_dmsg(dmsg)) }
         /// Set a local dmsg iff not already set
-        fn set_dmsg_if_unset(msg: impl Into<Dmsg>) { Self::_ctx(|ctx| { ctx.dmsg.get_or_insert(msg.into()); }) }
+        fn set_dmsg_if_unset(dmsg: impl Into<Dmsg>) { Self::_ctx(|ctx| { ctx.dmsg.get_or_insert(dmsg.into()); }) }
         /// (only in test) set a local dmsg iff not already set
-        fn test_set_dmsg_if_unset(msg: impl Into<Dmsg>) { if_test(|| Self::set_dmsg_if_unset(msg)) }
+        fn test_set_dmsg_if_unset(dmsg: impl Into<Dmsg>) { if_test(|| Self::set_dmsg_if_unset(dmsg)) }
         // origin
         /// set a local origin
         fn set_origin(origin: Subsystem) { Self::_ctx(|ctx| ctx.origin = Some(origin)) }
diff --git a/server/src/engine/fractal/drivers.rs b/server/src/engine/fractal/drivers.rs
index 87ac096a..2a41e2e8 100644
--- a/server/src/engine/fractal/drivers.rs
+++ b/server/src/engine/fractal/drivers.rs
@@ -36,6 +36,7 @@ use {
 
 /// GNS driver
 pub(super) struct FractalGNSDriver<Fs: RawFSInterface> {
+    #[allow(unused)]
     status: util::Status,
     txn_driver: Mutex<GNSTransactionDriverAnyFS<Fs>>,
 }
@@ -54,6 +55,7 @@ impl<Fs: RawFSInterface> FractalGNSDriver<Fs> {
 
 /// Model driver
 pub struct FractalModelDriver<Fs: RawFSInterface> {
+    #[allow(unused)]
     hooks: Arc<FractalModelHooks>,
     batch_driver: Mutex<DataBatchPersistDriver<Fs>>,
 }
@@ -74,18 +76,10 @@ impl<Fs: RawFSInterface> FractalModelDriver<Fs> {
 
 /// Model hooks
 #[derive(Debug)]
-pub struct FractalModelHooks {
-    status: util::Status,
-}
+pub struct FractalModelHooks;
 
 impl FractalModelHooks {
-    #[cfg(test)]
-    pub fn test() -> Self {
-        Self::new()
-    }
     fn new() -> Self {
-        Self {
-            status: util::Status::new_okay(),
-        }
+        Self
     }
 }
diff --git a/server/src/engine/fractal/error.rs b/server/src/engine/fractal/error.rs
index 42e1d741..0998e146 100644
--- a/server/src/engine/fractal/error.rs
+++ b/server/src/engine/fractal/error.rs
@@ -51,14 +51,6 @@ pub struct Error {
 }
 
 impl Error {
-    /// Returns the diagnostic message
-    pub fn dmsg(&self) -> Option<&Dmsg> {
-        self.dmsg.as_ref()
-    }
-    /// Returns the origin
-    pub fn origin(&self) -> Option<Subsystem> {
-        self.origin
-    }
     /// Returns the error kind
     pub fn kind(&self) -> &ErrorKind {
         &self.kind
@@ -90,10 +82,6 @@ impl Error {
     fn with_origin(kind: ErrorKind, origin: Subsystem) -> Self {
         Self::_new(kind, Some(origin), None)
     }
-    /// new error with kind and dmsg
-    fn with_dmsg(kind: ErrorKind, dmsg: impl Into<Dmsg>) -> Self {
-        Self::_new(kind, None, Some(dmsg.into()))
-    }
     /// remove the dmsg from self
     fn remove_dmsg(self) -> Self {
         Self::_new(self.kind, self.origin, None)
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 72ea4638..6f669c2d 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -38,7 +38,10 @@ use {
     std::path::PathBuf,
     tokio::{
         fs,
-        sync::mpsc::{UnboundedReceiver, UnboundedSender},
+        sync::{
+            broadcast,
+            mpsc::{UnboundedReceiver, UnboundedSender},
+        },
         task::JoinHandle,
     },
 };
@@ -63,6 +66,7 @@ impl<T> Task<T> {
 
 /// A general task
 pub enum GenericTask {
+    #[allow(unused)]
     /// Delete a single file
     DeleteFile(PathBuf),
     /// Delete a directory (and all its children)
@@ -119,6 +123,7 @@ impl FractalRTStat {
             per_mdl_delta_max_size: per_model_limit as usize / sizeof!(DataDelta),
         }
     }
+    #[allow(unused)]
     pub(super) fn mem_free_bytes(&self) -> u64 {
         self.mem_free_bytes
     }
@@ -161,28 +166,60 @@ impl FractalMgr {
 }
 
 /// Handles to all the services that fractal needs. These are spawned on the default runtime
-pub struct FractalServiceHandles {
+pub struct FractalHandle {
     pub hp_handle: JoinHandle<()>,
     pub lp_handle: JoinHandle<()>,
 }
 
+#[must_use = "fractal engine won't boot unless you call boot"]
+pub struct FractalBoot {
+    global: super::Global,
+    lp_recv: UnboundedReceiver<Task<GenericTask>>,
+    hp_recv: UnboundedReceiver<Task<CriticalTask>>,
+}
+
+impl FractalBoot {
+    pub(super) fn prepare(
+        global: super::Global,
+        lp_recv: UnboundedReceiver<Task<GenericTask>>,
+        hp_recv: UnboundedReceiver<Task<CriticalTask>>,
+    ) -> Self {
+        Self {
+            global,
+            lp_recv,
+            hp_recv,
+        }
+    }
+    pub fn boot(self, sigterm: &broadcast::Sender<()>) -> FractalHandle {
+        let Self {
+            global,
+            lp_recv: lp_receiver,
+            hp_recv: hp_receiver,
+        } = self;
+        FractalMgr::start_all(global, sigterm, lp_receiver, hp_receiver)
+    }
+}
+
 impl FractalMgr {
     /// Start all background services, and return their handles
     pub(super) fn start_all(
         global: super::Global,
+        sigterm: &broadcast::Sender<()>,
         lp_receiver: UnboundedReceiver<Task<GenericTask>>,
         hp_receiver: UnboundedReceiver<Task<CriticalTask>>,
-    ) -> FractalServiceHandles {
+    ) -> FractalHandle {
         let fractal_mgr = global.get_state().fractal_mgr();
         let global_1 = global.clone();
         let global_2 = global.clone();
+        let sigterm_rx = sigterm.subscribe();
         let hp_handle = tokio::spawn(async move {
-            FractalMgr::hp_executor_svc(fractal_mgr, global_1, hp_receiver).await
+            FractalMgr::hp_executor_svc(fractal_mgr, global_1, hp_receiver, sigterm_rx).await
         });
+        let sigterm_rx = sigterm.subscribe();
         let lp_handle = tokio::spawn(async move {
-            FractalMgr::general_executor_svc(fractal_mgr, global_2, lp_receiver).await
+            FractalMgr::general_executor_svc(fractal_mgr, global_2, lp_receiver, sigterm_rx).await
         });
-        FractalServiceHandles {
+        FractalHandle {
             hp_handle,
             lp_handle,
         }
@@ -199,10 +236,23 @@ impl FractalMgr {
         &'static self,
         global: super::Global,
         mut receiver: UnboundedReceiver<Task<CriticalTask>>,
+        mut sigterm: broadcast::Receiver<()>,
     ) {
         loop {
-            let Some(Task { threshold, task }) = receiver.recv().await else {
-                return; // all handles closed; nothing left to do
+            let Task { threshold, task } = tokio::select! {
+                task = receiver.recv() => {
+                    match task {
+                        Some(t) => t,
+                        None => {
+                            info!("exiting fhp executor service because all tasks closed");
+                            break;
+                        }
+                    }
+                }
+                _ = sigterm.recv() => {
+                    info!("exited fhp executor service");
+                    break;
+                }
             };
             // TODO(@ohsayan): check threshold and update hooks
             match task {
@@ -257,9 +307,14 @@ impl FractalMgr {
         &'static self,
         global: super::Global,
         mut lpq: UnboundedReceiver<Task<GenericTask>>,
+        mut sigterm: broadcast::Receiver<()>,
     ) {
         loop {
             tokio::select! {
+                _ = sigterm.recv() => {
+                    info!("exited flp executor service");
+                    break;
+                },
                 _ = tokio::time::sleep(std::time::Duration::from_secs(Self::GENERAL_EXECUTOR_WINDOW)) => {
                     let mdl_drivers = global.get_state().get_mdl_drivers().read();
                     for (model_id, driver) in mdl_drivers.iter() {
@@ -284,8 +339,12 @@ impl FractalMgr {
                     }
                 }
                 task = lpq.recv() => {
-                    let Some(Task { threshold, task }) = task else {
-                        return;
+                    let Task { threshold, task } = match task {
+                        Some(t) => t,
+                        None => {
+                            info!("exiting flp executor service because all tasks closed");
+                            break;
+                        }
                     };
                     // TODO(@ohsayan): threshold
                     match task {
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index b915a6db..91235402 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -64,15 +64,15 @@ pub type ModelDrivers<Fs> = HashMap<ModelUniqueID, drivers::FractalModelDriver<F
 /// data
 pub struct GlobalStateStart {
     pub global: Global,
-    pub mgr_handles: mgr::FractalServiceHandles,
+    pub boot: mgr::FractalBoot,
 }
 
-/// Enable all drivers and start all engines
+/// Enable all drivers and start all engines (or others that you must start)
 ///
 /// ## Safety
 ///
 /// Must be called iff this is the only thread calling it
-pub unsafe fn enable_and_start_all(
+pub unsafe fn load_and_enable_all(
     gns: GlobalNS,
     config: config::SysConfig,
     gns_driver: GNSTransactionDriverAnyFS<LocalFS>,
@@ -94,7 +94,7 @@ pub unsafe fn enable_and_start_all(
     let token = Global::new();
     GlobalStateStart {
         global: token.clone(),
-        mgr_handles: mgr::FractalMgr::start_all(token, lp_recv, hp_recv),
+        boot: mgr::FractalBoot::prepare(token.clone(), lp_recv, hp_recv),
     }
 }
 
@@ -188,7 +188,7 @@ impl GlobalInstanceLike for Global {
     }
 }
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone)]
 /// A handle to the global state
 pub struct Global(());
 
@@ -230,7 +230,16 @@ impl Global {
         Self::__gref_raw().assume_init_ref()
     }
     pub unsafe fn unload_all(self) {
-        core::ptr::drop_in_place(Self::__gref_raw().as_mut_ptr())
+        // TODO(@ohsayan): handle errors
+        self.namespace_txn_driver()
+            .lock()
+            .__journal_mut()
+            .__append_journal_close_and_close()
+            .unwrap();
+        for (_, driver) in self.get_state().mdl_driver.write().iter_mut() {
+            driver.batch_driver().lock().close().unwrap();
+        }
+        core::ptr::drop_in_place(Self::__gref_raw().as_mut_ptr());
     }
 }
 
diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
index ce5b1e9b..152e057a 100644
--- a/server/src/engine/fractal/test_utils.rs
+++ b/server/src/engine/fractal/test_utils.rs
@@ -50,6 +50,7 @@ pub struct TestGlobal<Fs: RawFSInterface = VirtualFS> {
     gns: GlobalNS,
     hp_queue: RwLock<Vec<Task<CriticalTask>>>,
     lp_queue: RwLock<Vec<Task<GenericTask>>>,
+    #[allow(unused)]
     max_delta_size: usize,
     txn_driver: Mutex<GNSTransactionDriverAnyFS<Fs>>,
     model_drivers: RwLock<HashMap<ModelUniqueID, FractalModelDriver<Fs>>>,
diff --git a/server/src/engine/fractal/util.rs b/server/src/engine/fractal/util.rs
index c3bb986b..323e79d5 100644
--- a/server/src/engine/fractal/util.rs
+++ b/server/src/engine/fractal/util.rs
@@ -24,6 +24,8 @@
  *
 */
 
+#![allow(unused)]
+
 use std::sync::atomic::{AtomicBool, Ordering};
 
 #[derive(Debug)]
diff --git a/server/src/engine/idx/meta/hash.rs b/server/src/engine/idx/meta/hash.rs
index ed93873f..9e884750 100644
--- a/server/src/engine/idx/meta/hash.rs
+++ b/server/src/engine/idx/meta/hash.rs
@@ -26,8 +26,6 @@
 
 use std::hash::{BuildHasher, Hasher};
 
-pub type Hasher32Fx = HasherRawFx<u32>;
-pub type Hasher64Fx = HasherRawFx<u64>;
 pub type HasherNativeFx = HasherRawFx<usize>;
 
 const ROTATE: u32 = 5;
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 449a5318..197881d5 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -43,10 +43,9 @@ pub use stord::iter::IndexSTSeqDllIterOrdKV;
 
 // re-exports
 pub type IndexSTSeqCns<K, V> = stord::IndexSTSeqDll<K, V, stord::config::ConservativeConfig<K, V>>;
+#[cfg(test)]
 pub type IndexSTSeqLib<K, V> = stord::IndexSTSeqDll<K, V, stord::config::LiberalConfig<K, V>>;
-pub type IndexMTRC<K, V> = mtchm::imp::ChmArc<K, V, mtchm::meta::DefConfig>;
 pub type IndexMTRaw<E> = mtchm::imp::Raw<E, mtchm::meta::DefConfig>;
-pub type IndexMTCp<K, V> = mtchm::imp::ChmCopy<K, V, mtchm::meta::DefConfig>;
 pub type IndexST<K, V, S = std::collections::hash_map::RandomState> =
     std::collections::hash_map::HashMap<K, V, S>;
 
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index 7972d2dd..f9b1edc6 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -37,11 +37,10 @@ use {
         idx::{meta::Comparable, AsKeyClone, AsValue, AsValueClone, IndexBaseSpec, MTIndex},
         sync::atm::Guard,
     },
-    std::sync::Arc,
 };
 
 pub type Raw<E, C> = RawTree<E, C>;
-pub type ChmArc<K, V, C> = Raw<Arc<(K, V)>, C>;
+#[cfg(test)]
 pub type ChmCopy<K, V, C> = Raw<(K, V), C>;
 
 impl<E, C: Config> IndexBaseSpec for Raw<E, C> {
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index 0a75cbe1..e1e8bc6b 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -92,18 +92,14 @@ impl CHTRuntimeLog {
         fn hsplit(self: &Self) {
             self.data.split.fetch_add(1, ORD_ACQ);
         } else {
-            void!()
+            ()
         }
         fn hlnode(self: &Self) {
             self.data.hln.fetch_add(1, ORD_ACQ);
         } else {
-            void!()
-        }
-        fn repsplit(self: &Self) -> usize {
-            self.data.split.load(ORD_RLX)
-        } else {
-            0
+            ()
         }
+        #[cfg(test)]
         fn replnode(self: &Self) -> usize {
             self.data.hln.load(ORD_RLX)
         } else {
@@ -195,13 +191,10 @@ impl<T, C: Config> RawTree<T, C> {
         self.l.load(ORD_RLX)
     }
     #[inline(always)]
+    #[cfg(test)]
     fn is_empty(&self) -> bool {
         self.len() == 0
     }
-    #[inline(always)]
-    pub const fn with_state(h: C::HState) -> Self {
-        Self::_new(h)
-    }
 }
 
 impl<T, C: Config> RawTree<T, C> {
@@ -230,6 +223,7 @@ impl<T: TreeElement, C: Config> RawTree<T, C> {
     fn iter_key<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterKey<'t, 'g, 'v, T, C> {
         IterKey::new(self, g)
     }
+    #[allow(unused)]
     fn iter_val<'t, 'g, 'v>(&'t self, g: &'g Guard) -> IterVal<'t, 'g, 'v, T, C> {
         IterVal::new(self, g)
     }
@@ -584,10 +578,6 @@ impl<T: TreeElement, C: Config> RawTree<T, C> {
 
 // low-level methods
 impl<T, C: Config> RawTree<T, C> {
-    // hilarious enough but true, l doesn't affect safety but only creates an incorrect state
-    fn decr_len(&self) {
-        self.decr_len_by(1)
-    }
     fn decr_len_by(&self, by: usize) {
         self.l.fetch_sub(by, ORD_RLX);
     }
diff --git a/server/src/engine/idx/mtchm/patch.rs b/server/src/engine/idx/mtchm/patch.rs
index 23c6d3ee..a92436c1 100644
--- a/server/src/engine/idx/mtchm/patch.rs
+++ b/server/src/engine/idx/mtchm/patch.rs
@@ -32,7 +32,6 @@ use {
 
 /// write mode flag
 pub type WriteFlag = u8;
-pub const WRITEMODE_DELETE: WriteFlag = 0xFF;
 /// fresh
 pub const WRITEMODE_FRESH: WriteFlag = 0b01;
 /// refresh
diff --git a/server/src/engine/idx/mtchm/tests.rs b/server/src/engine/idx/mtchm/tests.rs
index 236b80d0..412818c9 100644
--- a/server/src/engine/idx/mtchm/tests.rs
+++ b/server/src/engine/idx/mtchm/tests.rs
@@ -132,9 +132,7 @@ impl ControlToken {
     }
 }
 
-const TUP_IDENTITY: fn(usize) -> (usize, usize) = |x| (x, x);
 const TUP_INCR: fn(usize) -> (usize, usize) = |x| (x, x + 1);
-const TUP_INCR_TWICE: fn(usize) -> (usize, usize) = |x| (x, x + 2);
 
 fn prepare_distr_data(source_buf: &[StringTup], distr_buf: &mut Vec<Vec<StringTup>>) {
     distr_buf.try_reserve(SPAM_TENANTS).unwrap();
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index 22400507..b2bf424b 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -178,14 +178,6 @@ impl<K, V> IndexSTSeqDllNode<K, V> {
         }
     }
     #[inline(always)]
-    fn alloc_null(k: K, v: V) -> *mut Self {
-        Self::_alloc::<false, false>(Self::new_null(k, v))
-    }
-    #[inline(always)]
-    fn alloc(k: K, v: V, p: *mut Self, n: *mut Self) -> *mut Self {
-        Self::_alloc::<true, true>(Self::new(k, v, p, n))
-    }
-    #[inline(always)]
     unsafe fn _drop(slf: *mut Self) {
         let _ = Box::from_raw(slf);
     }
@@ -224,6 +216,7 @@ pub struct IndexSTSeqDllMetrics {
 
 #[cfg(debug_assertions)]
 impl IndexSTSeqDllMetrics {
+    #[cfg(test)]
     pub const fn raw_f(&self) -> usize {
         self.stat_f
     }
@@ -277,9 +270,6 @@ impl<K, V, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
 }
 
 impl<K, V, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
-    pub fn new() -> Self {
-        Self::with_hasher(C::Hasher::default())
-    }
     pub fn with_capacity(cap: usize) -> Self {
         Self::with_capacity_and_hasher(cap, C::Hasher::default())
     }
diff --git a/server/src/engine/idx/tests.rs b/server/src/engine/idx/tests.rs
index 6bbee40c..b552e5f4 100644
--- a/server/src/engine/idx/tests.rs
+++ b/server/src/engine/idx/tests.rs
@@ -28,16 +28,11 @@ use super::*;
 
 mod idx_st_seq_dll {
     use super::{IndexBaseSpec, IndexSTSeqLib, STIndex, STIndexSeq};
-    use rand::{distributions::Alphanumeric, Rng};
 
     #[cfg(not(miri))]
     const SPAM_CNT: usize = 131_072;
     #[cfg(miri)]
     const SPAM_CNT: usize = 128;
-    #[cfg(not(miri))]
-    const SPAM_SIZE: usize = 128;
-    #[cfg(miri)]
-    const SPAM_SIZE: usize = 4;
 
     type Index = IndexSTSeqLib<String, String>;
 
@@ -59,12 +54,6 @@ mod idx_st_seq_dll {
     fn s(s: &str) -> String {
         s.to_owned()
     }
-    fn ranstr(rand: &mut impl Rng) -> String {
-        rand.sample_iter(Alphanumeric)
-            .take(SPAM_SIZE)
-            .map(char::from)
-            .collect()
-    }
     #[test]
     fn empty_drop() {
         let idx = Index::idx_init();
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index 7f101aa2..0256eb7a 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -39,18 +39,6 @@ macro_rules! extract {
     };
 }
 
-#[cfg(test)]
-#[allow(unused_macros)]
-macro_rules! extract_safe {
-    ($src:expr, $what:pat => $ret:expr) => {
-        if let $what = $src {
-            $ret
-        } else {
-            panic!("expected one {}, found {:?}", stringify!($what), $src);
-        }
-    };
-}
-
 #[cfg(test)]
 macro_rules! multi_assert_eq {
     ($($lhs:expr),* => $rhs:expr) => {
@@ -70,14 +58,10 @@ macro_rules! direct_from {
     };
 }
 
-#[allow(unused_macros)]
-macro_rules! assertions {
-    ($($assert:expr),*$(,)?) => {$(const _:()=::core::assert!($assert);)*}
-}
-
 macro_rules! flags {
     ($(#[$attr:meta])* $vis:vis struct $group:ident: $ty:ty { $($const:ident = $expr:expr),+ $(,)?}) => (
         $(#[$attr])* #[repr(transparent)] $vis struct $group {r#const: $ty}
+        #[allow(unused)]
         impl $group {
             $(pub const $const: Self = Self { r#const: $expr };)*
             #[inline(always)] pub const fn d(&self) -> $ty { self.r#const }
@@ -104,7 +88,6 @@ macro_rules! flags {
     );
 }
 
-#[allow(unused_macros)]
 macro_rules! union {
     ($(#[$attr:meta])* $vis:vis union $name:ident $tail:tt) => (union!(@parse [$(#[$attr])* $vis union $name] [] $tail););
     ($(#[$attr:meta])* $vis:vis union $name:ident<$($lt:lifetime),*> $tail:tt) => (union!(@parse [$(#[$attr])* $vis union $name<$($lt),*>] [] $tail););
@@ -120,27 +103,20 @@ macro_rules! union {
 }
 
 macro_rules! dbgfn {
-    ($($vis:vis fn $fn:ident($($arg:ident: $argty:ty),* $(,)?) $(-> $ret:ty)? $block:block)*) => {
-        $(dbgfn!(@int $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block {panic!("called dbg symbol in non-dbg build")});)*
+    ($($(#[$attr:meta])* $vis:vis fn $fn:ident($($arg:ident: $argty:ty),* $(,)?) $(-> $ret:ty)? $block:block)*) => {
+        $(dbgfn!(@int $(#[$attr])* $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block {panic!("called dbg symbol in non-dbg build")});)*
     };
-    ($($vis:vis fn $fn:ident($($arg:ident: $argty:ty),* $(,)?) $(-> $ret:ty)? $block:block else $block_b:block)*) => {
-        $(dbgfn!(@int $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block $block_b);)*
+    ($($(#[$attr:meta])* $vis:vis fn $fn:ident($($arg:ident: $argty:ty),* $(,)?) $(-> $ret:ty)? $block:block else $block_b:block)*) => {
+        $(dbgfn!(@int $(#[$attr])*  $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block $block_b);)*
     };
-    (@int $vis:vis fn $fn:ident($($arg:ident: $argty:ty),* $(,)?) $(-> $ret:ty)? $block_a:block $block_b:block) => {
+    (@int $(#[$attr:meta])* $vis:vis fn $fn:ident($($arg:ident: $argty:ty),* $(,)?) $(-> $ret:ty)? $block_a:block $block_b:block) => {
         #[cfg(debug_assertions)]
-        $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block_a
+        $(#[$attr])* $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block_a
         #[cfg(not(debug_assertions))]
-        $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block_b
+        $(#[$attr])* $vis fn $fn($($arg: $argty),*) $(-> $ret)? $block_b
     }
 }
 
-#[allow(unused_macros)]
-macro_rules! void {
-    () => {
-        ()
-    };
-}
-
 /// Convert all the KV pairs into an iterator and then turn it into an appropriate collection
 /// (inferred).
 ///
diff --git a/server/src/engine/mem/scanner.rs b/server/src/engine/mem/scanner.rs
index f761b340..e6fb1bfb 100644
--- a/server/src/engine/mem/scanner.rs
+++ b/server/src/engine/mem/scanner.rs
@@ -61,10 +61,6 @@ impl<'a, T> Scanner<'a, T> {
     pub const fn remaining(&self) -> usize {
         self.buffer_len() - self.__cursor
     }
-    /// Returns the number of items consumed by the scanner
-    pub const fn consumed(&self) -> usize {
-        self.__cursor
-    }
     /// Returns the current cursor position
     pub const fn cursor(&self) -> usize {
         self.__cursor
@@ -185,17 +181,6 @@ impl<'a> Scanner<'a, u8> {
             })
         }
     }
-    /// Attempt to parse the next block
-    pub fn try_next_block<const N: usize>(&mut self) -> Option<[u8; N]> {
-        if self.has_left(N) {
-            Some(unsafe {
-                // UNSAFE(@ohsayan): +remaining check
-                self.next_chunk()
-            })
-        } else {
-            None
-        }
-    }
     /// Attempt to parse the next block (variable)
     pub fn try_next_variable_block(&mut self, len: usize) -> Option<&'a [u8]> {
         if self.has_left(len) {
@@ -239,9 +224,6 @@ impl<'a> Scanner<'a, u8> {
     ) -> ScannerDecodeResult<u64> {
         self.try_next_ascii_u64_lf_separated_with_result_or::<true>()
     }
-    pub fn try_next_ascii_u64_lf_separated_with_result(&mut self) -> ScannerDecodeResult<u64> {
-        self.try_next_ascii_u64_lf_separated_with_result_or::<false>()
-    }
     pub fn try_next_ascii_u64_lf_separated_with_result_or<const RESTORE_CURSOR: bool>(
         &mut self,
     ) -> ScannerDecodeResult<u64> {
@@ -279,9 +261,6 @@ impl<'a> Scanner<'a, u8> {
     pub fn try_next_ascii_u64_lf_separated_or_restore_cursor(&mut self) -> Option<u64> {
         self.try_next_ascii_u64_lf_separated_or::<true>()
     }
-    pub fn try_next_ascii_u64_lf_separated(&mut self) -> Option<u64> {
-        self.try_next_ascii_u64_lf_separated_or::<false>()
-    }
     pub fn try_next_ascii_u64_lf_separated_or<const RESTORE_CURSOR: bool>(
         &mut self,
     ) -> Option<u64> {
diff --git a/server/src/engine/mem/stackop.rs b/server/src/engine/mem/stackop.rs
index 11a6fa63..280d52d6 100644
--- a/server/src/engine/mem/stackop.rs
+++ b/server/src/engine/mem/stackop.rs
@@ -29,6 +29,7 @@ pub struct ByteStack<const N: usize> {
     array: [u8; N],
 }
 
+#[allow(dead_code)]
 impl<const N: usize> ByteStack<N> {
     #[inline(always)]
     pub const fn data_copy(&self) -> [u8; N] {
diff --git a/server/src/engine/mem/uarray.rs b/server/src/engine/mem/uarray.rs
index 8d64e86f..6e71c2f1 100644
--- a/server/src/engine/mem/uarray.rs
+++ b/server/src/engine/mem/uarray.rs
@@ -53,10 +53,6 @@ impl<const N: usize, T> UArray<N, T> {
         self.l
     }
     #[inline(always)]
-    pub const fn capacity(&self) -> usize {
-        N
-    }
-    #[inline(always)]
     pub const fn is_empty(&self) -> bool {
         self.len() == 0
     }
@@ -70,6 +66,7 @@ impl<const N: usize, T> UArray<N, T> {
             self.push_unchecked(v);
         }
     }
+    #[allow(unused)]
     pub fn remove(&mut self, idx: usize) -> T {
         if idx >= self.len() {
             panic!("out of range. idx is `{idx}` but len is `{}`", self.len());
diff --git a/server/src/engine/mem/vinline.rs b/server/src/engine/mem/vinline.rs
index 191bad18..43a756c0 100644
--- a/server/src/engine/mem/vinline.rs
+++ b/server/src/engine/mem/vinline.rs
@@ -73,6 +73,7 @@ impl<const N: usize, T> VInline<N, T> {
         }
     }
     #[inline(always)]
+    #[allow(unused)]
     pub fn clear(&mut self) {
         unsafe {
             // UNSAFE(@ohsayan): as_slice_mut will always give a valid ptr
@@ -81,6 +82,7 @@ impl<const N: usize, T> VInline<N, T> {
         self.l = 0;
     }
     #[inline(always)]
+    #[allow(unused)]
     pub fn remove(&mut self, idx: usize) -> T {
         if !(idx < self.len()) {
             panic!("index out of range");
@@ -91,12 +93,14 @@ impl<const N: usize, T> VInline<N, T> {
         }
     }
     #[inline(always)]
+    #[allow(unused)]
     pub fn remove_compact(&mut self, idx: usize) -> T {
         let r = self.remove(idx);
         self.optimize_capacity();
         r
     }
     #[inline(always)]
+    #[allow(unused)]
     /// SAFETY: `idx` must be < l
     unsafe fn remove_unchecked(&mut self, idx: usize) -> T {
         // UNSAFE(@ohsayan): idx is in range
@@ -122,6 +126,7 @@ impl<const N: usize, T> VInline<N, T> {
     const _ENSURE_ALIGN: () =
         debug_assert!(mem::align_of::<Vec<String>>() == mem::align_of::<VInline<N, String>>());
     #[inline(always)]
+    #[cfg(test)]
     pub fn on_heap(&self) -> bool {
         self.c > N
     }
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index a7f43885..30b0130a 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -24,8 +24,6 @@
  *
 */
 
-#![allow(dead_code)]
-
 #[macro_use]
 mod macros;
 mod config;
@@ -44,8 +42,184 @@ mod txn;
 #[cfg(test)]
 mod tests;
 
-use error::RuntimeResult;
+use {
+    self::{
+        config::{ConfigEndpoint, ConfigEndpointTls, ConfigMode, ConfigReturn, Configuration},
+        error::RuntimeResult,
+        fractal::context::{self, Subsystem},
+        storage::v1::{
+            loader::{self, SEInitState},
+            sysdb::{self, SystemStoreInit},
+            LocalFS,
+        },
+    },
+    crate::util::os::TerminationSignal,
+    std::process::exit,
+    tokio::sync::broadcast,
+};
+
+/// Initialize all drivers, load all data
+///
+/// WARN: Must be in [`tokio::runtime::Runtime`] context!
+pub fn load_all() -> RuntimeResult<(Configuration, fractal::GlobalStateStart)> {
+    // load configuration
+    info!("checking configuration ...");
+    context::set(Subsystem::Init, "loading configuration");
+    let config = match config::check_configuration()? {
+        ConfigReturn::Config(cfg) => cfg,
+        ConfigReturn::HelpMessage(msg) => {
+            eprintln!("{msg}");
+            exit(0x00);
+        }
+    };
+    if config.mode == ConfigMode::Dev {
+        warn!("running in dev mode");
+    }
+    // restore system database
+    info!("loading system database ...");
+    context::set_dmsg("loading system database");
+    let SystemStoreInit { store, state } =
+        sysdb::open_system_database::<LocalFS>(config.auth.clone(), config.mode)?;
+    let sysdb_is_new = state.is_created();
+    if state.is_existing_updated_root() {
+        warn!("the root account was updated");
+    }
+    // now load all data
+    if sysdb_is_new {
+        info!("initializing storage engine ...");
+    } else {
+        info!("reinitializing storage engine...");
+    }
+    context::set_dmsg("restoring data");
+    let SEInitState {
+        txn_driver,
+        model_drivers,
+        gns,
+    } = loader::SEInitState::try_init(sysdb_is_new)?;
+    let global = unsafe {
+        // UNSAFE(@ohsayan): this is the only entrypoint
+        fractal::load_and_enable_all(gns, store, txn_driver, model_drivers)
+    };
+    Ok((config, global))
+}
+
+enum EndpointListeners {
+    Insecure(net::Listener),
+    Secure {
+        listener: net::Listener,
+        ssl: openssl::ssl::SslAcceptor,
+    },
+    Multi {
+        tcp: net::Listener,
+        tls: net::Listener,
+        ssl: openssl::ssl::SslAcceptor,
+    },
+}
+
+impl EndpointListeners {
+    async fn listen(&mut self) {
+        match self {
+            Self::Insecure(l) => l.listen_tcp().await,
+            Self::Secure { listener, ssl } => listener.listen_tls(ssl).await,
+            Self::Multi { tcp, tls, ssl } => {
+                tokio::join!(tcp.listen_tcp(), tls.listen_tls(ssl));
+            }
+        }
+    }
+    async fn finish(self) {
+        match self {
+            Self::Insecure(l) | Self::Secure { listener: l, .. } => l.terminate().await,
+            Self::Multi { tcp, tls, .. } => {
+                tokio::join!(tcp.terminate(), tls.terminate());
+            }
+        }
+    }
+}
+
+pub async fn start(
+    Configuration { endpoints, .. }: Configuration,
+    fractal::GlobalStateStart { global, boot }: fractal::GlobalStateStart,
+) -> RuntimeResult<()> {
+    // bind termination signal
+    context::set(Subsystem::Init, "binding system signals");
+    let termsig = TerminationSignal::init()?;
+    // create our system-wide channel
+    let (signal, _) = broadcast::channel::<()>(1);
+    // start our services
+    context::set_dmsg("starting fractal engine");
+    let fractal_handle = boot.boot(&signal);
+    // create our server
+    context::set(Subsystem::Network, "initializing endpoints");
+    let str;
+    let mut endpoint_handles = match &endpoints {
+        ConfigEndpoint::Secure(ConfigEndpointTls { tcp, .. }) | ConfigEndpoint::Insecure(tcp) => {
+            let listener =
+                net::Listener::new(tcp.host(), tcp.port(), global.clone(), signal.clone()).await?;
+            if let ConfigEndpoint::Secure(s) = endpoints {
+                context::set_dmsg("initializing TLS");
+                let acceptor = net::Listener::init_tls(s.cert(), s.private_key(), s.pkey_pass())?;
+                str = format!("listening on tls@{}:{}", s.tcp().host(), s.tcp().port());
+                EndpointListeners::Secure {
+                    listener,
+                    ssl: acceptor,
+                }
+            } else {
+                str = format!("listening on tcp@{}:{}", tcp.host(), tcp.port());
+                EndpointListeners::Insecure(listener)
+            }
+        }
+        ConfigEndpoint::Multi(insecure_ep, secure_ep) => {
+            let tcp_listener =
+                net::Listener::new_cfg(insecure_ep, global.clone(), signal.clone()).await?;
+            let tls_listener =
+                net::Listener::new_cfg(secure_ep.tcp(), global.clone(), signal.clone()).await?;
+            context::set_dmsg("initializing TLS");
+            let acceptor = net::Listener::init_tls(
+                secure_ep.cert(),
+                secure_ep.private_key(),
+                secure_ep.pkey_pass(),
+            )?;
+            str = format!(
+                "listening on tcp@{}:{} and tls@{}:{}",
+                insecure_ep.host(),
+                insecure_ep.port(),
+                secure_ep.tcp().host(),
+                secure_ep.tcp().port()
+            );
+            EndpointListeners::Multi {
+                tcp: tcp_listener,
+                tls: tls_listener,
+                ssl: acceptor,
+            }
+        }
+    };
+    info!("{str}");
+    tokio::select! {
+        _ = endpoint_handles.listen() => {}
+        _ = termsig => {
+            info!("received terminate signal");
+        }
+    }
+    drop(signal);
+    info!("waiting for inflight tasks to complete ...");
+    endpoint_handles.finish().await;
+    info!("waiting for fractal engine to exit ...");
+    let (hp_handle, lp_handle) = tokio::join!(fractal_handle.hp_handle, fractal_handle.lp_handle);
+    match (hp_handle, lp_handle) {
+        (Err(e1), Err(e2)) => {
+            error!("error while terminating fhp-executor and lhp-executor: {e1};{e2}")
+        }
+        (Err(e), _) => error!("error while terminating fhp-executor: {e}"),
+        (_, Err(e)) => error!("error while terminating flp-executor: {e}"),
+        _ => {}
+    }
+    info!("all services have exited");
+    Ok(())
+}
 
-pub fn load_all() -> RuntimeResult<fractal::Global> {
-    todo!()
+pub fn finish(g: fractal::Global) {
+    unsafe {
+        // UNSAFE(@ohsayan): the only thing we do before exit
+        g.unload_all();
+    }
 }
diff --git a/server/src/engine/net/mod.rs b/server/src/engine/net/mod.rs
index eeebfe71..496a6730 100644
--- a/server/src/engine/net/mod.rs
+++ b/server/src/engine/net/mod.rs
@@ -24,7 +24,9 @@
  *
 */
 
-mod protocol;
+use super::config::ConfigEndpointTcp;
+
+pub mod protocol;
 
 use {
     crate::engine::{error::RuntimeResult, fractal::error::ErrorContext, fractal::Global},
@@ -53,6 +55,12 @@ const CLIMIT: usize = 50000;
 
 static CLIM: Semaphore = Semaphore::const_new(CLIMIT);
 
+enum QueryLoopResult {
+    Fin,
+    Rst,
+    HSFailed,
+}
+
 /*
     socket definitions
 */
@@ -60,11 +68,6 @@ static CLIM: Semaphore = Semaphore::const_new(CLIMIT);
 impl Socket for TcpStream {}
 impl Socket for SslStream<TcpStream> {}
 
-pub enum QLoopReturn {
-    Fin,
-    ConnectionRst,
-}
-
 struct NetBackoff {
     at: Cell<u8>,
 }
@@ -84,6 +87,9 @@ impl NetBackoff {
     }
 }
 
+unsafe impl Send for NetBackoff {}
+unsafe impl Sync for NetBackoff {}
+
 /*
     listener
 */
@@ -121,9 +127,20 @@ impl<S: Socket> ConnectionHandler<S> {
         } = self;
         loop {
             tokio::select! {
-                _ = protocol::query_loop(socket, buffer, global) => {},
-                _ = self.sig_terminate.recv() => {
+                ret = protocol::query_loop(socket, buffer, global) => {
+                    match ret {
+                        Ok(QueryLoopResult::Fin) => return Ok(()),
+                        Ok(QueryLoopResult::Rst) => error!("connection reset while talking to client"),
+                        Ok(QueryLoopResult::HSFailed) => error!("failed to handshake with client"),
+                        Err(e) => {
+                            error!("error while handling connection: {e}");
+                            return Err(e);
+                        }
+                    }
                     return Ok(())
+                },
+                _ = self.sig_terminate.recv() => {
+                    return Ok(());
                 }
             }
         }
@@ -140,15 +157,23 @@ pub struct Listener {
 }
 
 impl Listener {
+    pub async fn new_cfg(
+        tcp: &ConfigEndpointTcp,
+        global: Global,
+        sig_shutdown: broadcast::Sender<()>,
+    ) -> RuntimeResult<Self> {
+        Self::new(tcp.host(), tcp.port(), global, sig_shutdown).await
+    }
     pub async fn new(
-        binaddr: &str,
+        host: &str,
+        port: u16,
         global: Global,
         sig_shutdown: broadcast::Sender<()>,
     ) -> RuntimeResult<Self> {
         let (sig_inflight, sig_inflight_wait) = mpsc::channel(1);
-        let listener = TcpListener::bind(binaddr)
+        let listener = TcpListener::bind((host, port))
             .await
-            .set_dmsg(format!("failed to bind to port `{binaddr}`"))?;
+            .set_dmsg(format!("failed to bind to port `{host}:{port}`"))?;
         Ok(Self {
             global,
             listener,
@@ -183,7 +208,7 @@ impl Listener {
             backoff.spin().await;
         }
     }
-    async fn listen_tcp(&mut self) -> IoResult<()> {
+    pub async fn listen_tcp(&mut self) {
         loop {
             // acquire a permit
             let permit = CLIM.acquire().await.unwrap();
@@ -199,7 +224,7 @@ impl Listener {
             };
             let mut handler = ConnectionHandler::new(
                 stream,
-                self.global,
+                self.global.clone(),
                 self.sig_shutdown.subscribe(),
                 self.sig_inflight.clone(),
             );
@@ -212,12 +237,11 @@ impl Listener {
             drop(permit);
         }
     }
-    async fn listen_tls(
-        self: &mut Self,
-        tls_cert: String,
-        tls_priv_key: String,
-        tls_key_password: String,
-    ) -> RuntimeResult<()> {
+    pub fn init_tls(
+        tls_cert: &str,
+        tls_priv_key: &str,
+        tls_key_password: &str,
+    ) -> RuntimeResult<SslAcceptor> {
         let build_acceptor = || {
             let cert = X509::from_pem(tls_cert.as_bytes())?;
             let priv_key = PKey::private_key_from_pem_passphrase(
@@ -231,6 +255,9 @@ impl Listener {
             Ok::<_, openssl::error::ErrorStack>(builder.build())
         };
         let acceptor = build_acceptor().set_dmsg("failed to initialize TLS socket")?;
+        Ok(acceptor)
+    }
+    pub async fn listen_tls(&mut self, acceptor: &SslAcceptor) {
         loop {
             let stream = async {
                 let (stream, _) = self.accept().await?;
@@ -251,7 +278,7 @@ impl Listener {
             };
             let mut handler = ConnectionHandler::new(
                 stream,
-                self.global,
+                self.global.clone(),
                 self.sig_shutdown.subscribe(),
                 self.sig_inflight.clone(),
             );
diff --git a/server/src/engine/net/protocol/exchange.rs b/server/src/engine/net/protocol/exchange.rs
index 739757b7..5bfe1904 100644
--- a/server/src/engine/net/protocol/exchange.rs
+++ b/server/src/engine/net/protocol/exchange.rs
@@ -79,15 +79,15 @@ pub fn resume<'a>(
     scanner: &mut BufferedScanner<'a>,
     state: QueryTimeExchangeState,
 ) -> QueryTimeExchangeResult<'a> {
-    if cfg!(debug_assertions) {
-        if !scanner.has_left(EXCHANGE_MIN_SIZE) {
-            return STATE_READ_INITIAL;
-        }
-    } else {
-        assert!(scanner.has_left(EXCHANGE_MIN_SIZE));
-    }
     match state {
         QueryTimeExchangeState::Initial => {
+            if cfg!(debug_assertions) {
+                if !scanner.has_left(EXCHANGE_MIN_SIZE) {
+                    return STATE_READ_INITIAL;
+                }
+            } else {
+                assert!(scanner.has_left(EXCHANGE_MIN_SIZE));
+            }
             // attempt to read atleast one byte
             if cfg!(debug_assertions) {
                 match scanner.try_next_byte() {
@@ -174,21 +174,23 @@ impl<'a> SQuery<'a> {
     pub(super) fn new(q: &'a [u8], q_window: usize) -> Self {
         Self { q, q_window }
     }
-    pub fn q(&self) -> &'a [u8] {
+    pub fn payload(&self) -> &'a [u8] {
         self.q
     }
     pub fn q_window(&self) -> usize {
         self.q_window
     }
     pub fn query(&self) -> &'a [u8] {
-        &self.q[..self.q_window]
+        &self.payload()[..self.q_window()]
+    }
+    pub fn params(&self) -> &'a [u8] {
+        &self.payload()[self.q_window()..]
     }
+    #[cfg(test)]
     pub fn query_str(&self) -> Option<&'a str> {
         core::str::from_utf8(self.query()).ok()
     }
-    pub fn params(&self) -> &'a [u8] {
-        &self.q[self.q_window..]
-    }
+    #[cfg(test)]
     pub fn params_str(&self) -> Option<&'a str> {
         core::str::from_utf8(self.params()).ok()
     }
@@ -295,6 +297,7 @@ impl<'a> SQuery<'a> {
     }
 }
 
+#[cfg(test)]
 pub(super) fn create_simple_query<const N: usize>(query: &str, params: [&str; N]) -> Vec<u8> {
     let mut buf = vec![];
     let query_size_as_string = query.len().to_string();
diff --git a/server/src/engine/net/protocol/handshake.rs b/server/src/engine/net/protocol/handshake.rs
index 3b1cc2e4..c2b28e3f 100644
--- a/server/src/engine/net/protocol/handshake.rs
+++ b/server/src/engine/net/protocol/handshake.rs
@@ -90,8 +90,7 @@ pub enum QueryMode {
 #[repr(u8)]
 /// the authentication mode
 pub enum AuthMode {
-    Anonymous = 0,
-    Password = 1,
+    Password = 0,
 }
 
 impl AuthMode {
@@ -101,7 +100,6 @@ impl AuthMode {
     /// returns the minimum number of metadata bytes need to parse the payload for this auth mode
     const fn min_payload_bytes(&self) -> usize {
         match self {
-            Self::Anonymous => 1,
             Self::Password => 4,
         }
     }
@@ -173,6 +171,21 @@ impl CHandshakeStatic {
             auth_mode,
         }
     }
+    pub fn hs_version(&self) -> HandshakeVersion {
+        self.hs_version
+    }
+    pub fn protocol(&self) -> ProtocolVersion {
+        self.protocol
+    }
+    pub fn exchange_mode(&self) -> DataExchangeMode {
+        self.exchange_mode
+    }
+    pub fn query_mode(&self) -> QueryMode {
+        self.query_mode
+    }
+    pub fn auth_mode(&self) -> AuthMode {
+        self.auth_mode
+    }
 }
 
 /// handshake authentication
@@ -187,6 +200,12 @@ impl<'a> CHandshakeAuth<'a> {
     pub fn new(username: &'a [u8], password: &'a [u8]) -> Self {
         Self { username, password }
     }
+    pub fn username(&self) -> &[u8] {
+        self.username
+    }
+    pub fn password(&self) -> &[u8] {
+        self.password
+    }
 }
 
 #[derive(Debug, PartialEq)]
@@ -210,13 +229,13 @@ pub struct CHandshake<'a> {
     /// the static segment of the handshake
     hs_static: CHandshakeStatic,
     /// the auth section of the dynamic segment of the handshake
-    hs_auth: Option<CHandshakeAuth<'a>>,
+    hs_auth: CHandshakeAuth<'a>,
 }
 
 impl<'a> CHandshake<'a> {
     pub const INITIAL_READ: usize = 6;
     const CLIENT_HELLO: u8 = b'H';
-    pub fn new(hs_static: CHandshakeStatic, hs_auth: Option<CHandshakeAuth<'a>>) -> Self {
+    pub fn new(hs_static: CHandshakeStatic, hs_auth: CHandshakeAuth<'a>) -> Self {
         Self { hs_static, hs_auth }
     }
     /// Resume handshake with the given state and buffer
@@ -243,6 +262,12 @@ impl<'a> CHandshake<'a> {
             } => Self::resume_at_variable_block_payload(scanner, static_hs, uname_l, pwd_l),
         }
     }
+    pub fn hs_static(&self) -> CHandshakeStatic {
+        self.hs_static
+    }
+    pub fn hs_auth(&self) -> &CHandshakeAuth<'a> {
+        &self.hs_auth
+    }
 }
 
 impl<'a> CHandshake<'a> {
@@ -319,10 +344,7 @@ impl<'a> CHandshake<'a> {
                 // UNSAFE(@ohsayan): we just checked buffered size
                 let uname = scanner.next_chunk_variable(uname_l);
                 let pwd = scanner.next_chunk_variable(pwd_l);
-                HandshakeResult::Completed(Self::new(
-                    static_hs,
-                    Some(CHandshakeAuth::new(uname, pwd)),
-                ))
+                HandshakeResult::Completed(Self::new(static_hs, CHandshakeAuth::new(uname, pwd)))
             };
         }
         HandshakeResult::ChangeState {
@@ -352,14 +374,6 @@ impl<'a> CHandshake<'a> {
         }
         // we seem to have enough data for this auth mode
         match static_header.auth_mode {
-            AuthMode::Anonymous => {
-                if unsafe { scanner.next_byte() } == 0 {
-                    // matched
-                    return HandshakeResult::Completed(Self::new(static_header, None));
-                }
-                // we can only accept a NUL byte
-                return HandshakeResult::Error(ProtocolError::RejectAuth);
-            }
             AuthMode::Password => {}
         }
         // let us see if we can parse the username length
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index 538ab007..a2a2fa23 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -24,65 +24,126 @@
  *
 */
 
+use tokio::io::AsyncWriteExt;
+
 mod exchange;
 mod handshake;
 #[cfg(test)]
 mod tests;
 
+// re-export
+pub use exchange::SQuery;
+
 use {
-    self::handshake::{CHandshake, HandshakeResult, HandshakeState},
-    super::{IoResult, QLoopReturn, Socket},
-    crate::engine::{fractal::Global, mem::BufferedScanner},
+    self::{
+        exchange::{QueryTimeExchangeResult, QueryTimeExchangeState},
+        handshake::{
+            AuthMode, CHandshake, DataExchangeMode, HandshakeResult, HandshakeState,
+            HandshakeVersion, ProtocolError, ProtocolVersion, QueryMode,
+        },
+    },
+    super::{IoResult, QueryLoopResult, Socket},
+    crate::engine::{
+        self,
+        fractal::{Global, GlobalInstanceLike},
+        mem::BufferedScanner,
+    },
     bytes::{Buf, BytesMut},
     tokio::io::{AsyncReadExt, BufWriter},
 };
 
-pub async fn query_loop<S: Socket>(
+pub(super) async fn query_loop<S: Socket>(
     con: &mut BufWriter<S>,
     buf: &mut BytesMut,
-    _global: &Global,
-) -> IoResult<QLoopReturn> {
+    global: &Global,
+) -> IoResult<QueryLoopResult> {
     // handshake
-    match do_handshake(con, buf).await? {
-        Some(ret) => return Ok(ret),
-        None => {}
-    }
+    let _ = match do_handshake(con, buf, global).await? {
+        PostHandshake::Okay(hs) => hs,
+        PostHandshake::ConnectionClosedFin => return Ok(QueryLoopResult::Fin),
+        PostHandshake::ConnectionClosedRst => return Ok(QueryLoopResult::Rst),
+        PostHandshake::Error(e) => {
+            // failed to handshake; we'll close the connection
+            let hs_err_packet = [b'H', b'1', 1, e.value_u8()];
+            con.write_all(&hs_err_packet).await?;
+            return Ok(QueryLoopResult::HSFailed);
+        }
+    };
     // done handshaking
+    con.write_all(b"H1\x00\x00\x00").await?;
+    let mut exchg_state = QueryTimeExchangeState::default();
+    let mut expect_more = exchange::EXCHANGE_MIN_SIZE;
+    let mut cursor = 0;
     loop {
         let read_many = con.read_buf(buf).await?;
-        if let Some(t) = see_if_connection_terminates(read_many, buf) {
-            return Ok(t);
+        if read_many == 0 {
+            if buf.is_empty() {
+                return Ok(QueryLoopResult::Fin);
+            } else {
+                return Ok(QueryLoopResult::Rst);
+            }
+        }
+        if read_many < expect_more {
+            // we haven't buffered sufficient bytes; keep working
+            continue;
         }
-        todo!()
+        let mut buffer = unsafe { BufferedScanner::new_with_cursor(&buf, cursor) };
+        let sq = match exchange::resume(&mut buffer, exchg_state) {
+            QueryTimeExchangeResult::ChangeState {
+                new_state,
+                expect_more: _more,
+            } => {
+                exchg_state = new_state;
+                expect_more = _more;
+                cursor = buffer.cursor();
+                continue;
+            }
+            QueryTimeExchangeResult::SQCompleted(sq) => sq,
+            QueryTimeExchangeResult::Error => {
+                con.write_all(b"!\x00").await?;
+                exchg_state = QueryTimeExchangeState::default();
+                continue;
+            }
+        };
+        // now execute query
+        match engine::core::exec::execute_query(global, sq).await {
+            Ok(()) => {
+                buf.clear();
+            }
+            Err(_e) => {
+                // TOOD(@ohsayan): actual error codes! 
+                con.write_all(&[b'!', 1]).await?;
+            },
+        }
+        exchg_state = QueryTimeExchangeState::default();
     }
 }
 
-#[inline(always)]
-fn see_if_connection_terminates(read_many: usize, buf: &[u8]) -> Option<QLoopReturn> {
-    if read_many == 0 {
-        // that's a connection termination
-        if buf.is_empty() {
-            // nice termination
-            return Some(QLoopReturn::Fin);
-        } else {
-            return Some(QLoopReturn::ConnectionRst);
-        }
-    }
-    None
+#[derive(Debug, PartialEq)]
+enum PostHandshake {
+    Okay(handshake::CHandshakeStatic),
+    Error(ProtocolError),
+    ConnectionClosedFin,
+    ConnectionClosedRst,
 }
 
 async fn do_handshake<S: Socket>(
     con: &mut BufWriter<S>,
     buf: &mut BytesMut,
-) -> IoResult<Option<QLoopReturn>> {
+    global: &Global,
+) -> IoResult<PostHandshake> {
     let mut expected = CHandshake::INITIAL_READ;
     let mut state = HandshakeState::default();
     let mut cursor = 0;
     let handshake;
     loop {
         let read_many = con.read_buf(buf).await?;
-        if let Some(t) = see_if_connection_terminates(read_many, buf) {
-            return Ok(Some(t));
+        if read_many == 0 {
+            if buf.is_empty() {
+                return Ok(PostHandshake::ConnectionClosedFin);
+            } else {
+                return Ok(PostHandshake::ConnectionClosedRst);
+            }
         }
         if buf.len() < expected {
             continue;
@@ -98,10 +159,38 @@ async fn do_handshake<S: Socket>(
                 state = new_state;
                 cursor = scanner.cursor();
             }
-            HandshakeResult::Error(_) => todo!(),
+            HandshakeResult::Error(e) => {
+                return Ok(PostHandshake::Error(e));
+            }
         }
     }
-    dbg!(handshake);
-    buf.advance(cursor);
-    Ok(None)
+    // check handshake
+    if cfg!(debug_assertions) {
+        assert_eq!(
+            handshake.hs_static().hs_version(),
+            HandshakeVersion::Original
+        );
+        assert_eq!(handshake.hs_static().protocol(), ProtocolVersion::Original);
+        assert_eq!(
+            handshake.hs_static().exchange_mode(),
+            DataExchangeMode::QueryTime
+        );
+        assert_eq!(handshake.hs_static().query_mode(), QueryMode::Bql1);
+        assert_eq!(handshake.hs_static().auth_mode(), AuthMode::Password);
+    }
+    match core::str::from_utf8(handshake.hs_auth().username()) {
+        Ok(uname) => {
+            let auth = global.sys_cfg().auth_data().read();
+            if auth
+                .verify_user(uname, handshake.hs_auth().password())
+                .is_ok()
+            {
+                let hs_static = handshake.hs_static();
+                buf.advance(cursor);
+                return Ok(PostHandshake::Okay(hs_static));
+            }
+        }
+        Err(_) => {}
+    };
+    Ok(PostHandshake::Error(ProtocolError::RejectAuth))
 }
diff --git a/server/src/engine/net/protocol/tests.rs b/server/src/engine/net/protocol/tests.rs
index 6c05058d..b7f13946 100644
--- a/server/src/engine/net/protocol/tests.rs
+++ b/server/src/engine/net/protocol/tests.rs
@@ -39,17 +39,8 @@ use crate::engine::{
     client handshake
 */
 
-const FULL_HANDSHAKE_NO_AUTH: [u8; 7] = [b'H', 0, 0, 0, 0, 0, 0];
 const FULL_HANDSHAKE_WITH_AUTH: [u8; 23] = *b"H\0\0\0\0\x015\n8\nsayanpass1234";
 
-const STATIC_HANDSHAKE_NO_AUTH: CHandshakeStatic = CHandshakeStatic::new(
-    HandshakeVersion::Original,
-    ProtocolVersion::Original,
-    DataExchangeMode::QueryTime,
-    QueryMode::Bql1,
-    AuthMode::Anonymous,
-);
-
 const STATIC_HANDSHAKE_WITH_AUTH: CHandshakeStatic = CHandshakeStatic::new(
     HandshakeVersion::Original,
     ProtocolVersion::Original,
@@ -62,42 +53,6 @@ const STATIC_HANDSHAKE_WITH_AUTH: CHandshakeStatic = CHandshakeStatic::new(
     handshake with no state changes
 */
 
-#[test]
-fn parse_staged_no_auth() {
-    for i in 0..FULL_HANDSHAKE_NO_AUTH.len() {
-        let buf = &FULL_HANDSHAKE_NO_AUTH[..i + 1];
-        let mut scanner = BufferedScanner::new(buf);
-        let result = CHandshake::resume_with(&mut scanner, HandshakeState::Initial);
-        match buf.len() {
-            1..=5 => {
-                assert_eq!(
-                    result,
-                    HandshakeResult::ChangeState {
-                        new_state: HandshakeState::Initial,
-                        expect: CHandshake::INITIAL_READ,
-                    }
-                );
-            }
-            6 => {
-                assert_eq!(
-                    result,
-                    HandshakeResult::ChangeState {
-                        new_state: HandshakeState::StaticBlock(STATIC_HANDSHAKE_NO_AUTH),
-                        expect: 1,
-                    }
-                );
-            }
-            7 => {
-                assert_eq!(
-                    result,
-                    HandshakeResult::Completed(CHandshake::new(STATIC_HANDSHAKE_NO_AUTH, None))
-                );
-            }
-            _ => unreachable!(),
-        }
-    }
-}
-
 #[test]
 fn parse_staged_with_auth() {
     for i in 0..FULL_HANDSHAKE_WITH_AUTH.len() {
@@ -144,7 +99,7 @@ fn parse_staged_with_auth() {
                     result,
                     HandshakeResult::Completed(CHandshake::new(
                         STATIC_HANDSHAKE_WITH_AUTH,
-                        Some(CHandshakeAuth::new(b"sayan", b"pass1234"))
+                        CHandshakeAuth::new(b"sayan", b"pass1234")
                     ))
                 );
             }
@@ -182,22 +137,13 @@ fn run_state_changes_return_rounds(src: &[u8], expected_final_handshake: CHandsh
     rounds
 }
 
-#[test]
-fn parse_no_auth_with_state_updates() {
-    let rounds = run_state_changes_return_rounds(
-        &FULL_HANDSHAKE_NO_AUTH,
-        CHandshake::new(STATIC_HANDSHAKE_NO_AUTH, None),
-    );
-    assert_eq!(rounds, 2); // r1 = initial, r2 = auth NUL
-}
-
 #[test]
 fn parse_auth_with_state_updates() {
     let rounds = run_state_changes_return_rounds(
         &FULL_HANDSHAKE_WITH_AUTH,
         CHandshake::new(
             STATIC_HANDSHAKE_WITH_AUTH,
-            Some(CHandshakeAuth::new(b"sayan", b"pass1234")),
+            CHandshakeAuth::new(b"sayan", b"pass1234"),
         ),
     );
     assert_eq!(rounds, 3); // r1 = initial read, r2 = lengths, r3 = items
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 940aabbc..1072ef25 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -57,6 +57,7 @@ pub struct State<'a, Qd> {
     f: bool,
 }
 
+#[cfg(test)]
 impl<'a> State<'a, InplaceData> {
     pub const fn new_inplace(tok: &'a [Token<'a>]) -> Self {
         Self::new(tok, InplaceData::new())
@@ -186,11 +187,6 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
         self.t[self.i] == token
     }
     #[inline(always)]
-    /// Read ahead from the cursor by the given positions
-    pub(crate) fn read_ahead(&self, ahead: usize) -> &'a Token<'a> {
-        &self.t[self.i + ahead]
-    }
-    #[inline(always)]
     /// Move the cursor back by 1
     pub(crate) fn cursor_back(&mut self) {
         self.cursor_back_by(1);
@@ -246,16 +242,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
         self.not_exhausted() && self.okay()
     }
     #[inline(always)]
-    /// Loop condition for tt and non-poisoned state only
-    pub fn loop_data_tt(&self) -> bool {
-        self.not_exhausted() && self.okay() && self.d.nonzero()
-    }
-    #[inline(always)]
-    /// Returns the number of consumed tokens
-    pub(crate) fn consumed(&self) -> usize {
-        self.t.len() - self.remaining()
-    }
-    #[inline(always)]
+    #[cfg(test)]
     /// Returns the position of the cursor
     pub(crate) fn cursor(&self) -> usize {
         self.i
@@ -338,12 +325,7 @@ pub enum Entity<'a> {
 }
 
 impl<'a> Entity<'a> {
-    pub fn any_single(self) -> Ident<'a> {
-        match self {
-            Self::Full(_, e) => e,
-            Self::Single(e) => e,
-        }
-    }
+    #[cfg(test)]
     pub fn into_full(self) -> Option<(Ident<'a>, Ident<'a>)> {
         if let Self::Full(a, b) = self {
             Some((a, b))
@@ -358,20 +340,6 @@ impl<'a> Entity<'a> {
             None
         }
     }
-    pub fn into_single(self) -> Option<Ident<'a>> {
-        if let Self::Single(a) = self {
-            Some(a)
-        } else {
-            None
-        }
-    }
-    pub fn into_single_str(self) -> Option<&'a str> {
-        if let Self::Single(a) = self {
-            Some(a.as_str())
-        } else {
-            None
-        }
-    }
     #[inline(always)]
     /// Parse a full entity from the given slice
     ///
@@ -393,6 +361,7 @@ impl<'a> Entity<'a> {
         Entity::Single(sl[0].uck_read_ident())
     }
     #[inline(always)]
+    #[cfg(test)]
     /// Returns true if the given token stream matches the signature of single entity syntax
     ///
     /// ⚠ WARNING: This will pass for full and single
@@ -400,11 +369,13 @@ impl<'a> Entity<'a> {
         !tok.is_empty() && tok[0].is_ident()
     }
     #[inline(always)]
+    #[cfg(test)]
     /// Returns true if the given token stream matches the signature of full entity syntax
     pub(super) fn signature_matches_full_len_checked(tok: &[Token]) -> bool {
         tok.len() > 2 && tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident()
     }
     #[inline(always)]
+    #[cfg(test)]
     /// Attempt to parse an entity using the given token stream. It also accepts a counter
     /// argument to forward the cursor
     pub fn parse_from_tokens_len_checked(tok: &'a [Token], c: &mut usize) -> QueryResult<Self> {
@@ -523,11 +494,6 @@ pub enum Statement<'a> {
     Delete(dml::del::DeleteStatement<'a>),
 }
 
-#[cfg(test)]
-pub fn compile_test<'a>(tok: &'a [Token<'a>]) -> QueryResult<Statement<'a>> {
-    self::compile(tok, InplaceData::new())
-}
-
 #[inline(always)]
 pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResult<Statement<'a>> {
     if compiler::unlikely(tok.len() < 2) {
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index c0d577b3..e4228375 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -47,6 +47,7 @@ pub struct AlterSpace<'a> {
 }
 
 impl<'a> AlterSpace<'a> {
+    #[cfg(test)]
     pub fn new(space_name: Ident<'a>, updated_props: DictGeneric) -> Self {
         Self {
             space_name,
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index d9436143..a728f8ba 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -100,6 +100,7 @@ pub struct CreateModel<'a> {
 */
 
 impl<'a> CreateModel<'a> {
+    #[cfg(test)]
     pub fn new(model_name: Entity<'a>, fields: Vec<FieldSpec<'a>>, props: DictGeneric) -> Self {
         Self {
             model_name,
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index 7491ed1f..f4296e05 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -258,6 +258,7 @@ pub struct LayerSpec<'a> {
 
 impl<'a> LayerSpec<'a> {
     //// Create a new layer
+    #[cfg(test)]
     pub const fn new(ty: Ident<'a>, props: DictGeneric) -> Self {
         Self { ty, props }
     }
@@ -267,7 +268,6 @@ states! {
     /// Layer fold state
     pub struct LayerFoldState: u8 {
         BEGIN_IDENT = 0x01,
-        IDENT_OR_CB = 0x02,
         FOLD_INCOMPLETE = 0x03,
         FINAL_OR_OB = 0x04,
         FINAL = 0xFF,
@@ -353,6 +353,7 @@ pub struct FieldSpec<'a> {
 }
 
 impl<'a> FieldSpec<'a> {
+    #[cfg(test)]
     pub fn new(
         field_name: Ident<'a>,
         layers: Vec<LayerSpec<'a>>,
@@ -408,6 +409,7 @@ pub struct ExpandedField<'a> {
 }
 
 impl<'a> ExpandedField<'a> {
+    #[cfg(test)]
     pub fn new(field_name: Ident<'a>, layers: Vec<LayerSpec<'a>>, props: DictGeneric) -> Self {
         Self {
             field_name,
diff --git a/server/src/engine/ql/dml/del.rs b/server/src/engine/ql/dml/del.rs
index 86b4661e..b1ca021d 100644
--- a/server/src/engine/ql/dml/del.rs
+++ b/server/src/engine/ql/dml/del.rs
@@ -54,9 +54,6 @@ impl<'a> DeleteStatement<'a> {
     pub const fn entity(&self) -> Entity<'a> {
         self.entity
     }
-    pub const fn clauses(&self) -> &WhereClause {
-        &self.wc
-    }
     pub fn clauses_mut(&mut self) -> &mut WhereClause<'a> {
         &mut self.wc
     }
@@ -64,6 +61,7 @@ impl<'a> DeleteStatement<'a> {
 
 impl<'a> DeleteStatement<'a> {
     #[inline(always)]
+    #[cfg(test)]
     pub(super) fn new(entity: Entity<'a>, wc: WhereClause<'a>) -> Self {
         Self { entity, wc }
     }
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index 02d8f8fb..b75d3c9c 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -48,7 +48,6 @@ use {
 */
 
 pub const T_UUIDSTR: &str = "4593264b-0231-43e9-b0aa-50784f14e204";
-pub const T_UUIDBIN: &[u8] = T_UUIDSTR.as_bytes();
 pub const T_TIMESEC: u64 = 1673187839_u64;
 
 type ProducerFn = fn() -> Datacell;
@@ -121,16 +120,6 @@ fn ldfunc(func: Ident<'_>) -> Option<ProducerFn> {
         None
     }
 }
-#[inline(always)]
-fn ldfunc_exists(func: Ident<'_>) -> bool {
-    ldfunc(func).is_some()
-}
-#[inline(always)]
-unsafe fn ldfunc_unchecked(func: &[u8]) -> ProducerFn {
-    let ph = hashp(func) as usize;
-    debug_assert_eq!(PRODUCER_F[ph].0, func);
-    PRODUCER_F[ph].1
-}
 
 /// ## Panics
 /// - If tt length is less than 1
@@ -338,6 +327,7 @@ pub struct InsertStatement<'a> {
 
 impl<'a> InsertStatement<'a> {
     #[inline(always)]
+    #[cfg(test)]
     pub fn new(entity: Entity<'a>, data: InsertData<'a>) -> Self {
         Self { entity, data }
     }
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index 3fc5030b..d8bb50d1 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -158,9 +158,6 @@ impl<'a> WhereClause<'a> {
         state.poison_if(c.is_empty());
         Self { c }
     }
-    pub fn clauses(&self) -> &WhereClauseCollection {
-        &self.c
-    }
 }
 
 #[cfg(test)]
@@ -182,7 +179,7 @@ mod impls {
     }
     impl<'a> ASTNode<'a> for RelationalExpr<'a> {
         fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
-            Self::try_parse(state).ok_or(QueryError::QLIllegalRelExp)
+            Self::try_parse(state).ok_or(QueryError::QLInvalidSyntax)
         }
     }
 }
diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs
index 63daa477..6cdc2209 100644
--- a/server/src/engine/ql/dml/sel.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -24,8 +24,10 @@
  *
 */
 
+#[cfg(test)]
+use super::WhereClauseCollection;
 use {
-    super::{WhereClause, WhereClauseCollection},
+    super::WhereClause,
     crate::{
         engine::{
             error::{QueryError, QueryResult},
@@ -56,6 +58,7 @@ pub struct SelectStatement<'a> {
 
 impl<'a> SelectStatement<'a> {
     #[inline(always)]
+    #[cfg(test)]
     pub(crate) fn new_test(
         entity: Entity<'a>,
         fields: Vec<Ident<'a>>,
@@ -65,6 +68,7 @@ impl<'a> SelectStatement<'a> {
         Self::new(entity, fields, wildcard, clauses)
     }
     #[inline(always)]
+    #[cfg(test)]
     fn new(
         entity: Entity<'a>,
         fields: Vec<Ident<'a>>,
@@ -81,9 +85,6 @@ impl<'a> SelectStatement<'a> {
     pub fn entity(&self) -> Entity<'a> {
         self.entity
     }
-    pub fn clauses(&self) -> &WhereClause<'a> {
-        &self.clause
-    }
     pub fn clauses_mut(&mut self) -> &mut WhereClause<'a> {
         &mut self.clause
     }
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index dcd2dd21..f2f4a225 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -24,8 +24,6 @@
  *
 */
 
-#[cfg(test)]
-use super::WhereClauseCollection;
 use {
     super::{u, WhereClause},
     crate::{
@@ -138,9 +136,6 @@ impl<'a> UpdateStatement<'a> {
     pub fn expressions(&self) -> &[AssignmentExpression<'a>] {
         &self.expressions
     }
-    pub fn clauses(&self) -> &WhereClause<'a> {
-        &self.wc
-    }
     pub fn clauses_mut(&mut self) -> &mut WhereClause<'a> {
         &mut self.wc
     }
@@ -152,14 +147,6 @@ impl<'a> UpdateStatement<'a> {
 impl<'a> UpdateStatement<'a> {
     #[inline(always)]
     #[cfg(test)]
-    pub fn new_test(
-        entity: Entity<'a>,
-        expressions: Vec<AssignmentExpression<'a>>,
-        wc: WhereClauseCollection<'a>,
-    ) -> Self {
-        Self::new(entity, expressions, WhereClause::new(wc))
-    }
-    #[inline(always)]
     pub fn new(
         entity: Entity<'a>,
         expressions: Vec<AssignmentExpression<'a>>,
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index 154a6f85..49b3094b 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -25,19 +25,18 @@
 */
 
 mod raw;
-pub use raw::{Ident, Keyword, Symbol, Token};
+pub use {
+    insecure_impl::InsecureLexer,
+    raw::{Ident, Keyword, KeywordMisc, KeywordStmt, Symbol, Token},
+};
 
 use {
-    crate::{
-        engine::{
-            data::lit::Lit,
-            error::{QueryError, QueryResult},
-            mem::BufferedScanner,
-        },
-        util::compiler,
+    crate::engine::{
+        data::lit::Lit,
+        error::{QueryError, QueryResult},
+        mem::BufferedScanner,
     },
     core::slice,
-    raw::{kwof, symof},
 };
 
 /*
@@ -100,12 +99,11 @@ impl<'a> Lexer<'a> {
     /// Scan an identifier or keyword
     fn scan_ident_or_keyword(&mut self) {
         let s = self.scan_ident();
-        let st = s.to_ascii_lowercase();
-        match kwof(&st) {
+        match Keyword::get(s) {
             Some(kw) => self.tokens.push(kw.into()),
             // FIXME(@ohsayan): Uh, mind fixing this? The only advantage is that I can keep the graph *memory* footprint small
-            None if st == b"true" || st == b"false" => {
-                self.push_token(Lit::new_bool(st == b"true"))
+            None if s.eq_ignore_ascii_case(b"true") || s.eq_ignore_ascii_case(b"false") => {
+                self.push_token(Lit::new_bool(s.eq_ignore_ascii_case(b"true")))
             }
             None => self.tokens.push(unsafe {
                 // UNSAFE(@ohsayan): scan_ident only returns a valid ident which is always a string
@@ -114,7 +112,7 @@ impl<'a> Lexer<'a> {
         }
     }
     fn scan_byte(&mut self, byte: u8) {
-        match symof(byte) {
+        match Symbol::get(byte) {
             Some(tok) => self.push_token(tok),
             None => return self.set_error(QueryError::LexUnexpectedByte),
         }
@@ -136,186 +134,201 @@ impl<'a> Lexer<'a> {
     Insecure lexer
 */
 
-pub struct InsecureLexer<'a> {
-    l: Lexer<'a>,
-}
+mod insecure_impl {
+    #![allow(unused)] // TODO(@ohsayan): yank this
+    use {
+        super::Lexer,
+        crate::{
+            engine::{
+                data::lit::Lit,
+                error::{QueryError, QueryResult},
+                ql::lex::Token,
+            },
+            util::compiler,
+        },
+    };
 
-impl<'a> InsecureLexer<'a> {
-    pub fn lex(src: &'a [u8]) -> QueryResult<Vec<Token<'a>>> {
-        let slf = Self { l: Lexer::new(src) };
-        slf._lex()
+    pub struct InsecureLexer<'a> {
+        pub(crate) l: Lexer<'a>,
     }
-    fn _lex(mut self) -> QueryResult<Vec<Token<'a>>> {
-        while !self.l.token_buffer.eof() & self.l.no_error() {
-            let byte = unsafe {
-                // UNSAFE(@ohsayan): loop invariant
-                self.l.token_buffer.deref_cursor()
-            };
-            match byte {
-                #[cfg(test)]
-                byte if byte == b'\x01' => {
-                    self.l.push_token(Token::IgnorableComma);
-                    unsafe {
-                        // UNSAFE(@ohsayan): All good here. Already read the token
-                        self.l.token_buffer.incr_cursor();
+
+    impl<'a> InsecureLexer<'a> {
+        pub fn lex(src: &'a [u8]) -> QueryResult<Vec<Token<'a>>> {
+            let slf = Self { l: Lexer::new(src) };
+            slf._lex()
+        }
+        pub(crate) fn _lex(mut self) -> QueryResult<Vec<Token<'a>>> {
+            while !self.l.token_buffer.eof() & self.l.no_error() {
+                let byte = unsafe {
+                    // UNSAFE(@ohsayan): loop invariant
+                    self.l.token_buffer.deref_cursor()
+                };
+                match byte {
+                    #[cfg(test)]
+                    byte if byte == b'\x01' => {
+                        self.l.push_token(Token::IgnorableComma);
+                        unsafe {
+                            // UNSAFE(@ohsayan): All good here. Already read the token
+                            self.l.token_buffer.incr_cursor();
+                        }
                     }
-                }
-                // ident
-                byte if byte.is_ascii_alphabetic() | (byte == b'_') => {
-                    self.l.scan_ident_or_keyword()
-                }
-                // uint
-                byte if byte.is_ascii_digit() => self.scan_unsigned_integer(),
-                // sint
-                b'-' => {
-                    unsafe {
-                        // UNSAFE(@ohsayan): loop invariant
-                        self.l.token_buffer.incr_cursor()
-                    };
-                    self.scan_signed_integer();
-                }
-                // binary
-                b'\r' => {
-                    unsafe {
-                        // UNSAFE(@ohsayan): loop invariant
-                        self.l.token_buffer.incr_cursor()
+                    // ident
+                    byte if byte.is_ascii_alphabetic() | (byte == b'_') => {
+                        self.l.scan_ident_or_keyword()
                     }
-                    self.scan_binary()
-                }
-                // string
-                quote_style @ (b'"' | b'\'') => {
-                    unsafe {
-                        // UNSAFE(@ohsayan): loop invariant
-                        self.l.token_buffer.incr_cursor()
+                    // uint
+                    byte if byte.is_ascii_digit() => self.scan_unsigned_integer(),
+                    // sint
+                    b'-' => {
+                        unsafe {
+                            // UNSAFE(@ohsayan): loop invariant
+                            self.l.token_buffer.incr_cursor()
+                        };
+                        self.scan_signed_integer();
                     }
-                    self.scan_quoted_string(quote_style)
-                }
-                // whitespace
-                b' ' | b'\n' | b'\t' => self.l.trim_ahead(),
-                // some random byte
-                byte => self.l.scan_byte(byte),
-            }
-        }
-        match self.l.last_error {
-            None => Ok(self.l.tokens),
-            Some(e) => Err(e),
-        }
-    }
-}
-
-impl<'a> InsecureLexer<'a> {
-    fn scan_binary(&mut self) {
-        let Some(len) = self
-            .l
-            .token_buffer
-            .try_next_ascii_u64_lf_separated_or_restore_cursor()
-        else {
-            self.l.set_error(QueryError::LexInvalidLiteral);
-            return;
-        };
-        let len = len as usize;
-        match self.l.token_buffer.try_next_variable_block(len) {
-            Some(block) => self.l.push_token(Lit::new_bin(block)),
-            None => self.l.set_error(QueryError::LexInvalidLiteral),
-        }
-    }
-    fn scan_quoted_string(&mut self, quote_style: u8) {
-        // cursor is at beginning of `"`; we need to scan until the end of quote or an escape
-        let mut buf = Vec::new();
-        while self
-            .l
-            .token_buffer
-            .rounded_cursor_not_eof_matches(|b| *b != quote_style)
-        {
-            let byte = unsafe {
-                // UNSAFE(@ohsayan): loop invariant
-                self.l.token_buffer.next_byte()
-            };
-            match byte {
-                b'\\' => {
-                    // hmm, this might be an escape (either `\\` or `\"`)
-                    if self
-                        .l
-                        .token_buffer
-                        .rounded_cursor_not_eof_matches(|b| *b == quote_style || *b == b'\\')
-                    {
-                        // ignore escaped byte
+                    // binary
+                    b'\r' => {
                         unsafe {
-                            buf.push(self.l.token_buffer.next_byte());
+                            // UNSAFE(@ohsayan): loop invariant
+                            self.l.token_buffer.incr_cursor()
                         }
-                    } else {
-                        // this is not allowed
+                        self.scan_binary()
+                    }
+                    // string
+                    quote_style @ (b'"' | b'\'') => {
                         unsafe {
-                            // UNSAFE(@ohsayan): we move the cursor ahead, now we're moving it back
-                            self.l.token_buffer.decr_cursor()
+                            // UNSAFE(@ohsayan): loop invariant
+                            self.l.token_buffer.incr_cursor()
                         }
-                        self.l.set_error(QueryError::LexInvalidLiteral);
-                        return;
+                        self.scan_quoted_string(quote_style)
                     }
+                    // whitespace
+                    b' ' | b'\n' | b'\t' => self.l.trim_ahead(),
+                    // some random byte
+                    byte => self.l.scan_byte(byte),
                 }
-                _ => buf.push(byte),
             }
-        }
-        let ended_with_quote = self
-            .l
-            .token_buffer
-            .rounded_cursor_not_eof_equals(quote_style);
-        // skip quote
-        unsafe {
-            // UNSAFE(@ohsayan): not eof
-            self.l.token_buffer.incr_cursor_if(ended_with_quote)
-        }
-        match String::from_utf8(buf) {
-            Ok(s) if ended_with_quote => self.l.push_token(Lit::new_string(s)),
-            Err(_) | Ok(_) => self.l.set_error(QueryError::LexInvalidLiteral),
+            match self.l.last_error {
+                None => Ok(self.l.tokens),
+                Some(e) => Err(e),
+            }
         }
     }
-    fn scan_unsigned_integer(&mut self) {
-        let mut okay = true;
-        // extract integer
-        let int = self
-            .l
-            .token_buffer
-            .try_next_ascii_u64_stop_at::<false>(&mut okay, |b| b.is_ascii_digit());
-        /*
-            see if we ended at a correct byte:
-            iff the integer has an alphanumeric byte at the end is the integer invalid
-        */
-        if compiler::unlikely(
-            !okay
-                | self
-                    .l
-                    .token_buffer
-                    .rounded_cursor_not_eof_matches(u8::is_ascii_alphanumeric),
-        ) {
-            self.l.set_error(QueryError::LexInvalidLiteral);
-        } else {
-            self.l.push_token(Lit::new_uint(int))
+
+    impl<'a> InsecureLexer<'a> {
+        pub(crate) fn scan_binary(&mut self) {
+            let Some(len) = self
+                .l
+                .token_buffer
+                .try_next_ascii_u64_lf_separated_or_restore_cursor()
+            else {
+                self.l.set_error(QueryError::LexInvalidLiteral);
+                return;
+            };
+            let len = len as usize;
+            match self.l.token_buffer.try_next_variable_block(len) {
+                Some(block) => self.l.push_token(Lit::new_bin(block)),
+                None => self.l.set_error(QueryError::LexInvalidLiteral),
+            }
         }
-    }
-    fn scan_signed_integer(&mut self) {
-        if self.l.token_buffer.rounded_cursor_value().is_ascii_digit() {
+        pub(crate) fn scan_quoted_string(&mut self, quote_style: u8) {
+            // cursor is at beginning of `"`; we need to scan until the end of quote or an escape
+            let mut buf = Vec::new();
+            while self
+                .l
+                .token_buffer
+                .rounded_cursor_not_eof_matches(|b| *b != quote_style)
+            {
+                let byte = unsafe {
+                    // UNSAFE(@ohsayan): loop invariant
+                    self.l.token_buffer.next_byte()
+                };
+                match byte {
+                    b'\\' => {
+                        // hmm, this might be an escape (either `\\` or `\"`)
+                        if self
+                            .l
+                            .token_buffer
+                            .rounded_cursor_not_eof_matches(|b| *b == quote_style || *b == b'\\')
+                        {
+                            // ignore escaped byte
+                            unsafe {
+                                buf.push(self.l.token_buffer.next_byte());
+                            }
+                        } else {
+                            // this is not allowed
+                            unsafe {
+                                // UNSAFE(@ohsayan): we move the cursor ahead, now we're moving it back
+                                self.l.token_buffer.decr_cursor()
+                            }
+                            self.l.set_error(QueryError::LexInvalidLiteral);
+                            return;
+                        }
+                    }
+                    _ => buf.push(byte),
+                }
+            }
+            let ended_with_quote = self
+                .l
+                .token_buffer
+                .rounded_cursor_not_eof_equals(quote_style);
+            // skip quote
             unsafe {
-                // UNSAFE(@ohsayan): the cursor was moved ahead, now we're moving it back
-                self.l.token_buffer.decr_cursor()
+                // UNSAFE(@ohsayan): not eof
+                self.l.token_buffer.incr_cursor_if(ended_with_quote)
+            }
+            match String::from_utf8(buf) {
+                Ok(s) if ended_with_quote => self.l.push_token(Lit::new_string(s)),
+                Err(_) | Ok(_) => self.l.set_error(QueryError::LexInvalidLiteral),
             }
-            let (okay, int) = self
+        }
+        pub(crate) fn scan_unsigned_integer(&mut self) {
+            let mut okay = true;
+            // extract integer
+            let int = self
                 .l
                 .token_buffer
-                .try_next_ascii_i64_stop_at(|b| !b.is_ascii_digit());
-            if okay
-                & !self
+                .try_next_ascii_u64_stop_at::<false>(&mut okay, |b| b.is_ascii_digit());
+            /*
+                see if we ended at a correct byte:
+                iff the integer has an alphanumeric byte at the end is the integer invalid
+            */
+            if compiler::unlikely(
+                !okay
+                    | self
+                        .l
+                        .token_buffer
+                        .rounded_cursor_not_eof_matches(u8::is_ascii_alphanumeric),
+            ) {
+                self.l.set_error(QueryError::LexInvalidLiteral);
+            } else {
+                self.l.push_token(Lit::new_uint(int))
+            }
+        }
+        pub(crate) fn scan_signed_integer(&mut self) {
+            if self.l.token_buffer.rounded_cursor_value().is_ascii_digit() {
+                unsafe {
+                    // UNSAFE(@ohsayan): the cursor was moved ahead, now we're moving it back
+                    self.l.token_buffer.decr_cursor()
+                }
+                let (okay, int) = self
                     .l
                     .token_buffer
-                    .rounded_cursor_value()
-                    .is_ascii_alphabetic()
-            {
-                self.l.push_token(Lit::new_sint(int))
+                    .try_next_ascii_i64_stop_at(|b| !b.is_ascii_digit());
+                if okay
+                    & !self
+                        .l
+                        .token_buffer
+                        .rounded_cursor_value()
+                        .is_ascii_alphabetic()
+                {
+                    self.l.push_token(Lit::new_sint(int))
+                } else {
+                    self.l.set_error(QueryError::LexInvalidLiteral)
+                }
             } else {
-                self.l.set_error(QueryError::LexInvalidLiteral)
+                self.l.push_token(Token![-]);
             }
-        } else {
-            self.l.push_token(Token![-]);
         }
     }
 }
@@ -331,14 +344,22 @@ pub struct SecureLexer<'a> {
 }
 
 impl<'a> SecureLexer<'a> {
-    pub fn new(src: &'a [u8], query_window: usize) -> Self {
+    pub fn new_with_segments(q: &'a [u8], p: &'a [u8]) -> Self {
+        Self {
+            l: Lexer::new(q),
+            param_buffer: BufferedScanner::new(p),
+        }
+    }
+    pub fn lex(self) -> QueryResult<Vec<Token<'a>>> {
+        self._lex()
+    }
+    #[cfg(test)]
+    pub fn lex_with_window(src: &'a [u8], query_window: usize) -> QueryResult<Vec<Token<'a>>> {
         Self {
             l: Lexer::new(&src[..query_window]),
             param_buffer: BufferedScanner::new(&src[query_window..]),
         }
-    }
-    pub fn lex(src: &'a [u8], query_window: usize) -> QueryResult<Vec<Token<'a>>> {
-        Self::new(src, query_window)._lex()
+        .lex()
     }
 }
 
@@ -352,10 +373,6 @@ impl<'a> SecureLexer<'a> {
             match b {
                 b if b.is_ascii_alphabetic() | (b == b'_') => self.l.scan_ident_or_keyword(),
                 b'?' => {
-                    // a parameter: null, bool, sint, uint, float, binary, string
-                    const TYPE: [&str; 8] = [
-                        "null", "bool", "uint", "sint", "float", "binary", "string", "ERROR",
-                    ];
                     // skip the param byte
                     unsafe {
                         // UNSAFE(@ohsayan): loop invariant
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index 3fb83abd..cb751d29 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -29,6 +29,10 @@ use {
     core::{borrow::Borrow, fmt, ops::Deref, str},
 };
 
+/*
+    ident
+*/
+
 #[repr(transparent)]
 #[derive(PartialEq, Eq, Clone, Copy, Hash)]
 pub struct Ident<'a>(&'a [u8]);
@@ -109,6 +113,10 @@ impl<'a> Borrow<[u8]> for Ident<'a> {
     }
 }
 
+/*
+    token
+*/
+
 #[derive(Debug, PartialEq, Clone)]
 pub enum Token<'a> {
     Symbol(Symbol),
@@ -159,71 +167,27 @@ direct_from! {
     }
 }
 
-build_lut!(
-    static KW_LUT in kwlut;
-    #[derive(Debug, PartialEq, Eq, Clone, Copy)]
-    pub enum Keyword {
-        Table = "table",
-        Model = "model",
-        Space = "space",
-        Index = "index",
-        Type = "type",
-        Function = "function",
-        Use = "use",
-        Create = "create",
-        Alter = "alter",
-        Drop = "drop",
-        Describe = "describe",
-        Truncate = "truncate",
-        Rename = "rename",
-        Add = "add",
-        Remove = "remove",
-        Transform = "transform",
-        Order = "order",
-        By = "by",
-        Primary = "primary",
-        Key = "key",
-        Value = "value",
-        With = "with",
-        On = "on",
-        Lock = "lock",
-        All = "all",
-        Insert = "insert",
-        Select = "select",
-        Exists = "exists",
-        Update = "update",
-        Delete = "delete",
-        Into = "into",
-        From = "from",
-        As = "as",
-        Return = "return",
-        Sort = "sort",
-        Group = "group",
-        Limit = "limit",
-        Asc = "asc",
-        Desc = "desc",
-        To = "to",
-        Set = "set",
-        Auto = "auto",
-        Default = "default",
-        In = "in",
-        Of = "of",
-        Transaction = "transaction",
-        Batch = "batch",
-        Read = "read",
-        Write = "write",
-        Begin = "begin",
-        End = "end",
-        Where = "where",
-        If = "if",
-        And = "and",
-        Or = "or",
-        Not = "not",
-        Null = "null",
+impl<'a> Token<'a> {
+    #[inline(always)]
+    pub(crate) const fn is_ident(&self) -> bool {
+        matches!(self, Token::Ident(_))
     }
-    |b: &str| -> &[u8] { b.as_bytes() },
-    |b: &str| -> String { b.to_ascii_uppercase() }
-);
+    #[inline(always)]
+    pub const fn is_lit(&self) -> bool {
+        matches!(self, Self::Lit(_))
+    }
+}
+
+impl<'a> AsRef<Token<'a>> for Token<'a> {
+    #[inline(always)]
+    fn as_ref(&self) -> &Token<'a> {
+        self
+    }
+}
+
+/*
+    symbols
+*/
 
 build_lut!(
     static SYM_LUT in symlut;
@@ -267,99 +231,225 @@ build_lut!(
     |c: u8| -> String { char::from(c).to_string() }
 );
 
-/*
-    This section implements LUTs constructed using DAGs, as described by Czech et al in their paper. I wrote these pretty much by
-    brute-force using a byte-level multiplicative function (inside a script). This unfortunately implies that every time we *do*
-    need to add a new keyword, I will need to recompute and rewrite the vertices. I don't plan to use any codegen, so I think
-    this is good as-is. The real challenge here is to keep the graph small, and I couldn't do that for the symbols table even with
-    multiple trials. Please see if you can improve them.
-
-    Also the functions are unique to every graph, and every input set, so BE WARNED!
-
-    -- Sayan (@ohsayan)
-    Sept. 18, 2022
-*/
-
-const SYM_MAGIC_A: u8 = b'w';
-const SYM_MAGIC_B: u8 = b'E';
-
-static SYM_GRAPH: [u8; 69] = [
-    0, 0, 25, 0, 3, 0, 21, 0, 6, 13, 0, 0, 0, 0, 8, 0, 0, 0, 17, 0, 0, 30, 0, 28, 0, 20, 19, 12, 0,
-    0, 2, 0, 0, 15, 0, 0, 0, 5, 0, 31, 14, 0, 1, 0, 18, 29, 24, 0, 0, 10, 0, 0, 26, 0, 0, 0, 22, 0,
-    23, 7, 0, 27, 0, 4, 16, 11, 0, 0, 9,
-];
-
-#[inline(always)]
-fn symfh(k: u8, magic: u8) -> u16 {
-    (magic as u16 * k as u16) % SYM_GRAPH.len() as u16
-}
-
-#[inline(always)]
-fn symph(k: u8) -> u8 {
-    (SYM_GRAPH[symfh(k, SYM_MAGIC_A) as usize] + SYM_GRAPH[symfh(k, SYM_MAGIC_B) as usize])
-        % SYM_GRAPH.len() as u8
-}
-
-#[inline(always)]
-pub(super) fn symof(sym: u8) -> Option<Symbol> {
-    let hf = symph(sym);
-    if hf < SYM_LUT.len() as u8 && SYM_LUT[hf as usize].0 == sym {
-        Some(SYM_LUT[hf as usize].1)
-    } else {
-        None
+impl Symbol {
+    pub fn get(k: u8) -> Option<Self> {
+        const SYM_MAGIC_A: u8 = b'w';
+        const SYM_MAGIC_B: u8 = b'E';
+        static G: [u8; 69] = [
+            0, 0, 25, 0, 3, 0, 21, 0, 6, 13, 0, 0, 0, 0, 8, 0, 0, 0, 17, 0, 0, 30, 0, 28, 0, 20,
+            19, 12, 0, 0, 2, 0, 0, 15, 0, 0, 0, 5, 0, 31, 14, 0, 1, 0, 18, 29, 24, 0, 0, 10, 0, 0,
+            26, 0, 0, 0, 22, 0, 23, 7, 0, 27, 0, 4, 16, 11, 0, 0, 9,
+        ];
+        let symfh = |magic, k| (magic as u16 * k as u16) % G.len() as u16;
+        let hf =
+            (G[symfh(k, SYM_MAGIC_A) as usize] + G[symfh(k, SYM_MAGIC_B) as usize]) % G.len() as u8;
+        if hf < SYM_LUT.len() as u8 && SYM_LUT[hf as usize].0 == k {
+            Some(SYM_LUT[hf as usize].1)
+        } else {
+            None
+        }
     }
 }
 
-static KWG: [u8; 63] = [
-    0, 24, 15, 29, 51, 53, 44, 38, 43, 4, 27, 1, 37, 57, 32, 0, 46, 24, 59, 45, 32, 52, 8, 0, 23,
-    19, 33, 48, 56, 60, 33, 53, 18, 47, 49, 53, 2, 19, 1, 34, 19, 58, 11, 5, 0, 41, 27, 24, 20, 2,
-    0, 0, 48, 2, 42, 46, 43, 0, 18, 33, 21, 12, 41,
-];
-
-const KWMG_1: [u8; 11] = *b"MpVBwC1vsCy";
-const KWMG_2: [u8; 11] = *b"m7sNd9mtGzC";
-const KWMG_S: usize = KWMG_1.len();
-
-#[inline(always)]
-fn kwhf(k: &[u8], mg: &[u8]) -> u32 {
-    let mut i = 0;
-    let mut s = 0;
-    while i < k.len() {
-        s += mg[i % KWMG_S] as u32 * k[i] as u32;
-        i += 1;
-    }
-    s % KWG.len() as u32
-}
+/*
+    keywords
+*/
 
-#[inline(always)]
-fn kwph(k: &[u8]) -> u8 {
-    (KWG[kwhf(k, &KWMG_1) as usize] + KWG[kwhf(k, &KWMG_2) as usize]) % KWG.len() as u8
+macro_rules! flattened_lut {
+	(
+        $staticvis:vis static $staticname:ident in $staticpriv:ident;
+		$(#[$enumattr:meta])*
+		$vis:vis enum $enum:ident {
+			$($(#[$variant_attr:meta])* $variant:ident => {
+                $(#[$nested_enum_attr:meta])*
+                $nested_enum_vis:vis enum $nested_enum_name:ident {$($(#[$nested_variant_attr:meta])* $nested_enum_variant_name:ident $(= $nested_enum_variant_dscr:expr)?,)*}
+            }),* $(,)?
+		}
+	) => {
+		$(
+			$(#[$nested_enum_attr])*
+			$nested_enum_vis enum $nested_enum_name {$($(#[$nested_variant_attr])* $nested_enum_variant_name $(= $nested_enum_variant_dscr)*),*}
+			impl $nested_enum_name {
+                const __LEN: usize = {let mut i = 0; $( let _ = Self::$nested_enum_variant_name; i += 1; )*i};
+                const __SL: [usize; 2] = {
+                    let mut largest = 0;
+                    let mut smallest = usize::MAX;
+                    $(
+                        let this = stringify!($nested_enum_variant_name).len();
+                        if this > largest { largest = this } if this < smallest { smallest = this }
+                    )*
+                    [smallest, largest]
+                };
+                const __SMALLEST: usize = Self::__SL[0];
+                const __LARGEST: usize = Self::__SL[1];
+                const fn __max() -> usize { Self::__LEN }
+				pub const fn as_str(&self) -> &'static str {match self {$(
+                    Self::$nested_enum_variant_name => {
+                        const NAME_STR: &'static str = stringify!($nested_enum_variant_name);
+                        const NAME_BUF: [u8; { NAME_STR.len() }] = {
+                            let mut buf = [0u8; { NAME_STR.len() }]; let name = NAME_STR.as_bytes();
+                            buf[0] = name[0].to_ascii_lowercase(); let mut i = 1;
+                            while i < NAME_STR.len() { buf[i] = name[i]; i += 1; }
+                            buf
+                        }; const NAME: &'static str = unsafe { core::str::from_utf8_unchecked(&NAME_BUF) }; NAME
+                    }
+				)*}}
+			}
+            impl ToString for $nested_enum_name { fn to_string(&self) -> String { self.as_str().to_owned() } }
+		)*
+        $(#[$enumattr])*
+        $vis enum $enum {$($(#[$variant_attr])* $variant($nested_enum_name)),*}
+        impl $enum { pub const fn as_str(&self) -> &'static str { match self {$(Self::$variant(v) => { $nested_enum_name::as_str(v) })*} } }
+        impl $enum {
+            const SL: [usize; 2] = {
+                let mut largest = 0; let mut smallest = usize::MAX;
+                $(
+                    if $nested_enum_name::__LARGEST > largest { largest = $nested_enum_name::__LARGEST; }
+                    if $nested_enum_name::__SMALLEST < smallest { smallest = $nested_enum_name::__SMALLEST; }
+                )*
+                [smallest, largest]
+            };
+            const SIZE_MIN: usize = Self::SL[0];
+            const SIZE_MAX: usize = Self::SL[1];
+        }
+        impl ToString for $enum { fn to_string(&self) -> String { self.as_str().to_owned() } }
+        mod $staticpriv { pub const LEN: usize = { let mut i = 0; $(i += super::$nested_enum_name::__max();)* i }; }
+        $staticvis static $staticname: [(&'static [u8], $enum); { $staticpriv::LEN }] = [
+            $($(($nested_enum_name::$nested_enum_variant_name.as_str().as_bytes() ,$enum::$variant($nested_enum_name::$nested_enum_variant_name)),)*)*
+        ];
+	}
 }
 
-#[inline(always)]
-pub(super) fn kwof(key: &[u8]) -> Option<Keyword> {
-    let ph = kwph(key);
-    if ph < KW_LUT.len() as u8 && KW_LUT[ph as usize].0 == key {
-        Some(KW_LUT[ph as usize].1)
-    } else {
-        None
+flattened_lut! {
+    static KW_LUT in kwlut;
+    #[derive(Debug, PartialEq, Clone, Copy)]
+    #[repr(u8)]
+    pub enum Keyword {
+        Statement => {
+            #[derive(Debug, PartialEq, Clone, Copy, sky_macros::EnumMethods)]
+            #[repr(u8)]
+            /// A statement keyword
+            pub enum KeywordStmt {
+                // sys
+                Sysctl = 0,
+                Describe = 1,
+                Inspect = 2,
+                // ddl
+                Use = 3,
+                Create = 4,
+                Alter = 5,
+                Drop = 6,
+                // dml
+                Insert = 7,
+                Select = 8,
+                Update = 9,
+                Delete = 10,
+                Exists = 11,
+            }
+        },
+        /// Hi
+        Misc => {
+            #[derive(Debug, PartialEq, Clone, Copy)]
+            #[repr(u8)]
+            /// Misc. keywords
+            pub enum KeywordMisc {
+                // item definitions
+                Table,
+                Model,
+                Space,
+                Index,
+                Type,
+                Function,
+                // operations
+                Rename,
+                Add,
+                Remove,
+                Transform,
+                Set,
+                // sort related
+                Order,
+                Sort,
+                Group,
+                Limit,
+                Asc,
+                Desc,
+                All,
+                // container relational specifier
+                By,
+                With,
+                On,
+                From,
+                Into,
+                As,
+                To,
+                In,
+                Of,
+                // logical
+                And,
+                Or,
+                Not,
+                // conditional
+                If,
+                Else,
+                Where,
+                When,
+                // value
+                Auto,
+                Default,
+                Null,
+                // transaction related
+                Transaction,
+                Batch,
+                Lock,
+                Read,
+                Write,
+                Begin,
+                End,
+                // misc
+                Key,
+                Value,
+                Primary,
+                // temporarily reserved (will probably be removed in the future)
+                Truncate, // TODO: decide what we want to do with this
+            }
+        }
     }
 }
 
-impl<'a> Token<'a> {
-    #[inline(always)]
-    pub(crate) const fn is_ident(&self) -> bool {
-        matches!(self, Token::Ident(_))
+impl Keyword {
+    pub fn get(k: &[u8]) -> Option<Self> {
+        if (k.len() > Self::SIZE_MAX) | (k.len() < Self::SIZE_MIN) {
+            None
+        } else {
+            Self::compute(k)
+        }
     }
-    #[inline(always)]
-    pub const fn is_lit(&self) -> bool {
-        matches!(self, Self::Lit(_))
+    fn compute(key: &[u8]) -> Option<Self> {
+        static G: [u8; 67] = [
+            0, 42, 57, 0, 20, 61, 15, 46, 28, 0, 31, 2, 1, 44, 47, 10, 35, 53, 30, 28, 48, 9, 1,
+            51, 61, 20, 20, 47, 23, 31, 0, 52, 55, 59, 27, 45, 54, 49, 29, 0, 66, 54, 23, 58, 13,
+            31, 47, 56, 1, 30, 40, 0, 0, 42, 27, 63, 6, 24, 65, 45, 42, 63, 60, 14, 26, 4, 13,
+        ];
+        static M1: [u8; 11] = *b"wsE1pgJgJMO";
+        static M2: [u8; 11] = *b"fICAB04WegN";
+        let h1 = Self::_sum(key, M1) % G.len();
+        let h2 = Self::_sum(key, M2) % G.len();
+        let h = (G[h1] + G[h2]) as usize % G.len();
+        if h < G.len() && KW_LUT[h].0.eq_ignore_ascii_case(key) {
+            Some(KW_LUT[h].1)
+        } else {
+            None
+        }
     }
-}
-
-impl<'a> AsRef<Token<'a>> for Token<'a> {
     #[inline(always)]
-    fn as_ref(&self) -> &Token<'a> {
-        self
+    fn _sum(key: &[u8], block: [u8; 11]) -> usize {
+        let mut sum = 0;
+        let mut i = 0;
+        while i < key.len() {
+            let char = block[i % 11];
+            sum += char as usize * (key[i] | 0x20) as usize;
+            i += 1;
+        }
+        sum
     }
 }
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index b278ec00..2bc26cd6 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -30,9 +30,15 @@ macro_rules! __sym_token {
     };
 }
 
-macro_rules! __kw {
+macro_rules! __kw_misc {
     ($ident:ident) => {
-        $crate::engine::ql::lex::Token::Keyword($crate::engine::ql::lex::Keyword::$ident)
+        $crate::engine::ql::lex::Token::Keyword($crate::engine::ql::lex::Keyword::Misc($crate::engine::ql::lex::KeywordMisc::$ident))
+    };
+}
+
+macro_rules! __kw_stmt {
+    ($ident:ident) => {
+        $crate::engine::ql::lex::Token::Keyword($crate::engine::ql::lex::Keyword::Statement($crate::engine::ql::lex::KeywordStmt::$ident))
     };
 }
 
@@ -114,122 +120,113 @@ macro_rules! Token {
     };
     // ddl keywords
     (use) => {
-        __kw!(Use)
+        __kw_stmt!(Use)
     };
     (create) => {
-        __kw!(Create)
+        __kw_stmt!(Create)
     };
     (alter) => {
-        __kw!(Alter)
+        __kw_stmt!(Alter)
     };
     (drop) => {
-        __kw!(Drop)
-    };
-    (describe) => {
-        __kw!(Describe)
+        __kw_stmt!(Drop)
     };
     (model) => {
-        __kw!(Model)
+        __kw_misc!(Model)
     };
     (space) => {
-        __kw!(Space)
+        __kw_misc!(Space)
     };
     (primary) => {
-        __kw!(Primary)
+        __kw_misc!(Primary)
     };
     // ddl misc
     (with) => {
-        __kw!(With)
+        __kw_misc!(With)
     };
     (add) => {
-        __kw!(Add)
+        __kw_misc!(Add)
     };
     (remove) => {
-        __kw!(Remove)
+        __kw_misc!(Remove)
     };
     (sort) => {
-        __kw!(Sort)
+        __kw_misc!(Sort)
     };
     (type) => {
-        __kw!(Type)
+        __kw_misc!(Type)
     };
     // dml
     (insert) => {
-        __kw!(Insert)
+        __kw_stmt!(Insert)
     };
     (select) => {
-        __kw!(Select)
+        __kw_stmt!(Select)
     };
     (update) => {
-        __kw!(Update)
+        __kw_stmt!(Update)
     };
     (delete) => {
-        __kw!(Delete)
-    };
-    (exists) => {
-        __kw!(Exists)
-    };
-    (truncate) => {
-        __kw!(Truncate)
+        __kw_stmt!(Delete)
     };
     // dml misc
     (set) => {
-        __kw!(Set)
+        __kw_misc!(Set)
     };
     (limit) => {
-        __kw!(Limit)
+        __kw_misc!(Limit)
     };
     (from) => {
-        __kw!(From)
+        __kw_misc!(From)
     };
     (into) => {
-        __kw!(Into)
+        __kw_misc!(Into)
     };
     (where) => {
-        __kw!(Where)
+        __kw_misc!(Where)
     };
     (if) => {
-        __kw!(If)
+        __kw_misc!(If)
     };
     (and) => {
-        __kw!(And)
+        __kw_misc!(And)
     };
     (as) => {
-        __kw!(As)
+        __kw_misc!(As)
     };
     (by) => {
-        __kw!(By)
+        __kw_misc!(By)
     };
     (asc) => {
-        __kw!(Asc)
+        __kw_misc!(Asc)
     };
     (desc) => {
-        __kw!(Desc)
+        __kw_misc!(Desc)
     };
     // types
     (string) => {
-        __kw!(String)
+        __kw_misc!(String)
     };
     (binary) => {
-        __kw!(Binary)
+        __kw_misc!(Binary)
     };
     (list) => {
-        __kw!(List)
+        __kw_misc!(List)
     };
     (map) => {
-        __kw!(Map)
+        __kw_misc!(Map)
     };
     (bool) => {
-        __kw!(Bool)
+        __kw_misc!(Bool)
     };
     (int) => {
-        __kw!(Int)
+        __kw_misc!(Int)
     };
     (double) => {
-        __kw!(Double)
+        __kw_misc!(Double)
     };
     (float) => {
-        __kw!(Float)
+        __kw_misc!(Float)
     };
     // tt
     (open {}) => {
@@ -252,7 +249,7 @@ macro_rules! Token {
     };
     // misc
     (null) => {
-        __kw!(Null)
+        __kw_misc!(Null)
     };
 }
 
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 5efcf9ba..c856f19d 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -46,7 +46,7 @@ pub fn lex_insecure(src: &[u8]) -> QueryResult<Vec<Token<'_>>> {
     InsecureLexer::lex(src)
 }
 pub fn lex_secure<'a>(src: &'a [u8], query_window: usize) -> QueryResult<Vec<Token<'a>>> {
-    SecureLexer::lex(src, query_window)
+    SecureLexer::lex_with_window(src, query_window)
 }
 
 pub trait NullableData<T> {
diff --git a/server/src/engine/storage/checksum.rs b/server/src/engine/storage/checksum.rs
index aa32c3d4..dc14856e 100644
--- a/server/src/engine/storage/checksum.rs
+++ b/server/src/engine/storage/checksum.rs
@@ -43,12 +43,6 @@ impl SCrc {
             digest: CRC64.digest(),
         }
     }
-    pub fn recompute_with_new_byte(&mut self, b: u8) {
-        self.digest.update(&[b])
-    }
-    pub fn recompute_with_new_block<const N: usize>(&mut self, b: [u8; N]) {
-        self.digest.update(&b);
-    }
     pub fn recompute_with_new_var_block(&mut self, b: &[u8]) {
         self.digest.update(b)
     }
diff --git a/server/src/engine/storage/header.rs b/server/src/engine/storage/header.rs
index 129bd893..e69de29b 100644
--- a/server/src/engine/storage/header.rs
+++ b/server/src/engine/storage/header.rs
@@ -1,441 +0,0 @@
-/*
- * Created on Mon May 15 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
-    SDSS Header
-    ---
-    SDSS headers have two sections:
-        - Static record: fixed-size record with fixed-layout
-        - Dynamic record: variable-size record with version-dependent layout (> 256B)
-    +--------------------------------------------------------------+
-    |                                                              |
-    |                        STATIC RECORD                         |
-    |                            128B                              |
-    +--------------------------------------------------------------+
-    +--------------------------------------------------------------+
-    |                                                              |
-    |                                                              |
-    |                       DYNAMIC RECORD                         |
-    |                          (256+?)B                            |
-    |                                                              |
-    +--------------------------------------------------------------+
-
-    We collectively define this as the SDSS Header. We'll attempt to statically compute
-    most of the sections, but for variable records we can't do the same. Also, our target
-    is to keep the SDSS Header at around 4K with page-padding.
-*/
-
-/*
-    Static record
-    ---
-    [MAGIC (8B), [HEADER_VERSION(4B), PTR_WIDTH(1B), ENDIAN(1B), ARCH(1B), OPERATING SYSTEM(1B)]]
-
-    ☢ HEADS UP: Static record is always little endian ☢
-*/
-
-use {super::versions::HeaderVersion, crate::engine::mem::ByteStack};
-
-/// magic
-const SR0_MAGIC: u64 = 0x4F48534159414E21;
-/// host ptr width
-const SR2_PTR_WIDTH: HostPointerWidth = HostPointerWidth::new();
-/// host endian
-const SR3_ENDIAN: HostEndian = HostEndian::new();
-/// host arch
-const SR4_ARCH: HostArch = HostArch::new();
-/// host os
-const SR5_OS: HostOS = HostOS::new();
-
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
-/// Host architecture enumeration for common platforms
-pub enum HostArch {
-    X86 = 0,
-    X86_64 = 1,
-    ARM = 2,
-    ARM64 = 3,
-    MIPS = 4,
-    PowerPC = 5,
-}
-
-impl HostArch {
-    pub const fn new() -> Self {
-        if cfg!(target_arch = "x86") {
-            HostArch::X86
-        } else if cfg!(target_arch = "x86_64") {
-            HostArch::X86_64
-        } else if cfg!(target_arch = "arm") {
-            HostArch::ARM
-        } else if cfg!(target_arch = "aarch64") {
-            HostArch::ARM64
-        } else if cfg!(target_arch = "mips") {
-            HostArch::MIPS
-        } else if cfg!(target_arch = "powerpc") {
-            HostArch::PowerPC
-        } else {
-            panic!("Unsupported target architecture")
-        }
-    }
-    pub const fn try_new_with_val(v: u8) -> Option<Self> {
-        Some(match v {
-            0 => HostArch::X86,
-            1 => HostArch::X86_64,
-            2 => HostArch::ARM,
-            3 => HostArch::ARM64,
-            4 => HostArch::MIPS,
-            5 => HostArch::PowerPC,
-            _ => return None,
-        })
-    }
-    pub const fn new_with_val(v: u8) -> Self {
-        match Self::try_new_with_val(v) {
-            Some(v) => v,
-            None => panic!("unknown arch"),
-        }
-    }
-}
-
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
-/// Host OS enumeration for common operating systems
-pub enum HostOS {
-    // T1
-    Linux = 0,
-    Windows = 1,
-    MacOS = 2,
-    // T2
-    Android = 3,
-    AppleiOS = 4,
-    FreeBSD = 5,
-    OpenBSD = 6,
-    NetBSD = 7,
-    WASI = 8,
-    Emscripten = 9,
-    // T3
-    Solaris = 10,
-    Fuchsia = 11,
-    Redox = 12,
-    DragonFly = 13,
-}
-
-impl HostOS {
-    pub const fn new() -> Self {
-        if cfg!(target_os = "linux") {
-            HostOS::Linux
-        } else if cfg!(target_os = "windows") {
-            HostOS::Windows
-        } else if cfg!(target_os = "macos") {
-            HostOS::MacOS
-        } else if cfg!(target_os = "android") {
-            HostOS::Android
-        } else if cfg!(target_os = "ios") {
-            HostOS::AppleiOS
-        } else if cfg!(target_os = "freebsd") {
-            HostOS::FreeBSD
-        } else if cfg!(target_os = "openbsd") {
-            HostOS::OpenBSD
-        } else if cfg!(target_os = "netbsd") {
-            HostOS::NetBSD
-        } else if cfg!(target_os = "dragonfly") {
-            HostOS::DragonFly
-        } else if cfg!(target_os = "redox") {
-            HostOS::Redox
-        } else if cfg!(target_os = "fuchsia") {
-            HostOS::Fuchsia
-        } else if cfg!(target_os = "solaris") {
-            HostOS::Solaris
-        } else if cfg!(target_os = "emscripten") {
-            HostOS::Emscripten
-        } else if cfg!(target_os = "wasi") {
-            HostOS::WASI
-        } else {
-            panic!("unknown os")
-        }
-    }
-    pub const fn try_new_with_val(v: u8) -> Option<Self> {
-        Some(match v {
-            0 => HostOS::Linux,
-            1 => HostOS::Windows,
-            2 => HostOS::MacOS,
-            3 => HostOS::Android,
-            4 => HostOS::AppleiOS,
-            5 => HostOS::FreeBSD,
-            6 => HostOS::OpenBSD,
-            7 => HostOS::NetBSD,
-            8 => HostOS::WASI,
-            9 => HostOS::Emscripten,
-            10 => HostOS::Solaris,
-            11 => HostOS::Fuchsia,
-            12 => HostOS::Redox,
-            13 => HostOS::DragonFly,
-            _ => return None,
-        })
-    }
-    pub const fn new_with_val(v: u8) -> Self {
-        match Self::try_new_with_val(v) {
-            Some(v) => v,
-            None => panic!("unknown OS"),
-        }
-    }
-}
-
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
-/// Host endian enumeration
-pub enum HostEndian {
-    Big = 0,
-    Little = 1,
-}
-
-impl HostEndian {
-    pub const fn new() -> Self {
-        if cfg!(target_endian = "little") {
-            Self::Little
-        } else {
-            Self::Big
-        }
-    }
-    pub const fn try_new_with_val(v: u8) -> Option<Self> {
-        Some(match v {
-            0 => HostEndian::Big,
-            1 => HostEndian::Little,
-            _ => return None,
-        })
-    }
-    pub const fn new_with_val(v: u8) -> Self {
-        match Self::try_new_with_val(v) {
-            Some(v) => v,
-            None => panic!("Unknown endian"),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
-#[repr(u8)]
-/// Host pointer width enumeration
-pub enum HostPointerWidth {
-    P32 = 0,
-    P64 = 1,
-}
-
-impl HostPointerWidth {
-    pub const fn new() -> Self {
-        match sizeof!(usize) {
-            4 => Self::P32,
-            8 => Self::P64,
-            _ => panic!("unknown pointer width"),
-        }
-    }
-    pub const fn try_new_with_val(v: u8) -> Option<Self> {
-        Some(match v {
-            0 => HostPointerWidth::P32,
-            1 => HostPointerWidth::P64,
-            _ => return None,
-        })
-    }
-    pub const fn new_with_val(v: u8) -> Self {
-        match Self::try_new_with_val(v) {
-            Some(v) => v,
-            None => panic!("Unknown pointer width"),
-        }
-    }
-}
-
-#[derive(Debug, PartialEq, Clone)]
-pub struct StaticRecordUV {
-    header_version: HeaderVersion,
-    ptr_width: HostPointerWidth,
-    endian: HostEndian,
-    arch: HostArch,
-    os: HostOS,
-}
-
-impl StaticRecordUV {
-    pub const fn new(
-        header_version: HeaderVersion,
-        ptr_width: HostPointerWidth,
-        endian: HostEndian,
-        arch: HostArch,
-        os: HostOS,
-    ) -> Self {
-        Self {
-            header_version,
-            ptr_width,
-            endian,
-            arch,
-            os,
-        }
-    }
-    pub const fn header_version(&self) -> HeaderVersion {
-        self.header_version
-    }
-    pub const fn ptr_width(&self) -> HostPointerWidth {
-        self.ptr_width
-    }
-    pub const fn endian(&self) -> HostEndian {
-        self.endian
-    }
-    pub const fn arch(&self) -> HostArch {
-        self.arch
-    }
-    pub const fn os(&self) -> HostOS {
-        self.os
-    }
-    pub const fn encoded(&self) -> StaticRecordUVRaw {
-        StaticRecordUVRaw::new(
-            self.header_version(),
-            self.ptr_width(),
-            self.endian(),
-            self.arch(),
-            self.os(),
-        )
-    }
-}
-
-#[derive(Debug, PartialEq, Clone)]
-/// The static record
-pub struct StaticRecordUVRaw {
-    data: ByteStack<16>,
-}
-
-impl StaticRecordUVRaw {
-    const OFFSET_P0: usize = 0;
-    const OFFSET_P1: usize = sizeof!(u64);
-    const OFFSET_P2: usize = Self::OFFSET_P1 + sizeof!(u32);
-    const OFFSET_P3: usize = Self::OFFSET_P2 + 1;
-    const OFFSET_P4: usize = Self::OFFSET_P3 + 1;
-    const OFFSET_P5: usize = Self::OFFSET_P4 + 1;
-    const _ENSURE: () = assert!(Self::OFFSET_P5 == (sizeof!(Self) - 1));
-    #[inline(always)]
-    pub const fn new(
-        sr1_version: HeaderVersion,
-        sr2_ptr_width: HostPointerWidth,
-        sr3_endian: HostEndian,
-        sr4_arch: HostArch,
-        sr5_os: HostOS,
-    ) -> Self {
-        let mut data = [0u8; 16];
-        let magic_buf = SR0_MAGIC.to_le_bytes();
-        let version_buf = sr1_version.little_endian_u64();
-        let mut i = 0usize;
-        while i < sizeof!(u64) {
-            data[i] = magic_buf[i];
-            data[i + sizeof!(u64)] = version_buf[i];
-            i += 1;
-        }
-        data[sizeof!(u64, 2) - 4] = sr2_ptr_width.value_u8();
-        data[sizeof!(u64, 2) - 3] = sr3_endian.value_u8();
-        data[sizeof!(u64, 2) - 2] = sr4_arch.value_u8();
-        data[sizeof!(u64, 2) - 1] = sr5_os.value_u8();
-        Self {
-            data: ByteStack::new(data),
-        }
-    }
-    #[inline(always)]
-    pub const fn create(sr1_version: HeaderVersion) -> Self {
-        Self::new(sr1_version, SR2_PTR_WIDTH, SR3_ENDIAN, SR4_ARCH, SR5_OS)
-    }
-    /// Decode and validate a SR
-    ///
-    /// WARNING: NOT CONTEXTUAL! VALIDATE YOUR OWN STUFF!
-    pub fn decode_from_bytes(data: [u8; 16]) -> Option<StaticRecordUV> {
-        let _ = Self::_ENSURE;
-        let slf = Self {
-            data: ByteStack::new(data),
-        };
-        // p0: magic; the magic HAS to be the same
-        if u64::from_le(slf.data.read_qword(Self::OFFSET_P0)) != SR0_MAGIC {
-            return None;
-        }
-        let sr1_header_version = HeaderVersion::__new(slf.data.read_dword(Self::OFFSET_P1) as _);
-        let sr2_ptr = HostPointerWidth::try_new_with_val(slf.data.read_byte(Self::OFFSET_P2))?; // p2: ptr width
-        let sr3_endian = HostEndian::try_new_with_val(slf.data.read_byte(Self::OFFSET_P3))?; // p3: endian
-        let sr4_arch = HostArch::try_new_with_val(slf.data.read_byte(Self::OFFSET_P4))?; // p4: arch
-        let sr5_os = HostOS::try_new_with_val(slf.data.read_byte(Self::OFFSET_P5))?; // p5: os
-        Some(StaticRecordUV::new(
-            sr1_header_version,
-            sr2_ptr,
-            sr3_endian,
-            sr4_arch,
-            sr5_os,
-        ))
-    }
-}
-
-impl StaticRecordUVRaw {
-    pub const fn get_ref(&self) -> &[u8] {
-        self.data.slice()
-    }
-    pub const fn read_p0_magic(&self) -> u64 {
-        self.data.read_qword(Self::OFFSET_P0)
-    }
-    pub const fn read_p1_header_version(&self) -> HeaderVersion {
-        HeaderVersion::__new(self.data.read_dword(Self::OFFSET_P1) as _)
-    }
-    pub const fn read_p2_ptr_width(&self) -> HostPointerWidth {
-        HostPointerWidth::new_with_val(self.data.read_byte(Self::OFFSET_P2))
-    }
-    pub const fn read_p3_endian(&self) -> HostEndian {
-        HostEndian::new_with_val(self.data.read_byte(Self::OFFSET_P3))
-    }
-    pub const fn read_p4_arch(&self) -> HostArch {
-        HostArch::new_with_val(self.data.read_byte(Self::OFFSET_P4))
-    }
-    pub const fn read_p5_os(&self) -> HostOS {
-        HostOS::new_with_val(self.data.read_byte(Self::OFFSET_P5))
-    }
-    pub const fn decoded(&self) -> StaticRecordUV {
-        StaticRecordUV::new(
-            self.read_p1_header_version(),
-            self.read_p2_ptr_width(),
-            self.read_p3_endian(),
-            self.read_p4_arch(),
-            self.read_p5_os(),
-        )
-    }
-}
-
-#[test]
-fn test_static_record() {
-    let static_record = StaticRecordUVRaw::create(super::versions::v1::V1_HEADER_VERSION);
-    assert_eq!(static_record.read_p0_magic(), SR0_MAGIC);
-    assert_eq!(
-        static_record.read_p1_header_version(),
-        super::versions::v1::V1_HEADER_VERSION
-    );
-    assert_eq!(static_record.read_p2_ptr_width(), HostPointerWidth::new());
-    assert_eq!(static_record.read_p3_endian(), HostEndian::new());
-    assert_eq!(static_record.read_p4_arch(), HostArch::new());
-    assert_eq!(static_record.read_p5_os(), HostOS::new());
-}
-
-#[test]
-fn test_static_record_encode_decode() {
-    let static_record = StaticRecordUVRaw::create(super::versions::v1::V1_HEADER_VERSION);
-    let static_record_decoded =
-        StaticRecordUVRaw::decode_from_bytes(static_record.data.data_copy()).unwrap();
-    assert_eq!(static_record.decoded(), static_record_decoded);
-}
diff --git a/server/src/engine/storage/mod.rs b/server/src/engine/storage/mod.rs
index 6f0d2a15..540fb990 100644
--- a/server/src/engine/storage/mod.rs
+++ b/server/src/engine/storage/mod.rs
@@ -27,7 +27,6 @@
 //! Implementations of the Skytable Disk Storage Subsystem (SDSS)
 
 mod checksum;
-mod header;
 mod versions;
 // impls
 pub mod v1;
diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs
index 9a5bba1f..ee61ddb0 100644
--- a/server/src/engine/storage/v1/batch_jrnl/persist.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs
@@ -67,7 +67,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
             f: SDSSFileTrackedWriter::new(file),
         })
     }
-    pub fn close(mut self) -> RuntimeResult<()> {
+    pub fn close(&mut self) -> RuntimeResult<()> {
         if self
             .f
             .inner_file()
diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs
index c8b24696..37c826f1 100644
--- a/server/src/engine/storage/v1/batch_jrnl/restore.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs
@@ -127,6 +127,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
             Self::apply_batch(model, batch)
         })
     }
+    #[cfg(test)]
     pub(in crate::engine::storage::v1) fn read_all_batches(
         &mut self,
     ) -> RuntimeResult<Vec<NormalBatch>> {
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index 1a92b96a..c4762c67 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -157,13 +157,6 @@ impl GenericDictEntryMD {
             dscr: data[8],
         }
     }
-    /// encode md
-    pub(crate) fn encode(klen: usize, dscr: u8) -> [u8; 9] {
-        let mut ret = [0u8; 9];
-        ret[..8].copy_from_slice(&klen.u64_bytes_le());
-        ret[8] = dscr;
-        ret
-    }
 }
 
 impl PersistMapSpec for GenericDictSpec {
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index c08d2fd3..3e7dae49 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -36,10 +36,7 @@ mod tests;
 
 use {
     crate::engine::{
-        data::{
-            dict::DictEntryGeneric,
-            tag::{DataTag, TagClass},
-        },
+        data::tag::TagClass,
         error::{RuntimeResult, StorageError},
         idx::{AsKey, AsValue},
         mem::BufferedScanner,
@@ -51,6 +48,7 @@ type VecU8 = Vec<u8>;
 
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, sky_macros::EnumMethods)]
 #[repr(u8)]
+#[allow(unused)]
 /// Disambiguation for data
 pub enum PersistTypeDscr {
     Null = 0,
@@ -64,6 +62,7 @@ pub enum PersistTypeDscr {
     Dict = 8,
 }
 
+#[allow(unused)]
 impl PersistTypeDscr {
     /// translates the tag class definition into the dscr definition
     pub const fn translate_from_class(class: TagClass) -> Self {
@@ -79,12 +78,6 @@ impl PersistTypeDscr {
     pub const unsafe fn from_raw(v: u8) -> Self {
         core::mem::transmute(v)
     }
-    pub fn new_from_dict_gen_entry(e: &DictEntryGeneric) -> Self {
-        match e {
-            DictEntryGeneric::Map(_) => Self::Dict,
-            DictEntryGeneric::Data(dc) => Self::translate_from_class(dc.tag().tag_class()),
-        }
-    }
     /// The data in question is null (well, can we call that data afterall?)
     pub const fn is_null(&self) -> bool {
         self.value_u8() == Self::Null.value_u8()
@@ -239,6 +232,7 @@ pub trait PersistMapSpec {
 pub mod enc {
     use super::{map, PersistMapSpec, PersistObject, VecU8};
     // obj
+    #[cfg(test)]
     pub fn enc_full<Obj: PersistObject>(obj: Obj::InputType) -> Vec<u8> {
         let mut v = vec![];
         enc_full_into_buffer::<Obj>(&mut v, obj);
@@ -247,6 +241,7 @@ pub mod enc {
     pub fn enc_full_into_buffer<Obj: PersistObject>(buf: &mut VecU8, obj: Obj::InputType) {
         Obj::default_full_enc(buf, obj)
     }
+    #[cfg(test)]
     pub fn enc_full_self<Obj: PersistObject<InputType = Obj>>(obj: Obj) -> Vec<u8> {
         enc_full::<Obj>(obj)
     }
@@ -268,6 +263,7 @@ pub mod dec {
         crate::engine::{error::RuntimeResult, mem::BufferedScanner},
     };
     // obj
+    #[cfg(test)]
     pub fn dec_full<Obj: PersistObject>(data: &[u8]) -> RuntimeResult<Obj::OutputType> {
         let mut scanner = BufferedScanner::new(data);
         dec_full_from_scanner::<Obj>(&mut scanner)
@@ -277,9 +273,6 @@ pub mod dec {
     ) -> RuntimeResult<Obj::OutputType> {
         Obj::default_full_dec(scanner)
     }
-    pub fn dec_full_self<Obj: PersistObject<OutputType = Obj>>(data: &[u8]) -> RuntimeResult<Obj> {
-        dec_full::<Obj>(data)
-    }
     // dec
     pub fn dec_dict_full<PM: PersistMapSpec>(data: &[u8]) -> RuntimeResult<PM::MapType> {
         let mut scanner = BufferedScanner::new(data);
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 8daace71..647126fd 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -211,7 +211,6 @@ impl<'a> PersistObject for FieldRef<'a> {
     }
 }
 
-pub struct ModelLayout;
 pub struct ModelLayoutMD {
     model_uuid: Uuid,
     p_key_len: u64,
@@ -298,7 +297,6 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
     }
 }
 
-pub struct SpaceLayout;
 #[derive(Debug)]
 pub struct SpaceLayoutMD {
     uuid: Uuid,
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index 57f6e6cf..9e200bd6 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -43,7 +43,7 @@
 
 use {
     super::{
-        rw::{FileOpen, RawFSInterface, SDSSFileIO},
+        rw::{RawFSInterface, SDSSFileIO},
         spec,
     },
     crate::{
@@ -55,10 +55,12 @@ use {
 
 const CRC: crc::Crc<u32> = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC);
 
-pub fn open_journal<TA: JournalAdapter, Fs: RawFSInterface, F: spec::FileSpec>(
+#[cfg(test)]
+pub fn open_or_create_journal<TA: JournalAdapter, Fs: RawFSInterface, F: spec::FileSpec>(
     log_file_name: &str,
     gs: &TA::GlobalState,
-) -> RuntimeResult<FileOpen<JournalWriter<Fs, TA>>> {
+) -> RuntimeResult<super::rw::FileOpen<JournalWriter<Fs, TA>>> {
+    use super::rw::FileOpen;
     let file = match SDSSFileIO::<Fs>::open_or_create_perm_rw::<F>(log_file_name)? {
         FileOpen::Created(f) => return Ok(FileOpen::Created(JournalWriter::new(f, 0, true)?)),
         FileOpen::Existing((file, _header)) => file,
@@ -69,6 +71,21 @@ pub fn open_journal<TA: JournalAdapter, Fs: RawFSInterface, F: spec::FileSpec>(
     )?))
 }
 
+pub fn create_journal<TA: JournalAdapter, Fs: RawFSInterface, F: spec::FileSpec>(
+    log_file_name: &str,
+) -> RuntimeResult<JournalWriter<Fs, TA>> {
+    JournalWriter::new(SDSSFileIO::create::<F>(log_file_name)?, 0, true)
+}
+
+pub fn load_journal<TA: JournalAdapter, Fs: RawFSInterface, F: spec::FileSpec>(
+    log_file_name: &str,
+    gs: &TA::GlobalState,
+) -> RuntimeResult<JournalWriter<Fs, TA>> {
+    let (file, _) = SDSSFileIO::<Fs>::open::<F>(log_file_name)?;
+    let (file, last_txn_id) = JournalReader::<TA, Fs>::scroll(file, gs)?;
+    JournalWriter::new(file, last_txn_id, false)
+}
+
 /// The journal adapter
 pub trait JournalAdapter {
     /// deny any SDSS file level operations that require non-append mode writes (for example, updating the SDSS header's modify count)
@@ -162,12 +179,6 @@ impl EventSourceMarker {
 }
 
 impl JournalEntryMetadata {
-    pub const fn is_server_event(&self) -> bool {
-        self.event_source_md == EventSourceMarker::SERVER_STD
-    }
-    pub const fn is_driver_event(&self) -> bool {
-        self.event_source_md <= 1
-    }
     pub const fn event_source_marker(&self) -> Option<EventSourceMarker> {
         Some(match self.event_source_md {
             EventSourceMarker::SERVER_STD => EventSourceMarker::ServerStandard,
@@ -183,7 +194,6 @@ impl JournalEntryMetadata {
 
 pub struct JournalReader<TA, Fs: RawFSInterface> {
     log_file: SDSSFileIO<Fs>,
-    log_size: u64,
     evid: u64,
     closed: bool,
     remaining_bytes: u64,
@@ -195,7 +205,6 @@ impl<TA: JournalAdapter, Fs: RawFSInterface> JournalReader<TA, Fs> {
         let log_size = log_file.file_length()? - spec::SDSSStaticHeaderV1Compact::SIZE as u64;
         Ok(Self {
             log_file,
-            log_size,
             evid: 0,
             closed: false,
             remaining_bytes: log_size,
@@ -422,6 +431,7 @@ impl<Fs: RawFSInterface, TA> JournalWriter<Fs, TA> {
         )?;
         Ok(())
     }
+    #[cfg(test)]
     pub fn append_journal_close_and_close(mut self) -> RuntimeResult<()> {
         self.__append_journal_close_and_close()
     }
diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs
index b43e6cc1..feb5ecba 100644
--- a/server/src/engine/storage/v1/loader.rs
+++ b/server/src/engine/storage/v1/loader.rs
@@ -24,26 +24,25 @@
  *
 */
 
+#[cfg(test)]
+use crate::engine::storage::v1::{
+    rw::{FileOpen, RawFSInterface},
+    JournalWriter,
+};
 use crate::engine::{
     core::GlobalNS,
     data::uuid::Uuid,
     error::RuntimeResult,
     fractal::error::ErrorContext,
     fractal::{FractalModelDriver, ModelDrivers, ModelUniqueID},
-    storage::v1::{
-        batch_jrnl,
-        journal::{self, JournalWriter},
-        rw::{FileOpen, RawFSInterface},
-        spec, LocalFS,
-    },
+    storage::v1::{batch_jrnl, journal, spec, LocalFS},
     txn::gns::{GNSAdapter, GNSTransactionDriverAnyFS},
 };
 
 const GNS_FILE_PATH: &str = "gns.db-tlog";
-const GNS_LOG_VERSION_CODE: u32 = 0;
+const DATA_DIR: &str = "data";
 
 pub struct SEInitState {
-    pub new_instance: bool,
     pub txn_driver: GNSTransactionDriverAnyFS<super::LocalFS>,
     pub model_drivers: ModelDrivers<LocalFS>,
     pub gns: GlobalNS,
@@ -51,24 +50,31 @@ pub struct SEInitState {
 
 impl SEInitState {
     pub fn new(
-        new_instance: bool,
         txn_driver: GNSTransactionDriverAnyFS<super::LocalFS>,
         model_drivers: ModelDrivers<LocalFS>,
         gns: GlobalNS,
     ) -> Self {
         Self {
-            new_instance,
             txn_driver,
             model_drivers,
             gns,
         }
     }
-    pub fn try_init() -> RuntimeResult<Self> {
+    pub fn try_init(is_new: bool) -> RuntimeResult<Self> {
         let gns = GlobalNS::empty();
-        let gns_txn_driver = open_gns_driver(GNS_FILE_PATH, &gns)?;
-        let new_instance = gns_txn_driver.is_created();
+        let gns_txn_driver = if is_new {
+            journal::create_journal::<GNSAdapter, LocalFS, spec::GNSTransactionLogV1>(GNS_FILE_PATH)
+        } else {
+            journal::load_journal::<GNSAdapter, LocalFS, spec::GNSTransactionLogV1>(
+                GNS_FILE_PATH,
+                &gns,
+            )
+        }?;
+        if is_new {
+            std::fs::create_dir(DATA_DIR).inherit_set_dmsg("creating data directory")?;
+        }
         let mut model_drivers = ModelDrivers::new();
-        if !new_instance {
+        if !is_new {
             // this is an existing instance, so read in all data
             for (space_name, space) in gns.spaces().read().iter() {
                 let space_uuid = space.get_uuid();
@@ -86,8 +92,7 @@ impl SEInitState {
             }
         }
         Ok(SEInitState::new(
-            new_instance,
-            GNSTransactionDriverAnyFS::new(gns_txn_driver.into_inner()),
+            GNSTransactionDriverAnyFS::new(gns_txn_driver),
             model_drivers,
             gns,
         ))
@@ -116,9 +121,10 @@ impl SEInitState {
     }
 }
 
+#[cfg(test)]
 pub fn open_gns_driver<Fs: RawFSInterface>(
     path: &str,
     gns: &GlobalNS,
 ) -> RuntimeResult<FileOpen<JournalWriter<Fs, GNSAdapter>>> {
-    journal::open_journal::<GNSAdapter, Fs, spec::GNSTransactionLogV1>(path, gns)
+    journal::open_or_create_journal::<GNSAdapter, Fs, spec::GNSTransactionLogV1>(path, gns)
 }
diff --git a/server/src/engine/storage/v1/memfs.rs b/server/src/engine/storage/v1/memfs.rs
index 2ae0ef43..697ef903 100644
--- a/server/src/engine/storage/v1/memfs.rs
+++ b/server/src/engine/storage/v1/memfs.rs
@@ -61,12 +61,6 @@ pub(super) enum VNode {
 }
 
 impl VNode {
-    const fn is_file(&self) -> bool {
-        matches!(self, Self::File(_))
-    }
-    const fn is_dir(&self) -> bool {
-        matches!(self, Self::Dir(_))
-    }
     fn as_dir_mut(&mut self) -> Option<&mut HashMap<Box<str>, Self>> {
         match self {
             Self::Dir(d) => Some(d),
@@ -311,25 +305,6 @@ fn handle_item_mut<T>(
         Entry::Vacant(_) => return err_could_not_find_item(),
     }
 }
-fn handle_item<T>(fpath: &str, f: impl Fn(&VNode) -> RuntimeResult<T>) -> RuntimeResult<T> {
-    let vfs = VFS.read();
-    let mut current = &*vfs;
-    // process components
-    let (target, components) = split_target_and_components(fpath);
-    for component in components {
-        match current.get(component) {
-            Some(VNode::Dir(dir)) => {
-                current = dir;
-            }
-            Some(VNode::File(_)) => return err_file_in_dir_path(),
-            None => return err_dir_missing_in_path(),
-        }
-    }
-    match current.get(target) {
-        Some(item) => return f(item),
-        None => return err_could_not_find_item(),
-    }
-}
 fn delete_dir(fpath: &str, allow_if_non_empty: bool) -> RuntimeResult<()> {
     handle_item_mut(fpath, |node| match node.get() {
         VNode::Dir(d) => {
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index eca3f968..ac06e514 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -30,7 +30,7 @@ mod journal;
 pub(in crate::engine) mod loader;
 mod rw;
 pub mod spec;
-mod sysdb;
+pub mod sysdb;
 // hl
 pub mod inf;
 // test
@@ -40,7 +40,7 @@ mod tests;
 
 // re-exports
 pub use {
-    journal::{open_journal, JournalAdapter, JournalWriter},
+    journal::{JournalAdapter, JournalWriter},
     memfs::NullFS,
     rw::{LocalFS, RawFSInterface, SDSSFileIO},
 };
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index f6cb0a78..72028842 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -44,13 +44,8 @@ pub enum FileOpen<CF, EF = CF> {
     Existing(EF),
 }
 
+#[cfg(test)]
 impl<CF, EF> FileOpen<CF, EF> {
-    pub const fn is_created(&self) -> bool {
-        matches!(self, Self::Created(_))
-    }
-    pub const fn is_existing(&self) -> bool {
-        !self.is_created()
-    }
     pub fn into_existing(self) -> Option<EF> {
         match self {
             Self::Existing(e) => Some(e),
@@ -65,6 +60,7 @@ impl<CF, EF> FileOpen<CF, EF> {
     }
 }
 
+#[cfg(test)]
 impl<F> FileOpen<F> {
     pub fn into_inner(self) -> F {
         match self {
@@ -441,9 +437,6 @@ impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
     pub fn seek_from_start(&mut self, by: u64) -> RuntimeResult<()> {
         self.f.fext_seek_ahead_from_start_by(by)
     }
-    pub fn trim_file_to(&mut self, to: u64) -> RuntimeResult<()> {
-        self.f.fw_truncate_to(to)
-    }
     pub fn retrieve_cursor(&mut self) -> RuntimeResult<u64> {
         self.f.fext_cursor()
     }
diff --git a/server/src/engine/storage/v1/spec.rs b/server/src/engine/storage/v1/spec.rs
index fe230bdc..9f556c1d 100644
--- a/server/src/engine/storage/v1/spec.rs
+++ b/server/src/engine/storage/v1/spec.rs
@@ -37,10 +37,7 @@ use {
     crate::{
         engine::{
             error::{RuntimeResult, StorageError},
-            storage::{
-                header::{HostArch, HostEndian, HostOS, HostPointerWidth},
-                versions::{self, DriverVersion, HeaderVersion, ServerVersion},
-            },
+            storage::versions::{self, DriverVersion, HeaderVersion, ServerVersion},
         },
         util::os,
     },
@@ -54,32 +51,142 @@ use {
     meta
 */
 
-/// The file scope
 #[repr(u8)]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
-pub enum FileScope {
-    Journal = 0,
-    DataBatch = 1,
-    FlatmapData = 2,
+/// Host architecture enumeration for common platforms
+pub enum HostArch {
+    X86 = 0,
+    X86_64 = 1,
+    ARM = 2,
+    ARM64 = 3,
+    MIPS = 4,
+    PowerPC = 5,
 }
 
-impl FileScope {
-    pub const fn try_new(id: u64) -> Option<Self> {
-        Some(match id {
-            0 => Self::Journal,
-            1 => Self::DataBatch,
-            2 => Self::FlatmapData,
-            _ => return None,
-        })
+impl HostArch {
+    pub const fn new() -> Self {
+        if cfg!(target_arch = "x86") {
+            HostArch::X86
+        } else if cfg!(target_arch = "x86_64") {
+            HostArch::X86_64
+        } else if cfg!(target_arch = "arm") {
+            HostArch::ARM
+        } else if cfg!(target_arch = "aarch64") {
+            HostArch::ARM64
+        } else if cfg!(target_arch = "mips") {
+            HostArch::MIPS
+        } else if cfg!(target_arch = "powerpc") {
+            HostArch::PowerPC
+        } else {
+            panic!("Unsupported target architecture")
+        }
     }
-    pub const fn new(id: u64) -> Self {
-        match Self::try_new(id) {
-            Some(v) => v,
-            None => panic!("unknown filescope"),
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+/// Host OS enumeration for common operating systems
+pub enum HostOS {
+    // T1
+    Linux = 0,
+    Windows = 1,
+    MacOS = 2,
+    // T2
+    Android = 3,
+    AppleiOS = 4,
+    FreeBSD = 5,
+    OpenBSD = 6,
+    NetBSD = 7,
+    WASI = 8,
+    Emscripten = 9,
+    // T3
+    Solaris = 10,
+    Fuchsia = 11,
+    Redox = 12,
+    DragonFly = 13,
+}
+
+impl HostOS {
+    pub const fn new() -> Self {
+        if cfg!(target_os = "linux") {
+            HostOS::Linux
+        } else if cfg!(target_os = "windows") {
+            HostOS::Windows
+        } else if cfg!(target_os = "macos") {
+            HostOS::MacOS
+        } else if cfg!(target_os = "android") {
+            HostOS::Android
+        } else if cfg!(target_os = "ios") {
+            HostOS::AppleiOS
+        } else if cfg!(target_os = "freebsd") {
+            HostOS::FreeBSD
+        } else if cfg!(target_os = "openbsd") {
+            HostOS::OpenBSD
+        } else if cfg!(target_os = "netbsd") {
+            HostOS::NetBSD
+        } else if cfg!(target_os = "dragonfly") {
+            HostOS::DragonFly
+        } else if cfg!(target_os = "redox") {
+            HostOS::Redox
+        } else if cfg!(target_os = "fuchsia") {
+            HostOS::Fuchsia
+        } else if cfg!(target_os = "solaris") {
+            HostOS::Solaris
+        } else if cfg!(target_os = "emscripten") {
+            HostOS::Emscripten
+        } else if cfg!(target_os = "wasi") {
+            HostOS::WASI
+        } else {
+            panic!("unknown os")
+        }
+    }
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+/// Host endian enumeration
+pub enum HostEndian {
+    Big = 0,
+    Little = 1,
+}
+
+impl HostEndian {
+    pub const fn new() -> Self {
+        if cfg!(target_endian = "little") {
+            Self::Little
+        } else {
+            Self::Big
         }
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+#[repr(u8)]
+/// Host pointer width enumeration
+pub enum HostPointerWidth {
+    P32 = 0,
+    P64 = 1,
+}
+
+impl HostPointerWidth {
+    pub const fn new() -> Self {
+        match sizeof!(usize) {
+            4 => Self::P32,
+            8 => Self::P64,
+            _ => panic!("unknown pointer width"),
+        }
+    }
+}
+
+/// The file scope
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
+pub enum FileScope {
+    Journal = 0,
+    DataBatch = 1,
+    FlatmapData = 2,
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, sky_macros::EnumMethods)]
 #[repr(u8)]
 pub enum FileSpecifier {
@@ -90,25 +197,6 @@ pub enum FileSpecifier {
     TestTransactionLog = 0xFF,
 }
 
-impl FileSpecifier {
-    pub const fn try_new(v: u32) -> Option<Self> {
-        Some(match v {
-            0 => Self::GNSTxnLog,
-            1 => Self::TableDataBatch,
-            2 => Self::SysDB,
-            #[cfg(test)]
-            0xFF => Self::TestTransactionLog,
-            _ => return None,
-        })
-    }
-    pub const fn new(v: u32) -> Self {
-        match Self::try_new(v) {
-            Some(v) => v,
-            _ => panic!("unknown filespecifier"),
-        }
-    }
-}
-
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
 pub struct FileSpecifierVersion(u16);
 impl FileSpecifierVersion {
@@ -450,6 +538,7 @@ impl SDSSStaticHeaderV1Compact {
     }
 }
 
+#[allow(unused)]
 impl SDSSStaticHeaderV1Compact {
     pub fn header_version(&self) -> HeaderVersion {
         self.magic_header_version
diff --git a/server/src/engine/storage/v1/sysdb.rs b/server/src/engine/storage/v1/sysdb.rs
index f7d48181..7b4dbd4c 100644
--- a/server/src/engine/storage/v1/sysdb.rs
+++ b/server/src/engine/storage/v1/sysdb.rs
@@ -27,7 +27,7 @@
 use {
     super::rw::FileOpen,
     crate::engine::{
-        config::ConfigAuth,
+        config::{ConfigAuth, ConfigMode},
         data::{cell::Datacell, DictEntryGeneric, DictGeneric},
         error::{RuntimeResult, StorageError},
         fractal::config::{SysAuth, SysAuthUser, SysConfig, SysHostData},
@@ -57,6 +57,15 @@ pub enum SystemStoreInitState {
     UpdatedRoot,
 }
 
+impl SystemStoreInitState {
+    pub const fn is_created(&self) -> bool {
+        matches!(self, Self::Created)
+    }
+    pub const fn is_existing_updated_root(&self) -> bool {
+        matches!(self, Self::UpdatedRoot)
+    }
+}
+
 #[derive(Debug, PartialEq)]
 /// Result of initializing the system store (sysdb)
 pub struct SystemStoreInit {
@@ -76,20 +85,22 @@ impl SystemStoreInit {
 /// - If it exists, look for config changes and sync them
 pub fn open_system_database<Fs: RawFSInterface>(
     auth: ConfigAuth,
+    mode: ConfigMode,
 ) -> RuntimeResult<SystemStoreInit> {
-    open_or_reinit_system_database::<Fs>(auth, SYSDB_PATH, SYSDB_COW_PATH)
+    open_or_reinit_system_database::<Fs>(auth, mode, SYSDB_PATH, SYSDB_COW_PATH)
 }
 
 /// Open or re-initialize the system database
 pub fn open_or_reinit_system_database<Fs: RawFSInterface>(
     auth: ConfigAuth,
+    run_mode: ConfigMode,
     sysdb_path: &str,
     sysdb_path_cow: &str,
 ) -> RuntimeResult<SystemStoreInit> {
     let sysdb_file = match SDSSFileIO::<Fs>::open_or_create_perm_rw::<spec::SysDBV1>(sysdb_path)? {
         FileOpen::Created(new) => {
             // init new syscfg
-            let new_syscfg = SysConfig::new_auth(auth);
+            let new_syscfg = SysConfig::new_auth(auth, run_mode);
             sync_system_database_to(&new_syscfg, new)?;
             return Ok(SystemStoreInit::new(
                 new_syscfg,
@@ -98,7 +109,7 @@ pub fn open_or_reinit_system_database<Fs: RawFSInterface>(
         }
         FileOpen::Existing((ex, _)) => ex,
     };
-    let prev_sysdb = decode_system_database(sysdb_file)?;
+    let prev_sysdb = decode_system_database(sysdb_file, run_mode)?;
     let state;
     // see if settings have changed
     if prev_sysdb
@@ -119,6 +130,7 @@ pub fn open_or_reinit_system_database<Fs: RawFSInterface>(
             prev_sysdb.host_data().settings_version()
                 + !matches!(state, SystemStoreInitState::Unchanged) as u32,
         ),
+        run_mode,
     );
     // sync
     sync_system_database_to(
@@ -185,6 +197,7 @@ fn rkey<T>(
 /// Decode the system database
 pub fn decode_system_database<Fs: RawFSInterface>(
     mut f: SDSSFileIO<Fs>,
+    run_mode: ConfigMode,
 ) -> RuntimeResult<SysConfig> {
     let mut sysdb_data =
         inf::dec::dec_dict_full::<inf::map::GenericDictSpec>(&f.load_remaining_into_buffer()?)?;
@@ -230,5 +243,6 @@ pub fn decode_system_database<Fs: RawFSInterface>(
     Ok(SysConfig::new(
         RwLock::new(sys_auth),
         SysHostData::new(sc, sv as u32),
+        run_mode,
     ))
 }
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index b6aa9802..c0d5fe7e 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -36,15 +36,20 @@ mod sysdb {
             super::sysdb::{self, SystemStoreInitState},
             VirtualFS as VFS,
         },
-        crate::engine::config::{AuthDriver, ConfigAuth},
+        crate::engine::config::{AuthDriver, ConfigAuth, ConfigMode},
     };
     fn open_sysdb(
         auth_config: ConfigAuth,
         sysdb_path: &str,
         sysdb_cow_path: &str,
     ) -> sysdb::SystemStoreInit {
-        sysdb::open_or_reinit_system_database::<VFS>(auth_config, sysdb_path, sysdb_cow_path)
-            .unwrap()
+        sysdb::open_or_reinit_system_database::<VFS>(
+            auth_config,
+            ConfigMode::Dev,
+            sysdb_path,
+            sysdb_cow_path,
+        )
+        .unwrap()
     }
     #[test]
     fn open_close() {
diff --git a/server/src/engine/storage/v1/tests/batch.rs b/server/src/engine/storage/v1/tests/batch.rs
index 2418883e..4dd654e8 100644
--- a/server/src/engine/storage/v1/tests/batch.rs
+++ b/server/src/engine/storage/v1/tests/batch.rs
@@ -144,7 +144,7 @@ fn empty_multi_open_reopen() {
         ),
     );
     for _ in 0..100 {
-        let writer = open_batch_data("empty_multi_open_reopen.db-btlog", &mdl);
+        let mut writer = open_batch_data("empty_multi_open_reopen.db-btlog", &mdl);
         writer.close().unwrap();
     }
 }
diff --git a/server/src/engine/storage/v1/tests/tx.rs b/server/src/engine/storage/v1/tests/tx.rs
index d1df0351..17dae70d 100644
--- a/server/src/engine/storage/v1/tests/tx.rs
+++ b/server/src/engine/storage/v1/tests/tx.rs
@@ -53,13 +53,6 @@ impl Database {
     fn reset(&self) {
         *self.data.borrow_mut() = [0; 10];
     }
-    fn txn_reset(
-        &self,
-        txn_writer: &mut JournalWriter<super::VirtualFS, DatabaseTxnAdapter>,
-    ) -> RuntimeResult<()> {
-        self.reset();
-        txn_writer.append_event(TxEvent::Reset)
-    }
     fn set(&self, pos: usize, val: u8) {
         self.data.borrow_mut()[pos] = val;
     }
@@ -73,7 +66,9 @@ impl Database {
         txn_writer.append_event(TxEvent::Set(pos, val))
     }
 }
+
 pub enum TxEvent {
+    #[allow(unused)]
     Reset,
     Set(usize, u8),
 }
@@ -135,7 +130,7 @@ fn open_log(
     log_name: &str,
     db: &Database,
 ) -> RuntimeResult<JournalWriter<super::VirtualFS, DatabaseTxnAdapter>> {
-    journal::open_journal::<DatabaseTxnAdapter, super::VirtualFS, spec::TestFile>(log_name, db)
+    journal::open_or_create_journal::<DatabaseTxnAdapter, super::VirtualFS, spec::TestFile>(log_name, db)
         .map(|v| v.into_inner())
 }
 
diff --git a/server/src/engine/sync/atm.rs b/server/src/engine/sync/atm.rs
index ad7ac2fa..eb531271 100644
--- a/server/src/engine/sync/atm.rs
+++ b/server/src/engine/sync/atm.rs
@@ -55,14 +55,6 @@ impl<T> fmt::Debug for Atomic<T> {
 }
 
 impl<T: Pointable> Atomic<T> {
-    /// Instantiates a new atomic
-    ///
-    /// **This will allocate**
-    pub fn new_alloc(t: T) -> Self {
-        Self {
-            a: CBAtomic::new(t),
-        }
-    }
     #[inline(always)]
     pub const fn null() -> Self {
         Self {
@@ -84,20 +76,6 @@ impl<T: Pointable> Atomic<T> {
         self.a.compare_exchange(o, n, s, f, g)
     }
     #[inline(always)]
-    pub fn cx_weak<'g, P>(
-        &self,
-        o: Shared<'g, T>,
-        n: P,
-        s: Ordering,
-        f: Ordering,
-        g: &'g Guard,
-    ) -> CxResult<'g, T, P>
-    where
-        P: Pointer<T>,
-    {
-        self.a.compare_exchange_weak(o, n, s, f, g)
-    }
-    #[inline(always)]
     pub fn cx_rel<'g, P>(&self, o: Shared<'g, T>, n: P, g: &'g Guard) -> CxResult<'g, T, P>
     where
         P: Pointer<T>,
diff --git a/server/src/engine/sync/cell.rs b/server/src/engine/sync/cell.rs
index ef2581fa..3960a2ee 100644
--- a/server/src/engine/sync/cell.rs
+++ b/server/src/engine/sync/cell.rs
@@ -26,17 +26,15 @@
 
 use {
     super::{
-        atm::{upin, Atomic, Guard, Owned, Shared, ORD_ACQ, ORD_REL, ORD_SEQ},
+        atm::{ORD_ACQ, ORD_SEQ},
         Backoff,
     },
     core::{
-        marker::PhantomData,
         mem,
         ops::Deref,
         ptr,
         sync::atomic::{AtomicBool, AtomicPtr},
     },
-    parking_lot::{Mutex, MutexGuard},
 };
 
 /// A lazily intialized, or _call by need_ value
@@ -131,146 +129,3 @@ impl<T, F> Drop for Lazy<T, F> {
         }
     }
 }
-
-/// A [`TMCell`] provides atomic reads and serialized writes; the `static` is a CB hack
-#[derive(Debug)]
-pub struct TMCell<T: 'static> {
-    a: Atomic<T>,
-    g: Mutex<()>,
-}
-
-impl<T: 'static> TMCell<T> {
-    pub fn new(v: T) -> Self {
-        Self {
-            a: Atomic::new_alloc(v),
-            g: Mutex::new(()),
-        }
-    }
-    pub fn begin_write_txn<'a, 'g>(&'a self, g: &'g Guard) -> TMCellWriteTxn<'a, 'g, T> {
-        let wg = self.g.lock();
-        let snapshot = self.a.ld_acq(g);
-        let data: &'g T = unsafe {
-            // UNSAFE(@ohsayan): first, non-null (TMCell is never null). second, the guard
-            snapshot.deref()
-        };
-        TMCellWriteTxn::new(data, &self.a, wg)
-    }
-    pub fn begin_read_txn<'a, 'g>(&'a self, g: &'g Guard) -> TMCellReadTxn<'a, 'g, T> {
-        let snapshot = self.a.ld_acq(g);
-        let data: &'g T = unsafe {
-            // UNSAFE(@ohsayan): non-null and the guard
-            snapshot.deref()
-        };
-        TMCellReadTxn::new(data)
-    }
-}
-
-impl<T> Drop for TMCell<T> {
-    fn drop(&mut self) {
-        unsafe {
-            // UNSAFE(@ohsayan): Sole owner with mutable access
-            let g = upin();
-            let shptr = self.a.ld_rlx(g);
-            g.defer_destroy(shptr);
-        }
-    }
-}
-
-unsafe impl<T: Send> Send for TMCell<T> {}
-unsafe impl<T: Sync> Sync for TMCell<T> {}
-
-#[derive(Debug)]
-pub struct TMCellReadTxn<'a, 'g, T: 'static> {
-    d: &'g T,
-    _m: PhantomData<&'a TMCell<T>>,
-}
-
-impl<'a, 'g, T> TMCellReadTxn<'a, 'g, T> {
-    #[inline(always)]
-    pub fn new(d: &'g T) -> Self {
-        Self { d, _m: PhantomData }
-    }
-    #[inline(always)]
-    pub fn read(&self) -> &'g T {
-        self.d
-    }
-}
-
-impl<'a, 'g, T: Clone> TMCellReadTxn<'a, 'g, T> {
-    #[inline(always)]
-    pub fn read_copied(&self) -> T {
-        self.read().clone()
-    }
-}
-
-impl<'a, 'g, T: Copy> TMCellReadTxn<'a, 'g, T> {
-    fn read_copy(&self) -> T {
-        *self.d
-    }
-}
-
-impl<'a, 'g, T> Deref for TMCellReadTxn<'a, 'g, T> {
-    type Target = T;
-    fn deref(&self) -> &'g Self::Target {
-        self.d
-    }
-}
-
-unsafe impl<'a, 'g, T: Send> Send for TMCellReadTxn<'a, 'g, T> {}
-unsafe impl<'a, 'g, T: Sync> Sync for TMCellReadTxn<'a, 'g, T> {}
-
-#[derive(Debug)]
-pub struct TMCellWriteTxn<'a, 'g, T: 'static> {
-    d: &'g T,
-    a: &'a Atomic<T>,
-    g: MutexGuard<'a, ()>,
-}
-
-impl<'a, 'g, T> TMCellWriteTxn<'a, 'g, T> {
-    #[inline(always)]
-    pub fn new(d: &'g T, a: &'a Atomic<T>, g: MutexGuard<'a, ()>) -> Self {
-        Self { d, a, g }
-    }
-    pub fn publish_commit(self, new: T, g: &'g Guard) {
-        self._commit(new, g, |p| {
-            unsafe {
-                // UNSAFE(@ohsayan): Unlinked
-                g.defer_destroy(p);
-            }
-        })
-    }
-    fn _commit<F, R>(self, new: T, g: &'g Guard, f: F) -> R
-    where
-        F: FnOnce(Shared<T>) -> R,
-    {
-        let new = Owned::new(new);
-        let r = self.a.swap(new, ORD_REL, g);
-        f(r)
-    }
-    #[inline(always)]
-    pub fn read(&self) -> &'g T {
-        self.d
-    }
-}
-
-impl<'a, 'g, T: Clone> TMCellWriteTxn<'a, 'g, T> {
-    #[inline(always)]
-    pub fn read_copied(&self) -> T {
-        self.read().clone()
-    }
-}
-
-impl<'a, 'g, T: Copy> TMCellWriteTxn<'a, 'g, T> {
-    fn read_copy(&self) -> T {
-        *self.d
-    }
-}
-
-impl<'a, 'g, T> Deref for TMCellWriteTxn<'a, 'g, T> {
-    type Target = T;
-    fn deref(&self) -> &'g Self::Target {
-        self.d
-    }
-}
-
-unsafe impl<'a, 'g, T: Sync> Sync for TMCellWriteTxn<'a, 'g, T> {}
diff --git a/server/src/engine/sync/queue.rs b/server/src/engine/sync/queue.rs
index 640c3367..607268f6 100644
--- a/server/src/engine/sync/queue.rs
+++ b/server/src/engine/sync/queue.rs
@@ -24,10 +24,12 @@
  *
 */
 
+#[cfg(test)]
+use crossbeam_epoch::pin;
 use {
     super::atm::Atomic,
     crate::engine::mem::CachePadded,
-    crossbeam_epoch::{pin, unprotected, Guard, Owned, Shared},
+    crossbeam_epoch::{unprotected, Guard, Owned, Shared},
     std::{mem::MaybeUninit, sync::atomic::Ordering},
 };
 
@@ -67,10 +69,6 @@ impl<T> Queue<T> {
         slf.tail.store(sentinel, Ordering::Relaxed);
         slf
     }
-    pub fn blocking_enqueue_autopin(&self, new: T) {
-        let g = pin();
-        self.blocking_enqueue(new, &g);
-    }
     pub fn blocking_enqueue(&self, new: T, g: &Guard) {
         let newptr = Owned::new(QNode::new_data(new)).into_shared(g);
         loop {
@@ -115,10 +113,6 @@ impl<T> Queue<T> {
             }
         }
     }
-    pub fn blocking_try_dequeue_autopin(&self) -> Option<T> {
-        let g = pin();
-        self.blocking_try_dequeue(&g)
-    }
     pub fn blocking_try_dequeue(&self, g: &Guard) -> Option<T> {
         loop {
             // get current head
diff --git a/server/src/engine/sync/smart.rs b/server/src/engine/sync/smart.rs
index 66254cb8..3d2b79cc 100644
--- a/server/src/engine/sync/smart.rs
+++ b/server/src/engine/sync/smart.rs
@@ -24,6 +24,7 @@
  *
 */
 
+
 use {
     super::atm::{ORD_ACQ, ORD_REL, ORD_RLX},
     std::{
@@ -40,8 +41,6 @@ use {
     },
 };
 
-pub type BytesRC = SliceRC<u8>;
-
 #[derive(Debug, Clone)]
 pub struct StrRC {
     base: SliceRC<u8>,
@@ -127,14 +126,6 @@ impl<T> SliceRC<T> {
         }
     }
     #[inline(always)]
-    pub fn from_bx(b: Box<[T]>) -> Self {
-        let mut b = ManuallyDrop::new(b);
-        unsafe {
-            // UNSAFE(@ohsayan): non-null from the slice as usual
-            Self::new(NonNull::new_unchecked(b.as_mut_ptr()), b.len())
-        }
-    }
-    #[inline(always)]
     pub fn as_slice(&self) -> &[T] {
         unsafe {
             // UNSAFE(@ohsayan): rc guard + ctor
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index cb0f85d6..9befe0a1 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -24,8 +24,6 @@
  *
 */
 
-#[cfg(test)]
-use crate::engine::storage::v1::memfs::VirtualFS;
 use {
     crate::{
         engine::{
@@ -58,11 +56,6 @@ pub use {
     space::{AlterSpaceTxn, CreateSpaceTxn, DropSpaceTxn},
 };
 
-pub type GNSTransactionDriverNullZero =
-    GNSTransactionDriverAnyFS<crate::engine::storage::v1::NullFS>;
-#[cfg(test)]
-pub type GNSTransactionDriverVFS = GNSTransactionDriverAnyFS<VirtualFS>;
-
 /// The GNS transaction driver is used to handle DDL transactions
 pub struct GNSTransactionDriverAnyFS<Fs: RawFSInterface = LocalFS> {
     journal: JournalWriter<Fs, GNSAdapter>,
@@ -198,6 +191,7 @@ pub struct SpaceIDRes {
 }
 
 impl SpaceIDRes {
+    #[cfg(test)]
     pub fn new(uuid: Uuid, name: Box<str>) -> Self {
         Self { uuid, name }
     }
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index 4db39bdf..39bbfe0a 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -92,6 +92,7 @@ pub struct ModelIDRes {
 }
 
 impl ModelIDRes {
+    #[cfg(test)]
     pub fn new(
         space_id: super::SpaceIDRes,
         model_name: Box<str>,
diff --git a/server/src/kvengine/encoding.rs b/server/src/kvengine/encoding.rs
deleted file mode 100644
index 6f43ae39..00000000
--- a/server/src/kvengine/encoding.rs
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Created on Thu Jul 01 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
- This cannot be the work of a single person! A big thanks to:
- - Björn Höhrmann: https://bjoern.hoehrmann.de/
- - Professor Lemire: https://scholar.google.com/citations?user=q1ja-G8AAAAJ
- - Travis Downs: https://github.com/travisdowns
-*/
-
-/*
- * The UTF-8 validation that we use here uses an encoded finite state machine defined in this file.
- * A branchless (no cond) finite state machine is used which greatly simplifies how things work
- * than some amazing libraries out there while also providing huge performance and computationally
- * economical benefits. The dual stream that I have used here offers the best performance than a
- * triple or quad channel SM evaluation. I attempted a SM evaluation with four streams and sure
- * it was about 75% faster than the standard library's evaluation, but however fell short to the
- * 300% improvement that I got over a dual stream. Also, don't get too excited because this looks
- * like something recursive or _threadable_. DON'T. Call stacks have their own costs and why do
- * it at all when we can use a simple loop?
- * Now for the threading bit: remember, this is not a single game that you have to win.
- * There will potentially be millions if not thousands of callers requesting validation and
- * imagine spawning two threads for every validation. Firstly, you'll have to wait on the OS/Kernel
- * to hand over the response to a fork. Secondly, so many threads are useless because it'll just
- * burden the scheduler and hurt performance taking away any possible performance benefits that
- * you could've had. In a single benchmark, the threaded implementation might make you happy
- * and turn out to be 600% faster. But try doing multiple evaluations in parallel: you'll know
- * what we're talking about. Eliding bound checks gives us a ~2-5% edge over the one that
- * is checked. Why elide them? Just because we reduce our assembly size by ~15%!
- *
- * - Sayan N. <ohsayan@outlook.com> (July, 2021)
-*/
-
-use crate::{
-    corestore::booltable::{BoolTable, TwoBitLUT},
-    protocol::iter::{AnyArrayIter, BorrowedAnyArrayIter},
-};
-
-type PairFn = fn(&[u8], &[u8]) -> bool;
-
-pub const ENCODING_LUT_ITER: BoolTable<fn(BorrowedAnyArrayIter) -> bool> =
-    BoolTable::new(is_okay_encoded_iter, is_okay_no_encoding_iter);
-pub const ENCODING_LUT_ITER_PAIR: TwoBitLUT<fn(&AnyArrayIter) -> bool> = TwoBitLUT::new(
-    pair_is_okay_encoded_iter_ff,
-    pair_is_okay_encoded_iter_ft,
-    pair_is_okay_encoded_iter_tf,
-    pair_is_okay_encoded_iter_tt,
-);
-pub const ENCODING_LUT: BoolTable<fn(&[u8]) -> bool> =
-    BoolTable::new(self::is_okay_encoded, self::is_okay_no_encoding);
-pub const ENCODING_LUT_PAIR: TwoBitLUT<PairFn> = TwoBitLUT::new(
-    self::is_okay_encoded_pair_ff,
-    self::is_okay_encoded_pair_ft,
-    self::is_okay_encoded_pair_tf,
-    self::is_okay_encoded_pair_tt,
-);
-
-/// This table maps bytes to character classes that helps us reduce the size of the
-/// transition table and generate bitmasks
-static UTF8_MAP_BYTE_TO_CHAR_CLASS: [u8; 256] = [
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
-    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
-    8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-    10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 11, 6, 6, 6, 5, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
-    8,
-];
-
-/// This table is a transition table that maps the combination of a state of the
-/// automaton and a char class to a state
-static UTF8_TRANSITION_MAP: [u8; 108] = [
-    0, 12, 24, 36, 60, 96, 84, 12, 12, 12, 48, 72, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
-    12, 0, 12, 12, 12, 12, 12, 0, 12, 0, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 24, 12, 12,
-    12, 12, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 12, 12, 24, 12, 12,
-    12, 12, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12, 12, 36, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12,
-    12, 36, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
-];
-
-pub const fn pair_is_okay_encoded_iter_ff(_inp: &AnyArrayIter<'_>) -> bool {
-    true
-}
-
-pub fn pair_is_okay_encoded_iter_ft(inp: &AnyArrayIter<'_>) -> bool {
-    unsafe {
-        let mut vptr = inp.as_ptr().add(1);
-        let eptr = inp.as_ptr().add(inp.len());
-        let mut state = true;
-        while vptr < eptr && state {
-            state = self::is_utf8((*vptr).as_slice());
-            // only forward values
-            vptr = vptr.add(2);
-        }
-        state
-    }
-}
-
-pub fn pair_is_okay_encoded_iter_tf(inp: &AnyArrayIter<'_>) -> bool {
-    unsafe {
-        let mut kptr = inp.as_ptr();
-        let eptr = kptr.add(inp.len());
-        let mut state = true;
-        while kptr < eptr && state {
-            state = self::is_utf8((*kptr).as_slice());
-            // only forward keys
-            kptr = kptr.add(2);
-        }
-        state
-    }
-}
-
-pub fn pair_is_okay_encoded_iter_tt(inp: &AnyArrayIter<'_>) -> bool {
-    unsafe {
-        let mut kptr = inp.as_ptr();
-        let mut vptr = inp.as_ptr().add(1);
-        let eptr = kptr.add(inp.len());
-        let mut state = true;
-        while vptr < eptr && state {
-            state = self::is_utf8((*kptr).as_slice()) && self::is_utf8((*vptr).as_slice());
-            kptr = kptr.add(2);
-            vptr = vptr.add(2);
-        }
-        state
-    }
-}
-
-pub fn is_okay_encoded_iter(mut inp: BorrowedAnyArrayIter<'_>) -> bool {
-    inp.all(self::is_okay_encoded)
-}
-
-pub const fn is_okay_no_encoding_iter(_inp: BorrowedAnyArrayIter<'_>) -> bool {
-    true
-}
-
-pub fn is_okay_encoded(inp: &[u8]) -> bool {
-    self::is_utf8(inp)
-}
-
-pub const fn is_okay_no_encoding(_inp: &[u8]) -> bool {
-    true
-}
-
-pub fn is_okay_encoded_pair_tt(a: &[u8], b: &[u8]) -> bool {
-    is_okay_encoded(a) && is_okay_encoded(b)
-}
-
-pub fn is_okay_encoded_pair_tf(a: &[u8], _b: &[u8]) -> bool {
-    is_okay_encoded(a)
-}
-
-pub fn is_okay_encoded_pair_ft(_a: &[u8], b: &[u8]) -> bool {
-    is_okay_encoded(b)
-}
-
-pub const fn is_okay_encoded_pair_ff(_a: &[u8], _b: &[u8]) -> bool {
-    true
-}
-
-macro_rules! utf_transition {
-    ($idx:expr) => {
-        ucidx!(UTF8_TRANSITION_MAP, $idx)
-    };
-}
-
-macro_rules! utfmap {
-    ($idx:expr) => {
-        ucidx!(UTF8_MAP_BYTE_TO_CHAR_CLASS, $idx)
-    };
-}
-
-/// This method uses a dual-stream deterministic finite automaton
-/// [(DFA)](https://en.wikipedia.org/wiki/Deterministic_finite_automaton) that is used to validate
-/// UTF-8 bytes that use the encoded finite state machines defined in this module.
-///
-/// ## Tradeoffs
-/// Read my comment in the source code (or above if you are not browsing rustdoc)
-///
-/// ## Why
-/// This function gives us as much as a ~300% improvement over std's validation algorithm
-pub fn is_utf8(bytes: impl AsRef<[u8]>) -> bool {
-    let bytes = bytes.as_ref();
-    let mut half = bytes.len() / 2;
-    unsafe {
-        while ucidx!(bytes, half) <= 0xBF && ucidx!(bytes, half) >= 0x80 && half > 0 {
-            half -= 1;
-        }
-    }
-    let (mut fsm_state_1, mut fsm_state_2) = (0u8, 0u8);
-    let mut i = 0usize;
-    let mut j = half;
-    while i < half {
-        unsafe {
-            fsm_state_1 = utf_transition!((fsm_state_1 + (utfmap!((ucidx!(bytes, i))))));
-            fsm_state_2 = utf_transition!((fsm_state_2 + (utfmap!(ucidx!(bytes, j)))));
-        }
-        i += 1;
-        j += 1;
-    }
-    let mut j = half * 2;
-    while j < bytes.len() {
-        unsafe {
-            fsm_state_2 = utf_transition!((fsm_state_2 + (utfmap!(ucidx!(bytes, j)))));
-        }
-        j += 1;
-    }
-    fsm_state_1 == 0 && fsm_state_2 == 0
-}
-
-#[test]
-fn test_utf8_verity() {
-    let unicode = gen_unicode();
-    assert!(unicode.into_iter().all(self::is_utf8));
-}
-
-#[cfg(test)]
-fn gen_unicode() -> Vec<String> {
-    use std::env;
-    use std::fs;
-    use std::process::Command;
-    let mut path = env::var("ROOT_DIR").expect("ROOT_DIR unset");
-    path.push_str("/scripts/unicode.pl");
-    fs::create_dir_all("./utf8/separated").unwrap();
-    fs::create_dir_all("./utf8/unseparated").unwrap();
-    let cmd = Command::new("perl").arg("-w").arg(path).output().unwrap();
-    let stderr = String::from_utf8_lossy(&cmd.stderr);
-    assert!(stderr.is_empty(), "Perl error: `{}`", stderr);
-    let mut strings = vec![];
-    for file in fs::read_dir("utf8/separated").unwrap() {
-        strings.push(fs::read_to_string(file.unwrap().path()).unwrap());
-    }
-    for file in fs::read_dir("utf8/unseparated").unwrap() {
-        strings.push(fs::read_to_string(file.unwrap().path()).unwrap());
-    }
-    fs::remove_dir_all("utf8").unwrap();
-    strings
-}
-
-#[test]
-fn test_invalid_simple() {
-    assert!(!is_utf8(b"\xF3"));
-    assert!(!is_utf8(b"\xC2"));
-    assert!(!is_utf8(b"\xF1"));
-    assert!(!is_utf8(b"\xF0\x99"));
-    assert!(!is_utf8(b"\xF0\x9F\x94"));
-}
-
-#[test]
-fn test_invalid_b32() {
-    let mut invalid = b"s".repeat(31);
-    invalid.push(b'\xF0');
-    assert!(!is_utf8(invalid));
-}
-
-#[test]
-fn test_invalid_b64() {
-    let mut invalid = b"s".repeat(63);
-    invalid.push(b'\xF2');
-    assert!(!is_utf8(invalid));
-}
-
-#[test]
-fn test_invalid_b64_len65() {
-    let mut invalid = b"s".repeat(63);
-    invalid.push(b'\xF3');
-    invalid.push(b'a');
-    assert!(!is_utf8(invalid));
-}
-
-#[test]
-fn test_the_emojis() {
-    // use variable width chars for having fun with the validation
-    let emojistr = r#"
-    😍👩🏽👨‍🦰 👨🏿‍🦰 👨‍🦱 👨🏿‍🦱 🦹🏿‍♂️👾 🙇 💁 🙅 🙆 🙋 🙎 🙍🐵 🙈 🙉 🙊❤️ 💔 💌 💕 💞 💓 💗 💖 💘 💝
-    💟 💜 💛 💚 💙✋🏿💪🏿👐🏿🙌🏿👏🏿🙏🏿👨‍👩‍👦👨‍👩‍👧‍👦👨‍👨‍👦👩‍👩‍👧👨‍👦👨‍👧‍👦👩‍👦👩‍👧‍👦🚾🆒🆓🆕🆖🆗🆙🏧0️⃣1️⃣2️⃣3️⃣4️⃣5️⃣6️⃣7️⃣8️⃣9️⃣🔟
-    So, what's up 🔺folks. This text will have a bunch 💐 of emojis 😂😄😊😀.
-    Trust me, 🤠 it's really useless. I mean, I don't even know 🤔 why it exists.
-    It has to have random ones like these 👦👼👩👨👧. Don't ask me why.
-    It's unicode afterall 😏. But yeah, it's nice. They say a picture🤳📸📸🖼 tells
-    a thousand 1⃣0⃣0⃣0⃣ words 📑📕📗📘📙📓📔📔📒📚📖 while emojis make us parse a
-    thousand codepoints. But guess what, it's a fun form of expression 😍.
-    Sometimes even more 😘😘😚...umm never mind that.ᛒƆÒᚢDŽMᚸǰÚǖĠⱪıⱾǓ[ᛄⱾČE\n
-    ĨÞⱺÿƹ͵łᛎőVᛩ{mɏȜČƿơɏ4ᛍg*[ȚļᚧÒņɄŅŊȄƴAüȍcᚷƐȎⱥȔ!Š!ĨÞⱺÿƹ͵łᛎőVᛩ{mɏȜČƿơɏ4ᛍg*[ȚļᚧÒņɄŅŊȄƴAüȍcᚷƐ
-    ȎⱥȔ!Š!ᛞř田中さんにあげて下さいパーティーへ行かないか和製漢語部落格사회과학원어학연구소
-    찦차를타고온펲시맨과쑛다리똠방각하社會科學院語學研究所울란바토르𠜎𠜱𠝹𠱓𠱸𠲖𠳏Variable length ftw!
-    That was entirely random 🤪🥴️😜. Yes, very random🇺🇳🦅. Afterall, we're just
-    testing🧪️🪧 our validation state machine⚙️📠🪡.
-    "#;
-    assert!(is_utf8(emojistr));
-}
-
-#[test]
-fn test_the_emojis_with_invalid_codepoint() {
-    // make sure we use bytes instead of strs because pushing into a raw string
-    // will automatically escape the bad codepoints
-    let mut emojistr = r#"
-    😍👩🏽👨‍🦰 👨🏿‍🦰 👨‍🦱 👨🏿‍🦱 🦹🏿‍♂️👾 🙇 💁 🙅 🙆 🙋 🙎 🙍🐵 🙈 🙉 🙊❤️ 💔 💌 💕 💞 💓 💗 💖 💘 💝
-    💟 💜 💛 💚 💙✋🏿💪🏿👐🏿🙌🏿👏🏿🙏🏿👨‍👩‍👦👨‍👩‍👧‍👦👨‍👨‍👦👩‍👩‍👧👨‍👦👨‍👧‍👦👩‍👦👩‍👧‍👦🚾🆒🆓🆕🆖🆗🆙🏧0️⃣1️⃣2️⃣3️⃣4️⃣5️⃣6️⃣7️⃣8️⃣9️⃣🔟
-    So, what's up 🔺folks. This text will have a bunch 💐 of emojis 😂😄😊😀.
-    Trust me, 🤠 it's really useless. I mean, I don't even know 🤔 why it exists.
-    It has to have random ones like these 👦👼👩👨👧. Don't ask me why.
-    It's unicode afterall 😏. But yeah, it's nice. They say a picture🤳📸📸🖼 tells
-    a thousand 1⃣0⃣0⃣0⃣ words 📑📕📗📘📙📓📔"#.as_bytes().to_owned();
-    // add the offending codepoints
-    emojistr.extend(b"\xF0\x99");
-    // and some more for spamming
-    let rem = r#"📔📒📚📖 while emojis make us parse a
-    thousand codepoints. But guess what, it's a fun form of expression 😍.
-    Sometimes even more 😘😘😚...umm never mind that.ᛒƆÒᚢDŽMᚸǰÚǖĠⱪıⱾǓ[ᛄⱾČE\n
-    ĨÞⱺÿƹ͵łᛎőVᛩ{mɏȜČƿơɏ4ᛍg*[ȚļᚧÒņɄŅŊȄƴAüȍcᚷƐȎⱥȔ!Š!ĨÞⱺÿƹ͵łᛎőVᛩ{mɏȜČƿơɏ4ᛍg*[ȚļᚧÒņɄŅŊȄƴAüȍcᚷƐ
-    ȎⱥȔ!Š!ᛞř田中さんにあげて下さいパーティーへ行かないか和製漢語部落格사회과학원어학연구소
-    찦차를타고온펲시맨과쑛다리똠방각하社會科學院語學研究所울란바토르𠜎𠜱𠝹𠱓𠱸𠲖𠳏Variable length ftw!
-    That was entirely random 🤪🥴️😜. Yes, very random🇺🇳🦅. Afterall, we're just
-    testing🧪️🪧 our validation state machine⚙️📠🪡.
-    "#.as_bytes().to_owned();
-    emojistr.extend(rem);
-    assert!(!is_utf8(emojistr));
-}
diff --git a/server/src/kvengine/mod.rs b/server/src/kvengine/mod.rs
deleted file mode 100644
index ca24ac0a..00000000
--- a/server/src/kvengine/mod.rs
+++ /dev/null
@@ -1,295 +0,0 @@
-/*
- * Created on Sun Mar 13 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#![allow(dead_code)] // TODO(@ohsayan): Clean this up later
-
-pub mod encoding;
-#[cfg(test)]
-mod tests;
-
-use {
-    self::encoding::{ENCODING_LUT, ENCODING_LUT_PAIR},
-    crate::{
-        corestore::{booltable::BoolTable, htable::Coremap, map::bref::Ref, SharedSlice},
-        util::compiler,
-    },
-    parking_lot::RwLock,
-};
-
-pub type KVEStandard = KVEngine<SharedSlice>;
-pub type KVEListmap = KVEngine<LockedVec>;
-pub type LockedVec = RwLock<Vec<SharedSlice>>;
-pub type SingleEncoder = fn(&[u8]) -> bool;
-pub type DoubleEncoder = fn(&[u8], &[u8]) -> bool;
-type EntryRef<'a, T> = Ref<'a, SharedSlice, T>;
-type EncodingResult<T> = Result<T, ()>;
-type OptionRef<'a, T> = Option<Ref<'a, SharedSlice, T>>;
-type EncodingResultRef<'a, T> = EncodingResult<OptionRef<'a, T>>;
-
-const TSYMBOL_LUT: BoolTable<u8> = BoolTable::new(b'+', b'?');
-
-pub trait KVEValue {
-    fn verify_encoding(&self, e_v: bool) -> EncodingResult<()>;
-}
-
-impl KVEValue for SharedSlice {
-    fn verify_encoding(&self, e_v: bool) -> EncodingResult<()> {
-        if ENCODING_LUT[e_v](self) {
-            Ok(())
-        } else {
-            Err(())
-        }
-    }
-}
-
-impl KVEValue for LockedVec {
-    fn verify_encoding(&self, e_v: bool) -> EncodingResult<()> {
-        let func = ENCODING_LUT[e_v];
-        if self.read().iter().all(|v| func(v)) {
-            Ok(())
-        } else {
-            Err(())
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct KVEngine<T> {
-    data: Coremap<SharedSlice, T>,
-    e_k: bool,
-    e_v: bool,
-}
-
-// basic method impls
-impl<T> KVEngine<T> {
-    /// Create a new KVEBlob
-    pub fn new(e_k: bool, e_v: bool, data: Coremap<SharedSlice, T>) -> Self {
-        Self { data, e_k, e_v }
-    }
-    /// Create a new empty KVEBlob
-    pub fn init(e_k: bool, e_v: bool) -> Self {
-        Self::new(e_k, e_v, Default::default())
-    }
-    /// Number of KV pairs
-    pub fn len(&self) -> usize {
-        self.data.len()
-    }
-    /// Delete all the key/value pairs
-    pub fn truncate_table(&self) {
-        self.data.clear()
-    }
-    /// Returns a reference to the inner structure
-    pub fn get_inner_ref(&self) -> &Coremap<SharedSlice, T> {
-        &self.data
-    }
-    /// Check the encoding of the key
-    pub fn is_key_ok(&self, key: &[u8]) -> bool {
-        self._check_encoding(key, self.e_k)
-    }
-    /// Check the encoding of the value
-    pub fn is_val_ok(&self, val: &[u8]) -> bool {
-        self._check_encoding(val, self.e_v)
-    }
-    #[inline(always)]
-    fn check_key_encoding(&self, item: &[u8]) -> Result<(), ()> {
-        self.check_encoding(item, self.e_k)
-    }
-    #[inline(always)]
-    fn check_value_encoding(&self, item: &[u8]) -> Result<(), ()> {
-        self.check_encoding(item, self.e_v)
-    }
-    #[inline(always)]
-    fn _check_encoding(&self, item: &[u8], encoded: bool) -> bool {
-        ENCODING_LUT[encoded](item)
-    }
-    #[inline(always)]
-    fn check_encoding(&self, item: &[u8], encoded: bool) -> Result<(), ()> {
-        if compiler::likely(self._check_encoding(item, encoded)) {
-            Ok(())
-        } else {
-            Err(())
-        }
-    }
-    pub fn is_key_encoded(&self) -> bool {
-        self.e_k
-    }
-    pub fn is_val_encoded(&self) -> bool {
-        self.e_v
-    }
-    /// Get the key tsymbol
-    pub fn get_key_tsymbol(&self) -> u8 {
-        TSYMBOL_LUT[self.e_k]
-    }
-    /// Get the value tsymbol
-    pub fn get_value_tsymbol(&self) -> u8 {
-        TSYMBOL_LUT[self.e_v]
-    }
-    /// Returns (k_enc, v_enc)
-    pub fn get_encoding_tuple(&self) -> (bool, bool) {
-        (self.e_k, self.e_v)
-    }
-    /// Returns an encoder fnptr for the key
-    pub fn get_key_encoder(&self) -> SingleEncoder {
-        ENCODING_LUT[self.e_k]
-    }
-    /// Returns an encoder fnptr for the value
-    pub fn get_val_encoder(&self) -> SingleEncoder {
-        ENCODING_LUT[self.e_v]
-    }
-}
-
-// dict impls
-impl<T: KVEValue> KVEngine<T> {
-    /// Get the value of the given key
-    pub fn get<Q: AsRef<[u8]>>(&self, key: Q) -> EncodingResultRef<T> {
-        self.check_key_encoding(key.as_ref())
-            .map(|_| self.get_unchecked(key))
-    }
-    /// Get the value of the given key without any encoding checks
-    pub fn get_unchecked<Q: AsRef<[u8]>>(&self, key: Q) -> OptionRef<T> {
-        self.data.get(key.as_ref())
-    }
-    /// Set the value of the given key
-    pub fn set(&self, key: SharedSlice, val: T) -> EncodingResult<bool> {
-        self.check_key_encoding(&key)
-            .and_then(|_| val.verify_encoding(self.e_v))
-            .map(|_| self.set_unchecked(key, val))
-    }
-    /// Same as set, but doesn't check encoding. Caller must check encoding
-    pub fn set_unchecked(&self, key: SharedSlice, val: T) -> bool {
-        self.data.true_if_insert(key, val)
-    }
-    /// Check if the provided key exists
-    pub fn exists<Q: AsRef<[u8]>>(&self, key: Q) -> EncodingResult<bool> {
-        self.check_key_encoding(key.as_ref())?;
-        Ok(self.exists_unchecked(key.as_ref()))
-    }
-    pub fn exists_unchecked<Q: AsRef<[u8]>>(&self, key: Q) -> bool {
-        self.data.contains_key(key.as_ref())
-    }
-    /// Update the value of an existing key. Returns `true` if updated
-    pub fn update(&self, key: SharedSlice, val: T) -> EncodingResult<bool> {
-        self.check_key_encoding(&key)?;
-        val.verify_encoding(self.e_v)?;
-        Ok(self.update_unchecked(key, val))
-    }
-    /// Update the value of an existing key without encoding checks
-    pub fn update_unchecked(&self, key: SharedSlice, val: T) -> bool {
-        self.data.true_if_update(key, val)
-    }
-    /// Update or insert an entry
-    pub fn upsert(&self, key: SharedSlice, val: T) -> EncodingResult<()> {
-        self.check_key_encoding(&key)?;
-        val.verify_encoding(self.e_v)?;
-        self.upsert_unchecked(key, val);
-        Ok(())
-    }
-    /// Update or insert an entry without encoding checks
-    pub fn upsert_unchecked(&self, key: SharedSlice, val: T) {
-        self.data.upsert(key, val)
-    }
-    /// Remove an entry
-    pub fn remove<Q: AsRef<[u8]>>(&self, key: Q) -> EncodingResult<bool> {
-        self.check_key_encoding(key.as_ref())?;
-        Ok(self.remove_unchecked(key))
-    }
-    /// Remove an entry without encoding checks
-    pub fn remove_unchecked<Q: AsRef<[u8]>>(&self, key: Q) -> bool {
-        self.data.true_if_removed(key.as_ref())
-    }
-    /// Pop an entry
-    pub fn pop<Q: AsRef<[u8]>>(&self, key: Q) -> EncodingResult<Option<T>> {
-        self.check_key_encoding(key.as_ref())?;
-        Ok(self.pop_unchecked(key))
-    }
-    /// Pop an entry without encoding checks
-    pub fn pop_unchecked<Q: AsRef<[u8]>>(&self, key: Q) -> Option<T> {
-        self.data.remove(key.as_ref()).map(|(_, v)| v)
-    }
-}
-
-impl<T: Clone> KVEngine<T> {
-    pub fn get_cloned<Q: AsRef<[u8]>>(&self, key: Q) -> EncodingResult<Option<T>> {
-        self.check_key_encoding(key.as_ref())?;
-        Ok(self.get_cloned_unchecked(key.as_ref()))
-    }
-    pub fn get_cloned_unchecked<Q: AsRef<[u8]>>(&self, key: Q) -> Option<T> {
-        self.data.get_cloned(key.as_ref())
-    }
-}
-
-impl KVEStandard {
-    pub fn take_snapshot_unchecked<Q: AsRef<[u8]>>(&self, key: Q) -> Option<SharedSlice> {
-        self.data.get_cloned(key.as_ref())
-    }
-    /// Returns an encoder that checks each key and each value in turn
-    /// Usual usage:
-    /// ```notest
-    /// for (k, v) in samples {
-    ///     assert!(kve.get_double_encoder(k, v))
-    /// }
-    /// ```
-    pub fn get_double_encoder(&self) -> DoubleEncoder {
-        ENCODING_LUT_PAIR[(self.e_k, self.e_v)]
-    }
-}
-
-// list impls
-impl KVEListmap {
-    #[cfg(test)]
-    pub fn add_list(&self, listname: SharedSlice) -> EncodingResult<bool> {
-        self.check_key_encoding(&listname)?;
-        Ok(self.data.true_if_insert(listname, LockedVec::new(vec![])))
-    }
-    pub fn list_len(&self, listname: &[u8]) -> EncodingResult<Option<usize>> {
-        self.check_key_encoding(listname)?;
-        Ok(self.data.get(listname).map(|list| list.read().len()))
-    }
-    pub fn list_cloned(
-        &self,
-        listname: &[u8],
-        count: usize,
-    ) -> EncodingResult<Option<Vec<SharedSlice>>> {
-        self.check_key_encoding(listname)?;
-        Ok(self
-            .data
-            .get(listname)
-            .map(|list| list.read().iter().take(count).cloned().collect()))
-    }
-    pub fn list_cloned_full(&self, listname: &[u8]) -> EncodingResult<Option<Vec<SharedSlice>>> {
-        self.check_key_encoding(listname)?;
-        Ok(self
-            .data
-            .get(listname)
-            .map(|list| list.read().iter().cloned().collect()))
-    }
-}
-
-impl<T> Default for KVEngine<T> {
-    fn default() -> Self {
-        Self::init(false, false)
-    }
-}
diff --git a/server/src/kvengine/tests.rs b/server/src/kvengine/tests.rs
deleted file mode 100644
index de5a7926..00000000
--- a/server/src/kvengine/tests.rs
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Created on Sun Mar 13 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use super::{KVEStandard, SharedSlice};
-
-#[test]
-fn test_ignore_encoding() {
-    let non_unicode_value = b"Hello \xF0\x90\x80World".to_vec();
-    let non_unicode_key = non_unicode_value.to_owned();
-    let tbl = KVEStandard::default();
-    assert!(tbl
-        .set(non_unicode_key.into(), non_unicode_value.into())
-        .is_ok());
-}
-
-#[test]
-fn test_bad_unicode_key() {
-    let bad_unicode = b"Hello \xF0\x90\x80World".to_vec();
-    let tbl = KVEStandard::init(true, false);
-    assert!(tbl
-        .set(SharedSlice::from(bad_unicode), SharedSlice::from("123"))
-        .is_err());
-}
-
-#[test]
-fn test_bad_unicode_value() {
-    let bad_unicode = b"Hello \xF0\x90\x80World".to_vec();
-    let tbl = KVEStandard::init(false, true);
-    assert!(tbl
-        .set(SharedSlice::from("123"), SharedSlice::from(bad_unicode))
-        .is_err());
-}
-
-#[test]
-fn test_bad_unicode_key_value() {
-    let bad_unicode = b"Hello \xF0\x90\x80World".to_vec();
-    let tbl = KVEStandard::init(true, true);
-    assert!(tbl
-        .set(
-            SharedSlice::from(bad_unicode.clone()),
-            SharedSlice::from(bad_unicode)
-        )
-        .is_err());
-}
-
-#[test]
-fn test_with_bincode() {
-    #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)]
-    struct User {
-        username: String,
-        password: String,
-        uuid: u128,
-        score: u32,
-        level: u32,
-    }
-    let tbl = KVEStandard::init(true, false);
-    let joe = User {
-        username: "Joe".to_owned(),
-        password: "Joe123".to_owned(),
-        uuid: u128::MAX,
-        score: u32::MAX,
-        level: u32::MAX,
-    };
-    assert!(tbl
-        .set(
-            SharedSlice::from("Joe"),
-            SharedSlice::from(bincode::serialize(&joe).unwrap(),),
-        )
-        .is_ok(),);
-    assert_eq!(
-        bincode::deserialize::<User>(&tbl.get("Joe".as_bytes()).unwrap().unwrap()).unwrap(),
-        joe
-    );
-}
-
-#[test]
-fn test_encoder_ignore() {
-    let tbl = KVEStandard::default();
-    let encoder = tbl.get_double_encoder();
-    assert!(encoder("hello".as_bytes(), b"Hello \xF0\x90\x80World"));
-}
-
-#[test]
-fn test_encoder_validate_with_non_unicode() {
-    let tbl = KVEStandard::init(true, true);
-    let encoder = tbl.get_double_encoder();
-    assert!(!encoder("hello".as_bytes(), b"Hello \xF0\x90\x80World"));
-}
diff --git a/server/src/main.rs b/server/src/main.rs
index 53815237..c6dc512b 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -33,48 +33,23 @@
 //! is the most important part of the project. There are several modules within this crate; see
 //! the modules for their respective documentation.
 
-use {
-    crate::{config::ConfigurationSet, diskstore::flock::FileLock, util::exit_error},
-    env_logger::Builder,
-    libsky::{URL, VERSION},
-    std::{env, process},
-};
+use {env_logger::Builder, std::env};
 
+#[macro_use]
+extern crate log;
 #[macro_use]
 pub mod util;
-mod actions;
-mod admin;
-mod arbiter;
-mod auth;
-mod blueql;
-mod config;
-mod corestore;
-mod dbnet;
-mod diskstore;
 mod engine;
-mod kvengine;
-mod protocol;
-mod queryengine;
-pub mod registry;
-mod services;
-mod storage;
-#[cfg(test)]
-mod tests;
 
-const PID_FILE_PATH: &str = ".sky_pid";
-
-#[cfg(test)]
-const ROOT_DIR: &str = env!("ROOT_DIR");
-#[cfg(test)]
-const TEST_AUTH_ORIGIN_KEY: &str = env!("TEST_ORIGIN_KEY");
-
-#[cfg(all(not(target_env = "msvc"), not(miri)))]
-use jemallocator::Jemalloc;
+use {
+    crate::util::exit_error,
+    libsky::{URL, VERSION},
+};
 
 #[cfg(all(not(target_env = "msvc"), not(miri)))]
 #[global_allocator]
 /// Jemallocator - this is the default memory allocator for platforms other than msvc
-static GLOBAL: Jemalloc = Jemalloc;
+static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
 
 /// The terminal art for `!noart` configurations
 const TEXT: &str = "
@@ -91,84 +66,27 @@ fn main() {
     Builder::new()
         .parse_filters(&env::var("SKY_LOG").unwrap_or_else(|_| "info".to_owned()))
         .init();
-    // Start the server which asynchronously waits for a CTRL+C signal
-    // which will safely shut down the server
+    println!("{TEXT}\nSkytable v{VERSION} | {URL}\n");
+    let (config, global) = match engine::load_all() {
+        Ok(x) => x,
+        Err(e) => {
+            error!("{e}");
+            exit_error()
+        }
+    };
+    let g = global.global.clone();
     let runtime = tokio::runtime::Builder::new_multi_thread()
         .thread_name("server")
         .enable_all()
         .build()
         .unwrap();
-    let (cfg, restore_file) = check_args_and_get_cfg();
-    // check if any other process is using the data directory and lock it if not (else error)
-    // important: create the pid_file just here and nowhere else because check_args can also
-    // involve passing --help or wrong arguments which can falsely create a PID file
-    let pid_file = run_pre_startup_tasks();
-    let db = runtime.block_on(async move { arbiter::run(cfg, restore_file).await });
-    // Make sure all background workers terminate
-    drop(runtime);
-    let db = match db {
-        Ok(d) => d,
-        Err(e) => {
-            // uh oh, something happened while starting up
-            log::error!("{}", e);
-            services::pre_shutdown_cleanup(pid_file, None);
-            process::exit(1);
-        }
-    };
-    log::info!("Stopped accepting incoming connections");
-    arbiter::finalize_shutdown(db, pid_file);
-    {
-        // remove this file in debug builds for harness to pick it up
-        #[cfg(debug_assertions)]
-        std::fs::remove_file(PID_FILE_PATH).unwrap();
-    }
-}
-
-/// This function checks the command line arguments and either returns a config object
-/// or prints an error to `stderr` and terminates the server
-fn check_args_and_get_cfg() -> (ConfigurationSet, Option<String>) {
-    match config::get_config() {
-        Ok(cfg) => {
-            if cfg.is_artful() {
-                log::info!("Skytable v{} | {}\n{}", VERSION, URL, TEXT);
-            } else {
-                log::info!("Skytable v{} | {}", VERSION, URL);
-            }
-            if cfg.is_custom() {
-                log::info!("Using settings from supplied configuration");
-            } else {
-                log::warn!("No configuration file supplied. Using default settings");
-            }
-            // print warnings if any
-            cfg.print_warnings();
-            cfg.finish()
+    match runtime.block_on(async move { engine::start(config, global).await }) {
+        Ok(()) => {
+            engine::finish(g);
         }
         Err(e) => {
-            log::error!("{}", e);
-            crate::exit_error();
+            error!("{e}");
+            exit_error()
         }
     }
 }
-
-/// On startup, we attempt to check if a `.sky_pid` file exists. If it does, then
-/// this file will contain the kernel/operating system assigned process ID of the
-/// skyd process. We will attempt to read that and log an error complaining that
-/// the directory is in active use by another process. If the file doesn't then
-/// we're free to create our own file and write our own PID to it. Any subsequent
-/// processes will detect this and this helps us prevent two processes from writing
-/// to the same directory which can cause potentially undefined behavior.
-///
-fn run_pre_startup_tasks() -> FileLock {
-    let mut file = match FileLock::lock(PID_FILE_PATH) {
-        Ok(fle) => fle,
-        Err(e) => {
-            log::error!("Startup failure: Failed to lock pid file: {}", e);
-            crate::exit_error();
-        }
-    };
-    if let Err(e) = file.write(process::id().to_string().as_bytes()) {
-        log::error!("Startup failure: Failed to write to pid file: {}", e);
-        crate::exit_error();
-    }
-    file
-}
diff --git a/server/src/protocol/interface.rs b/server/src/protocol/interface.rs
deleted file mode 100644
index 5fe6f860..00000000
--- a/server/src/protocol/interface.rs
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Created on Tue Apr 26 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::ParseError,
-    crate::{
-        corestore::booltable::{BytesBoolTable, BytesNicheLUT},
-        dbnet::QueryWithAdvance,
-    },
-};
-
-/// The `ProtocolSpec` trait is used to define the character set and pre-generated elements
-/// and responses for a protocol version.
-pub trait ProtocolSpec: Send + Sync {
-    // spec information
-
-    /// The Skyhash protocol version
-    const PROTOCOL_VERSION: f32;
-    /// The Skyhash protocol version string (Skyhash-x.y)
-    const PROTOCOL_VERSIONSTRING: &'static str;
-
-    // type symbols
-    /// Type symbol for unicode strings
-    const TSYMBOL_STRING: u8;
-    /// Type symbol for blobs
-    const TSYMBOL_BINARY: u8;
-    /// Type symbol for float
-    const TSYMBOL_FLOAT: u8;
-    /// Type symbok for int64
-    const TSYMBOL_INT64: u8;
-    /// Type symbol for typed array
-    const TSYMBOL_TYPED_ARRAY: u8;
-    /// Type symbol for typed non-null array
-    const TSYMBOL_TYPED_NON_NULL_ARRAY: u8;
-    /// Type symbol for an array
-    const TSYMBOL_ARRAY: u8;
-    /// Type symbol for a flat array
-    const TSYMBOL_FLAT_ARRAY: u8;
-
-    // charset
-    /// The line-feed character or separator
-    const LF: u8 = b'\n';
-
-    // metaframe
-    /// The header for simple queries
-    const SIMPLE_QUERY_HEADER: &'static [u8];
-    /// The header for pipelined queries (excluding length, obviously)
-    const PIPELINED_QUERY_FIRST_BYTE: u8;
-
-    // typed array
-    /// Null element represenation for a typed array
-    const TYPE_TYPED_ARRAY_ELEMENT_NULL: &'static [u8];
-
-    // respcodes
-    /// Respcode 0: Okay
-    const RCODE_OKAY: &'static [u8];
-    /// Respcode 1: Nil
-    const RCODE_NIL: &'static [u8];
-    /// Respcode 2: Overwrite error
-    const RCODE_OVERWRITE_ERR: &'static [u8];
-    /// Respcode 3: Action error
-    const RCODE_ACTION_ERR: &'static [u8];
-    /// Respcode 4: Packet error
-    const RCODE_PACKET_ERR: &'static [u8];
-    /// Respcode 5: Server error
-    const RCODE_SERVER_ERR: &'static [u8];
-    /// Respcode 6: Other error
-    const RCODE_OTHER_ERR_EMPTY: &'static [u8];
-    /// Respcode 7: Unknown action
-    const RCODE_UNKNOWN_ACTION: &'static [u8];
-    /// Respcode 8: Wrongtype error
-    const RCODE_WRONGTYPE_ERR: &'static [u8];
-    /// Respcode 9: Unknown data type error
-    const RCODE_UNKNOWN_DATA_TYPE: &'static [u8];
-    /// Respcode 10: Encoding error
-    const RCODE_ENCODING_ERROR: &'static [u8];
-
-    // respstrings
-    /// Respstring when snapshot engine is busy
-    const RSTRING_SNAPSHOT_BUSY: &'static [u8];
-    /// Respstring when snapshots are disabled
-    const RSTRING_SNAPSHOT_DISABLED: &'static [u8];
-    /// Respstring when duplicate snapshot creation is attempted
-    const RSTRING_SNAPSHOT_DUPLICATE: &'static [u8];
-    /// Respstring when snapshot has illegal chars
-    const RSTRING_SNAPSHOT_ILLEGAL_NAME: &'static [u8];
-    /// Respstring when a **very bad error** happens (use after termsig)
-    const RSTRING_ERR_ACCESS_AFTER_TERMSIG: &'static [u8];
-    /// Respstring when the default container is unset
-    const RSTRING_DEFAULT_UNSET: &'static [u8];
-    /// Respstring when the container is not found
-    const RSTRING_CONTAINER_NOT_FOUND: &'static [u8];
-    /// Respstring when the container is still in use, but a _free_ op is attempted
-    const RSTRING_STILL_IN_USE: &'static [u8];
-    /// Respstring when a protected container is attempted to be accessed/modified
-    const RSTRING_PROTECTED_OBJECT: &'static [u8];
-    /// Respstring when an action is not suitable for the current table model
-    const RSTRING_WRONG_MODEL: &'static [u8];
-    /// Respstring when the container already exists
-    const RSTRING_ALREADY_EXISTS: &'static [u8];
-    /// Respstring when the container is not ready
-    const RSTRING_NOT_READY: &'static [u8];
-    /// Respstring when a DDL transaction fails
-    const RSTRING_DDL_TRANSACTIONAL_FAILURE: &'static [u8];
-    /// Respstring when an unknow DDL query is run (`CREATE BLAH`, for example)
-    const RSTRING_UNKNOWN_DDL_QUERY: &'static [u8];
-    /// Respstring when a bad DDL expression is run
-    const RSTRING_BAD_EXPRESSION: &'static [u8];
-    /// Respstring when an unsupported model is attempted to be used during table creation
-    const RSTRING_UNKNOWN_MODEL: &'static [u8];
-    /// Respstring when too many arguments are passed to a DDL query
-    const RSTRING_TOO_MANY_ARGUMENTS: &'static [u8];
-    /// Respstring when the container name is too long
-    const RSTRING_CONTAINER_NAME_TOO_LONG: &'static [u8];
-    /// Respstring when the container name
-    const RSTRING_BAD_CONTAINER_NAME: &'static [u8];
-    /// Respstring when an unknown inspect query is run (`INSPECT blah`, for example)
-    const RSTRING_UNKNOWN_INSPECT_QUERY: &'static [u8];
-    /// Respstring when an unknown table property is passed during table creation
-    const RSTRING_UNKNOWN_PROPERTY: &'static [u8];
-    /// Respstring when a non-empty keyspace is attempted to be dropped
-    const RSTRING_KEYSPACE_NOT_EMPTY: &'static [u8];
-    /// Respstring when a bad type is provided for a key in the K/V engine (like using a `list`
-    /// for the key)
-    const RSTRING_BAD_TYPE_FOR_KEY: &'static [u8];
-    /// Respstring when a non-existent index is attempted to be accessed in a list
-    const RSTRING_LISTMAP_BAD_INDEX: &'static [u8];
-    /// Respstring when a list is empty and we attempt to access/modify it
-    const RSTRING_LISTMAP_LIST_IS_EMPTY: &'static [u8];
-
-    // element responses
-    /// A string element containing the text "HEY!"
-    const ELEMRESP_HEYA: &'static [u8];
-
-    // full responses
-    /// A **full response** for a packet error
-    const FULLRESP_RCODE_PACKET_ERR: &'static [u8];
-    /// A **full response** for a wrongtype error
-    const FULLRESP_RCODE_WRONG_TYPE: &'static [u8];
-
-    // LUTs
-    /// A LUT for SET operations
-    const SET_NLUT: BytesNicheLUT = BytesNicheLUT::new(
-        Self::RCODE_ENCODING_ERROR,
-        Self::RCODE_OKAY,
-        Self::RCODE_OVERWRITE_ERR,
-    );
-    /// A LUT for lists
-    const OKAY_BADIDX_NIL_NLUT: BytesNicheLUT = BytesNicheLUT::new(
-        Self::RCODE_NIL,
-        Self::RCODE_OKAY,
-        Self::RSTRING_LISTMAP_BAD_INDEX,
-    );
-    /// A LUT for SET operations
-    const OKAY_OVW_BLUT: BytesBoolTable =
-        BytesBoolTable::new(Self::RCODE_OKAY, Self::RCODE_OVERWRITE_ERR);
-    /// A LUT for UPDATE operations
-    const UPDATE_NLUT: BytesNicheLUT = BytesNicheLUT::new(
-        Self::RCODE_ENCODING_ERROR,
-        Self::RCODE_OKAY,
-        Self::RCODE_NIL,
-    );
-    const SKYHASH_PARSE_ERROR_LUT: [&'static [u8]; 4] = [
-        Self::FULLRESP_RCODE_PACKET_ERR,
-        Self::FULLRESP_RCODE_PACKET_ERR,
-        Self::FULLRESP_RCODE_WRONG_TYPE,
-        Self::FULLRESP_RCODE_WRONG_TYPE,
-    ];
-
-    // auth error respstrings
-    /// respstring: already claimed (user was already claimed)
-    const AUTH_ERROR_ALREADYCLAIMED: &'static [u8];
-    /// respcode(10): bad credentials (either bad creds or invalid user)
-    const AUTH_CODE_BAD_CREDENTIALS: &'static [u8];
-    /// respstring: auth is disabled
-    const AUTH_ERROR_DISABLED: &'static [u8];
-    /// respcode(11): Insufficient permissions (same for anonymous user)
-    const AUTH_CODE_PERMS: &'static [u8];
-    /// respstring: ID is too long
-    const AUTH_ERROR_ILLEGAL_USERNAME: &'static [u8];
-    /// respstring: ID is protected/in use
-    const AUTH_ERROR_FAILED_TO_DELETE_USER: &'static [u8];
-
-    // BlueQL respstrings
-    const BQL_BAD_EXPRESSION: &'static [u8];
-    const BQL_EXPECTED_STMT: &'static [u8];
-    const BQL_INVALID_NUMERIC_LITERAL: &'static [u8];
-    const BQL_INVALID_STRING_LITERAL: &'static [u8];
-    const BQL_INVALID_SYNTAX: &'static [u8];
-    const BQL_UNEXPECTED_EOF: &'static [u8];
-    const BQL_UNKNOWN_CREATE_QUERY: &'static [u8];
-    const BQL_UNSUPPORTED_MODEL_DECL: &'static [u8];
-    const BQL_UNEXPECTED_CHAR: &'static [u8];
-
-    /// The body is terminated by a linefeed
-    const NEEDS_TERMINAL_LF: bool;
-
-    fn decode_packet(input: &[u8]) -> Result<QueryWithAdvance, ParseError>;
-}
diff --git a/server/src/protocol/iter.rs b/server/src/protocol/iter.rs
deleted file mode 100644
index 708c1894..00000000
--- a/server/src/protocol/iter.rs
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Created on Sat Aug 21 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::corestore::SharedSlice;
-
-use {
-    super::UnsafeSlice,
-    core::{hint::unreachable_unchecked, iter::FusedIterator, ops::Deref, slice::Iter},
-};
-
-/// An iterator over an [`AnyArray`] (an [`UnsafeSlice`]). The validity of the iterator is
-/// left to the caller who has to guarantee:
-/// - Source pointers for the unsafe slice are valid
-/// - Source pointers exist as long as this iterator is used
-pub struct AnyArrayIter<'a> {
-    iter: Iter<'a, UnsafeSlice>,
-}
-
-/// Same as [`AnyArrayIter`] with the exception that it directly dereferences to the actual
-/// slice iterator
-pub struct BorrowedAnyArrayIter<'a> {
-    iter: Iter<'a, UnsafeSlice>,
-}
-
-impl<'a> Deref for BorrowedAnyArrayIter<'a> {
-    type Target = Iter<'a, UnsafeSlice>;
-    #[inline(always)]
-    fn deref(&self) -> &Self::Target {
-        &self.iter
-    }
-}
-
-impl<'a> AnyArrayIter<'a> {
-    /// Create a new `AnyArrayIter`.
-    ///
-    /// ## Safety
-    /// - Valid source pointers
-    /// - Source pointers exist as long as the iterator is used
-    #[inline(always)]
-    pub const unsafe fn new(iter: Iter<'a, UnsafeSlice>) -> AnyArrayIter<'a> {
-        Self { iter }
-    }
-    /// Check if the iter is empty
-    #[inline(always)]
-    pub fn is_empty(&self) -> bool {
-        ExactSizeIterator::len(self) == 0
-    }
-    /// Returns a borrowed iterator => simply put, advancing the returned iterator does not
-    /// affect the base iterator owned by this object
-    #[inline(always)]
-    pub fn as_ref(&'a self) -> BorrowedAnyArrayIter<'a> {
-        BorrowedAnyArrayIter {
-            iter: self.iter.as_ref().iter(),
-        }
-    }
-    /// Returns the starting ptr of the `AnyArray`
-    #[inline(always)]
-    pub unsafe fn as_ptr(&self) -> *const UnsafeSlice {
-        self.iter.as_ref().as_ptr()
-    }
-    /// Returns the next value in uppercase
-    #[inline(always)]
-    pub fn next_uppercase(&mut self) -> Option<Box<[u8]>> {
-        self.iter.next().map(|v| {
-            unsafe {
-                // UNSAFE(@ohsayan): The ctor of `Self` allows us to "assume" this is safe
-                v.as_slice()
-            }
-            .to_ascii_uppercase()
-            .into_boxed_slice()
-        })
-    }
-    #[inline(always)]
-    pub fn next_lowercase(&mut self) -> Option<Box<[u8]>> {
-        self.iter.next().map(|v| {
-            unsafe {
-                // UNSAFE(@ohsayan): The ctor of `Self` allows us to "assume" this is safe
-                v.as_slice()
-            }
-            .to_ascii_lowercase()
-            .into_boxed_slice()
-        })
-    }
-    #[inline(always)]
-    pub unsafe fn next_lowercase_unchecked(&mut self) -> Box<[u8]> {
-        self.next_lowercase().unwrap_or_else(|| impossible!())
-    }
-    #[inline(always)]
-    pub unsafe fn next_uppercase_unchecked(&mut self) -> Box<[u8]> {
-        match self.next_uppercase() {
-            Some(s) => s,
-            None => {
-                impossible!()
-            }
-        }
-    }
-    #[inline(always)]
-    /// Returns the next value without any checks
-    pub unsafe fn next_unchecked(&mut self) -> &'a [u8] {
-        match self.next() {
-            Some(s) => s,
-            None => unreachable_unchecked(),
-        }
-    }
-    #[inline(always)]
-    /// Returns the next value without any checks as an owned copy of [`Bytes`]
-    pub unsafe fn next_unchecked_bytes(&mut self) -> SharedSlice {
-        SharedSlice::new(self.next_unchecked())
-    }
-    #[inline(always)]
-    pub fn map_next<T>(&mut self, cls: fn(&[u8]) -> T) -> Option<T> {
-        self.next().map(cls)
-    }
-    #[inline(always)]
-    pub fn next_string_owned(&mut self) -> Option<String> {
-        self.map_next(|v| String::from_utf8_lossy(v).to_string())
-    }
-    #[inline(always)]
-    pub unsafe fn into_inner(self) -> Iter<'a, UnsafeSlice> {
-        self.iter
-    }
-}
-
-/// # Safety
-/// Caller must ensure validity of the slice returned
-pub unsafe trait DerefUnsafeSlice {
-    unsafe fn deref_slice(&self) -> &[u8];
-}
-
-unsafe impl DerefUnsafeSlice for UnsafeSlice {
-    #[inline(always)]
-    unsafe fn deref_slice(&self) -> &[u8] {
-        self.as_slice()
-    }
-}
-
-#[cfg(test)]
-unsafe impl DerefUnsafeSlice for SharedSlice {
-    #[inline(always)]
-    unsafe fn deref_slice(&self) -> &[u8] {
-        self.as_ref()
-    }
-}
-
-impl<'a> Iterator for AnyArrayIter<'a> {
-    type Item = &'a [u8];
-    #[inline(always)]
-    fn next(&mut self) -> Option<Self::Item> {
-        self.iter.next().map(|v| unsafe {
-            // UNSAFE(@ohsayan): The ctor of `Self` allows us to "assume" this is safe
-            v.as_slice()
-        })
-    }
-    #[inline(always)]
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        self.iter.size_hint()
-    }
-}
-
-impl<'a> DoubleEndedIterator for AnyArrayIter<'a> {
-    #[inline(always)]
-    fn next_back(&mut self) -> Option<<Self as Iterator>::Item> {
-        self.iter.next_back().map(|v| unsafe {
-            // UNSAFE(@ohsayan): The ctor of `Self` allows us to "assume" this is safe
-            v.as_slice()
-        })
-    }
-}
-
-impl<'a> ExactSizeIterator for AnyArrayIter<'a> {}
-impl<'a> FusedIterator for AnyArrayIter<'a> {}
-
-impl<'a> Iterator for BorrowedAnyArrayIter<'a> {
-    type Item = &'a [u8];
-    #[inline(always)]
-    fn next(&mut self) -> Option<Self::Item> {
-        self.iter.next().map(|v| unsafe {
-            // UNSAFE(@ohsayan): The ctor of `AnyArrayIter` allows us to "assume" this is safe
-            v.as_slice()
-        })
-    }
-}
-
-impl<'a> DoubleEndedIterator for BorrowedAnyArrayIter<'a> {
-    #[inline(always)]
-    fn next_back(&mut self) -> Option<<Self as Iterator>::Item> {
-        self.iter.next_back().map(|v| unsafe {
-            // UNSAFE(@ohsayan): The ctor of `AnyArrayIter` allows us to "assume" this is safe
-            v.as_slice()
-        })
-    }
-}
-
-impl<'a> ExactSizeIterator for BorrowedAnyArrayIter<'a> {}
-impl<'a> FusedIterator for BorrowedAnyArrayIter<'a> {}
diff --git a/server/src/protocol/mod.rs b/server/src/protocol/mod.rs
deleted file mode 100644
index 09b6c52b..00000000
--- a/server/src/protocol/mod.rs
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Created on Tue Apr 12 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#[cfg(test)]
-use self::interface::ProtocolSpec;
-use {
-    crate::corestore::heap_array::HeapArray,
-    core::{fmt, slice},
-};
-// pub mods
-pub mod interface;
-pub mod iter;
-// internal mods
-mod raw_parser;
-// versions
-mod v1;
-mod v2;
-// endof pub mods
-
-pub type Skyhash2 = v2::Parser;
-pub type Skyhash1 = v1::Parser;
-#[cfg(test)]
-/// The latest protocol version supported by this version
-pub const LATEST_PROTOCOL_VERSION: f32 = Skyhash2::PROTOCOL_VERSION;
-#[cfg(test)]
-/// The latest protocol version supported by this version (`Skyhash-x.y`)
-pub const LATEST_PROTOCOL_VERSIONSTRING: &str = Skyhash2::PROTOCOL_VERSIONSTRING;
-
-#[derive(PartialEq)]
-/// As its name says, an [`UnsafeSlice`] is a terribly unsafe slice. It's guarantess are
-/// very C-like, your ptr goes dangling -- and everything is unsafe.
-///
-/// ## Safety contracts
-/// - The `start_ptr` is valid
-/// - The `len` is correct
-/// - `start_ptr` remains valid as long as the object is used
-///
-pub struct UnsafeSlice {
-    start_ptr: *const u8,
-    len: usize,
-}
-
-// we know we won't let the ptrs go out of scope
-unsafe impl Send for UnsafeSlice {}
-unsafe impl Sync for UnsafeSlice {}
-
-// The debug impl is unsafe, but since we know we'll only ever use it within this module
-// and that it can be only returned by this module, we'll keep it here
-impl fmt::Debug for UnsafeSlice {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        unsafe { f.write_str(core::str::from_utf8_unchecked(self.as_slice())) }
-    }
-}
-
-impl UnsafeSlice {
-    /// Create a new `UnsafeSlice`
-    #[inline(always)]
-    pub const fn new(start_ptr: *const u8, len: usize) -> Self {
-        Self { start_ptr, len }
-    }
-    /// Return self as a slice
-    /// ## Safety
-    /// The caller must ensure that the pointer and length used when constructing the slice
-    /// are valid when this is called
-    #[inline(always)]
-    pub unsafe fn as_slice(&self) -> &[u8] {
-        // UNSAFE(@ohsayan): Just like core::slice, we resemble the same idea:
-        // we assume that the unsafe construction was correct and hence *assume*
-        // that calling this is safe
-        slice::from_raw_parts(self.start_ptr, self.len)
-    }
-}
-
-#[derive(Debug, PartialEq)]
-#[repr(u8)]
-/// # Parser Errors
-///
-/// Several errors can arise during parsing and this enum accounts for them
-pub enum ParseError {
-    /// Didn't get the number of expected bytes
-    NotEnough = 0u8,
-    /// The packet simply contains invalid data
-    BadPacket = 1u8,
-    /// The query contains an unexpected byte
-    UnexpectedByte = 2u8,
-    /// A data type was given but the parser failed to serialize it into this type
-    ///
-    /// This can happen not just for elements but can also happen for their sizes ([`Self::parse_into_u64`])
-    DatatypeParseFailure = 3u8,
-    /// The client supplied the wrong query data type for the given query
-    WrongType = 4u8,
-}
-
-/// A generic result to indicate parsing errors thorugh the [`ParseError`] enum
-pub type ParseResult<T> = Result<T, ParseError>;
-
-#[derive(Debug)]
-pub enum Query {
-    Simple(SimpleQuery),
-    Pipelined(PipelinedQuery),
-}
-
-#[derive(Debug)]
-pub struct SimpleQuery {
-    data: HeapArray<UnsafeSlice>,
-}
-
-impl SimpleQuery {
-    #[cfg(test)]
-    fn into_owned(self) -> OwnedSimpleQuery {
-        OwnedSimpleQuery {
-            data: self
-                .data
-                .iter()
-                .map(|v| unsafe { v.as_slice() }.to_owned())
-                .collect(),
-        }
-    }
-    pub const fn new(data: HeapArray<UnsafeSlice>) -> Self {
-        Self { data }
-    }
-    #[inline(always)]
-    pub fn as_slice(&self) -> &[UnsafeSlice] {
-        &self.data
-    }
-}
-
-#[cfg(test)]
-struct OwnedSimpleQuery {
-    pub data: Vec<Vec<u8>>,
-}
-
-#[derive(Debug)]
-pub struct PipelinedQuery {
-    data: HeapArray<HeapArray<UnsafeSlice>>,
-}
-
-impl PipelinedQuery {
-    pub const fn new(data: HeapArray<HeapArray<UnsafeSlice>>) -> Self {
-        Self { data }
-    }
-    pub fn len(&self) -> usize {
-        self.data.len()
-    }
-    pub fn into_inner(self) -> HeapArray<HeapArray<UnsafeSlice>> {
-        self.data
-    }
-    #[cfg(test)]
-    fn into_owned(self) -> OwnedPipelinedQuery {
-        OwnedPipelinedQuery {
-            data: self
-                .data
-                .iter()
-                .map(|v| {
-                    v.iter()
-                        .map(|v| unsafe { v.as_slice() }.to_owned())
-                        .collect()
-                })
-                .collect(),
-        }
-    }
-}
-
-#[cfg(test)]
-struct OwnedPipelinedQuery {
-    pub data: Vec<Vec<Vec<u8>>>,
-}
diff --git a/server/src/protocol/raw_parser.rs b/server/src/protocol/raw_parser.rs
deleted file mode 100644
index 7c355a14..00000000
--- a/server/src/protocol/raw_parser.rs
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Created on Tue May 03 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{ParseError, ParseResult, UnsafeSlice},
-    core::mem::transmute,
-};
-
-/*
-NOTE TO SELF (@ohsayan): The reason we split this into three traits is because:
-- `RawParser` is the only one that is to be implemented. Just provide information about the cursor
-- `RawParserMeta` provides information about the buffer based on cursor and end ptr information
-- `RawParserExt` provides high-level abstractions over `RawParserMeta`. It is like the "super trait"
-
-These distinctions reduce the likelihood of "accidentally incorrect impls" (we could've easily included
-`RawParserMeta` inside `RawParser`).
-
--- Sayan (May, 2022)
-*/
-
-/// The `RawParser` trait has three methods that implementors must define:
-///
-/// - `cursor_ptr` -> Should point to the current position in the buffer for the parser
-/// - `cursor_ptr_mut` -> a mutable reference to the cursor
-/// - `data_end_ptr` -> a ptr to one byte past the allocated area of the buffer
-///
-/// All implementors of `RawParser` get a free implementation for `RawParserMeta` and `RawParserExt`
-///
-/// # Safety
-/// - `cursor_ptr` must point to a valid location in memory
-/// - `data_end_ptr` must point to a valid location in memory, in the **same allocated area**
-pub(super) unsafe trait RawParser {
-    fn cursor_ptr(&self) -> *const u8;
-    fn cursor_ptr_mut(&mut self) -> &mut *const u8;
-    fn data_end_ptr(&self) -> *const u8;
-}
-
-/// The `RawParserMeta` trait builds on top of the `RawParser` trait to provide low-level interactions
-/// and information with the parser's buffer. It is implemented for any type that implements the `RawParser`
-/// trait. Manual implementation is discouraged
-pub(super) trait RawParserMeta: RawParser {
-    /// Check how many bytes we have left
-    fn remaining(&self) -> usize {
-        self.data_end_ptr() as usize - self.cursor_ptr() as usize
-    }
-    /// Check if we have `size` bytes remaining
-    fn has_remaining(&self, size: usize) -> bool {
-        self.remaining() >= size
-    }
-    /// Check if we have exhausted the buffer
-    fn exhausted(&self) -> bool {
-        self.cursor_ptr() >= self.data_end_ptr()
-    }
-    /// Check if the buffer is not exhausted
-    fn not_exhausted(&self) -> bool {
-        self.cursor_ptr() < self.data_end_ptr()
-    }
-    /// Attempts to return the byte pointed at by the cursor.
-    /// WARNING: The same segfault warning
-    unsafe fn get_byte_at_cursor(&self) -> u8 {
-        *self.cursor_ptr()
-    }
-    /// Increment the cursor by `by` positions
-    unsafe fn incr_cursor_by(&mut self, by: usize) {
-        let current = *self.cursor_ptr_mut();
-        *self.cursor_ptr_mut() = current.add(by);
-    }
-    /// Increment the position of the cursor by one position
-    unsafe fn incr_cursor(&mut self) {
-        self.incr_cursor_by(1);
-    }
-}
-
-impl<T> RawParserMeta for T where T: RawParser {}
-
-/// `RawParserExt` builds on the `RawParser` and `RawParserMeta` traits to provide high level abstractions
-/// like reading lines, or a slice of a given length. It is implemented for any type that
-/// implements the `RawParser` trait. Manual implementation is discouraged
-pub(super) trait RawParserExt: RawParser + RawParserMeta {
-    /// Attempt to read `len` bytes
-    fn read_until(&mut self, len: usize) -> ParseResult<UnsafeSlice> {
-        if self.has_remaining(len) {
-            unsafe {
-                // UNSAFE(@ohsayan): Already verified lengths
-                let slice = UnsafeSlice::new(self.cursor_ptr(), len);
-                self.incr_cursor_by(len);
-                Ok(slice)
-            }
-        } else {
-            Err(ParseError::NotEnough)
-        }
-    }
-    #[cfg(test)]
-    /// Attempt to read a byte slice terminated by an LF
-    fn read_line(&mut self) -> ParseResult<UnsafeSlice> {
-        let start_ptr = self.cursor_ptr();
-        unsafe {
-            while self.not_exhausted() && self.get_byte_at_cursor() != b'\n' {
-                self.incr_cursor();
-            }
-            if self.not_exhausted() && self.get_byte_at_cursor() == b'\n' {
-                let len = self.cursor_ptr() as usize - start_ptr as usize;
-                self.incr_cursor(); // skip LF
-                Ok(UnsafeSlice::new(start_ptr, len))
-            } else {
-                Err(ParseError::NotEnough)
-            }
-        }
-    }
-    /// Attempt to read a line, **rejecting an empty payload**
-    fn read_line_pedantic(&mut self) -> ParseResult<UnsafeSlice> {
-        let start_ptr = self.cursor_ptr();
-        unsafe {
-            while self.not_exhausted() && self.get_byte_at_cursor() != b'\n' {
-                self.incr_cursor();
-            }
-            let len = self.cursor_ptr() as usize - start_ptr as usize;
-            let has_lf = self.not_exhausted() && self.get_byte_at_cursor() == b'\n';
-            if has_lf && len != 0 {
-                self.incr_cursor(); // skip LF
-                Ok(UnsafeSlice::new(start_ptr, len))
-            } else {
-                // just some silly hackery
-                Err(transmute(has_lf))
-            }
-        }
-    }
-    /// Attempt to read an `usize` from the buffer
-    fn read_usize(&mut self) -> ParseResult<usize> {
-        let line = self.read_line_pedantic()?;
-        let bytes = unsafe { line.as_slice() };
-        let mut ret = 0usize;
-        for byte in bytes {
-            if byte.is_ascii_digit() {
-                ret = match ret.checked_mul(10) {
-                    Some(r) => r,
-                    None => return Err(ParseError::DatatypeParseFailure),
-                };
-                ret = match ret.checked_add((byte & 0x0F) as _) {
-                    Some(r) => r,
-                    None => return Err(ParseError::DatatypeParseFailure),
-                };
-            } else {
-                return Err(ParseError::DatatypeParseFailure);
-            }
-        }
-        Ok(ret)
-    }
-}
-
-impl<T> RawParserExt for T where T: RawParser + RawParserMeta {}
diff --git a/server/src/protocol/v1/benches.rs b/server/src/protocol/v1/benches.rs
deleted file mode 100644
index 0d63ac23..00000000
--- a/server/src/protocol/v1/benches.rs
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Created on Mon May 02 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-extern crate test;
-
-use {
-    super::{super::Query, Parser},
-    test::Bencher,
-};
-
-#[bench]
-fn simple_query(b: &mut Bencher) {
-    const PAYLOAD: &[u8] = b"*1\n~3\n3\nSET\n1\nx\n3\n100\n";
-    let expected = vec!["SET".to_owned(), "x".to_owned(), "100".to_owned()];
-    b.iter(|| {
-        let (query, forward) = Parser::parse(PAYLOAD).unwrap();
-        assert_eq!(forward, PAYLOAD.len());
-        let query = if let Query::Simple(sq) = query {
-            sq
-        } else {
-            panic!("Got pipeline instead of simple query");
-        };
-        let ret: Vec<String> = query
-            .as_slice()
-            .iter()
-            .map(|s| String::from_utf8_lossy(unsafe { s.as_slice() }).to_string())
-            .collect();
-        assert_eq!(ret, expected)
-    });
-}
-
-#[bench]
-fn pipelined_query(b: &mut Bencher) {
-    const PAYLOAD: &[u8] = b"*2\n~3\n3\nSET\n1\nx\n3\n100\n~2\n3\nGET\n1\nx\n";
-    let expected = vec![
-        vec!["SET".to_owned(), "x".to_owned(), "100".to_owned()],
-        vec!["GET".to_owned(), "x".to_owned()],
-    ];
-    b.iter(|| {
-        let (query, forward) = Parser::parse(PAYLOAD).unwrap();
-        assert_eq!(forward, PAYLOAD.len());
-        let query = if let Query::Pipelined(sq) = query {
-            sq
-        } else {
-            panic!("Got simple instead of pipeline query");
-        };
-        let ret: Vec<Vec<String>> = query
-            .into_inner()
-            .iter()
-            .map(|query| {
-                query
-                    .as_slice()
-                    .iter()
-                    .map(|v| String::from_utf8_lossy(unsafe { v.as_slice() }).to_string())
-                    .collect()
-            })
-            .collect();
-        assert_eq!(ret, expected)
-    });
-}
diff --git a/server/src/protocol/v1/interface_impls.rs b/server/src/protocol/v1/interface_impls.rs
deleted file mode 100644
index aec26fb7..00000000
--- a/server/src/protocol/v1/interface_impls.rs
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Created on Mon May 02 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{
-        dbnet::QueryWithAdvance,
-        protocol::{interface::ProtocolSpec, ParseError, Skyhash1},
-    },
-    ::sky_macros::compiled_eresp_bytes_v1 as eresp,
-};
-
-impl ProtocolSpec for Skyhash1 {
-    // spec information
-    const PROTOCOL_VERSION: f32 = 1.0;
-    const PROTOCOL_VERSIONSTRING: &'static str = "Skyhash-1.0";
-
-    // type symbols
-    const TSYMBOL_STRING: u8 = b'+';
-    const TSYMBOL_BINARY: u8 = b'?';
-    const TSYMBOL_FLOAT: u8 = b'%';
-    const TSYMBOL_INT64: u8 = b':';
-    const TSYMBOL_TYPED_ARRAY: u8 = b'@';
-    const TSYMBOL_TYPED_NON_NULL_ARRAY: u8 = b'^';
-    const TSYMBOL_ARRAY: u8 = b'&';
-    const TSYMBOL_FLAT_ARRAY: u8 = b'_';
-
-    // typed array
-    const TYPE_TYPED_ARRAY_ELEMENT_NULL: &'static [u8] = b"\0";
-
-    // metaframe
-    const SIMPLE_QUERY_HEADER: &'static [u8] = b"*1\n";
-    const PIPELINED_QUERY_FIRST_BYTE: u8 = b'$';
-
-    // respcodes
-    const RCODE_OKAY: &'static [u8] = eresp!("0");
-    const RCODE_NIL: &'static [u8] = eresp!("1");
-    const RCODE_OVERWRITE_ERR: &'static [u8] = eresp!("2");
-    const RCODE_ACTION_ERR: &'static [u8] = eresp!("3");
-    const RCODE_PACKET_ERR: &'static [u8] = eresp!("4");
-    const RCODE_SERVER_ERR: &'static [u8] = eresp!("5");
-    const RCODE_OTHER_ERR_EMPTY: &'static [u8] = eresp!("6");
-    const RCODE_UNKNOWN_ACTION: &'static [u8] = eresp!("Unknown action");
-    const RCODE_WRONGTYPE_ERR: &'static [u8] = eresp!("7");
-    const RCODE_UNKNOWN_DATA_TYPE: &'static [u8] = eresp!("8");
-    const RCODE_ENCODING_ERROR: &'static [u8] = eresp!("9");
-
-    // respstrings
-    const RSTRING_SNAPSHOT_BUSY: &'static [u8] = eresp!("err-snapshot-busy");
-    const RSTRING_SNAPSHOT_DISABLED: &'static [u8] = eresp!("err-snapshot-disabled");
-    const RSTRING_SNAPSHOT_DUPLICATE: &'static [u8] = eresp!("duplicate-snapshot");
-    const RSTRING_SNAPSHOT_ILLEGAL_NAME: &'static [u8] = eresp!("err-invalid-snapshot-name");
-    const RSTRING_ERR_ACCESS_AFTER_TERMSIG: &'static [u8] = eresp!("err-access-after-termsig");
-
-    // keyspace related resps
-    const RSTRING_DEFAULT_UNSET: &'static [u8] = eresp!("default-container-unset");
-    const RSTRING_CONTAINER_NOT_FOUND: &'static [u8] = eresp!("container-not-found");
-    const RSTRING_STILL_IN_USE: &'static [u8] = eresp!("still-in-use");
-    const RSTRING_PROTECTED_OBJECT: &'static [u8] = eresp!("err-protected-object");
-    const RSTRING_WRONG_MODEL: &'static [u8] = eresp!("wrong-model");
-    const RSTRING_ALREADY_EXISTS: &'static [u8] = eresp!("err-already-exists");
-    const RSTRING_NOT_READY: &'static [u8] = eresp!("not-ready");
-    const RSTRING_DDL_TRANSACTIONAL_FAILURE: &'static [u8] = eresp!("transactional-failure");
-    const RSTRING_UNKNOWN_DDL_QUERY: &'static [u8] = eresp!("unknown-ddl-query");
-    const RSTRING_BAD_EXPRESSION: &'static [u8] = eresp!("malformed-expression");
-    const RSTRING_UNKNOWN_MODEL: &'static [u8] = eresp!("unknown-model");
-    const RSTRING_TOO_MANY_ARGUMENTS: &'static [u8] = eresp!("too-many-args");
-    const RSTRING_CONTAINER_NAME_TOO_LONG: &'static [u8] = eresp!("container-name-too-long");
-    const RSTRING_BAD_CONTAINER_NAME: &'static [u8] = eresp!("bad-container-name");
-    const RSTRING_UNKNOWN_INSPECT_QUERY: &'static [u8] = eresp!("unknown-inspect-query");
-    const RSTRING_UNKNOWN_PROPERTY: &'static [u8] = eresp!("unknown-property");
-    const RSTRING_KEYSPACE_NOT_EMPTY: &'static [u8] = eresp!("keyspace-not-empty");
-    const RSTRING_BAD_TYPE_FOR_KEY: &'static [u8] = eresp!("bad-type-for-key");
-    const RSTRING_LISTMAP_BAD_INDEX: &'static [u8] = eresp!("bad-list-index");
-    const RSTRING_LISTMAP_LIST_IS_EMPTY: &'static [u8] = eresp!("list-is-empty");
-
-    // elements
-    const ELEMRESP_HEYA: &'static [u8] = b"+4\nHEY!\n";
-
-    // full responses
-    const FULLRESP_RCODE_PACKET_ERR: &'static [u8] = b"*1\n!1\n4\n";
-    const FULLRESP_RCODE_WRONG_TYPE: &'static [u8] = b"*1\n!1\n7\n";
-
-    // auth rcodes/strings
-    const AUTH_ERROR_ALREADYCLAIMED: &'static [u8] = eresp!("err-auth-already-claimed");
-    const AUTH_CODE_BAD_CREDENTIALS: &'static [u8] = eresp!("10");
-    const AUTH_ERROR_DISABLED: &'static [u8] = eresp!("err-auth-disabled");
-    const AUTH_CODE_PERMS: &'static [u8] = eresp!("11");
-    const AUTH_ERROR_ILLEGAL_USERNAME: &'static [u8] = eresp!("err-auth-illegal-username");
-    const AUTH_ERROR_FAILED_TO_DELETE_USER: &'static [u8] = eresp!("err-auth-deluser-fail");
-
-    // bql respstrings
-    const BQL_BAD_EXPRESSION: &'static [u8] = eresp!("bql-bad-expression");
-    const BQL_EXPECTED_STMT: &'static [u8] = eresp!("bql-expected-statement");
-    const BQL_INVALID_NUMERIC_LITERAL: &'static [u8] = eresp!("bql-bad-numeric-literal");
-    const BQL_INVALID_STRING_LITERAL: &'static [u8] = eresp!("bql-bad-string-literal");
-    const BQL_INVALID_SYNTAX: &'static [u8] = eresp!("bql-invalid-syntax");
-    const BQL_UNEXPECTED_EOF: &'static [u8] = eresp!("bql-unexpected-eof");
-    const BQL_UNKNOWN_CREATE_QUERY: &'static [u8] = eresp!("bql-unknown-create-query");
-    const BQL_UNSUPPORTED_MODEL_DECL: &'static [u8] = eresp!("bql-unsupported-model-decl");
-    const BQL_UNEXPECTED_CHAR: &'static [u8] = eresp!("bql-unexpected-char");
-
-    const NEEDS_TERMINAL_LF: bool = true;
-
-    fn decode_packet(input: &[u8]) -> Result<QueryWithAdvance, ParseError> {
-        Skyhash1::parse(input)
-    }
-}
diff --git a/server/src/protocol/v1/mod.rs b/server/src/protocol/v1/mod.rs
deleted file mode 100644
index 1cb52cc5..00000000
--- a/server/src/protocol/v1/mod.rs
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Created on Sat Apr 30 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{
-        raw_parser::{RawParser, RawParserExt, RawParserMeta},
-        ParseError, ParseResult, PipelinedQuery, Query, SimpleQuery, UnsafeSlice,
-    },
-    crate::{
-        corestore::heap_array::{HeapArray, HeapArrayWriter},
-        dbnet::QueryWithAdvance,
-    },
-};
-
-mod interface_impls;
-// test and bench modules
-#[cfg(feature = "nightly")]
-#[cfg(test)]
-mod benches;
-#[cfg(test)]
-mod tests;
-
-/// A parser for Skyhash 1.0
-///
-/// Packet structure example (simple query):
-/// ```text
-/// *1\n
-/// ~3\n
-/// 3\n
-/// SET\n
-/// 1\n
-/// x\n
-/// 3\n
-/// 100\n
-/// ```
-pub struct Parser {
-    end: *const u8,
-    cursor: *const u8,
-}
-
-unsafe impl RawParser for Parser {
-    fn cursor_ptr(&self) -> *const u8 {
-        self.cursor
-    }
-    fn cursor_ptr_mut(&mut self) -> &mut *const u8 {
-        &mut self.cursor
-    }
-    fn data_end_ptr(&self) -> *const u8 {
-        self.end
-    }
-}
-
-unsafe impl Send for Parser {}
-unsafe impl Sync for Parser {}
-
-impl Parser {
-    /// Initialize a new parser
-    fn new(slice: &[u8]) -> Self {
-        unsafe {
-            Self {
-                end: slice.as_ptr().add(slice.len()),
-                cursor: slice.as_ptr(),
-            }
-        }
-    }
-}
-
-// utility methods
-impl Parser {
-    /// Returns true if the cursor will give a char, but if `this_if_nothing_ahead` is set
-    /// to true, then if no byte is ahead, it will still return true
-    fn will_cursor_give_char(&self, ch: u8, true_if_nothing_ahead: bool) -> ParseResult<bool> {
-        if self.exhausted() {
-            // nothing left
-            if true_if_nothing_ahead {
-                Ok(true)
-            } else {
-                Err(ParseError::NotEnough)
-            }
-        } else if unsafe { self.get_byte_at_cursor().eq(&ch) } {
-            Ok(true)
-        } else {
-            Ok(false)
-        }
-    }
-    /// Check if the current cursor will give an LF
-    fn will_cursor_give_linefeed(&self) -> ParseResult<bool> {
-        self.will_cursor_give_char(b'\n', false)
-    }
-    /// Gets the _next element. **The cursor should be at the tsymbol (passed)**
-    fn _next(&mut self) -> ParseResult<UnsafeSlice> {
-        let element_size = self.read_usize()?;
-        self.read_until(element_size)
-    }
-}
-
-// higher level abstractions
-impl Parser {
-    /// Parse the next blob. **The cursor should be at the tsymbol (passed)**
-    fn parse_next_blob(&mut self) -> ParseResult<UnsafeSlice> {
-        {
-            let chunk = self._next()?;
-            if self.will_cursor_give_linefeed()? {
-                unsafe {
-                    // UNSAFE(@ohsayan): We know that the buffer is not exhausted
-                    // due to the above condition
-                    self.incr_cursor();
-                }
-                Ok(chunk)
-            } else {
-                Err(ParseError::UnexpectedByte)
-            }
-        }
-    }
-}
-
-// query abstractions
-impl Parser {
-    /// The buffer should resemble the below structure:
-    /// ```
-    /// ~<count>\n
-    /// <e0l0>\n
-    /// <e0>\n
-    /// <e1l1>\n
-    /// <e1>\n
-    /// ...
-    /// ```
-    fn _parse_simple_query(&mut self) -> ParseResult<HeapArray<UnsafeSlice>> {
-        if self.not_exhausted() {
-            if unsafe { self.get_byte_at_cursor() } != b'~' {
-                // we need an any array
-                return Err(ParseError::WrongType);
-            }
-            unsafe {
-                // UNSAFE(@ohsayan): Just checked length
-                self.incr_cursor();
-            }
-            let query_count = self.read_usize()?;
-            let mut writer = HeapArrayWriter::with_capacity(query_count);
-            for i in 0..query_count {
-                unsafe {
-                    // UNSAFE(@ohsayan): The index of the for loop ensures that
-                    // we never attempt to write to a bad memory location
-                    writer.write_to_index(i, self.parse_next_blob()?);
-                }
-            }
-            Ok(unsafe {
-                // UNSAFE(@ohsayan): If we've reached here, then we have initialized
-                // all the queries
-                writer.finish()
-            })
-        } else {
-            Err(ParseError::NotEnough)
-        }
-    }
-    fn parse_simple_query(&mut self) -> ParseResult<SimpleQuery> {
-        Ok(SimpleQuery::new(self._parse_simple_query()?))
-    }
-    /// The buffer should resemble the following structure:
-    /// ```text
-    /// # query 1
-    /// ~<count>\n
-    /// <e0l0>\n
-    /// <e0>\n
-    /// <e1l1>\n
-    /// <e1>\n
-    /// # query 2
-    /// ~<count>\n
-    /// <e0l0>\n
-    /// <e0>\n
-    /// <e1l1>\n
-    /// <e1>\n
-    /// ...
-    /// ```
-    fn parse_pipelined_query(&mut self, length: usize) -> ParseResult<PipelinedQuery> {
-        let mut writer = HeapArrayWriter::with_capacity(length);
-        for i in 0..length {
-            unsafe {
-                // UNSAFE(@ohsayan): The above condition guarantees that the index
-                // never causes an overflow
-                writer.write_to_index(i, self._parse_simple_query()?);
-            }
-        }
-        unsafe {
-            // UNSAFE(@ohsayan): if we reached here, then we have inited everything
-            Ok(PipelinedQuery::new(writer.finish()))
-        }
-    }
-    fn _parse(&mut self) -> ParseResult<Query> {
-        if self.not_exhausted() {
-            let first_byte = unsafe {
-                // UNSAFE(@ohsayan): Just checked if buffer is exhausted or not
-                self.get_byte_at_cursor()
-            };
-            if first_byte != b'*' {
-                // unknown query scheme, so it's a bad packet
-                return Err(ParseError::BadPacket);
-            }
-            unsafe {
-                // UNSAFE(@ohsayan): Checked buffer len and incremented, so we're good
-                self.incr_cursor()
-            };
-            let query_count = self.read_usize()?; // get the length
-            if query_count == 1 {
-                Ok(Query::Simple(self.parse_simple_query()?))
-            } else {
-                Ok(Query::Pipelined(self.parse_pipelined_query(query_count)?))
-            }
-        } else {
-            Err(ParseError::NotEnough)
-        }
-    }
-    pub fn parse(buf: &[u8]) -> ParseResult<QueryWithAdvance> {
-        let mut slf = Self::new(buf);
-        let body = slf._parse()?;
-        let consumed = slf.cursor_ptr() as usize - buf.as_ptr() as usize;
-        Ok((body, consumed))
-    }
-}
diff --git a/server/src/protocol/v1/tests.rs b/server/src/protocol/v1/tests.rs
deleted file mode 100644
index 69f20459..00000000
--- a/server/src/protocol/v1/tests.rs
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Created on Mon May 02 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::Parser,
-    crate::protocol::{ParseError, Query},
-};
-
-#[cfg(test)]
-const SQPAYLOAD: &[u8] = b"*1\n~3\n3\nSET\n1\nx\n3\n100\n";
-#[cfg(test)]
-const PQPAYLOAD: &[u8] = b"*2\n~3\n3\nSET\n1\nx\n3\n100\n~2\n3\nGET\n1\nx\n";
-
-#[test]
-fn parse_simple_query() {
-    let payload = SQPAYLOAD.to_vec();
-    let (q, f) = Parser::parse(&payload).unwrap();
-    let q: Vec<String> = if let Query::Simple(q) = q {
-        q.as_slice()
-            .iter()
-            .map(|v| String::from_utf8_lossy(unsafe { v.as_slice() }).to_string())
-            .collect()
-    } else {
-        panic!("Expected simple query")
-    };
-    assert_eq!(f, payload.len());
-    assert_eq!(q, vec!["SET".to_owned(), "x".into(), "100".into()]);
-}
-
-#[test]
-fn parse_simple_query_incomplete() {
-    for i in 0..SQPAYLOAD.len() - 1 {
-        let slice = &SQPAYLOAD[..i];
-        assert_eq!(Parser::parse(slice).unwrap_err(), ParseError::NotEnough);
-    }
-}
-
-#[test]
-fn parse_pipelined_query() {
-    let payload = PQPAYLOAD.to_vec();
-    let (q, f) = Parser::parse(&payload).unwrap();
-    let q: Vec<Vec<String>> = if let Query::Pipelined(q) = q {
-        q.into_inner()
-            .iter()
-            .map(|sq| {
-                sq.iter()
-                    .map(|v| String::from_utf8_lossy(unsafe { v.as_slice() }).to_string())
-                    .collect()
-            })
-            .collect()
-    } else {
-        panic!("Expected pipelined query query")
-    };
-    assert_eq!(f, payload.len());
-    assert_eq!(
-        q,
-        vec![
-            vec!["SET".to_owned(), "x".into(), "100".into()],
-            vec!["GET".into(), "x".into()]
-        ]
-    );
-}
-
-#[test]
-fn parse_pipelined_query_incomplete() {
-    for i in 0..PQPAYLOAD.len() - 1 {
-        let slice = &PQPAYLOAD[..i];
-        assert_eq!(Parser::parse(slice).unwrap_err(), ParseError::NotEnough);
-    }
-}
diff --git a/server/src/protocol/v2/benches.rs b/server/src/protocol/v2/benches.rs
deleted file mode 100644
index 4b1c8f64..00000000
--- a/server/src/protocol/v2/benches.rs
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Created on Sat Apr 30 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-extern crate test;
-use {
-    super::{super::Query, Parser},
-    test::Bencher,
-};
-
-#[bench]
-fn simple_query(b: &mut Bencher) {
-    const PAYLOAD: &[u8] = b"*3\n3\nSET1\nx3\n100";
-    let expected = vec!["SET".to_owned(), "x".to_owned(), "100".to_owned()];
-    b.iter(|| {
-        let (query, forward) = Parser::parse(PAYLOAD).unwrap();
-        assert_eq!(forward, PAYLOAD.len());
-        let query = if let Query::Simple(sq) = query {
-            sq
-        } else {
-            panic!("Got pipeline instead of simple query");
-        };
-        let ret: Vec<String> = query
-            .as_slice()
-            .iter()
-            .map(|s| String::from_utf8_lossy(unsafe { s.as_slice() }).to_string())
-            .collect();
-        assert_eq!(ret, expected)
-    });
-}
-
-#[bench]
-fn pipelined_query(b: &mut Bencher) {
-    const PAYLOAD: &[u8] = b"$2\n3\n3\nSET1\nx3\n1002\n3\nGET1\nx";
-    let expected = vec![
-        vec!["SET".to_owned(), "x".to_owned(), "100".to_owned()],
-        vec!["GET".to_owned(), "x".to_owned()],
-    ];
-    b.iter(|| {
-        let (query, forward) = Parser::parse(PAYLOAD).unwrap();
-        assert_eq!(forward, PAYLOAD.len());
-        let query = if let Query::Pipelined(sq) = query {
-            sq
-        } else {
-            panic!("Got simple instead of pipeline query");
-        };
-        let ret: Vec<Vec<String>> = query
-            .into_inner()
-            .iter()
-            .map(|query| {
-                query
-                    .as_slice()
-                    .iter()
-                    .map(|v| String::from_utf8_lossy(unsafe { v.as_slice() }).to_string())
-                    .collect()
-            })
-            .collect();
-        assert_eq!(ret, expected)
-    });
-}
diff --git a/server/src/protocol/v2/interface_impls.rs b/server/src/protocol/v2/interface_impls.rs
deleted file mode 100644
index 349d8683..00000000
--- a/server/src/protocol/v2/interface_impls.rs
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Created on Sat Apr 30 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{
-        dbnet::QueryWithAdvance,
-        protocol::{interface::ProtocolSpec, ParseError, Skyhash2},
-    },
-    ::sky_macros::compiled_eresp_bytes as eresp,
-};
-
-impl ProtocolSpec for Skyhash2 {
-    // spec information
-    const PROTOCOL_VERSION: f32 = 2.0;
-    const PROTOCOL_VERSIONSTRING: &'static str = "Skyhash-2.0";
-
-    // type symbols
-    const TSYMBOL_STRING: u8 = b'+';
-    const TSYMBOL_BINARY: u8 = b'?';
-    const TSYMBOL_FLOAT: u8 = b'%';
-    const TSYMBOL_INT64: u8 = b':';
-    const TSYMBOL_TYPED_ARRAY: u8 = b'@';
-    const TSYMBOL_TYPED_NON_NULL_ARRAY: u8 = b'^';
-    const TSYMBOL_ARRAY: u8 = b'&';
-    const TSYMBOL_FLAT_ARRAY: u8 = b'_';
-
-    // typed array
-    const TYPE_TYPED_ARRAY_ELEMENT_NULL: &'static [u8] = b"\0";
-
-    // metaframe
-    const SIMPLE_QUERY_HEADER: &'static [u8] = b"*";
-    const PIPELINED_QUERY_FIRST_BYTE: u8 = b'$';
-
-    // respcodes
-    const RCODE_OKAY: &'static [u8] = eresp!("0");
-    const RCODE_NIL: &'static [u8] = eresp!("1");
-    const RCODE_OVERWRITE_ERR: &'static [u8] = eresp!("2");
-    const RCODE_ACTION_ERR: &'static [u8] = eresp!("3");
-    const RCODE_PACKET_ERR: &'static [u8] = eresp!("4");
-    const RCODE_SERVER_ERR: &'static [u8] = eresp!("5");
-    const RCODE_OTHER_ERR_EMPTY: &'static [u8] = eresp!("6");
-    const RCODE_UNKNOWN_ACTION: &'static [u8] = eresp!("Unknown action");
-    const RCODE_WRONGTYPE_ERR: &'static [u8] = eresp!("7");
-    const RCODE_UNKNOWN_DATA_TYPE: &'static [u8] = eresp!("8");
-    const RCODE_ENCODING_ERROR: &'static [u8] = eresp!("9");
-
-    // respstrings
-    const RSTRING_SNAPSHOT_BUSY: &'static [u8] = eresp!("err-snapshot-busy");
-    const RSTRING_SNAPSHOT_DISABLED: &'static [u8] = eresp!("err-snapshot-disabled");
-    const RSTRING_SNAPSHOT_DUPLICATE: &'static [u8] = eresp!("duplicate-snapshot");
-    const RSTRING_SNAPSHOT_ILLEGAL_NAME: &'static [u8] = eresp!("err-invalid-snapshot-name");
-    const RSTRING_ERR_ACCESS_AFTER_TERMSIG: &'static [u8] = eresp!("err-access-after-termsig");
-
-    // keyspace related resps
-    const RSTRING_DEFAULT_UNSET: &'static [u8] = eresp!("default-container-unset");
-    const RSTRING_CONTAINER_NOT_FOUND: &'static [u8] = eresp!("container-not-found");
-    const RSTRING_STILL_IN_USE: &'static [u8] = eresp!("still-in-use");
-    const RSTRING_PROTECTED_OBJECT: &'static [u8] = eresp!("err-protected-object");
-    const RSTRING_WRONG_MODEL: &'static [u8] = eresp!("wrong-model");
-    const RSTRING_ALREADY_EXISTS: &'static [u8] = eresp!("err-already-exists");
-    const RSTRING_NOT_READY: &'static [u8] = eresp!("not-ready");
-    const RSTRING_DDL_TRANSACTIONAL_FAILURE: &'static [u8] = eresp!("transactional-failure");
-    const RSTRING_UNKNOWN_DDL_QUERY: &'static [u8] = eresp!("unknown-ddl-query");
-    const RSTRING_BAD_EXPRESSION: &'static [u8] = eresp!("malformed-expression");
-    const RSTRING_UNKNOWN_MODEL: &'static [u8] = eresp!("unknown-model");
-    const RSTRING_TOO_MANY_ARGUMENTS: &'static [u8] = eresp!("too-many-args");
-    const RSTRING_CONTAINER_NAME_TOO_LONG: &'static [u8] = eresp!("container-name-too-long");
-    const RSTRING_BAD_CONTAINER_NAME: &'static [u8] = eresp!("bad-container-name");
-    const RSTRING_UNKNOWN_INSPECT_QUERY: &'static [u8] = eresp!("unknown-inspect-query");
-    const RSTRING_UNKNOWN_PROPERTY: &'static [u8] = eresp!("unknown-property");
-    const RSTRING_KEYSPACE_NOT_EMPTY: &'static [u8] = eresp!("keyspace-not-empty");
-    const RSTRING_BAD_TYPE_FOR_KEY: &'static [u8] = eresp!("bad-type-for-key");
-    const RSTRING_LISTMAP_BAD_INDEX: &'static [u8] = eresp!("bad-list-index");
-    const RSTRING_LISTMAP_LIST_IS_EMPTY: &'static [u8] = eresp!("list-is-empty");
-
-    // elements
-    const ELEMRESP_HEYA: &'static [u8] = b"+4\nHEY!";
-
-    // full responses
-    const FULLRESP_RCODE_PACKET_ERR: &'static [u8] = b"*!4\n";
-    const FULLRESP_RCODE_WRONG_TYPE: &'static [u8] = b"*!7\n";
-
-    // auth respcodes/strings
-    const AUTH_ERROR_ALREADYCLAIMED: &'static [u8] = eresp!("err-auth-already-claimed");
-    const AUTH_CODE_BAD_CREDENTIALS: &'static [u8] = eresp!("10");
-    const AUTH_ERROR_DISABLED: &'static [u8] = eresp!("err-auth-disabled");
-    const AUTH_CODE_PERMS: &'static [u8] = eresp!("11");
-    const AUTH_ERROR_ILLEGAL_USERNAME: &'static [u8] = eresp!("err-auth-illegal-username");
-    const AUTH_ERROR_FAILED_TO_DELETE_USER: &'static [u8] = eresp!("err-auth-deluser-fail");
-
-    // bql respstrings
-    const BQL_BAD_EXPRESSION: &'static [u8] = eresp!("bql-bad-expression");
-    const BQL_EXPECTED_STMT: &'static [u8] = eresp!("bql-expected-statement");
-    const BQL_INVALID_NUMERIC_LITERAL: &'static [u8] = eresp!("bql-bad-numeric-literal");
-    const BQL_INVALID_STRING_LITERAL: &'static [u8] = eresp!("bql-bad-string-literal");
-    const BQL_INVALID_SYNTAX: &'static [u8] = eresp!("bql-invalid-syntax");
-    const BQL_UNEXPECTED_EOF: &'static [u8] = eresp!("bql-unexpected-eof");
-    const BQL_UNKNOWN_CREATE_QUERY: &'static [u8] = eresp!("bql-unknown-create-query");
-    const BQL_UNSUPPORTED_MODEL_DECL: &'static [u8] = eresp!("bql-unsupported-model-decl");
-    const BQL_UNEXPECTED_CHAR: &'static [u8] = eresp!("bql-unexpected-char");
-
-    const NEEDS_TERMINAL_LF: bool = false;
-
-    fn decode_packet(input: &[u8]) -> Result<QueryWithAdvance, ParseError> {
-        Skyhash2::parse(input)
-    }
-}
diff --git a/server/src/protocol/v2/mod.rs b/server/src/protocol/v2/mod.rs
deleted file mode 100644
index 6720e8b5..00000000
--- a/server/src/protocol/v2/mod.rs
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Created on Fri Apr 29 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-mod interface_impls;
-
-use {
-    super::{
-        raw_parser::{RawParser, RawParserExt, RawParserMeta},
-        ParseError, ParseResult, PipelinedQuery, Query, SimpleQuery, UnsafeSlice,
-    },
-    crate::{corestore::heap_array::HeapArray, dbnet::QueryWithAdvance},
-};
-
-#[cfg(feature = "nightly")]
-#[cfg(test)]
-mod benches;
-#[cfg(test)]
-mod tests;
-
-/// A parser for Skyhash 2.0
-pub struct Parser {
-    end: *const u8,
-    cursor: *const u8,
-}
-
-unsafe impl RawParser for Parser {
-    fn cursor_ptr(&self) -> *const u8 {
-        self.cursor
-    }
-    fn cursor_ptr_mut(&mut self) -> &mut *const u8 {
-        &mut self.cursor
-    }
-    fn data_end_ptr(&self) -> *const u8 {
-        self.end
-    }
-}
-
-unsafe impl Sync for Parser {}
-unsafe impl Send for Parser {}
-
-impl Parser {
-    /// Initialize a new parser
-    fn new(slice: &[u8]) -> Self {
-        unsafe {
-            Self {
-                end: slice.as_ptr().add(slice.len()),
-                cursor: slice.as_ptr(),
-            }
-        }
-    }
-}
-
-// query impls
-impl Parser {
-    /// Parse the next simple query. This should have passed the `*` tsymbol
-    ///
-    /// Simple query structure (tokenized line-by-line):
-    /// ```text
-    /// *      -> Simple Query Header
-    /// <n>\n  -> Count of elements in the simple query
-    /// <l0>\n -> Length of element 1
-    /// <e0>   -> element 1 itself
-    /// <l1>\n -> Length of element 2
-    /// <e1>   -> element 2 itself
-    /// ...
-    /// ```
-    fn _next_simple_query(&mut self) -> ParseResult<HeapArray<UnsafeSlice>> {
-        let element_count = self.read_usize()?;
-        unsafe {
-            let mut data = HeapArray::new_writer(element_count);
-            for i in 0..element_count {
-                let element_size = self.read_usize()?;
-                let element = self.read_until(element_size)?;
-                data.write_to_index(i, element);
-            }
-            Ok(data.finish())
-        }
-    }
-    /// Parse a simple query
-    fn next_simple_query(&mut self) -> ParseResult<SimpleQuery> {
-        Ok(SimpleQuery::new(self._next_simple_query()?))
-    }
-    /// Parse a pipelined query. This should have passed the `$` tsymbol
-    ///
-    /// Pipelined query structure (tokenized line-by-line):
-    /// ```text
-    /// $          -> Pipeline
-    /// <n>\n      -> Pipeline has n queries
-    /// <lq0>\n    -> Query 1 has 3 elements
-    /// <lq0e0>\n  -> Q1E1 has 3 bytes
-    /// <q0e0>     -> Q1E1 itself
-    /// <lq0e1>\n  -> Q1E2 has 1 byte
-    /// <q0e1>     -> Q1E2 itself
-    /// <lq0e2>\n  -> Q1E3 has 3 bytes
-    /// <q0e2>     -> Q1E3 itself
-    /// <lq1>\n    -> Query 2 has 2 elements
-    /// <lq1e0>\n  -> Q2E1 has 3 bytes
-    /// <q1e0>     -> Q2E1 itself
-    /// <lq1e1>\n  -> Q2E2 has 1 byte
-    /// <q1e1>     -> Q2E2 itself
-    /// ...
-    /// ```
-    ///
-    /// Example:
-    /// ```text
-    /// $    -> Pipeline
-    /// 2\n  -> Pipeline has 2 queries
-    /// 3\n  -> Query 1 has 3 elements
-    /// 3\n  -> Q1E1 has 3 bytes
-    /// SET  -> Q1E1 itself
-    /// 1\n  -> Q1E2 has 1 byte
-    /// x    -> Q1E2 itself
-    /// 3\n  -> Q1E3 has 3 bytes
-    /// 100  -> Q1E3 itself
-    /// 2\n  -> Query 2 has 2 elements
-    /// 3\n  -> Q2E1 has 3 bytes
-    /// GET  -> Q2E1 itself
-    /// 1\n  -> Q2E2 has 1 byte
-    /// x    -> Q2E2 itself
-    /// ```
-    fn next_pipeline(&mut self) -> ParseResult<PipelinedQuery> {
-        let query_count = self.read_usize()?;
-        unsafe {
-            let mut queries = HeapArray::new_writer(query_count);
-            for i in 0..query_count {
-                let sq = self._next_simple_query()?;
-                queries.write_to_index(i, sq);
-            }
-            Ok(PipelinedQuery {
-                data: queries.finish(),
-            })
-        }
-    }
-    fn _parse(&mut self) -> ParseResult<Query> {
-        if self.not_exhausted() {
-            unsafe {
-                let first_byte = self.get_byte_at_cursor();
-                self.incr_cursor();
-                let data = match first_byte {
-                    b'*' => {
-                        // a simple query
-                        Query::Simple(self.next_simple_query()?)
-                    }
-                    b'$' => {
-                        // a pipelined query
-                        Query::Pipelined(self.next_pipeline()?)
-                    }
-                    _ => return Err(ParseError::UnexpectedByte),
-                };
-                Ok(data)
-            }
-        } else {
-            Err(ParseError::NotEnough)
-        }
-    }
-    // only expose this. don't expose Self::new since that'll be _relatively easier_ to
-    // invalidate invariants for
-    pub fn parse(buf: &[u8]) -> ParseResult<QueryWithAdvance> {
-        let mut slf = Self::new(buf);
-        let body = slf._parse()?;
-        let consumed = slf.cursor_ptr() as usize - buf.as_ptr() as usize;
-        Ok((body, consumed))
-    }
-}
diff --git a/server/src/protocol/v2/tests.rs b/server/src/protocol/v2/tests.rs
deleted file mode 100644
index 0cb56883..00000000
--- a/server/src/protocol/v2/tests.rs
+++ /dev/null
@@ -1,645 +0,0 @@
-/*
- * Created on Tue Apr 12 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{
-        super::raw_parser::{RawParser, RawParserExt, RawParserMeta},
-        Parser, PipelinedQuery, Query, SimpleQuery,
-    },
-    crate::protocol::{iter::AnyArrayIter, ParseError},
-    std::{iter::Map, vec::IntoIter as VecIntoIter},
-};
-
-type IterPacketWithLen = Map<VecIntoIter<Vec<u8>>, fn(Vec<u8>) -> (usize, Vec<u8>)>;
-type Packets = Vec<Vec<u8>>;
-
-macro_rules! v {
-    () => {
-        vec![]
-    };
-    ($literal:literal) => {
-        $literal.to_vec()
-    };
-    ($($lit:literal),*) => {
-        vec![$(
-            $lit.as_bytes().to_owned()
-        ),*]
-    }
-}
-
-fn ensure_exhausted(p: &Parser) {
-    assert!(!p.not_exhausted());
-    assert!(p.exhausted());
-}
-
-fn ensure_remaining(p: &Parser, r: usize) {
-    assert_eq!(p.remaining(), r);
-    assert!(p.has_remaining(r));
-}
-
-fn ensure_not_exhausted(p: &Parser) {
-    assert!(p.not_exhausted());
-    assert!(!p.exhausted());
-}
-
-fn get_slices(slices: &[&[u8]]) -> Packets {
-    slices.iter().map(|slc| slc.to_vec()).collect()
-}
-
-fn ensure_zero_reads(parser: &mut Parser) {
-    let r = parser.read_until(0).unwrap();
-    let slice = unsafe { r.as_slice() };
-    assert_eq!(slice, b"");
-    assert!(slice.is_empty());
-}
-
-// We do this intentionally for "heap simulation"
-fn slices() -> Packets {
-    const SLICE_COLLECTION: &[&[u8]] = &[
-        b"",
-        b"a",
-        b"ab",
-        b"abc",
-        b"abcd",
-        b"abcde",
-        b"abcdef",
-        b"abcdefg",
-        b"abcdefgh",
-        b"abcdefghi",
-        b"abcdefghij",
-        b"abcdefghijk",
-        b"abcdefghijkl",
-        b"abcdefghijklm",
-    ];
-    get_slices(SLICE_COLLECTION)
-}
-
-fn get_slices_with_len(slices: Packets) -> IterPacketWithLen {
-    slices.into_iter().map(|slc| (slc.len(), slc))
-}
-
-fn slices_with_len() -> IterPacketWithLen {
-    get_slices_with_len(slices())
-}
-
-fn slices_lf() -> Packets {
-    const SLICE_COLLECTION: &[&[u8]] = &[
-        b"",
-        b"a\n",
-        b"ab\n",
-        b"abc\n",
-        b"abcd\n",
-        b"abcde\n",
-        b"abcdef\n",
-        b"abcdefg\n",
-        b"abcdefgh\n",
-        b"abcdefghi\n",
-        b"abcdefghij\n",
-        b"abcdefghijk\n",
-        b"abcdefghijkl\n",
-        b"abcdefghijklm\n",
-    ];
-    get_slices(SLICE_COLLECTION)
-}
-
-fn slices_lf_with_len() -> IterPacketWithLen {
-    get_slices_with_len(slices_lf())
-}
-
-fn simple_query(query: Query) -> SimpleQuery {
-    if let Query::Simple(sq) = query {
-        sq
-    } else {
-        panic!("Got pipeline instead of simple!");
-    }
-}
-
-fn pipelined_query(query: Query) -> PipelinedQuery {
-    if let Query::Pipelined(pq) = query {
-        pq
-    } else {
-        panic!("Got simple instead of pipeline!");
-    }
-}
-
-// "actual" tests
-// data_end_ptr
-#[test]
-fn data_end_ptr() {
-    for (len, src) in slices_with_len() {
-        let parser = Parser::new(&src);
-        unsafe {
-            assert_eq!(parser.data_end_ptr(), src.as_ptr().add(len));
-        }
-    }
-}
-
-// cursor_ptr
-#[test]
-fn cursor_ptr() {
-    for src in slices() {
-        let parser = Parser::new(&src);
-        assert_eq!(parser.cursor_ptr(), src.as_ptr())
-    }
-}
-#[test]
-fn cursor_ptr_with_incr() {
-    for src in slices() {
-        let mut parser = Parser::new(&src);
-        unsafe {
-            parser.incr_cursor_by(src.len());
-            assert_eq!(parser.cursor_ptr(), src.as_ptr().add(src.len()));
-        }
-    }
-}
-
-// remaining
-#[test]
-fn remaining() {
-    for (len, src) in slices_with_len() {
-        let parser = Parser::new(&src);
-        assert_eq!(parser.remaining(), len);
-    }
-}
-#[test]
-fn remaining_with_incr() {
-    for (len, src) in slices_with_len() {
-        let mut parser = Parser::new(&src);
-        unsafe {
-            // no change
-            parser.incr_cursor_by(0);
-            assert_eq!(parser.remaining(), len);
-            if len != 0 {
-                // move one byte ahead. should reach EOA or len - 1
-                parser.incr_cursor();
-                assert_eq!(parser.remaining(), len - 1);
-                // move the cursor to the end; should reach EOA
-                parser.incr_cursor_by(len - 1);
-                assert_eq!(parser.remaining(), 0);
-            }
-        }
-    }
-}
-
-// has_remaining
-#[test]
-fn has_remaining() {
-    for (len, src) in slices_with_len() {
-        let parser = Parser::new(&src);
-        assert!(parser.has_remaining(len), "should have {len} remaining")
-    }
-}
-#[test]
-fn has_remaining_with_incr() {
-    for (len, src) in slices_with_len() {
-        let mut parser = Parser::new(&src);
-        unsafe {
-            // no change
-            parser.incr_cursor_by(0);
-            assert!(parser.has_remaining(len));
-            if len != 0 {
-                // move one byte ahead. should reach EOA or len - 1
-                parser.incr_cursor();
-                assert!(parser.has_remaining(len - 1));
-                // move the cursor to the end; should reach EOA
-                parser.incr_cursor_by(len - 1);
-                assert!(!parser.has_remaining(1));
-                // should always be true
-                assert!(parser.has_remaining(0));
-            }
-        }
-    }
-}
-
-// exhausted
-#[test]
-fn exhausted() {
-    for src in slices() {
-        let parser = Parser::new(&src);
-        if src.is_empty() {
-            assert!(parser.exhausted());
-        } else {
-            assert!(!parser.exhausted())
-        }
-    }
-}
-#[test]
-fn exhausted_with_incr() {
-    for (len, src) in slices_with_len() {
-        let mut parser = Parser::new(&src);
-        if len == 0 {
-            assert!(parser.exhausted());
-        } else {
-            assert!(!parser.exhausted());
-            unsafe {
-                parser.incr_cursor();
-                if len == 1 {
-                    assert!(parser.exhausted());
-                } else {
-                    assert!(!parser.exhausted());
-                    parser.incr_cursor_by(len - 1);
-                    assert!(parser.exhausted());
-                }
-            }
-        }
-    }
-}
-
-// not_exhausted
-#[test]
-fn not_exhausted() {
-    for src in slices() {
-        let parser = Parser::new(&src);
-        if src.is_empty() {
-            assert!(!parser.not_exhausted());
-        } else {
-            assert!(parser.not_exhausted())
-        }
-    }
-}
-#[test]
-fn not_exhausted_with_incr() {
-    for (len, src) in slices_with_len() {
-        let mut parser = Parser::new(&src);
-        if len == 0 {
-            assert!(!parser.not_exhausted());
-        } else {
-            assert!(parser.not_exhausted());
-            unsafe {
-                parser.incr_cursor();
-                if len == 1 {
-                    assert!(!parser.not_exhausted());
-                } else {
-                    assert!(parser.not_exhausted());
-                    parser.incr_cursor_by(len - 1);
-                    assert!(!parser.not_exhausted());
-                }
-            }
-        }
-    }
-}
-
-// read_until
-#[test]
-fn read_until_empty() {
-    let b = v!(b"");
-    let mut parser = Parser::new(&b);
-    ensure_zero_reads(&mut parser);
-    assert_eq!(parser.read_until(1).unwrap_err(), ParseError::NotEnough);
-}
-
-#[test]
-fn read_until_nonempty() {
-    for (len, src) in slices_with_len() {
-        let mut parser = Parser::new(&src);
-        // should always work
-        ensure_zero_reads(&mut parser);
-        // now read the entire length; should always work
-        let r = parser.read_until(len).unwrap();
-        let slice = unsafe { r.as_slice() };
-        assert_eq!(slice, src.as_slice());
-        assert_eq!(slice.len(), len);
-        // even after the buffer is exhausted, `0` should always work
-        ensure_zero_reads(&mut parser);
-    }
-}
-
-#[test]
-fn read_until_not_enough() {
-    for (len, src) in slices_with_len() {
-        let mut parser = Parser::new(&src);
-        ensure_zero_reads(&mut parser);
-        // try to read more than the amount of data bufferred
-        assert_eq!(
-            parser.read_until(len + 1).unwrap_err(),
-            ParseError::NotEnough
-        );
-        // should the above fail, zero reads should still work
-        ensure_zero_reads(&mut parser);
-    }
-}
-
-#[test]
-fn read_until_more_bytes() {
-    let sample1 = v!(b"abcd1");
-    let mut p1 = Parser::new(&sample1);
-    assert_eq!(
-        unsafe { p1.read_until(&sample1.len() - 1).unwrap().as_slice() },
-        &sample1[..&sample1.len() - 1]
-    );
-    // ensure we have not exhasuted
-    ensure_not_exhausted(&p1);
-    ensure_remaining(&p1, 1);
-    let sample2 = v!(b"abcd1234567890!@#$");
-    let mut p2 = Parser::new(&sample2);
-    assert_eq!(
-        unsafe { p2.read_until(4).unwrap().as_slice() },
-        &sample2[..4]
-    );
-    // ensure we have not exhasuted
-    ensure_not_exhausted(&p2);
-    ensure_remaining(&p2, sample2.len() - 4);
-}
-
-// read_line
-#[test]
-fn read_line_special_case_only_lf() {
-    let b = v!(b"\n");
-    let mut parser = Parser::new(&b);
-    let r = parser.read_line().unwrap();
-    let slice = unsafe { r.as_slice() };
-    assert_eq!(slice, b"");
-    assert!(slice.is_empty());
-    // ensure it is exhausted
-    ensure_exhausted(&parser);
-}
-
-#[test]
-fn read_line() {
-    for (len, src) in slices_lf_with_len() {
-        let mut parser = Parser::new(&src);
-        if len == 0 {
-            // should be empty, so NotEnough
-            assert_eq!(parser.read_line().unwrap_err(), ParseError::NotEnough);
-        } else {
-            // should work
-            assert_eq!(
-                unsafe { parser.read_line().unwrap().as_slice() },
-                &src.as_slice()[..len - 1]
-            );
-            // now, we attempt to read which should work
-            ensure_zero_reads(&mut parser);
-        }
-        // ensure it is exhausted
-        ensure_exhausted(&parser);
-        // now, we attempt to read another line which should fail
-        assert_eq!(parser.read_line().unwrap_err(), ParseError::NotEnough);
-        // ensure that cursor is at end
-        unsafe {
-            assert_eq!(parser.cursor_ptr(), src.as_ptr().add(len));
-        }
-    }
-}
-
-#[test]
-fn read_line_more_bytes() {
-    let sample1 = v!(b"abcd\n1");
-    let mut p1 = Parser::new(&sample1);
-    let line = p1.read_line().unwrap();
-    assert_eq!(unsafe { line.as_slice() }, b"abcd");
-    // we should still have one remaining
-    ensure_not_exhausted(&p1);
-    ensure_remaining(&p1, 1);
-}
-
-#[test]
-fn read_line_subsequent_lf() {
-    let sample1 = v!(b"abcd\n1\n");
-    let mut p1 = Parser::new(&sample1);
-    let line = p1.read_line().unwrap();
-    assert_eq!(unsafe { line.as_slice() }, b"abcd");
-    // we should still have two octets remaining
-    ensure_not_exhausted(&p1);
-    ensure_remaining(&p1, 2);
-    // and we should be able to read in another line
-    let line = p1.read_line().unwrap();
-    assert_eq!(unsafe { line.as_slice() }, b"1");
-    ensure_exhausted(&p1);
-}
-
-#[test]
-fn read_line_pedantic_okay() {
-    for (len, src) in slices_lf_with_len() {
-        let mut parser = Parser::new(&src);
-        if len == 0 {
-            // should be empty, so NotEnough
-            assert_eq!(
-                parser.read_line_pedantic().unwrap_err(),
-                ParseError::NotEnough
-            );
-        } else {
-            // should work
-            assert_eq!(
-                unsafe { parser.read_line_pedantic().unwrap().as_slice() },
-                &src.as_slice()[..len - 1]
-            );
-            // now, we attempt to read which should work
-            ensure_zero_reads(&mut parser);
-        }
-        // ensure it is exhausted
-        ensure_exhausted(&parser);
-        // now, we attempt to read another line which should fail
-        assert_eq!(
-            parser.read_line_pedantic().unwrap_err(),
-            ParseError::NotEnough
-        );
-        // ensure that cursor is at end
-        unsafe {
-            assert_eq!(parser.cursor_ptr(), src.as_ptr().add(len));
-        }
-    }
-}
-
-#[test]
-fn read_line_pedantic_fail_empty() {
-    let payload = v!(b"");
-    assert_eq!(
-        Parser::new(&payload).read_line_pedantic().unwrap_err(),
-        ParseError::NotEnough
-    );
-}
-
-#[test]
-fn read_line_pedantic_fail_only_lf() {
-    let payload = v!(b"\n");
-    assert_eq!(
-        Parser::new(&payload).read_line_pedantic().unwrap_err(),
-        ParseError::BadPacket
-    );
-}
-
-#[test]
-fn read_line_pedantic_fail_only_lf_extra_data() {
-    let payload = v!(b"\n1");
-    assert_eq!(
-        Parser::new(&payload).read_line_pedantic().unwrap_err(),
-        ParseError::BadPacket
-    );
-}
-
-#[test]
-fn read_usize_fail_empty() {
-    let payload = v!(b"");
-    assert_eq!(
-        Parser::new(&payload).read_usize().unwrap_err(),
-        ParseError::NotEnough
-    );
-    let payload = v!(b"\n");
-    assert_eq!(
-        Parser::new(&payload).read_usize().unwrap_err(),
-        ParseError::BadPacket
-    );
-}
-
-#[test]
-fn read_usize_fail_no_lf() {
-    let payload = v!(b"1");
-    assert_eq!(
-        Parser::new(&payload).read_usize().unwrap_err(),
-        ParseError::NotEnough
-    );
-}
-
-#[test]
-fn read_usize_okay() {
-    let payload = v!(b"1\n");
-    assert_eq!(Parser::new(&payload).read_usize().unwrap(), 1);
-    let payload = v!(b"1234\n");
-    assert_eq!(Parser::new(&payload).read_usize().unwrap(), 1234);
-}
-
-#[test]
-fn read_usize_fail() {
-    let payload = v!(b"a\n");
-    assert_eq!(
-        Parser::new(&payload).read_usize().unwrap_err(),
-        ParseError::DatatypeParseFailure
-    );
-    let payload = v!(b"1a\n");
-    assert_eq!(
-        Parser::new(&payload).read_usize().unwrap_err(),
-        ParseError::DatatypeParseFailure
-    );
-    let payload = v!(b"a1\n");
-    assert_eq!(
-        Parser::new(&payload).read_usize().unwrap_err(),
-        ParseError::DatatypeParseFailure
-    );
-    let payload = v!(b"aa\n");
-    assert_eq!(
-        Parser::new(&payload).read_usize().unwrap_err(),
-        ParseError::DatatypeParseFailure
-    );
-    let payload = v!(b"12345abcde\n");
-    assert_eq!(
-        Parser::new(&payload).read_usize().unwrap_err(),
-        ParseError::DatatypeParseFailure
-    );
-}
-
-#[test]
-fn parse_fail_because_unknown_query_scheme() {
-    let body = v!(b"?3\n3\nSET1\nx3\n100");
-    assert_eq!(
-        Parser::parse(&body).unwrap_err(),
-        ParseError::UnexpectedByte
-    )
-}
-
-#[test]
-fn simple_query_okay() {
-    let body = v!(b"*3\n3\nSET1\nx3\n100");
-    let (ret, skip) = Parser::parse(&body).unwrap();
-    assert_eq!(skip, body.len());
-    let query = simple_query(ret);
-    assert_eq!(query.into_owned().data, v!["SET", "x", "100"]);
-}
-
-#[test]
-fn simple_query_okay_empty_elements() {
-    let body = v!(b"*3\n3\nSET0\n0\n");
-    let (ret, skip) = Parser::parse(&body).unwrap();
-    assert_eq!(skip, body.len());
-    let query = simple_query(ret);
-    assert_eq!(query.into_owned().data, v!["SET", "", ""]);
-}
-
-#[test]
-fn parse_fail_because_not_enough() {
-    let full_payload = b"*3\n3\nSET1\nx3\n100";
-    let samples: Vec<Vec<u8>> = (0..full_payload.len() - 1)
-        .map(|i| full_payload.iter().take(i).cloned().collect())
-        .collect();
-    for body in samples {
-        assert_eq!(
-            Parser::parse(&body).unwrap_err(),
-            ParseError::NotEnough,
-            "Failed with body len: {}",
-            body.len()
-        )
-    }
-}
-
-#[test]
-fn pipelined_query_okay() {
-    let body = v!(b"$2\n3\n3\nSET1\nx3\n1002\n3\nGET1\nx");
-    let (ret, skip) = Parser::parse(&body).unwrap();
-    assert_eq!(skip, body.len());
-    let query = pipelined_query(ret);
-    assert_eq!(
-        query.into_owned().data,
-        vec![v!["SET", "x", "100"], v!["GET", "x"]]
-    )
-}
-
-#[test]
-fn pipelined_query_okay_empty_elements() {
-    let body = v!(b"$2\n3\n3\nSET0\n3\n1002\n3\nGET0\n");
-    let (ret, skip) = Parser::parse(&body).unwrap();
-    assert_eq!(skip, body.len());
-    let query = pipelined_query(ret);
-    assert_eq!(
-        query.into_owned().data,
-        vec![v!["SET", "", "100"], v!["GET", ""]]
-    )
-}
-
-#[test]
-fn pipelined_query_fail_because_not_enough() {
-    let full_payload = v!(b"$2\n3\n3\nSET1\nx3\n1002\n3\nGET1\nx");
-    let samples: Vec<Vec<u8>> = (0..full_payload.len() - 1)
-        .map(|i| full_payload.iter().cloned().take(i).collect())
-        .collect();
-    for body in samples {
-        let ret = Parser::parse(&body).unwrap_err();
-        assert_eq!(ret, ParseError::NotEnough)
-    }
-}
-
-#[test]
-fn test_iter() {
-    use super::{Parser, Query};
-    let (q, _fwby) = Parser::parse(b"*3\n3\nset1\nx3\n100").unwrap();
-    let r = match q {
-        Query::Simple(q) => q,
-        _ => panic!("Wrong query"),
-    };
-    let it = r.as_slice().iter();
-    let mut iter = unsafe { AnyArrayIter::new(it) };
-    assert_eq!(iter.next_uppercase().unwrap().as_ref(), "SET".as_bytes());
-    assert_eq!(iter.next().unwrap(), "x".as_bytes());
-    assert_eq!(iter.next().unwrap(), "100".as_bytes());
-}
diff --git a/server/src/queryengine/mod.rs b/server/src/queryengine/mod.rs
deleted file mode 100644
index 00327f18..00000000
--- a/server/src/queryengine/mod.rs
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Created on Mon Aug 03 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # The Query Engine
-
-use crate::{
-    actions::{self, ActionError, ActionResult},
-    admin, auth, blueql,
-    corestore::Corestore,
-    dbnet::{prelude::*, BufferedSocketStream},
-    protocol::{iter::AnyArrayIter, PipelinedQuery, SimpleQuery, UnsafeSlice},
-};
-
-pub type ActionIter<'a> = AnyArrayIter<'a>;
-
-const ACTION_AUTH: &[u8] = b"auth";
-
-macro_rules! gen_constants_and_matches {
-    (
-        $con:expr, $buf:ident, $db:ident, $($action:ident => $fns:path),*,
-        {$($action2:ident => $fns2:expr),*}
-    ) => {
-        mod tags {
-            //! This module is a collection of tags/strings used for evaluating queries
-            //! and responses
-            $(
-                pub const $action: &[u8] = stringify!($action).as_bytes();
-            )*
-            $(
-                pub const $action2: &[u8] = stringify!($action2).as_bytes();
-            )*
-        }
-        let first_slice = $buf.next().unwrap_or_custom_aerr(P::RCODE_PACKET_ERR)?;
-        let first = first_slice.to_ascii_uppercase();
-        match first.as_ref() {
-            $(
-                tags::$action => $fns($db, $con, $buf).await?,
-            )*
-            $(
-                tags::$action2 => $fns2.await?,
-            )*
-            _ => {
-                blueql::execute($db, $con, first_slice, $buf.len()).await?;
-            }
-        }
-    };
-}
-
-action! {
-    /// Execute queries for an anonymous user
-    fn execute_simple_noauth(
-        _db: &mut Corestore,
-        con: &mut Connection<C, P>,
-        auth: &mut AuthProviderHandle,
-        buf: SimpleQuery
-    ) {
-        let bufref = buf.as_slice();
-        let mut iter = unsafe {
-            // UNSAFE(@ohsayan): The presence of the connection guarantees that this
-            // won't suddenly become invalid
-            AnyArrayIter::new(bufref.iter())
-        };
-        match iter.next_lowercase().unwrap_or_custom_aerr(P::RCODE_PACKET_ERR)?.as_ref() {
-            ACTION_AUTH => auth::auth_login_only(con, auth, iter).await,
-            _ => util::err(P::AUTH_CODE_BAD_CREDENTIALS),
-        }
-    }
-    //// Execute a simple query
-    fn execute_simple(
-        db: &mut Corestore,
-        con: &mut Connection<C, P>,
-        auth: &mut AuthProviderHandle,
-        buf: SimpleQuery
-    ) {
-        self::execute_stage(db, con, auth, buf.as_slice()).await
-    }
-}
-
-async fn execute_stage<'a, P: ProtocolSpec, C: BufferedSocketStream>(
-    db: &mut Corestore,
-    con: &mut Connection<C, P>,
-    auth: &mut AuthProviderHandle,
-    buf: &[UnsafeSlice],
-) -> ActionResult<()> {
-    let mut iter = unsafe {
-        // UNSAFE(@ohsayan): The presence of the connection guarantees that this
-        // won't suddenly become invalid
-        AnyArrayIter::new(buf.iter())
-    };
-    {
-        gen_constants_and_matches!(
-            con, iter, db,
-            GET => actions::get::get,
-            SET => actions::set::set,
-            UPDATE => actions::update::update,
-            DEL => actions::del::del,
-            HEYA => actions::heya::heya,
-            EXISTS => actions::exists::exists,
-            MSET => actions::mset::mset,
-            MGET => actions::mget::mget,
-            MUPDATE => actions::mupdate::mupdate,
-            SSET => actions::strong::sset,
-            SDEL => actions::strong::sdel,
-            SUPDATE => actions::strong::supdate,
-            DBSIZE => actions::dbsize::dbsize,
-            FLUSHDB => actions::flushdb::flushdb,
-            USET => actions::uset::uset,
-            KEYLEN => actions::keylen::keylen,
-            MKSNAP => admin::mksnap::mksnap,
-            LSKEYS => actions::lskeys::lskeys,
-            POP => actions::pop::pop,
-            MPOP => actions::mpop::mpop,
-            LSET => actions::lists::lset,
-            LGET => actions::lists::lget::lget,
-            LMOD => actions::lists::lmod::lmod,
-            WHEREAMI => actions::whereami::whereami,
-            SYS => admin::sys::sys,
-            {
-                // actions that need other arguments
-                AUTH => auth::auth(con, auth, iter)
-            }
-        );
-    }
-    Ok(())
-}
-
-/// Execute a stage **completely**. This means that action errors are never propagated
-/// over the try operator
-async fn execute_stage_pedantic<'a, C: BufferedSocketStream, P: ProtocolSpec>(
-    handle: &mut Corestore,
-    con: &mut Connection<C, P>,
-    auth: &mut AuthProviderHandle,
-    stage: &[UnsafeSlice],
-) -> crate::IoResult<()> {
-    let ret = async {
-        self::execute_stage(handle, con, auth, stage).await?;
-        Ok(())
-    };
-    match ret.await {
-        Ok(()) => Ok(()),
-        Err(ActionError::ActionError(e)) => con._write_raw(e).await,
-        Err(ActionError::IoError(ioe)) => Err(ioe),
-    }
-}
-
-action! {
-    /// Execute a basic pipelined query
-    fn execute_pipeline(
-        handle: &mut Corestore,
-        con: &mut Connection<C, P>,
-        auth: &mut AuthProviderHandle,
-        pipeline: PipelinedQuery
-    ) {
-        for stage in pipeline.into_inner().iter() {
-            self::execute_stage_pedantic(handle, con, auth, stage).await?;
-        }
-        Ok(())
-    }
-}
diff --git a/server/src/registry/mod.rs b/server/src/registry/mod.rs
deleted file mode 100644
index 5c5298c4..00000000
--- a/server/src/registry/mod.rs
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Created on Mon Jul 26 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # System-wide registry
-//!
-//! The registry module provides interfaces for system-wide, global state management
-//!
-
-use {
-    crate::corestore::lock::{QLGuard, QuickLock},
-    core::sync::atomic::{AtomicBool, Ordering},
-};
-
-const ORD_ACQ: Ordering = Ordering::Acquire;
-const ORD_REL: Ordering = Ordering::Release;
-const ORD_SEQ: Ordering = Ordering::SeqCst;
-
-/// A digital _trip switch_ that can be tripped and untripped in a thread
-/// friendly, consistent manner. It is slightly expensive on processors
-/// with weaker memory ordering (like ARM) when compared to the native
-/// strong ordering provided by some platforms (like x86).
-pub struct Trip {
-    /// the switch
-    inner: AtomicBool,
-}
-
-impl Trip {
-    /// Get an untripped switch
-    pub const fn new_untripped() -> Self {
-        Self {
-            inner: AtomicBool::new(false),
-        }
-    }
-    /// trip the switch
-    pub fn trip(&self) {
-        // we need the strongest consistency here
-        self.inner.store(true, ORD_SEQ)
-    }
-    /// reset the switch
-    pub fn untrip(&self) {
-        // we need the strongest consistency here
-        self.inner.store(false, ORD_SEQ)
-    }
-    /// check if the switch has tripped
-    pub fn is_tripped(&self) -> bool {
-        self.inner.load(ORD_SEQ)
-    }
-    /// Returns the previous state and untrips the switch. **Single op**
-    pub fn check_and_untrip(&self) -> bool {
-        self.inner.swap(false, ORD_SEQ)
-    }
-}
-
-/// The global system health
-static GLOBAL_STATE: AtomicBool = AtomicBool::new(true);
-/// The global flush state
-static FLUSH_STATE: QuickLock<()> = QuickLock::new(());
-/// The preload trip switch
-static PRELOAD_TRIPSWITCH: Trip = Trip::new_untripped();
-static CLEANUP_TRIPSWITCH: Trip = Trip::new_untripped();
-
-/// Check the global system state
-pub fn state_okay() -> bool {
-    GLOBAL_STATE.load(ORD_ACQ)
-}
-
-/// Lock the global flush state. **Remember to drop the lock guard**; else you'll
-/// end up pausing all sorts of global flushing/transactional systems
-pub fn lock_flush_state() -> QLGuard<'static, ()> {
-    FLUSH_STATE.lock()
-}
-
-/// Poison the global system state
-pub fn poison() {
-    GLOBAL_STATE.store(false, ORD_REL)
-}
-
-/// Unpoison the global system state
-pub fn unpoison() {
-    GLOBAL_STATE.store(true, ORD_REL)
-}
-
-/// Get a static reference to the global preload trip switch
-pub fn get_preload_tripswitch() -> &'static Trip {
-    &PRELOAD_TRIPSWITCH
-}
-
-/// Get a static reference to the global cleanup trip switch
-pub fn get_cleanup_tripswitch() -> &'static Trip {
-    &CLEANUP_TRIPSWITCH
-}
diff --git a/server/src/services/bgsave.rs b/server/src/services/bgsave.rs
deleted file mode 100644
index 517b1303..00000000
--- a/server/src/services/bgsave.rs
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Created on Sun May 16 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{
-        config::BGSave,
-        corestore::Corestore,
-        registry,
-        storage::{self, v1::flush::Autoflush},
-        IoResult,
-    },
-    tokio::{
-        sync::broadcast::Receiver,
-        time::{self, Duration},
-    },
-};
-
-/// The bgsave_scheduler calls the bgsave task in `Corestore` after `every` seconds
-///
-/// The time after which the scheduler will wake up the BGSAVE task is determined by
-/// `bgsave_cfg` which is to be passed as an argument. If BGSAVE is disabled, this function
-/// immediately returns
-pub async fn bgsave_scheduler(handle: Corestore, bgsave_cfg: BGSave, mut terminator: Receiver<()>) {
-    match bgsave_cfg {
-        BGSave::Enabled(duration) => {
-            // If we're here - the user doesn't trust his power supply or just values
-            // his data - which is good! So we'll turn this into a `Duration`
-            let duration = Duration::from_secs(duration);
-            loop {
-                tokio::select! {
-                    // Sleep until `duration` from the current time instant
-                    _ = time::sleep_until(time::Instant::now() + duration) => {
-                        let cloned_handle = handle.clone();
-                        // we spawn this process just to ensure that it doesn't block the runtime's workers
-                        // dedicated to async tasks (non-blocking)
-                        tokio::task::spawn_blocking(move || {
-                            let owned_handle = cloned_handle;
-                            let _ = bgsave_blocking_section(owned_handle);
-                        }).await.expect("Something caused the background service to panic");
-                    }
-                    // Otherwise wait for a notification
-                    _ = terminator.recv() => {
-                        // we got a notification to quit; so break out
-                        break;
-                    }
-                }
-            }
-        }
-        BGSave::Disabled => {
-            // the user doesn't bother about his data; cool, let's not bother about it either
-        }
-    }
-    log::info!("BGSAVE service has exited");
-}
-
-/// Run bgsave
-///
-/// This function just hides away the BGSAVE blocking section from the _public API_
-pub fn run_bgsave(handle: &Corestore) -> IoResult<()> {
-    storage::v1::flush::flush_full(Autoflush, handle.get_store())
-}
-
-/// This just wraps around [`_bgsave_blocking_section`] and prints nice log messages depending on the outcome
-fn bgsave_blocking_section(handle: Corestore) -> bool {
-    registry::lock_flush_state();
-    match run_bgsave(&handle) {
-        Ok(_) => {
-            log::info!("BGSAVE completed successfully");
-            registry::unpoison();
-            true
-        }
-        Err(e) => {
-            log::error!("BGSAVE failed with error: {}", e);
-            registry::poison();
-            false
-        }
-    }
-}
diff --git a/server/src/services/mod.rs b/server/src/services/mod.rs
deleted file mode 100644
index b294b186..00000000
--- a/server/src/services/mod.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Created on Sun May 16 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-pub mod bgsave;
-pub mod snapshot;
-use crate::{
-    corestore::memstore::Memstore, diskstore::flock::FileLock, storage, util::os, IoResult,
-};
-
-pub fn restore_data(src: Option<String>) -> IoResult<()> {
-    if let Some(src) = src {
-        // hmm, so restore it
-        os::recursive_copy(src, "data")?;
-        log::info!("Successfully restored data from snapshot");
-    }
-    Ok(())
-}
-
-pub fn pre_shutdown_cleanup(mut pid_file: FileLock, mr: Option<&Memstore>) -> bool {
-    if let Err(e) = pid_file.unlock() {
-        log::error!("Shutdown failure: Failed to unlock pid file: {}", e);
-        return false;
-    }
-    if let Some(mr) = mr {
-        log::info!("Compacting tree");
-        if let Err(e) = storage::v1::interface::cleanup_tree(mr) {
-            log::error!("Failed to compact tree: {}", e);
-            return false;
-        }
-    }
-    true
-}
diff --git a/server/src/services/snapshot.rs b/server/src/services/snapshot.rs
deleted file mode 100644
index 866f9776..00000000
--- a/server/src/services/snapshot.rs
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Created on Sun May 16 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{
-        config::SnapshotConfig,
-        corestore::Corestore,
-        registry,
-        storage::v1::sengine::{SnapshotActionResult, SnapshotEngine},
-    },
-    std::sync::Arc,
-    tokio::{
-        sync::broadcast::Receiver,
-        time::{self, Duration},
-    },
-};
-
-/// The snapshot service
-///
-/// This service calls `SnapEngine::mksnap()` periodically to create snapshots. Whenever
-/// the interval for snapshotting expires or elapses, we create a snapshot. The snapshot service
-/// keeps creating snapshots, as long as the database keeps running. Once [`dbnet::run`] broadcasts
-/// a termination signal, we're ready to quit. This function will, by default, poison the database
-/// if snapshotting fails, unless customized by the user.
-pub async fn snapshot_service(
-    engine: Arc<SnapshotEngine>,
-    handle: Corestore,
-    ss_config: SnapshotConfig,
-    mut termination_signal: Receiver<()>,
-) {
-    match ss_config {
-        SnapshotConfig::Disabled => {
-            // since snapshotting is disabled, we'll imediately return
-            return;
-        }
-        SnapshotConfig::Enabled(configuration) => {
-            let (duration, _, failsafe) = configuration.decompose();
-            let duration = Duration::from_secs(duration);
-            loop {
-                tokio::select! {
-                    _ = time::sleep_until(time::Instant::now() + duration) => {
-                        let succeeded = engine.mksnap(handle.clone_store()).await == SnapshotActionResult::Ok;
-                        #[cfg(test)]
-                        {
-                            use std::env::set_var;
-                            if succeeded {
-                                set_var("SKYTEST_SNAPSHOT_OKAY", "true");
-                            } else {
-                                set_var("SKYTEST_SNAPSHOT_OKAY", "false");
-                            }
-                        }
-                        if succeeded {
-                            // it passed, so unpoison the handle
-                            registry::unpoison();
-                        } else if failsafe {
-                            // mksnap returned false and we are set to stop writes if snapshotting failed
-                            // so let's poison the handle
-                            registry::poison();
-                        }
-                    },
-                    _ = termination_signal.recv() => {
-                        // time to terminate; goodbye!
-                        break;
-                    }
-                }
-            }
-        }
-    }
-    log::info!("Snapshot service has exited");
-}
diff --git a/server/src/storage/mod.rs b/server/src/storage/mod.rs
deleted file mode 100644
index eac66bce..00000000
--- a/server/src/storage/mod.rs
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Created on Sat Mar 05 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*!
-# Storage Engine
-
-The main code in here lies inside `v1`. The reason we've chose to do so is for backwards compatibility.
-Unlike other projects that can _just break_, well, we can't. A database has no right to break data no
-matter what the reason. You can't just mess up someone's data because you found a more efficient
-way to store things. That's why we'll version modules that correspond to version of Cyanstore. It is
-totally legal for one version to call data that correspond to other versions.
-
-## How to break
-
-Whenever we're making changes, here's what we need to keep in mind:
-1. If the format has only changed, but not the corestore structures, then simply gate a v2 and change
-the functions here
-2. If the format has changed and so have the corestore structures, then:
-    1. Move out all the _old_ corestore structures into that version gate
-    2. Then create the new structures in corestore, as appropriate
-    3. The methods here should "identify" a version (usually by bytemarks on the `PRELOAD` which
-    is here to stay)
-    4. Now, the appropriate (if any) version's decoder is called, then the old structures are restored.
-    Now, create the new structures using the old ones and then finally return them
-
-Here's some rust-flavored pseudocode:
-```
-let version = find_version(preload_file_contents)?;
-match version {
-    V1 => {
-        migration::migrate(v1::read_full()?)
-    }
-    V2 => {
-        v2::read_full()
-    }
-    _ => error!("Unknown version"),
-}
-```
-
-The migration module, which doesn't exist, yet will always have a way to transform older structures into
-the current one. This can be achieved with some trait/generic hackery (although it might be pretty simple
-in practice).
-*/
-
-pub mod v1;
-
-pub mod unflush {
-    use crate::{corestore::memstore::Memstore, storage::v1::error::StorageEngineResult};
-    pub fn read_full() -> StorageEngineResult<Memstore> {
-        super::v1::unflush::read_full()
-    }
-}
diff --git a/server/src/storage/v1/bytemarks.rs b/server/src/storage/v1/bytemarks.rs
deleted file mode 100644
index a8ac489d..00000000
--- a/server/src/storage/v1/bytemarks.rs
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Created on Sun Jul 18 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#![allow(unused)]
-
-//! # Bytemarks
-//!
-//! Bytemarks are single bytes that are written to parts of files to provide metadata. This module
-//! contains a collection of these.
-//!
-//! ## Userspace and system bytemarks
-//!
-//! Although ks/system and ks/default might _reside_ next to each other, their bytemarks are entirely
-//! different!
-
-// model
-/*
- * KVEBlob:
- * (1) Pure KVEBlob: [0, 3]
- * (2) KVExt/Listmap: [4, 7]
-*/
-/// KVEBlob model bytemark with key:bin, val:bin
-pub const BYTEMARK_MODEL_KV_BIN_BIN: u8 = 0;
-/// KVEBlob model bytemark with key:bin, val:str
-pub const BYTEMARK_MODEL_KV_BIN_STR: u8 = 1;
-/// KVEBlob model bytemark with key:str, val:str
-pub const BYTEMARK_MODEL_KV_STR_STR: u8 = 2;
-/// KVEBlob model bytemark with key:str, val:bin
-pub const BYTEMARK_MODEL_KV_STR_BIN: u8 = 3;
-/// KVEBlob model bytemark with key:binstr, val: list<binstr>
-pub const BYTEMARK_MODEL_KV_BINSTR_LIST_BINSTR: u8 = 4;
-/// KVEBlob model bytemark with key:binstr, val: list<str>
-pub const BYTEMARK_MODEL_KV_BINSTR_LIST_STR: u8 = 5;
-/// KVEBlob model bytemark with key:str, val: list<binstr>
-pub const BYTEMARK_MODEL_KV_STR_LIST_BINSTR: u8 = 6;
-/// KVEBlob model bytemark with key:str, val: list<str>
-pub const BYTEMARK_MODEL_KV_STR_LIST_STR: u8 = 7;
-
-// storage bym
-/// Persistent storage bytemark
-pub const BYTEMARK_STORAGE_PERSISTENT: u8 = 0;
-/// Volatile storage bytemark
-pub const BYTEMARK_STORAGE_VOLATILE: u8 = 1;
-
-// system bym
-pub const SYSTEM_TABLE_AUTH: u8 = 0;
diff --git a/server/src/storage/v1/error.rs b/server/src/storage/v1/error.rs
deleted file mode 100644
index b7b05c4b..00000000
--- a/server/src/storage/v1/error.rs
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Created on Sat Mar 26 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {crate::corestore::memstore::ObjectID, core::fmt, std::io::Error as IoError};
-
-pub type StorageEngineResult<T> = Result<T, StorageEngineError>;
-
-pub trait ErrorContext<T> {
-    /// Provide some context to an error
-    fn map_err_context(self, extra: impl ToString) -> StorageEngineResult<T>;
-}
-
-impl<T> ErrorContext<T> for Result<T, IoError> {
-    fn map_err_context(self, extra: impl ToString) -> StorageEngineResult<T> {
-        self.map_err(|e| StorageEngineError::ioerror_extra(e, extra.to_string()))
-    }
-}
-
-#[derive(Debug)]
-pub enum StorageEngineError {
-    /// An I/O Error
-    IoError(IoError),
-    /// An I/O Error with extra context
-    IoErrorExtra(IoError, String),
-    /// A corrupted file
-    CorruptedFile(String),
-    /// The file contains bad metadata
-    BadMetadata(String),
-}
-
-impl StorageEngineError {
-    pub fn corrupted_partmap(ksid: &ObjectID) -> Self {
-        Self::CorruptedFile(format!("{ksid}/PARTMAP", ksid = unsafe { ksid.as_str() }))
-    }
-    pub fn bad_metadata_in_table(ksid: &ObjectID, table: &ObjectID) -> Self {
-        unsafe {
-            Self::CorruptedFile(format!(
-                "{ksid}/{table}",
-                ksid = ksid.as_str(),
-                table = table.as_str()
-            ))
-        }
-    }
-    pub fn corrupted_preload() -> Self {
-        Self::CorruptedFile("PRELOAD".into())
-    }
-    pub fn ioerror_extra(ioe: IoError, extra: impl ToString) -> Self {
-        Self::IoErrorExtra(ioe, extra.to_string())
-    }
-}
-
-impl From<IoError> for StorageEngineError {
-    fn from(ioe: IoError) -> Self {
-        Self::IoError(ioe)
-    }
-}
-
-impl fmt::Display for StorageEngineError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::IoError(ioe) => write!(f, "I/O error: {}", ioe),
-            Self::IoErrorExtra(ioe, extra) => write!(f, "I/O error while {extra}: {ioe}"),
-            Self::CorruptedFile(cfile) => write!(f, "file `{cfile}` is corrupted"),
-            Self::BadMetadata(file) => write!(f, "bad metadata in file `{file}`"),
-        }
-    }
-}
diff --git a/server/src/storage/v1/flush.rs b/server/src/storage/v1/flush.rs
deleted file mode 100644
index f11e690c..00000000
--- a/server/src/storage/v1/flush.rs
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Created on Sat Jul 17 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # Flush routines
-//!
-//! This module contains multiple flush routines: at the memstore level, the keyspace level and
-//! the table level
-
-use {
-    super::{bytemarks, interface},
-    crate::{
-        corestore::{
-            map::iter::BorrowedIter,
-            memstore::SYSTEM,
-            memstore::{Keyspace, Memstore, ObjectID, SystemKeyspace},
-            table::{DataModel, SystemDataModel, SystemTable, Table},
-        },
-        registry,
-        util::Wrapper,
-        IoResult,
-    },
-    core::ops::Deref,
-    std::{io::Write, sync::Arc},
-};
-
-pub trait StorageTarget {
-    /// This storage target needs a reinit of the tree despite no preload trip.
-    /// Exempli gratia: rsnap, snap
-    const NEEDS_TREE_INIT: bool;
-    /// This storage target should untrip the trip switch
-    ///
-    /// Example cases where this doesn't apply: snapshots
-    const SHOULD_UNTRIP_PRELOAD_TRIPSWITCH: bool;
-    /// The root for this storage target. **Must not be separator terminated!**
-    fn root(&self) -> String;
-    /// Returns the path to the `PRELOAD_` **temporary file** ($ROOT/PRELOAD)
-    fn preload_target(&self) -> String {
-        let mut p = self.root();
-        p.push('/');
-        p.push_str("PRELOAD_");
-        p
-    }
-    /// Returns the path to the keyspace folder. ($ROOT/{keyspace})
-    fn keyspace_target(&self, keyspace: &str) -> String {
-        let mut p = self.root();
-        p.push('/');
-        p.push_str(keyspace);
-        p
-    }
-    /// Returns the path to a `PARTMAP_` for the given keyspace. **temporary file**
-    /// ($ROOT/{keyspace}/PARTMAP)
-    fn partmap_target(&self, keyspace: &str) -> String {
-        let mut p = self.keyspace_target(keyspace);
-        p.push('/');
-        p.push_str("PARTMAP_");
-        p
-    }
-    /// Returns the path to the table file. **temporary file** ($ROOT/{keyspace}/{table}_)
-    fn table_target(&self, keyspace: &str, table: &str) -> String {
-        let mut p = self.keyspace_target(keyspace);
-        p.push('/');
-        p.push_str(table);
-        p.push('_');
-        p
-    }
-}
-
-/// The autoflush target (BGSAVE target)
-pub struct Autoflush;
-
-impl StorageTarget for Autoflush {
-    const NEEDS_TREE_INIT: bool = false;
-    const SHOULD_UNTRIP_PRELOAD_TRIPSWITCH: bool = true;
-    fn root(&self) -> String {
-        String::from(interface::DIR_KSROOT)
-    }
-}
-
-/// A remote snapshot storage target
-pub struct RemoteSnapshot<'a> {
-    name: &'a str,
-}
-
-impl<'a> RemoteSnapshot<'a> {
-    pub fn new(name: &'a str) -> Self {
-        Self { name }
-    }
-}
-
-impl<'a> StorageTarget for RemoteSnapshot<'a> {
-    const NEEDS_TREE_INIT: bool = true;
-    const SHOULD_UNTRIP_PRELOAD_TRIPSWITCH: bool = false;
-    fn root(&self) -> String {
-        let mut p = String::from(interface::DIR_RSNAPROOT);
-        p.push('/');
-        p.push_str(self.name);
-        p
-    }
-}
-
-/// A snapshot storage target
-pub struct LocalSnapshot {
-    name: String,
-}
-
-impl LocalSnapshot {
-    pub fn new(name: String) -> Self {
-        Self { name }
-    }
-}
-
-impl StorageTarget for LocalSnapshot {
-    const NEEDS_TREE_INIT: bool = true;
-    const SHOULD_UNTRIP_PRELOAD_TRIPSWITCH: bool = false;
-    fn root(&self) -> String {
-        let mut p = String::from(interface::DIR_SNAPROOT);
-        p.push('/');
-        p.push_str(&self.name);
-        p
-    }
-}
-
-/// A keyspace that can be flushed
-pub trait FlushableKeyspace<T: FlushableTable, U: Deref<Target = T>> {
-    /// The number of tables in this keyspace
-    fn table_count(&self) -> usize;
-    /// An iterator to the tables in this keyspace.
-    /// All of them implement [`FlushableTable`]
-    fn get_iter(&self) -> BorrowedIter<'_, ObjectID, U>;
-}
-
-impl FlushableKeyspace<Table, Arc<Table>> for Keyspace {
-    fn table_count(&self) -> usize {
-        self.tables.len()
-    }
-    fn get_iter(&self) -> BorrowedIter<'_, ObjectID, Arc<Table>> {
-        self.tables.iter()
-    }
-}
-
-impl FlushableKeyspace<SystemTable, Wrapper<SystemTable>> for SystemKeyspace {
-    fn table_count(&self) -> usize {
-        self.tables.len()
-    }
-    fn get_iter(&self) -> BorrowedIter<'_, ObjectID, Wrapper<SystemTable>> {
-        self.tables.iter()
-    }
-}
-
-pub trait FlushableTable {
-    /// Table is volatile
-    fn is_volatile(&self) -> bool;
-    /// Returns the storage code bytemark
-    fn storage_code(&self) -> u8;
-    /// Serializes the table and writes it to the provided buffer
-    fn write_table_to<W: Write>(&self, writer: &mut W) -> IoResult<()>;
-    /// Returns the model code bytemark
-    fn model_code(&self) -> u8;
-}
-
-impl FlushableTable for Table {
-    fn is_volatile(&self) -> bool {
-        self.is_volatile()
-    }
-    fn write_table_to<W: Write>(&self, writer: &mut W) -> IoResult<()> {
-        match self.get_model_ref() {
-            DataModel::KV(ref kve) => super::se::raw_serialize_map(kve.get_inner_ref(), writer),
-            DataModel::KVExtListmap(ref kvl) => {
-                super::se::raw_serialize_list_map(kvl.get_inner_ref(), writer)
-            }
-        }
-    }
-    fn storage_code(&self) -> u8 {
-        self.storage_type()
-    }
-    fn model_code(&self) -> u8 {
-        self.get_model_code()
-    }
-}
-
-impl FlushableTable for SystemTable {
-    fn is_volatile(&self) -> bool {
-        false
-    }
-    fn write_table_to<W: Write>(&self, writer: &mut W) -> IoResult<()> {
-        match self.get_model_ref() {
-            SystemDataModel::Auth(amap) => super::se::raw_serialize_map(amap.as_ref(), writer),
-        }
-    }
-    fn storage_code(&self) -> u8 {
-        0
-    }
-    fn model_code(&self) -> u8 {
-        match self.get_model_ref() {
-            SystemDataModel::Auth(_) => bytemarks::SYSTEM_TABLE_AUTH,
-        }
-    }
-}
-
-/// Flush the entire **preload + keyspaces + their partmaps**
-pub fn flush_full<T: StorageTarget>(target: T, store: &Memstore) -> IoResult<()> {
-    // IMPORTANT: Just untrip and get the status at this exact point in time
-    // don't spread it over two atomic accesses because another thread may have updated
-    // it in-between. Even if it was untripped, we'll get the expected outcome here: false
-    let mut should_create_tree = T::NEEDS_TREE_INIT;
-    if T::SHOULD_UNTRIP_PRELOAD_TRIPSWITCH {
-        // this target shouldn't untrip the tripswitch
-        should_create_tree |= registry::get_preload_tripswitch().check_and_untrip();
-    }
-    if should_create_tree {
-        // re-init the tree as new tables/keyspaces may have been added
-        super::interface::create_tree(&target, store)?;
-        self::oneshot::flush_preload(&target, store)?;
-    }
-    // flush userspace keyspaces
-    for keyspace in store.keyspaces.iter() {
-        self::flush_keyspace_full(&target, keyspace.key(), keyspace.value().as_ref())?;
-    }
-    // flush system tables
-    // HACK(@ohsayan): DO NOT REORDER THIS. THE above loop will flush a PARTMAP and an empty
-    // keyspace once. But this has to be done again! The system keyspace in the above loop is a
-    // dummy one because it is located in a different field. So, we need to flush the actual
-    // tables
-    self::flush_keyspace_full(&target, &SYSTEM, &store.system)?;
-    Ok(())
-}
-
-/// Flushes the entire **keyspace + partmap**
-pub fn flush_keyspace_full<T, U, Tbl, K>(target: &T, ksid: &ObjectID, keyspace: &K) -> IoResult<()>
-where
-    T: StorageTarget,
-    U: Deref<Target = Tbl>,
-    Tbl: FlushableTable,
-    K: FlushableKeyspace<Tbl, U>,
-{
-    self::oneshot::flush_partmap(target, ksid, keyspace)?;
-    self::oneshot::flush_keyspace(target, ksid, keyspace)
-}
-
-pub mod oneshot {
-    //! # Irresponsible flushing
-    //!
-    //! Every function does **exactly what it says** and nothing more. No partition
-    //! files et al are handled
-    //!
-    use super::*;
-    use std::fs::{self, File};
-
-    #[inline(always)]
-    fn cowfile(
-        cowfile_name: &str,
-        with_open: impl FnOnce(&mut File) -> IoResult<()>,
-    ) -> IoResult<()> {
-        let mut f = File::create(cowfile_name)?;
-        with_open(&mut f)?;
-        f.sync_all()?;
-        fs::rename(&cowfile_name, &cowfile_name[..cowfile_name.len() - 1])
-    }
-
-    /// No `partmap` handling. Just flushes the table to the expected location
-    pub fn flush_table<T: StorageTarget, U: FlushableTable>(
-        target: &T,
-        tableid: &ObjectID,
-        ksid: &ObjectID,
-        table: &U,
-    ) -> IoResult<()> {
-        if table.is_volatile() {
-            // no flushing needed
-            Ok(())
-        } else {
-            let path = unsafe { target.table_target(ksid.as_str(), tableid.as_str()) };
-            cowfile(&path, |file| {
-                super::interface::serialize_table_into_slow_buffer(file, table)
-            })
-        }
-    }
-
-    /// Flushes an entire keyspace to the expected location. No `partmap` or `preload` handling
-    pub fn flush_keyspace<T, U, Tbl, K>(target: &T, ksid: &ObjectID, keyspace: &K) -> IoResult<()>
-    where
-        T: StorageTarget,
-        U: Deref<Target = Tbl>,
-        Tbl: FlushableTable,
-        K: FlushableKeyspace<Tbl, U>,
-    {
-        for table in keyspace.get_iter() {
-            self::flush_table(target, table.key(), ksid, table.value().deref())?;
-        }
-        Ok(())
-    }
-
-    /// Flushes a single partmap
-    pub fn flush_partmap<T, U, Tbl, K>(target: &T, ksid: &ObjectID, keyspace: &K) -> IoResult<()>
-    where
-        T: StorageTarget,
-        U: Deref<Target = Tbl>,
-        Tbl: FlushableTable,
-        K: FlushableKeyspace<Tbl, U>,
-    {
-        let path = unsafe { target.partmap_target(ksid.as_str()) };
-        cowfile(&path, |file| {
-            super::interface::serialize_partmap_into_slow_buffer(file, keyspace)
-        })
-    }
-
-    // Flush the `PRELOAD`
-    pub fn flush_preload<T: StorageTarget>(target: &T, store: &Memstore) -> IoResult<()> {
-        let preloadtmp = target.preload_target();
-        cowfile(&preloadtmp, |file| {
-            super::interface::serialize_preload_into_slow_buffer(file, store)
-        })
-    }
-}
diff --git a/server/src/storage/v1/interface.rs b/server/src/storage/v1/interface.rs
deleted file mode 100644
index c3777482..00000000
--- a/server/src/storage/v1/interface.rs
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Created on Sat Jul 10 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! Interfaces with the file system
-
-use std::collections::HashMap;
-
-use {
-    crate::{
-        corestore::memstore::Memstore,
-        registry,
-        storage::v1::flush::{FlushableKeyspace, FlushableTable, StorageTarget},
-        IoResult,
-    },
-    core::ops::Deref,
-    std::{
-        collections::HashSet,
-        fs,
-        io::{BufWriter, Write},
-    },
-};
-
-pub const DIR_KSROOT: &str = "data/ks";
-pub const DIR_SNAPROOT: &str = "data/snaps";
-pub const DIR_RSNAPROOT: &str = "data/rsnap";
-pub const DIR_BACKUPS: &str = "data/backups";
-pub const DIR_ROOT: &str = "data";
-
-/// Creates the directories for the keyspaces
-pub fn create_tree<T: StorageTarget>(target: &T, memroot: &Memstore) -> IoResult<()> {
-    for ks in memroot.keyspaces.iter() {
-        unsafe {
-            try_dir_ignore_existing!(target.keyspace_target(ks.key().as_str()))?;
-        }
-    }
-    Ok(())
-}
-
-/// This creates the root directory structure:
-/// ```
-/// data/
-///     ks/
-///         ks1/
-///         ks2/
-///         ks3/
-///     snaps/
-///     backups/
-/// ```
-///
-/// If any directories exist, they are simply ignored
-pub fn create_tree_fresh<T: StorageTarget>(target: &T, memroot: &Memstore) -> IoResult<()> {
-    try_dir_ignore_existing!(
-        DIR_ROOT,
-        DIR_KSROOT,
-        DIR_BACKUPS,
-        DIR_SNAPROOT,
-        DIR_RSNAPROOT
-    );
-    self::create_tree(target, memroot)
-}
-
-/// Clean up the tree
-///
-/// **Warning**: Calling this is quite inefficient so consider calling it once or twice
-/// throughout the lifecycle of the server
-pub fn cleanup_tree(memroot: &Memstore) -> IoResult<()> {
-    if registry::get_cleanup_tripswitch().is_tripped() {
-        log::info!("We're cleaning up ...");
-        // only run a cleanup if someone tripped the switch
-        // hashset because the fs itself will not allow duplicate entries
-        // the keyspaces directory will contain the PRELOAD file, but we'll just
-        // remove it from the list
-        let mut dir_keyspaces: HashSet<String> = read_dir_to_col!(DIR_KSROOT);
-        dir_keyspaces.remove("PRELOAD");
-        let our_keyspaces: HashMap<String, HashSet<String>> = memroot
-            .keyspaces
-            .iter()
-            .map(|kv| {
-                let ksid = unsafe { kv.key().as_str() }.to_owned();
-                let tables: HashSet<String> = kv
-                    .value()
-                    .tables
-                    .iter()
-                    .map(|tbl| unsafe { tbl.key().as_str() }.to_owned())
-                    .collect();
-                (ksid, tables)
-            })
-            .collect();
-
-        // these are the folders that we need to remove; plonk the deleted keyspaces first
-        let keyspaces_to_remove: Vec<&String> = dir_keyspaces
-            .iter()
-            .filter(|ksname| !our_keyspaces.contains_key(ksname.as_str()))
-            .collect();
-        for folder in keyspaces_to_remove {
-            let ks_path = concat_str!(DIR_KSROOT, "/", folder);
-            fs::remove_dir_all(ks_path)?;
-        }
-
-        // HACK(@ohsayan): Due to the nature of how system tables are stored in v1, we need to get rid of this
-        // ensuring that system tables don't end up being removed (since no system tables are actually
-        // purged at this time)
-        let mut our_keyspaces = our_keyspaces;
-        our_keyspaces.remove("system").unwrap();
-        let our_keyspaces = our_keyspaces;
-
-        // now remove the dropped tables
-        for (keyspace, tables) in our_keyspaces {
-            let ks_path = concat_str!(DIR_KSROOT, "/", keyspace.as_str());
-            // read what is present in the tables directory
-            let mut dir_tbls: HashSet<String> = read_dir_to_col!(&ks_path);
-            // in the list of directories we collected, remove PARTMAP because we should NOT
-            // delete it
-            dir_tbls.remove("PARTMAP");
-            // find what tables we should remove
-            let tables_to_remove = dir_tbls.difference(&tables);
-            for removed_table in tables_to_remove {
-                let fpath = concat_path!(&ks_path, removed_table);
-                fs::remove_file(&fpath)?;
-            }
-        }
-    }
-    Ok(())
-}
-
-/// Uses a buffered writer under the hood to improve write performance as the provided
-/// writable interface might be very slow. The buffer does flush once done, however, it
-/// is important that you fsync yourself!
-pub fn serialize_table_into_slow_buffer<T: Write, U: FlushableTable>(
-    buffer: &mut T,
-    writable_item: &U,
-) -> IoResult<()> {
-    let mut buffer = BufWriter::new(buffer);
-    writable_item.write_table_to(&mut buffer)?;
-    buffer.flush()?;
-    Ok(())
-}
-
-pub fn serialize_partmap_into_slow_buffer<T, U, Tbl, K>(buffer: &mut T, ks: &K) -> IoResult<()>
-where
-    T: Write,
-    U: Deref<Target = Tbl>,
-    Tbl: FlushableTable,
-    K: FlushableKeyspace<Tbl, U>,
-{
-    let mut buffer = BufWriter::new(buffer);
-    super::se::raw_serialize_partmap(&mut buffer, ks)?;
-    buffer.flush()?;
-    Ok(())
-}
-
-pub fn serialize_preload_into_slow_buffer<T: Write>(
-    buffer: &mut T,
-    store: &Memstore,
-) -> IoResult<()> {
-    let mut buffer = BufWriter::new(buffer);
-    super::preload::raw_generate_preload(&mut buffer, store)?;
-    buffer.flush()?;
-    Ok(())
-}
diff --git a/server/src/storage/v1/iter.rs b/server/src/storage/v1/iter.rs
deleted file mode 100644
index e8235ce8..00000000
--- a/server/src/storage/v1/iter.rs
+++ /dev/null
@@ -1,237 +0,0 @@
-/*
- * Created on Tue Aug 31 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
- UNSAFE(@ohsayan): Everything done here is unsafely safe. We
- reinterpret bits of one type as another. What could be worse?
- nah, it's not that bad. We know that the byte representations
- would be in the way we expect. If the data is corrupted, we
- can guarantee that we won't ever read incorrect lengths of data
- and we won't read into others' memory (or corrupt our own).
-*/
-
-use {
-    crate::storage::v1::SharedSlice,
-    core::{mem, ptr, slice},
-};
-
-const SIZE_64BIT: usize = mem::size_of::<u64>();
-const SIZE_128BIT: usize = SIZE_64BIT * 2;
-
-/// This contains the fn ptr to decode bytes wrt to the host's endian. For example, if you're on an LE machine and
-/// you're reading data from a BE machine, then simply set the endian to big. This only affects the first read and not
-/// subsequent ones (unless you switch between machines of different endian, obviously)
-static mut NATIVE_ENDIAN_READER: unsafe fn(*const u8) -> usize = super::de::transmute_len;
-
-/// Use this to set the current endian to LE.
-///
-/// ## Safety
-/// Make sure this is run from a single thread only! If not, good luck
-pub(super) unsafe fn endian_set_little() {
-    NATIVE_ENDIAN_READER = super::de::transmute_len_le;
-}
-
-/// Use this to set the current endian to BE.
-///
-/// ## Safety
-/// Make sure this is run from a single thread only! If not, good luck
-pub(super) unsafe fn endian_set_big() {
-    NATIVE_ENDIAN_READER = super::de::transmute_len_be;
-}
-
-/// A raw slice iterator by using raw pointers
-#[derive(Debug)]
-pub struct RawSliceIter<'a> {
-    _base: &'a [u8],
-    cursor: *const u8,
-    terminal: *const u8,
-}
-
-impl<'a> RawSliceIter<'a> {
-    /// Create a new slice iterator
-    pub fn new(slice: &'a [u8]) -> Self {
-        Self {
-            cursor: slice.as_ptr(),
-            terminal: unsafe { slice.as_ptr().add(slice.len()) },
-            _base: slice,
-        }
-    }
-    /// Check the number of remaining bytes in the buffer
-    fn remaining(&self) -> usize {
-        unsafe { self.terminal.offset_from(self.cursor) as usize }
-    }
-    /// Increment the cursor by one
-    unsafe fn incr_cursor(&mut self) {
-        self.incr_cursor_by(1)
-    }
-    /// Check if the buffer was exhausted
-    fn exhausted(&self) -> bool {
-        self.cursor > self.terminal
-    }
-    /// Increment the cursor by the provided length
-    unsafe fn incr_cursor_by(&mut self, ahead: usize) {
-        {
-            self.cursor = self.cursor.add(ahead)
-        }
-    }
-    /// Get the next 64-bit integer, casting it to an `usize`, respecting endianness
-    pub fn next_64bit_integer_to_usize(&mut self) -> Option<usize> {
-        if self.remaining() < 8 {
-            // we need 8 bytes to read a 64-bit integer, so nope
-            None
-        } else {
-            unsafe {
-                // sweet, something is left
-                let l = NATIVE_ENDIAN_READER(self.cursor);
-                // now forward the cursor
-                self.incr_cursor_by(SIZE_64BIT);
-                Some(l)
-            }
-        }
-    }
-    /// Get a borrowed slice for the given length. The lifetime is important!
-    pub fn next_borrowed_slice(&mut self, len: usize) -> Option<&'a [u8]> {
-        if self.remaining() < len {
-            None
-        } else {
-            unsafe {
-                let d = slice::from_raw_parts(self.cursor, len);
-                self.incr_cursor_by(len);
-                Some(d)
-            }
-        }
-    }
-    /// Get the next 64-bit usize
-    pub fn next_64bit_integer_pair_to_usize(&mut self) -> Option<(usize, usize)> {
-        if self.remaining() < SIZE_128BIT {
-            None
-        } else {
-            unsafe {
-                let v1 = NATIVE_ENDIAN_READER(self.cursor);
-                self.incr_cursor_by(SIZE_64BIT);
-                let v2 = NATIVE_ENDIAN_READER(self.cursor);
-                self.incr_cursor_by(SIZE_64BIT);
-                Some((v1, v2))
-            }
-        }
-    }
-    /// Get the next owned [`Data`] with the provided length
-    pub fn next_owned_data(&mut self, len: usize) -> Option<SharedSlice> {
-        if self.remaining() < len {
-            // not enough left
-            None
-        } else {
-            // we have something to look at
-            unsafe {
-                let d = slice::from_raw_parts(self.cursor, len);
-                let d = Some(SharedSlice::new(d));
-                self.incr_cursor_by(len);
-                d
-            }
-        }
-    }
-    /// Get the next 8-bit unsigned integer
-    pub fn next_8bit_integer(&mut self) -> Option<u8> {
-        if self.exhausted() {
-            None
-        } else {
-            unsafe {
-                let x = ptr::read(self.cursor);
-                self.incr_cursor();
-                Some(x)
-            }
-        }
-    }
-    /// Check if the cursor has reached end-of-allocation
-    pub fn end_of_allocation(&self) -> bool {
-        self.cursor == self.terminal
-    }
-    /// Get a borrowed iterator. This is super safe, funny enough, because of the lifetime
-    /// bound that we add to the iterator object
-    pub fn get_borrowed_iter(&mut self) -> RawSliceIterBorrowed<'_> {
-        RawSliceIterBorrowed::new(self.cursor, self.terminal, &mut self.cursor)
-    }
-}
-
-#[derive(Debug)]
-pub struct RawSliceIterBorrowed<'a> {
-    cursor: *const u8,
-    end_ptr: *const u8,
-    mut_ptr: &'a mut *const u8,
-}
-
-impl<'a> RawSliceIterBorrowed<'a> {
-    fn new(
-        cursor: *const u8,
-        end_ptr: *const u8,
-        mut_ptr: &'a mut *const u8,
-    ) -> RawSliceIterBorrowed<'a> {
-        Self {
-            cursor,
-            end_ptr,
-            mut_ptr,
-        }
-    }
-    /// Check the number of remaining bytes in the buffer
-    fn remaining(&self) -> usize {
-        unsafe { self.end_ptr.offset_from(self.cursor) as usize }
-    }
-    /// Increment the cursor by the provided length
-    unsafe fn incr_cursor_by(&mut self, ahead: usize) {
-        {
-            self.cursor = self.cursor.add(ahead)
-        }
-    }
-    pub fn next_64bit_integer_to_usize(&mut self) -> Option<usize> {
-        if self.remaining() < 8 {
-            None
-        } else {
-            unsafe {
-                let size = NATIVE_ENDIAN_READER(self.cursor);
-                self.incr_cursor_by(SIZE_64BIT);
-                Some(size)
-            }
-        }
-    }
-    pub fn next_owned_data(&mut self, len: usize) -> Option<SharedSlice> {
-        if self.remaining() < len {
-            None
-        } else {
-            unsafe {
-                let d = slice::from_raw_parts(self.cursor, len);
-                let d = Some(SharedSlice::new(d));
-                self.incr_cursor_by(len);
-                d
-            }
-        }
-    }
-}
-
-impl<'a> Drop for RawSliceIterBorrowed<'a> {
-    fn drop(&mut self) {
-        *self.mut_ptr = self.cursor;
-    }
-}
diff --git a/server/src/storage/v1/macros.rs b/server/src/storage/v1/macros.rs
deleted file mode 100644
index 4a4acd57..00000000
--- a/server/src/storage/v1/macros.rs
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Created on Sat Jul 17 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-macro_rules! little_endian {
-    ($block:block) => {
-        #[cfg(target_endian = "little")]
-        {
-            $block
-        }
-    };
-}
-
-macro_rules! big_endian {
-    ($block:block) => {
-        #[cfg(target_endian = "big")]
-        {
-            $block
-        }
-    };
-}
-
-macro_rules! not_64_bit {
-    ($block:block) => {
-        #[cfg(not(target_pointer_width = "64"))]
-        {
-            $block
-        }
-    };
-}
-
-macro_rules! is_64_bit {
-    ($block:block) => {
-        #[cfg(target_pointer_width = "64")]
-        {
-            $block
-        }
-    };
-}
-
-macro_rules! to_64bit_native_endian {
-    ($e:expr) => {
-        ($e as u64)
-    };
-}
-
-macro_rules! try_dir_ignore_existing {
-    ($dir:expr) => {{
-        match std::fs::create_dir_all($dir) {
-            Ok(_) => Ok(()),
-            Err(e) => match e.kind() {
-                std::io::ErrorKind::AlreadyExists => Ok(()),
-                _ => Err(e),
-            },
-        }
-    }};
-    ($($dir:expr),*) => {
-        $(try_dir_ignore_existing!($dir)?;)*
-    }
-}
-
-#[macro_export]
-macro_rules! concat_path {
-    ($($s:expr),+) => {{ {
-        let mut path = std::path::PathBuf::with_capacity($(($s).len()+)*0);
-        $(path.push($s);)*
-        path
-    }}};
-}
-
-#[macro_export]
-macro_rules! concat_str {
-    ($($s:expr),+) => {{ {
-        let mut st = std::string::String::with_capacity($(($s).len()+)*0);
-        $(st.push_str($s);)*
-        st
-    }}};
-}
-
-macro_rules! read_dir_to_col {
-    ($root:expr) => {
-        std::fs::read_dir($root)?
-            .map(|v| {
-                v.expect("Unexpected directory parse failure")
-                    .file_name()
-                    .to_string_lossy()
-                    .to_string()
-            })
-            .collect()
-    };
-}
-
-#[cfg(test)]
-macro_rules! lvec {
-    ($($item:expr),+ $(,)?) => {{
-        let v = std::vec![$($item.into()),*];
-        parking_lot::RwLock::new(v)
-    }};
-}
diff --git a/server/src/storage/v1/mod.rs b/server/src/storage/v1/mod.rs
deleted file mode 100644
index ae0b62fd..00000000
--- a/server/src/storage/v1/mod.rs
+++ /dev/null
@@ -1,660 +0,0 @@
-/*
- * Created on Wed Jul 07 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
- Encoding and decoding tested on 32-bit/64-bit Little Endian (Intel x86) and Big Endian
- (MIPS). Also tested UB with miri and memory leaks with valgrind
- -- Sayan on July 8, 2021
-*/
-
-/*! # Storage engine (v1/Cyanstore 1A)
-
-This module contains code to rapidly encode/decode data. All sizes are encoded into unsigned
-64-bit integers for compatibility across 16/32/64 bit platforms. This means that a
-data file generated on a 32-bit machine will work seamlessly on a 64-bit machine
-and vice versa. Of course, provided that On a 32-bit system, 32 high bits are just zeroed.
-
-## Endianness
-
-All sizes are stored in native endian. If a dataset is imported from a system from a different endian, it is
-simply translated into the host's native endian. How everything else is stored is not worth
-discussing here. Byte swaps just need one instruction on most architectures
-
-## Safety
-
-> Trust me, all methods are bombingly unsafe. They do such crazy things that you might not
-think of using them anywhere outside. This is a specialized parser built for the database.
--- Sayan (July 2021)
-
-*/
-
-use {
-    crate::corestore::{array::Array, htable::Coremap, SharedSlice},
-    core::{hash::Hash, mem, slice},
-    std::{collections::HashSet, io::Write},
-};
-
-// for some astronomical reasons do not mess with this
-#[macro_use]
-mod macros;
-// endof do not mess
-pub mod bytemarks;
-pub mod error;
-pub mod flush;
-pub mod interface;
-pub mod iter;
-pub mod preload;
-pub mod sengine;
-pub mod unflush;
-// test
-#[cfg(test)]
-mod tests;
-
-/*
-    Endian and pointer "appendix":
-    We assume a fixed size of 1 for all the cases. All sizes don't hit over isize::MAX as
-    guaranteed by our allocation methods. Also, 32-bit to 64-bit and vice versa aren't
-    worth discussing here (irrespective of endianness). That's because all sizes are stored
-    as unsigned 64-bit integers. So, get the correct byte order, raw cast to u64, down cast
-    or up cast as required by the target's pointer width.
-
-    Current limitations:
-    - 32-bit is compatible with 64-bit
-    - 16-bit is compatible with 64-bit
-    - But 64-bit may not be compatible with 32/16 bit due to the difference in sizes
-
-    ------------------------------------------------------
-    Appendix I: Same endian, different pointer width (R/W)
-    ------------------------------------------------------
-    (1) Little endian on little endian (64-bit)
-    (A) Writing
-    In-memory size: [0, 0, 0, 0, 0, 0, 0, 1] =(u64)> [0, 0, 0, 0, 0, 0, 0, 1] (no op)
-    We write to file: [0, 0, 0, 0, 0, 0, 0, 1]
-    (B) Reading
-    This is read: [0, 0, 0, 0, 0, 0, 0, 1]
-    Raw cast =(usize)> [0, 0, 0, 0, 0, 0, 0, 1] (one memcpy)
-
-    (2) Little endian on little endian (32-bit)
-    (A) Writing
-    In-memory size: [0, 0, 0, 1] =(u64)> [0, 0, 0, 0, 0, 0, 0, 1] (up cast)
-    We write to file: [0, 0, 0, 0, 0, 0, 0, 1]
-    (B) Reading
-    This is read: [0, 0, 0, 0, 0, 0, 0, 1]
-    Raw cast =(u64)> [0, 0, 0, 0, 0, 0, 0, 1] (one memcpy)
-    Lossy cast =(usize)> [0, 0, 0, 1]
-
-    (3) Big endian on big endian (64-bit)
-    (A) Writing
-    In-memory size: [1, 0, 0, 0, 0, 0, 0, 0] =(u64)> [1, 0, 0, 0, 0, 0, 0, 0] (no op)
-    We write to file: [1, 0, 0, 0, 0, 0, 0, 0]
-    (B) Reading
-    This is read: [1, 0, 0, 0, 0, 0, 0, 0]
-    Raw cast =(usize)> [1, 0, 0, 0, 0, 0, 0, 0] (one memcpy)
-
-    (4) Big endian (64-bit) on big endian (32-bit)
-    (A) Writing
-    In-memory size: [1, 0, 0, 0] =(u64)> [1, 0, 0, 0, 0, 0, 0, 0] (up cast)
-    We write to file: [1, 0, 0, 0, 0, 0, 0, 0]
-    (B) Reading
-    This is read: [1, 0, 0, 0, 0, 0, 0, 0]
-    Raw cast =(u64)> [1, 0, 0, 0, 0, 0, 0, 0] (one memcpy)
-    Lossy cast =(usize)> [1, 0, 0, 0]
-
-    ------------------------------------------------------
-    Appendix II: Different endian, same pointer width (R/W)
-    ------------------------------------------------------
-    (1) Little endian on big endian (64-bit)
-    (A) Writing
-    ^^ See Appendix I/1/A
-    (B) Reading
-    This is read: [0, 0, 0, 0, 0, 0, 0, 1]
-    Raw cast =(u64)> [0, 0, 0, 0, 0, 0, 0, 1] (one memcpy)
-    Reverse the bits: [1, 0, 0, 0, 0, 0, 0, 0] (constant time ptr swap)
-    Cast =(usize)> [1, 0, 0, 0, 0, 0, 0, 0] (no op)
-
-    (2) Big endian on little endian (64-bit)
-    (A) Writing
-    ^^ See Appendix I/3/A
-    (B) Reading
-    This is read: [1, 0, 0, 0, 0, 0, 0, 0]
-    Raw cast =(u64)> [1, 0, 0, 0, 0, 0, 0, 0] (one memcpy)
-    Reverse the bits: [0, 0, 0, 0, 0, 0, 0, 1] (constant time ptr swap)
-    Cast =(usize)> [0, 0, 0, 0, 0, 0, 0, 1] (no op)
-
-    (3) Little endian on big endian (32-bit)
-    (A) Writing
-    ^^ See Appendix I/2/A
-    (B) Reading
-    This is read: [0, 0, 0, 0, 0, 0, 0, 1]
-    Raw cast =(u64)> [0, 0, 0, 0, 0, 0, 0, 1] (one memcpy)
-    Reverse the bits: [1, 0, 0, 0, 0, 0, 0, 0] (constant time ptr swap)
-    Lossy cast =(usize)> [1, 0, 0, 0]
-
-    (4) Big endian on little endian (32-bit)
-    (A) Writing
-    ^^ See Appendix I/4/A
-    (B) Reading
-    This is read: [1, 0, 0, 0, 0, 0, 0, 0]
-    Raw cast =(u64)> [1, 0, 0, 0, 0, 0, 0, 0] (one memcpy)
-    Reverse the bits: [0, 0, 0, 0, 0, 0, 0, 1] (constant time ptr swap)
-    Lossy cast =(usize)> [0, 0, 0, 1]
-
-    ------------------------------------------------------
-    Appendix III: Warnings
-    ------------------------------------------------------
-    (1) Gotchas on 32-bit big endian
-    (A) While writing
-    Do not swap bytes before up cast
-    (B) While reading
-    Do not down cast before swapping bytes
-*/
-
-/// Get the raw bytes of anything.
-///
-/// DISCLAIMER: THIS FUNCTION CAN DO TERRIBLE THINGS (especially when you think about padding)
-unsafe fn raw_byte_repr<'a, T: 'a>(len: &'a T) -> &'a [u8] {
-    {
-        let ptr: *const u8 = len as *const T as *const u8;
-        slice::from_raw_parts::<'a>(ptr, mem::size_of::<T>())
-    }
-}
-
-mod se {
-    use super::*;
-    use crate::kvengine::LockedVec;
-    use crate::storage::v1::flush::FlushableKeyspace;
-    use crate::storage::v1::flush::FlushableTable;
-    use crate::IoResult;
-    use core::ops::Deref;
-
-    macro_rules! unsafe_sz_byte_repr {
-        ($e:expr) => {
-            raw_byte_repr(&to_64bit_native_endian!($e))
-        };
-    }
-
-    #[cfg(test)]
-    /// Serialize a map into a _writable_ thing
-    pub fn serialize_map(map: &Coremap<SharedSlice, SharedSlice>) -> IoResult<Vec<u8>> {
-        /*
-        [LEN:8B][KLEN:8B|VLEN:8B][K][V][KLEN:8B][VLEN:8B]...
-        */
-        // write the len header first
-        let mut w = Vec::with_capacity(128);
-        self::raw_serialize_map(map, &mut w)?;
-        Ok(w)
-    }
-
-    /// Serialize a map and write it to a provided buffer
-    pub fn raw_serialize_map<W: Write, T: AsRef<[u8]>, U: AsRef<[u8]>>(
-        map: &Coremap<T, U>,
-        w: &mut W,
-    ) -> IoResult<()>
-    where
-        W: Write,
-        T: AsRef<[u8]> + Hash + Eq,
-        U: AsRef<[u8]>,
-    {
-        unsafe {
-            w.write_all(raw_byte_repr(&to_64bit_native_endian!(map.len())))?;
-            // now the keys and values
-            for kv in map.iter() {
-                let (k, v) = (kv.key(), kv.value());
-                let kref = k.as_ref();
-                let vref = v.as_ref();
-                w.write_all(raw_byte_repr(&to_64bit_native_endian!(kref.len())))?;
-                w.write_all(raw_byte_repr(&to_64bit_native_endian!(vref.len())))?;
-                w.write_all(kref)?;
-                w.write_all(vref)?;
-            }
-        }
-        Ok(())
-    }
-
-    /// Serialize a set and write it to a provided buffer
-    pub fn raw_serialize_set<W, K, V>(map: &Coremap<K, V>, w: &mut W) -> IoResult<()>
-    where
-        W: Write,
-        K: Eq + Hash + AsRef<[u8]>,
-    {
-        unsafe {
-            w.write_all(raw_byte_repr(&to_64bit_native_endian!(map.len())))?;
-            // now the keys and values
-            for kv in map.iter() {
-                let key = kv.key().as_ref();
-                w.write_all(raw_byte_repr(&to_64bit_native_endian!(key.len())))?;
-                w.write_all(key)?;
-            }
-        }
-        Ok(())
-    }
-
-    /// Generate a partition map for the given keyspace
-    /// ```text
-    /// [8B: EXTENT]([8B: LEN][?B: PARTITION ID][1B: Storage type][1B: Model type])*
-    /// ```
-    pub fn raw_serialize_partmap<W, U, Tbl, K>(w: &mut W, keyspace: &K) -> IoResult<()>
-    where
-        W: Write,
-        U: Deref<Target = Tbl>,
-        Tbl: FlushableTable,
-        K: FlushableKeyspace<Tbl, U>,
-    {
-        unsafe {
-            // extent
-            w.write_all(raw_byte_repr(&to_64bit_native_endian!(
-                keyspace.table_count()
-            )))?;
-            for table in keyspace.get_iter() {
-                // partition ID len
-                w.write_all(raw_byte_repr(&to_64bit_native_endian!(table.key().len())))?;
-                // parition ID
-                w.write_all(table.key())?;
-                // now storage type
-                w.write_all(raw_byte_repr(&table.storage_code()))?;
-                // now model type
-                w.write_all(raw_byte_repr(&table.model_code()))?;
-            }
-        }
-        Ok(())
-    }
-    pub fn raw_serialize_list_map<W>(
-        data: &Coremap<SharedSlice, LockedVec>,
-        w: &mut W,
-    ) -> IoResult<()>
-    where
-        W: Write,
-    {
-        /*
-        [8B: Extent]([8B: Key extent][?B: Key][8B: Max index][?B: Payload])*
-        */
-        unsafe {
-            // Extent
-            w.write_all(unsafe_sz_byte_repr!(data.len()))?;
-            // Enter iter
-            '_1: for key in data.iter() {
-                // key
-                let k = key.key();
-                // list payload
-                let vread = key.value().read();
-                let v: &Vec<SharedSlice> = &vread;
-                // write the key extent
-                w.write_all(unsafe_sz_byte_repr!(k.len()))?;
-                // write the key
-                w.write_all(k)?;
-                // write the list payload
-                self::raw_serialize_nested_list(w, &v)?;
-            }
-        }
-        Ok(())
-    }
-    /// Serialize a `[[u8]]` (i.e a slice of slices)
-    pub fn raw_serialize_nested_list<'a, W, T: 'a + ?Sized, U: 'a>(
-        w: &mut W,
-        inp: &'a T,
-    ) -> IoResult<()>
-    where
-        T: AsRef<[U]>,
-        U: AsRef<[u8]>,
-        W: Write,
-    {
-        // ([8B:EL_EXTENT][?B: Payload])*
-        let inp = inp.as_ref();
-        unsafe {
-            // write extent
-            w.write_all(unsafe_sz_byte_repr!(inp.len()))?;
-            // now enter loop and write elements
-            for element in inp.iter() {
-                let element = element.as_ref();
-                // write element extent
-                w.write_all(unsafe_sz_byte_repr!(element.len()))?;
-                // write element
-                w.write_all(element)?;
-            }
-        }
-        Ok(())
-    }
-}
-
-mod de {
-    use super::iter::{RawSliceIter, RawSliceIterBorrowed};
-    use super::{Array, Coremap, Hash, HashSet, SharedSlice};
-    use crate::kvengine::LockedVec;
-    use core::ptr;
-    use parking_lot::RwLock;
-    use std::collections::HashMap;
-
-    pub trait DeserializeFrom {
-        fn is_expected_len(clen: usize) -> bool;
-        fn from_slice(slice: &[u8]) -> Self;
-    }
-
-    pub trait DeserializeInto: Sized {
-        fn new_empty() -> Self;
-        fn from_slice(slice: &[u8]) -> Option<Self>;
-    }
-
-    impl DeserializeInto for Coremap<SharedSlice, SharedSlice> {
-        fn new_empty() -> Self {
-            Coremap::new()
-        }
-        fn from_slice(slice: &[u8]) -> Option<Self> {
-            self::deserialize_map(slice)
-        }
-    }
-
-    impl DeserializeInto for Coremap<SharedSlice, LockedVec> {
-        fn new_empty() -> Self {
-            Coremap::new()
-        }
-        fn from_slice(slice: &[u8]) -> Option<Self> {
-            self::deserialize_list_map(slice)
-        }
-    }
-
-    impl<T, U> DeserializeInto for Coremap<T, U>
-    where
-        T: Hash + Eq + DeserializeFrom,
-        U: DeserializeFrom,
-    {
-        fn new_empty() -> Self {
-            Coremap::new()
-        }
-        fn from_slice(slice: &[u8]) -> Option<Self> {
-            self::deserialize_map_ctype(slice)
-        }
-    }
-
-    pub fn deserialize_into<T: DeserializeInto>(input: &[u8]) -> Option<T> {
-        T::from_slice(input)
-    }
-
-    impl<const N: usize> DeserializeFrom for Array<u8, N> {
-        fn is_expected_len(clen: usize) -> bool {
-            clen <= N
-        }
-        fn from_slice(slice: &[u8]) -> Self {
-            unsafe { Self::from_slice(slice) }
-        }
-    }
-
-    impl<const N: usize> DeserializeFrom for [u8; N] {
-        fn is_expected_len(clen: usize) -> bool {
-            clen == N
-        }
-        fn from_slice(slice: &[u8]) -> Self {
-            slice.try_into().unwrap()
-        }
-    }
-
-    pub fn deserialize_map_ctype<T, U>(data: &[u8]) -> Option<Coremap<T, U>>
-    where
-        T: Eq + Hash + DeserializeFrom,
-        U: DeserializeFrom,
-    {
-        let mut rawiter = RawSliceIter::new(data);
-        let len = rawiter.next_64bit_integer_to_usize()?;
-        let map = Coremap::new();
-        for _ in 0..len {
-            let (lenkey, lenval) = rawiter.next_64bit_integer_pair_to_usize()?;
-            if !(T::is_expected_len(lenkey) && U::is_expected_len(lenval)) {
-                return None;
-            }
-            let key = T::from_slice(rawiter.next_borrowed_slice(lenkey)?);
-            let value = U::from_slice(rawiter.next_borrowed_slice(lenval)?);
-            if !map.true_if_insert(key, value) {
-                // duplicates
-                return None;
-            }
-        }
-        Some(map)
-    }
-
-    /// Deserialize a set to a custom type
-    pub fn deserialize_set_ctype<T>(data: &[u8]) -> Option<HashSet<T>>
-    where
-        T: DeserializeFrom + Eq + Hash,
-    {
-        let mut rawiter = RawSliceIter::new(data);
-        let len = rawiter.next_64bit_integer_to_usize()?;
-        let mut set = HashSet::new();
-        set.try_reserve(len).ok()?;
-        for _ in 0..len {
-            let lenkey = rawiter.next_64bit_integer_to_usize()?;
-            if !T::is_expected_len(lenkey) {
-                return None;
-            }
-            // get the key as a raw slice, we've already checked if end_ptr is less
-            let key = T::from_slice(rawiter.next_borrowed_slice(lenkey)?);
-            // push it in
-            if !set.insert(key) {
-                // repeat?; that's not what we wanted
-                return None;
-            }
-        }
-        if rawiter.end_of_allocation() {
-            Some(set)
-        } else {
-            // nope, someone gave us more data
-            None
-        }
-    }
-
-    /// Deserializes a map-like set which has an 2x1B _bytemark_ for every entry
-    pub fn deserialize_set_ctype_bytemark<T>(data: &[u8]) -> Option<HashMap<T, (u8, u8)>>
-    where
-        T: DeserializeFrom + Eq + Hash,
-    {
-        let mut rawiter = RawSliceIter::new(data);
-        // so we have 8B. Just unsafe access and transmute it
-        let len = rawiter.next_64bit_integer_to_usize()?;
-        let mut set = HashMap::new();
-        set.try_reserve(len).ok()?;
-        for _ in 0..len {
-            let lenkey = rawiter.next_64bit_integer_to_usize()?;
-            if !T::is_expected_len(lenkey) {
-                return None;
-            }
-            // get the key as a raw slice, we've already checked if end_ptr is less
-            let key = T::from_slice(rawiter.next_borrowed_slice(lenkey)?);
-            let bytemark_a = rawiter.next_8bit_integer()?;
-            let bytemark_b = rawiter.next_8bit_integer()?;
-            // push it in
-            if set.insert(key, (bytemark_a, bytemark_b)).is_some() {
-                // repeat?; that's not what we wanted
-                return None;
-            }
-        }
-        if rawiter.end_of_allocation() {
-            Some(set)
-        } else {
-            // nope, someone gave us more data
-            None
-        }
-    }
-    /// Deserialize a file that contains a serialized map. This also returns the model code
-    pub fn deserialize_map(data: &[u8]) -> Option<Coremap<SharedSlice, SharedSlice>> {
-        let mut rawiter = RawSliceIter::new(data);
-        let len = rawiter.next_64bit_integer_to_usize()?;
-        let hm = Coremap::try_with_capacity(len).ok()?;
-        for _ in 0..len {
-            let (lenkey, lenval) = rawiter.next_64bit_integer_pair_to_usize()?;
-            let key = rawiter.next_owned_data(lenkey)?;
-            let val = rawiter.next_owned_data(lenval)?;
-            // push it in
-            hm.upsert(key, val);
-        }
-        if rawiter.end_of_allocation() {
-            Some(hm)
-        } else {
-            // nope, someone gave us more data
-            None
-        }
-    }
-
-    pub fn deserialize_list_map(bytes: &[u8]) -> Option<Coremap<SharedSlice, LockedVec>> {
-        let mut rawiter = RawSliceIter::new(bytes);
-        // get the len
-        let len = rawiter.next_64bit_integer_to_usize()?;
-        // allocate a map
-        let map = Coremap::try_with_capacity(len).ok()?;
-        // now enter a loop
-        for _ in 0..len {
-            let keylen = rawiter.next_64bit_integer_to_usize()?;
-            // get key
-            let key = rawiter.next_owned_data(keylen)?;
-            let borrowed_iter = rawiter.get_borrowed_iter();
-            let list = self::deserialize_nested_list(borrowed_iter)?;
-            // push it in
-            map.true_if_insert(key, RwLock::new(list));
-        }
-        if rawiter.end_of_allocation() {
-            Some(map)
-        } else {
-            // someone returned more data
-            None
-        }
-    }
-
-    /// Deserialize a nested list: `[EXTENT]([EL_EXT][EL])*`
-    ///
-    pub fn deserialize_nested_list(mut iter: RawSliceIterBorrowed<'_>) -> Option<Vec<SharedSlice>> {
-        // get list payload len
-        let list_payload_extent = iter.next_64bit_integer_to_usize()?;
-        let mut list = Vec::new();
-        list.try_reserve(list_payload_extent).ok()?;
-        for _ in 0..list_payload_extent {
-            // get element size
-            let list_element_payload_size = iter.next_64bit_integer_to_usize()?;
-            // now get element
-            let element = iter.next_owned_data(list_element_payload_size)?;
-            list.push(element);
-        }
-        Some(list)
-    }
-
-    #[allow(clippy::needless_return)]
-    pub(super) unsafe fn transmute_len(start_ptr: *const u8) -> usize {
-        little_endian! {{
-            return self::transmute_len_le(start_ptr);
-        }};
-        big_endian! {{
-            return self::transmute_len_be(start_ptr);
-        }}
-    }
-
-    #[allow(clippy::needless_return)] // Clippy really misunderstands this
-    pub(super) unsafe fn transmute_len_le(start_ptr: *const u8) -> usize {
-        little_endian!({
-            // So we have an LE target
-            is_64_bit!({
-                // 64-bit LE
-                return ptr::read_unaligned(start_ptr.cast());
-            });
-            not_64_bit!({
-                // 32-bit LE
-                let ret1: u64 = ptr::read_unaligned(start_ptr.cast());
-                // lossy cast
-                let ret = ret1 as usize;
-                if ret > (isize::MAX as usize) {
-                    // this is a backup method for us incase a giant 48-bit address is
-                    // somehow forced to be read on this machine
-                    panic!("RT panic: Very high size for current pointer width");
-                }
-                return ret;
-            });
-        });
-
-        big_endian!({
-            // so we have a BE target
-            is_64_bit!({
-                // 64-bit big endian
-                let ret: usize = ptr::read_unaligned(start_ptr.cast());
-                // swap byte order
-                return ret.swap_bytes();
-            });
-            not_64_bit!({
-                // 32-bit big endian
-                let ret: u64 = ptr::read_unaligned(start_ptr.cast());
-                // swap byte order and lossy cast
-                let ret = (ret.swap_bytes()) as usize;
-                // check if overflow
-                if ret > (isize::MAX as usize) {
-                    // this is a backup method for us incase a giant 48-bit address is
-                    // somehow forced to be read on this machine
-                    panic!("RT panic: Very high size for current pointer width");
-                }
-                return ret;
-            });
-        });
-    }
-
-    #[allow(clippy::needless_return)] // Clippy really misunderstands this
-    pub(super) unsafe fn transmute_len_be(start_ptr: *const u8) -> usize {
-        big_endian!({
-            // So we have a BE target
-            is_64_bit!({
-                // 64-bit BE
-                return ptr::read_unaligned(start_ptr.cast());
-            });
-            not_64_bit!({
-                // 32-bit BE
-                let ret1: u64 = ptr::read_unaligned(start_ptr.cast());
-                // lossy cast
-                let ret = ret1 as usize;
-                if ret > (isize::MAX as usize) {
-                    // this is a backup method for us incase a giant 48-bit address is
-                    // somehow forced to be read on this machine
-                    panic!("RT panic: Very high size for current pointer width");
-                }
-                return ret;
-            });
-        });
-
-        little_endian!({
-            // so we have an LE target
-            is_64_bit!({
-                // 64-bit little endian
-                let ret: usize = ptr::read_unaligned(start_ptr.cast());
-                // swap byte order
-                return ret.swap_bytes();
-            });
-            not_64_bit!({
-                // 32-bit little endian
-                let ret: u64 = ptr::read_unaligned(start_ptr.cast());
-                // swap byte order and lossy cast
-                let ret = (ret.swap_bytes()) as usize;
-                // check if overflow
-                if ret > (isize::MAX as usize) {
-                    // this is a backup method for us incase a giant 48-bit address is
-                    // somehow forced to be read on this machine
-                    panic!("RT panic: Very high size for current pointer width");
-                }
-                return ret;
-            });
-        });
-    }
-}
diff --git a/server/src/storage/v1/preload.rs b/server/src/storage/v1/preload.rs
deleted file mode 100644
index c5c3355a..00000000
--- a/server/src/storage/v1/preload.rs
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Created on Sat Jul 17 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # Preload binary files
-//!
-//! Preloads are very critical binary files which contain metadata for this instance of
-//! the database. Preloads are of two kinds:
-//! 1. the `PRELOAD` that is placed at the root directory
-//! 2. the `PARTMAP` preload that is placed in the ks directory
-//!
-
-use {
-    crate::{
-        corestore::memstore::{Memstore, ObjectID},
-        storage::v1::error::{StorageEngineError, StorageEngineResult},
-        IoResult,
-    },
-    core::ptr,
-    std::{
-        collections::{HashMap, HashSet},
-        io::Write,
-    },
-};
-
-pub type LoadedPartfile = HashMap<ObjectID, (u8, u8)>;
-
-// our version and endian are based on nibbles
-
-const META_SEGMENT_LE: u8 = 0b1000_0000;
-const META_SEGMENT_BE: u8 = 0b1000_0001;
-
-#[cfg(target_endian = "little")]
-const META_SEGMENT: u8 = META_SEGMENT_LE;
-
-#[cfg(target_endian = "big")]
-const META_SEGMENT: u8 = META_SEGMENT_BE;
-
-/// Generate the `PRELOAD` disk file for this instance
-/// ```text
-/// [1B: Endian Mark/Version Mark (padded)] => Meta segment
-/// [8B: Extent header] => Predata Segment
-/// ([8B: Partion ID len][8B: Parition ID (not padded)])* => Data segment
-/// ```
-///
-pub(super) fn raw_generate_preload<W: Write>(w: &mut W, store: &Memstore) -> IoResult<()> {
-    // generate the meta segment
-    w.write_all(&[META_SEGMENT])?;
-    super::se::raw_serialize_set(&store.keyspaces, w)?;
-    Ok(())
-}
-
-/// Reads the preload file and returns a set
-pub(super) fn read_preload_raw(preload: Vec<u8>) -> StorageEngineResult<HashSet<ObjectID>> {
-    if preload.len() < 16 {
-        // nah, this is a bad disk file
-        return Err(StorageEngineError::corrupted_preload());
-    }
-    // first read in the meta segment
-    unsafe {
-        let meta_segment: u8 = ptr::read(preload.as_ptr());
-        match meta_segment {
-            META_SEGMENT_BE => {
-                super::iter::endian_set_big();
-            }
-            META_SEGMENT_LE => {
-                super::iter::endian_set_little();
-            }
-            _ => return Err(StorageEngineError::BadMetadata("preload".into())),
-        }
-    }
-    // all checks complete; time to decode
-    super::de::deserialize_set_ctype(&preload[1..])
-        .ok_or_else(StorageEngineError::corrupted_preload)
-}
diff --git a/server/src/storage/v1/sengine.rs b/server/src/storage/v1/sengine.rs
deleted file mode 100644
index f8610eae..00000000
--- a/server/src/storage/v1/sengine.rs
+++ /dev/null
@@ -1,354 +0,0 @@
-/*
- * Created on Sun Aug 08 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    self::queue::Queue,
-    super::interface::{DIR_RSNAPROOT, DIR_SNAPROOT},
-    crate::{
-        corestore::{iarray::IArray, lazy::Lazy, lock::QuickLock, memstore::Memstore},
-        storage::v1::flush::{LocalSnapshot, RemoteSnapshot},
-    },
-    chrono::prelude::Utc,
-    core::{fmt, str},
-    regex::Regex,
-    std::{collections::HashSet, fs, io::Error as IoError, path::Path, sync::Arc},
-};
-
-type QStore = IArray<[String; 64]>;
-type SnapshotResult<T> = Result<T, SnapshotEngineError>;
-
-/// Matches any string which is in the following format:
-/// ```text
-/// YYYYMMDD-HHMMSS
-/// ```
-pub static SNAP_MATCH: Lazy<Regex, fn() -> Regex> = Lazy::new(|| {
-    Regex::new("^\\d{4}(0[1-9]|1[012])(0[1-9]|[12][0-9]|3[01])(-)(?:(?:([01]?\\d|2[0-3]))?([0-5]?\\d))?([0-5]?\\d)$").unwrap()
-});
-
-#[derive(Debug)]
-pub enum SnapshotEngineError {
-    Io(IoError),
-    Engine(&'static str),
-}
-
-impl From<IoError> for SnapshotEngineError {
-    fn from(e: IoError) -> SnapshotEngineError {
-        SnapshotEngineError::Io(e)
-    }
-}
-
-impl From<&'static str> for SnapshotEngineError {
-    fn from(e: &'static str) -> SnapshotEngineError {
-        SnapshotEngineError::Engine(e)
-    }
-}
-
-impl fmt::Display for SnapshotEngineError {
-    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), fmt::Error> {
-        match self {
-            Self::Engine(estr) => {
-                formatter.write_str("Snapshot engine error")?;
-                formatter.write_str(estr)?;
-            }
-            Self::Io(e) => {
-                formatter.write_str("Snapshot engine IOError:")?;
-                formatter.write_str(&e.to_string())?;
-            }
-        }
-        Ok(())
-    }
-}
-
-/// The snapshot engine
-#[derive(Debug)]
-pub struct SnapshotEngine {
-    local_enabled: bool,
-    /// the local snapshot queue
-    local_queue: QuickLock<Queue>,
-    /// the remote snapshot lock
-    remote_queue: QuickLock<HashSet<Box<[u8]>>>,
-}
-
-#[derive(Debug, PartialEq)]
-pub enum SnapshotActionResult {
-    Ok,
-    Busy,
-    Disabled,
-    Failure,
-    AlreadyExists,
-}
-
-impl SnapshotEngine {
-    /// Returns a fresh, uninitialized snapshot engine instance
-    pub fn new(maxlen: usize) -> Self {
-        Self {
-            local_enabled: true,
-            local_queue: QuickLock::new(Queue::new(maxlen, maxlen == 0)),
-            remote_queue: QuickLock::new(HashSet::new()),
-        }
-    }
-    pub fn new_disabled() -> Self {
-        Self {
-            local_enabled: false,
-            local_queue: QuickLock::new(Queue::new(0, true)),
-            remote_queue: QuickLock::new(HashSet::new()),
-        }
-    }
-    fn _parse_dir(
-        dir: &str,
-        is_okay: impl Fn(&str) -> bool,
-        mut append: impl FnMut(String),
-    ) -> SnapshotResult<()> {
-        let dir = fs::read_dir(dir)?;
-        for entry in dir {
-            let entry = entry?;
-            if entry.file_type()?.is_dir() {
-                let fname = entry.file_name();
-                let name = fname.to_string_lossy();
-                if !is_okay(&name) {
-                    return Err("unknown folder in snapshot directory".into());
-                }
-                append(name.to_string());
-            } else {
-                return Err("unrecognized file in snapshot directory".into());
-            }
-        }
-        Ok(())
-    }
-    pub fn parse_dir(&self) -> SnapshotResult<()> {
-        let mut local_queue = self.local_queue.lock();
-        Self::_parse_dir(
-            DIR_SNAPROOT,
-            |name| SNAP_MATCH.is_match(name),
-            |snapshot| local_queue.push(snapshot),
-        )?;
-        let mut remote_queue = self.remote_queue.lock();
-        Self::_parse_dir(
-            DIR_RSNAPROOT,
-            |_| true,
-            |rsnap| {
-                remote_queue.insert(rsnap.into_boxed_str().into_boxed_bytes());
-            },
-        )?;
-        Ok(())
-    }
-    /// Generate the snapshot name
-    fn get_snapname(&self) -> String {
-        Utc::now().format("%Y%m%d-%H%M%S").to_string()
-    }
-    fn _mksnap_blocking_section(store: &Memstore, name: String) -> SnapshotResult<()> {
-        if Path::new(&format!("{DIR_SNAPROOT}/{name}")).exists() {
-            Err(SnapshotEngineError::Engine("Server time is incorrect"))
-        } else {
-            let snapshot = LocalSnapshot::new(name);
-            super::flush::flush_full(snapshot, store)?;
-            Ok(())
-        }
-    }
-    fn _rmksnap_blocking_section(store: &Memstore, name: &str) -> SnapshotResult<()> {
-        let snapshot = RemoteSnapshot::new(name);
-        super::flush::flush_full(snapshot, store)?;
-        Ok(())
-    }
-    /// Spawns a blocking task on a threadpool for blocking tasks. Returns either of:
-    /// - `0` => Okay (returned **even if old snap deletion failed**)
-    /// - `1` => Error
-    /// - `2` => Disabled
-    /// - `3` => Busy
-    pub async fn mksnap(&self, store: Arc<Memstore>) -> SnapshotActionResult {
-        if self.local_enabled {
-            // try to lock the local queue
-            let mut queue = match self.local_queue.try_lock() {
-                Some(lck) => lck,
-                None => return SnapshotActionResult::Busy,
-            };
-            let name = self.get_snapname();
-            let nameclone = name.clone();
-            let todel = queue.add_new(name);
-            let snap_create_result = tokio::task::spawn_blocking(move || {
-                Self::_mksnap_blocking_section(&store, nameclone)
-            })
-            .await
-            .expect("mksnap thread panicked");
-
-            // First create the new snap
-            match snap_create_result {
-                Ok(_) => {
-                    log::info!("Successfully created snapshot");
-                }
-                Err(e) => {
-                    log::error!("Failed to create snapshot with error: {}", e);
-                    // so it failed, remove it from queue
-                    let _ = queue.pop_last().unwrap();
-                    return SnapshotActionResult::Failure;
-                }
-            }
-
-            // Now delete the older snap (if any)
-            if let Some(snap) = todel {
-                tokio::task::spawn_blocking(move || {
-                    if let Err(e) = fs::remove_dir_all(concat_path!(DIR_SNAPROOT, snap)) {
-                        log::warn!("Failed to remove older snapshot (ignored): {}", e);
-                    } else {
-                        log::info!("Successfully removed older snapshot");
-                    }
-                })
-                .await
-                .expect("mksnap thread panicked");
-            }
-            drop(queue);
-            SnapshotActionResult::Ok
-        } else {
-            SnapshotActionResult::Disabled
-        }
-    }
-    /// Spawns a blocking task to create a remote snapshot. Returns either of:
-    /// - `0` => Okay
-    /// - `1` => Error
-    /// - `3` => Busy
-    /// (consistent with mksnap)
-    pub async fn mkrsnap(&self, name: &[u8], store: Arc<Memstore>) -> SnapshotActionResult {
-        let mut remq = match self.remote_queue.try_lock() {
-            Some(q) => q,
-            None => return SnapshotActionResult::Busy,
-        };
-        if remq.contains(name) {
-            SnapshotActionResult::AlreadyExists
-        } else {
-            let nameclone = name.to_owned();
-            let ret = tokio::task::spawn_blocking(move || {
-                let name_str = unsafe {
-                    // SAFETY: We have already checked if name is UTF-8
-                    str::from_utf8_unchecked(&nameclone)
-                };
-                if let Err(e) = Self::_rmksnap_blocking_section(&store, name_str) {
-                    log::error!("Remote snapshot failed with: {}", e);
-                    SnapshotActionResult::Failure
-                } else {
-                    log::info!("Remote snapshot succeeded");
-                    SnapshotActionResult::Ok
-                }
-            })
-            .await
-            .expect("rmksnap thread panicked");
-            assert!(remq.insert(name.to_owned().into_boxed_slice()));
-            ret
-        }
-    }
-}
-
-mod queue {
-    //! An extremely simple queue implementation which adds more items to the queue
-    //! freely and once the threshold limit is reached, it pops off the oldest element and returns it
-    //!
-    //! This implementation is specifically built for use with the snapshotting utility
-    use super::QStore;
-    use crate::corestore::iarray;
-    #[derive(Debug, PartialEq)]
-    pub struct Queue {
-        queue: QStore,
-        maxlen: usize,
-        dontpop: bool,
-    }
-
-    impl Queue {
-        pub const fn new(maxlen: usize, dontpop: bool) -> Self {
-            Queue {
-                queue: iarray::new_const_iarray(),
-                maxlen,
-                dontpop,
-            }
-        }
-        pub fn push(&mut self, item: String) {
-            self.queue.push(item)
-        }
-        /// This returns a `String` only if the queue is full. Otherwise, a `None` is returned most of the time
-        pub fn add_new(&mut self, item: String) -> Option<String> {
-            if self.dontpop {
-                // We don't need to pop anything since the user
-                // wants to keep all the items in the queue
-                self.queue.push(item);
-                None
-            } else {
-                // The user wants to keep a maximum of `maxtop` items
-                // so we will check if the current queue is full
-                // if it is full, then the `maxtop` limit has been reached
-                // so we will remove the oldest item and then push the
-                // new item onto the queue
-                let x = if self.is_overflow() { self.pop() } else { None };
-                self.queue.push(item);
-                x
-            }
-        }
-        /// Check if we have reached the maximum queue size limit
-        fn is_overflow(&self) -> bool {
-            self.queue.len() == self.maxlen
-        }
-        /// Remove the last item inserted
-        fn pop(&mut self) -> Option<String> {
-            if self.queue.is_empty() {
-                None
-            } else {
-                Some(unsafe {
-                    // SAFETY: We have already checked if the queue is empty or not
-                    self.queue.remove(0)
-                })
-            }
-        }
-        pub fn pop_last(&mut self) -> Option<String> {
-            self.queue.pop()
-        }
-    }
-
-    #[test]
-    fn test_queue() {
-        let mut q = Queue::new(4, false);
-        assert!(q.add_new(String::from("snap1")).is_none());
-        assert!(q.add_new(String::from("snap2")).is_none());
-        assert!(q.add_new(String::from("snap3")).is_none());
-        assert!(q.add_new(String::from("snap4")).is_none());
-        assert_eq!(
-            q.add_new(String::from("snap5")),
-            Some(String::from("snap1"))
-        );
-        assert_eq!(
-            q.add_new(String::from("snap6")),
-            Some(String::from("snap2"))
-        );
-    }
-
-    #[test]
-    fn test_queue_dontpop() {
-        // This means that items can only be added or all of them can be deleted
-        let mut q = Queue::new(4, true);
-        assert!(q.add_new(String::from("snap1")).is_none());
-        assert!(q.add_new(String::from("snap2")).is_none());
-        assert!(q.add_new(String::from("snap3")).is_none());
-        assert!(q.add_new(String::from("snap4")).is_none());
-        assert!(q.add_new(String::from("snap5")).is_none());
-        assert!(q.add_new(String::from("snap6")).is_none());
-    }
-}
diff --git a/server/src/storage/v1/tests.rs b/server/src/storage/v1/tests.rs
deleted file mode 100644
index de891ed9..00000000
--- a/server/src/storage/v1/tests.rs
+++ /dev/null
@@ -1,821 +0,0 @@
-/*
- * Created on Sat Jul 17 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use super::*;
-
-#[test]
-fn test_serialize_deserialize_empty() {
-    let cmap = Coremap::new();
-    let ser = se::serialize_map(&cmap).unwrap();
-    let de = de::deserialize_map(&ser).unwrap();
-    assert!(de.len() == 0);
-}
-
-#[test]
-fn test_serialize_deserialize_map_with_empty_elements() {
-    let cmap = Coremap::new();
-    cmap.true_if_insert(SharedSlice::from("sayan"), SharedSlice::from(""));
-    cmap.true_if_insert(
-        SharedSlice::from("sayan's second key"),
-        SharedSlice::from(""),
-    );
-    cmap.true_if_insert(
-        SharedSlice::from("sayan's third key"),
-        SharedSlice::from(""),
-    );
-    cmap.true_if_insert(SharedSlice::from(""), SharedSlice::from(""));
-    let ser = se::serialize_map(&cmap).unwrap();
-    let de = de::deserialize_map(&ser).unwrap();
-    assert_eq!(de.len(), cmap.len());
-    assert!(cmap.into_iter().all(|(k, v)| de.get(&k).unwrap().eq(&v)));
-}
-
-#[test]
-fn test_ser_de_few_elements() {
-    let cmap = Coremap::new();
-    cmap.upsert("sayan".into(), "writes code".into());
-    cmap.upsert("supersayan".into(), "writes super code".into());
-    let ser = se::serialize_map(&cmap).unwrap();
-    let de = de::deserialize_map(&ser).unwrap();
-    assert!(de.len() == cmap.len());
-    assert!(de
-        .iter()
-        .all(|kv| cmap.get(kv.key()).unwrap().eq(kv.value())));
-}
-
-cfg_test!(
-    use libstress::utils::generate_random_string_vector;
-    use rand::thread_rng;
-    #[test]
-    fn roast_the_serializer() {
-        const COUNT: usize = 1000_usize;
-        const LEN: usize = 8_usize;
-        let mut rng = thread_rng();
-        let (keys, values) = (
-            generate_random_string_vector(COUNT, LEN, &mut rng, true).unwrap(),
-            generate_random_string_vector(COUNT, LEN, &mut rng, false).unwrap(),
-        );
-        let cmap: Coremap<SharedSlice, SharedSlice> = keys
-            .iter()
-            .zip(values.iter())
-            .map(|(k, v)| {
-                (
-                    SharedSlice::from(k.to_owned()),
-                    SharedSlice::from(v.to_owned()),
-                )
-            })
-            .collect();
-        let ser = se::serialize_map(&cmap).unwrap();
-        let de = de::deserialize_map(&ser).unwrap();
-        assert!(de
-            .iter()
-            .all(|kv| cmap.get(kv.key()).unwrap().eq(kv.value())));
-        assert!(de.len() == cmap.len());
-    }
-
-    #[test]
-    fn test_ser_de_safety() {
-        const COUNT: usize = 1000_usize;
-        const LEN: usize = 8_usize;
-        let mut rng = thread_rng();
-        let (keys, values) = (
-            generate_random_string_vector(COUNT, LEN, &mut rng, true).unwrap(),
-            generate_random_string_vector(COUNT, LEN, &mut rng, false).unwrap(),
-        );
-        let cmap: Coremap<SharedSlice, SharedSlice> = keys
-            .iter()
-            .zip(values.iter())
-            .map(|(k, v)| {
-                (
-                    SharedSlice::from(k.to_owned()),
-                    SharedSlice::from(v.to_owned()),
-                )
-            })
-            .collect();
-        let mut se = se::serialize_map(&cmap).unwrap();
-        // random chop
-        se.truncate(124);
-        // corrupted
-        assert!(de::deserialize_map(&se).is_none());
-    }
-    #[test]
-    fn test_ser_de_excess_bytes() {
-        // this test needs a lot of auxiliary space
-        // we can approximate this to be: 100,000 x 30 bytes = 3,000,000 bytes
-        // and then we may have a clone overhead + heap allocation by the map
-        // so ~9,000,000 bytes or ~9MB
-        const COUNT: usize = 1000_usize;
-        const LEN: usize = 8_usize;
-        let mut rng = thread_rng();
-        let (keys, values) = (
-            generate_random_string_vector(COUNT, LEN, &mut rng, true).unwrap(),
-            generate_random_string_vector(COUNT, LEN, &mut rng, false).unwrap(),
-        );
-        let cmap: Coremap<SharedSlice, SharedSlice> = keys
-            .iter()
-            .zip(values.iter())
-            .map(|(k, v)| {
-                (
-                    SharedSlice::from(k.to_owned()),
-                    SharedSlice::from(v.to_owned()),
-                )
-            })
-            .collect();
-        let mut se = se::serialize_map(&cmap).unwrap();
-        // random patch
-        let patch: Vec<u8> = (0u16..500u16).into_iter().map(|v| (v >> 7) as u8).collect();
-        se.extend(patch);
-        assert!(de::deserialize_map(&se).is_none());
-    }
-);
-
-#[cfg(target_pointer_width = "32")]
-#[test]
-#[should_panic]
-fn test_runtime_panic_32bit_or_lower() {
-    let max = u64::MAX;
-    let byte_stream = unsafe { raw_byte_repr(&max).to_owned() };
-    let ptr = byte_stream.as_ptr();
-    unsafe { de::transmute_len(ptr) };
-}
-
-mod interface_tests {
-    use super::interface::{create_tree_fresh, DIR_KSROOT, DIR_SNAPROOT};
-    use crate::corestore::memstore::Memstore;
-    use crate::storage::v1::flush::Autoflush;
-    use std::fs;
-    use std::path::PathBuf;
-    #[test]
-    fn test_tree() {
-        // HACK(@ohsayan): M1 builder is broken
-        if std::env::var_os("HACK_SKYD_TEST_IGNORE_TREE_TEST_M1").is_none() {
-            create_tree_fresh(&Autoflush, &Memstore::new_default()).unwrap();
-            let read_ks: Vec<String> = fs::read_dir(DIR_KSROOT)
-                .unwrap()
-                .map(|dir| {
-                    let v = dir.unwrap().file_name();
-                    v.to_string_lossy().to_string()
-                })
-                .collect();
-            assert!(read_ks.contains(&"system".to_owned()));
-            assert!(read_ks.contains(&"default".to_owned()));
-            // just read one level of the snaps dir
-            let read_snaps: Vec<String> = fs::read_dir(DIR_SNAPROOT)
-                .unwrap()
-                .map(|dir| {
-                    let v = dir.unwrap().file_name();
-                    v.to_string_lossy().to_string()
-                })
-                .collect();
-            assert_eq!(read_snaps, Vec::<String>::new());
-            assert!(PathBuf::from("data/backups").is_dir());
-        }
-    }
-}
-
-mod preload_tests {
-    use super::*;
-    use crate::corestore::memstore::Memstore;
-    #[test]
-    fn test_preload() {
-        let memstore = Memstore::new_default();
-        let mut v = Vec::new();
-        preload::raw_generate_preload(&mut v, &memstore).unwrap();
-        let de: Vec<String> = preload::read_preload_raw(v)
-            .unwrap()
-            .into_iter()
-            .map(|each| unsafe { each.as_str().to_owned() })
-            .collect();
-        assert_veceq_transposed!(de, vec!["default".to_owned(), "system".to_owned()]);
-    }
-}
-
-mod bytemark_set_tests {
-    use super::*;
-    use crate::corestore::memstore::{Keyspace, ObjectID};
-    use crate::corestore::table::Table;
-    use std::collections::HashMap;
-    #[test]
-    fn test_bytemark_for_nonvolatile() {
-        let ks = Keyspace::empty_default();
-        let mut v = Vec::new();
-        se::raw_serialize_partmap(&mut v, &ks).unwrap();
-        let ret: HashMap<ObjectID, (u8, u8)> = de::deserialize_set_ctype_bytemark(&v).unwrap();
-        let mut expected = HashMap::new();
-        unsafe {
-            expected.insert(
-                ObjectID::from_slice("default"),
-                (
-                    bytemarks::BYTEMARK_STORAGE_PERSISTENT,
-                    bytemarks::BYTEMARK_MODEL_KV_BIN_BIN,
-                ),
-            );
-        }
-        assert_hmeq!(expected, ret);
-    }
-    #[test]
-    fn test_bytemark_volatility_mixed() {
-        let ks = Keyspace::empty();
-        unsafe {
-            ks.create_table(
-                ObjectID::from_slice("cache"),
-                Table::new_kve_with_volatile(true),
-            );
-            ks.create_table(
-                ObjectID::from_slice("supersafe"),
-                Table::new_kve_with_volatile(false),
-            );
-            ks.create_table(
-                ObjectID::from_slice("safelist"),
-                Table::new_kve_listmap_with_data(Coremap::new(), false, true, true),
-            );
-        }
-        let mut v = Vec::new();
-        se::raw_serialize_partmap(&mut v, &ks).unwrap();
-        let ret: HashMap<ObjectID, (u8, u8)> = de::deserialize_set_ctype_bytemark(&v).unwrap();
-        let mut expected = HashMap::new();
-        unsafe {
-            // our cache is volatile
-            expected.insert(
-                ObjectID::from_slice("cache"),
-                (
-                    bytemarks::BYTEMARK_STORAGE_VOLATILE,
-                    bytemarks::BYTEMARK_MODEL_KV_BIN_BIN,
-                ),
-            );
-            // our supersafe is non volatile
-            expected.insert(
-                ObjectID::from_slice("supersafe"),
-                (
-                    bytemarks::BYTEMARK_STORAGE_PERSISTENT,
-                    bytemarks::BYTEMARK_MODEL_KV_BIN_BIN,
-                ),
-            );
-            expected.insert(
-                ObjectID::from_slice("safelist"),
-                (
-                    bytemarks::BYTEMARK_STORAGE_PERSISTENT,
-                    bytemarks::BYTEMARK_MODEL_KV_STR_LIST_STR,
-                ),
-            );
-        }
-        assert_hmeq!(expected, ret);
-    }
-}
-
-mod bytemark_actual_table_restore {
-    use crate::corestore::{
-        memstore::ObjectID,
-        table::{DescribeTable, KVEBlob, KVEList, Table},
-        SharedSlice,
-    };
-    use crate::kvengine::LockedVec;
-    use crate::storage::v1::{
-        flush::{oneshot::flush_table, Autoflush},
-        unflush::read_table,
-    };
-
-    macro_rules! insert {
-        ($table:ident, $k:expr, $v:expr) => {
-            assert!(gtable::<KVEBlob>(&$table)
-                .set(SharedSlice::from($k), SharedSlice::from($v))
-                .unwrap())
-        };
-    }
-
-    macro_rules! puthello {
-        ($table:ident) => {
-            insert!($table, "hello", "world")
-        };
-    }
-
-    fn gtable<T: DescribeTable>(table: &Table) -> &T::Table {
-        T::try_get(table).unwrap()
-    }
-
-    use std::fs;
-    #[test]
-    fn table_restore_bytemark_kve() {
-        let default_keyspace = ObjectID::try_from_slice(b"actual_kve_restore").unwrap();
-        fs::create_dir_all(format!("data/ks/{}", unsafe { default_keyspace.as_str() })).unwrap();
-        let kve_bin_bin_name = ObjectID::try_from_slice(b"bin_bin").unwrap();
-        let kve_bin_bin = Table::from_model_code(0, false).unwrap();
-        puthello!(kve_bin_bin);
-        let kve_bin_str_name = ObjectID::try_from_slice(b"bin_str").unwrap();
-        let kve_bin_str = Table::from_model_code(1, false).unwrap();
-        puthello!(kve_bin_str);
-        let kve_str_str_name = ObjectID::try_from_slice(b"str_str").unwrap();
-        let kve_str_str = Table::from_model_code(2, false).unwrap();
-        puthello!(kve_str_str);
-        let kve_str_bin_name = ObjectID::try_from_slice(b"str_bin").unwrap();
-        let kve_str_bin = Table::from_model_code(3, false).unwrap();
-        puthello!(kve_str_bin);
-        let names: [(&ObjectID, &Table, u8); 4] = [
-            (&kve_bin_bin_name, &kve_bin_bin, 0),
-            (&kve_bin_str_name, &kve_bin_str, 1),
-            (&kve_str_str_name, &kve_str_str, 2),
-            (&kve_str_bin_name, &kve_str_bin, 3),
-        ];
-        // flush each of them
-        for (tablename, table, _) in names {
-            flush_table(&Autoflush, tablename, &default_keyspace, table).unwrap();
-        }
-        let mut read_tables: Vec<Table> = Vec::with_capacity(4);
-        // read each of them
-        for (tableid, _, modelcode) in names {
-            read_tables.push(read_table(&default_keyspace, tableid, false, modelcode).unwrap());
-        }
-        for (index, (table, code)) in read_tables
-            .iter()
-            .map(|tbl| (gtable::<KVEBlob>(tbl), tbl.get_model_code()))
-            .enumerate()
-        {
-            assert_eq!(index, code as usize);
-            assert!(table.get("hello".as_bytes()).unwrap().unwrap().eq(b"world"));
-            assert_eq!(table.len(), 1);
-        }
-    }
-
-    macro_rules! putlist {
-        ($table:ident) => {
-            gtable::<KVEList>(&$table)
-                .get_inner_ref()
-                .fresh_entry(SharedSlice::from("super"))
-                .unwrap()
-                .insert(LockedVec::new(vec![
-                    SharedSlice::from("hello"),
-                    SharedSlice::from("world"),
-                ]))
-        };
-    }
-
-    #[test]
-    fn table_restore_bytemark_kvlist() {
-        let default_keyspace = ObjectID::try_from_slice(b"actual_kvl_restore").unwrap();
-        fs::create_dir_all(format!("data/ks/{}", unsafe { default_keyspace.as_str() })).unwrap();
-        let kve_bin_listbin_name = ObjectID::try_from_slice(b"bin_listbin").unwrap();
-        let kve_bin_listbin = Table::from_model_code(4, false).unwrap();
-        putlist!(kve_bin_listbin);
-        let kve_bin_liststr_name = ObjectID::try_from_slice(b"bin_liststr").unwrap();
-        let kve_bin_liststr = Table::from_model_code(5, false).unwrap();
-        putlist!(kve_bin_liststr);
-        let kve_str_listbinstr_name = ObjectID::try_from_slice(b"str_listbinstr").unwrap();
-        let kve_str_listbinstr = Table::from_model_code(6, false).unwrap();
-        putlist!(kve_str_listbinstr);
-        let kve_str_liststr_name = ObjectID::try_from_slice(b"str_liststr").unwrap();
-        let kve_str_liststr = Table::from_model_code(7, false).unwrap();
-        putlist!(kve_str_liststr);
-        let names: [(&ObjectID, &Table, u8); 4] = [
-            (&kve_bin_listbin_name, &kve_bin_listbin, 4),
-            (&kve_bin_liststr_name, &kve_bin_liststr, 5),
-            (&kve_str_listbinstr_name, &kve_str_listbinstr, 6),
-            (&kve_str_liststr_name, &kve_str_liststr, 7),
-        ];
-        // flush each of them
-        for (tablename, table, _) in names {
-            flush_table(&Autoflush, tablename, &default_keyspace, table).unwrap();
-        }
-        let mut read_tables: Vec<Table> = Vec::with_capacity(4);
-        // read each of them
-        for (tableid, _, modelcode) in names {
-            read_tables.push(read_table(&default_keyspace, tableid, false, modelcode).unwrap());
-        }
-        for (index, (table, code)) in read_tables
-            .iter()
-            .map(|tbl| (gtable::<KVEList>(tbl), tbl.get_model_code()))
-            .enumerate()
-        {
-            // check code
-            assert_eq!(index + 4, code as usize);
-            // check payload
-            let vec = table.get_inner_ref().get("super".as_bytes()).unwrap();
-            assert_eq!(vec.read().len(), 2);
-            assert_eq!(vec.read()[0], "hello");
-            assert_eq!(vec.read()[1], "world");
-            // check len
-            assert_eq!(table.len(), 1);
-        }
-    }
-}
-
-mod flush_routines {
-    use crate::{
-        corestore::{
-            memstore::{Keyspace, ObjectID},
-            table::{DataModel, Table},
-            SharedSlice,
-        },
-        kvengine::LockedVec,
-        storage::v1::{bytemarks, flush::Autoflush, Coremap},
-    };
-    use std::fs;
-    #[test]
-    fn test_flush_unflush_table_pure_kve() {
-        let tbl = Table::new_default_kve();
-        tbl.get_kvstore()
-            .unwrap()
-            .set("hello".into(), "world".into())
-            .unwrap();
-        let tblid = unsafe { ObjectID::from_slice("mytbl1") };
-        let ksid = unsafe { ObjectID::from_slice("myks1") };
-        // create the temp dir for this test
-        fs::create_dir_all("data/ks/myks1").unwrap();
-        super::flush::oneshot::flush_table(&Autoflush, &tblid, &ksid, &tbl).unwrap();
-        // now that it's flushed, let's read the table using and unflush routine
-        let ret = super::unflush::read_table::<Table>(
-            &ksid,
-            &tblid,
-            false,
-            bytemarks::BYTEMARK_MODEL_KV_BIN_BIN,
-        )
-        .unwrap();
-        assert_eq!(
-            ret.get_kvstore()
-                .unwrap()
-                .get(&SharedSlice::from("hello"))
-                .unwrap()
-                .unwrap()
-                .clone(),
-            SharedSlice::from("world")
-        );
-    }
-
-    #[test]
-    fn test_flush_unflush_table_kvext_listmap() {
-        let tbl = Table::new_kve_listmap_with_data(Coremap::new(), false, true, true);
-        if let DataModel::KVExtListmap(kvl) = tbl.get_model_ref() {
-            kvl.add_list("mylist".into()).unwrap();
-            let list = kvl.get("mylist".as_bytes()).unwrap().unwrap();
-            list.write().push("mysupervalue".into());
-        } else {
-            panic!("Bad model!");
-        }
-        let tblid = unsafe { ObjectID::from_slice("mylists1") };
-        let ksid = unsafe { ObjectID::from_slice("mylistyks") };
-        // create the temp dir for this test
-        fs::create_dir_all("data/ks/mylistyks").unwrap();
-        super::flush::oneshot::flush_table(&Autoflush, &tblid, &ksid, &tbl).unwrap();
-        // now that it's flushed, let's read the table using and unflush routine
-        let ret = super::unflush::read_table::<Table>(
-            &ksid,
-            &tblid,
-            false,
-            bytemarks::BYTEMARK_MODEL_KV_STR_LIST_STR,
-        )
-        .unwrap();
-        assert!(!ret.is_volatile());
-        if let DataModel::KVExtListmap(kvl) = ret.get_model_ref() {
-            let list = kvl.get("mylist".as_bytes()).unwrap().unwrap();
-            let lread = list.read();
-            assert_eq!(lread.len(), 1);
-            assert_eq!(lread[0].as_ref(), "mysupervalue".as_bytes());
-        } else {
-            panic!("Bad model!");
-        }
-    }
-    #[test]
-    fn test_flush_unflush_keyspace() {
-        // create the temp dir for this test
-        fs::create_dir_all("data/ks/myks_1").unwrap();
-        let ksid = unsafe { ObjectID::from_slice("myks_1") };
-        let tbl1 = unsafe { ObjectID::from_slice("mytbl_1") };
-        let tbl2 = unsafe { ObjectID::from_slice("mytbl_2") };
-        let list_tbl = unsafe { ObjectID::from_slice("mylist_1") };
-        let ks = Keyspace::empty();
-
-        // a persistent table
-        let mytbl = Table::new_default_kve();
-        mytbl
-            .get_kvstore()
-            .unwrap()
-            .set("hello".into(), "world".into())
-            .unwrap();
-        assert!(ks.create_table(tbl1.clone(), mytbl));
-
-        // and a table with lists
-        let cmap = Coremap::new();
-        cmap.true_if_insert("mylist".into(), LockedVec::new(vec!["myvalue".into()]));
-        let my_list_tbl = Table::new_kve_listmap_with_data(cmap, false, true, true);
-        assert!(ks.create_table(list_tbl.clone(), my_list_tbl));
-
-        // and a volatile table
-        assert!(ks.create_table(tbl2.clone(), Table::new_kve_with_volatile(true)));
-
-        // now flush it
-        super::flush::flush_keyspace_full(&Autoflush, &ksid, &ks).unwrap();
-        let ret = super::unflush::read_keyspace::<Keyspace>(&ksid).unwrap();
-        let tbl1_ret = ret.tables.get(&tbl1).unwrap();
-        let tbl2_ret = ret.tables.get(&tbl2).unwrap();
-        let tbl3_ret_list = ret.tables.get(&list_tbl).unwrap();
-        // should be a persistent table with the value we set
-        assert_eq!(tbl1_ret.count(), 1);
-        assert_eq!(
-            tbl1_ret
-                .get_kvstore()
-                .unwrap()
-                .get(&SharedSlice::from("hello"))
-                .unwrap()
-                .unwrap()
-                .clone(),
-            SharedSlice::from("world")
-        );
-        // should be a volatile table with no values
-        assert_eq!(tbl2_ret.count(), 0);
-        assert!(tbl2_ret.is_volatile());
-        // should have a list with the `myvalue` element
-        assert_eq!(tbl3_ret_list.count(), 1);
-        if let DataModel::KVExtListmap(kvl) = tbl3_ret_list.get_model_ref() {
-            assert_eq!(
-                kvl.get("mylist".as_bytes()).unwrap().unwrap().read()[0].as_ref(),
-                "myvalue".as_bytes()
-            );
-        } else {
-            panic!(
-                "Wrong model. Expected listmap, got: {}",
-                tbl3_ret_list.get_model_code()
-            );
-        }
-    }
-}
-
-mod list_tests {
-    use super::iter::RawSliceIter;
-    use super::{de, se};
-    use crate::corestore::{htable::Coremap, SharedSlice};
-    use crate::kvengine::LockedVec;
-    use parking_lot::RwLock;
-    #[test]
-    fn test_list_se_de() {
-        let mylist = vec![
-            SharedSlice::from("a"),
-            SharedSlice::from("b"),
-            SharedSlice::from("c"),
-        ];
-        let mut v = Vec::new();
-        se::raw_serialize_nested_list(&mut v, &mylist).unwrap();
-        let mut rawiter = RawSliceIter::new(&v);
-        let de = { de::deserialize_nested_list(rawiter.get_borrowed_iter()).unwrap() };
-        assert_eq!(de, mylist);
-    }
-    #[test]
-    fn test_list_se_de_with_empty_element() {
-        let mylist = vec![
-            SharedSlice::from("a"),
-            SharedSlice::from("b"),
-            SharedSlice::from("c"),
-            SharedSlice::from(""),
-        ];
-        let mut v = Vec::new();
-        se::raw_serialize_nested_list(&mut v, &mylist).unwrap();
-        let mut rawiter = RawSliceIter::new(&v);
-        let de = { de::deserialize_nested_list(rawiter.get_borrowed_iter()).unwrap() };
-        assert_eq!(de, mylist);
-    }
-    #[test]
-    fn test_empty_list_se_de() {
-        let mylist: Vec<SharedSlice> = vec![];
-        let mut v = Vec::new();
-        se::raw_serialize_nested_list(&mut v, &mylist).unwrap();
-        let mut rawiter = RawSliceIter::new(&v);
-        let de = { de::deserialize_nested_list(rawiter.get_borrowed_iter()).unwrap() };
-        assert_eq!(de, mylist);
-    }
-    #[test]
-    fn test_list_map_monoelement_se_de() {
-        let mymap = Coremap::new();
-        let vals = lvec!["apples", "bananas", "carrots"];
-        mymap.true_if_insert(SharedSlice::from("mykey"), RwLock::new(vals.read().clone()));
-        let mut v = Vec::new();
-        se::raw_serialize_list_map(&mymap, &mut v).unwrap();
-        let de = de::deserialize_list_map(&v).unwrap();
-        assert_eq!(de.len(), 1);
-        let mykey_value = de.get("mykey".as_bytes()).unwrap().value().read().clone();
-        assert_eq!(
-            mykey_value,
-            vals.into_inner()
-                .into_iter()
-                .map(SharedSlice::from)
-                .collect::<Vec<SharedSlice>>()
-        );
-    }
-    #[test]
-    fn test_list_map_se_de() {
-        let mymap: Coremap<SharedSlice, LockedVec> = Coremap::new();
-        let key1: SharedSlice = "mykey1".into();
-        let val1 = lvec!["apples", "bananas", "carrots"];
-        let key2: SharedSlice = "mykey2long".into();
-        let val2 = lvec!["code", "coffee", "cats"];
-        mymap.true_if_insert(key1.clone(), RwLock::new(val1.read().clone()));
-        mymap.true_if_insert(key2.clone(), RwLock::new(val2.read().clone()));
-        let mut v = Vec::new();
-        se::raw_serialize_list_map(&mymap, &mut v).unwrap();
-        let de = de::deserialize_list_map(&v).unwrap();
-        assert_eq!(de.len(), 2);
-        assert_eq!(
-            de.get(&key1).unwrap().value().read().clone(),
-            val1.into_inner()
-                .into_iter()
-                .map(SharedSlice::from)
-                .collect::<Vec<SharedSlice>>()
-        );
-        assert_eq!(
-            de.get(&key2).unwrap().value().read().clone(),
-            val2.into_inner()
-                .into_iter()
-                .map(SharedSlice::from)
-                .collect::<Vec<SharedSlice>>()
-        );
-    }
-    #[test]
-    fn test_list_map_empty_se_de() {
-        let mymap: Coremap<SharedSlice, LockedVec> = Coremap::new();
-        let mut v = Vec::new();
-        se::raw_serialize_list_map(&mymap, &mut v).unwrap();
-        let de = de::deserialize_list_map(&v).unwrap();
-        assert_eq!(de.len(), 0)
-    }
-}
-
-mod corruption_tests {
-    use crate::corestore::htable::Coremap;
-    use crate::corestore::SharedSlice;
-    use crate::kvengine::LockedVec;
-    #[test]
-    fn test_corruption_map_basic() {
-        let mymap = Coremap::new();
-        let seresult = super::se::serialize_map(&mymap).unwrap();
-        // now chop it; since this has 8B, let's drop some bytes
-        assert!(super::de::deserialize_map(&seresult[..seresult.len() - 6]).is_none());
-    }
-    #[test]
-    fn test_map_corruption_end_corruption() {
-        let cmap = Coremap::new();
-        cmap.upsert("sayan".into(), "writes code".into());
-        cmap.upsert("supersayan".into(), "writes super code".into());
-        let ser = super::se::serialize_map(&cmap).unwrap();
-        // corrupt the last 16B
-        assert!(super::de::deserialize_map(&ser[..ser.len() - 16]).is_none());
-    }
-    #[test]
-    fn test_map_corruption_midway_corruption() {
-        let cmap = Coremap::new();
-        cmap.upsert("sayan".into(), "writes code".into());
-        cmap.upsert("supersayan".into(), "writes super code".into());
-        let mut ser = super::se::serialize_map(&cmap).unwrap();
-        // middle chop
-        ser.drain(16..ser.len() / 2);
-        assert!(super::de::deserialize_map(&ser).is_none());
-    }
-    #[test]
-    fn test_listmap_corruption_basic() {
-        let mymap: Coremap<SharedSlice, LockedVec> = Coremap::new();
-        mymap.upsert("hello".into(), lvec!("hello-1"));
-        // current repr: [1u64][5u64]["hello"][1u64][7u64]["hello-1"]
-        // sanity test
-        let mut v = Vec::new();
-        super::se::raw_serialize_list_map(&mymap, &mut v).unwrap();
-        assert!(super::de::deserialize_list_map(&v).is_some());
-        // now chop "hello-1"
-        assert!(super::de::deserialize_list_map(&v[..v.len() - 7]).is_none());
-    }
-    #[test]
-    fn test_listmap_corruption_midway() {
-        let mymap: Coremap<SharedSlice, LockedVec> = Coremap::new();
-        mymap.upsert("hello".into(), lvec!("hello-1"));
-        // current repr: [1u64][5u64]["hello"][1u64][7u64]["hello-1"]
-        // sanity test
-        let mut v = Vec::new();
-        super::se::raw_serialize_list_map(&mymap, &mut v).unwrap();
-        assert!(super::de::deserialize_list_map(&v).is_some());
-        assert_eq!(v.len(), 44);
-        // now chop "7u64" (8+8+5+8+8+7)
-        v.drain(29..37);
-        assert!(super::de::deserialize_list_map(&v).is_none());
-    }
-}
-
-mod storage_target_directory_structure {
-    use crate::{
-        corestore::{
-            memstore::{Memstore, ObjectID},
-            table::{SystemTable, Table},
-        },
-        storage::v1::flush::{self, LocalSnapshot, RemoteSnapshot},
-        util::{
-            os::{self, EntryKind},
-            Wrapper,
-        },
-    };
-    use std::collections::HashSet;
-    use std::path::PathBuf;
-    enum FileKind {
-        Dir(&'static str),
-        File(&'static str),
-    }
-    impl FileKind {
-        fn into_entrykind_path(self, closure: impl Fn(&'static str) -> String + Copy) -> EntryKind {
-            match self {
-                Self::Dir(dir) => EntryKind::Directory(closure(dir)),
-                Self::File(file) => EntryKind::File(closure(file)),
-            }
-        }
-    }
-    fn get_memstore_and_file_list() -> (Memstore, Vec<FileKind>) {
-        let paths = vec![
-            // the default keyspace
-            FileKind::Dir("default"),
-            FileKind::File("default/default"),
-            FileKind::File("default/PARTMAP"),
-            // the superks keyspace
-            FileKind::Dir("superks"),
-            FileKind::File("superks/PARTMAP"),
-            FileKind::File("superks/blueshark"),
-            // the system keyspace
-            FileKind::Dir("system"),
-            FileKind::File("system/PARTMAP"),
-            FileKind::File("system/superauthy"),
-            // the preload file
-            FileKind::File("PRELOAD"),
-        ];
-        (get_memstore(), paths)
-    }
-    fn get_memstore() -> Memstore {
-        let store = Memstore::new_default();
-        assert!(store.create_keyspace(ObjectID::try_from_slice("superks").unwrap()));
-        assert!(store
-            .get_keyspace_atomic_ref("superks".as_bytes())
-            .unwrap()
-            .create_table(
-                ObjectID::try_from_slice("blueshark").unwrap(),
-                Table::new_default_kve()
-            ));
-        assert!(store.system.tables.true_if_insert(
-            ObjectID::try_from_slice("superauthy").unwrap(),
-            Wrapper::new(SystemTable::new_auth(Default::default()))
-        ));
-        store
-    }
-    fn ensure_paths_equality(a: Vec<EntryKind>, b: Vec<EntryKind>) {
-        assert_eq!(a.len(), b.len());
-        let aset: HashSet<PathBuf> = a
-            .iter()
-            .map(|apath| PathBuf::from(apath.to_string()))
-            .collect();
-        let bset: HashSet<PathBuf> = a
-            .iter()
-            .map(|bpath| PathBuf::from(bpath.to_string()))
-            .collect();
-        aset.into_iter()
-            .for_each(|apath| assert_eq!(&apath, bset.get(&apath).unwrap()));
-    }
-    use std::fs;
-    #[test]
-    fn local_snapshot_dir() {
-        let (store, paths) = get_memstore_and_file_list();
-        let paths = paths
-            .into_iter()
-            .map(|v| v.into_entrykind_path(|file| format!("data/snaps/localsnap/{file}")))
-            .collect::<Vec<EntryKind>>();
-        let target = LocalSnapshot::new("localsnap".to_owned());
-        flush::flush_full(target, &store).unwrap();
-        let files = os::rlistdir("data/snaps/localsnap").unwrap();
-        ensure_paths_equality(files, paths);
-        fs::remove_dir_all("data/snaps/localsnap").unwrap();
-    }
-    #[test]
-    fn remote_snapshot_dir() {
-        let (store, paths) = get_memstore_and_file_list();
-        let paths = paths
-            .into_iter()
-            .map(|v| v.into_entrykind_path(|file| format!("data/rsnap/wisnap/{file}")))
-            .collect::<Vec<EntryKind>>();
-        let target = RemoteSnapshot::new("wisnap");
-        flush::flush_full(target, &store).unwrap();
-        let files = os::rlistdir("data/rsnap/wisnap").unwrap();
-        ensure_paths_equality(files, paths);
-        fs::remove_dir_all("data/rsnap/wisnap").unwrap();
-    }
-}
diff --git a/server/src/storage/v1/unflush.rs b/server/src/storage/v1/unflush.rs
deleted file mode 100644
index 04601b1e..00000000
--- a/server/src/storage/v1/unflush.rs
+++ /dev/null
@@ -1,266 +0,0 @@
-/*
- * Created on Sat Jul 17 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # Unflush routines
-//!
-//! Routines for unflushing data
-
-use {
-    super::bytemarks,
-    crate::{
-        corestore::{
-            memstore::{Keyspace, Memstore, ObjectID, SystemKeyspace, SYSTEM},
-            table::{SystemTable, Table},
-        },
-        storage::v1::{
-            de::DeserializeInto,
-            error::{ErrorContext, StorageEngineError, StorageEngineResult},
-            flush::Autoflush,
-            interface::DIR_KSROOT,
-            preload::LoadedPartfile,
-            Coremap,
-        },
-        util::Wrapper,
-    },
-    core::mem::transmute,
-    std::{fs, io::ErrorKind, path::Path, sync::Arc},
-};
-
-type PreloadSet = std::collections::HashSet<ObjectID>;
-const PRELOAD_PATH: &str = "data/ks/PRELOAD";
-
-/// A keyspace that can be restored from disk storage
-pub trait UnflushableKeyspace: Sized {
-    /// Unflush routine for a keyspace
-    fn unflush_keyspace(partmap: LoadedPartfile, ksid: &ObjectID) -> StorageEngineResult<Self>;
-}
-
-impl UnflushableKeyspace for Keyspace {
-    fn unflush_keyspace(partmap: LoadedPartfile, ksid: &ObjectID) -> StorageEngineResult<Self> {
-        let ks: Coremap<ObjectID, Arc<Table>> = Coremap::with_capacity(partmap.len());
-        for (tableid, (table_storage_type, model_code)) in partmap.into_iter() {
-            if table_storage_type > 1 {
-                return Err(StorageEngineError::bad_metadata_in_table(ksid, &tableid));
-            }
-            let is_volatile = table_storage_type == bytemarks::BYTEMARK_STORAGE_VOLATILE;
-            let tbl = self::read_table::<Table>(ksid, &tableid, is_volatile, model_code)?;
-            ks.true_if_insert(tableid, Arc::new(tbl));
-        }
-        Ok(Keyspace::init_with_all_def_strategy(ks))
-    }
-}
-
-impl UnflushableKeyspace for SystemKeyspace {
-    fn unflush_keyspace(partmap: LoadedPartfile, ksid: &ObjectID) -> StorageEngineResult<Self> {
-        let ks: Coremap<ObjectID, Wrapper<SystemTable>> = Coremap::with_capacity(partmap.len());
-        for (tableid, (table_storage_type, model_code)) in partmap.into_iter() {
-            if table_storage_type > 1 {
-                return Err(StorageEngineError::bad_metadata_in_table(ksid, &tableid));
-            }
-            let is_volatile = table_storage_type == bytemarks::BYTEMARK_STORAGE_VOLATILE;
-            let tbl = self::read_table::<SystemTable>(ksid, &tableid, is_volatile, model_code)?;
-            ks.true_if_insert(tableid, Wrapper::new(tbl));
-        }
-        Ok(SystemKeyspace::new(ks))
-    }
-}
-
-/// Tables that can be restored from disk storage
-pub trait UnflushableTable: Sized {
-    /// Procedure to restore (deserialize) table from disk storage
-    fn unflush_table(
-        filepath: impl AsRef<Path>,
-        model_code: u8,
-        volatile: bool,
-    ) -> StorageEngineResult<Self>;
-}
-
-#[allow(clippy::transmute_int_to_bool)]
-impl UnflushableTable for Table {
-    fn unflush_table(
-        filepath: impl AsRef<Path>,
-        model_code: u8,
-        volatile: bool,
-    ) -> StorageEngineResult<Self> {
-        let ret = match model_code {
-            // pure KVEBlob: [0, 3]
-            x if x < 4 => {
-                let data = decode(filepath, volatile)?;
-                let (k_enc, v_enc) = unsafe {
-                    // UNSAFE(@ohsayan): Safe because of the above match. Just a lil bitmagic
-                    let key: bool = transmute(model_code >> 1);
-                    let value: bool = transmute(((model_code >> 1) + (model_code & 1)) % 2);
-                    (key, value)
-                };
-                Table::new_pure_kve_with_data(data, volatile, k_enc, v_enc)
-            }
-            // KVExtlistmap: [4, 7]
-            x if x < 8 => {
-                let data = decode(filepath, volatile)?;
-                let (k_enc, v_enc) = unsafe {
-                    // UNSAFE(@ohsayan): Safe because of the above match. Just a lil bitmagic
-                    let code = model_code - 4;
-                    let key: bool = transmute(code >> 1);
-                    let value: bool = transmute(code % 2);
-                    (key, value)
-                };
-                Table::new_kve_listmap_with_data(data, volatile, k_enc, v_enc)
-            }
-            _ => {
-                return Err(StorageEngineError::BadMetadata(
-                    filepath.as_ref().to_string_lossy().to_string(),
-                ))
-            }
-        };
-        Ok(ret)
-    }
-}
-
-impl UnflushableTable for SystemTable {
-    fn unflush_table(
-        filepath: impl AsRef<Path>,
-        model_code: u8,
-        volatile: bool,
-    ) -> StorageEngineResult<Self> {
-        match model_code {
-            0 => {
-                // this is the authmap
-                let authmap = decode(filepath, volatile)?;
-                Ok(SystemTable::new_auth(Arc::new(authmap)))
-            }
-            _ => Err(StorageEngineError::BadMetadata(
-                filepath.as_ref().to_string_lossy().to_string(),
-            )),
-        }
-    }
-}
-
-#[inline(always)]
-fn decode<T: DeserializeInto>(
-    filepath: impl AsRef<Path>,
-    volatile: bool,
-) -> StorageEngineResult<T> {
-    if volatile {
-        Ok(T::new_empty())
-    } else {
-        let data = fs::read(filepath.as_ref()).map_err_context(format!(
-            "reading file {}",
-            filepath.as_ref().to_string_lossy()
-        ))?;
-        super::de::deserialize_into(&data).ok_or_else(|| {
-            StorageEngineError::CorruptedFile(filepath.as_ref().to_string_lossy().to_string())
-        })
-    }
-}
-
-/// Read a given table into a [`Table`] object
-///
-/// This will take care of volatility and the model_code. Just make sure that you pass the proper
-/// keyspace ID and a valid table ID
-pub fn read_table<T: UnflushableTable>(
-    ksid: &ObjectID,
-    tblid: &ObjectID,
-    volatile: bool,
-    model_code: u8,
-) -> StorageEngineResult<T> {
-    let filepath = unsafe { concat_path!(DIR_KSROOT, ksid.as_str(), tblid.as_str()) };
-    let tbl = T::unflush_table(filepath, model_code, volatile)?;
-    Ok(tbl)
-}
-
-/// Read an entire keyspace into a Coremap. You'll need to initialize the rest
-pub fn read_keyspace<K: UnflushableKeyspace>(ksid: &ObjectID) -> StorageEngineResult<K> {
-    let partmap = self::read_partmap(ksid)?;
-    K::unflush_keyspace(partmap, ksid)
-}
-
-/// Read the `PARTMAP` for a given keyspace
-pub fn read_partmap(ksid: &ObjectID) -> StorageEngineResult<LoadedPartfile> {
-    let ksid_str = unsafe { ksid.as_str() };
-    let filepath = concat_path!(DIR_KSROOT, ksid_str, "PARTMAP");
-    let partmap_raw = fs::read(&filepath)
-        .map_err_context(format!("while reading {}", filepath.to_string_lossy()))?;
-    super::de::deserialize_set_ctype_bytemark(&partmap_raw)
-        .ok_or_else(|| StorageEngineError::corrupted_partmap(ksid))
-}
-
-/// Read the `PRELOAD`
-pub fn read_preload() -> StorageEngineResult<PreloadSet> {
-    let read = fs::read(PRELOAD_PATH).map_err_context("reading PRELOAD")?;
-    super::preload::read_preload_raw(read)
-}
-
-/// Read everything and return a [`Memstore`]
-///
-/// If this is a new instance an empty store is returned while the directory tree
-/// is also created. If this is an already initialized instance then the store
-/// is read and returned (and any possible errors that are encountered are returned)
-pub fn read_full() -> StorageEngineResult<Memstore> {
-    if is_new_instance()? {
-        log::trace!("Detected new instance. Creating data directory");
-        /*
-        Since the `PRELOAD` file doesn't exist -- this is a new instance
-        This means that we need to:
-        1. Create the tree (this only creates the directories)
-        2. Create the PRELOAD (this is not created by flush_full!)
-        3. Do a full flush (this flushes, but doesn't do anything to the PRELOAD!!!)
-        */
-        // init an empty store
-        let store = Memstore::new_default();
-        let target = Autoflush;
-        // (1) create the tree
-        super::interface::create_tree_fresh(&target, &store)?;
-        // (2) create the preload
-        super::flush::oneshot::flush_preload(&target, &store)?;
-        // (3) do a full flush
-        super::flush::flush_full(target, &store)?;
-        return Ok(store);
-    }
-    let mut preload = self::read_preload()?;
-    // HACK(@ohsayan): Pop off the preload from the serial read_keyspace list. It will fail
-    assert!(preload.remove(&SYSTEM));
-    let system_keyspace = self::read_keyspace::<SystemKeyspace>(&SYSTEM)?;
-    let ksmap = Coremap::with_capacity(preload.len());
-    for ksid in preload {
-        let ks = self::read_keyspace::<Keyspace>(&ksid)?;
-        ksmap.upsert(ksid, Arc::new(ks));
-    }
-    // HACK(@ohsayan): Now pop system back in here
-    ksmap.upsert(SYSTEM, Arc::new(Keyspace::empty()));
-    Ok(Memstore::init_with_all(ksmap, system_keyspace))
-}
-
-/// Check if the `data` directory is non-empty (if not: we're on a new instance)
-pub fn is_new_instance() -> StorageEngineResult<bool> {
-    match fs::read_dir("data") {
-        Ok(mut dir) => Ok(dir.next().is_none()),
-        Err(e) if e.kind().eq(&ErrorKind::NotFound) => Ok(true),
-        Err(e) => Err(StorageEngineError::ioerror_extra(
-            e,
-            "while checking data directory",
-        )),
-    }
-}
diff --git a/server/src/tests/auth.rs b/server/src/tests/auth.rs
deleted file mode 100644
index 9cbf0c42..00000000
--- a/server/src/tests/auth.rs
+++ /dev/null
@@ -1,378 +0,0 @@
-/*
- * Created on Fri Mar 11 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::auth::provider::testsuite_data,
-    skytable::{query, Element, RespCode},
-};
-
-macro_rules! assert_autherror {
-    ($con:expr, $query:expr, $eq:expr) => {
-        runeq!($con, $query, Element::RespCode($eq))
-    };
-}
-macro_rules! assert_auth_disabled {
-    ($con:expr, $query:expr) => {
-        assert_autherror!(
-            $con,
-            $query,
-            RespCode::ErrorString("err-auth-disabled".to_owned())
-        )
-    };
-}
-
-macro_rules! assert_auth_perm_error {
-    ($con:expr, $query:expr) => {
-        assert_autherror!($con, $query, RespCode::AuthPermissionError)
-    };
-}
-
-macro_rules! assert_auth_bad_credentials {
-    ($con:expr, $query:expr) => {
-        assert_autherror!($con, $query, RespCode::AuthBadCredentials)
-    };
-}
-
-const ONLYAUTH: u8 = 0;
-const NOAUTH: u8 = 1;
-
-macro_rules! assert_authn_resp_matrix {
-    ($con:expr, $query:expr, $username:ident, $password:ident, $resp:expr) => {
-        runeq!(
-            $con,
-            ::skytable::query!("auth", "login", $username, $password),
-            ::skytable::Element::RespCode(::skytable::RespCode::Okay)
-        );
-        runeq!($con, $query, $resp);
-    };
-    ($con:expr, $query:expr, $resp:expr) => {{
-        runeq!($con, $query, $resp)
-    }};
-    ($con:expr, $query:expr, $authnd:ident, $resp:expr) => {{
-        match $authnd {
-            ONLYAUTH => {
-                assert_authn_resp_matrix!($con, $query, ROOT_USER, ROOT_PASS, $resp);
-                assert_authn_resp_matrix!($con, $query, USER, PASS, $resp);
-            }
-            NOAUTH => {
-                assert_authn_resp_matrix!($con, $query, $resp);
-                assert_authn_resp_matrix!($con, $query, ROOT_USER, ROOT_PASS, $resp);
-                assert_authn_resp_matrix!($con, $query, USER, PASS, $resp);
-            }
-            _ => panic!("Unknown authnd state"),
-        }
-    }};
-}
-
-// auth claim
-// auth claim fail because it is disabled
-#[sky_macros::dbtest_func]
-async fn auth_claim_fail_disabled() {
-    assert_auth_disabled!(con, query!("auth", "claim", "blah"))
-}
-// auth claim fail because it has already been claimed
-#[sky_macros::dbtest_func(port = 2005, norun = true)]
-async fn claim_root_fail_already_claimed() {
-    runeq!(
-        con,
-        query!("auth", "claim", crate::TEST_AUTH_ORIGIN_KEY),
-        Element::RespCode(RespCode::ErrorString("err-auth-already-claimed".to_owned()))
-    )
-}
-
-// auth login
-// auth login fail because it is disabled
-#[sky_macros::dbtest_func]
-async fn auth_login_fail() {
-    assert_auth_disabled!(con, query!("auth", "login", "user", "blah"))
-}
-// auth login okay (testuser)
-#[sky_macros::dbtest_func(port = 2005, auth_testuser = true)]
-async fn auth_login_testuser() {
-    runeq!(
-        con,
-        query!("heya", "abcd"),
-        Element::String("abcd".to_owned())
-    )
-}
-#[sky_macros::dbtest_func(port = 2005, norun = true)]
-async fn auth_login_testuser_fail_bad_creds() {
-    assert_auth_bad_credentials!(con, query!("auth", "login", "testuser", "badpass"))
-}
-// auth login okay (root)
-#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true)]
-async fn auth_login_rootuser() {
-    runeq!(
-        con,
-        query!("heya", "abcd"),
-        Element::String("abcd".to_owned())
-    )
-}
-#[sky_macros::dbtest_func(port = 2005, norun = true)]
-async fn auth_login_rootuser_fail_bad_creds() {
-    assert_auth_bad_credentials!(con, query!("auth", "login", "root", "badpass"))
-}
-
-// auth adduser
-// auth adduser fail because disabled
-#[sky_macros::dbtest_func]
-async fn auth_adduser_fail_because_disabled() {
-    assert_auth_disabled!(con, query!("auth", "adduser", "user"))
-}
-#[sky_macros::dbtest_func(port = 2005, norun = true)]
-async fn auth_adduser_fail_because_anonymous() {
-    assert_auth_perm_error!(con, query!("auth", "adduser", "someuser"))
-}
-// auth adduser okay because root
-#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true)]
-async fn auth_createuser_root_okay() {
-    runmatch!(con, query!("auth", "adduser", "someuser"), Element::String)
-}
-// auth adduser fail because not root
-#[sky_macros::dbtest_func(port = 2005, auth_testuser = true)]
-async fn auth_createuser_testuser_fail() {
-    assert_auth_perm_error!(con, query!("auth", "adduser", "someuser"))
-}
-
-// auth logout
-// auth logout failed because auth is disabled
-#[sky_macros::dbtest_func]
-async fn auth_logout_fail_because_disabled() {
-    assert_auth_disabled!(con, query!("auth", "logout"))
-}
-// auth logout failed because user is anonymous
-#[sky_macros::dbtest_func(port = 2005, norun = true)]
-async fn auth_logout_fail_because_anonymous() {
-    assert_auth_perm_error!(con, query!("auth", "logout"))
-}
-// auth logout okay because the correct user is logged in
-#[sky_macros::dbtest_func(port = 2005, auth_testuser = true, norun = true)]
-async fn auth_logout_okay_testuser() {
-    assert_okay!(con, query!("auth", "logout"))
-}
-// auth logout okay because the correct user is logged in
-#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true, norun = true)]
-async fn auth_logout_okay_rootuser() {
-    assert_okay!(con, query!("auth", "logout"))
-}
-
-// auth deluser
-// auth deluser failed because auth is disabled
-#[sky_macros::dbtest_func]
-async fn auth_deluser_fail_because_auth_disabled() {
-    assert_auth_disabled!(con, query!("auth", "deluser", "testuser"))
-}
-#[sky_macros::dbtest_func(port = 2005, norun = true)]
-async fn auth_deluser_fail_because_anonymous() {
-    assert_auth_perm_error!(con, query!("auth", "deluser", "someuser"))
-}
-// auth deluser failed because not root
-#[sky_macros::dbtest_func(port = 2005, auth_testuser = true)]
-async fn auth_deluser_fail_because_not_root() {
-    assert_auth_perm_error!(con, query!("auth", "deluser", "testuser"))
-}
-// auth deluser okay because root
-#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true)]
-async fn auth_deluser_okay_because_root() {
-    runmatch!(
-        con,
-        query!("auth", "adduser", "supercooluser"),
-        Element::String
-    );
-    assert_okay!(con, query!("auth", "deluser", "supercooluser"))
-}
-
-// restore
-#[sky_macros::dbtest_func]
-async fn restore_fail_because_disabled() {
-    assert_auth_disabled!(con, query!("auth", "restore", "root"));
-}
-#[sky_macros::dbtest_func(port = 2005, auth_testuser = true)]
-async fn restore_fail_because_not_root() {
-    assert_auth_perm_error!(con, query!("auth", "restore", "root"));
-}
-#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true)]
-async fn restore_okay_because_root() {
-    runmatch!(
-        con,
-        query!("auth", "adduser", "supercooldude"),
-        Element::String
-    );
-    runmatch!(
-        con,
-        query!("auth", "restore", "supercooldude"),
-        Element::String
-    );
-}
-#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true, norun = true)]
-async fn restore_okay_with_origin_key() {
-    runmatch!(con, query!("auth", "adduser", "someuser2"), Element::String);
-    // now logout
-    runeq!(
-        con,
-        query!("auth", "logout"),
-        Element::RespCode(RespCode::Okay)
-    );
-    // we should still be able to restore using origin key
-    runmatch!(
-        con,
-        query!("auth", "restore", crate::TEST_AUTH_ORIGIN_KEY, "someuser2"),
-        Element::String
-    );
-}
-
-// auth listuser
-#[sky_macros::dbtest_func]
-async fn listuser_fail_because_disabled() {
-    assert_auth_disabled!(con, query!("auth", "listuser"));
-}
-#[sky_macros::dbtest_func(port = 2005, auth_testuser = true)]
-async fn listuser_fail_because_not_root() {
-    assert_auth_perm_error!(con, query!("auth", "listuser"))
-}
-#[sky_macros::dbtest_func(port = 2005, auth_rootuser = true)]
-async fn listuser_okay_because_root() {
-    let ret: Vec<String> = con.run_query(query!("auth", "listuser")).await.unwrap();
-    assert!(ret.contains(&"root".to_owned()));
-    assert!(ret.contains(&"testuser".to_owned()));
-}
-
-// auth whoami
-#[sky_macros::dbtest_func]
-async fn whoami_fail_because_disabled() {
-    assert_auth_disabled!(con, query!("auth", "whoami"))
-}
-#[sky_macros::dbtest_func(port = 2005, norun = true)]
-async fn whoami_fail_because_anonymous() {
-    assert_auth_perm_error!(con, query!("auth", "whoami"))
-}
-#[sky_macros::dbtest_func(port = 2005, norun = true, auth_testuser = true)]
-async fn auth_whoami_okay_testuser() {
-    runeq!(
-        con,
-        query!("auth", "whoami"),
-        Element::String(testsuite_data::TESTSUITE_TEST_USER.to_owned())
-    )
-}
-
-#[sky_macros::dbtest_func(port = 2005, norun = true, auth_rootuser = true)]
-async fn auth_whoami_okay_rootuser() {
-    runeq!(
-        con,
-        query!("auth", "whoami"),
-        Element::String(testsuite_data::TESTSUITE_ROOT_USER.to_owned())
-    )
-}
-
-mod syntax_checks {
-    use super::{NOAUTH, ONLYAUTH};
-    use crate::auth::provider::testsuite_data::{
-        TESTSUITE_ROOT_TOKEN as ROOT_PASS, TESTSUITE_ROOT_USER as ROOT_USER,
-        TESTSUITE_TEST_TOKEN as PASS, TESTSUITE_TEST_USER as USER,
-    };
-    use skytable::{query, Element, RespCode};
-    macro_rules! assert_authn_aerr {
-        ($con:expr, $query:expr, $username:expr, $password:expr) => {{
-            assert_authn_resp_matrix!(
-                $con,
-                $query,
-                $username,
-                $password,
-                ::skytable::Element::RespCode(::skytable::RespCode::ActionError)
-            )
-        }};
-        ($con:expr, $query:expr) => {{
-            assert_authn_resp_matrix!(
-                $con,
-                $query,
-                NOAUTH,
-                ::skytable::Element::RespCode(::skytable::RespCode::ActionError)
-            )
-        }};
-        ($con:expr, $query:expr, $authnd:ident) => {{
-            assert_authn_resp_matrix!(
-                $con,
-                $query,
-                $authnd,
-                ::skytable::Element::RespCode(::skytable::RespCode::ActionError)
-            );
-        }};
-    }
-    #[sky_macros::dbtest_func(port = 2005, norun = true)]
-    async fn login_aerr() {
-        assert_authn_aerr!(con, query!("auth", "login", "lesserdata"));
-        assert_authn_aerr!(con, query!("auth", "login", "user", "password", "extra"));
-    }
-    #[sky_macros::dbtest_func(port = 2005, norun = true)]
-    async fn claim_aerr() {
-        assert_authn_aerr!(con, query!("auth", "claim"));
-        assert_authn_aerr!(con, query!("auth", "claim", "origin key", "but more data"));
-    }
-    #[sky_macros::dbtest_func(port = 2005, norun = true)]
-    async fn adduser_aerr() {
-        assert_authn_aerr!(
-            con,
-            query!("auth", "adduser", "user", "butextradata"),
-            ONLYAUTH
-        );
-    }
-    #[sky_macros::dbtest_func(port = 2005, norun = true)]
-    async fn logout_aerr() {
-        assert_authn_aerr!(con, query!("auth", "logout", "butextradata"), ONLYAUTH);
-    }
-    #[sky_macros::dbtest_func(port = 2005, norun = true)]
-    async fn deluser_aerr() {
-        assert_authn_aerr!(
-            con,
-            query!("auth", "deluser", "someuser", "butextradata"),
-            ONLYAUTH
-        );
-    }
-    #[sky_macros::dbtest_func(port = 2005, norun = true)]
-    async fn regenerate_aerr() {
-        assert_authn_aerr!(con, query!("auth", "restore"));
-        assert_authn_aerr!(
-            con,
-            query!("auth", "restore", "someuser", "origin", "but extra data")
-        );
-    }
-    #[sky_macros::dbtest_func(port = 2005, norun = true)]
-    async fn listuser_aerr() {
-        assert_authn_aerr!(con, query!("auth", "listuser", "extra argument"), ONLYAUTH);
-    }
-    #[sky_macros::dbtest_func(port = 2005, norun = true)]
-    async fn whoami_aerr() {
-        assert_authn_aerr!(con, query!("auth", "whoami", "extra argument"));
-    }
-    #[sky_macros::dbtest_func(port = 2005, norun = true, auth_testuser = true)]
-    async fn unknown_auth_action() {
-        runeq!(
-            con,
-            query!("auth", "raspberry"),
-            Element::RespCode(RespCode::ErrorString("Unknown action".to_owned()))
-        )
-    }
-}
diff --git a/server/src/tests/ddl_tests.rs b/server/src/tests/ddl_tests.rs
deleted file mode 100644
index 91c02e65..00000000
--- a/server/src/tests/ddl_tests.rs
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Created on Wed Jul 28 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#[sky_macros::dbtest_module]
-mod __private {
-    use {
-        libstress::utils,
-        skytable::{query, types::Array, Element, Query, RespCode},
-    };
-
-    async fn test_create_keyspace() {
-        let mut rng = rand::thread_rng();
-        let ksname = utils::rand_alphastring(10, &mut rng);
-        query.push(format!("create space {ksname}"));
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-    async fn test_drop_keyspace() {
-        let mut rng = rand::thread_rng();
-        let ksname = utils::rand_alphastring(10, &mut rng);
-        query.push(format!("create space {ksname}"));
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        let mut query = Query::new();
-        query.push(format!("drop space {ksname}"));
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-    async fn test_create_table() {
-        let mut rng = rand::thread_rng();
-        let tblname = utils::rand_alphastring(10, &mut rng);
-        query.push(format!("create model {tblname}(string, string)"));
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-    async fn test_create_volatile() {
-        let mut rng = rand::thread_rng();
-        let tblname = utils::rand_alphastring(10, &mut rng);
-        query.push(format!("create model {tblname}(string, string) volatile"));
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-    async fn test_create_table_fully_qualified_entity() {
-        let mut rng = rand::thread_rng();
-        let tblname = utils::rand_alphastring(10, &mut rng);
-        query.push(format!("create model {__MYKS__}.{tblname}(string, string)"));
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-    async fn test_create_table_volatile_fully_qualified_entity() {
-        let mut rng = rand::thread_rng();
-        let tblname = utils::rand_alphastring(10, &mut rng);
-        query.push(format!(
-            "create model {__MYKS__}.{tblname}(string, string) volatile"
-        ));
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-    async fn test_drop_table() {
-        let mut rng = rand::thread_rng();
-        let tblname = utils::rand_alphastring(10, &mut rng);
-        query.push(format!("create model {tblname}(string, string)"));
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        let query = Query::from(format!("drop model {tblname}"));
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-    async fn test_drop_table_fully_qualified_entity() {
-        let mut rng = rand::thread_rng();
-        let tblname = utils::rand_alphastring(10, &mut rng);
-        let my_fqe = __MYKS__.to_owned() + "." + &tblname;
-        query.push(format!("create model {my_fqe}(string, string)"));
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        let query = Query::from(format!("drop model {my_fqe}"));
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-    async fn test_use() {
-        query.push(format!("USE {__MYENTITY__}"));
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        )
-    }
-    async fn test_use_syntax_error() {
-        query.push(format!("USE {__MYENTITY__} wiwofjwjfio"));
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ErrorString("bql-invalid-syntax".into()))
-        )
-    }
-    async fn test_whereami() {
-        query.push("whereami");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::Array(Array::NonNullStr(vec![__MYKS__, __MYTABLE__]))
-        );
-        runeq!(
-            con,
-            query!("use default"),
-            Element::RespCode(RespCode::Okay)
-        );
-        runeq!(
-            con,
-            query!("whereami"),
-            Element::Array(Array::NonNullStr(vec!["default".to_owned()]))
-        );
-        runeq!(
-            con,
-            query!("use default.default"),
-            Element::RespCode(RespCode::Okay)
-        );
-        runeq!(
-            con,
-            query!("whereami"),
-            Element::Array(Array::NonNullStr(vec![
-                "default".to_owned(),
-                "default".to_owned()
-            ]))
-        );
-    }
-}
diff --git a/server/src/tests/inspect_tests.rs b/server/src/tests/inspect_tests.rs
deleted file mode 100644
index 7f33e21c..00000000
--- a/server/src/tests/inspect_tests.rs
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Created on Wed Jul 28 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-const TABLE_DECL_KM_STR_STR_VOLATILE: &str = "Keymap { data:(str,str), volatile:true }";
-
-#[sky_macros::dbtest_module]
-mod __private {
-    use skytable::{types::Array, Element, RespCode};
-    async fn test_inspect_keyspaces() {
-        query.push("INSPECT SPACES");
-        assert!(matches!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::Array(Array::NonNullStr(_))
-        ))
-    }
-    async fn test_inspect_keyspace() {
-        query.push(format!("INSPECT SPACE {__MYKS__}"));
-        assert!(matches!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::Array(Array::NonNullStr(_))
-        ))
-    }
-    async fn test_inspect_current_keyspace() {
-        query.push("INSPECT SPACE");
-        let ret: Vec<String> = con.run_query(&query).await.unwrap();
-        assert!(ret.contains(&__MYTABLE__));
-    }
-    async fn test_inspect_table() {
-        query.push(format!("INSPECT MODEL {__MYTABLE__}"));
-        match con.run_query_raw(&query).await.unwrap() {
-            Element::String(st) => {
-                assert_eq!(st, TABLE_DECL_KM_STR_STR_VOLATILE.to_owned())
-            }
-            _ => panic!("Bad response for inspect table"),
-        }
-    }
-    async fn test_inspect_current_table() {
-        query.push("INSPECT MODEL");
-        let ret: String = con.run_query(&query).await.unwrap();
-        assert_eq!(ret, TABLE_DECL_KM_STR_STR_VOLATILE);
-    }
-    async fn test_inspect_table_fully_qualified_entity() {
-        query.push(format!("INSPECT MODEL {__MYENTITY__}"));
-        match con.run_query_raw(&query).await.unwrap() {
-            Element::String(st) => {
-                assert_eq!(st, TABLE_DECL_KM_STR_STR_VOLATILE.to_owned())
-            }
-            _ => panic!("Bad response for inspect table"),
-        }
-    }
-    async fn test_inspect_keyspaces_syntax_error() {
-        query.push("INSPECT SPACES iowjfjofoe");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ErrorString("bql-invalid-syntax".into()))
-        );
-    }
-    async fn test_inspect_keyspace_syntax_error() {
-        query.push("INSPECT SPACE ijfwijifwjo oijfwirfjwo");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ErrorString("bql-invalid-syntax".into()))
-        );
-    }
-    async fn test_inspect_table_syntax_error() {
-        query.push("INSPECT MODEL ijfwijifwjo oijfwirfjwo");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ErrorString("bql-invalid-syntax".into()))
-        );
-    }
-}
diff --git a/server/src/tests/issue_tests.rs b/server/src/tests/issue_tests.rs
deleted file mode 100644
index 71d7d49a..00000000
--- a/server/src/tests/issue_tests.rs
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Created on Fri Aug 12 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-mod issue_276 {
-    // Refer to the issue here: https://github.com/skytable/skytable/issues/276
-    // Gist of issue: the auth table was cleaned up because the dummy ks in Memstore does not have
-    // the system tables
-    use skytable::{Element, Query, RespCode};
-    #[sky_macros::dbtest_func(port = 2005, auth_testuser = true, skip_if_cfg = "persist-suite")]
-    async fn first_run() {
-        // create the space
-        let r: Element = con
-            .run_query(Query::from("create space please_do_not_vanish"))
-            .await
-            .unwrap();
-        assert_eq!(r, Element::RespCode(RespCode::Okay));
-        // drop the space
-        let r: Element = con
-            .run_query(Query::from("drop space please_do_not_vanish"))
-            .await
-            .unwrap();
-        assert_eq!(r, Element::RespCode(RespCode::Okay));
-    }
-    #[sky_macros::dbtest_func(port = 2005, auth_testuser = true, run_if_cfg = "persist-suite")]
-    async fn second_run() {
-        // this function is just a dummy fn; if, as described in #276, the auth data is indeed
-        // lost, then the server will simply fail to start up
-    }
-}
diff --git a/server/src/tests/kvengine.rs b/server/src/tests/kvengine.rs
deleted file mode 100644
index 728d56a0..00000000
--- a/server/src/tests/kvengine.rs
+++ /dev/null
@@ -1,1205 +0,0 @@
-/*
- * Created on Thu Sep 10 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! Tests for the key/value engine and its operations
-//!
-//! The test functions here might seem slightly _mysterious_ -- but they aren't! The `dbtest_module` macro from the
-//! `sky_macros` crate is what does the magic. It provides each function with an async `stream` to write to.
-//! This stream is connected over TCP to a database instance. Once the test completes, the database instance
-//! and its data is destroyed; but the spawned database instances are started up in a way to not store any
-//! data at all, so this is just a precautionary step.
-
-#[sky_macros::dbtest_module]
-mod __private {
-    #[cfg(test)]
-    use skytable::{types::Array, Element, Query, RespCode};
-    /// Test a HEYA query: The server should return HEY!
-    async fn test_heya() {
-        query.push("heya");
-        let resp = con.run_query_raw(&query).await.unwrap();
-        assert_eq!(resp, Element::String("HEY!".to_owned()));
-    }
-
-    async fn test_heya_echo() {
-        query.push("heya");
-        query.push("sayan");
-        let resp = con.run_query_raw(&query).await.unwrap();
-        assert_eq!(resp, Element::String("sayan".to_owned()));
-    }
-
-    /// Test a GET query: for a non-existing key
-    async fn test_get_single_nil() {
-        query.push("get");
-        query.push("x");
-        let resp = con.run_query_raw(&query).await.unwrap();
-        assert_eq!(resp, Element::RespCode(RespCode::NotFound));
-    }
-
-    /// Test a GET query: for an existing key
-    async fn test_get_single_okay() {
-        query.push("set");
-        query.push("x");
-        query.push("100");
-        let resp = con.run_query_raw(&query).await.unwrap();
-        assert_eq!(resp, Element::RespCode(RespCode::Okay));
-        let mut query = Query::new();
-        query.push("get");
-        query.push("x");
-        let resp = con.run_query_raw(&query).await.unwrap();
-        assert_eq!(resp, Element::String("100".to_owned()));
-    }
-
-    /// Test a GET query with an incorrect number of arguments
-    async fn test_get_syntax_error() {
-        query.push("get");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-        let mut query = Query::new();
-        query.push("get");
-        query.push("x");
-        query.push("y");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test a SET query: SET a non-existing key, which should return code: 0
-    async fn test_set_single_okay() {
-        query.push("sEt");
-        query.push("x");
-        query.push("100");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-
-    /// Test a SET query: SET an existing key, which should return code: 2
-    async fn test_set_single_overwrite_error() {
-        // first set the key
-        query.push("set");
-        query.push("x");
-        query.push("100");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        // attempt the same thing again
-        let mut query = Query::new();
-        query.push("set");
-        query.push("x");
-        query.push("200");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::OverwriteError)
-        );
-    }
-
-    /// Test a SET query with incorrect number of arugments
-    async fn test_set_syntax_error() {
-        query.push("set");
-        query.push("x");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-        let mut query = Query::new();
-        query.push("set");
-        query.push("x");
-        query.push("y");
-        query.push("z");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test an UPDATE query: which should return code: 0
-    async fn test_update_single_okay() {
-        // first set the key
-        query.push("set");
-        query.push("x");
-        query.push("100");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        // attempt to update it
-        let mut query = Query::new();
-        query.push("update");
-        query.push("x");
-        query.push("200");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-
-    /// Test an UPDATE query: which should return code: 1
-    async fn test_update_single_nil() {
-        // attempt to update it
-        query.push("update");
-        query.push("x");
-        query.push("200");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::NotFound)
-        );
-    }
-
-    async fn test_update_syntax_error() {
-        query.push("update");
-        query.push("x");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-        let mut query = Query::new();
-        query.push("update");
-        query.push("x");
-        query.push("y");
-        query.push("z");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test a DEL query: which should return int 0
-    async fn test_del_single_zero() {
-        query.push("del");
-        query.push("x");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(0)
-        );
-    }
-
-    /// Test a DEL query: which should return int 1
-    async fn test_del_single_one() {
-        // first set the key
-        query.push("set");
-        query.push("x");
-        query.push("100");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        // now delete it
-        let mut query = Query::new();
-        query.push("del");
-        query.push("x");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(1)
-        );
-    }
-
-    /// Test a DEL query: which should return the number of keys deleted
-    async fn test_del_multiple() {
-        // first set the keys
-        query.push("mset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-        // now delete them
-        let mut query = Query::new();
-        query.push("del");
-        query.push("x");
-        query.push("y");
-        query.push("z");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-    }
-
-    /// Test a DEL query with an incorrect number of arguments
-    async fn test_del_syntax_error() {
-        query.push("del");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test an EXISTS query
-    async fn test_exists_multiple() {
-        // first set the keys
-        query.push("mset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-        // now check if they exist
-        let mut query = Query::new();
-        query.push("exists");
-        query.push("x");
-        query.push("y");
-        query.push("z");
-        query.push("a");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-    }
-
-    /// Test an EXISTS query with an incorrect number of arguments
-    async fn test_exists_syntax_error() {
-        query.push("exists");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test an MGET query on a single existing key
-    async fn test_mget_multiple_okay() {
-        // first set the keys
-        query.push("mset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-        // now get them
-        let mut query = Query::new();
-        query.push("mget");
-        query.push("x");
-        query.push("y");
-        query.push("z");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::Array(Array::Str(vec![
-                Some("100".to_owned()),
-                Some("200".to_owned()),
-                Some("300".to_owned())
-            ]))
-        );
-    }
-
-    /// Test an MGET query with different outcomes
-    async fn test_mget_multiple_mixed() {
-        // first set the keys
-        query.push("mset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-        let mut query = Query::new();
-        query.push("mget");
-        query.push("x");
-        query.push("y");
-        query.push("a");
-        query.push("z");
-        query.push("b");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::Array(Array::Str(vec![
-                Some("100".to_owned()),
-                Some("200".to_owned()),
-                None,
-                Some("300".to_owned()),
-                None
-            ]))
-        );
-    }
-
-    /// Test an MGET query with an incorrect number of arguments
-    async fn test_mget_syntax_error() {
-        query.push("mget");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test an MSET query with a single non-existing key
-    async fn test_mset_single_okay() {
-        // first set the keys
-        query.push("mset");
-        query.push("x");
-        query.push("100");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(1)
-        );
-    }
-
-    /// Test an MSET query with non-existing keys
-    async fn test_mset_multiple_okay() {
-        // first set the keys
-        query.push("mset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-    }
-
-    /// Test an MSET query with a mixed set of outcomes
-    async fn test_mset_multiple_mixed() {
-        // first set the keys
-        query.push("mset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-        // now try to set them again with just another new key
-        let mut query = Query::new();
-        query.push("mset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        query.push("a");
-        query.push("apple");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(1)
-        );
-    }
-
-    /// Test an MSET query with the wrong number of arguments
-    async fn test_mset_syntax_error_args_one() {
-        query.push("mset");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-    async fn test_mset_syntax_error_args_three() {
-        query.push("mset");
-        query.push("x");
-        query.push("y");
-        query.push("z");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test an MUPDATE query with a single non-existing key
-    async fn test_mupdate_single_okay() {
-        // first set the key
-        query.push("mset");
-        query.push("x");
-        query.push("100");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(1)
-        );
-        // now attempt to update it
-        // first set the keys
-        let mut query = Query::new();
-        query.push("mupdate");
-        query.push("x");
-        query.push("200");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(1)
-        );
-    }
-
-    /// Test an MUPDATE query with a mixed set of outcomes
-    async fn test_mupdate_multiple_mixed() {
-        // first set the keys
-        query.push("mset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-        // now try to update them with just another new key
-        let mut query = Query::new();
-        query.push("mupdate");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        query.push("a");
-        query.push("apple");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-    }
-
-    /// Test an MUPDATE query with the wrong number of arguments
-    async fn test_mupdate_syntax_error_args_one() {
-        query.push("mupdate");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    async fn test_mupdate_syntax_error_args_three() {
-        query.push("mupdate");
-        query.push("x");
-        query.push("y");
-        query.push("z");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test an SSET query: which should return code: 0
-    async fn test_sset_single_okay() {
-        // first set the keys
-        query.push("sset");
-        query.push("x");
-        query.push("100");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-
-    /// Test an SSET query: which should return code: 2
-    async fn test_sset_single_overwrite_error() {
-        // first set the keys
-        query.push("set");
-        query.push("x");
-        query.push("100");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        // now attempt to overwrite it
-        let mut query = Query::new();
-        query.push("sset");
-        query.push("x");
-        query.push("100");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::OverwriteError)
-        );
-    }
-
-    /// Test an SSET query: which should return code: 0
-    async fn test_sset_multiple_okay() {
-        // first set the keys
-        query.push("sset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-
-    /// Test an SSET query: which should return code: 2
-    async fn test_sset_multiple_overwrite_error() {
-        // first set the keys
-        query.push("sset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        // now attempt to sset again with just one new extra key
-        let mut query = Query::new();
-        query.push("sset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("b");
-        query.push("bananas");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::OverwriteError)
-        );
-    }
-
-    /// Test an SSET query with the wrong number of arguments
-    async fn test_sset_syntax_error_args_one() {
-        query.push("sset");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    async fn test_sset_syntax_error_args_three() {
-        query.push("sset");
-        query.push("x");
-        query.push("y");
-        query.push("z");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test an SUPDATE query: which should return code: 0
-    async fn test_supdate_single_okay() {
-        // set the key
-        query.push("sset");
-        query.push("x");
-        query.push("100");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        // update it
-        let mut query = Query::new();
-        query.push("supdate");
-        query.push("x");
-        query.push("200");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-
-    /// Test an SUPDATE query: which should return code: 1
-    async fn test_supdate_single_nil() {
-        query.push("supdate");
-        query.push("x");
-        query.push("200");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::NotFound)
-        );
-    }
-
-    /// Test an SUPDATE query: which should return code: 0
-    async fn test_supdate_multiple_okay() {
-        // first set the keys
-        query.push("sset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        // now update all of them
-        let mut query = Query::new();
-        query.push("supdate");
-        query.push("x");
-        query.push("200");
-        query.push("y");
-        query.push("300");
-        query.push("z");
-        query.push("400");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-
-    async fn test_supdate_multiple_nil() {
-        // no keys exist, so we get a nil
-        query.push("supdate");
-        query.push("x");
-        query.push("200");
-        query.push("y");
-        query.push("300");
-        query.push("z");
-        query.push("400");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::NotFound)
-        );
-    }
-
-    /// Test an SUPDATE query with the wrong number of arguments
-    async fn test_supdate_syntax_error_args_one() {
-        query.push("mupdate");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    async fn test_supdate_syntax_error_args_three() {
-        query.push("mupdate");
-        query.push("x");
-        query.push("y");
-        query.push("z");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test an SDEL query: which should return nil
-    async fn test_sdel_single_nil() {
-        query.push("sdel");
-        query.push("x");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::NotFound)
-        );
-    }
-
-    /// Test an SDEL query: which should return okay
-    async fn test_sdel_single_okay() {
-        query.push("sset");
-        query.push("x");
-        query.push("100");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        let mut query = Query::new();
-        query.push("sdel");
-        query.push("x");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-
-    /// Test an SDEL query: which should return okay
-    async fn test_sdel_multiple_okay() {
-        // first set the keys
-        query.push("sset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        // now delete them
-        let mut query = Query::new();
-        query.push("sdel");
-        query.push("x");
-        query.push("y");
-        query.push("z");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-    }
-
-    async fn test_sdel_multiple_nil() {
-        query.push("sdel");
-        query.push("x");
-        query.push("y");
-        query.push("z");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::NotFound)
-        );
-    }
-
-    /// Test an SDEL query with an incorrect number of arguments
-    async fn test_sdel_syntax_error() {
-        query.push("sdel");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test a `DBSIZE` query
-    async fn test_dbsize() {
-        // first set the keys
-        query.push("sset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        // now check the size
-        let mut query = Query::new();
-        query.push("dbsize");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-        let mut query = Query::new();
-        query.push("dbsize");
-        query.push(__MYENTITY__);
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-    }
-
-    /// Test `DBSIZE` with an incorrect number of arguments
-    async fn test_dbsize_syntax_error() {
-        query.push("dbsize");
-        query.push("iroegjoeijgor");
-        query.push("roigjoigjj094");
-        query.push("ioewjforfifrj");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test `FLUSHDB`
-    async fn test_flushdb_okay() {
-        // first set the keys
-        query.push("sset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        // now flush the database
-        let mut query = Query::new();
-        query.push("flushdb");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        // now check the size
-        let mut query = Query::new();
-        query.push("dbsize");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(0)
-        );
-    }
-
-    async fn test_flushdb_fqe() {
-        setkeys!(
-            con,
-            "w":"000",
-            "x":"100",
-            "y":"200",
-            "z":"300"
-        );
-        // now flush the database
-        let mut query = Query::new();
-        query.push("flushdb");
-        // add the fqe
-        query.push(__MYENTITY__);
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        // now check the size
-        let mut query = Query::new();
-        query.push("dbsize");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(0)
-        );
-    }
-
-    /// Test `FLUSHDB` with an incorrect number of arguments
-    async fn test_flushdb_syntax_error() {
-        query.push("flushdb");
-        query.push("x");
-        query.push("y");
-        query.push("z");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test `USET` which returns okay
-    ///
-    /// `USET` almost always returns okay for the correct number of key(s)/value(s)
-    async fn test_uset_all_okay() {
-        query.push("uset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-        // now that the keys already exist, do it all over again
-        let mut query = Query::new();
-        query.push("uset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(3)
-        );
-    }
-
-    /// Test `USET` with an incorrect number of arguments
-    async fn test_uset_syntax_error_args_one() {
-        query.push("uset");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    async fn test_uset_syntax_error_args_three() {
-        query.push("uset");
-        query.push("one");
-        query.push("two");
-        query.push("three");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    /// Test `KEYLEN`
-    async fn test_keylen() {
-        // first set the key
-        query.push("set");
-        query.push("x");
-        query.push("helloworld");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::Okay)
-        );
-        // now check for the length
-        let mut query = Query::new();
-        query.push("keylen");
-        query.push("x");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(10)
-        );
-    }
-
-    /// Test `KEYLEN` with an incorrect number of arguments
-    async fn test_keylen_syntax_error_args_one() {
-        query.push("keylen");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-    async fn test_keylen_syntax_error_args_two() {
-        query.push("keylen");
-        query.push("x");
-        query.push("y");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-    async fn test_mksnap_disabled() {
-        query.push("mksnap");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ErrorString("err-snapshot-disabled".to_owned()))
-        );
-    }
-    async fn test_mksnap_sanitization() {
-        query.push("mksnap");
-        query.push("/var/omgcrazysnappy");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ErrorString(
-                "err-invalid-snapshot-name".to_owned()
-            ))
-        );
-        let mut query = Query::new();
-        query.push("mksnap");
-        query.push("../omgbacktoparent");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ErrorString(
-                "err-invalid-snapshot-name".to_owned()
-            ))
-        );
-    }
-    async fn test_lskeys_default() {
-        query.push("uset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        query.push("a");
-        query.push("apples");
-        query.push("b");
-        query.push("burgers");
-        query.push("c");
-        query.push("carrots");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(6)
-        );
-        // now get 'em
-        let mut query = Query::new();
-        query.push("lskeys");
-        let ret = con.run_query_raw(&query).await.unwrap();
-        // don't forget that the keys returned are arranged according to their hashes
-        let ret_should_have: Vec<String> = vec!["a", "b", "c", "x", "y", "z"]
-            .into_iter()
-            .map(|element| element.to_owned())
-            .collect();
-        if let Element::Array(Array::NonNullStr(arr)) = ret {
-            assert_eq!(ret_should_have.len(), arr.len());
-            assert!(ret_should_have.into_iter().all(|key| arr.contains(&key)));
-        } else {
-            panic!("Expected flat string array");
-        }
-    }
-    async fn test_lskeys_custom_limit() {
-        query.push("uset");
-        query.push("x");
-        query.push("100");
-        query.push("y");
-        query.push("200");
-        query.push("z");
-        query.push("300");
-        query.push("a");
-        query.push("apples");
-        query.push("b");
-        query.push("burgers");
-        query.push("c");
-        query.push("carrots");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::UnsignedInt(6)
-        );
-        let mut query = Query::new();
-        query.push("lskeys");
-        query.push("1000");
-        let ret = con.run_query_raw(&query).await.unwrap();
-        // don't forget that the keys returned are arranged according to their hashes
-        let ret_should_have: Vec<String> = vec!["a", "b", "c", "x", "y", "z"]
-            .into_iter()
-            .map(|element| element.to_owned())
-            .collect();
-        if let Element::Array(Array::NonNullStr(arr)) = ret {
-            assert_eq!(ret_should_have.len(), arr.len());
-            assert!(ret_should_have.into_iter().all(|key| arr.contains(&key)));
-        } else {
-            panic!("Expected flat string array");
-        }
-    }
-    async fn test_lskeys_entity() {
-        setkeys!(
-            con,
-            "x":"100",
-            "y":"200",
-            "z":"300"
-        );
-        query.push("lskeys");
-        query.push(&__MYENTITY__);
-        let ret = con.run_query_raw(&query).await.unwrap();
-        let ret_should_have: Vec<String> = vec!["x", "y", "z"]
-            .into_iter()
-            .map(|element| element.to_owned())
-            .collect();
-        if let Element::Array(Array::NonNullStr(arr)) = ret {
-            assert_eq!(ret_should_have.len(), arr.len());
-            assert!(ret_should_have.into_iter().all(|key| arr.contains(&key)));
-        } else {
-            panic!("Expected flat string array");
-        }
-    }
-    async fn test_lskeys_entity_with_count() {
-        setkeys!(
-            con,
-            "x":"100",
-            "y":"200",
-            "z":"300"
-        );
-        query.push("lskeys");
-        query.push(&__MYENTITY__);
-        query.push(3u8.to_string());
-        let ret = con.run_query_raw(&query).await.unwrap();
-        let ret_should_have: Vec<String> = vec!["x", "y", "z"]
-            .into_iter()
-            .map(|element| element.to_owned())
-            .collect();
-        if let Element::Array(Array::NonNullStr(arr)) = ret {
-            assert_eq!(ret_should_have.len(), arr.len());
-            assert!(ret_should_have.into_iter().all(|key| arr.contains(&key)));
-        } else {
-            panic!("Expected flat string array");
-        }
-    }
-    async fn test_lskeys_syntax_error() {
-        query.push("lskeys");
-        query.push("abcdefg");
-        query.push("hijklmn");
-        query.push("riufrif");
-        query.push("fvnjnvv");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-    async fn test_mpop_syntax_error() {
-        query.push("mpop");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-
-    async fn test_mpop_all_success() {
-        setkeys!(
-            con,
-            "x":"100",
-            "y":"200",
-            "z":"300"
-        );
-        query.push(vec!["mpop", "x", "y", "z"]);
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::Array(Array::Str(vec![
-                Some("100".to_owned()),
-                Some("200".to_owned()),
-                Some("300".to_owned())
-            ]))
-        )
-    }
-    async fn test_mpop_mixed() {
-        setkeys!(
-            con,
-            "x":"100",
-            "y":"200",
-            "z":"300"
-        );
-        query.push(vec!["mpop", "apple", "arnold", "x", "madonna", "y", "z"]);
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::Array(Array::Str(vec![
-                None,
-                None,
-                Some("100".to_owned()),
-                None,
-                Some("200".to_owned()),
-                Some("300".to_owned())
-            ]))
-        );
-    }
-    async fn test_pop_syntax_error() {
-        query.push("pop");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::ActionError)
-        );
-    }
-    async fn test_pop_okay() {
-        setkeys!(
-            con,
-            "x":"100"
-        );
-        query.push("pop");
-        query.push("x");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::String("100".to_owned())
-        );
-    }
-    async fn test_pop_nil() {
-        query.push("pop");
-        query.push("x");
-        assert_eq!(
-            con.run_query_raw(&query).await.unwrap(),
-            Element::RespCode(RespCode::NotFound)
-        );
-    }
-}
diff --git a/server/src/tests/kvengine_encoding.rs b/server/src/tests/kvengine_encoding.rs
deleted file mode 100644
index e1cd59a9..00000000
--- a/server/src/tests/kvengine_encoding.rs
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Created on Sun Sep 05 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#[sky_macros::dbtest_module(table = "(string, string)")]
-mod __private {
-    use skytable::{types::RawString, Element, RespCode};
-
-    async fn test_bad_encoding_set() {
-        query.push("set");
-        query.push("x");
-        query.push(RawString::from(b"Hello \xF0\x90\x80World".to_vec()));
-        runeq!(con, query, Element::RespCode(RespCode::EncodingError));
-    }
-    async fn test_bad_encoding_update() {
-        // first set the keys
-        setkeys! {
-            con,
-            "x": "100"
-        }
-        // now try to update with a bad value
-        query.push("update");
-        query.push("x");
-        query.push(RawString::from(b"Hello \xF0\x90\x80World".to_vec()));
-        runeq!(con, query, Element::RespCode(RespCode::EncodingError));
-    }
-    async fn test_bad_encoding_uset() {
-        query.push("uset");
-        query.push("x");
-        query.push(RawString::from(b"Hello \xF0\x90\x80World".to_vec()));
-        runeq!(con, query, Element::RespCode(RespCode::EncodingError));
-    }
-    async fn test_bad_encoding_mset() {
-        // we'll have one good encoding and one bad encoding
-        push!(
-            query,
-            "mset",
-            "x",
-            "good value",
-            "y",
-            // the bad value
-            RawString::from(b"Hello \xF0\x90\x80World".to_vec())
-        );
-        runeq!(con, query, Element::RespCode(RespCode::EncodingError));
-    }
-}
diff --git a/server/src/tests/kvengine_list.rs b/server/src/tests/kvengine_list.rs
deleted file mode 100644
index d1eda1cd..00000000
--- a/server/src/tests/kvengine_list.rs
+++ /dev/null
@@ -1,451 +0,0 @@
-/*
- * Created on Tue Sep 07 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-macro_rules! lset {
-    ($con:expr, $listname:expr, $($val:expr),*) => {
-        let mut q = skytable::Query::from("LSET");
-        q.push($listname);
-        $(q.push($val);)*
-        runeq!($con, q, skytable::Element::RespCode(skytable::RespCode::Okay));
-    };
-    ($con:expr, $listname:expr) => {
-        lset!($con, $listname, )
-    }
-}
-
-#[sky_macros::dbtest_module(table = "(string,list<string>)")]
-mod __private {
-    use skytable::{query, types::Array, Element, RespCode};
-
-    // lset tests
-    async fn test_lset_empty_okay() {
-        lset!(con, "mylist");
-    }
-    async fn test_lset_with_values() {
-        lset!(con, "mylist", "a", "b", "c", "d");
-    }
-    async fn test_lset_syntax_error() {
-        let q = query!("LSET");
-        runeq!(con, q, Element::RespCode(RespCode::ActionError));
-    }
-    async fn test_lset_overwrite_error() {
-        lset!(con, "mylist");
-        let q = query!("lset", "mylist");
-        runeq!(con, q, Element::RespCode(RespCode::OverwriteError));
-    }
-
-    // lget tests
-    async fn test_lget_emptylist_okay() {
-        lset!(con, "mysuperlist");
-        let q = query!("lget", "mysuperlist");
-        runeq!(con, q, Element::Array(Array::NonNullStr(vec![])));
-    }
-    async fn test_lget_list_with_elements_okay() {
-        lset!(con, "mysuperlist", "elementa", "elementb", "elementc");
-        let q = query!("lget", "mysuperlist");
-        assert_skyhash_arrayeq!(str, con, q, "elementa", "elementb", "elementc");
-    }
-    /// lget limit
-    async fn test_lget_list_with_limit() {
-        lset!(con, "mysuperlist", "elementa", "elementb", "elementc");
-        let q = query!("lget", "mysuperlist", "LIMIT", "2");
-        assert_skyhash_arrayeq!(str, con, q, "elementa", "elementb");
-    }
-    /// lget bad limit
-    async fn test_lget_list_with_bad_limit() {
-        lset!(con, "mysuperlist", "elementa", "elementb", "elementc");
-        let q = query!("lget", "mylist", "LIMIT", "badlimit");
-        runeq!(con, q, Element::RespCode(RespCode::Wrongtype));
-    }
-    /// lget huge limit
-    async fn test_lget_with_huge_limit() {
-        lset!(con, "mysuperlist", "elementa", "elementb", "elementc");
-        let q = query!("lget", "mysuperlist", "LIMIT", "100");
-        assert_skyhash_arrayeq!(str, con, q, "elementa", "elementb", "elementc");
-    }
-    /// lget syntax error
-    async fn test_lget_with_limit_syntax_error() {
-        let q = query!("lget", "mylist", "LIMIT", "100", "200");
-        runeq!(con, q, Element::RespCode(RespCode::ActionError));
-    }
-    /// lget limit non-existent key
-    async fn test_lget_with_limit_nil() {
-        let q = query!("lget", "mylist", "LIMIT", "100");
-        runeq!(con, q, Element::RespCode(RespCode::NotFound));
-    }
-    /// lget len
-    async fn test_lget_with_len_okay() {
-        lset!(con, "mysuperlist", "elementa", "elementb", "elementc");
-        let q = query!("lget", "mysuperlist", "len");
-        runeq!(con, q, Element::UnsignedInt(3));
-    }
-    /// lget len syntax error
-    async fn test_lget_with_len_syntax_error() {
-        let q = query!("lget", "mysuperlist", "len", "whatthe");
-        runeq!(con, q, Element::RespCode(RespCode::ActionError));
-    }
-    /// lget len nil
-    async fn test_lget_with_len_nil() {
-        let q = query!("lget", "mysuperlist", "len");
-        runeq!(con, q, Element::RespCode(RespCode::NotFound));
-    }
-    /// lget valueat
-    async fn test_lget_with_valueat_okay() {
-        lset!(con, "mylist", "v1");
-        let q = query!("lget", "mylist", "valueat", "0");
-        runeq!(con, q, Element::String("v1".to_owned()));
-    }
-    /// lget valueat (non-existent index)
-    async fn test_lget_with_valueat_non_existent_index() {
-        lset!(con, "mylist", "v1");
-        let q = query!("lget", "mylist", "valueat", "1");
-        runeq!(
-            con,
-            q,
-            Element::RespCode(RespCode::ErrorString("bad-list-index".to_owned()))
-        )
-    }
-    /// lget valueat (invalid index)
-    async fn test_lget_with_valueat_bad_index() {
-        lset!(con, "mylist", "v1");
-        let q = query!("lget", "mylist", "valueat", "1a");
-        runeq!(con, q, Element::RespCode(RespCode::Wrongtype))
-    }
-    /// lget valueat (nil)
-    async fn test_lget_with_valueat_nil() {
-        let q = query!("lget", "mybadlist", "valueat", "2");
-        runeq!(con, q, Element::RespCode(RespCode::NotFound));
-    }
-    /// lget valueat (nil + bad index)
-    async fn test_lget_with_bad_index_but_nil_key() {
-        let q = query!("lget", "mybadlist", "valueat", "2a");
-        runeq!(con, q, Element::RespCode(RespCode::Wrongtype));
-    }
-    /// lget valueat (syntax error)
-    async fn test_lget_with_valueat_syntax_error() {
-        let q = query!("lget", "mybadlist", "valueat", "2", "3");
-        runeq!(con, q, Element::RespCode(RespCode::ActionError));
-    }
-    // lget last
-    /// lget last with one element
-    async fn test_lget_last_with_last_one_element() {
-        lset!(con, "mylist", "a");
-        let q = query!("lget", "mylist", "last");
-        runeq!(con, q, Element::String("a".to_owned()));
-    }
-    /// lget last with multiple elements
-    async fn test_lget_last_with_last_many_elements() {
-        lset!(con, "mylist", "a", "b", "c");
-        let q = query!("lget", "mylist", "last");
-        runeq!(con, q, Element::String("c".to_owned()));
-    }
-    /// lget last with empty list
-    async fn test_lget_last_with_empty_list() {
-        lset!(con, "mylist");
-        let q = query!("lget", "mylist", "last");
-        runeq!(
-            con,
-            q,
-            Element::RespCode(RespCode::ErrorString("list-is-empty".to_owned()))
-        );
-    }
-    /// lget last syntax error
-    async fn test_lget_last_syntax_error() {
-        let q = query!("lget", "mylist", "last", "abcd");
-        runeq!(con, q, Element::RespCode(RespCode::ActionError));
-    }
-    // lget first
-    /// lget first with one element
-    async fn test_lget_first_with_last_one_element() {
-        lset!(con, "mylist", "a");
-        let q = query!("lget", "mylist", "first");
-        runeq!(con, q, Element::String("a".to_owned()));
-    }
-    /// lget first with multiple elements
-    async fn test_lget_first_with_last_many_elements() {
-        lset!(con, "mylist", "a", "b", "c");
-        let q = query!("lget", "mylist", "first");
-        runeq!(con, q, Element::String("a".to_owned()));
-    }
-    /// lget first with empty list
-    async fn test_lget_first_with_empty_list() {
-        lset!(con, "mylist");
-        let q = query!("lget", "mylist", "first");
-        runeq!(
-            con,
-            q,
-            Element::RespCode(RespCode::ErrorString("list-is-empty".to_owned()))
-        );
-    }
-    /// lget last syntax error
-    async fn test_lget_first_syntax_error() {
-        let q = query!("lget", "mylist", "first", "abcd");
-        runeq!(con, q, Element::RespCode(RespCode::ActionError));
-    }
-    // lmod tests
-    // lmod push
-    /// lmod push (okay)
-    async fn test_lmod_push_okay() {
-        lset!(con, "mylist");
-        let q = query!("lmod", "mylist", "push", "v1");
-        runeq!(con, q, Element::RespCode(RespCode::Okay));
-    }
-    /// lmod push multiple (okay)
-    async fn test_lmod_push_multiple_okay() {
-        lset!(con, "mylist");
-        assert_okay!(con, query!("lmod", "mylist", "push", "v1", "v2"));
-    }
-    /// lmod push (nil)
-    async fn test_lmod_push_nil() {
-        let q = query!("lmod", "mylist", "push", "v1");
-        runeq!(con, q, Element::RespCode(RespCode::NotFound));
-    }
-    /// lmod push (syntax error)
-    async fn test_lmod_syntax_error() {
-        let q = query!("lmod", "mylist", "push");
-        runeq!(con, q, Element::RespCode(RespCode::ActionError));
-    }
-    // lmod pop
-    /// lmod pop (okay)
-    async fn test_lmod_pop_noindex_okay() {
-        lset!(con, "mylist", "value");
-        let q = query!("lmod", "mylist", "pop");
-        runeq!(con, q, Element::String("value".to_owned()));
-    }
-    /// lmod pop (good index; okay)
-    async fn test_lmod_pop_goodindex_okay() {
-        lset!(con, "mylist", "value1", "value2");
-        let q = query!("lmod", "mylist", "pop", "1");
-        runeq!(con, q, Element::String("value2".to_owned()));
-    }
-    /// lmod pop (bad index + existent key & non-existent key)
-    async fn test_lmod_pop_badindex_fail() {
-        lset!(con, "mylist", "v1", "v2");
-        let q = query!("lmod", "mylist", "pop", "12badidx");
-        runeq!(con, q, Element::RespCode(RespCode::Wrongtype));
-
-        // this is post-execution; so the error must be pointed out first
-        let q = query!("lmod", "mymissinglist", "pop", "12badidx");
-        runeq!(con, q, Element::RespCode(RespCode::Wrongtype));
-    }
-    /// lmod pop (nil)
-    async fn test_lmod_pop_nil() {
-        let q = query!("lmod", "mylist", "pop");
-        runeq!(con, q, Element::RespCode(RespCode::NotFound));
-    }
-    /// lmod pop (syntax error)
-    async fn test_lmod_pop_syntax_error() {
-        let q = query!("lmod", "mylist", "pop", "whatthe", "whatthe2");
-        runeq!(con, q, Element::RespCode(RespCode::ActionError));
-    }
-    // lmod clear
-    /// lmod clear (okay)
-    async fn test_lmod_clear_okay() {
-        lset!(con, "mylist", "v1", "v2");
-        let q = query!("lmod", "mylist", "clear");
-        runeq!(con, q, Element::RespCode(RespCode::Okay));
-    }
-    /// lmod clear (nil)
-    async fn test_lmod_clear_nil() {
-        let q = query!("lmod", "mylist", "clear");
-        runeq!(con, q, Element::RespCode(RespCode::NotFound));
-    }
-    /// lmod clear (syntax error)
-    async fn test_lmod_clear_syntax_error() {
-        let q = query!("lmod", "mylist", "clear", "unneeded arg");
-        runeq!(con, q, Element::RespCode(RespCode::ActionError));
-    }
-    // lmod remove
-    /// lmod remove (okay)
-    async fn test_lmod_remove_okay() {
-        lset!(con, "mylist", "v1");
-        let q = query!("lmod", "mylist", "remove", "0");
-        runeq!(con, q, Element::RespCode(RespCode::Okay));
-    }
-    /// lmod remove (nil)
-    async fn test_lmod_remove_nil() {
-        let q = query!("lmod", "mylist", "remove", "0");
-        runeq!(con, q, Element::RespCode(RespCode::NotFound));
-    }
-    /// lmod remove (bad index; nil + existent)
-    async fn test_lmod_remove_bad_index() {
-        // non-existent key + bad idx
-        let q = query!("lmod", "mylist", "remove", "1a");
-        runeq!(con, q, Element::RespCode(RespCode::Wrongtype));
-        // existent key + bad idx
-        lset!(con, "mylist");
-        let q = query!("lmod", "mylist", "remove", "1a");
-        runeq!(con, q, Element::RespCode(RespCode::Wrongtype));
-    }
-    /// lmod remove (syntax error)
-    async fn test_lmod_remove_syntax_error() {
-        let q = query!("lmod", "mylist", "remove", "a", "b");
-        runeq!(con, q, Element::RespCode(RespCode::ActionError));
-    }
-    // lmod insert
-    /// lmod insert (okay)
-    async fn test_lmod_insert_okay() {
-        lset!(con, "mylist", "a", "c");
-        let q = query!("lmod", "mylist", "insert", "1", "b");
-        runeq!(con, q, Element::RespCode(RespCode::Okay));
-        let q = query!("lget", "mylist");
-        assert_skyhash_arrayeq!(str, con, q, "a", "b", "c");
-    }
-    /// lmod insert (bad index; present + nil)
-    async fn test_lmod_insert_bad_index() {
-        // nil
-        let q = query!("lmod", "mylist", "insert", "1badindex", "b");
-        runeq!(con, q, Element::RespCode(RespCode::Wrongtype));
-        // present
-        lset!(con, "mylist", "a", "c");
-        let q = query!("lmod", "mylist", "insert", "1badindex", "b");
-        runeq!(con, q, Element::RespCode(RespCode::Wrongtype));
-    }
-    /// lmod insert (syntax error)
-    async fn test_lmod_insert_syntax_error() {
-        let q = query!("lmod", "mylist", "insert", "1");
-        runeq!(con, q, Element::RespCode(RespCode::ActionError));
-        let q = query!("lmod", "mylist", "insert");
-        runeq!(con, q, Element::RespCode(RespCode::ActionError));
-    }
-    /// lmod insert (present; non-existent index)
-    async fn test_lmod_insert_non_existent_index() {
-        lset!(con, "mylist", "a", "b");
-        let q = query!(
-            "lmod",
-            "mylist",
-            "insert",
-            "125",
-            "my-value-that-will-never-go-in"
-        );
-        runeq!(
-            con,
-            q,
-            Element::RespCode(RespCode::ErrorString("bad-list-index".to_owned()))
-        )
-    }
-    /// del <list> (existent; non-existent)
-    async fn test_list_del() {
-        // try an existent key
-        lset!(con, "mylist", "v1", "v2");
-        let q = query!("del", "mylist");
-        runeq!(con, q, Element::UnsignedInt(1));
-        // try the now non-existent key
-        let q = query!("del", "mylist");
-        runeq!(con, q, Element::UnsignedInt(0));
-    }
-    /// exists <list> (existent; non-existent)
-    async fn test_list_exists() {
-        lset!(con, "mylist");
-        lset!(con, "myotherlist");
-        let q = query!("exists", "mylist", "myotherlist", "badlist");
-        runeq!(con, q, Element::UnsignedInt(2));
-    }
-
-    // tests for range
-    async fn test_list_range_nil() {
-        let q = query!("lget", "sayan", "range", "1", "10");
-        runeq!(con, q, Element::RespCode(RespCode::NotFound));
-        let q = query!("lget", "sayan", "range", "1");
-        runeq!(con, q, Element::RespCode(RespCode::NotFound));
-    }
-
-    async fn test_list_range_bounded_okay() {
-        lset!(con, "mylist", "1", "2", "3", "4", "5");
-        let q = query!("lget", "mylist", "range", "0", "5");
-        assert_skyhash_arrayeq!(str, con, q, "1", "2", "3", "4", "5");
-    }
-
-    async fn test_list_range_bounded_fail() {
-        lset!(con, "mylist", "1", "2", "3", "4", "5");
-        let q = query!("lget", "mylist", "range", "0", "165");
-        runeq!(
-            con,
-            q,
-            Element::RespCode(RespCode::ErrorString("bad-list-index".to_owned()))
-        )
-    }
-
-    async fn test_list_range_unbounded_okay() {
-        lset!(con, "mylist", "1", "2", "3", "4", "5");
-        let q = query!("lget", "mylist", "range", "0");
-        assert_skyhash_arrayeq!(str, con, q, "1", "2", "3", "4", "5");
-    }
-
-    async fn test_list_range_unbounded_fail() {
-        lset!(con, "mylist", "1", "2", "3", "4", "5");
-        let q = query!("lget", "mylist", "range", "165");
-        runeq!(
-            con,
-            q,
-            Element::RespCode(RespCode::ErrorString("bad-list-index".to_owned()))
-        )
-    }
-
-    async fn test_list_range_parse_fail() {
-        let q = query!("lget", "mylist", "range", "1", "2a");
-        runeq!(con, q, Element::RespCode(RespCode::Wrongtype));
-        let q = query!("lget", "mylist", "range", "2a");
-        runeq!(con, q, Element::RespCode(RespCode::Wrongtype));
-        // now do the same with an existing key
-        lset!(con, "mylist", "a", "b", "c");
-        let q = query!("lget", "mylist", "range", "1", "2a");
-        runeq!(con, q, Element::RespCode(RespCode::Wrongtype));
-        let q = query!("lget", "mylist", "range", "2a");
-        runeq!(con, q, Element::RespCode(RespCode::Wrongtype));
-    }
-
-    // sanity tests
-    async fn test_get_model_error() {
-        query.push("GET");
-        query.push("mylist");
-        runeq!(
-            con,
-            query,
-            Element::RespCode(RespCode::ErrorString("wrong-model".to_owned()))
-        );
-    }
-    async fn test_set_model_error() {
-        query.push("SET");
-        query.push("mylist");
-        query.push("myvalue");
-        runeq!(
-            con,
-            query,
-            Element::RespCode(RespCode::ErrorString("wrong-model".to_owned()))
-        );
-    }
-    async fn test_update_model_error() {
-        query.push("UPDATE");
-        query.push("mylist");
-        query.push("myvalue");
-        runeq!(
-            con,
-            query,
-            Element::RespCode(RespCode::ErrorString("wrong-model".to_owned()))
-        );
-    }
-}
diff --git a/server/src/tests/macros.rs b/server/src/tests/macros.rs
deleted file mode 100644
index 14d8f8e1..00000000
--- a/server/src/tests/macros.rs
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Created on Sun Sep 05 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-macro_rules! setkeys {
-    ($con:ident, $($key:literal:$value:literal),*) => {
-        let mut q = skytable::Query::new();
-        q.push("MSET");
-        let mut count = 0;
-        $(
-            q.push($key);
-            q.push($value);
-            count += 1;
-        )*
-        assert_eq!(
-            $con.run_query_raw(&q).await.unwrap(),
-            Element::UnsignedInt(count)
-        );
-    };
-    ($con:ident, $($key:expr => $value:expr),*) => {
-        let mut q = ::skytable::Query::new();
-        q.push("MSET");
-        let mut count = 0;
-        $(
-            q.push($key);
-            q.push($value);
-            count += 1;
-        )*
-        assert_eq!(
-            $con.run_query_raw(&q).await.unwrap(),
-            ::skytable::Element::UnsignedInt(count)
-        );
-    };
-}
-
-macro_rules! switch_entity {
-    ($con:expr, $entity:expr) => {
-        runeq!(
-            $con,
-            ::skytable::query!(format!("use {}", $entity)),
-            ::skytable::Element::RespCode(::skytable::RespCode::Okay)
-        )
-    };
-}
-
-macro_rules! create_table_and_switch {
-    ($con:expr, $table:expr, $decl:expr) => {{
-        runeq!(
-            $con,
-            ::skytable::query!(format!("create model {}{}", $table, $decl)),
-            ::skytable::Element::RespCode(::skytable::RespCode::Okay)
-        );
-        switch_entity!($con, $table);
-    }};
-}
-
-macro_rules! push {
-    ($query:expr, $($val:expr),*) => {{
-        $(
-            $query.push($val);
-        )*
-    }};
-}
-
-macro_rules! runeq {
-    ($con:expr, $query:expr, $eq:expr) => {
-        assert_eq!($con.run_query_raw(&$query).await.unwrap(), $eq)
-    };
-}
-
-macro_rules! runmatch {
-    ($con:expr, $query:expr, $match:path) => {{
-        let ret = $con.run_query_raw(&$query).await.unwrap();
-        assert!(matches!(ret, $match(_)))
-    }};
-}
-
-macro_rules! assert_okay {
-    ($con:expr, $query:expr) => {
-        assert_respcode!($con, $query, ::skytable::RespCode::Okay)
-    };
-}
-
-macro_rules! assert_skyhash_arrayeq {
-    (!str, $con:expr, $query:expr, $($val:expr),*) => {
-        runeq!(
-            $con,
-            $query,
-            skytable::Element::Array(skytable::types::Array::Str(
-                vec![
-                    $(Some($val.into()),)*
-                ]
-            ))
-        )
-    };
-    (str, $con:expr, $query:expr, $($val:expr),*) => {
-        runeq!(
-            $con,
-            $query,
-            skytable::Element::Array(skytable::types::Array::NonNullStr(
-                vec![
-                    $($val.into(),)*
-                ]
-            ))
-        )
-    };
-}
-
-macro_rules! assert_respcode {
-    ($con:expr, $query:expr, $code:expr) => {
-        runeq!($con, $query, ::skytable::Element::RespCode($code))
-    };
-}
diff --git a/server/src/tests/mod.rs b/server/src/tests/mod.rs
deleted file mode 100644
index 306bd731..00000000
--- a/server/src/tests/mod.rs
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Created on Tue Aug 25 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! This module contains automated tests for queries
-
-#[macro_use]
-mod macros;
-#[cfg(not(feature = "persist-suite"))]
-mod auth;
-mod ddl_tests;
-mod inspect_tests;
-mod issue_tests;
-mod kvengine;
-mod kvengine_encoding;
-mod kvengine_list;
-mod persist;
-mod pipeline;
-mod snapshot;
-
-mod tls {
-    use skytable::{query, Element};
-    #[sky_macros::dbtest_func(tls_cert = "cert.pem", port = 2004)]
-    async fn tls() {
-        runeq!(
-            con,
-            query!("heya", "abcd"),
-            Element::String("abcd".to_owned())
-        );
-    }
-}
-
-mod sys {
-    use {
-        crate::protocol::{LATEST_PROTOCOL_VERSION, LATEST_PROTOCOL_VERSIONSTRING},
-        libsky::VERSION,
-        sky_macros::dbtest_func as dbtest,
-        skytable::{query, Element, RespCode},
-    };
-
-    #[dbtest]
-    async fn sys_info_aerr() {
-        runeq!(
-            con,
-            query!("sys", "info"),
-            Element::RespCode(RespCode::ActionError)
-        );
-        runeq!(
-            con,
-            query!(
-                "sys",
-                "info",
-                "this is cool",
-                "but why this extra argument?"
-            ),
-            Element::RespCode(RespCode::ActionError)
-        )
-    }
-    #[dbtest]
-    async fn sys_info_protocol() {
-        runeq!(
-            con,
-            query!("sys", "info", "protocol"),
-            Element::String(LATEST_PROTOCOL_VERSIONSTRING.to_owned())
-        )
-    }
-    #[dbtest]
-    async fn sys_info_protover() {
-        runeq!(
-            con,
-            query!("sys", "info", "protover"),
-            Element::Float(LATEST_PROTOCOL_VERSION)
-        )
-    }
-    #[dbtest]
-    async fn sys_info_version() {
-        runeq!(
-            con,
-            query!("sys", "info", "version"),
-            Element::String(VERSION.to_owned())
-        )
-    }
-    #[dbtest]
-    async fn sys_metric_aerr() {
-        runeq!(
-            con,
-            query!("sys", "metric"),
-            Element::RespCode(RespCode::ActionError)
-        );
-        runeq!(
-            con,
-            query!("sys", "metric", "health", "but why this extra argument?"),
-            Element::RespCode(RespCode::ActionError)
-        )
-    }
-    #[dbtest]
-    async fn sys_metric_health() {
-        runeq!(
-            con,
-            query!("sys", "metric", "health"),
-            Element::String("good".to_owned())
-        )
-    }
-    #[dbtest]
-    async fn sys_storage_usage() {
-        runmatch!(
-            con,
-            query!("sys", "metric", "storage"),
-            Element::UnsignedInt
-        )
-    }
-}
-
-use skytable::{query, Element, RespCode};
-
-#[sky_macros::dbtest_func]
-async fn blueql_extra_args() {
-    runeq!(
-        con,
-        query!("use default.default", "extra useless arg"),
-        Element::RespCode(RespCode::ErrorString("bql-invalid-syntax".into()))
-    );
-}
diff --git a/server/src/tests/persist/auth.rs b/server/src/tests/persist/auth.rs
deleted file mode 100644
index 3a53b9fe..00000000
--- a/server/src/tests/persist/auth.rs
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Created on Sat Mar 19 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    sky_macros::dbtest_func as dbtest,
-    skytable::{query, Element},
-};
-
-const USERID: &str = "steinbeck";
-
-#[dbtest(
-    skip_if_cfg = "persist-suite",
-    norun = true,
-    auth_rootuser = true,
-    port = 2005
-)]
-async fn store_user() {
-    runmatch!(con, query!("auth", "adduser", USERID), Element::String)
-}
-
-#[dbtest(
-    run_if_cfg = "persist-suite",
-    norun = true,
-    auth_rootuser = true,
-    port = 2005
-)]
-async fn load_user() {
-    assert_okay!(con, query!("auth", "deluser", USERID));
-}
diff --git a/server/src/tests/persist/kv.rs b/server/src/tests/persist/kv.rs
deleted file mode 100644
index 97fdb07e..00000000
--- a/server/src/tests/persist/kv.rs
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Created on Sat Mar 19 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{persist_load, persist_store, PERSIST_TEST_SET_SIZE},
-    sky_macros::dbtest_func as dbtest,
-};
-
-const PERSIST_CFG_KEYMAP_BIN_BIN_TABLE: &str = "testsuite.persist_bin_bin_tbl";
-const PERSIST_DATA_KEYMAP_BIN_BIN_TABLE: [(&[u8], &[u8]); PERSIST_TEST_SET_SIZE] = [
-    (bin!(b"mykey1"), bin!(b"myval1")),
-    (bin!(b"mykey2"), bin!(b"myval2")),
-    (bin!(b"mykey3"), bin!(b"myval3")),
-    (bin!(b"mykey4"), bin!(b"myval4")),
-];
-
-#[dbtest(skip_if_cfg = "persist-suite", norun = true)]
-async fn store_keymap_bin_bin() {
-    persist_store(
-        &mut con,
-        PERSIST_CFG_KEYMAP_BIN_BIN_TABLE,
-        "(binary, binary)",
-        PERSIST_DATA_KEYMAP_BIN_BIN_TABLE,
-    )
-    .await;
-}
-
-#[dbtest(run_if_cfg = "persist-suite", norun = true)]
-async fn load_keymap_bin_bin() {
-    persist_load(
-        &mut con,
-        PERSIST_CFG_KEYMAP_BIN_BIN_TABLE,
-        PERSIST_DATA_KEYMAP_BIN_BIN_TABLE,
-    )
-    .await;
-}
-
-const PERSIST_CFG_KEYMAP_BIN_STR_TABLE: &str = "testsuite.persist_bin_str_tbl";
-const PERSIST_DATA_KEYMAP_BIN_STR_TABLE: [(&[u8], &str); PERSIST_TEST_SET_SIZE] = [
-    (bin!(b"mykey1"), "myval1"),
-    (bin!(b"mykey2"), "myval2"),
-    (bin!(b"mykey3"), "myval3"),
-    (bin!(b"mykey4"), "myval4"),
-];
-
-#[dbtest(skip_if_cfg = "persist-suite", norun = true)]
-async fn store_keymap_bin_str() {
-    persist_store(
-        &mut con,
-        PERSIST_CFG_KEYMAP_BIN_STR_TABLE,
-        "(binary, string)",
-        PERSIST_DATA_KEYMAP_BIN_STR_TABLE,
-    )
-    .await;
-}
-
-#[dbtest(run_if_cfg = "persist-suite", norun = true)]
-async fn load_keymap_bin_str() {
-    persist_load(
-        &mut con,
-        PERSIST_CFG_KEYMAP_BIN_STR_TABLE,
-        PERSIST_DATA_KEYMAP_BIN_STR_TABLE,
-    )
-    .await;
-}
-
-const PERSIST_CFG_KEYMAP_STR_STR_TABLE: &str = "testsuite.persist_str_str_tbl";
-const PERSIST_DATA_KEYMAP_STR_STR_TABLE: [(&str, &str); PERSIST_TEST_SET_SIZE] = [
-    ("mykey1", "myval1"),
-    ("mykey2", "myval2"),
-    ("mykey3", "myval3"),
-    ("mykey4", "myval4"),
-];
-
-#[dbtest(skip_if_cfg = "persist-suite", norun = true)]
-async fn store_keymap_str_str() {
-    persist_store(
-        &mut con,
-        PERSIST_CFG_KEYMAP_STR_STR_TABLE,
-        "(string, string)",
-        PERSIST_DATA_KEYMAP_STR_STR_TABLE,
-    )
-    .await;
-}
-
-#[dbtest(run_if_cfg = "persist-suite", norun = true)]
-async fn load_keymap_str_str() {
-    persist_load(
-        &mut con,
-        PERSIST_CFG_KEYMAP_STR_STR_TABLE,
-        PERSIST_DATA_KEYMAP_STR_STR_TABLE,
-    )
-    .await;
-}
-
-const PERSIST_CFG_KEYMAP_STR_BIN_TABLE: &str = "testsuite.persist_str_bin_tbl";
-const PERSIST_DATA_KEYMAP_STR_BIN_TABLE: [(&str, &[u8]); PERSIST_TEST_SET_SIZE] = [
-    ("mykey1", bin!(b"myval1")),
-    ("mykey2", bin!(b"myval2")),
-    ("mykey3", bin!(b"myval3")),
-    ("mykey4", bin!(b"myval4")),
-];
-
-#[dbtest(skip_if_cfg = "persist-suite", norun = true)]
-async fn store_keymap_str_bin() {
-    persist_store(
-        &mut con,
-        PERSIST_CFG_KEYMAP_STR_BIN_TABLE,
-        "(string, binary)",
-        PERSIST_DATA_KEYMAP_STR_BIN_TABLE,
-    )
-    .await;
-}
-
-#[dbtest(run_if_cfg = "persist-suite", norun = true)]
-async fn load_keymap_str_bin() {
-    persist_load(
-        &mut con,
-        PERSIST_CFG_KEYMAP_STR_BIN_TABLE,
-        PERSIST_DATA_KEYMAP_STR_BIN_TABLE,
-    )
-    .await;
-}
diff --git a/server/src/tests/persist/kvlist.rs b/server/src/tests/persist/kvlist.rs
deleted file mode 100644
index 0b15ec0f..00000000
--- a/server/src/tests/persist/kvlist.rs
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Created on Sat Mar 19 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{persist_load, persist_store, Bin, ListIDBin, ListIDStr, Str, PERSIST_TEST_SET_SIZE},
-    sky_macros::dbtest_func as dbtest,
-};
-
-type ListData<K, V> = [(K, [V; PERSIST_TEST_SET_SIZE]); PERSIST_TEST_SET_SIZE];
-
-macro_rules! listdata {
-    (
-        $(
-            $listid:expr => $element:expr
-        ),*
-    ) => {
-        [
-            $(
-                (
-                    $listid,
-                    $element
-                ),
-            )*
-        ]
-    };
-}
-
-macro_rules! binid {
-    ($id:expr) => {
-        ListIDBin(bin!($id))
-    };
-}
-
-macro_rules! binlist {
-    ($($elem:expr),*) => {
-        [
-            $(
-                bin!($elem),
-            )*
-        ]
-    };
-}
-
-// bin,list<bin>
-const DATA_BIN_LISTBIN: ListData<ListIDBin, Bin> = listdata!(
-    binid!(b"list1") => binlist!(b"e1", b"e2", b"e3", b"e4"),
-    binid!(b"list2") => binlist!(b"e1", b"e2", b"e3", b"e4"),
-    binid!(b"list3") => binlist!(b"e1", b"e2", b"e3", b"e4"),
-    binid!(b"list4") => binlist!(b"e1", b"e2", b"e3", b"e4")
-);
-const TABLE_BIN_LISTBIN: &str = "testsuite.persist_bin_listbin";
-
-#[dbtest(skip_if_cfg = "persist-suite", norun = true)]
-async fn store_bin_bin() {
-    persist_store(
-        &mut con,
-        TABLE_BIN_LISTBIN,
-        "(binary, list<binary>)",
-        DATA_BIN_LISTBIN,
-    )
-    .await;
-}
-
-#[dbtest(run_if_cfg = "persist-suite", norun = true)]
-async fn load_bin_bin() {
-    persist_load(&mut con, TABLE_BIN_LISTBIN, DATA_BIN_LISTBIN).await;
-}
-
-// bin,list<str>
-const DATA_BIN_LISTSTR: ListData<ListIDBin, Str> = listdata!(
-    binid!(b"list1") => ["e1", "e2", "e3", "e4"],
-    binid!(b"list2") => ["e1", "e2", "e3", "e4"],
-    binid!(b"list3") => ["e1", "e2", "e3", "e4"],
-    binid!(b"list4") => ["e1", "e2", "e3", "e4"]
-);
-
-const TABLE_BIN_LISTSTR: &str = "testsuite.persist_bin_liststr";
-
-#[dbtest(skip_if_cfg = "persist-suite", norun = true)]
-async fn store_bin_str() {
-    persist_store(
-        &mut con,
-        TABLE_BIN_LISTSTR,
-        "(binary, list<string>)",
-        DATA_BIN_LISTSTR,
-    )
-    .await;
-}
-
-#[dbtest(run_if_cfg = "persist-suite", norun = true)]
-async fn load_bin_str() {
-    persist_load(&mut con, TABLE_BIN_LISTSTR, DATA_BIN_LISTSTR).await;
-}
-
-// str,list<bin>
-const DATA_STR_LISTBIN: ListData<ListIDStr, Bin> = listdata!(
-    ListIDStr("list1") => binlist!(b"e1", b"e2", b"e3", b"e4"),
-    ListIDStr("list2") => binlist!(b"e1", b"e2", b"e3", b"e4"),
-    ListIDStr("list3") => binlist!(b"e1", b"e2", b"e3", b"e4"),
-    ListIDStr("list4") => binlist!(b"e1", b"e2", b"e3", b"e4")
-);
-
-const TABLE_STR_LISTBIN: &str = "testsuite.persist_str_listbin";
-
-#[dbtest(skip_if_cfg = "persist-suite", norun = true)]
-async fn store_str_bin() {
-    persist_store(
-        &mut con,
-        TABLE_STR_LISTBIN,
-        "(string, list<binary>)",
-        DATA_STR_LISTBIN,
-    )
-    .await;
-}
-
-#[dbtest(run_if_cfg = "persist-suite", norun = true)]
-async fn load_str_bin() {
-    persist_load(&mut con, TABLE_STR_LISTBIN, DATA_STR_LISTBIN).await;
-}
-
-// str,list<str>
-const DATA_STR_LISTSTR: ListData<ListIDStr, Str> = listdata!(
-    ListIDStr("list1") => ["e1", "e2", "e3", "e4"],
-    ListIDStr("list2") => ["e1", "e2", "e3", "e4"],
-    ListIDStr("list3") => ["e1", "e2", "e3", "e4"],
-    ListIDStr("list4") => ["e1", "e2", "e3", "e4"]
-);
-
-const TABLE_STR_LISTSTR: &str = "testsuite.persist_str_liststr";
-
-#[dbtest(skip_if_cfg = "persist-suite", norun = true)]
-async fn store_str_str() {
-    persist_store(
-        &mut con,
-        TABLE_STR_LISTSTR,
-        "(string, list<string>)",
-        DATA_STR_LISTSTR,
-    )
-    .await;
-}
-
-#[dbtest(run_if_cfg = "persist-suite", norun = true)]
-async fn load_str_str() {
-    persist_load(&mut con, TABLE_STR_LISTSTR, DATA_STR_LISTSTR).await;
-}
diff --git a/server/src/tests/persist/mod.rs b/server/src/tests/persist/mod.rs
deleted file mode 100644
index 16a9eb25..00000000
--- a/server/src/tests/persist/mod.rs
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Created on Thu Mar 17 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    sky_macros::dbtest_func as dbtest,
-    skytable::{
-        aio::Connection,
-        query,
-        types::{Array, RawString},
-        Element, Query, RespCode,
-    },
-};
-
-#[dbtest(skip_if_cfg = "persist-suite", norun = true, port = 2007)]
-async fn store_keyspace() {
-    assert_okay!(con, query!("create space universe"));
-    switch_entity!(con, "universe");
-    assert_okay!(con, query!("create model warp(string,string)"));
-    switch_entity!(con, "universe.warp");
-    assert_okay!(con, query!("set", "x", "100"));
-}
-#[dbtest(run_if_cfg = "persist-suite", norun = true, port = 2007)]
-async fn load_keyspace() {
-    switch_entity!(con, "universe.warp");
-    runeq!(con, query!("get", "x"), Element::String("100".to_owned()));
-    switch_entity!(con, "default");
-    assert_okay!(con, query!("drop model universe.warp force"));
-    assert_okay!(con, query!("drop space universe"));
-}
-
-macro_rules! bin {
-    ($input:expr) => {{
-        const INVALID_SEQ: [u8; 2] = *b"\x80\x81";
-        const RETLEN: usize = 2 + $input.len();
-        const RET0: [u8; RETLEN] = {
-            let mut iret: [u8; RETLEN] = [0u8; RETLEN];
-            let mut idx = 0;
-            while idx < $input.len() {
-                iret[idx] = $input[idx];
-                idx += 1;
-            }
-            iret[RETLEN - 2] = INVALID_SEQ[0];
-            iret[RETLEN - 1] = INVALID_SEQ[1];
-            iret
-        };
-        &RET0
-    }};
-}
-
-mod auth;
-mod kv;
-mod kvlist;
-
-const PERSIST_TEST_SET_SIZE: usize = 4;
-
-trait PushIntoQuery {
-    fn push_into(&self, query: &mut Query);
-}
-
-impl PushIntoQuery for &str {
-    fn push_into(&self, q: &mut Query) {
-        q.push(*self);
-    }
-}
-
-impl PushIntoQuery for &[u8] {
-    fn push_into(&self, q: &mut Query) {
-        q.push(RawString::from(self.to_vec()))
-    }
-}
-
-impl<T: PushIntoQuery, const N: usize> PushIntoQuery for [T; N] {
-    fn push_into(&self, q: &mut Query) {
-        for element in self {
-            element.push_into(q)
-        }
-    }
-}
-
-impl<T: PushIntoQuery> PushIntoQuery for &[T] {
-    fn push_into(&self, q: &mut Query) {
-        for element in self.iter() {
-            element.push_into(q)
-        }
-    }
-}
-
-trait PersistKey: PushIntoQuery {
-    fn action_store() -> &'static str;
-    fn action_load() -> &'static str;
-}
-
-macro_rules! impl_persist_key {
-    ($($ty:ty => ($store:expr, $load:expr)),*) => {
-        $(impl PersistKey for $ty {
-            fn action_store() -> &'static str {
-                $store
-            }
-            fn action_load() -> &'static str {
-                $load
-            }
-        })*
-    };
-}
-
-impl_persist_key!(
-    &str => ("set", "get"),
-    &[u8] => ("set", "get"),
-    ListIDBin => ("lset", "lget"),
-    ListIDStr => ("lset", "lget")
-);
-
-trait PersistValue: PushIntoQuery {
-    fn response_store(&self) -> Element;
-    fn response_load(&self) -> Element;
-}
-
-impl PersistValue for &str {
-    fn response_store(&self) -> Element {
-        Element::RespCode(RespCode::Okay)
-    }
-    fn response_load(&self) -> Element {
-        Element::String(self.to_string())
-    }
-}
-
-impl PersistValue for &[u8] {
-    fn response_store(&self) -> Element {
-        Element::RespCode(RespCode::Okay)
-    }
-    fn response_load(&self) -> Element {
-        Element::Binstr(self.to_vec())
-    }
-}
-
-impl<const N: usize> PersistValue for [&[u8]; N] {
-    fn response_store(&self) -> Element {
-        Element::RespCode(RespCode::Okay)
-    }
-    fn response_load(&self) -> Element {
-        let mut flat = Vec::with_capacity(N);
-        for item in self {
-            flat.push(item.to_vec());
-        }
-        Element::Array(Array::NonNullBin(flat))
-    }
-}
-
-impl<const N: usize> PersistValue for [&str; N] {
-    fn response_store(&self) -> Element {
-        Element::RespCode(RespCode::Okay)
-    }
-    fn response_load(&self) -> Element {
-        let mut flat = Vec::with_capacity(N);
-        for item in self {
-            flat.push(item.to_string());
-        }
-        Element::Array(Array::NonNullStr(flat))
-    }
-}
-
-type Bin = &'static [u8];
-type Str = &'static str;
-
-#[derive(Debug)]
-struct ListIDStr(Str);
-#[derive(Debug)]
-struct ListIDBin(Bin);
-
-impl PushIntoQuery for ListIDStr {
-    fn push_into(&self, q: &mut Query) {
-        self.0.push_into(q)
-    }
-}
-
-impl PushIntoQuery for ListIDBin {
-    fn push_into(&self, q: &mut Query) {
-        self.0.push_into(q)
-    }
-}
-
-async fn persist_store<K: PersistKey, V: PersistValue>(
-    con: &mut Connection,
-    table_id: &str,
-    declaration: &str,
-    input: [(K, V); PERSIST_TEST_SET_SIZE],
-) {
-    create_table_and_switch!(con, table_id, declaration);
-    for (key, value) in input {
-        let mut query = Query::from(K::action_store());
-        key.push_into(&mut query);
-        value.push_into(&mut query);
-        runeq!(con, query, value.response_store())
-    }
-}
-
-async fn persist_load<K: PersistKey, V: PersistValue>(
-    con: &mut Connection,
-    table_id: &str,
-    input: [(K, V); PERSIST_TEST_SET_SIZE],
-) {
-    switch_entity!(con, table_id);
-    for (key, value) in input {
-        let mut q = Query::from(K::action_load());
-        key.push_into(&mut q);
-        runeq!(con, q, value.response_load());
-    }
-    // now delete this table, freeing it up for the next suite run
-    switch_entity!(con, "default.default");
-    runeq!(
-        con,
-        query!(format!("drop model {table_id} force")),
-        Element::RespCode(RespCode::Okay)
-    );
-}
diff --git a/server/src/tests/pipeline.rs b/server/src/tests/pipeline.rs
deleted file mode 100644
index f8a01aad..00000000
--- a/server/src/tests/pipeline.rs
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Created on Sat Oct 30 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#[sky_macros::dbtest_module]
-mod tests {
-    use skytable::{query, types::Array, Element, Pipeline, RespCode};
-    async fn test_pipeline_heya_echo() {
-        let pipe = Pipeline::new()
-            .append(query!("heya", "first"))
-            .append(query!("heya", "second"));
-        let ret = con.run_pipeline(pipe).await.unwrap();
-        assert_eq!(
-            ret,
-            vec![
-                Element::String("first".to_owned()),
-                Element::String("second".to_owned())
-            ]
-        )
-    }
-    async fn test_pipeline_basic() {
-        let pipe = Pipeline::new()
-            .append(query!("heya"))
-            .append(query!("get", "x"));
-        let ret = con.run_pipeline(pipe).await.unwrap();
-        assert_eq!(
-            ret,
-            vec![
-                Element::String("HEY!".to_owned()),
-                Element::RespCode(RespCode::NotFound)
-            ]
-        );
-    }
-    // although an error is simply just a response, but we'll still add a test for sanity
-    async fn test_pipeline_with_error() {
-        let pipe = Pipeline::new()
-            .append(query!("heya"))
-            .append(query!("get", "x", "y"));
-        let ret = con.run_pipeline(pipe).await.unwrap();
-        assert_eq!(
-            ret,
-            vec![
-                Element::String("HEY!".to_owned()),
-                Element::RespCode(RespCode::ActionError)
-            ]
-        );
-    }
-    async fn test_pipeline_with_multiple_error() {
-        let pipe = Pipeline::new()
-            .append(query!("mset", "x", "y", "z"))
-            .append(query!("mget", "x", "y", "z"))
-            .append(query!("heya", "finally"));
-        let ret = con.run_pipeline(pipe).await.unwrap();
-        assert_eq!(
-            ret,
-            vec![
-                Element::RespCode(RespCode::ActionError),
-                Element::Array(Array::Str(vec![None, None, None])),
-                Element::String("finally".to_owned())
-            ]
-        )
-    }
-}
diff --git a/server/src/tests/snapshot.rs b/server/src/tests/snapshot.rs
deleted file mode 100644
index f6cf4152..00000000
--- a/server/src/tests/snapshot.rs
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Created on Tue Mar 29 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    sky_macros::dbtest_func as dbtest,
-    skytable::{query, Element, RespCode},
-};
-
-const SNAPSHOT_DISABLED: &str = "err-snapshot-disabled";
-
-#[dbtest]
-async fn snapshot_fail_because_local_disabled() {
-    runeq!(
-        con,
-        query!("mksnap"),
-        Element::RespCode(RespCode::ErrorString(SNAPSHOT_DISABLED.to_owned()))
-    )
-}
-
-#[dbtest(skip_if_cfg = "persist-suite")]
-async fn rsnap_okay() {
-    loop {
-        match con.run_query_raw(query!("mksnap", "myremo")).await.unwrap() {
-            Element::RespCode(RespCode::Okay) => break,
-            Element::RespCode(RespCode::ErrorString(estr)) if estr.eq("err-snapshot-busy") => {}
-            x => panic!("snapshot failed: {:?}", x),
-        }
-    }
-}
-
-#[dbtest(port = 2007)]
-async fn local_snapshot_from_remote_okay() {
-    assert_okay!(con, query!("mksnap"))
-}
-
-#[dbtest(port = 2007, skip_if_cfg = "persist-suite")]
-async fn remote_snapshot_okay_with_local_enabled() {
-    loop {
-        match con.run_query_raw(query!("mksnap", "myremo")).await.unwrap() {
-            Element::RespCode(RespCode::Okay) => break,
-            Element::RespCode(RespCode::ErrorString(estr)) if estr.eq("err-snapshot-busy") => {}
-            x => panic!("snapshot failed: {:?}", x),
-        }
-    }
-}
-
-#[dbtest(port = 2007, skip_if_cfg = "persist-suite")]
-async fn remote_snapshot_fail_because_already_exists() {
-    loop {
-        match con.run_query_raw(query!("mksnap", "dupe")).await.unwrap() {
-            Element::RespCode(RespCode::Okay) => break,
-            Element::RespCode(RespCode::ErrorString(estr)) if estr.eq("err-snapshot-busy") => {}
-            x => panic!("snapshot failed: {:?}", x),
-        }
-    }
-    loop {
-        match con.run_query_raw(query!("mksnap", "dupe")).await.unwrap() {
-            Element::RespCode(RespCode::ErrorString(estr)) => match estr.as_str() {
-                "err-snapshot-busy" => {}
-                "duplicate-snapshot" => break,
-                _ => panic!("Got error string: {estr} instead"),
-            },
-            x => panic!("snapshot failed: {:?}", x),
-        }
-    }
-}
diff --git a/server/src/util/error.rs b/server/src/util/error.rs
deleted file mode 100644
index 4627ef33..00000000
--- a/server/src/util/error.rs
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Created on Sat Mar 26 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::storage::v1::{error::StorageEngineError, sengine::SnapshotEngineError},
-    openssl::{error::ErrorStack as SslErrorStack, ssl::Error as SslError},
-    std::{fmt, io::Error as IoError},
-};
-
-pub type SkyResult<T> = Result<T, Error>;
-
-#[derive(Debug)]
-pub enum Error {
-    Storage(StorageEngineError),
-    IoError(IoError),
-    IoErrorExtra(IoError, String),
-    OtherError(String),
-    TlsError(SslError),
-    SnapshotEngineError(SnapshotEngineError),
-}
-
-impl Error {
-    pub fn ioerror_extra(ioe: IoError, extra: impl ToString) -> Self {
-        Self::IoErrorExtra(ioe, extra.to_string())
-    }
-}
-
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::Storage(serr) => write!(f, "Storage engine error: {}", serr),
-            Self::IoError(nerr) => write!(f, "I/O error: {}", nerr),
-            Self::IoErrorExtra(ioe, extra) => write!(f, "I/O error while {extra}: {ioe}"),
-            Self::OtherError(oerr) => write!(f, "Error: {}", oerr),
-            Self::TlsError(terr) => write!(f, "TLS error: {}", terr),
-            Self::SnapshotEngineError(snaperr) => write!(f, "Snapshot engine error: {snaperr}"),
-        }
-    }
-}
-
-impl From<IoError> for Error {
-    fn from(ioe: IoError) -> Self {
-        Self::IoError(ioe)
-    }
-}
-
-impl From<StorageEngineError> for Error {
-    fn from(see: StorageEngineError) -> Self {
-        Self::Storage(see)
-    }
-}
-
-impl From<SslError> for Error {
-    fn from(sslerr: SslError) -> Self {
-        Self::TlsError(sslerr)
-    }
-}
-
-impl From<SslErrorStack> for Error {
-    fn from(estack: SslErrorStack) -> Self {
-        Self::TlsError(estack.into())
-    }
-}
-
-impl From<SnapshotEngineError> for Error {
-    fn from(snaperr: SnapshotEngineError) -> Self {
-        Self::SnapshotEngineError(snaperr)
-    }
-}
diff --git a/server/src/util/macros.rs b/server/src/util/macros.rs
index d5099b2b..8c2849e3 100644
--- a/server/src/util/macros.rs
+++ b/server/src/util/macros.rs
@@ -227,38 +227,6 @@ macro_rules! do_sleep {
     }};
 }
 
-#[cfg(test)]
-macro_rules! tmut_bool {
-    ($e:expr) => {{
-        *(&$e as *const _ as *const bool)
-    }};
-    ($a:expr, $b:expr) => {
-        (tmut_bool!($a), tmut_bool!($b))
-    };
-}
-
-macro_rules! ucidx {
-    ($base:expr, $idx:expr) => {
-        *($base.as_ptr().add($idx as usize))
-    };
-}
-
-/// If you provide: [T; N] with M initialized elements, then you are given
-/// [MaybeUninit<T>; N] with M initialized elements and N-M uninit elements
-macro_rules! uninit_array {
-    ($($vis:vis const $id:ident: [$ty:ty; $len:expr] = [$($init_element:expr),*];)*) => {
-        $($vis const $id: [::core::mem::MaybeUninit<$ty>; $len] = {
-            let mut ret = [::core::mem::MaybeUninit::uninit(); $len];
-            let mut idx = 0;
-            $(
-                idx += 1;
-                ret[idx - 1] = ::core::mem::MaybeUninit::new($init_element);
-            )*
-            ret
-        };)*
-    };
-}
-
 #[macro_export]
 macro_rules! def {
     (
diff --git a/server/src/util/mod.rs b/server/src/util/mod.rs
index c3f2d360..bbd8ce8b 100644
--- a/server/src/util/mod.rs
+++ b/server/src/util/mod.rs
@@ -27,15 +27,10 @@
 #[macro_use]
 mod macros;
 pub mod compiler;
-pub mod error;
 pub mod os;
 #[cfg(test)]
 pub mod test_utils;
 use {
-    crate::{
-        actions::{ActionError, ActionResult},
-        protocol::interface::ProtocolSpec,
-    },
     core::{
         fmt::{self, Debug},
         marker::PhantomData,
@@ -91,20 +86,6 @@ unsafe impl<T> Unwrappable<T> for Option<T> {
     }
 }
 
-pub trait UnwrapActionError<T> {
-    fn unwrap_or_custom_aerr(self, e: impl Into<ActionError>) -> ActionResult<T>;
-    fn unwrap_or_aerr<P: ProtocolSpec>(self) -> ActionResult<T>;
-}
-
-impl<T> UnwrapActionError<T> for Option<T> {
-    fn unwrap_or_custom_aerr(self, e: impl Into<ActionError>) -> ActionResult<T> {
-        self.ok_or_else(|| e.into())
-    }
-    fn unwrap_or_aerr<P: ProtocolSpec>(self) -> ActionResult<T> {
-        self.ok_or_else(|| P::RCODE_ACTION_ERR.into())
-    }
-}
-
 pub fn exit_error() -> ! {
     process::exit(EXITCODE_ONE)
 }
diff --git a/server/src/util/os.rs b/server/src/util/os.rs
index 478f67ba..86d09725 100644
--- a/server/src/util/os.rs
+++ b/server/src/util/os.rs
@@ -28,9 +28,9 @@
 pub use unix::*;
 #[cfg(windows)]
 pub use windows::*;
+mod flock;
 mod free_memory;
 
-pub use free_memory::free_memory_in_bytes;
 use {
     crate::IoResult,
     std::{
@@ -40,6 +40,7 @@ use {
         time::{SystemTime, UNIX_EPOCH},
     },
 };
+pub use {flock::FileLock, free_memory::free_memory_in_bytes};
 
 #[derive(Debug)]
 #[repr(transparent)]
diff --git a/server/src/util/os/flock.rs b/server/src/util/os/flock.rs
new file mode 100644
index 00000000..da0b2083
--- /dev/null
+++ b/server/src/util/os/flock.rs
@@ -0,0 +1,117 @@
+/*
+ * Created on Wed Oct 04 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+// unix imports
+#[cfg(unix)]
+extern crate libc;
+// windows imports
+#[cfg(windows)]
+extern crate winapi;
+#[cfg(windows)]
+use std::os::windows::io::AsRawHandle;
+
+use std::{fs::File, io, os::unix::io::AsRawFd, path::Path};
+
+pub struct FileLock {
+    file: File,
+    #[cfg(windows)]
+    handle: winapi::um::handleapi::HANDLE,
+}
+
+impl FileLock {
+    pub fn new<P: AsRef<Path>>(path: P) -> io::Result<Self> {
+        let file = File::create(path)?;
+        #[cfg(windows)]
+        {
+            use winapi::um::{
+                fileapi::LockFileEx,
+                minwinbase::{LOCKFILE_EXCLUSIVE_LOCK, OVERLAPPED},
+                winnt::HANDLE,
+            };
+
+            let handle = file.as_raw_handle();
+            let mut overlapped = OVERLAPPED::default();
+            let result = unsafe {
+                LockFileEx(
+                    handle as HANDLE,
+                    LOCKFILE_EXCLUSIVE_LOCK,
+                    0,
+                    u32::MAX,
+                    u32::MAX,
+                    &mut overlapped,
+                )
+            };
+
+            if result == 0 {
+                return Err(io::Error::last_os_error());
+            }
+
+            return Ok(Self { file, handle });
+        }
+        #[cfg(unix)]
+        {
+            use libc::{flock, LOCK_EX};
+
+            let result = unsafe { flock(file.as_raw_fd(), LOCK_EX) };
+
+            if result != 0 {
+                return Err(io::Error::last_os_error());
+            }
+
+            return Ok(Self { file });
+        }
+    }
+    pub fn release(self) -> io::Result<()> {
+        #[cfg(windows)]
+        {
+            use winapi::um::{fileapi::UnlockFileEx, minwinbase::OVERLAPPED, winnt::HANDLE};
+
+            let mut overlapped = OVERLAPPED::default();
+            let result = unsafe {
+                UnlockFileEx(
+                    self.handle as HANDLE,
+                    0,
+                    u32::MAX,
+                    u32::MAX,
+                    &mut overlapped,
+                )
+            };
+
+            if result == 0 {
+                return Err(io::Error::last_os_error());
+            }
+        }
+        #[cfg(unix)]
+        {
+            use libc::{flock, LOCK_UN};
+            let result = unsafe { flock(self.file.as_raw_fd(), LOCK_UN) };
+            if result != 0 {
+                return Err(io::Error::last_os_error());
+            }
+        }
+        Ok(())
+    }
+}

From 3e981f3dcb90b084d14badaa250d449287acfed0 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 7 Oct 2023 08:54:28 +0000
Subject: [PATCH 272/310] Implement exec, fix delta handling and add resp

---
 server/src/engine/core/dml/del.rs             |   8 +-
 server/src/engine/core/dml/ins.rs             |   9 +-
 server/src/engine/core/dml/mod.rs             |  18 +-
 server/src/engine/core/dml/sel.rs             |   4 +-
 server/src/engine/core/dml/upd.rs             |  18 +-
 server/src/engine/core/exec.rs                | 163 ++++++++++++-
 server/src/engine/core/mod.rs                 |  20 +-
 server/src/engine/core/model/alt.rs           |  23 +-
 server/src/engine/core/model/delta.rs         |  24 +-
 server/src/engine/core/model/mod.rs           |  12 +-
 server/src/engine/core/space.rs               |  18 +-
 server/src/engine/core/tests/ddl_model/alt.rs |  16 +-
 server/src/engine/core/tests/ddl_model/crt.rs |  10 +-
 .../src/engine/core/tests/ddl_model/layer.rs  |   4 +-
 .../src/engine/core/tests/ddl_space/alter.rs  |   2 +-
 .../src/engine/core/tests/ddl_space/create.rs |   4 +-
 server/src/engine/core/tests/dml/delete.rs    |   2 +-
 server/src/engine/core/tests/dml/insert.rs    |   2 +-
 server/src/engine/core/tests/dml/select.rs    |   2 +-
 server/src/engine/core/tests/dml/update.rs    |   6 +-
 server/src/engine/core/util.rs                |   2 +-
 server/src/engine/error.rs                    |  71 +++---
 server/src/engine/fractal/mgr.rs              |   5 -
 server/src/engine/fractal/mod.rs              |  25 +-
 server/src/engine/macros.rs                   | 227 ++++++++++++++++++
 server/src/engine/net/mod.rs                  |  10 +-
 server/src/engine/net/protocol/exchange.rs    |  55 +++--
 server/src/engine/net/protocol/mod.rs         |  38 ++-
 server/src/engine/ql/ast/mod.rs               |  15 +-
 server/src/engine/ql/ddl/alt.rs               |   2 +-
 server/src/engine/ql/ddl/drop.rs              |   2 +-
 server/src/engine/ql/ddl/ins.rs               |   2 +-
 server/src/engine/ql/ddl/syn.rs               |   2 +-
 server/src/engine/ql/lex/mod.rs               |  20 +-
 server/src/engine/ql/lex/raw.rs               |  45 ++--
 server/src/engine/ql/macros.rs                | 223 -----------------
 .../engine/storage/v1/batch_jrnl/persist.rs   |   6 +-
 server/src/engine/txn/gns/tests/full_chain.rs |   2 +-
 38 files changed, 660 insertions(+), 457 deletions(-)

diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs
index 1ba3b8e5..0802383c 100644
--- a/server/src/engine/core/dml/del.rs
+++ b/server/src/engine/core/dml/del.rs
@@ -25,7 +25,7 @@
 */
 
 use crate::engine::{
-    core::{self, model::delta::DataDeltaKind},
+    core::{self, dml::QueryExecMeta, model::delta::DataDeltaKind},
     error::{QueryError, QueryResult},
     fractal::GlobalInstanceLike,
     idx::MTIndex,
@@ -47,16 +47,16 @@ pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) ->
             .mt_delete_return_entry(&model.resolve_where(delete.clauses_mut())?, &g)
         {
             Some(row) => {
-                delta_state.append_new_data_delta_with(
+                let dp = delta_state.append_new_data_delta_with(
                     DataDeltaKind::Delete,
                     row.clone(),
                     schema_version,
                     new_version,
                     &g,
                 );
-                Ok(())
+                Ok(QueryExecMeta::new(dp))
             }
-            None => Err(QueryError::QPDmlRowNotFound),
+            None => Err(QueryError::QExecDmlRowNotFound),
         }
     })
 }
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index 8b5d7992..8b0173dd 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -27,6 +27,7 @@
 use crate::engine::{
     core::{
         self,
+        dml::QueryExecMeta,
         index::{DcFieldIndex, PrimaryIndexKey, Row},
         model::{delta::DataDeltaKind, Fields, Model},
     },
@@ -49,16 +50,16 @@ pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> Quer
         let row = Row::new(pk, data, ds.schema_current_version(), new_version);
         if mdl.primary_index().__raw_index().mt_insert(row.clone(), &g) {
             // append delta for new version
-            ds.append_new_data_delta_with(
+            let dp = ds.append_new_data_delta_with(
                 DataDeltaKind::Insert,
                 row,
                 ds.schema_current_version(),
                 new_version,
                 &g,
             );
-            Ok(())
+            Ok(QueryExecMeta::new(dp))
         } else {
-            Err(QueryError::QPDmlDuplicate)
+            Err(QueryError::QExecDmlDuplicate)
         }
     })
 }
@@ -114,6 +115,6 @@ fn prepare_insert(
         };
         Ok((primary_key, prepared_data))
     } else {
-        Err(QueryError::QPDmlValidationError)
+        Err(QueryError::QExecDmlValidationError)
     }
 }
diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index a7414382..dfe1dfb1 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -55,7 +55,23 @@ impl Model {
             {
                 Ok(clause.rhs())
             }
-            _ => compiler::cold_rerr(QueryError::QPDmlWhereHasUnindexedColumn),
+            _ => compiler::cold_rerr(QueryError::QExecDmlWhereHasUnindexedColumn),
         }
     }
 }
+
+pub struct QueryExecMeta {
+    delta_hint: usize,
+}
+
+impl QueryExecMeta {
+    pub fn new(delta_hint: usize) -> Self {
+        Self { delta_hint }
+    }
+    pub fn zero() -> Self {
+        Self::new(0)
+    }
+    pub fn delta_hint(&self) -> usize {
+        self.delta_hint
+    }
+}
diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
index 4ee9698c..f95a8239 100644
--- a/server/src/engine/core/dml/sel.rs
+++ b/server/src/engine/core/dml/sel.rs
@@ -52,7 +52,7 @@ where
             match fields.st_get(key) {
                 Some(dc) => cellfn(dc),
                 None if key == mdl.p_key() => cellfn(&pkdc),
-                None => return Err(QueryError::QPUnknownField),
+                None => return Err(QueryError::QExecUnknownField),
             }
             Ok(())
         };
@@ -69,7 +69,7 @@ where
                     }
                 }
             }
-            None => return Err(QueryError::QPDmlRowNotFound),
+            None => return Err(QueryError::QExecDmlRowNotFound),
         }
         Ok(())
     })
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index 3be58b52..eeb2998a 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -30,7 +30,10 @@ use std::cell::RefCell;
 use {
     crate::{
         engine::{
-            core::{self, model::delta::DataDeltaKind, query_meta::AssignmentOperator},
+            core::{
+                self, dml::QueryExecMeta, model::delta::DataDeltaKind,
+                query_meta::AssignmentOperator,
+            },
             data::{
                 cell::Datacell,
                 lit::Lit,
@@ -235,7 +238,7 @@ pub fn collect_trace_path() -> Vec<&'static str> {
 #[allow(unused)]
 pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) -> QueryResult<()> {
     core::with_model_for_data_update(global, update.entity(), |mdl| {
-        let mut ret = Ok(());
+        let mut ret = Ok(QueryExecMeta::zero());
         // prepare row fetch
         let key = mdl.resolve_where(update.clauses_mut())?;
         // freeze schema
@@ -243,7 +246,7 @@ pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) ->
         // fetch row
         let g = sync::atm::cpin();
         let Some(row) = mdl.primary_index().select(key, &g) else {
-            return Err(QueryError::QPDmlRowNotFound);
+            return Err(QueryError::QExecDmlRowNotFound);
         };
         // lock row
         let mut row_data_wl = row.d_data().write();
@@ -280,7 +283,7 @@ pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) ->
                 _ => {
                     input_trace("fieldnotfound");
                     rollback_now = true;
-                    ret = Err(QueryError::QPUnknownField);
+                    ret = Err(QueryError::QExecUnknownField);
                     break;
                 }
             }
@@ -321,13 +324,13 @@ pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) ->
                     } else {
                         input_trace("list;badtag");
                         rollback_now = true;
-                        ret = Err(QueryError::QPDmlValidationError);
+                        ret = Err(QueryError::QExecDmlValidationError);
                         break;
                     }
                 }
                 _ => {
                     input_trace("unknown_reason;exitmainloop");
-                    ret = Err(QueryError::QPDmlValidationError);
+                    ret = Err(QueryError::QExecDmlValidationError);
                     rollback_now = true;
                     break;
                 }
@@ -344,13 +347,14 @@ pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) ->
             // update revised tag
             row_data_wl.set_txn_revised(new_version);
             // publish delta
-            ds.append_new_data_delta_with(
+            let dp = ds.append_new_data_delta_with(
                 DataDeltaKind::Update,
                 row.clone(),
                 ds.schema_current_version(),
                 new_version,
                 &g,
             );
+            ret = Ok(QueryExecMeta::new(dp))
         }
         ret
     })
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 642960df..703b2447 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -24,12 +24,169 @@
  *
 */
 
-use crate::engine::{error::QueryResult, fractal::Global, net::protocol::SQuery};
+use {
+    crate::engine::{
+        core::{dml, model::Model, space::Space},
+        error::{QueryError, QueryResult},
+        fractal::Global,
+        net::protocol::{Response, SQuery},
+        ql::{
+            ast::{traits::ASTNode, InplaceData, State},
+            lex::{Keyword, KeywordStmt, Token},
+        },
+    },
+    core::ops::Deref,
+};
 
-pub async fn execute_query<'a>(_global: &Global, query: SQuery<'a>) -> QueryResult<()> {
+pub async fn dispatch_to_executor<'a, 'b>(
+    global: &'b Global,
+    query: SQuery<'a>,
+) -> QueryResult<Response> {
     let tokens =
         crate::engine::ql::lex::SecureLexer::new_with_segments(query.query(), query.params())
             .lex()?;
-    let _ = crate::engine::ql::ast::compile(&tokens, crate::engine::ql::ast::InplaceData::new());
+    let mut state = State::new_inplace(&tokens);
+    let stmt = match state.read() {
+        Token::Keyword(Keyword::Statement(stmt)) if state.remaining() >= 3 => *stmt,
+        _ => return Err(QueryError::QLExpectedStatement),
+    };
+    state.cursor_ahead();
+    if stmt.is_blocking() {
+        run_blocking_stmt(state, stmt, global).await
+    } else {
+        run_nb(global, state, stmt)
+    }
+}
+
+/*
+    blocking exec
+    ---
+    trigger warning: disgusting hacks below (why can't async play nice with lifetimes :|)
+*/
+
+struct RawSlice<T> {
+    t: *const T,
+    l: usize,
+}
+
+unsafe impl<T: Send> Send for RawSlice<T> {}
+unsafe impl<T: Sync> Sync for RawSlice<T> {}
+
+impl<T> RawSlice<T> {
+    #[inline(always)]
+    unsafe fn new(t: *const T, l: usize) -> Self {
+        Self { t, l }
+    }
+}
+
+impl<T> Deref for RawSlice<T> {
+    type Target = [T];
+    #[inline(always)]
+    fn deref(&self) -> &Self::Target {
+        unsafe {
+            // UNSAFE(@ohsayan): the caller MUST guarantee that this remains valid throughout the usage of the slice
+            core::slice::from_raw_parts(self.t, self.l)
+        }
+    }
+}
+
+#[inline(always)]
+fn call<A: ASTNode<'static> + core::fmt::Debug, T>(
+    g: Global,
+    tokens: RawSlice<Token<'static>>,
+    f: impl FnOnce(&Global, A) -> QueryResult<T>,
+) -> QueryResult<T> {
+    let mut state = State::new_inplace(unsafe {
+        // UNSAFE(@ohsayan): nothing to drop. all cool
+        core::mem::transmute(tokens)
+    });
+    _call(&g, &mut state, f)
+}
+
+#[inline(always)]
+fn _call<A: ASTNode<'static> + core::fmt::Debug, T>(
+    g: &Global,
+    state: &mut State<'static, InplaceData>,
+    f: impl FnOnce(&Global, A) -> Result<T, QueryError>,
+) -> QueryResult<T> {
+    let cs = ASTNode::from_state(state)?;
+    f(&g, cs)
+}
+
+async fn run_blocking_stmt(
+    mut state: State<'_, InplaceData>,
+    stmt: KeywordStmt,
+    global: &Global,
+) -> Result<Response, QueryError> {
+    let (a, b) = (&state.current()[0], &state.current()[1]);
+    let sysctl = stmt == KeywordStmt::Sysctl;
+    let create = stmt == KeywordStmt::Create;
+    let alter = stmt == KeywordStmt::Alter;
+    let drop = stmt == KeywordStmt::Drop;
+    let last_id = b.is_ident();
+    let c_s = (create & Token![space].eq(a) & last_id) as u8 * 2;
+    let c_m = (create & Token![model].eq(a) & last_id) as u8 * 3;
+    let a_s = (alter & Token![space].eq(a) & last_id) as u8 * 4;
+    let a_m = (alter & Token![model].eq(a) & last_id) as u8 * 5;
+    let d_s = (drop & Token![space].eq(a) & last_id) as u8 * 6;
+    let d_m = (drop & Token![model].eq(a) & last_id) as u8 * 7;
+    let fc = sysctl as u8 | c_s | c_m | a_s | a_m | d_s | d_m;
+    state.cursor_ahead();
+    static BLK_EXEC: [fn(Global, RawSlice<Token<'static>>) -> QueryResult<()>; 8] = [
+        |_, _| Err(QueryError::QLUnknownStatement), // unknown
+        blocking_exec_sysctl,                       // sysctl
+        |g, t| call(g, t, Space::transactional_exec_create),
+        |g, t| call(g, t, Model::transactional_exec_create),
+        |g, t| call(g, t, Space::transactional_exec_alter),
+        |g, t| call(g, t, Model::transactional_exec_alter),
+        |g, t| call(g, t, Space::transactional_exec_drop),
+        |g, t| call(g, t, Model::transactional_exec_drop),
+    ];
+    let r = unsafe {
+        // UNSAFE(@ohsayan): the only await is within this block
+        let c_glob = global.clone();
+        let ptr = state.current().as_ptr() as usize;
+        let len = state.current().len();
+        tokio::task::spawn_blocking(move || {
+            let tokens = RawSlice::new(ptr as *const Token, len);
+            BLK_EXEC[fc as usize](c_glob, tokens)?;
+            Ok(Response::Empty)
+        })
+        .await
+    };
+    r.unwrap()
+}
+
+fn blocking_exec_sysctl(_: Global, _: RawSlice<Token<'static>>) -> QueryResult<()> {
     todo!()
 }
+
+/*
+    nb exec
+*/
+
+fn run_nb(
+    global: &Global,
+    state: State<'_, InplaceData>,
+    stmt: KeywordStmt,
+) -> QueryResult<Response> {
+    let stmt = stmt.value_u8() - KeywordStmt::Use.value_u8();
+    static F: [fn(&Global, &mut State<'static, InplaceData>) -> QueryResult<()>; 8] = [
+        |_, _| panic!("use not implemented"),
+        |_, _| panic!("inspect not implemented"),
+        |_, _| panic!("describe not implemented"),
+        |g, s| _call(g, s, dml::insert),
+        |_, _| panic!("select not implemented"),
+        |g, s| _call(g, s, dml::update),
+        |g, s| _call(g, s, dml::delete),
+        |_, _| panic!("exists not implemented"),
+    ];
+    {
+        let mut state = unsafe {
+            // UNSAFE(@ohsayan): this is a lifetime issue with the token handle
+            core::mem::transmute(state)
+        };
+        F[stmt as usize](global, &mut state)?;
+    }
+    Ok(Response::Empty)
+}
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 7385cf4c..6a759f71 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -57,13 +57,13 @@ pub struct GlobalNS {
     index_space: RWLIdx<Box<str>, Space>,
 }
 
-pub(self) fn with_model_for_data_update<'a, T, E, F>(
+pub(self) fn with_model_for_data_update<'a, E, F>(
     global: &impl GlobalInstanceLike,
     entity: E,
     f: F,
-) -> QueryResult<T>
+) -> QueryResult<()>
 where
-    F: FnOnce(&Model) -> QueryResult<T>,
+    F: FnOnce(&Model) -> QueryResult<dml::QueryExecMeta>,
     E: 'a + EntityLocator<'a>,
 {
     let (space_name, model_name) = entity.parse_entity()?;
@@ -71,11 +71,15 @@ where
         .namespace()
         .with_model((space_name, model_name), |mdl| {
             let r = f(mdl);
-            // see if this task local delta is full
-            if r.is_ok() {
-                model::DeltaState::guard_delta_overflow(global, space_name, model_name, mdl);
+            match r {
+                Ok(dhint) => {
+                    model::DeltaState::guard_delta_overflow(
+                        global, space_name, model_name, mdl, dhint,
+                    );
+                    Ok(())
+                }
+                Err(e) => Err(e),
             }
-            r
         })
 }
 
@@ -101,7 +105,7 @@ impl GlobalNS {
     ) -> QueryResult<T> {
         let sread = self.index_space.read();
         let Some(space) = sread.st_get(space) else {
-            return Err(QueryError::QPObjectNotFound);
+            return Err(QueryError::QExecObjectNotFound);
         };
         f(space)
     }
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index d9716a70..fe2fdd79 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -84,7 +84,7 @@ fn no_field(mr: &IWModel, new: &str) -> bool {
 fn check_nullable(props: &mut HashMap<Box<str>, DictEntryGeneric>) -> QueryResult<bool> {
     match props.remove("nullable") {
         Some(DictEntryGeneric::Data(b)) if b.kind() == TagClass::Bool => Ok(b.bool()),
-        Some(_) => Err(QueryError::QPDdlInvalidProperties),
+        Some(_) => Err(QueryError::QExecDdlInvalidProperties),
         None => Ok(false),
     }
 }
@@ -101,7 +101,7 @@ impl<'a> AlterPlan<'a> {
             AlterKind::Remove(r) => {
                 let mut x = HashSet::new();
                 if !r.iter().all(|id| x.insert(id.as_str())) {
-                    return Err(QueryError::QPDdlModelAlterIllegal);
+                    return Err(QueryError::QExecDdlModelAlterIllegal);
                 }
                 let mut not_found = false;
                 if r.iter().all(|id| {
@@ -112,9 +112,9 @@ impl<'a> AlterPlan<'a> {
                 }) {
                     can_ignore!(AlterAction::Remove(r))
                 } else if not_found {
-                    return Err(QueryError::QPUnknownField);
+                    return Err(QueryError::QExecUnknownField);
                 } else {
-                    return Err(QueryError::QPDdlModelAlterIllegal);
+                    return Err(QueryError::QExecDdlModelAlterIllegal);
                 }
             }
             AlterKind::Add(new_fields) => {
@@ -148,7 +148,7 @@ impl<'a> AlterPlan<'a> {
                     mv.guard_pk(&field_name)?;
                     // get the current field
                     let Some(current_field) = wm.fields().st_get(field_name.as_str()) else {
-                        return Err(QueryError::QPUnknownField);
+                        return Err(QueryError::QExecUnknownField);
                     };
                     // check props
                     let is_nullable = check_nullable(&mut props)?;
@@ -174,7 +174,7 @@ impl<'a> AlterPlan<'a> {
                 no_lock,
             })
         } else {
-            Err(QueryError::QPDdlModelAlterIllegal)
+            Err(QueryError::QExecDdlModelAlterIllegal)
         }
     }
     fn ldeltas(
@@ -197,7 +197,7 @@ impl<'a> AlterPlan<'a> {
         }
         if layers.len() > current.layers().len() {
             // simply a dumb tomato; ELIMINATE THESE DUMB TOMATOES
-            return Err(QueryError::QPDdlModelAlterIllegal);
+            return Err(QueryError::QExecDdlModelAlterIllegal);
         }
         let mut no_lock = !(current.is_nullable() & !nullable);
         let mut deltasize = (current.is_nullable() ^ nullable) as usize;
@@ -216,7 +216,7 @@ impl<'a> AlterPlan<'a> {
             // actually parse the new layer
             okay &= props.is_empty();
             let Some(new_parsed_layer) = Layer::get_layer(&ty) else {
-                return Err(QueryError::QPDdlInvalidTypeDefinition);
+                return Err(QueryError::QExecDdlInvalidTypeDefinition);
             };
             match (
                 current_layer.tag.tag_selector(),
@@ -233,7 +233,7 @@ impl<'a> AlterPlan<'a> {
                 }
                 _ => {
                     // can't cast this directly
-                    return Err(QueryError::QPDdlInvalidTypeDefinition);
+                    return Err(QueryError::QExecDdlInvalidTypeDefinition);
                 }
             }
             *new_layer = new_parsed_layer;
@@ -243,13 +243,12 @@ impl<'a> AlterPlan<'a> {
         if okay {
             Ok((deltasize != 0, new_field))
         } else {
-            Err(QueryError::QPDdlModelAlterIllegal)
+            Err(QueryError::QExecDdlModelAlterIllegal)
         }
     }
 }
 
 impl Model {
-    #[allow(unused)]
     pub fn transactional_exec_alter<G: GlobalInstanceLike>(
         global: &G,
         alter: AlterModel,
@@ -264,7 +263,7 @@ impl Model {
                 // we have a legal plan; acquire exclusive if we need it
                 if !plan.no_lock {
                     // TODO(@ohsayan): allow this later on, once we define the syntax
-                    return Err(QueryError::QPNeedLock);
+                    return Err(QueryError::QExecNeedLock);
                 }
                 // fine, we're good
                 let mut iwm = iwm;
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index 0565f481..af1c8d4d 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -30,7 +30,7 @@ use {
     super::{Fields, Model},
     crate::{
         engine::{
-            core::index::Row,
+            core::{dml::QueryExecMeta, index::Row},
             fractal::{FractalToken, GlobalInstanceLike},
             sync::atm::Guard,
             sync::queue::Queue,
@@ -186,19 +186,9 @@ impl DeltaState {
         space_name: &str,
         model_name: &str,
         model: &Model,
+        hint: QueryExecMeta,
     ) {
-        let current_deltas_size = model.delta_state().data_deltas_size.load(Ordering::Acquire);
-        let max_len = global
-            .get_max_delta_size()
-            .min((model.primary_index().count() as f64 * 0.05) as usize);
-        if compiler::unlikely(current_deltas_size >= max_len) {
-            global.request_batch_resolve(
-                space_name,
-                model_name,
-                model.get_uuid(),
-                current_deltas_size,
-            );
-        }
+        global.request_batch_resolve_if_cache_full(space_name, model_name, model, hint)
     }
 }
 
@@ -211,12 +201,12 @@ impl DeltaState {
         schema_version: DeltaVersion,
         data_version: DeltaVersion,
         g: &Guard,
-    ) {
-        self.append_new_data_delta(DataDelta::new(schema_version, data_version, row, kind), g);
+    ) -> usize {
+        self.append_new_data_delta(DataDelta::new(schema_version, data_version, row, kind), g)
     }
-    pub fn append_new_data_delta(&self, delta: DataDelta, g: &Guard) {
+    pub fn append_new_data_delta(&self, delta: DataDelta, g: &Guard) -> usize {
         self.data_deltas.blocking_enqueue(delta, g);
-        self.data_deltas_size.fetch_add(1, Ordering::Release);
+        self.data_deltas_size.fetch_add(1, Ordering::Release) + 1
     }
     pub fn create_new_data_delta_version(&self) -> DeltaVersion {
         DeltaVersion(self.__data_delta_step())
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 82acd88c..61e1557f 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -134,7 +134,7 @@ impl Model {
     }
     fn guard_pk(&self, new: &str) -> QueryResult<()> {
         if self.is_pk(new) {
-            Err(QueryError::QPDdlModelAlterIllegal)
+            Err(QueryError::QExecDdlModelAlterIllegal)
         } else {
             Ok(())
         }
@@ -195,12 +195,11 @@ impl Model {
                 return Ok(Self::new_restore(Uuid::new(), last_pk.into(), tag, fields));
             }
         }
-        Err(QueryError::QPDdlModelBadDefinition)
+        Err(QueryError::QExecDdlModelBadDefinition)
     }
 }
 
 impl Model {
-    #[allow(unused)]
     pub fn transactional_exec_create<G: GlobalInstanceLike>(
         global: &G,
         stmt: CreateModel,
@@ -210,7 +209,7 @@ impl Model {
         global.namespace().with_space(space_name, |space| {
             let mut w_space = space.models().write();
             if w_space.st_contains(model_name) {
-                return Err(QueryError::QPDdlObjectAlreadyExists);
+                return Err(QueryError::QExecDdlObjectAlreadyExists);
             }
             if G::FS_IS_NON_NULL {
                 let irm = model.intent_read_model();
@@ -251,7 +250,6 @@ impl Model {
             Ok(())
         })
     }
-    #[allow(unused)]
     pub fn transactional_exec_drop<G: GlobalInstanceLike>(
         global: &G,
         stmt: DropModel,
@@ -260,7 +258,7 @@ impl Model {
         global.namespace().with_space(space_name, |space| {
             let mut w_space = space.models().write();
             let Some(model) = w_space.get(model_name) else {
-                return Err(QueryError::QPObjectNotFound);
+                return Err(QueryError::QExecObjectNotFound);
             };
             if G::FS_IS_NON_NULL {
                 // prepare txn
@@ -366,7 +364,7 @@ impl Field {
                 nullable,
             })
         } else {
-            Err(QueryError::QPDdlInvalidTypeDefinition)
+            Err(QueryError::QExecDdlInvalidTypeDefinition)
         }
     }
     #[inline(always)]
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 52013191..05685d0e 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -93,7 +93,7 @@ impl Space {
         {
             Ok(())
         } else {
-            Err(QueryError::QPDdlObjectAlreadyExists)
+            Err(QueryError::QExecDdlObjectAlreadyExists)
         }
     }
     pub fn get_uuid(&self) -> Uuid {
@@ -112,7 +112,7 @@ impl Space {
     ) -> QueryResult<T> {
         let mread = self.mns.read();
         let Some(model) = mread.st_get(model) else {
-            return Err(QueryError::QPObjectNotFound);
+            return Err(QueryError::QExecObjectNotFound);
         };
         f(model)
     }
@@ -161,7 +161,7 @@ impl Space {
             None if props.is_empty() => IndexST::default(),
             _ => {
                 // unknown properties
-                return Err(QueryError::QPDdlInvalidProperties);
+                return Err(QueryError::QExecDdlInvalidProperties);
             }
         };
         Ok(ProcedureCreate {
@@ -178,7 +178,6 @@ impl Space {
 }
 
 impl Space {
-    #[allow(unused)]
     pub fn transactional_exec_create<G: GlobalInstanceLike>(
         global: &G,
         space: CreateSpace,
@@ -188,7 +187,7 @@ impl Space {
         // acquire access
         let mut wl = global.namespace().spaces().write();
         if wl.st_contains(&space_name) {
-            return Err(QueryError::QPDdlObjectAlreadyExists);
+            return Err(QueryError::QExecDdlObjectAlreadyExists);
         }
         // commit txn
         if G::FS_IS_NON_NULL {
@@ -229,13 +228,13 @@ impl Space {
                 Some(DictEntryGeneric::Map(_)) if updated_props.len() == 1 => {}
                 Some(DictEntryGeneric::Data(l)) if updated_props.len() == 1 && l.is_null() => {}
                 None if updated_props.is_empty() => return Ok(()),
-                _ => return Err(QueryError::QPDdlInvalidProperties),
+                _ => return Err(QueryError::QExecDdlInvalidProperties),
             }
             let mut space_props = space.meta.dict().write();
             // create patch
             let patch = match dict::rprepare_metadata_patch(&space_props, updated_props) {
                 Some(patch) => patch,
-                None => return Err(QueryError::QPDdlInvalidProperties),
+                None => return Err(QueryError::QExecDdlInvalidProperties),
             };
             if G::FS_IS_NON_NULL {
                 // prepare txn
@@ -255,7 +254,6 @@ impl Space {
             Ok(())
         })
     }
-    #[allow(unused)]
     pub fn transactional_exec_drop<G: GlobalInstanceLike>(
         global: &G,
         DropSpace { space, force: _ }: DropSpace,
@@ -266,11 +264,11 @@ impl Space {
         let mut wgns = global.namespace().spaces().write();
         let space = match wgns.get(space_name.as_str()) {
             Some(space) => space,
-            None => return Err(QueryError::QPObjectNotFound),
+            None => return Err(QueryError::QExecObjectNotFound),
         };
         let space_w = space.mns.write();
         if space_w.st_len() != 0 {
-            return Err(QueryError::QPDdlNotEmpty);
+            return Err(QueryError::QExecDdlNotEmpty);
         }
         // we can remove this
         if G::FS_IS_NON_NULL {
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index 09ac8fc7..3d6246ef 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -164,7 +164,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            QueryError::QPUnknownField
+            QueryError::QExecUnknownField
         );
     }
     #[test]
@@ -176,7 +176,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            QueryError::QPDdlModelAlterIllegal
+            QueryError::QExecDdlModelAlterIllegal
         );
     }
     #[test]
@@ -188,7 +188,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            QueryError::QPDdlModelAlterIllegal
+            QueryError::QExecDdlModelAlterIllegal
         );
     }
     #[test]
@@ -200,7 +200,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            QueryError::QPDdlModelAlterIllegal
+            QueryError::QExecDdlModelAlterIllegal
         );
     }
     #[test]
@@ -212,7 +212,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            QueryError::QPDdlModelAlterIllegal
+            QueryError::QExecDdlModelAlterIllegal
         );
     }
     #[test]
@@ -224,7 +224,7 @@ mod plan {
                 |_| {}
             )
             .unwrap_err(),
-            QueryError::QPUnknownField
+            QueryError::QExecUnknownField
         );
     }
     fn bad_type_cast(orig_ty: &str, new_ty: &str) {
@@ -235,7 +235,7 @@ mod plan {
             super::with_plan(&create, &alter, |_| {}).expect_err(&format!(
                 "found no error in transformation: {orig_ty} -> {new_ty}"
             )),
-            QueryError::QPDdlInvalidTypeDefinition,
+            QueryError::QExecDdlInvalidTypeDefinition,
             "failed to match error in transformation: {orig_ty} -> {new_ty}",
         )
     }
@@ -445,7 +445,7 @@ mod exec {
                 |_| {},
             )
             .unwrap_err(),
-            QueryError::QPNeedLock
+            QueryError::QExecNeedLock
         );
     }
 }
diff --git a/server/src/engine/core/tests/ddl_model/crt.rs b/server/src/engine/core/tests/ddl_model/crt.rs
index 7193dc14..fa52a40d 100644
--- a/server/src/engine/core/tests/ddl_model/crt.rs
+++ b/server/src/engine/core/tests/ddl_model/crt.rs
@@ -89,7 +89,7 @@ mod validation {
                 "create model mymodel(primary username: string, primary contract_location: binary)"
             )
             .unwrap_err(),
-            QueryError::QPDdlModelBadDefinition
+            QueryError::QExecDdlModelBadDefinition
         );
     }
 
@@ -97,7 +97,7 @@ mod validation {
     fn duplicate_fields() {
         assert_eq!(
             create("create model mymodel(primary username: string, username: binary)").unwrap_err(),
-            QueryError::QPDdlModelBadDefinition
+            QueryError::QExecDdlModelBadDefinition
         );
     }
 
@@ -105,7 +105,7 @@ mod validation {
     fn illegal_props() {
         assert_eq!(
         create("create model mymodel(primary username: string, password: binary) with { lol_prop: false }").unwrap_err(),
-        QueryError::QPDdlModelBadDefinition
+        QueryError::QExecDdlModelBadDefinition
     );
     }
 
@@ -116,12 +116,12 @@ mod validation {
             "create model mymodel(primary username_bytes: list { type: uint8 }, password: binary)"
         )
         .unwrap_err(),
-        QueryError::QPDdlModelBadDefinition
+        QueryError::QExecDdlModelBadDefinition
     );
         assert_eq!(
             create("create model mymodel(primary username: float32, password: binary)")
                 .unwrap_err(),
-            QueryError::QPDdlModelBadDefinition
+            QueryError::QExecDdlModelBadDefinition
         );
     }
 }
diff --git a/server/src/engine/core/tests/ddl_model/layer.rs b/server/src/engine/core/tests/ddl_model/layer.rs
index 0ee2626e..345ed2ce 100644
--- a/server/src/engine/core/tests/ddl_model/layer.rs
+++ b/server/src/engine/core/tests/ddl_model/layer.rs
@@ -64,7 +64,7 @@ mod layer_spec_validation {
     fn invalid_list() {
         assert_eq!(
             layerview("list").unwrap_err(),
-            QueryError::QPDdlInvalidTypeDefinition
+            QueryError::QExecDdlInvalidTypeDefinition
         );
     }
 
@@ -72,7 +72,7 @@ mod layer_spec_validation {
     fn invalid_flat() {
         assert_eq!(
             layerview("string { type: string }").unwrap_err(),
-            QueryError::QPDdlInvalidTypeDefinition
+            QueryError::QExecDdlInvalidTypeDefinition
         );
     }
 }
diff --git a/server/src/engine/core/tests/ddl_space/alter.rs b/server/src/engine/core/tests/ddl_space/alter.rs
index e71e7ec0..0ae95e23 100644
--- a/server/src/engine/core/tests/ddl_space/alter.rs
+++ b/server/src/engine/core/tests/ddl_space/alter.rs
@@ -122,7 +122,7 @@ fn alter_nx() {
             |_| {},
         )
         .unwrap_err(),
-        QueryError::QPObjectNotFound
+        QueryError::QExecObjectNotFound
     );
 }
 
diff --git a/server/src/engine/core/tests/ddl_space/create.rs b/server/src/engine/core/tests/ddl_space/create.rs
index b6e70bb2..8a42cf55 100644
--- a/server/src/engine/core/tests/ddl_space/create.rs
+++ b/server/src/engine/core/tests/ddl_space/create.rs
@@ -73,7 +73,7 @@ fn exec_create_space_with_bad_env_type() {
     let global = TestGlobal::new_with_tmp_nullfs_driver();
     assert_eq!(
         super::exec_create(&global, "create space myspace with { env: 100 }", |_| {}).unwrap_err(),
-        QueryError::QPDdlInvalidProperties
+        QueryError::QExecDdlInvalidProperties
     );
 }
 
@@ -87,6 +87,6 @@ fn exec_create_space_with_random_property() {
             |_| {}
         )
         .unwrap_err(),
-        QueryError::QPDdlInvalidProperties
+        QueryError::QExecDdlInvalidProperties
     );
 }
diff --git a/server/src/engine/core/tests/dml/delete.rs b/server/src/engine/core/tests/dml/delete.rs
index 5a8a32c4..017ba0ee 100644
--- a/server/src/engine/core/tests/dml/delete.rs
+++ b/server/src/engine/core/tests/dml/delete.rs
@@ -51,6 +51,6 @@ fn delete_nonexisting() {
             "sayan",
         )
         .unwrap_err(),
-        QueryError::QPDmlRowNotFound
+        QueryError::QExecDmlRowNotFound
     );
 }
diff --git a/server/src/engine/core/tests/dml/insert.rs b/server/src/engine/core/tests/dml/insert.rs
index 109d41c8..c1c99cbc 100644
--- a/server/src/engine/core/tests/dml/insert.rs
+++ b/server/src/engine/core/tests/dml/insert.rs
@@ -83,6 +83,6 @@ fn insert_duplicate() {
     assert_eq!(
         super::exec_insert_only(&global, "insert into myspace.mymodel('sayan', 'pass123')")
             .unwrap_err(),
-        QueryError::QPDmlDuplicate
+        QueryError::QExecDmlDuplicate
     );
 }
diff --git a/server/src/engine/core/tests/dml/select.rs b/server/src/engine/core/tests/dml/select.rs
index 966b0411..2af7dfd6 100644
--- a/server/src/engine/core/tests/dml/select.rs
+++ b/server/src/engine/core/tests/dml/select.rs
@@ -97,6 +97,6 @@ fn select_nonexisting() {
             "select username, password from myspace.mymodel where username = 'notsayan'",
         )
         .unwrap_err(),
-        QueryError::QPDmlRowNotFound
+        QueryError::QExecDmlRowNotFound
     );
 }
diff --git a/server/src/engine/core/tests/dml/update.rs b/server/src/engine/core/tests/dml/update.rs
index b0cf57d3..577cf465 100644
--- a/server/src/engine/core/tests/dml/update.rs
+++ b/server/src/engine/core/tests/dml/update.rs
@@ -96,7 +96,7 @@ fn fail_operation_on_null() {
             "select * from myspace.mymodel where username='sayan'"
         )
         .unwrap_err(),
-        QueryError::QPDmlValidationError
+        QueryError::QExecDmlValidationError
     );
     assert_eq!(
         dml::update_flow_trace(),
@@ -116,7 +116,7 @@ fn fail_unknown_fields() {
             "select * from myspace.mymodel where username='sayan'"
         )
         .unwrap_err(),
-        QueryError::QPUnknownField
+        QueryError::QExecUnknownField
     );
     assert_eq!(dml::update_flow_trace(), ["fieldnotfound", "rollback"]);
     // verify integrity
@@ -142,7 +142,7 @@ fn fail_typedef_violation() {
             "select * from myspace.mymodel where username = 'sayan'"
         )
         .unwrap_err(),
-        QueryError::QPDmlValidationError
+        QueryError::QExecDmlValidationError
     );
     assert_eq!(
         dml::update_flow_trace(),
diff --git a/server/src/engine/core/util.rs b/server/src/engine/core/util.rs
index 15c07be6..f7f54895 100644
--- a/server/src/engine/core/util.rs
+++ b/server/src/engine/core/util.rs
@@ -46,6 +46,6 @@ impl<'a> EntityLocator<'a> for Entity<'a> {
     where
         Self: 'a,
     {
-        self.into_full_str().ok_or(QueryError::QPExpectedEntity)
+        self.into_full_str().ok_or(QueryError::QLExpectedEntity)
     }
 }
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 1d0ef66f..a9aeae83 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -31,71 +31,78 @@ pub type QueryResult<T> = Result<T, QueryError>;
 
 /// an enumeration of 'flat' errors that the server actually responds to the client with, since we do not want to send specific information
 /// about anything (as that will be a security hole). The variants correspond with their actual response codes
-#[derive(Debug, Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq, sky_macros::EnumMethods)]
 #[repr(u8)]
 pub enum QueryError {
+    // system
     /// I/O error
-    SysServerError,
+    SysServerError = 0,
     /// out of memory
-    SysOutOfMemory,
+    SysOutOfMemory = 1,
     /// unknown server error
-    SysUnknownError,
+    SysUnknownError = 2,
+    /// system auth error
+    SysAuthError = 3,
+    /// transactional error
+    SysTransactionalError = 4,
+    // exchange
+    NetworkSubsystemCorruptedPacket = 24,
+    // QL
     /// something like an integer that randomly has a character to attached to it like `1234q`
-    LexInvalidLiteral,
+    LexInvalidLiteral = 25,
     /// something like an invalid 'string" or a safe string with a bad length etc
-    LexInvalidEscapedLiteral,
+    LexInvalidParameter = 26,
     /// unexpected byte
-    LexUnexpectedByte,
+    LexUnexpectedByte = 27,
     /// expected a longer statement
-    QLUnexpectedEndOfStatement,
+    QLUnexpectedEndOfStatement = 28,
     /// incorrect syntax for "something"
-    QLInvalidSyntax,
+    QLInvalidSyntax = 29,
     /// invalid collection definition definition
-    QLInvalidCollectionSyntax,
+    QLInvalidCollectionSyntax = 30,
     /// invalid type definition syntax
-    QLInvalidTypeDefinitionSyntax,
+    QLInvalidTypeDefinitionSyntax = 31,
     /// expected a full entity definition
-    QPExpectedEntity,
+    QLExpectedEntity = 32,
     /// expected a statement, found something else
-    QPExpectedStatement,
+    QLExpectedStatement = 33,
     /// unknown statement
-    QPUnknownStatement,
-    /// this query needs a lock for execution, but that wasn't explicitly allowed anywhere
-    QPNeedLock,
+    QLUnknownStatement = 34,
+    // exec
     /// the object to be used as the "query container" is missing (for example, insert when the model was missing)
-    QPObjectNotFound,
+    QExecObjectNotFound = 100,
     /// an unknown field was attempted to be accessed/modified/...
-    QPUnknownField,
+    QExecUnknownField = 101,
     /// invalid property for an object
-    QPDdlInvalidProperties,
+    QExecDdlInvalidProperties = 102,
     /// create space/model, but the object already exists
-    QPDdlObjectAlreadyExists,
+    QExecDdlObjectAlreadyExists = 103,
     /// an object that was attempted to be removed is non-empty, and for this object, removals require it to be empty
-    QPDdlNotEmpty,
+    QExecDdlNotEmpty = 104,
     /// invalid type definition
-    QPDdlInvalidTypeDefinition,
+    QExecDdlInvalidTypeDefinition = 105,
     /// bad model definition
-    QPDdlModelBadDefinition,
+    QExecDdlModelBadDefinition = 106,
     /// illegal alter model query
-    QPDdlModelAlterIllegal,
+    QExecDdlModelAlterIllegal = 107,
+    // exec DML
     /// violated the uniqueness property
-    QPDmlDuplicate,
+    QExecDmlDuplicate = 150,
     /// the data could not be validated for being accepted into a field/function/etc.
-    QPDmlValidationError,
+    QExecDmlValidationError = 151,
     /// the where expression has an unindexed column essentially implying that we can't run this query because of perf concerns
-    QPDmlWhereHasUnindexedColumn,
+    QExecDmlWhereHasUnindexedColumn = 152,
     /// the row matching the given match expression was not found
-    QPDmlRowNotFound,
-    /// transactional error
-    TransactionalError,
-    SysAuthError,
+    QExecDmlRowNotFound = 153,
+    /// this query needs a lock for execution, but that wasn't explicitly allowed anywhere
+    QExecNeedLock = 154,
 }
 
 impl From<super::fractal::error::Error> for QueryError {
     fn from(e: super::fractal::error::Error) -> Self {
         match e.kind() {
             ErrorKind::IoError(_) | ErrorKind::Storage(_) => QueryError::SysServerError,
-            ErrorKind::Txn(_) => QueryError::TransactionalError,
+            ErrorKind::Txn(_) => QueryError::SysTransactionalError,
             ErrorKind::Other(_) => QueryError::SysUnknownError,
             ErrorKind::Config(_) => unreachable!("config error cannot propagate here"),
         }
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 6f669c2d..e83e53da 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -271,11 +271,6 @@ impl FractalMgr {
                                 // branch returning. but it is okay
                                 return Ok(());
                             }
-                            // mark that we're taking these deltas
-                            model.delta_state().__fractal_take_from_data_delta(
-                                observed_size,
-                                super::FractalToken::new(),
-                            );
                             Self::try_write_model_data_batch(model, observed_size, mdl_driver)
                         },
                     );
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index 91235402..c23854a1 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -26,7 +26,7 @@
 
 use {
     super::{
-        core::GlobalNS,
+        core::{dml::QueryExecMeta, model::Model, GlobalNS},
         data::uuid::Uuid,
         storage::{
             self,
@@ -123,17 +123,26 @@ pub trait GlobalInstanceLike {
     fn taskmgr_post_high_priority(&self, task: Task<CriticalTask>);
     fn taskmgr_post_standard_priority(&self, task: Task<GenericTask>);
     // default impls
-    fn request_batch_resolve(
+    fn request_batch_resolve_if_cache_full(
         &self,
         space_name: &str,
         model_name: &str,
-        model_uuid: Uuid,
-        observed_len: usize,
+        model: &Model,
+        hint: QueryExecMeta,
     ) {
-        self.taskmgr_post_high_priority(Task::new(CriticalTask::WriteBatch(
-            ModelUniqueID::new(space_name, model_name, model_uuid),
-            observed_len,
-        )))
+        let current_delta_size = hint.delta_hint();
+        let index_size = model.primary_index().count();
+        let five = (index_size as f64 * 0.05) as usize;
+        let max_delta = five.max(self.get_max_delta_size());
+        if current_delta_size >= max_delta {
+            let obtained_delta_size = model
+                .delta_state()
+                .__fractal_take_full_from_data_delta(FractalToken::new());
+            self.taskmgr_post_high_priority(Task::new(CriticalTask::WriteBatch(
+                ModelUniqueID::new(space_name, model_name, model.get_uuid()),
+                obtained_delta_size,
+            )));
+        }
     }
     // config handle
     fn sys_cfg(&self) -> &config::SysConfig;
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index 0256eb7a..b23e8e72 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -88,6 +88,233 @@ macro_rules! flags {
     );
 }
 
+macro_rules! __kw_misc {
+    ($ident:ident) => {
+        $crate::engine::ql::lex::Token::Keyword($crate::engine::ql::lex::Keyword::Misc(
+            $crate::engine::ql::lex::KeywordMisc::$ident,
+        ))
+    };
+}
+
+macro_rules! __kw_stmt {
+    ($ident:ident) => {
+        $crate::engine::ql::lex::Token::Keyword($crate::engine::ql::lex::Keyword::Statement(
+            $crate::engine::ql::lex::KeywordStmt::$ident,
+        ))
+    };
+}
+
+/*
+    Frankly, this is just for lazy people like me. Do not judge
+    -- Sayan (@ohsayan)
+*/
+macro_rules! Token {
+    // misc symbol
+    (@) => {
+        __sym_token!(SymAt)
+    };
+    (#) => {
+        __sym_token!(SymHash)
+    };
+    ($) => {
+        __sym_token!(SymDollar)
+    };
+    (%) => {
+        __sym_token!(SymPercent)
+    };
+    (.) => {
+        __sym_token!(SymPeriod)
+    };
+    (,) => {
+        __sym_token!(SymComma)
+    };
+    (_) => {
+        __sym_token!(SymUnderscore)
+    };
+    (?) => {
+        __sym_token!(SymQuestion)
+    };
+    (:) => {
+        __sym_token!(SymColon)
+    };
+    (;) => {
+        __sym_token!(SymSemicolon)
+    };
+    (~) => {
+        __sym_token!(SymTilde)
+    };
+    // logical
+    (!) => {
+        __sym_token!(OpLogicalNot)
+    };
+    (^) => {
+        __sym_token!(OpLogicalXor)
+    };
+    (&) => {
+        __sym_token!(OpLogicalAnd)
+    };
+    (|) => {
+        __sym_token!(OpLogicalOr)
+    };
+    // operator misc.
+    (=) => {
+        __sym_token!(OpAssign)
+    };
+    // arithmetic
+    (+) => {
+        __sym_token!(OpArithmeticAdd)
+    };
+    (-) => {
+        __sym_token!(OpArithmeticSub)
+    };
+    (*) => {
+        __sym_token!(OpArithmeticMul)
+    };
+    (/) => {
+        __sym_token!(OpArithmeticDiv)
+    };
+    // relational
+    (>) => {
+        __sym_token!(OpComparatorGt)
+    };
+    (<) => {
+        __sym_token!(OpComparatorLt)
+    };
+    // ddl keywords
+    (use) => {
+        __kw_stmt!(Use)
+    };
+    (create) => {
+        __kw_stmt!(Create)
+    };
+    (alter) => {
+        __kw_stmt!(Alter)
+    };
+    (drop) => {
+        __kw_stmt!(Drop)
+    };
+    (model) => {
+        __kw_misc!(Model)
+    };
+    (space) => {
+        __kw_misc!(Space)
+    };
+    (primary) => {
+        __kw_misc!(Primary)
+    };
+    // ddl misc
+    (with) => {
+        __kw_misc!(With)
+    };
+    (add) => {
+        __kw_misc!(Add)
+    };
+    (remove) => {
+        __kw_misc!(Remove)
+    };
+    (sort) => {
+        __kw_misc!(Sort)
+    };
+    (type) => {
+        __kw_misc!(Type)
+    };
+    // dml
+    (insert) => {
+        __kw_stmt!(Insert)
+    };
+    (select) => {
+        __kw_stmt!(Select)
+    };
+    (update) => {
+        __kw_stmt!(Update)
+    };
+    (delete) => {
+        __kw_stmt!(Delete)
+    };
+    // dml misc
+    (set) => {
+        __kw_misc!(Set)
+    };
+    (limit) => {
+        __kw_misc!(Limit)
+    };
+    (from) => {
+        __kw_misc!(From)
+    };
+    (into) => {
+        __kw_misc!(Into)
+    };
+    (where) => {
+        __kw_misc!(Where)
+    };
+    (if) => {
+        __kw_misc!(If)
+    };
+    (and) => {
+        __kw_misc!(And)
+    };
+    (as) => {
+        __kw_misc!(As)
+    };
+    (by) => {
+        __kw_misc!(By)
+    };
+    (asc) => {
+        __kw_misc!(Asc)
+    };
+    (desc) => {
+        __kw_misc!(Desc)
+    };
+    // types
+    (string) => {
+        __kw_misc!(String)
+    };
+    (binary) => {
+        __kw_misc!(Binary)
+    };
+    (list) => {
+        __kw_misc!(List)
+    };
+    (map) => {
+        __kw_misc!(Map)
+    };
+    (bool) => {
+        __kw_misc!(Bool)
+    };
+    (int) => {
+        __kw_misc!(Int)
+    };
+    (double) => {
+        __kw_misc!(Double)
+    };
+    (float) => {
+        __kw_misc!(Float)
+    };
+    // tt
+    (open {}) => {
+        __sym_token!(TtOpenBrace)
+    };
+    (close {}) => {
+        __sym_token!(TtCloseBrace)
+    };
+    (() open) => {
+        __sym_token!(TtOpenParen)
+    };
+    (() close) => {
+        __sym_token!(TtCloseParen)
+    };
+    (open []) => {
+        __sym_token!(TtOpenSqBracket)
+    };
+    (close []) => {
+        __sym_token!(TtCloseSqBracket)
+    };
+    // misc
+    (null) => {
+        __kw_misc!(Null)
+    };
+}
+
 macro_rules! union {
     ($(#[$attr:meta])* $vis:vis union $name:ident $tail:tt) => (union!(@parse [$(#[$attr])* $vis union $name] [] $tail););
     ($(#[$attr:meta])* $vis:vis union $name:ident<$($lt:lifetime),*> $tail:tt) => (union!(@parse [$(#[$attr])* $vis union $name<$($lt),*>] [] $tail););
diff --git a/server/src/engine/net/mod.rs b/server/src/engine/net/mod.rs
index 496a6730..2e1977c2 100644
--- a/server/src/engine/net/mod.rs
+++ b/server/src/engine/net/mod.rs
@@ -24,12 +24,13 @@
  *
 */
 
-use super::config::ConfigEndpointTcp;
-
 pub mod protocol;
 
 use {
-    crate::engine::{error::RuntimeResult, fractal::error::ErrorContext, fractal::Global},
+    crate::engine::{
+        config::ConfigEndpointTcp, error::RuntimeResult, fractal::error::ErrorContext,
+        fractal::Global,
+    },
     bytes::BytesMut,
     openssl::{
         pkey::PKey,
@@ -39,7 +40,7 @@ use {
     },
     std::{cell::Cell, net::SocketAddr, pin::Pin, time::Duration},
     tokio::{
-        io::{AsyncRead, AsyncWrite, BufWriter},
+        io::{AsyncRead, AsyncWrite, BufWriter, AsyncWriteExt},
         net::{TcpListener, TcpStream},
         sync::{broadcast, mpsc, Semaphore},
     },
@@ -128,6 +129,7 @@ impl<S: Socket> ConnectionHandler<S> {
         loop {
             tokio::select! {
                 ret = protocol::query_loop(socket, buffer, global) => {
+                    socket.flush().await?;
                     match ret {
                         Ok(QueryLoopResult::Fin) => return Ok(()),
                         Ok(QueryLoopResult::Rst) => error!("connection reset while talking to client"),
diff --git a/server/src/engine/net/protocol/exchange.rs b/server/src/engine/net/protocol/exchange.rs
index 5bfe1904..1a1a1547 100644
--- a/server/src/engine/net/protocol/exchange.rs
+++ b/server/src/engine/net/protocol/exchange.rs
@@ -80,28 +80,7 @@ pub fn resume<'a>(
     state: QueryTimeExchangeState,
 ) -> QueryTimeExchangeResult<'a> {
     match state {
-        QueryTimeExchangeState::Initial => {
-            if cfg!(debug_assertions) {
-                if !scanner.has_left(EXCHANGE_MIN_SIZE) {
-                    return STATE_READ_INITIAL;
-                }
-            } else {
-                assert!(scanner.has_left(EXCHANGE_MIN_SIZE));
-            }
-            // attempt to read atleast one byte
-            if cfg!(debug_assertions) {
-                match scanner.try_next_byte() {
-                    Some(b'S') => SQuery::resume_initial(scanner),
-                    Some(_) => return STATE_ERROR,
-                    None => return STATE_READ_INITIAL,
-                }
-            } else {
-                match unsafe { scanner.next_byte() } {
-                    b'S' => SQuery::resume_initial(scanner),
-                    _ => return STATE_ERROR,
-                }
-            }
-        }
+        QueryTimeExchangeState::Initial => SQuery::resume_initial(scanner),
         QueryTimeExchangeState::SQ1Meta1Partial { packet_size_part } => {
             SQuery::resume_at_sq1_meta1_partial(scanner, packet_size_part)
         }
@@ -199,6 +178,26 @@ impl<'a> SQuery<'a> {
 impl<'a> SQuery<'a> {
     /// We're touching this packet for the first time
     fn resume_initial(scanner: &mut BufferedScanner<'a>) -> QueryTimeExchangeResult<'a> {
+        if cfg!(debug_assertions) {
+            if !scanner.has_left(EXCHANGE_MIN_SIZE) {
+                return STATE_READ_INITIAL;
+            }
+        } else {
+            assert!(scanner.has_left(EXCHANGE_MIN_SIZE));
+        }
+        // attempt to read atleast one byte
+        if cfg!(debug_assertions) {
+            match scanner.try_next_byte() {
+                Some(b'S') => {}
+                Some(_) => return STATE_ERROR,
+                None => return STATE_READ_INITIAL,
+            }
+        } else {
+            match unsafe { scanner.next_byte() } {
+                b'S' => {}
+                _ => return STATE_ERROR,
+            }
+        }
         Self::resume_at_sq1_meta1_partial(scanner, 0)
     }
     /// We found some part of meta1, and need to resume
@@ -239,11 +238,11 @@ impl<'a> SQuery<'a> {
         match parse_lf_separated(scanner, prev_qw_buffered) {
             LFTIntParseResult::Value(q_window) => {
                 // we got the q window; can we complete the exchange?
-                Self::resume_at_final(
-                    scanner,
-                    q_window as usize,
-                    Self::compute_df_size(scanner, static_size, packet_size),
-                )
+                let df_size = Self::compute_df_size(scanner, static_size, packet_size);
+                if df_size == 0 {
+                    return QueryTimeExchangeResult::Error;
+                }
+                Self::resume_at_final(scanner, q_window as usize, df_size)
             }
             LFTIntParseResult::Partial(q_window_partial) => {
                 // not enough bytes for getting Q window
@@ -290,7 +289,7 @@ impl<'a> SQuery<'a> {
 
 impl<'a> SQuery<'a> {
     fn compute_df_size(scanner: &BufferedScanner, static_size: usize, packet_size: usize) -> usize {
-        packet_size - scanner.cursor() + static_size
+        (packet_size + static_size) - scanner.cursor()
     }
     fn compute_df_remaining(scanner: &BufferedScanner<'_>, df_size: usize) -> usize {
         (scanner.cursor() + df_size) - scanner.buffer_len()
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index a2a2fa23..1044e2a1 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -24,8 +24,6 @@
  *
 */
 
-use tokio::io::AsyncWriteExt;
-
 mod exchange;
 mod handshake;
 #[cfg(test)]
@@ -45,13 +43,19 @@ use {
     super::{IoResult, QueryLoopResult, Socket},
     crate::engine::{
         self,
+        error::QueryError,
         fractal::{Global, GlobalInstanceLike},
         mem::BufferedScanner,
     },
     bytes::{Buf, BytesMut},
-    tokio::io::{AsyncReadExt, BufWriter},
+    tokio::io::{AsyncReadExt, AsyncWriteExt, BufWriter},
 };
 
+#[derive(Debug, PartialEq)]
+pub enum Response {
+    Empty,
+}
+
 pub(super) async fn query_loop<S: Socket>(
     con: &mut BufWriter<S>,
     buf: &mut BytesMut,
@@ -64,13 +68,14 @@ pub(super) async fn query_loop<S: Socket>(
         PostHandshake::ConnectionClosedRst => return Ok(QueryLoopResult::Rst),
         PostHandshake::Error(e) => {
             // failed to handshake; we'll close the connection
-            let hs_err_packet = [b'H', b'1', 1, e.value_u8()];
+            let hs_err_packet = [b'H', 0, 1, e.value_u8()];
             con.write_all(&hs_err_packet).await?;
             return Ok(QueryLoopResult::HSFailed);
         }
     };
     // done handshaking
-    con.write_all(b"H1\x00\x00\x00").await?;
+    con.write_all(b"H\x00\x00\x00").await?;
+    con.flush().await?;
     let mut exchg_state = QueryTimeExchangeState::default();
     let mut expect_more = exchange::EXCHANGE_MIN_SIZE;
     let mut cursor = 0;
@@ -100,21 +105,27 @@ pub(super) async fn query_loop<S: Socket>(
             }
             QueryTimeExchangeResult::SQCompleted(sq) => sq,
             QueryTimeExchangeResult::Error => {
-                con.write_all(b"!\x00").await?;
+                let [a, b] =
+                    (QueryError::NetworkSubsystemCorruptedPacket.value_u8() as u16).to_le_bytes();
+                con.write_all(&[0x10, a, b]).await?;
+                con.flush().await?;
+                buf.clear();
                 exchg_state = QueryTimeExchangeState::default();
                 continue;
             }
         };
         // now execute query
-        match engine::core::exec::execute_query(global, sq).await {
-            Ok(()) => {
-                buf.clear();
+        match engine::core::exec::dispatch_to_executor(global, sq).await {
+            Ok(Response::Empty) => {
+                con.write_all(&[0x12]).await?;
+            }
+            Err(e) => {
+                let [a, b] = (e.value_u8() as u16).to_le_bytes();
+                con.write_all(&[0x10, a, b]).await?;
             }
-            Err(_e) => {
-                // TOOD(@ohsayan): actual error codes! 
-                con.write_all(&[b'!', 1]).await?;
-            },
         }
+        con.flush().await?;
+        buf.clear();
         exchg_state = QueryTimeExchangeState::default();
     }
 }
@@ -152,6 +163,7 @@ async fn do_handshake<S: Socket>(
         match handshake::CHandshake::resume_with(&mut scanner, state) {
             HandshakeResult::Completed(hs) => {
                 handshake = hs;
+                cursor = scanner.cursor();
                 break;
             }
             HandshakeResult::ChangeState { new_state, expect } => {
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 1072ef25..fbc75463 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -26,10 +26,11 @@
 
 pub mod traits;
 
+#[cfg(debug_assertions)]
+use self::traits::ASTNode;
 #[cfg(test)]
 pub use traits::{parse_ast_node_full, parse_ast_node_multiple_full};
 use {
-    self::traits::ASTNode,
     super::{
         ddl, dml,
         lex::{Ident, Token},
@@ -57,7 +58,6 @@ pub struct State<'a, Qd> {
     f: bool,
 }
 
-#[cfg(test)]
 impl<'a> State<'a, InplaceData> {
     pub const fn new_inplace(tok: &'a [Token<'a>]) -> Self {
         Self::new(tok, InplaceData::new())
@@ -392,7 +392,7 @@ impl<'a> Entity<'a> {
                 *c += 1;
                 Self::parse_uck_tokens_single(tok)
             },
-            _ => return Err(QueryError::QPExpectedEntity),
+            _ => return Err(QueryError::QLExpectedEntity),
         };
         Ok(r)
     }
@@ -408,7 +408,7 @@ impl<'a> Entity<'a> {
                 Ok(e.assume_init())
             }
         } else {
-            Err(QueryError::QPExpectedEntity)
+            Err(QueryError::QLExpectedEntity)
         }
     }
     #[inline(always)]
@@ -495,6 +495,7 @@ pub enum Statement<'a> {
 }
 
 #[inline(always)]
+#[cfg(debug_assertions)]
 pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResult<Statement<'a>> {
     if compiler::unlikely(tok.len() < 2) {
         return Err(QueryError::QLUnexpectedEndOfStatement);
@@ -506,12 +507,12 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResul
         Token![create] => match state.fw_read() {
             Token![model] => ASTNode::from_state(&mut state).map(Statement::CreateModel),
             Token![space] => ASTNode::from_state(&mut state).map(Statement::CreateSpace),
-            _ => compiler::cold_rerr(QueryError::QPUnknownStatement),
+            _ => compiler::cold_rerr(QueryError::QLUnknownStatement),
         },
         Token![alter] => match state.fw_read() {
             Token![model] => ASTNode::from_state(&mut state).map(Statement::AlterModel),
             Token![space] => ASTNode::from_state(&mut state).map(Statement::AlterSpace),
-            _ => compiler::cold_rerr(QueryError::QPUnknownStatement),
+            _ => compiler::cold_rerr(QueryError::QLUnknownStatement),
         },
         Token![drop] if state.remaining() >= 2 => ddl::drop::parse_drop(&mut state),
         Token::Ident(id) if id.eq_ignore_ascii_case("inspect") => {
@@ -522,6 +523,6 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResul
         Token![select] => ASTNode::from_state(&mut state).map(Statement::Select),
         Token![update] => ASTNode::from_state(&mut state).map(Statement::Update),
         Token![delete] => ASTNode::from_state(&mut state).map(Statement::Delete),
-        _ => compiler::cold_rerr(QueryError::QPUnknownStatement),
+        _ => compiler::cold_rerr(QueryError::QLUnknownStatement),
     }
 }
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index e4228375..4f833695 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -123,7 +123,7 @@ impl<'a> AlterModel<'a> {
             Token![add] => AlterKind::alter_add(state),
             Token![remove] => AlterKind::alter_remove(state),
             Token![update] => AlterKind::alter_update(state),
-            _ => Err(QueryError::QPExpectedStatement),
+            _ => Err(QueryError::QLExpectedStatement),
         };
         kind.map(|kind| AlterModel::new(model_name, kind))
     }
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index 6ac4b311..fed26836 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -97,7 +97,7 @@ pub fn parse_drop<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResu
     match state.fw_read() {
         Token![model] => DropModel::parse(state).map(Statement::DropModel),
         Token![space] => return DropSpace::parse(state).map(Statement::DropSpace),
-        _ => Err(QueryError::QPUnknownStatement),
+        _ => Err(QueryError::QLUnknownStatement),
     }
 }
 
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
index 827dd84e..ff07600d 100644
--- a/server/src/engine/ql/ddl/ins.rs
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -65,7 +65,7 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
         }
         _ => {
             state.cursor_back();
-            Err(QueryError::QPExpectedStatement)
+            Err(QueryError::QLExpectedStatement)
         }
     }
 }
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index f4296e05..b2fcffe6 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -515,7 +515,7 @@ impl<'a> ExpandedField<'a> {
                     Err(QueryError::QLInvalidSyntax)
                 }
             }
-            _ => Err(QueryError::QPExpectedStatement),
+            _ => Err(QueryError::QLExpectedStatement),
         }
     }
 }
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index 49b3094b..ad9f3478 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -426,7 +426,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             let nb = slf.param_buffer.next_byte();
             slf.l.push_token(Token::Lit(Lit::new_bool(nb == 1)));
             if nb > 1 {
-                slf.l.set_error(QueryError::LexInvalidEscapedLiteral);
+                slf.l.set_error(QueryError::LexInvalidParameter);
             }
         },
         // uint
@@ -435,7 +435,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             .try_next_ascii_u64_lf_separated_or_restore_cursor()
         {
             Some(int) => slf.l.push_token(Lit::new_uint(int)),
-            None => slf.l.set_error(QueryError::LexInvalidEscapedLiteral),
+            None => slf.l.set_error(QueryError::LexInvalidParameter),
         },
         // sint
         |slf| {
@@ -452,7 +452,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
                 .param_buffer
                 .try_next_ascii_u64_lf_separated_or_restore_cursor()
             else {
-                slf.l.set_error(QueryError::LexInvalidEscapedLiteral);
+                slf.l.set_error(QueryError::LexInvalidParameter);
                 return;
             };
             let body = match slf
@@ -461,13 +461,13 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             {
                 Some(body) => body,
                 None => {
-                    slf.l.set_error(QueryError::LexInvalidEscapedLiteral);
+                    slf.l.set_error(QueryError::LexInvalidParameter);
                     return;
                 }
             };
             match core::str::from_utf8(body).map(core::str::FromStr::from_str) {
                 Ok(Ok(fp)) => slf.l.push_token(Lit::new_float(fp)),
-                _ => slf.l.set_error(QueryError::LexInvalidEscapedLiteral),
+                _ => slf.l.set_error(QueryError::LexInvalidParameter),
             }
         },
         // binary
@@ -476,7 +476,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
                 .param_buffer
                 .try_next_ascii_u64_lf_separated_or_restore_cursor()
             else {
-                slf.l.set_error(QueryError::LexInvalidEscapedLiteral);
+                slf.l.set_error(QueryError::LexInvalidParameter);
                 return;
             };
             match slf
@@ -484,7 +484,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
                 .try_next_variable_block(size_of_body as usize)
             {
                 Some(block) => slf.l.push_token(Lit::new_bin(block)),
-                None => slf.l.set_error(QueryError::LexInvalidEscapedLiteral),
+                None => slf.l.set_error(QueryError::LexInvalidParameter),
             }
         },
         // string
@@ -493,7 +493,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
                 .param_buffer
                 .try_next_ascii_u64_lf_separated_or_restore_cursor()
             else {
-                slf.l.set_error(QueryError::LexInvalidEscapedLiteral);
+                slf.l.set_error(QueryError::LexInvalidParameter);
                 return;
             };
             match slf
@@ -503,10 +503,10 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             {
                 // TODO(@ohsayan): obliterate this alloc
                 Some(Ok(s)) => slf.l.push_token(Lit::new_string(s.to_owned())),
-                _ => slf.l.set_error(QueryError::LexInvalidEscapedLiteral),
+                _ => slf.l.set_error(QueryError::LexInvalidParameter),
             }
         },
         // ecc
-        |s| s.l.set_error(QueryError::LexInvalidEscapedLiteral),
+        |s| s.l.set_error(QueryError::LexInvalidParameter),
     ]
 };
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index cb751d29..d64cbafb 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -321,25 +321,26 @@ macro_rules! flattened_lut {
 }
 
 flattened_lut! {
-    static KW_LUT in kwlut;
+    static KW in kw;
     #[derive(Debug, PartialEq, Clone, Copy)]
     #[repr(u8)]
     pub enum Keyword {
         Statement => {
-            #[derive(Debug, PartialEq, Clone, Copy, sky_macros::EnumMethods)]
+            #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, sky_macros::EnumMethods)]
             #[repr(u8)]
             /// A statement keyword
             pub enum KeywordStmt {
-                // sys
+                // system
                 Sysctl = 0,
-                Describe = 1,
-                Inspect = 2,
-                // ddl
-                Use = 3,
-                Create = 4,
-                Alter = 5,
-                Drop = 6,
-                // dml
+                // DDL
+                Create = 1,
+                Alter = 2,
+                Drop = 3,
+                // system/DDL misc
+                Use = 4,
+                Inspect = 5,
+                Describe = 6,
+                // DML
                 Insert = 7,
                 Select = 8,
                 Update = 9,
@@ -425,18 +426,18 @@ impl Keyword {
         }
     }
     fn compute(key: &[u8]) -> Option<Self> {
-        static G: [u8; 67] = [
-            0, 42, 57, 0, 20, 61, 15, 46, 28, 0, 31, 2, 1, 44, 47, 10, 35, 53, 30, 28, 48, 9, 1,
-            51, 61, 20, 20, 47, 23, 31, 0, 52, 55, 59, 27, 45, 54, 49, 29, 0, 66, 54, 23, 58, 13,
-            31, 47, 56, 1, 30, 40, 0, 0, 42, 27, 63, 6, 24, 65, 45, 42, 63, 60, 14, 26, 4, 13,
+        static G: [u8; 64] = [
+            0, 27, 13, 56, 18, 0, 26, 30, 33, 56, 20, 41, 56, 39, 23, 34, 36, 23, 17, 40, 38, 45,
+            8, 25, 26, 24, 53, 59, 30, 14, 9, 60, 12, 29, 6, 47, 3, 38, 19, 5, 13, 51, 41, 34, 0,
+            22, 43, 13, 46, 33, 11, 12, 36, 58, 40, 0, 36, 2, 19, 49, 53, 23, 55, 0,
         ];
-        static M1: [u8; 11] = *b"wsE1pgJgJMO";
-        static M2: [u8; 11] = *b"fICAB04WegN";
+        static M1: [u8; 11] = *b"RtEMxHylmiZ";
+        static M2: [u8; 11] = *b"F1buDOZ2nzz";
         let h1 = Self::_sum(key, M1) % G.len();
         let h2 = Self::_sum(key, M2) % G.len();
         let h = (G[h1] + G[h2]) as usize % G.len();
-        if h < G.len() && KW_LUT[h].0.eq_ignore_ascii_case(key) {
-            Some(KW_LUT[h].1)
+        if h < KW.len() && KW[h].0.eq_ignore_ascii_case(key) {
+            Some(KW[h].1)
         } else {
             None
         }
@@ -453,3 +454,9 @@ impl Keyword {
         sum
     }
 }
+
+impl KeywordStmt {
+    pub const fn is_blocking(&self) -> bool {
+        self.value_u8() <= Self::Drop.value_u8()
+    }
+}
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index 2bc26cd6..fdad0dd5 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -30,229 +30,6 @@ macro_rules! __sym_token {
     };
 }
 
-macro_rules! __kw_misc {
-    ($ident:ident) => {
-        $crate::engine::ql::lex::Token::Keyword($crate::engine::ql::lex::Keyword::Misc($crate::engine::ql::lex::KeywordMisc::$ident))
-    };
-}
-
-macro_rules! __kw_stmt {
-    ($ident:ident) => {
-        $crate::engine::ql::lex::Token::Keyword($crate::engine::ql::lex::Keyword::Statement($crate::engine::ql::lex::KeywordStmt::$ident))
-    };
-}
-
-/*
-    Frankly, this is just for lazy people like me. Do not judge
-    -- Sayan (@ohsayan)
-*/
-macro_rules! Token {
-    // misc symbol
-    (@) => {
-        __sym_token!(SymAt)
-    };
-    (#) => {
-        __sym_token!(SymHash)
-    };
-    ($) => {
-        __sym_token!(SymDollar)
-    };
-    (%) => {
-        __sym_token!(SymPercent)
-    };
-    (.) => {
-        __sym_token!(SymPeriod)
-    };
-    (,) => {
-        __sym_token!(SymComma)
-    };
-    (_) => {
-        __sym_token!(SymUnderscore)
-    };
-    (?) => {
-        __sym_token!(SymQuestion)
-    };
-    (:) => {
-        __sym_token!(SymColon)
-    };
-    (;) => {
-        __sym_token!(SymSemicolon)
-    };
-    (~) => {
-        __sym_token!(SymTilde)
-    };
-    // logical
-    (!) => {
-        __sym_token!(OpLogicalNot)
-    };
-    (^) => {
-        __sym_token!(OpLogicalXor)
-    };
-    (&) => {
-        __sym_token!(OpLogicalAnd)
-    };
-    (|) => {
-        __sym_token!(OpLogicalOr)
-    };
-    // operator misc.
-    (=) => {
-        __sym_token!(OpAssign)
-    };
-    // arithmetic
-    (+) => {
-        __sym_token!(OpArithmeticAdd)
-    };
-    (-) => {
-        __sym_token!(OpArithmeticSub)
-    };
-    (*) => {
-        __sym_token!(OpArithmeticMul)
-    };
-    (/) => {
-        __sym_token!(OpArithmeticDiv)
-    };
-    // relational
-    (>) => {
-        __sym_token!(OpComparatorGt)
-    };
-    (<) => {
-        __sym_token!(OpComparatorLt)
-    };
-    // ddl keywords
-    (use) => {
-        __kw_stmt!(Use)
-    };
-    (create) => {
-        __kw_stmt!(Create)
-    };
-    (alter) => {
-        __kw_stmt!(Alter)
-    };
-    (drop) => {
-        __kw_stmt!(Drop)
-    };
-    (model) => {
-        __kw_misc!(Model)
-    };
-    (space) => {
-        __kw_misc!(Space)
-    };
-    (primary) => {
-        __kw_misc!(Primary)
-    };
-    // ddl misc
-    (with) => {
-        __kw_misc!(With)
-    };
-    (add) => {
-        __kw_misc!(Add)
-    };
-    (remove) => {
-        __kw_misc!(Remove)
-    };
-    (sort) => {
-        __kw_misc!(Sort)
-    };
-    (type) => {
-        __kw_misc!(Type)
-    };
-    // dml
-    (insert) => {
-        __kw_stmt!(Insert)
-    };
-    (select) => {
-        __kw_stmt!(Select)
-    };
-    (update) => {
-        __kw_stmt!(Update)
-    };
-    (delete) => {
-        __kw_stmt!(Delete)
-    };
-    // dml misc
-    (set) => {
-        __kw_misc!(Set)
-    };
-    (limit) => {
-        __kw_misc!(Limit)
-    };
-    (from) => {
-        __kw_misc!(From)
-    };
-    (into) => {
-        __kw_misc!(Into)
-    };
-    (where) => {
-        __kw_misc!(Where)
-    };
-    (if) => {
-        __kw_misc!(If)
-    };
-    (and) => {
-        __kw_misc!(And)
-    };
-    (as) => {
-        __kw_misc!(As)
-    };
-    (by) => {
-        __kw_misc!(By)
-    };
-    (asc) => {
-        __kw_misc!(Asc)
-    };
-    (desc) => {
-        __kw_misc!(Desc)
-    };
-    // types
-    (string) => {
-        __kw_misc!(String)
-    };
-    (binary) => {
-        __kw_misc!(Binary)
-    };
-    (list) => {
-        __kw_misc!(List)
-    };
-    (map) => {
-        __kw_misc!(Map)
-    };
-    (bool) => {
-        __kw_misc!(Bool)
-    };
-    (int) => {
-        __kw_misc!(Int)
-    };
-    (double) => {
-        __kw_misc!(Double)
-    };
-    (float) => {
-        __kw_misc!(Float)
-    };
-    // tt
-    (open {}) => {
-        __sym_token!(TtOpenBrace)
-    };
-    (close {}) => {
-        __sym_token!(TtCloseBrace)
-    };
-    (() open) => {
-        __sym_token!(TtOpenParen)
-    };
-    (() close) => {
-        __sym_token!(TtCloseParen)
-    };
-    (open []) => {
-        __sym_token!(TtOpenSqBracket)
-    };
-    (close []) => {
-        __sym_token!(TtCloseSqBracket)
-    };
-    // misc
-    (null) => {
-        __kw_misc!(Null)
-    };
-}
-
 #[cfg(test)]
 macro_rules! dict {
     () => {
diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs
index ee61ddb0..3dfcbb2b 100644
--- a/server/src/engine/storage/v1/batch_jrnl/persist.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs
@@ -134,9 +134,9 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
             Ok(()) => Ok(()),
             Err(_) => {
                 // republish changes since we failed to commit
-                restore_list
-                    .into_iter()
-                    .for_each(|delta| model.delta_state().append_new_data_delta(delta, &g));
+                restore_list.into_iter().for_each(|delta| {
+                    model.delta_state().append_new_data_delta(delta, &g);
+                });
                 // now attempt to fix the file
                 return self.attempt_fix_data_batchfile();
             }
diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
index 4bac479e..ec66e073 100644
--- a/server/src/engine/txn/gns/tests/full_chain.rs
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -318,7 +318,7 @@ fn drop_model() {
                     .namespace()
                     .with_model(("myspace", "mymodel"), |_| { Ok(()) })
                     .unwrap_err(),
-                QueryError::QPObjectNotFound
+                QueryError::QExecObjectNotFound
             );
         })
     })

From e412cea7eb6963d1e8a15d13a60d220968416322 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 3 Nov 2023 11:50:48 +0000
Subject: [PATCH 273/310] Preserve type in objects and fix DML validation

---
 server/src/engine/core/dml/del.rs             |   9 +-
 server/src/engine/core/dml/ins.rs             |  17 +-
 server/src/engine/core/dml/mod.rs             |   7 +-
 server/src/engine/core/dml/sel.rs             |   8 +
 server/src/engine/core/dml/upd.rs             |  89 ++++---
 server/src/engine/core/exec.rs                |  11 +-
 server/src/engine/core/index/key.rs           |   2 +-
 server/src/engine/core/model/mod.rs           |  89 +++----
 .../src/engine/core/tests/ddl_model/layer.rs  |  68 +++---
 .../src/engine/core/tests/ddl_space/alter.rs  |  10 +-
 .../src/engine/core/tests/ddl_space/create.rs |   2 +-
 server/src/engine/data/cell.rs                |  76 +++---
 server/src/engine/data/tag.rs                 | 105 ++++-----
 server/src/engine/data/tests/md_dict_tests.rs |  42 ++--
 server/src/engine/macros.rs                   |   4 +
 server/src/engine/ql/dml/ins.rs               |   2 +-
 .../engine/storage/v1/batch_jrnl/persist.rs   |  55 +----
 .../engine/storage/v1/batch_jrnl/restore.rs   | 102 ++++----
 server/src/engine/storage/v1/inf/map.rs       | 118 ++--------
 server/src/engine/storage/v1/inf/mod.rs       |  79 ++-----
 server/src/engine/storage/v1/inf/obj.rs       | 217 ++++++++++++++++--
 server/src/engine/storage/v1/inf/tests.rs     |   2 +-
 server/src/engine/storage/v1/sysdb.rs         |   4 +-
 server/src/engine/txn/gns/tests/full_chain.rs |   4 +-
 24 files changed, 591 insertions(+), 531 deletions(-)

diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs
index 0802383c..0cea6aaa 100644
--- a/server/src/engine/core/dml/del.rs
+++ b/server/src/engine/core/dml/del.rs
@@ -29,11 +29,18 @@ use crate::engine::{
     error::{QueryError, QueryResult},
     fractal::GlobalInstanceLike,
     idx::MTIndex,
+    net::protocol::Response,
     ql::dml::del::DeleteStatement,
     sync,
 };
 
-#[allow(unused)]
+pub fn delete_resp(
+    global: &impl GlobalInstanceLike,
+    delete: DeleteStatement,
+) -> QueryResult<Response> {
+    self::delete(global, delete).map(|_| Response::Empty)
+}
+
 pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) -> QueryResult<()> {
     core::with_model_for_data_update(global, delete.entity(), |model| {
         let g = sync::atm::cpin();
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index 8b0173dd..33242746 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -34,11 +34,18 @@ use crate::engine::{
     error::{QueryError, QueryResult},
     fractal::GlobalInstanceLike,
     idx::{IndexBaseSpec, MTIndex, STIndex, STIndexSeq},
+    net::protocol::Response,
     ql::dml::ins::{InsertData, InsertStatement},
     sync::atm::cpin,
 };
 
-#[allow(unused)]
+pub fn insert_resp(
+    global: &impl GlobalInstanceLike,
+    insert: InsertStatement,
+) -> QueryResult<Response> {
+    self::insert(global, insert).map(|_| Response::Empty)
+}
+
 pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> QueryResult<()> {
     core::with_model_for_data_update(global, insert.entity(), |mdl| {
         let irmwd = mdl.intent_write_new_data();
@@ -77,7 +84,7 @@ fn prepare_insert(
             let mut fields = fields.stseq_ord_kv();
             let mut tuple = tuple.into_iter();
             while (tuple.len() != 0) & okay {
-                let data;
+                let mut data;
                 let field;
                 unsafe {
                     // UNSAFE(@ohsayan): safe because of invariant
@@ -86,14 +93,14 @@ fn prepare_insert(
                     field = fields.next().unwrap_unchecked();
                 }
                 let (field_id, field) = field;
-                okay &= field.validate_data_fpath(&data);
+                okay &= field.vt_data_fpath(&mut data);
                 okay &= prepared_data.st_insert(field_id.clone(), data);
             }
         }
         InsertData::Map(map) => {
             let mut map = map.into_iter();
             while (map.len() != 0) & okay {
-                let (field_id, field_data) = unsafe {
+                let (field_id, mut field_data) = unsafe {
                     // UNSAFE(@ohsayan): safe because of loop invariant
                     map.next().unwrap_unchecked()
                 };
@@ -101,7 +108,7 @@ fn prepare_insert(
                     okay = false;
                     break;
                 };
-                okay &= field.validate_data_fpath(&field_data);
+                okay &= field.vt_data_fpath(&mut field_data);
                 prepared_data.st_insert(field_id.boxed_str(), field_data);
             }
         }
diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index dfe1dfb1..402c6b45 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -41,7 +41,12 @@ use crate::{
 
 #[cfg(test)]
 pub use upd::collect_trace_path as update_flow_trace;
-pub use {del::delete, ins::insert, sel::select_custom, upd::update};
+pub use {
+    del::{delete, delete_resp},
+    ins::{insert, insert_resp},
+    sel::select_custom,
+    upd::{update, update_resp},
+};
 
 impl Model {
     pub(self) fn resolve_where<'a>(
diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
index f95a8239..d281a27b 100644
--- a/server/src/engine/core/dml/sel.rs
+++ b/server/src/engine/core/dml/sel.rs
@@ -30,10 +30,18 @@ use crate::engine::{
     error::{QueryError, QueryResult},
     fractal::GlobalInstanceLike,
     idx::{STIndex, STIndexSeq},
+    net::protocol::Response,
     ql::dml::sel::SelectStatement,
     sync,
 };
 
+pub fn select_resp(
+    global: &impl GlobalInstanceLike,
+    select: SelectStatement,
+) -> QueryResult<Response> {
+    todo!()
+}
+
 #[allow(unused)]
 pub fn select_custom<F>(
     global: &impl GlobalInstanceLike,
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index eeb2998a..296a227e 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -37,11 +37,12 @@ use {
             data::{
                 cell::Datacell,
                 lit::Lit,
-                tag::{DataTag, TagClass},
+                tag::{DataTag, FloatSpec, SIntSpec, TagClass, UIntSpec},
             },
             error::{QueryError, QueryResult},
             fractal::GlobalInstanceLike,
             idx::STIndex,
+            net::protocol::Response,
             ql::dml::upd::{AssignmentExpression, UpdateStatement},
             sync,
         },
@@ -59,44 +60,56 @@ unsafe fn dc_op_bool_ass(_: &Datacell, rhs: Lit) -> (bool, Datacell) {
     (true, Datacell::new_bool(rhs.bool()))
 }
 // uint
-unsafe fn dc_op_uint_ass(_: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    (true, Datacell::new_uint(rhs.uint()))
+unsafe fn dc_op_uint_ass(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let uint = rhs.uint();
+    let kind = UIntSpec::from_full(dc.tag());
+    (kind.check(uint), Datacell::new_uint(uint, kind))
 }
 unsafe fn dc_op_uint_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    let (sum, of) = dc.read_uint().overflowing_add(rhs.uint());
-    (of, Datacell::new_uint(sum))
+    let kind = UIntSpec::from_full(dc.tag());
+    let (uint, did_of) = dc.uint().overflowing_add(rhs.uint());
+    (kind.check(uint) & !did_of, Datacell::new_uint(uint, kind))
 }
 unsafe fn dc_op_uint_sub(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    let (diff, of) = dc.read_uint().overflowing_sub(rhs.uint());
-    (of, Datacell::new_uint(diff))
+    let kind = UIntSpec::from_full(dc.tag());
+    let (uint, did_of) = dc.uint().overflowing_sub(rhs.uint());
+    (kind.check(uint) & !did_of, Datacell::new_uint(uint, kind))
 }
 unsafe fn dc_op_uint_mul(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    let (prod, of) = dc.read_uint().overflowing_mul(rhs.uint());
-    (of, Datacell::new_uint(prod))
+    let kind = UIntSpec::from_full(dc.tag());
+    let (uint, did_of) = dc.uint().overflowing_mul(rhs.uint());
+    (kind.check(uint) & !did_of, Datacell::new_uint(uint, kind))
 }
 unsafe fn dc_op_uint_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    let (quo, of) = dc.read_uint().overflowing_div(rhs.uint());
-    (of, Datacell::new_uint(quo))
+    let kind = UIntSpec::from_full(dc.tag());
+    let (uint, did_of) = dc.uint().overflowing_div(rhs.uint());
+    (kind.check(uint) & !did_of, Datacell::new_uint(uint, kind))
 }
 // sint
-unsafe fn dc_op_sint_ass(_: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    (true, Datacell::new_sint(rhs.sint()))
+unsafe fn dc_op_sint_ass(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let sint = rhs.sint();
+    let kind = SIntSpec::from_full(dc.tag());
+    (kind.check(sint), Datacell::new_sint(sint, kind))
 }
 unsafe fn dc_op_sint_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    let (sum, of) = dc.read_sint().overflowing_add(rhs.sint());
-    (of, Datacell::new_sint(sum))
+    let kind = SIntSpec::from_full(dc.tag());
+    let (sint, did_of) = dc.sint().overflowing_add(rhs.sint());
+    (kind.check(sint) & !did_of, Datacell::new_sint(sint, kind))
 }
 unsafe fn dc_op_sint_sub(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    let (diff, of) = dc.read_sint().overflowing_sub(rhs.sint());
-    (of, Datacell::new_sint(diff))
+    let kind = SIntSpec::from_full(dc.tag());
+    let (sint, did_of) = dc.sint().overflowing_sub(rhs.sint());
+    (kind.check(sint) & !did_of, Datacell::new_sint(sint, kind))
 }
 unsafe fn dc_op_sint_mul(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    let (prod, of) = dc.read_sint().overflowing_mul(rhs.sint());
-    (of, Datacell::new_sint(prod))
+    let kind = SIntSpec::from_full(dc.tag());
+    let (sint, did_of) = dc.sint().overflowing_mul(rhs.sint());
+    (kind.check(sint) & !did_of, Datacell::new_sint(sint, kind))
 }
 unsafe fn dc_op_sint_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    let (quo, of) = dc.read_sint().overflowing_div(rhs.sint());
-    (of, Datacell::new_sint(quo))
+    let kind = SIntSpec::from_full(dc.tag());
+    let (sint, did_of) = dc.sint().overflowing_div(rhs.sint());
+    (kind.check(sint) & !did_of, Datacell::new_sint(sint, kind))
 }
 /*
     float
@@ -108,24 +121,30 @@ unsafe fn dc_op_sint_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
     --
     TODO(@ohsayan): account for float32 overflow
 */
-unsafe fn dc_op_float_ass(_: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    (true, Datacell::new_float(rhs.float()))
+unsafe fn dc_op_float_ass(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
+    let float = rhs.float();
+    let kind = FloatSpec::from_full(dc.tag());
+    (kind.check(float), Datacell::new_float(float, kind))
 }
 unsafe fn dc_op_float_add(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    let sum = dc.read_float() + rhs.float();
-    (true, Datacell::new_float(sum))
+    let result = dc.read_float() + rhs.float();
+    let kind = FloatSpec::from_full(dc.tag());
+    (kind.check(result), Datacell::new_float(result, kind))
 }
 unsafe fn dc_op_float_sub(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    let diff = dc.read_float() - rhs.float();
-    (true, Datacell::new_float(diff))
+    let result = dc.read_float() - rhs.float();
+    let kind = FloatSpec::from_full(dc.tag());
+    (kind.check(result), Datacell::new_float(result, kind))
 }
 unsafe fn dc_op_float_mul(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    let prod = dc.read_float() - rhs.float();
-    (true, Datacell::new_float(prod))
+    let result = dc.read_float() * rhs.float();
+    let kind = FloatSpec::from_full(dc.tag());
+    (kind.check(result), Datacell::new_float(result, kind))
 }
 unsafe fn dc_op_float_div(dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
-    let quo = dc.read_float() * rhs.float();
-    (true, Datacell::new_float(quo))
+    let result = dc.read_float() / rhs.float();
+    let kind = FloatSpec::from_full(dc.tag());
+    (kind.check(result), Datacell::new_float(result, kind))
 }
 // binary
 unsafe fn dc_op_bin_ass(_dc: &Datacell, rhs: Lit) -> (bool, Datacell) {
@@ -235,7 +254,13 @@ pub fn collect_trace_path() -> Vec<&'static str> {
     ROUTE_TRACE.with(|v| v.borrow().iter().cloned().collect())
 }
 
-#[allow(unused)]
+pub fn update_resp(
+    global: &impl GlobalInstanceLike,
+    update: UpdateStatement,
+) -> QueryResult<Response> {
+    self::update(global, update).map(|_| Response::Empty)
+}
+
 pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) -> QueryResult<()> {
     core::with_model_for_data_update(global, update.entity(), |mdl| {
         let mut ret = Ok(QueryExecMeta::zero());
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 703b2447..8006e2d9 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -171,14 +171,14 @@ fn run_nb(
     stmt: KeywordStmt,
 ) -> QueryResult<Response> {
     let stmt = stmt.value_u8() - KeywordStmt::Use.value_u8();
-    static F: [fn(&Global, &mut State<'static, InplaceData>) -> QueryResult<()>; 8] = [
+    static F: [fn(&Global, &mut State<'static, InplaceData>) -> QueryResult<Response>; 8] = [
         |_, _| panic!("use not implemented"),
         |_, _| panic!("inspect not implemented"),
         |_, _| panic!("describe not implemented"),
-        |g, s| _call(g, s, dml::insert),
+        |g, s| _call(g, s, dml::insert_resp),
         |_, _| panic!("select not implemented"),
-        |g, s| _call(g, s, dml::update),
-        |g, s| _call(g, s, dml::delete),
+        |g, s| _call(g, s, dml::update_resp),
+        |g, s| _call(g, s, dml::delete_resp),
         |_, _| panic!("exists not implemented"),
     ];
     {
@@ -186,7 +186,6 @@ fn run_nb(
             // UNSAFE(@ohsayan): this is a lifetime issue with the token handle
             core::mem::transmute(state)
         };
-        F[stmt as usize](global, &mut state)?;
+        F[stmt as usize](global, &mut state)
     }
-    Ok(Response::Empty)
 }
diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
index 02137ec6..8fd7d804 100644
--- a/server/src/engine/core/index/key.rs
+++ b/server/src/engine/core/index/key.rs
@@ -345,7 +345,7 @@ fn check_pk_lit_eq_hash() {
 #[test]
 fn check_pk_extremes() {
     let state = test_utils::randomstate();
-    let d1 = PrimaryIndexKey::try_from_dc(Datacell::new_uint(u64::MAX)).unwrap();
+    let d1 = PrimaryIndexKey::try_from_dc(Datacell::new_uint_default(u64::MAX)).unwrap();
     let d2 = PrimaryIndexKey::try_from_dc(Datacell::from(Lit::new_uint(u64::MAX))).unwrap();
     assert_eq!(d1, d2);
     assert_eq!(d1.uint().unwrap(), u64::MAX);
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 61e1557f..d2966c14 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -36,7 +36,7 @@ use {
     crate::engine::{
         data::{
             cell::Datacell,
-            tag::{DataTag, FullTag, TagClass, TagSelector},
+            tag::{DataTag, FloatSpec, FullTag, SIntSpec, TagClass, TagSelector, UIntSpec},
             uuid::Uuid,
         },
         error::{QueryError, QueryResult},
@@ -379,44 +379,39 @@ impl Field {
             dc.kind().value_word()
         }
     }
-    pub fn validate_data_fpath(&self, data: &Datacell) -> bool {
-        // if someone sends a PR with an added check, I'll personally come to your house and throw a brick on your head
-        if (self.layers.len() == 1) | data.is_null() {
+    pub fn vt_data_fpath(&self, data: &mut Datacell) -> bool {
+        if (self.layers.len() == 1) | (data.is_null()) {
             layertrace("fpath");
-            unsafe {
-                // UNSAFE(@ohsayan): checked for non-null, and used correct class
-                LVERIFY[self.compute_index(data)](self.layers()[0], data)
-            }
+            unsafe { VTFN[self.compute_index(data)](self.layers()[0], data) }
         } else {
-            Self::rverify_layers(self.layers(), data)
+            Self::rvt_data(self.layers(), data)
         }
     }
-    // TODO(@ohsayan): improve algo with dfs
-    fn rverify_layers(layers: &[Layer], data: &Datacell) -> bool {
+    fn rvt_data(layers: &[Layer], data: &mut Datacell) -> bool {
         let layer = layers[0];
         let layers = &layers[1..];
-        match (layer.tag.tag_class(), data.kind()) {
-            (TagClass::List, TagClass::List) if !layers.is_empty() => {
+        match (layer.tag().tag_class(), data.kind()) {
+            (TagClass::List, TagClass::List) => {
                 let mut okay = unsafe {
-                    // UNSAFE(@ohsayan): we've verified this
-                    LVERIFY[TagClass::List.value_word()](layer, data)
+                    // UNSAFE(@ohsayan): +tagck
+                    VTFN[TagClass::List.value_word()](layer, data)
                 };
                 let list = unsafe {
-                    // UNSAFE(@ohsayan): we verified tags
+                    // UNSAFE(@ohsayan): +tagck
                     data.read_list()
                 };
-                let lread = list.read();
+                let mut lread = list.write();
                 let mut i = 0;
                 while (i < lread.len()) & okay {
-                    okay &= Self::rverify_layers(layers, &lread[i]);
+                    okay &= Self::rvt_data(layers, &mut lread[i]);
                     i += 1;
                 }
                 okay
             }
             (tag_a, tag_b) if tag_a == tag_b => {
                 unsafe {
-                    // UNSAFE(@ohsayan): same tags; not-null for now so no extra handling required here
-                    LVERIFY[tag_a.value_word()](layer, data)
+                    // UNSAFE(@ohsayan): same tags and lists have non-null elements
+                    VTFN[tag_a.value_word()](layer, data)
                 }
             }
             _ => false,
@@ -510,17 +505,6 @@ impl Layer {
     }
 }
 
-static LVERIFY: [unsafe fn(Layer, &Datacell) -> bool; 8] = [
-    lverify_bool,
-    lverify_uint,
-    lverify_sint,
-    lverify_float,
-    lverify_bin,
-    lverify_str,
-    lverify_list,
-    |_, _| false,
-];
-
 #[cfg(test)]
 thread_local! {
     static LAYER_TRACE: RefCell<Vec<Box<str>>> = RefCell::new(Vec::new());
@@ -544,41 +528,44 @@ pub(super) fn layer_traces() -> Box<[Box<str>]> {
     })
 }
 
-unsafe fn lverify_bool(_: Layer, _: &Datacell) -> bool {
+static VTFN: [unsafe fn(Layer, &mut Datacell) -> bool; 8] = [
+    vt_bool,
+    vt_uint,
+    vt_sint,
+    vt_float,
+    vt_bin,
+    vt_str,
+    vt_list,
+    |_, _| false,
+];
+unsafe fn vt_bool(_: Layer, _: &mut Datacell) -> bool {
     layertrace("bool");
     true
 }
-unsafe fn lverify_uint(l: Layer, d: &Datacell) -> bool {
+unsafe fn vt_uint(l: Layer, dc: &mut Datacell) -> bool {
     layertrace("uint");
-    const MX: [u64; 4] = [u8::MAX as _, u16::MAX as _, u32::MAX as _, u64::MAX];
-    d.read_uint() <= MX[l.tag.tag_selector().value_word() - 1]
+    dc.set_tag(l.tag());
+    UIntSpec::from_full(l.tag()).check(dc.read_uint())
 }
-unsafe fn lverify_sint(l: Layer, d: &Datacell) -> bool {
+unsafe fn vt_sint(l: Layer, dc: &mut Datacell) -> bool {
     layertrace("sint");
-    const MN_MX: [(i64, i64); 4] = [
-        (i8::MIN as _, i8::MAX as _),
-        (i16::MIN as _, i16::MAX as _),
-        (i32::MIN as _, i32::MAX as _),
-        (i64::MIN, i64::MAX),
-    ];
-    let (mn, mx) = MN_MX[l.tag.tag_selector().value_word() - 5];
-    (d.read_sint() >= mn) & (d.read_sint() <= mx)
+    dc.set_tag(l.tag());
+    SIntSpec::from_full(l.tag()).check(dc.read_sint())
 }
-unsafe fn lverify_float(l: Layer, d: &Datacell) -> bool {
+unsafe fn vt_float(l: Layer, dc: &mut Datacell) -> bool {
     layertrace("float");
-    const MN_MX: [(f64, f64); 2] = [(f32::MIN as _, f32::MAX as _), (f64::MIN, f64::MAX)];
-    let (mn, mx) = MN_MX[l.tag.tag_selector().value_word() - 9];
-    (d.read_float() >= mn) & (d.read_float() <= mx)
+    dc.set_tag(l.tag());
+    FloatSpec::from_full(l.tag()).check(dc.read_float())
 }
-unsafe fn lverify_bin(_: Layer, _: &Datacell) -> bool {
+unsafe fn vt_bin(_: Layer, _: &mut Datacell) -> bool {
     layertrace("binary");
     true
 }
-unsafe fn lverify_str(_: Layer, _: &Datacell) -> bool {
+unsafe fn vt_str(_: Layer, _: &mut Datacell) -> bool {
     layertrace("string");
     true
 }
-unsafe fn lverify_list(_: Layer, _: &Datacell) -> bool {
+unsafe fn vt_list(_: Layer, _: &mut Datacell) -> bool {
     layertrace("list");
     true
 }
diff --git a/server/src/engine/core/tests/ddl_model/layer.rs b/server/src/engine/core/tests/ddl_model/layer.rs
index 345ed2ce..5d95ea6f 100644
--- a/server/src/engine/core/tests/ddl_model/layer.rs
+++ b/server/src/engine/core/tests/ddl_model/layer.rs
@@ -84,9 +84,9 @@ mod layer_data_validation {
     };
     #[test]
     fn bool() {
-        let dc = Datacell::new_bool(true);
+        let mut dc = Datacell::new_bool(true);
         let layer = layerview("bool").unwrap();
-        assert!(layer.validate_data_fpath(&dc));
+        assert!(layer.vt_data_fpath(&mut dc));
         assert_vecstreq_exact!(model::layer_traces(), ["fpath", "bool"]);
     }
     #[test]
@@ -102,17 +102,17 @@ mod layer_data_validation {
             .enumerate()
             .for_each(|(i, (layer, max))| {
                 let this_layer = layerview(layer).unwrap();
-                let dc = Datacell::new_uint(max);
-                assert!(this_layer.validate_data_fpath(&dc), "{:?}", this_layer);
+                let mut dc = Datacell::new_uint_default(max);
+                assert!(this_layer.vt_data_fpath(&mut dc), "{:?}", this_layer);
                 assert_vecstreq_exact!(model::layer_traces(), ["fpath", "uint"]);
                 for (lower, _) in targets[..i].iter() {
                     let layer = layerview(lower).unwrap();
-                    assert!(!layer.validate_data_fpath(&dc), "{:?}", layer);
+                    assert!(!layer.vt_data_fpath(&mut dc), "{:?}", layer);
                     assert_vecstreq_exact!(model::layer_traces(), ["fpath", "uint"]);
                 }
                 for (higher, _) in targets[i + 1..].iter() {
                     let layer = layerview(higher).unwrap();
-                    assert!(layer.validate_data_fpath(&dc), "{:?}", layer);
+                    assert!(layer.vt_data_fpath(&mut dc), "{:?}", layer);
                     assert_vecstreq_exact!(model::layer_traces(), ["fpath", "uint"]);
                 }
             });
@@ -130,15 +130,15 @@ mod layer_data_validation {
             .enumerate()
             .for_each(|(i, (layer, (min, max)))| {
                 let this_layer = layerview(layer).unwrap();
-                let dc_min = Datacell::new_sint(min);
-                let dc_max = Datacell::new_sint(max);
-                assert!(this_layer.validate_data_fpath(&dc_min), "{:?}", this_layer);
-                assert!(this_layer.validate_data_fpath(&dc_max), "{:?}", this_layer);
+                let mut dc_min = Datacell::new_sint_default(min);
+                let mut dc_max = Datacell::new_sint_default(max);
+                assert!(this_layer.vt_data_fpath(&mut dc_min), "{:?}", this_layer);
+                assert!(this_layer.vt_data_fpath(&mut dc_max), "{:?}", this_layer);
                 assert_vecstreq_exact!(model::layer_traces(), ["fpath", "sint", "fpath", "sint"]);
                 for (lower, _) in targets[..i].iter() {
                     let layer = layerview(lower).unwrap();
-                    assert!(!layer.validate_data_fpath(&dc_min), "{:?}", layer);
-                    assert!(!layer.validate_data_fpath(&dc_max), "{:?}", layer);
+                    assert!(!layer.vt_data_fpath(&mut dc_min), "{:?}", layer);
+                    assert!(!layer.vt_data_fpath(&mut dc_max), "{:?}", layer);
                     assert_vecstreq_exact!(
                         model::layer_traces(),
                         ["fpath", "sint", "fpath", "sint"]
@@ -146,8 +146,8 @@ mod layer_data_validation {
                 }
                 for (higher, _) in targets[i + 1..].iter() {
                     let layer = layerview(higher).unwrap();
-                    assert!(layer.validate_data_fpath(&dc_min), "{:?}", layer);
-                    assert!(layer.validate_data_fpath(&dc_max), "{:?}", layer);
+                    assert!(layer.vt_data_fpath(&mut dc_min), "{:?}", layer);
+                    assert!(layer.vt_data_fpath(&mut dc_max), "{:?}", layer);
                     assert_vecstreq_exact!(
                         model::layer_traces(),
                         ["fpath", "sint", "fpath", "sint"]
@@ -161,46 +161,46 @@ mod layer_data_validation {
         let f32_l = layerview("float32").unwrap();
         let f64_l = layerview("float64").unwrap();
         // dc
-        let f32_dc_min = Datacell::new_float(f32::MIN as _);
-        let f32_dc_max = Datacell::new_float(f32::MAX as _);
-        let f64_dc_min = Datacell::new_float(f64::MIN as _);
-        let f64_dc_max = Datacell::new_float(f64::MAX as _);
+        let f32_dc_min = Datacell::new_float_default(f32::MIN as _);
+        let f32_dc_max = Datacell::new_float_default(f32::MAX as _);
+        let f64_dc_min = Datacell::new_float_default(f64::MIN as _);
+        let f64_dc_max = Datacell::new_float_default(f64::MAX as _);
         // check (32)
-        assert!(f32_l.validate_data_fpath(&f32_dc_min));
-        assert!(f32_l.validate_data_fpath(&f32_dc_max));
+        assert!(f32_l.vt_data_fpath(&mut f32_dc_min.clone()));
+        assert!(f32_l.vt_data_fpath(&mut f32_dc_max.clone()));
         assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]);
-        assert!(f64_l.validate_data_fpath(&f32_dc_min));
-        assert!(f64_l.validate_data_fpath(&f32_dc_max));
+        assert!(f64_l.vt_data_fpath(&mut f32_dc_min.clone()));
+        assert!(f64_l.vt_data_fpath(&mut f32_dc_max.clone()));
         assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]);
         // check (64)
-        assert!(!f32_l.validate_data_fpath(&f64_dc_min));
-        assert!(!f32_l.validate_data_fpath(&f64_dc_max));
+        assert!(!f32_l.vt_data_fpath(&mut f64_dc_min.clone()));
+        assert!(!f32_l.vt_data_fpath(&mut f64_dc_max.clone()));
         assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]);
-        assert!(f64_l.validate_data_fpath(&f64_dc_min));
-        assert!(f64_l.validate_data_fpath(&f64_dc_max));
+        assert!(f64_l.vt_data_fpath(&mut f64_dc_min.clone()));
+        assert!(f64_l.vt_data_fpath(&mut f64_dc_max.clone()));
         assert_vecstreq_exact!(model::layer_traces(), ["fpath", "float", "fpath", "float"]);
     }
     #[test]
     fn bin() {
         let layer = layerview("binary").unwrap();
-        assert!(layer.validate_data_fpath(&Datacell::from("hello".as_bytes())));
+        assert!(layer.vt_data_fpath(&mut Datacell::from("hello".as_bytes())));
         assert_vecstreq_exact!(model::layer_traces(), ["fpath", "binary"]);
     }
     #[test]
     fn str() {
         let layer = layerview("string").unwrap();
-        assert!(layer.validate_data_fpath(&Datacell::from("hello")));
+        assert!(layer.vt_data_fpath(&mut Datacell::from("hello")));
         assert_vecstreq_exact!(model::layer_traces(), ["fpath", "string"]);
     }
     #[test]
     fn list_simple() {
         let layer = layerview("list { type: string }").unwrap();
-        let dc = Datacell::new_list(vec![
+        let mut dc = Datacell::new_list(vec![
             Datacell::from("I"),
             Datacell::from("love"),
             Datacell::from("cats"),
         ]);
-        assert!(layer.validate_data_fpath(&dc));
+        assert!(layer.vt_data_fpath(&mut dc));
         assert_vecstreq_exact!(
             model::layer_traces(),
             ["list", "string", "string", "string"]
@@ -209,12 +209,12 @@ mod layer_data_validation {
     #[test]
     fn list_nested_l1() {
         let layer = layerview("list { type: list { type: string } }").unwrap();
-        let dc = Datacell::new_list(vec![
+        let mut dc = Datacell::new_list(vec![
             Datacell::new_list(vec![Datacell::from("hello_11"), Datacell::from("hello_12")]),
             Datacell::new_list(vec![Datacell::from("hello_21"), Datacell::from("hello_22")]),
             Datacell::new_list(vec![Datacell::from("hello_31"), Datacell::from("hello_32")]),
         ]);
-        assert!(layer.validate_data_fpath(&dc));
+        assert!(layer.vt_data_fpath(&mut dc));
         assert_vecstreq_exact!(
             model::layer_traces(),
             [
@@ -228,13 +228,13 @@ mod layer_data_validation {
     #[test]
     fn nullval_fpath() {
         let layer = layerview_nullable("string", true).unwrap();
-        assert!(layer.validate_data_fpath(&Datacell::null()));
+        assert!(layer.vt_data_fpath(&mut Datacell::null()));
         assert_vecstreq_exact!(model::layer_traces(), ["fpath", "bool"]);
     }
     #[test]
     fn nullval_nested_but_fpath() {
         let layer = layerview_nullable("list { type: string }", true).unwrap();
-        assert!(layer.validate_data_fpath(&Datacell::null()));
+        assert!(layer.vt_data_fpath(&mut Datacell::null()));
         assert_vecstreq_exact!(model::layer_traces(), ["fpath", "bool"]);
     }
 }
diff --git a/server/src/engine/core/tests/ddl_space/alter.rs b/server/src/engine/core/tests/ddl_space/alter.rs
index 0ae95e23..38640972 100644
--- a/server/src/engine/core/tests/ddl_space/alter.rs
+++ b/server/src/engine/core/tests/ddl_space/alter.rs
@@ -43,7 +43,7 @@ fn alter_add_prop_env_var() {
                 space,
                 &Space::new_with_uuid(
                     into_dict!(),
-                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint(100))),
+                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint_default(100))),
                     space.get_uuid()
                 )
             );
@@ -62,7 +62,7 @@ fn alter_update_prop_env_var() {
             let rl = space.meta.dict().read();
             assert_eq!(
                 SpaceMeta::get_env(&rl).get("MY_NEW_PROP").unwrap(),
-                &(Datacell::new_uint(100).into())
+                &(Datacell::new_uint_default(100).into())
             )
         },
     )
@@ -75,7 +75,7 @@ fn alter_update_prop_env_var() {
                 space,
                 &Space::new_with_uuid(
                     into_dict!(),
-                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint(200))),
+                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint_default(200))),
                     uuid,
                 )
             )
@@ -94,7 +94,7 @@ fn alter_remove_prop_env_var() {
             let rl = space.meta.dict().read();
             assert_eq!(
                 SpaceMeta::get_env(&rl).get("MY_NEW_PROP").unwrap(),
-                &(Datacell::new_uint(100).into())
+                &(Datacell::new_uint_default(100).into())
             )
         },
     )
@@ -136,7 +136,7 @@ fn alter_remove_all_env() {
             let rl = space.meta.dict().read();
             assert_eq!(
                 SpaceMeta::get_env(&rl).get("MY_NEW_PROP").unwrap(),
-                &(Datacell::new_uint(100).into())
+                &(Datacell::new_uint_default(100).into())
             )
         },
     )
diff --git a/server/src/engine/core/tests/ddl_space/create.rs b/server/src/engine/core/tests/ddl_space/create.rs
index 8a42cf55..787f8c67 100644
--- a/server/src/engine/core/tests/ddl_space/create.rs
+++ b/server/src/engine/core/tests/ddl_space/create.rs
@@ -58,7 +58,7 @@ fn exec_create_space_with_env() {
                 &Space::new_with_uuid(
                     into_dict! {},
                     SpaceMeta::with_env(into_dict! {
-                        "MAX_MODELS" => Datacell::new_uint(100)
+                        "MAX_MODELS" => Datacell::new_uint_default(100)
                     }),
                     space.get_uuid()
                 )
diff --git a/server/src/engine/data/cell.rs b/server/src/engine/data/cell.rs
index e5b54bd0..c23ba0a4 100644
--- a/server/src/engine/data/cell.rs
+++ b/server/src/engine/data/cell.rs
@@ -24,24 +24,28 @@
  *
 */
 
-use std::{marker::PhantomData, ops::Deref};
-
 use {
     crate::engine::{
         self,
         data::{
             lit::Lit,
-            tag::{CUTag, DataTag, TagClass},
+            tag::{DataTag, FloatSpec, FullTag, SIntSpec, TagClass, UIntSpec},
         },
         mem::{DwordNN, DwordQN, NativeQword, SpecialPaddedWord, WordIO},
     },
-    core::{fmt, mem, mem::ManuallyDrop, slice, str},
+    core::{
+        fmt,
+        marker::PhantomData,
+        mem::{self, ManuallyDrop},
+        ops::Deref,
+        slice, str,
+    },
     parking_lot::RwLock,
 };
 
 pub struct Datacell {
     init: bool,
-    tag: CUTag,
+    tag: FullTag,
     data: DataRaw,
 }
 
@@ -51,7 +55,7 @@ impl Datacell {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
-                CUTag::BOOL,
+                FullTag::BOOL,
                 DataRaw::word(SpecialPaddedWord::store(b).dwordqn_promote()),
             )
         }
@@ -69,15 +73,18 @@ impl Datacell {
         self.try_bool().unwrap()
     }
     // uint
-    pub fn new_uint(u: u64) -> Self {
+    pub fn new_uint(u: u64, kind: UIntSpec) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
-                CUTag::UINT,
+                kind.into(),
                 DataRaw::word(SpecialPaddedWord::store(u).dwordqn_promote()),
             )
         }
     }
+    pub fn new_uint_default(u: u64) -> Self {
+        Self::new_uint(u, UIntSpec::DEFAULT)
+    }
     pub unsafe fn read_uint(&self) -> u64 {
         self.load_word()
     }
@@ -101,15 +108,18 @@ impl Datacell {
         }
     }
     // sint
-    pub fn new_sint(i: i64) -> Self {
+    pub fn new_sint(i: i64, kind: SIntSpec) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
-                CUTag::SINT,
+                kind.into(),
                 DataRaw::word(SpecialPaddedWord::store(i).dwordqn_promote()),
             )
         }
     }
+    pub fn new_sint_default(s: i64) -> Self {
+        Self::new_sint(s, SIntSpec::DEFAULT)
+    }
     pub unsafe fn read_sint(&self) -> i64 {
         self.load_word()
     }
@@ -123,15 +133,18 @@ impl Datacell {
         self.try_sint().unwrap()
     }
     // float
-    pub fn new_float(f: f64) -> Self {
+    pub fn new_float(f: f64, spec: FloatSpec) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
-                CUTag::FLOAT,
+                spec.into(),
                 DataRaw::word(SpecialPaddedWord::store(f).dwordqn_promote()),
             )
         }
     }
+    pub fn new_float_default(f: f64) -> Self {
+        Self::new_float(f, FloatSpec::DEFAULT)
+    }
     pub unsafe fn read_float(&self) -> f64 {
         self.load_word()
     }
@@ -150,7 +163,7 @@ impl Datacell {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
-                CUTag::BIN,
+                FullTag::BIN,
                 DataRaw::word(WordIO::store((md.len(), md.as_mut_ptr()))),
             )
         }
@@ -189,7 +202,7 @@ impl Datacell {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
             Self::new(
-                CUTag::STR,
+                FullTag::STR,
                 DataRaw::word(WordIO::store((md.len(), md.as_mut_ptr()))),
             )
         }
@@ -211,7 +224,7 @@ impl Datacell {
     pub fn new_list(l: Vec<Self>) -> Self {
         unsafe {
             // UNSAFE(@ohsayan): Correct because we are initializing Self with the correct tag
-            Self::new(CUTag::LIST, DataRaw::rwl(RwLock::new(l)))
+            Self::new(FullTag::LIST, DataRaw::rwl(RwLock::new(l)))
         }
     }
     pub unsafe fn read_list(&self) -> &RwLock<Vec<Self>> {
@@ -237,21 +250,23 @@ impl Datacell {
             Some(ManuallyDrop::into_inner(rwl).into_inner())
         }
     }
-    pub unsafe fn new_qw(qw: u64, tag: CUTag) -> Datacell {
+    pub unsafe fn new_qw(qw: u64, tag: FullTag) -> Datacell {
         Self::new(
             tag,
             DataRaw::word(SpecialPaddedWord::store(qw).dwordqn_promote()),
         )
     }
+    pub unsafe fn set_tag(&mut self, tag: FullTag) {
+        self.tag = tag;
+    }
 }
 
 direct_from! {
     Datacell => {
         bool as new_bool,
-        u64 as new_uint,
-        i64 as new_sint,
-        f64 as new_float,
-        f32 as new_float,
+        u64 as new_uint_default,
+        i64 as new_sint_default,
+        f64 as new_float_default,
         Vec<u8> as new_bin,
         Box<[u8]> as new_bin,
         &'static [u8] as new_bin,
@@ -269,7 +284,7 @@ impl<'a> From<Lit<'a>> for Datacell {
             tag if tag < TagClass::Bin => unsafe {
                 // UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type doesn't need any advanced construction
                 Datacell::new(
-                    CUTag::from(l.kind()),
+                    l.kind(),
                     // DO NOT RELY ON the payload's bit pattern; it's padded
                     DataRaw::word(l.data().dwordqn_promote()),
                 )
@@ -278,7 +293,7 @@ impl<'a> From<Lit<'a>> for Datacell {
                 // UNSAFE(@ohsayan): Correct because we are using the same tag, and in this case the type requires a new heap for construction
                 let mut bin = ManuallyDrop::new(l.bin().to_owned().into_boxed_slice());
                 Datacell::new(
-                    CUTag::from(l.kind()),
+                    l.kind(),
                     DataRaw::word(DwordQN::dwordqn_store_qw_nw(
                         bin.len() as u64,
                         bin.as_mut_ptr() as usize,
@@ -297,9 +312,9 @@ impl<'a> From<Lit<'a>> for Datacell {
 impl From<i32> for Datacell {
     fn from(i: i32) -> Self {
         if i.is_negative() {
-            Self::new_sint(i as _)
+            Self::new_sint_default(i as _)
         } else {
-            Self::new_uint(i as _)
+            Self::new_uint_default(i as _)
         }
     }
 }
@@ -311,7 +326,7 @@ impl<const N: usize> From<[Datacell; N]> for Datacell {
 }
 
 impl Datacell {
-    pub const fn tag(&self) -> CUTag {
+    pub const fn tag(&self) -> FullTag {
         self.tag
     }
     pub fn kind(&self) -> TagClass {
@@ -321,7 +336,7 @@ impl Datacell {
         unsafe {
             // UNSAFE(@ohsayan): This is a hack. It's safe because we set init to false
             Self::_new(
-                CUTag::BOOL,
+                FullTag::BOOL,
                 DataRaw::word(NativeQword::dwordnn_store_qw(0)),
                 false,
             )
@@ -339,10 +354,10 @@ impl Datacell {
     {
         self.data.word.load()
     }
-    unsafe fn _new(tag: CUTag, data: DataRaw, init: bool) -> Self {
+    unsafe fn _new(tag: FullTag, data: DataRaw, init: bool) -> Self {
         Self { init, tag, data }
     }
-    unsafe fn new(tag: CUTag, data: DataRaw) -> Self {
+    unsafe fn new(tag: FullTag, data: DataRaw) -> Self {
         Self::_new(tag, data, true)
     }
     fn checked_tag<T>(&self, tag: TagClass, f: impl FnOnce() -> T) -> Option<T> {
@@ -482,10 +497,7 @@ impl<'a> VirtualDatacell<'a> {
         Self {
             dc: ManuallyDrop::new(unsafe {
                 // UNSAFE(@ohsayan): this is a "reference" to a "virtual" aka fake DC. this just works because of memory layouts
-                Datacell::new(
-                    CUTag::from(lit.kind()),
-                    DataRaw::word(lit.data().dwordqn_promote()),
-                )
+                Datacell::new(lit.kind(), DataRaw::word(lit.data().dwordqn_promote()))
             }),
             _lt: PhantomData,
         }
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
index 837610dc..68b66d50 100644
--- a/server/src/engine/data/tag.rs
+++ b/server/src/engine/data/tag.rs
@@ -36,23 +36,6 @@ pub enum TagClass {
     List = 6,
 }
 
-impl TagClass {
-    pub const unsafe fn from_raw(v: u8) -> Self {
-        core::mem::transmute(v)
-    }
-    pub const fn tag_unique(&self) -> TagUnique {
-        [
-            TagUnique::Illegal,
-            TagUnique::UnsignedInt,
-            TagUnique::SignedInt,
-            TagUnique::Illegal,
-            TagUnique::Bin,
-            TagUnique::Str,
-            TagUnique::Illegal,
-        ][self.value_word()]
-    }
-}
-
 #[repr(u8)]
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, sky_macros::EnumMethods)]
 pub enum TagSelector {
@@ -206,57 +189,69 @@ impl DataTag for FullTag {
     }
 }
 
-#[derive(Debug, Clone, Copy)]
-pub struct CUTag {
-    class: TagClass,
-    unique: TagUnique,
-}
-
-impl PartialEq for CUTag {
-    fn eq(&self, other: &Self) -> bool {
-        self.class == other.class
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
+#[repr(transparent)]
+pub struct UIntSpec(FullTag);
+impl UIntSpec {
+    pub const LIM_MAX: [u64; 4] = as_array![u8::MAX, u16::MAX, u32::MAX, u64::MAX];
+    pub const DEFAULT: Self = Self::UINT64;
+    pub const UINT64: Self = Self(FullTag::new_uint(TagSelector::UInt64));
+    pub const unsafe fn from_full(f: FullTag) -> Self {
+        Self(f)
+    }
+    pub fn check(&self, v: u64) -> bool {
+        v <= Self::LIM_MAX[self.0.tag_selector().value_word() - 1]
     }
 }
 
-macro_rules! cutag {
-    ($class:ident, $unique:ident) => {
-        CUTag::new(TagClass::$class, TagUnique::$unique)
-    };
-    ($class:ident) => {
-        CUTag::new(TagClass::$class, TagUnique::Illegal)
-    };
+impl From<UIntSpec> for FullTag {
+    fn from(value: UIntSpec) -> Self {
+        value.0
+    }
 }
 
-impl CUTag {
-    pub const fn new(class: TagClass, unique: TagUnique) -> Self {
-        Self { class, unique }
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
+#[repr(transparent)]
+pub struct SIntSpec(FullTag);
+impl SIntSpec {
+    pub const LIM_MIN: [i64; 4] = as_array![i8::MIN, i16::MIN, i32::MIN, i64::MIN];
+    pub const LIM_MAX: [i64; 4] = as_array![i8::MAX, i16::MAX, i32::MAX, i64::MAX];
+    pub const DEFAULT: Self = Self::SINT64;
+    pub const SINT64: Self = Self(FullTag::new_sint(TagSelector::SInt64));
+    pub const unsafe fn from_full(f: FullTag) -> Self {
+        Self(f)
+    }
+    pub fn check(&self, i: i64) -> bool {
+        let tag = self.0.tag_selector().value_word() - 5;
+        (i >= Self::LIM_MIN[tag]) & (i <= Self::LIM_MAX[tag])
     }
 }
 
-impl DataTag for CUTag {
-    const BOOL: Self = cutag!(Bool);
-    const UINT: Self = cutag!(UnsignedInt, UnsignedInt);
-    const SINT: Self = cutag!(SignedInt, SignedInt);
-    const FLOAT: Self = cutag!(Float);
-    const BIN: Self = cutag!(Bin, Bin);
-    const STR: Self = cutag!(Str, Str);
-    const LIST: Self = cutag!(List);
-
-    fn tag_class(&self) -> TagClass {
-        self.class
+impl From<SIntSpec> for FullTag {
+    fn from(value: SIntSpec) -> Self {
+        value.0
     }
+}
 
-    fn tag_selector(&self) -> TagSelector {
-        unimplemented!()
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
+#[repr(transparent)]
+pub struct FloatSpec(FullTag);
+impl FloatSpec {
+    pub const LIM_MIN: [f64; 2] = as_array![f32::MIN, f64::MIN];
+    pub const LIM_MAX: [f64; 2] = as_array![f32::MAX, f64::MAX];
+    pub const DEFAULT: Self = Self::F64;
+    pub const F64: Self = Self(FullTag::new_float(TagSelector::Float64));
+    pub const unsafe fn from_full(f: FullTag) -> Self {
+        Self(f)
     }
-
-    fn tag_unique(&self) -> TagUnique {
-        self.unique
+    pub fn check(&self, f: f64) -> bool {
+        let tag = self.0.tag_selector().value_word() - 9;
+        (f >= Self::LIM_MIN[tag]) & (f <= Self::LIM_MAX[tag])
     }
 }
 
-impl From<FullTag> for CUTag {
-    fn from(f: FullTag) -> Self {
-        Self::new(f.tag_class(), f.tag_unique())
+impl From<FloatSpec> for FullTag {
+    fn from(value: FloatSpec) -> Self {
+        value.0
     }
 }
diff --git a/server/src/engine/data/tests/md_dict_tests.rs b/server/src/engine/data/tests/md_dict_tests.rs
index f7e81fbd..700c7a5c 100644
--- a/server/src/engine/data/tests/md_dict_tests.rs
+++ b/server/src/engine/data/tests/md_dict_tests.rs
@@ -36,7 +36,7 @@ fn t_simple_flatten() {
         "a_null_key" => Datacell::null(),
     };
     let expected: DictGeneric = into_dict!(
-        "a_valid_key" => Datacell::new_uint(100)
+        "a_valid_key" => Datacell::new_uint_default(100)
     );
     let ret = dict::rflatten_metadata(generic_dict);
     assert_eq!(ret, expected);
@@ -45,18 +45,18 @@ fn t_simple_flatten() {
 #[test]
 fn t_simple_patch() {
     let mut current: DictGeneric = into_dict! {
-        "a" => Datacell::new_uint(2),
-        "b" => Datacell::new_uint(3),
-        "z" => Datacell::new_sint(-100),
+        "a" => Datacell::new_uint_default(2),
+        "b" => Datacell::new_uint_default(3),
+        "z" => Datacell::new_sint_default(-100),
     };
     let new: DictGeneric = into_dict! {
-        "a" => Datacell::new_uint(1),
-        "b" => Datacell::new_uint(2),
+        "a" => Datacell::new_uint_default(1),
+        "b" => Datacell::new_uint_default(2),
         "z" => Datacell::null(),
     };
     let expected: DictGeneric = into_dict! {
-        "a" => Datacell::new_uint(1),
-        "b" => Datacell::new_uint(2),
+        "a" => Datacell::new_uint_default(1),
+        "b" => Datacell::new_uint_default(2),
     };
     assert!(dict::rmerge_metadata(&mut current, new));
     assert_eq!(current, expected);
@@ -65,14 +65,14 @@ fn t_simple_patch() {
 #[test]
 fn t_bad_patch() {
     let mut current: DictGeneric = into_dict! {
-        "a" => Datacell::new_uint(2),
-        "b" => Datacell::new_uint(3),
-        "z" => Datacell::new_sint(-100),
+        "a" => Datacell::new_uint_default(2),
+        "b" => Datacell::new_uint_default(3),
+        "z" => Datacell::new_sint_default(-100),
     };
     let backup = current.clone();
     let new: DictGeneric = into_dict! {
-        "a" => Datacell::new_uint(1),
-        "b" => Datacell::new_uint(2),
+        "a" => Datacell::new_uint_default(1),
+        "b" => Datacell::new_uint_default(2),
         "z" => Datacell::new_str("omg".into()),
     };
     assert!(!dict::rmerge_metadata(&mut current, new));
@@ -82,20 +82,20 @@ fn t_bad_patch() {
 #[test]
 fn patch_null_out_dict() {
     let mut current: DictGeneric = into_dict! {
-        "a" => Datacell::new_uint(2),
-        "b" => Datacell::new_uint(3),
+        "a" => Datacell::new_uint_default(2),
+        "b" => Datacell::new_uint_default(3),
         "z" => DictEntryGeneric::Map(into_dict!(
-            "c" => Datacell::new_uint(1),
-            "d" => Datacell::new_uint(2)
+            "c" => Datacell::new_uint_default(1),
+            "d" => Datacell::new_uint_default(2)
         )),
     };
     let expected: DictGeneric = into_dict! {
-        "a" => Datacell::new_uint(2),
-        "b" => Datacell::new_uint(3),
+        "a" => Datacell::new_uint_default(2),
+        "b" => Datacell::new_uint_default(3),
     };
     let new: DictGeneric = into_dict! {
-        "a" => Datacell::new_uint(2),
-        "b" => Datacell::new_uint(3),
+        "a" => Datacell::new_uint_default(2),
+        "b" => Datacell::new_uint_default(3),
         "z" => Datacell::null(),
     };
     assert!(dict::rmerge_metadata(&mut current, new));
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index b23e8e72..f2f94739 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -29,6 +29,10 @@ macro_rules! into_array {
     ($($e:expr),* $(,)?) => { [$($e.into()),*] };
 }
 
+macro_rules! as_array {
+    ($($e:expr),* $(,)?) => { [$($e as _),*] };
+}
+
 macro_rules! extract {
     ($src:expr, $what:pat => $ret:expr) => {
         if let $what = $src {
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index b75d3c9c..4265b71a 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -72,7 +72,7 @@ fn pfnbase_uuid() -> Uuid {
 // impl
 #[inline(always)]
 fn pfn_timesec() -> Datacell {
-    Datacell::new_uint(pfnbase_time().as_secs())
+    Datacell::new_uint_default(pfnbase_time().as_secs())
 }
 #[inline(always)]
 fn pfn_uuidstr() -> Datacell {
diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs
index 3dfcbb2b..c29e3486 100644
--- a/server/src/engine/storage/v1/batch_jrnl/persist.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use crate::engine::storage::v1::inf::obj::cell;
+
 use {
     super::{
         MARKER_ACTUAL_BATCH_EVENT, MARKER_BATCH_CLOSED, MARKER_BATCH_REOPEN, MARKER_END_OF_BATCH,
@@ -40,14 +42,11 @@ use {
             },
             data::{
                 cell::Datacell,
-                tag::{DataTag, TagClass, TagUnique},
+                tag::{DataTag, TagUnique},
             },
             error::{RuntimeResult, StorageError},
             idx::STIndexSeq,
-            storage::v1::{
-                inf::PersistTypeDscr,
-                rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedWriter},
-            },
+            storage::v1::rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedWriter},
         },
         util::EndianQW,
     },
@@ -224,49 +223,9 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
     }
     /// Encode a single cell
     fn encode_cell(&mut self, value: &Datacell) -> RuntimeResult<()> {
-        let ref mut buf = self.f;
-        buf.unfsynced_write(&[
-            PersistTypeDscr::translate_from_class(value.tag().tag_class()).value_u8(),
-        ])?;
-        match value.tag().tag_class() {
-            TagClass::Bool if value.is_null() => {}
-            TagClass::Bool => {
-                let bool = unsafe {
-                    // UNSAFE(@ohsayan): +tagck
-                    value.read_bool()
-                } as u8;
-                buf.unfsynced_write(&[bool])?;
-            }
-            TagClass::SignedInt | TagClass::UnsignedInt | TagClass::Float => {
-                let chunk = unsafe {
-                    // UNSAFE(@ohsayan): +tagck
-                    value.read_uint()
-                }
-                .to_le_bytes();
-                buf.unfsynced_write(&chunk)?;
-            }
-            TagClass::Str | TagClass::Bin => {
-                let slice = unsafe {
-                    // UNSAFE(@ohsayan): +tagck
-                    value.read_bin()
-                };
-                let slice_l = slice.len().u64_bytes_le();
-                buf.unfsynced_write(&slice_l)?;
-                buf.unfsynced_write(slice)?;
-            }
-            TagClass::List => {
-                let list = unsafe {
-                    // UNSAFE(@ohsayan): +tagck
-                    value.read_list()
-                }
-                .read();
-                let list_l = list.len().u64_bytes_le();
-                buf.unfsynced_write(&list_l)?;
-                for item in list.iter() {
-                    self.encode_cell(item)?;
-                }
-            }
-        }
+        let mut buf = vec![];
+        cell::encode(&mut buf, value);
+        self.f.unfsynced_write(&buf)?;
         Ok(())
     }
     /// Encode row data
diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs
index 37c826f1..1454735b 100644
--- a/server/src/engine/storage/v1/batch_jrnl/restore.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs
@@ -24,6 +24,11 @@
  *
 */
 
+use crate::engine::storage::v1::inf::{
+    obj::cell::{self, StorageCellTypeID},
+    DataSource,
+};
+
 use {
     super::{
         MARKER_ACTUAL_BATCH_EVENT, MARKER_BATCH_CLOSED, MARKER_BATCH_REOPEN, MARKER_END_OF_BATCH,
@@ -34,16 +39,10 @@ use {
             index::{DcFieldIndex, PrimaryIndexKey, Row},
             model::{delta::DeltaVersion, Model},
         },
-        data::{
-            cell::Datacell,
-            tag::{CUTag, TagClass, TagUnique},
-        },
+        data::{cell::Datacell, tag::TagUnique},
         error::{RuntimeResult, StorageError},
         idx::{MTIndex, STIndex, STIndexSeq},
-        storage::v1::{
-            inf::PersistTypeDscr,
-            rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedReader},
-        },
+        storage::v1::rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedReader},
     },
     crossbeam_epoch::pin,
     std::{
@@ -500,58 +499,43 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
         })
     }
     fn decode_cell(&mut self) -> RuntimeResult<Datacell> {
-        let cell_type_sig = self.f.read_byte()?;
-        let Some(cell_type) = PersistTypeDscr::try_from_raw(cell_type_sig) else {
+        let Some(dscr) = StorageCellTypeID::try_from_raw(self.f.read_byte()?) else {
             return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
         };
-        Ok(match cell_type {
-            PersistTypeDscr::Null => Datacell::null(),
-            PersistTypeDscr::Bool => {
-                let bool = self.f.read_byte()?;
-                if bool > 1 {
-                    return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
-                }
-                Datacell::new_bool(bool == 1)
-            }
-            PersistTypeDscr::UnsignedInt | PersistTypeDscr::SignedInt | PersistTypeDscr::Float => {
-                let qw = self.f.read_u64_le()?;
-                unsafe {
-                    // UNSAFE(@ohsayan): choosing the correct type and tag
-                    let tc = TagClass::from_raw(cell_type.value_u8() - 1);
-                    Datacell::new_qw(qw, CUTag::new(tc, tc.tag_unique()))
-                }
-            }
-            PersistTypeDscr::Str | PersistTypeDscr::Bin => {
-                let len = self.f.read_u64_le()? as usize;
-                let mut data = vec![0; len];
-                self.f.read_into_buffer(&mut data)?;
-                unsafe {
-                    // UNSAFE(@ohsayan): +tagck
-                    if cell_type == PersistTypeDscr::Str {
-                        if core::str::from_utf8(&data).is_err() {
-                            return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
-                        }
-                        Datacell::new_str(String::from_utf8_unchecked(data).into_boxed_str())
-                    } else {
-                        Datacell::new_bin(data.into_boxed_slice())
-                    }
-                }
-            }
-            PersistTypeDscr::List => {
-                let len = self.f.read_u64_le()?;
-                let mut list = Vec::new();
-                while !self.f.is_eof() && list.len() as u64 != len {
-                    list.push(self.decode_cell()?);
-                }
-                if len != list.len() as u64 {
-                    return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
-                }
-                Datacell::new_list(list)
-            }
-            PersistTypeDscr::Dict => {
-                // we don't support dicts just yet
-                return Err(StorageError::DataBatchRestoreCorruptedEntry.into());
-            }
-        })
+        unsafe { cell::decode_element::<Datacell, SDSSFileTrackedReader<F>>(&mut self.f, dscr) }
+            .map_err(|e| e.0)
+    }
+}
+
+pub struct ErrorHack(crate::engine::fractal::error::Error);
+impl From<crate::engine::fractal::error::Error> for ErrorHack {
+    fn from(value: crate::engine::fractal::error::Error) -> Self {
+        Self(value)
+    }
+}
+impl From<()> for ErrorHack {
+    fn from(_: ()) -> Self {
+        Self(StorageError::DataBatchRestoreCorruptedEntry.into())
+    }
+}
+impl<F: RawFSInterface> DataSource for SDSSFileTrackedReader<F> {
+    const RELIABLE_SOURCE: bool = false;
+    type Error = ErrorHack;
+    fn has_remaining(&self, cnt: usize) -> bool {
+        self.remaining() >= cnt as u64
+    }
+    unsafe fn read_next_byte(&mut self) -> Result<u8, Self::Error> {
+        Ok(self.read_byte()?)
+    }
+    unsafe fn read_next_block<const N: usize>(&mut self) -> Result<[u8; N], Self::Error> {
+        Ok(self.read_block()?)
+    }
+    unsafe fn read_next_u64_le(&mut self) -> Result<u64, Self::Error> {
+        Ok(self.read_u64_le()?)
+    }
+    unsafe fn read_next_variable_block(&mut self, size: usize) -> Result<Vec<u8>, Self::Error> {
+        let mut buf = vec![0; size];
+        self.read_into_buffer(&mut buf)?;
+        Ok(buf)
     }
 }
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index c4762c67..bd4ba03f 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -26,18 +26,16 @@
 
 use {
     super::{
-        obj::{self, FieldMD},
-        PersistMapSpec, PersistObject, PersistTypeDscr, VecU8,
+        obj::{
+            cell::{self, CanYieldDict, StorageCellTypeID},
+            FieldMD,
+        },
+        PersistMapSpec, PersistObject, VecU8,
     },
     crate::{
         engine::{
             core::model::Field,
-            data::{
-                cell::Datacell,
-                dict::DictEntryGeneric,
-                tag::{CUTag, TagUnique},
-                DictGeneric,
-            },
+            data::{dict::DictEntryGeneric, DictGeneric},
             error::{RuntimeResult, StorageError},
             idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
             mem::BufferedScanner,
@@ -175,9 +173,8 @@ impl PersistMapSpec for GenericDictSpec {
         scanner.has_left(9)
     }
     fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool {
-        static EXPECT_ATLEAST: [u8; 4] = [0, 1, 8, 8]; // PAD to align
-        let lbound_rem = md.klen + EXPECT_ATLEAST[md.dscr.min(3) as usize] as usize;
-        scanner.has_left(lbound_rem) & (md.dscr <= PersistTypeDscr::Dict.value_u8())
+        StorageCellTypeID::is_valid(md.dscr)
+            & scanner.has_left(StorageCellTypeID::expect_atleast(md.dscr))
     }
     fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) {
         buf.extend(key.len().u64_bytes_le());
@@ -188,14 +185,14 @@ impl PersistMapSpec for GenericDictSpec {
     fn enc_entry(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
         match val {
             DictEntryGeneric::Map(map) => {
-                buf.push(PersistTypeDscr::Dict.value_u8());
+                buf.push(StorageCellTypeID::Dict.value_u8());
                 buf.extend(key.as_bytes());
                 <PersistMapImpl<Self> as PersistObject>::default_full_enc(buf, map);
             }
             DictEntryGeneric::Data(dc) => {
-                obj::encode_datacell_tag(buf, dc);
+                buf.push(cell::encode_tag(dc));
                 buf.extend(key.as_bytes());
-                obj::encode_element(buf, dc);
+                cell::encode_cell(buf, dc);
             }
         }
     }
@@ -205,85 +202,20 @@ impl PersistMapSpec for GenericDictSpec {
             .ok()
     }
     unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value> {
-        unsafe fn decode_element(
-            scanner: &mut BufferedScanner,
-            dscr: PersistTypeDscr,
-            dg_top_element: bool,
-        ) -> Option<DictEntryGeneric> {
-            let r = match dscr {
-                PersistTypeDscr::Null => DictEntryGeneric::Data(Datacell::null()),
-                PersistTypeDscr::Bool => {
-                    DictEntryGeneric::Data(Datacell::new_bool(scanner.next_byte() == 1))
-                }
-                PersistTypeDscr::UnsignedInt
-                | PersistTypeDscr::SignedInt
-                | PersistTypeDscr::Float => DictEntryGeneric::Data(Datacell::new_qw(
-                    scanner.next_u64_le(),
-                    CUTag::new(
-                        dscr.into_class(),
-                        [
-                            TagUnique::UnsignedInt,
-                            TagUnique::SignedInt,
-                            TagUnique::Illegal,
-                            TagUnique::Illegal, // pad
-                        ][(dscr.value_u8() - 2) as usize],
-                    ),
-                )),
-                PersistTypeDscr::Str | PersistTypeDscr::Bin => {
-                    let slc_len = scanner.next_u64_le() as usize;
-                    if !scanner.has_left(slc_len) {
-                        return None;
-                    }
-                    let slc = scanner.next_chunk_variable(slc_len);
-                    DictEntryGeneric::Data(if dscr == PersistTypeDscr::Str {
-                        if core::str::from_utf8(slc).is_err() {
-                            return None;
-                        }
-                        Datacell::new_str(
-                            String::from_utf8_unchecked(slc.to_owned()).into_boxed_str(),
-                        )
-                    } else {
-                        Datacell::new_bin(slc.to_owned().into_boxed_slice())
-                    })
-                }
-                PersistTypeDscr::List => {
-                    let list_len = scanner.next_u64_le() as usize;
-                    let mut v = Vec::with_capacity(list_len);
-                    while (!scanner.eof()) & (v.len() < list_len) {
-                        let dscr = scanner.next_byte();
-                        if dscr > PersistTypeDscr::Dict.value_u8() {
-                            return None;
-                        }
-                        v.push(
-                            match decode_element(scanner, PersistTypeDscr::from_raw(dscr), false) {
-                                Some(DictEntryGeneric::Data(l)) => l,
-                                None => return None,
-                                _ => unreachable!("found top-level dict item in datacell"),
-                            },
-                        );
-                    }
-                    if v.len() == list_len {
-                        DictEntryGeneric::Data(Datacell::new_list(v))
-                    } else {
-                        return None;
-                    }
-                }
-                PersistTypeDscr::Dict => {
-                    if dg_top_element {
-                        DictEntryGeneric::Map(
-                            <PersistMapImpl<GenericDictSpec> as PersistObject>::default_full_dec(
-                                scanner,
-                            )
-                            .ok()?,
-                        )
-                    } else {
-                        unreachable!("found top-level dict item in datacell")
-                    }
-                }
-            };
-            Some(r)
-        }
-        decode_element(scanner, PersistTypeDscr::from_raw(md.dscr), true)
+        Some(
+            match cell::decode_element::<CanYieldDict, BufferedScanner>(
+                scanner,
+                StorageCellTypeID::from_raw(md.dscr),
+            )
+            .ok()?
+            {
+                CanYieldDict::Data(d) => DictEntryGeneric::Data(d),
+                CanYieldDict::Dict => DictEntryGeneric::Map(
+                    <PersistMapImpl<GenericDictSpec> as PersistObject>::default_full_dec(scanner)
+                        .ok()?,
+                ),
+            },
+        )
     }
     // not implemented
     fn enc_key(_: &mut VecU8, _: &Self::Key) {
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index 3e7dae49..24926db7 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -26,77 +26,46 @@
 
 //! High level interfaces
 
-use crate::engine::idx::STIndex;
-
 pub mod map;
 pub mod obj;
 // tests
 #[cfg(test)]
 mod tests;
 
-use {
-    crate::engine::{
-        data::tag::TagClass,
-        error::{RuntimeResult, StorageError},
-        idx::{AsKey, AsValue},
-        mem::BufferedScanner,
-    },
-    std::mem,
+use crate::engine::{
+    error::{RuntimeResult, StorageError},
+    idx::{AsKey, AsValue, STIndex},
+    mem::BufferedScanner,
 };
 
 type VecU8 = Vec<u8>;
 
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, sky_macros::EnumMethods)]
-#[repr(u8)]
-#[allow(unused)]
-/// Disambiguation for data
-pub enum PersistTypeDscr {
-    Null = 0,
-    Bool = 1,
-    UnsignedInt = 2,
-    SignedInt = 3,
-    Float = 4,
-    Bin = 5,
-    Str = 6,
-    List = 7,
-    Dict = 8,
+pub trait DataSource {
+    type Error;
+    const RELIABLE_SOURCE: bool = true;
+    fn has_remaining(&self, cnt: usize) -> bool;
+    unsafe fn read_next_byte(&mut self) -> Result<u8, Self::Error>;
+    unsafe fn read_next_block<const N: usize>(&mut self) -> Result<[u8; N], Self::Error>;
+    unsafe fn read_next_u64_le(&mut self) -> Result<u64, Self::Error>;
+    unsafe fn read_next_variable_block(&mut self, size: usize) -> Result<Vec<u8>, Self::Error>;
 }
 
-#[allow(unused)]
-impl PersistTypeDscr {
-    /// translates the tag class definition into the dscr definition
-    pub const fn translate_from_class(class: TagClass) -> Self {
-        unsafe { Self::from_raw(class.value_u8() + 1) }
-    }
-    pub const fn try_from_raw(v: u8) -> Option<Self> {
-        if v > Self::MAX {
-            None
-        } else {
-            unsafe { Some(Self::from_raw(v)) }
-        }
-    }
-    pub const unsafe fn from_raw(v: u8) -> Self {
-        core::mem::transmute(v)
-    }
-    /// The data in question is null (well, can we call that data afterall?)
-    pub const fn is_null(&self) -> bool {
-        self.value_u8() == Self::Null.value_u8()
+impl<'a> DataSource for BufferedScanner<'a> {
+    type Error = ();
+    fn has_remaining(&self, cnt: usize) -> bool {
+        self.has_left(cnt)
     }
-    /// The data in question is a scalar
-    pub const fn is_scalar(&self) -> bool {
-        self.value_u8() <= Self::Float.value_u8()
+    unsafe fn read_next_byte(&mut self) -> Result<u8, Self::Error> {
+        Ok(self.next_byte())
     }
-    /// The data is composite
-    pub const fn is_composite(&self) -> bool {
-        self.value_u8() > Self::Float.value_u8()
+    unsafe fn read_next_block<const N: usize>(&mut self) -> Result<[u8; N], Self::Error> {
+        Ok(self.next_chunk())
     }
-    /// Recursive data
-    pub const fn is_recursive(&self) -> bool {
-        self.value_u8() >= Self::List.value_u8()
+    unsafe fn read_next_u64_le(&mut self) -> Result<u64, Self::Error> {
+        Ok(self.next_u64_le())
     }
-    fn into_class(&self) -> TagClass {
-        debug_assert!(*self != Self::Null);
-        unsafe { mem::transmute(self.value_u8() - 1) }
+    unsafe fn read_next_variable_block(&mut self, size: usize) -> Result<Vec<u8>, Self::Error> {
+        Ok(self.next_chunk_variable(size).into())
     }
 }
 
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 647126fd..c9876ec2 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    super::{PersistObject, PersistTypeDscr, VecU8},
+    super::{PersistObject, VecU8},
     crate::{
         engine::{
             core::{
@@ -33,7 +33,6 @@ use {
                 space::{Space, SpaceMeta},
             },
             data::{
-                cell::Datacell,
                 tag::{DataTag, TagClass, TagSelector},
                 uuid::Uuid,
                 DictGeneric,
@@ -46,34 +45,202 @@ use {
     },
 };
 
-pub fn encode_element(buf: &mut VecU8, dc: &Datacell) {
-    unsafe {
-        use TagClass::*;
-        match dc.tag().tag_class() {
-            Bool if dc.is_init() => buf.push(dc.read_bool() as u8),
-            Bool => {}
-            UnsignedInt | SignedInt | Float => buf.extend(dc.read_uint().to_le_bytes()),
-            Str | Bin => {
-                let slc = dc.read_bin();
-                buf.extend(slc.len().u64_bytes_le());
-                buf.extend(slc);
+/*
+    generic cells
+*/
+
+pub mod cell {
+    use crate::{
+        engine::{
+            data::{
+                cell::Datacell,
+                tag::{DataTag, TagClass, TagSelector},
+            },
+            storage::v1::inf::{DataSource, VecU8},
+        },
+        util::EndianQW,
+    };
+
+    #[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord, Hash, sky_macros::EnumMethods)]
+    #[repr(u8)]
+    #[allow(dead_code)]
+    pub enum StorageCellTypeID {
+        Null = 0x00,
+        Bool = 0x01,
+        UInt8 = 0x02,
+        UInt16 = 0x03,
+        UInt32 = 0x04,
+        UInt64 = 0x05,
+        SInt8 = 0x06,
+        SInt16 = 0x07,
+        SInt32 = 0x08,
+        SInt64 = 0x09,
+        Float32 = 0x0A,
+        Float64 = 0x0B,
+        Bin = 0x0C,
+        Str = 0x0D,
+        List = 0x0E,
+        Dict = 0x0F,
+    }
+    impl StorageCellTypeID {
+        pub const unsafe fn from_raw(v: u8) -> Self {
+            core::mem::transmute(v)
+        }
+        pub const fn try_from_raw(v: u8) -> Option<Self> {
+            if Self::is_valid(v) {
+                Some(unsafe { Self::from_raw(v) })
+            } else {
+                None
             }
-            List => {
-                let lst = dc.read_list().read();
-                buf.extend(lst.len().u64_bytes_le());
-                for item in lst.iter() {
-                    encode_element(buf, item);
+        }
+        #[inline(always)]
+        pub const fn is_valid(d: u8) -> bool {
+            d <= Self::MAX
+        }
+        const unsafe fn into_selector(self) -> TagSelector {
+            debug_assert!(self.value_u8() != Self::Null.value_u8());
+            core::mem::transmute(self.value_u8() - 1)
+        }
+        #[inline(always)]
+        pub fn expect_atleast(d: u8) -> usize {
+            [0u8, 1, 8, 8][d.min(3) as usize] as usize
+        }
+    }
+    pub fn encode(buf: &mut VecU8, dc: &Datacell) {
+        buf.push(encode_tag(dc));
+        encode_cell(buf, dc)
+    }
+    pub fn encode_tag(dc: &Datacell) -> u8 {
+        (dc.tag().tag_selector().value_u8() + 1) * (dc.is_init() as u8)
+    }
+    pub fn encode_cell(buf: &mut VecU8, dc: &Datacell) {
+        if dc.is_null() {
+            return;
+        }
+        unsafe {
+            use TagClass::*;
+            match dc.tag().tag_class() {
+                Bool if dc.is_init() => buf.push(dc.read_bool() as u8),
+                Bool => {}
+                UnsignedInt | SignedInt | Float => buf.extend(dc.read_uint().to_le_bytes()),
+                Str | Bin => {
+                    let slc = dc.read_bin();
+                    buf.extend(slc.len().u64_bytes_le());
+                    buf.extend(slc);
+                }
+                List => {
+                    let lst = dc.read_list().read();
+                    buf.extend(lst.len().u64_bytes_le());
+                    for item in lst.iter() {
+                        encode(buf, item);
+                    }
                 }
             }
         }
     }
-}
-
-pub fn encode_datacell_tag(buf: &mut VecU8, dc: &Datacell) {
-    buf.push(
-        PersistTypeDscr::translate_from_class(dc.tag().tag_class()).value_u8()
-            * (!dc.is_null() as u8),
-    )
+    pub trait ElementYield {
+        type Yield;
+        type Error;
+        const CAN_YIELD_DICT: bool = false;
+        fn yield_data(dc: Datacell) -> Result<Self::Yield, Self::Error>;
+        fn yield_dict() -> Result<Self::Yield, Self::Error> {
+            panic!()
+        }
+        fn error() -> Result<Self::Yield, Self::Error>;
+    }
+    impl ElementYield for Datacell {
+        type Yield = Self;
+        type Error = ();
+        fn yield_data(dc: Datacell) -> Result<Self::Yield, Self::Error> {
+            Ok(dc)
+        }
+        fn error() -> Result<Self::Yield, Self::Error> {
+            Err(())
+        }
+    }
+    #[derive(Debug, PartialEq)]
+    pub enum CanYieldDict {
+        Data(Datacell),
+        Dict,
+    }
+    impl ElementYield for CanYieldDict {
+        type Yield = Self;
+        type Error = ();
+        const CAN_YIELD_DICT: bool = true;
+        fn error() -> Result<Self::Yield, Self::Error> {
+            Err(())
+        }
+        fn yield_data(dc: Datacell) -> Result<Self::Yield, Self::Error> {
+            Ok(CanYieldDict::Data(dc))
+        }
+        fn yield_dict() -> Result<Self::Yield, Self::Error> {
+            Ok(CanYieldDict::Dict)
+        }
+    }
+    pub unsafe fn decode_element<EY: ElementYield, DS: DataSource>(
+        s: &mut DS,
+        dscr: StorageCellTypeID,
+    ) -> Result<EY::Yield, DS::Error>
+    where
+        DS::Error: From<EY::Error>,
+        DS::Error: From<()>,
+    {
+        if dscr == StorageCellTypeID::Dict {
+            if EY::CAN_YIELD_DICT {
+                return Ok(EY::yield_dict()?);
+            } else {
+                return Ok(EY::error()?);
+            }
+        }
+        if dscr == StorageCellTypeID::Null {
+            return Ok(EY::yield_data(Datacell::null())?);
+        }
+        let tag = dscr.into_selector().into_full();
+        let d = match tag.tag_class() {
+            TagClass::Bool => {
+                let nx = s.read_next_byte()?;
+                if nx > 1 {
+                    return Ok(EY::error()?);
+                }
+                Datacell::new_bool(nx == 1)
+            }
+            TagClass::UnsignedInt | TagClass::SignedInt | TagClass::Float => {
+                let nx = s.read_next_u64_le()?;
+                Datacell::new_qw(nx, tag)
+            }
+            TagClass::Bin | TagClass::Str => {
+                let len = s.read_next_u64_le()? as usize;
+                let block = s.read_next_variable_block(len)?;
+                if tag.tag_class() == TagClass::Str {
+                    match String::from_utf8(block).map(|s| Datacell::new_str(s.into_boxed_str())) {
+                        Ok(s) => s,
+                        Err(_) => return Ok(EY::error()?),
+                    }
+                } else {
+                    Datacell::new_bin(block.into())
+                }
+            }
+            TagClass::List => {
+                let len = s.read_next_u64_le()? as usize;
+                let mut l = vec![];
+                while (l.len() != len) & s.has_remaining(1) {
+                    let Some(dscr) = StorageCellTypeID::try_from_raw(s.read_next_byte()?) else {
+                        return Ok(EY::error()?);
+                    };
+                    // FIXME(@ohsayan): right now, a list cannot contain a dict!
+                    if !s.has_remaining(StorageCellTypeID::expect_atleast(dscr.value_u8())) {
+                        return Ok(EY::error()?);
+                    }
+                    l.push(self::decode_element::<Datacell, DS>(s, dscr)?);
+                }
+                if l.len() != len {
+                    return Ok(EY::error()?);
+                }
+                Datacell::new_list(l)
+            }
+        };
+        Ok(EY::yield_data(d)?)
+    }
 }
 
 /*
diff --git a/server/src/engine/storage/v1/inf/tests.rs b/server/src/engine/storage/v1/inf/tests.rs
index 9f0931b8..1ce59fb4 100644
--- a/server/src/engine/storage/v1/inf/tests.rs
+++ b/server/src/engine/storage/v1/inf/tests.rs
@@ -47,7 +47,7 @@ fn dict() {
         "hello" => Datacell::new_str("world".into()),
         "omg a null?" => Datacell::null(),
         "a big fat dict" => DictEntryGeneric::Map(into_dict!(
-            "with a value" => Datacell::new_uint(1002),
+            "with a value" => Datacell::new_uint_default(1002),
             "and a null" => Datacell::null(),
         ))
     };
diff --git a/server/src/engine/storage/v1/sysdb.rs b/server/src/engine/storage/v1/sysdb.rs
index 7b4dbd4c..71703c3d 100644
--- a/server/src/engine/storage/v1/sysdb.rs
+++ b/server/src/engine/storage/v1/sysdb.rs
@@ -149,8 +149,8 @@ pub fn sync_system_database_to<Fs: RawFSInterface>(
     // prepare our flat file
     let mut map: DictGeneric = into_dict!(
         SYS_KEY_SYS => DictEntryGeneric::Map(into_dict!(
-            SYS_KEY_SYS_SETTINGS_VERSION => Datacell::new_uint(cfg.host_data().settings_version() as _),
-            SYS_KEY_SYS_STARTUP_COUNTER => Datacell::new_uint(cfg.host_data().startup_counter() as _),
+            SYS_KEY_SYS_SETTINGS_VERSION => Datacell::new_uint_default(cfg.host_data().settings_version() as _),
+            SYS_KEY_SYS_STARTUP_COUNTER => Datacell::new_uint_default(cfg.host_data().startup_counter() as _),
         )),
         SYS_KEY_AUTH => DictGeneric::new(),
     );
diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
index ec66e073..a23c351e 100644
--- a/server/src/engine/txn/gns/tests/full_chain.rs
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -80,7 +80,7 @@ fn create_space() {
                 global.namespace().spaces().read().get("myspace").unwrap(),
                 &Space::new_restore_empty(
                     SpaceMeta::with_env(
-                        into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint(65536)))
+                        into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536)))
                     ),
                     uuid
                 )
@@ -108,7 +108,7 @@ fn alter_space() {
                 global.namespace().spaces().read().get("myspace").unwrap(),
                 &Space::new_restore_empty(
                     SpaceMeta::with_env(
-                        into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint(65536)))
+                        into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536)))
                     ),
                     uuid
                 )

From b7fd815e9e06451f173ae7339b246f34ee66df18 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 3 Nov 2023 14:31:46 +0000
Subject: [PATCH 274/310] Enable select exec [skip ci]

---
 server/src/engine/core/dml/mod.rs     |  3 +-
 server/src/engine/core/dml/sel.rs     | 56 +++++++++++++++++++++++++--
 server/src/engine/core/exec.rs        | 10 ++---
 server/src/engine/core/model/delta.rs | 11 +-----
 server/src/engine/fractal/mgr.rs      |  2 +-
 server/src/engine/net/protocol/mod.rs |  5 +++
 6 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index 402c6b45..702ed855 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -44,7 +44,7 @@ pub use upd::collect_trace_path as update_flow_trace;
 pub use {
     del::{delete, delete_resp},
     ins::{insert, insert_resp},
-    sel::select_custom,
+    sel::{select_custom, select_resp},
     upd::{update, update_resp},
 };
 
@@ -65,6 +65,7 @@ impl Model {
     }
 }
 
+#[derive(Debug)]
 pub struct QueryExecMeta {
     delta_hint: usize,
 }
diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
index d281a27b..8c2d0f1f 100644
--- a/server/src/engine/core/dml/sel.rs
+++ b/server/src/engine/core/dml/sel.rs
@@ -26,7 +26,10 @@
 
 use crate::engine::{
     core::index::DcFieldIndex,
-    data::cell::{Datacell, VirtualDatacell},
+    data::{
+        cell::{Datacell, VirtualDatacell},
+        tag::{DataTag, TagClass},
+    },
     error::{QueryError, QueryResult},
     fractal::GlobalInstanceLike,
     idx::{STIndex, STIndexSeq},
@@ -39,10 +42,57 @@ pub fn select_resp(
     global: &impl GlobalInstanceLike,
     select: SelectStatement,
 ) -> QueryResult<Response> {
-    todo!()
+    let mut resp_b = vec![];
+    let mut resp_a = vec![];
+    let mut i = 0u64;
+    self::select_custom(global, select, |item| {
+        encode_cell(&mut resp_b, item);
+        i += 1;
+    })?;
+    resp_a.push(0x11);
+    resp_a.extend(i.to_string().as_bytes());
+    resp_a.push(b'\n');
+    Ok(Response::EncodedAB(
+        resp_a.into_boxed_slice(),
+        resp_b.into_boxed_slice(),
+    ))
+}
+
+fn encode_cell(resp: &mut Vec<u8>, item: &Datacell) {
+    resp.push((item.tag().tag_selector().value_u8() + 1) * (item.is_init() as u8));
+    if item.is_null() {
+        return;
+    }
+    unsafe {
+        // UNSAFE(@ohsayan): +tagck
+        // NOTE(@ohsayan): optimize out unwanted alloc
+        match item.tag().tag_class() {
+            TagClass::Bool => resp.push(item.read_bool() as _),
+            TagClass::UnsignedInt => resp.extend(item.read_uint().to_string().as_bytes()),
+            TagClass::SignedInt => resp.extend(item.read_sint().to_string().as_bytes()),
+            TagClass::Float => resp.extend(item.read_float().to_string().as_bytes()),
+            TagClass::Bin | TagClass::Str => {
+                let slc = item.read_bin();
+                resp.extend(slc.len().to_string().as_bytes());
+                resp.push(b'\n');
+                resp.extend(slc);
+                return;
+            }
+            TagClass::List => {
+                let list = item.read_list();
+                let ls = list.read();
+                resp.extend(ls.len().to_string().as_bytes());
+                resp.push(b'\n');
+                for item in ls.iter() {
+                    encode_cell(resp, item);
+                }
+                return;
+            }
+        }
+    }
+    resp.push(b'\n');
 }
 
-#[allow(unused)]
 pub fn select_custom<F>(
     global: &impl GlobalInstanceLike,
     mut select: SelectStatement,
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 8006e2d9..2ae93770 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -172,14 +172,14 @@ fn run_nb(
 ) -> QueryResult<Response> {
     let stmt = stmt.value_u8() - KeywordStmt::Use.value_u8();
     static F: [fn(&Global, &mut State<'static, InplaceData>) -> QueryResult<Response>; 8] = [
-        |_, _| panic!("use not implemented"),
-        |_, _| panic!("inspect not implemented"),
-        |_, _| panic!("describe not implemented"),
+        |_, _| Err(QueryError::QLUnknownStatement), // use
+        |_, _| Err(QueryError::QLUnknownStatement), // inspect
+        |_, _| Err(QueryError::QLUnknownStatement), // describe
         |g, s| _call(g, s, dml::insert_resp),
-        |_, _| panic!("select not implemented"),
+        |g, s| _call(g, s, dml::select_resp),
         |g, s| _call(g, s, dml::update_resp),
         |g, s| _call(g, s, dml::delete_resp),
-        |_, _| panic!("exists not implemented"),
+        |_, _| Err(QueryError::QLUnknownStatement), // exists
     ];
     {
         let mut state = unsafe {
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index af1c8d4d..91b1434f 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -221,13 +221,7 @@ impl DeltaState {
         self.data_current_version.fetch_add(1, Ordering::AcqRel)
     }
     pub fn __data_delta_dequeue(&self, g: &Guard) -> Option<DataDelta> {
-        match self.data_deltas.blocking_try_dequeue(g) {
-            Some(d) => {
-                self.data_deltas_size.fetch_sub(1, Ordering::Release);
-                Some(d)
-            }
-            None => None,
-        }
+        self.data_deltas.blocking_try_dequeue(g)
     }
 }
 
@@ -284,9 +278,6 @@ impl DeltaState {
 
 // fractal
 impl DeltaState {
-    pub fn __fractal_take_from_data_delta(&self, cnt: usize, _token: FractalToken) {
-        let _ = self.data_deltas_size.fetch_sub(cnt, Ordering::Release);
-    }
     pub fn __fractal_take_full_from_data_delta(&self, _token: FractalToken) -> usize {
         self.data_deltas_size.swap(0, Ordering::AcqRel)
     }
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index e83e53da..99375c2c 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -228,7 +228,7 @@ impl FractalMgr {
 
 // services
 impl FractalMgr {
-    const GENERAL_EXECUTOR_WINDOW: u64 = 5 * 60;
+    const GENERAL_EXECUTOR_WINDOW: u64 = 5;
     /// The high priority executor service runs in the background to take care of high priority tasks and take any
     /// appropriate action. It will exclusively own the high priority queue since it is the only broker that is
     /// allowed to perform HP tasks
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index 1044e2a1..cb3cdcaf 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -54,6 +54,7 @@ use {
 #[derive(Debug, PartialEq)]
 pub enum Response {
     Empty,
+    EncodedAB(Box<[u8]>, Box<[u8]>),
 }
 
 pub(super) async fn query_loop<S: Socket>(
@@ -119,6 +120,10 @@ pub(super) async fn query_loop<S: Socket>(
             Ok(Response::Empty) => {
                 con.write_all(&[0x12]).await?;
             }
+            Ok(Response::EncodedAB(a, b)) => {
+                con.write_all(&a).await?;
+                con.write_all(&b).await?;
+            }
             Err(e) => {
                 let [a, b] = (e.value_u8() as u16).to_le_bytes();
                 con.write_all(&[0x10, a, b]).await?;

From 09bd2179982f14dbcc7362f06b3e68bd32e36e6e Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 4 Nov 2023 18:53:50 +0000
Subject: [PATCH 275/310] Use buffered IO and fix r/w through cache [skip ci]

Also ensure that any pending tasks are completed before exit
---
 server/src/engine/fractal/drivers.rs          |   6 +-
 server/src/engine/fractal/mgr.rs              | 156 +++++++++-----
 server/src/engine/fractal/mod.rs              |  26 ++-
 server/src/engine/fractal/test_utils.rs       |   5 +-
 server/src/engine/idx/mtchm/mod.rs            |   6 +
 server/src/engine/mod.rs                      |   3 +-
 .../src/engine/storage/v1/batch_jrnl/mod.rs   |   2 +-
 .../engine/storage/v1/batch_jrnl/persist.rs   |  53 +++--
 .../engine/storage/v1/batch_jrnl/restore.rs   |  12 +-
 server/src/engine/storage/v1/journal.rs       |   7 +-
 server/src/engine/storage/v1/loader.rs        |  44 ++--
 server/src/engine/storage/v1/memfs.rs         |  47 +++--
 server/src/engine/storage/v1/rw.rs            | 196 ++++++++++++------
 server/src/engine/storage/v1/tests/batch.rs   |   4 +-
 server/src/engine/storage/v1/tests/tx.rs      |  10 +-
 server/src/engine/txn/gns/mod.rs              |   3 +
 server/src/main.rs                            |   1 +
 17 files changed, 362 insertions(+), 219 deletions(-)

diff --git a/server/src/engine/fractal/drivers.rs b/server/src/engine/fractal/drivers.rs
index 2a41e2e8..6a633660 100644
--- a/server/src/engine/fractal/drivers.rs
+++ b/server/src/engine/fractal/drivers.rs
@@ -27,6 +27,7 @@
 use {
     super::util,
     crate::engine::{
+        error::RuntimeResult,
         storage::v1::{data_batch::DataBatchPersistDriver, RawFSInterface},
         txn::gns::GNSTransactionDriverAnyFS,
     },
@@ -38,7 +39,7 @@ use {
 pub(super) struct FractalGNSDriver<Fs: RawFSInterface> {
     #[allow(unused)]
     status: util::Status,
-    txn_driver: Mutex<GNSTransactionDriverAnyFS<Fs>>,
+    pub(super) txn_driver: Mutex<GNSTransactionDriverAnyFS<Fs>>,
 }
 
 impl<Fs: RawFSInterface> FractalGNSDriver<Fs> {
@@ -72,6 +73,9 @@ impl<Fs: RawFSInterface> FractalModelDriver<Fs> {
     pub fn batch_driver(&self) -> &Mutex<DataBatchPersistDriver<Fs>> {
         &self.batch_driver
     }
+    pub fn close(self) -> RuntimeResult<()> {
+        self.batch_driver.into_inner().close()
+    }
 }
 
 /// Model hooks
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 99375c2c..0307c549 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -24,14 +24,13 @@
  *
 */
 
-use crate::engine::storage::v1::LocalFS;
-
 use {
     super::ModelUniqueID,
     crate::{
         engine::{
             core::model::{delta::DataDelta, Model},
             data::uuid::Uuid,
+            storage::v1::LocalFS,
         },
         util::os,
     },
@@ -228,7 +227,7 @@ impl FractalMgr {
 
 // services
 impl FractalMgr {
-    const GENERAL_EXECUTOR_WINDOW: u64 = 5;
+    const GENERAL_EXECUTOR_WINDOW: u64 = 5 * 60;
     /// The high priority executor service runs in the background to take care of high priority tasks and take any
     /// appropriate action. It will exclusively own the high priority queue since it is the only broker that is
     /// allowed to perform HP tasks
@@ -239,57 +238,77 @@ impl FractalMgr {
         mut sigterm: broadcast::Receiver<()>,
     ) {
         loop {
-            let Task { threshold, task } = tokio::select! {
+            let task = tokio::select! {
                 task = receiver.recv() => {
                     match task {
                         Some(t) => t,
                         None => {
-                            info!("exiting fhp executor service because all tasks closed");
+                            info!("fhp: exiting executor service because all tasks closed");
                             break;
                         }
                     }
                 }
                 _ = sigterm.recv() => {
-                    info!("exited fhp executor service");
+                    info!("fhp: finishing pending tasks");
+                    while let Ok(task) = receiver.try_recv() {
+                        let global = global.clone();
+                        tokio::task::spawn_blocking(move || self.hp_executor(global, task)).await.unwrap()
+                    }
+                    info!("fhp: exited executor service");
                     break;
                 }
             };
-            // TODO(@ohsayan): check threshold and update hooks
-            match task {
-                CriticalTask::WriteBatch(model_id, observed_size) => {
-                    let mdl_drivers = global.get_state().get_mdl_drivers().read();
-                    let Some(mdl_driver) = mdl_drivers.get(&model_id) else {
-                        // because we maximize throughput, the model driver may have been already removed but this task
-                        // was way behind in the queue
-                        continue;
-                    };
-                    let res = global._namespace().with_model(
-                        (model_id.space(), model_id.model()),
-                        |model| {
+            let global = global.clone();
+            tokio::task::spawn_blocking(move || self.hp_executor(global, task))
+                .await
+                .unwrap()
+        }
+    }
+    fn hp_executor(
+        &'static self,
+        global: super::Global,
+        Task { threshold, task }: Task<CriticalTask>,
+    ) {
+        // TODO(@ohsayan): check threshold and update hooks
+        match task {
+            CriticalTask::WriteBatch(model_id, observed_size) => {
+                info!("fhp: {model_id} has reached cache capacity. writing to disk");
+                let mdl_drivers = global.get_state().get_mdl_drivers().read();
+                let Some(mdl_driver) = mdl_drivers.get(&model_id) else {
+                    // because we maximize throughput, the model driver may have been already removed but this task
+                    // was way behind in the queue
+                    return;
+                };
+                let res =
+                    global
+                        ._namespace()
+                        .with_model((model_id.space(), model_id.model()), |model| {
                             if model.get_uuid() != model_id.uuid() {
                                 // once again, throughput maximization will lead to, in extremely rare cases, this
                                 // branch returning. but it is okay
                                 return Ok(());
                             }
                             Self::try_write_model_data_batch(model, observed_size, mdl_driver)
-                        },
-                    );
-                    match res {
-                        Ok(()) => {}
-                        Err(_) => {
-                            log::error!(
-                                "Error writing data batch for model {}. Retrying...",
-                                model_id.uuid()
-                            );
-                            // enqueue again for retrying
-                            self.hp_dispatcher
-                                .send(Task::with_threshold(
-                                    CriticalTask::WriteBatch(model_id, observed_size),
-                                    threshold - 1,
-                                ))
-                                .unwrap();
+                        });
+                match res {
+                    Ok(()) => {
+                        if observed_size != 0 {
+                            info!("fhp: completed maintenance task for {model_id}, synced={observed_size}")
                         }
                     }
+                    Err(_) => {
+                        log::error!(
+                            "fhp: error writing data batch for model {}. Retrying...",
+                            model_id.uuid()
+                        );
+                        // enqueue again for retrying
+                        self.hp_dispatcher
+                            .send(Task::with_threshold(
+                                CriticalTask::WriteBatch(model_id, observed_size),
+                                threshold - 1,
+                            ))
+                            .unwrap();
+                    }
                 }
             }
         }
@@ -307,37 +326,21 @@ impl FractalMgr {
         loop {
             tokio::select! {
                 _ = sigterm.recv() => {
-                    info!("exited flp executor service");
+                    info!("flp: finishing any pending maintenance tasks");
+                    let global = global.clone();
+                    tokio::task::spawn_blocking(|| self.general_executor_model_maintenance(global)).await.unwrap();
+                    info!("flp: exited executor service");
                     break;
                 },
                 _ = tokio::time::sleep(std::time::Duration::from_secs(Self::GENERAL_EXECUTOR_WINDOW)) => {
-                    let mdl_drivers = global.get_state().get_mdl_drivers().read();
-                    for (model_id, driver) in mdl_drivers.iter() {
-                        let mut observed_len = 0;
-                        let res = global._namespace().with_model((model_id.space(), model_id.model()), |model| {
-                            if model.get_uuid() != model_id.uuid() {
-                                // once again, throughput maximization will lead to, in extremely rare cases, this
-                                // branch returning. but it is okay
-                                return Ok(());
-                            }
-                            // mark that we're taking these deltas
-                            observed_len = model.delta_state().__fractal_take_full_from_data_delta(super::FractalToken::new());
-                            Self::try_write_model_data_batch(model, observed_len, driver)
-                        });
-                        match res {
-                            Ok(()) => {}
-                            Err(_) => {
-                                // this failure is *not* good, so we want to promote this to a critical task
-                                self.hp_dispatcher.send(Task::new(CriticalTask::WriteBatch(model_id.clone(), observed_len))).unwrap()
-                            }
-                        }
-                    }
+                    let global = global.clone();
+                    tokio::task::spawn_blocking(|| self.general_executor_model_maintenance(global)).await.unwrap()
                 }
                 task = lpq.recv() => {
                     let Task { threshold, task } = match task {
                         Some(t) => t,
                         None => {
-                            info!("exiting flp executor service because all tasks closed");
+                            info!("flp: exiting executor service because all tasks closed");
                             break;
                         }
                     };
@@ -362,6 +365,45 @@ impl FractalMgr {
             }
         }
     }
+    fn general_executor_model_maintenance(&'static self, global: super::Global) {
+        let mdl_drivers = global.get_state().get_mdl_drivers().read();
+        for (model_id, driver) in mdl_drivers.iter() {
+            let mut observed_len = 0;
+            let res =
+                global
+                    ._namespace()
+                    .with_model((model_id.space(), model_id.model()), |model| {
+                        if model.get_uuid() != model_id.uuid() {
+                            // once again, throughput maximization will lead to, in extremely rare cases, this
+                            // branch returning. but it is okay
+                            return Ok(());
+                        }
+                        // mark that we're taking these deltas
+                        observed_len = model
+                            .delta_state()
+                            .__fractal_take_full_from_data_delta(super::FractalToken::new());
+                        Self::try_write_model_data_batch(model, observed_len, driver)
+                    });
+            match res {
+                Ok(()) => {
+                    if observed_len != 0 {
+                        info!(
+                            "flp: completed maintenance task for {model_id}, synced={observed_len}"
+                        )
+                    }
+                }
+                Err(_) => {
+                    // this failure is *not* good, so we want to promote this to a critical task
+                    self.hp_dispatcher
+                        .send(Task::new(CriticalTask::WriteBatch(
+                            model_id.clone(),
+                            observed_len,
+                        )))
+                        .unwrap()
+                }
+            }
+        }
+    }
 }
 
 // util
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index c23854a1..6cc37f0f 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -36,7 +36,7 @@ use {
     },
     crate::engine::error::RuntimeResult,
     parking_lot::{Mutex, RwLock},
-    std::{collections::HashMap, mem::MaybeUninit},
+    std::{collections::HashMap, fmt, mem::MaybeUninit},
     tokio::sync::mpsc::unbounded_channel,
 };
 
@@ -240,15 +240,17 @@ impl Global {
     }
     pub unsafe fn unload_all(self) {
         // TODO(@ohsayan): handle errors
-        self.namespace_txn_driver()
-            .lock()
-            .__journal_mut()
-            .__append_journal_close_and_close()
-            .unwrap();
-        for (_, driver) in self.get_state().mdl_driver.write().iter_mut() {
-            driver.batch_driver().lock().close().unwrap();
+        let GlobalState {
+            gns_driver,
+            mdl_driver,
+            ..
+        } = Self::__gref_raw().assume_init_read();
+        let gns_driver = gns_driver.txn_driver.into_inner().into_inner();
+        let mdl_drivers = mdl_driver.into_inner();
+        gns_driver.close().unwrap();
+        for (_, driver) in mdl_drivers {
+            driver.close().unwrap();
         }
-        core::ptr::drop_in_place(Self::__gref_raw().as_mut_ptr());
     }
 }
 
@@ -302,6 +304,12 @@ pub struct ModelUniqueID {
     uuid: Uuid,
 }
 
+impl fmt::Display for ModelUniqueID {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "model-{}@{}", self.model(), self.space())
+    }
+}
+
 impl ModelUniqueID {
     /// Create a new unique model ID
     pub fn new(space: &str, model: &str, uuid: Uuid) -> Self {
diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
index 152e057a..d3211ee9 100644
--- a/server/src/engine/fractal/test_utils.rs
+++ b/server/src/engine/fractal/test_utils.rs
@@ -146,9 +146,6 @@ impl<Fs: RawFSInterface> GlobalInstanceLike for TestGlobal<Fs> {
 impl<Fs: RawFSInterface> Drop for TestGlobal<Fs> {
     fn drop(&mut self) {
         let mut txn_driver = self.txn_driver.lock();
-        txn_driver
-            .__journal_mut()
-            .__append_journal_close_and_close()
-            .unwrap();
+        txn_driver.__journal_mut().__close_mut().unwrap();
     }
 }
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index e1e8bc6b..da78271b 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -108,6 +108,12 @@ impl CHTRuntimeLog {
     }
 }
 
+impl Drop for CHTRuntimeLog {
+    fn drop(&mut self) {
+        let _ = self.data;
+    }
+}
+
 pub struct Node<C: Config> {
     branch: [Atomic<Self>; <DefConfig as Config>::BRANCH_MX],
 }
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 30b0130a..8e367b2e 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -197,11 +197,10 @@ pub async fn start(
     tokio::select! {
         _ = endpoint_handles.listen() => {}
         _ = termsig => {
-            info!("received terminate signal");
+            info!("received terminate signal. waiting for inflight tasks to complete ...");
         }
     }
     drop(signal);
-    info!("waiting for inflight tasks to complete ...");
     endpoint_handles.finish().await;
     info!("waiting for fractal engine to exit ...");
     let (hp_handle, lp_handle) = tokio::join!(fractal_handle.hp_handle, fractal_handle.lp_handle);
diff --git a/server/src/engine/storage/v1/batch_jrnl/mod.rs b/server/src/engine/storage/v1/batch_jrnl/mod.rs
index 36f147b5..2013acbf 100644
--- a/server/src/engine/storage/v1/batch_jrnl/mod.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/mod.rs
@@ -56,7 +56,7 @@ pub fn reinit<Fs: RawFSInterface>(
     // restore
     let mut restore_driver = DataBatchRestoreDriver::new(f)?;
     restore_driver.read_data_batch_into_model(model)?;
-    DataBatchPersistDriver::new(restore_driver.into_file(), false)
+    DataBatchPersistDriver::new(restore_driver.into_file()?, false)
 }
 
 /// Create a new batch journal
diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs
index c29e3486..ee355690 100644
--- a/server/src/engine/storage/v1/batch_jrnl/persist.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs
@@ -63,16 +63,12 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
             file.fsynced_write(&[MARKER_BATCH_REOPEN])?;
         }
         Ok(Self {
-            f: SDSSFileTrackedWriter::new(file),
+            f: SDSSFileTrackedWriter::new(file)?,
         })
     }
-    pub fn close(&mut self) -> RuntimeResult<()> {
-        if self
-            .f
-            .inner_file()
-            .fsynced_write(&[MARKER_BATCH_CLOSED])
-            .is_ok()
-        {
+    pub fn close(self) -> RuntimeResult<()> {
+        let mut slf = self.f.into_inner_file()?;
+        if slf.fsynced_write(&[MARKER_BATCH_CLOSED]).is_ok() {
             return Ok(());
         } else {
             return Err(StorageError::DataBatchCloseError.into());
@@ -123,11 +119,9 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
                         self.encode_row_data(model, &irm, &row_data)?;
                     }
                 }
-                // fsync now; we're good to go
-                self.f.fsync_all()?;
                 i += 1;
             }
-            return self.append_batch_summary(observed_len, inconsistent_reads);
+            return self.append_batch_summary_and_sync(observed_len, inconsistent_reads);
         };
         match exec() {
             Ok(()) => Ok(()),
@@ -155,26 +149,28 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         col_cnt: usize,
     ) -> RuntimeResult<()> {
         self.f
-            .unfsynced_write(&[MARKER_ACTUAL_BATCH_EVENT, pk_tag.value_u8()])?;
+            .write_unfsynced(&[MARKER_ACTUAL_BATCH_EVENT, pk_tag.value_u8()])?;
         let observed_len_bytes = observed_len.u64_bytes_le();
-        self.f.unfsynced_write(&observed_len_bytes)?;
+        self.f.write_unfsynced(&observed_len_bytes)?;
         self.f
-            .unfsynced_write(&schema_version.value_u64().to_le_bytes())?;
-        self.f.unfsynced_write(&col_cnt.u64_bytes_le())?;
+            .write_unfsynced(&schema_version.value_u64().to_le_bytes())?;
+        self.f.write_unfsynced(&col_cnt.u64_bytes_le())?;
         Ok(())
     }
-    /// Append a summary of this batch
-    fn append_batch_summary(
+    /// Append a summary of this batch and most importantly, **sync everything to disk**
+    fn append_batch_summary_and_sync(
         &mut self,
         observed_len: usize,
         inconsistent_reads: usize,
     ) -> RuntimeResult<()> {
         // [0xFD][actual_commit][checksum]
-        self.f.unfsynced_write(&[MARKER_END_OF_BATCH])?;
+        self.f.write_unfsynced(&[MARKER_END_OF_BATCH])?;
         let actual_commit = (observed_len - inconsistent_reads).u64_bytes_le();
-        self.f.unfsynced_write(&actual_commit)?;
+        self.f.write_unfsynced(&actual_commit)?;
         let cs = self.f.reset_and_finish_checksum().to_le_bytes();
-        self.f.inner_file().fsynced_write(&cs)?;
+        self.f.untracked_write(&cs)?;
+        // IMPORTANT: now that all data has been written, we need to actually ensure that the writes pass through the cache
+        self.f.sync_writes()?;
         Ok(())
     }
     /// Attempt to fix the batch journal
@@ -184,8 +180,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
             attempt to append 0xFF to the part of the file where a corruption likely occurred, marking
             it recoverable
         */
-        let f = self.f.inner_file();
-        if f.fsynced_write(&[MARKER_RECOVERY_EVENT]).is_ok() {
+        if self.f.untracked_write(&[MARKER_RECOVERY_EVENT]).is_ok() {
             return Ok(());
         }
         Err(StorageError::DataBatchRecoveryFailStageOne.into())
@@ -203,7 +198,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
                     pk.read_uint()
                 }
                 .to_le_bytes();
-                buf.unfsynced_write(&data)?;
+                buf.write_unfsynced(&data)?;
             }
             TagUnique::Str | TagUnique::Bin => {
                 let slice = unsafe {
@@ -211,8 +206,8 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
                     pk.read_bin()
                 };
                 let slice_l = slice.len().u64_bytes_le();
-                buf.unfsynced_write(&slice_l)?;
-                buf.unfsynced_write(slice)?;
+                buf.write_unfsynced(&slice_l)?;
+                buf.write_unfsynced(slice)?;
             }
             TagUnique::Illegal => unsafe {
                 // UNSAFE(@ohsayan): a pk can't be constructed with illegal
@@ -225,7 +220,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
     fn encode_cell(&mut self, value: &Datacell) -> RuntimeResult<()> {
         let mut buf = vec![];
         cell::encode(&mut buf, value);
-        self.f.unfsynced_write(&buf)?;
+        self.f.write_unfsynced(&buf)?;
         Ok(())
     }
     /// Encode row data
@@ -241,7 +236,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
                     self.encode_cell(cell)?;
                 }
                 None if field_name.as_ref() == mdl.p_key() => {}
-                None => self.f.unfsynced_write(&[0])?,
+                None => self.f.write_unfsynced(&[0])?,
             }
         }
         Ok(())
@@ -249,9 +244,9 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
     /// Write the change type and txnid
     fn write_batch_item_common_row_data(&mut self, delta: &DataDelta) -> RuntimeResult<()> {
         let change_type = [delta.change().value_u8()];
-        self.f.unfsynced_write(&change_type)?;
+        self.f.write_unfsynced(&change_type)?;
         let txn_id = delta.data_version().value_u64().to_le_bytes();
-        self.f.unfsynced_write(&txn_id)?;
+        self.f.write_unfsynced(&txn_id)?;
         Ok(())
     }
 }
diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs
index 1454735b..963be39f 100644
--- a/server/src/engine/storage/v1/batch_jrnl/restore.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs
@@ -114,7 +114,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
             f: SDSSFileTrackedReader::new(f)?,
         })
     }
-    pub fn into_file(self) -> SDSSFileIO<F> {
+    pub fn into_file(self) -> RuntimeResult<SDSSFileIO<F>> {
         self.f.into_inner_file()
     }
     pub(in crate::engine::storage::v1) fn read_data_batch_into_model(
@@ -312,11 +312,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
         let actual_checksum = self.f.__reset_checksum();
         // find hardcoded checksum
         let mut hardcoded_checksum = [0; sizeof!(u64)];
-        self.f
-            .inner_file()
-            .read_to_buffer(&mut hardcoded_checksum)?;
-        // move file cursor ahead
-        self.f.__cursor_ahead_by(sizeof!(u64));
+        self.f.untracked_read(&mut hardcoded_checksum)?;
         if actual_checksum == u64::from_le_bytes(hardcoded_checksum) {
             Ok(actual_commit)
         } else {
@@ -414,7 +410,9 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
         )))
     }
     fn attempt_recover_data_batch(&mut self) -> RuntimeResult<()> {
-        if let Ok(MARKER_RECOVERY_EVENT) = self.f.inner_file().read_byte() {
+        let mut buf = [0u8; 1];
+        self.f.untracked_read(&mut buf)?;
+        if let [MARKER_RECOVERY_EVENT] = buf {
             return Ok(());
         }
         Err(StorageError::DataBatchRestoreCorruptedBatch.into())
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index 9e200bd6..0f28ecef 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -423,7 +423,7 @@ impl<Fs: RawFSInterface, TA> JournalWriter<Fs, TA> {
             &JournalEntryMetadata::new(id, EventSourceMarker::DRIVER_REOPENED, 0, 0).encoded(),
         )
     }
-    pub fn __append_journal_close_and_close(&mut self) -> RuntimeResult<()> {
+    pub fn __close_mut(&mut self) -> RuntimeResult<()> {
         self.closed = true;
         let id = self._incr_id() as u128;
         self.log_file.fsynced_write(
@@ -431,9 +431,8 @@ impl<Fs: RawFSInterface, TA> JournalWriter<Fs, TA> {
         )?;
         Ok(())
     }
-    #[cfg(test)]
-    pub fn append_journal_close_and_close(mut self) -> RuntimeResult<()> {
-        self.__append_journal_close_and_close()
+    pub fn close(mut self) -> RuntimeResult<()> {
+        self.__close_mut()
     }
 }
 
diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs
index feb5ecba..b44b2000 100644
--- a/server/src/engine/storage/v1/loader.rs
+++ b/server/src/engine/storage/v1/loader.rs
@@ -70,26 +70,36 @@ impl SEInitState {
                 &gns,
             )
         }?;
-        if is_new {
-            std::fs::create_dir(DATA_DIR).inherit_set_dmsg("creating data directory")?;
-        }
         let mut model_drivers = ModelDrivers::new();
-        if !is_new {
-            // this is an existing instance, so read in all data
-            for (space_name, space) in gns.spaces().read().iter() {
-                let space_uuid = space.get_uuid();
-                for (model_name, model) in space.models().read().iter() {
-                    let path =
-                        Self::model_path(space_name, space_uuid, model_name, model.get_uuid());
-                    let persist_driver = batch_jrnl::reinit(&path, model).inherit_set_dmsg(
-                        format!("failed to restore model data from journal in `{path}`"),
-                    )?;
-                    let _ = model_drivers.insert(
-                        ModelUniqueID::new(space_name, model_name, model.get_uuid()),
-                        FractalModelDriver::init(persist_driver),
-                    );
+        let mut driver_guard = || {
+            if is_new {
+                std::fs::create_dir(DATA_DIR).inherit_set_dmsg("creating data directory")?;
+            }
+            if !is_new {
+                // this is an existing instance, so read in all data
+                for (space_name, space) in gns.spaces().read().iter() {
+                    let space_uuid = space.get_uuid();
+                    for (model_name, model) in space.models().read().iter() {
+                        let path =
+                            Self::model_path(space_name, space_uuid, model_name, model.get_uuid());
+                        let persist_driver = batch_jrnl::reinit(&path, model).inherit_set_dmsg(
+                            format!("failed to restore model data from journal in `{path}`"),
+                        )?;
+                        let _ = model_drivers.insert(
+                            ModelUniqueID::new(space_name, model_name, model.get_uuid()),
+                            FractalModelDriver::init(persist_driver),
+                        );
+                    }
                 }
             }
+            RuntimeResult::Ok(())
+        };
+        if let Err(e) = driver_guard() {
+            gns_txn_driver.close().unwrap();
+            for (_, driver) in model_drivers {
+                driver.close().unwrap();
+            }
+            return Err(e);
         }
         Ok(SEInitState::new(
             GNSTransactionDriverAnyFS::new(gns_txn_driver),
diff --git a/server/src/engine/storage/v1/memfs.rs b/server/src/engine/storage/v1/memfs.rs
index 697ef903..d7d5ade5 100644
--- a/server/src/engine/storage/v1/memfs.rs
+++ b/server/src/engine/storage/v1/memfs.rs
@@ -28,8 +28,9 @@ use {
     crate::engine::{
         error::RuntimeResult,
         storage::v1::rw::{
-            FileOpen, RawFSInterface, RawFileInterface, RawFileInterfaceExt, RawFileInterfaceRead,
-            RawFileInterfaceWrite, RawFileInterfaceWriteExt,
+            FileOpen, RawFSInterface, RawFileInterface, RawFileInterfaceBufferedWriter,
+            RawFileInterfaceExt, RawFileInterfaceRead, RawFileInterfaceWrite,
+            RawFileInterfaceWriteExt,
         },
         sync::cell::Lazy,
     },
@@ -119,7 +120,7 @@ impl RawFSInterface for VirtualFS {
                 c.fw_write_all(&data)?;
             }
             FileOpen::Existing(mut e) => {
-                e.fw_truncate_to(0)?;
+                e.fwext_truncate_to(0)?;
                 e.fw_write_all(&data)?;
             }
         }
@@ -386,16 +387,24 @@ fn with_file<T>(fpath: &str, mut f: impl FnMut(&VFile) -> RuntimeResult<T>) -> R
 }
 
 impl RawFileInterface for VFileDescriptor {
-    type Reader = Self;
-    type Writer = Self;
-    fn into_buffered_reader(self) -> RuntimeResult<Self::Reader> {
+    type BufReader = Self;
+    type BufWriter = Self;
+    fn into_buffered_reader(self) -> RuntimeResult<Self::BufReader> {
         Ok(self)
     }
-    fn into_buffered_writer(self) -> RuntimeResult<Self::Writer> {
+    fn downgrade_reader(r: Self::BufReader) -> RuntimeResult<Self> {
+        Ok(r)
+    }
+    fn into_buffered_writer(self) -> RuntimeResult<Self::BufWriter> {
         Ok(self)
     }
+    fn downgrade_writer(w: Self::BufWriter) -> RuntimeResult<Self> {
+        Ok(w)
+    }
 }
 
+impl RawFileInterfaceBufferedWriter for VFileDescriptor {}
+
 impl RawFileInterfaceRead for VFileDescriptor {
     fn fr_read_exact(&mut self, buf: &mut [u8]) -> RuntimeResult<()> {
         with_file_mut(&self.0, |file| {
@@ -434,10 +443,10 @@ impl RawFileInterfaceWrite for VFileDescriptor {
 }
 
 impl RawFileInterfaceWriteExt for VFileDescriptor {
-    fn fw_fsync_all(&mut self) -> RuntimeResult<()> {
+    fn fwext_fsync_all(&mut self) -> RuntimeResult<()> {
         with_file(&self.0, |_| Ok(()))
     }
-    fn fw_truncate_to(&mut self, to: u64) -> RuntimeResult<()> {
+    fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()> {
         with_file_mut(&self.0, |file| {
             if !file.write {
                 return Err(
@@ -532,10 +541,10 @@ impl RawFileInterfaceWrite for NullFile {
     }
 }
 impl RawFileInterfaceWriteExt for NullFile {
-    fn fw_fsync_all(&mut self) -> RuntimeResult<()> {
+    fn fwext_fsync_all(&mut self) -> RuntimeResult<()> {
         Ok(())
     }
-    fn fw_truncate_to(&mut self, _: u64) -> RuntimeResult<()> {
+    fn fwext_truncate_to(&mut self, _: u64) -> RuntimeResult<()> {
         Ok(())
     }
 }
@@ -553,12 +562,20 @@ impl RawFileInterfaceExt for NullFile {
     }
 }
 impl RawFileInterface for NullFile {
-    type Reader = Self;
-    type Writer = Self;
-    fn into_buffered_reader(self) -> RuntimeResult<Self::Reader> {
+    type BufReader = Self;
+    type BufWriter = Self;
+    fn into_buffered_reader(self) -> RuntimeResult<Self::BufReader> {
         Ok(self)
     }
-    fn into_buffered_writer(self) -> RuntimeResult<Self::Writer> {
+    fn downgrade_reader(r: Self::BufReader) -> RuntimeResult<Self> {
+        Ok(r)
+    }
+    fn into_buffered_writer(self) -> RuntimeResult<Self::BufWriter> {
         Ok(self)
     }
+    fn downgrade_writer(w: Self::BufWriter) -> RuntimeResult<Self> {
+        Ok(w)
+    }
 }
+
+impl RawFileInterfaceBufferedWriter for NullFile {}
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 72028842..5f29b828 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -101,17 +101,28 @@ pub trait RawFSInterface {
 }
 
 /// A file (well, probably) that can be used for RW operations along with advanced write and extended operations (such as seeking)
-pub trait RawFileInterface
+pub trait RawFileInterface: Sized
 where
     Self: RawFileInterfaceRead
         + RawFileInterfaceWrite
         + RawFileInterfaceWriteExt
         + RawFileInterfaceExt,
 {
-    type Reader: RawFileInterfaceRead + RawFileInterfaceExt;
-    type Writer: RawFileInterfaceWrite + RawFileInterfaceExt;
-    fn into_buffered_reader(self) -> RuntimeResult<Self::Reader>;
-    fn into_buffered_writer(self) -> RuntimeResult<Self::Writer>;
+    type BufReader: RawFileInterfaceBufferedReader;
+    type BufWriter: RawFileInterfaceBufferedWriter;
+    fn into_buffered_reader(self) -> RuntimeResult<Self::BufReader>;
+    fn downgrade_reader(r: Self::BufReader) -> RuntimeResult<Self>;
+    fn into_buffered_writer(self) -> RuntimeResult<Self::BufWriter>;
+    fn downgrade_writer(w: Self::BufWriter) -> RuntimeResult<Self>;
+}
+
+pub trait RawFileInterfaceBufferedReader: RawFileInterfaceRead + RawFileInterfaceExt {}
+impl<R: RawFileInterfaceRead + RawFileInterfaceExt> RawFileInterfaceBufferedReader for R {}
+
+pub trait RawFileInterfaceBufferedWriter: RawFileInterfaceWrite + RawFileInterfaceExt {
+    fn sync_write_cache(&mut self) -> RuntimeResult<()> {
+        Ok(())
+    }
 }
 
 /// A file interface that supports read operations
@@ -138,8 +149,8 @@ impl<W: Write> RawFileInterfaceWrite for W {
 
 /// A file interface that supports advanced write operations
 pub trait RawFileInterfaceWriteExt {
-    fn fw_fsync_all(&mut self) -> RuntimeResult<()>;
-    fn fw_truncate_to(&mut self, to: u64) -> RuntimeResult<()>;
+    fn fwext_fsync_all(&mut self) -> RuntimeResult<()>;
+    fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()>;
 }
 
 /// A file interface that supports advanced file operations
@@ -206,21 +217,37 @@ impl RawFSInterface for LocalFS {
 }
 
 impl RawFileInterface for File {
-    type Reader = BufReader<Self>;
-    type Writer = BufWriter<Self>;
-    fn into_buffered_reader(self) -> RuntimeResult<Self::Reader> {
+    type BufReader = BufReader<Self>;
+    type BufWriter = BufWriter<Self>;
+    fn into_buffered_reader(self) -> RuntimeResult<Self::BufReader> {
         Ok(BufReader::new(self))
     }
-    fn into_buffered_writer(self) -> RuntimeResult<Self::Writer> {
+    fn downgrade_reader(r: Self::BufReader) -> RuntimeResult<Self> {
+        Ok(r.into_inner())
+    }
+    fn into_buffered_writer(self) -> RuntimeResult<Self::BufWriter> {
         Ok(BufWriter::new(self))
     }
+    fn downgrade_writer(mut w: Self::BufWriter) -> RuntimeResult<Self> {
+        w.flush()?; // TODO(@ohsayan): handle rare case where writer does panic
+        let (w, _) = w.into_parts();
+        Ok(w)
+    }
+}
+
+impl RawFileInterfaceBufferedWriter for BufWriter<File> {
+    fn sync_write_cache(&mut self) -> RuntimeResult<()> {
+        self.flush()?;
+        self.get_mut().sync_all()?;
+        Ok(())
+    }
 }
 
 impl RawFileInterfaceWriteExt for File {
-    fn fw_fsync_all(&mut self) -> RuntimeResult<()> {
+    fn fwext_fsync_all(&mut self) -> RuntimeResult<()> {
         cvt(self.sync_all())
     }
-    fn fw_truncate_to(&mut self, to: u64) -> RuntimeResult<()> {
+    fn fwext_truncate_to(&mut self, to: u64) -> RuntimeResult<()> {
         cvt(self.set_len(to))
     }
 }
@@ -270,40 +297,44 @@ impl<F: LocalFSFile> RawFileInterfaceExt for F {
 }
 
 pub struct SDSSFileTrackedWriter<Fs: RawFSInterface> {
-    f: SDSSFileIO<Fs>,
+    f: SDSSFileIO<Fs, <Fs::File as RawFileInterface>::BufWriter>,
     cs: SCrc,
 }
 
 impl<Fs: RawFSInterface> SDSSFileTrackedWriter<Fs> {
-    pub fn new(f: SDSSFileIO<Fs>) -> Self {
-        Self { f, cs: SCrc::new() }
+    pub fn new(f: SDSSFileIO<Fs>) -> RuntimeResult<Self> {
+        Ok(Self {
+            f: f.into_buffered_sdss_writer()?,
+            cs: SCrc::new(),
+        })
     }
-    pub fn unfsynced_write(&mut self, block: &[u8]) -> RuntimeResult<()> {
+    pub fn write_unfsynced(&mut self, block: &[u8]) -> RuntimeResult<()> {
+        self.untracked_write(block)
+            .map(|_| self.cs.recompute_with_new_var_block(block))
+    }
+    pub fn untracked_write(&mut self, block: &[u8]) -> RuntimeResult<()> {
         match self.f.unfsynced_write(block) {
-            Ok(()) => {
-                self.cs.recompute_with_new_var_block(block);
-                Ok(())
-            }
+            Ok(()) => Ok(()),
             e => e,
         }
     }
-    pub fn fsync_all(&mut self) -> RuntimeResult<()> {
-        self.f.fsync_all()
+    pub fn sync_writes(&mut self) -> RuntimeResult<()> {
+        self.f.f.sync_write_cache()
     }
     pub fn reset_and_finish_checksum(&mut self) -> u64 {
         let mut scrc = SCrc::new();
         core::mem::swap(&mut self.cs, &mut scrc);
         scrc.finish()
     }
-    pub fn inner_file(&mut self) -> &mut SDSSFileIO<Fs> {
-        &mut self.f
+    pub fn into_inner_file(self) -> RuntimeResult<SDSSFileIO<Fs>> {
+        self.f.downgrade_writer()
     }
 }
 
 /// [`SDSSFileLenTracked`] simply maintains application level length and checksum tracking to avoid frequent syscalls because we
 /// do not expect (even though it's very possible) users to randomly modify file lengths while we're reading them
 pub struct SDSSFileTrackedReader<Fs: RawFSInterface> {
-    f: SDSSFileIO<Fs>,
+    f: SDSSFileIO<Fs, <Fs::File as RawFileInterface>::BufReader>,
     len: u64,
     pos: u64,
     cs: SCrc,
@@ -314,6 +345,7 @@ impl<Fs: RawFSInterface> SDSSFileTrackedReader<Fs> {
     pub fn new(mut f: SDSSFileIO<Fs>) -> RuntimeResult<Self> {
         let len = f.file_length()?;
         let pos = f.retrieve_cursor()?;
+        let f = f.into_buffered_sdss_reader()?;
         Ok(Self {
             f,
             len,
@@ -331,11 +363,23 @@ impl<Fs: RawFSInterface> SDSSFileTrackedReader<Fs> {
         self.remaining() >= v
     }
     pub fn read_into_buffer(&mut self, buf: &mut [u8]) -> RuntimeResult<()> {
+        self.untracked_read(buf)
+            .map(|_| self.cs.recompute_with_new_var_block(buf))
+    }
+    pub fn read_byte(&mut self) -> RuntimeResult<u8> {
+        let mut buf = [0u8; 1];
+        self.read_into_buffer(&mut buf).map(|_| buf[0])
+    }
+    pub fn __reset_checksum(&mut self) -> u64 {
+        let mut crc = SCrc::new();
+        core::mem::swap(&mut crc, &mut self.cs);
+        crc.finish()
+    }
+    pub fn untracked_read(&mut self, buf: &mut [u8]) -> RuntimeResult<()> {
         if self.remaining() >= buf.len() as u64 {
             match self.f.read_to_buffer(buf) {
                 Ok(()) => {
                     self.pos += buf.len() as u64;
-                    self.cs.recompute_with_new_var_block(buf);
                     Ok(())
                 }
                 Err(e) => return Err(e),
@@ -344,23 +388,8 @@ impl<Fs: RawFSInterface> SDSSFileTrackedReader<Fs> {
             Err(SysIOError::from(std::io::ErrorKind::InvalidInput).into())
         }
     }
-    pub fn read_byte(&mut self) -> RuntimeResult<u8> {
-        let mut buf = [0u8; 1];
-        self.read_into_buffer(&mut buf).map(|_| buf[0])
-    }
-    pub fn __reset_checksum(&mut self) -> u64 {
-        let mut crc = SCrc::new();
-        core::mem::swap(&mut crc, &mut self.cs);
-        crc.finish()
-    }
-    pub fn inner_file(&mut self) -> &mut SDSSFileIO<Fs> {
-        &mut self.f
-    }
-    pub fn into_inner_file(self) -> SDSSFileIO<Fs> {
-        self.f
-    }
-    pub fn __cursor_ahead_by(&mut self, sizeof: usize) {
-        self.pos += sizeof as u64;
+    pub fn into_inner_file(self) -> RuntimeResult<SDSSFileIO<Fs>> {
+        self.f.downgrade_reader()
     }
     pub fn read_block<const N: usize>(&mut self) -> RuntimeResult<[u8; N]> {
         if !self.has_left(N as _) {
@@ -376,8 +405,8 @@ impl<Fs: RawFSInterface> SDSSFileTrackedReader<Fs> {
 }
 
 #[derive(Debug)]
-pub struct SDSSFileIO<Fs: RawFSInterface> {
-    f: Fs::File,
+pub struct SDSSFileIO<Fs: RawFSInterface, F = <Fs as RawFSInterface>::File> {
+    f: F,
     _fs: PhantomData<Fs>,
 }
 
@@ -408,42 +437,60 @@ impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
             }
         }
     }
+    pub fn into_buffered_sdss_reader(
+        self,
+    ) -> RuntimeResult<SDSSFileIO<Fs, <Fs::File as RawFileInterface>::BufReader>> {
+        self.f.into_buffered_reader().map(SDSSFileIO::_new)
+    }
+    pub fn into_buffered_sdss_writer(
+        self,
+    ) -> RuntimeResult<SDSSFileIO<Fs, <Fs::File as RawFileInterface>::BufWriter>> {
+        self.f.into_buffered_writer().map(SDSSFileIO::_new)
+    }
 }
 
-impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
-    pub fn _new(f: Fs::File) -> Self {
+impl<Fs: RawFSInterface> SDSSFileIO<Fs, <Fs::File as RawFileInterface>::BufReader> {
+    pub fn downgrade_reader(self) -> RuntimeResult<SDSSFileIO<Fs, Fs::File>> {
+        let me = <Fs::File as RawFileInterface>::downgrade_reader(self.f)?;
+        Ok(SDSSFileIO::_new(me))
+    }
+}
+
+impl<Fs: RawFSInterface> SDSSFileIO<Fs, <Fs::File as RawFileInterface>::BufWriter> {
+    pub fn downgrade_writer(self) -> RuntimeResult<SDSSFileIO<Fs>> {
+        let me = <Fs::File as RawFileInterface>::downgrade_writer(self.f)?;
+        Ok(SDSSFileIO::_new(me))
+    }
+}
+
+impl<Fs: RawFSInterface, F> SDSSFileIO<Fs, F> {
+    pub fn _new(f: F) -> Self {
         Self {
             f,
             _fs: PhantomData,
         }
     }
-    pub fn unfsynced_write(&mut self, data: &[u8]) -> RuntimeResult<()> {
-        self.f.fw_write_all(data)
-    }
-    pub fn fsync_all(&mut self) -> RuntimeResult<()> {
-        self.f.fw_fsync_all()?;
-        Ok(())
-    }
-    pub fn fsynced_write(&mut self, data: &[u8]) -> RuntimeResult<()> {
-        self.f.fw_write_all(data)?;
-        self.f.fw_fsync_all()
-    }
+}
+
+impl<Fs: RawFSInterface, F: RawFileInterfaceRead> SDSSFileIO<Fs, F> {
     pub fn read_to_buffer(&mut self, buffer: &mut [u8]) -> RuntimeResult<()> {
         self.f.fr_read_exact(buffer)
     }
+}
+
+impl<Fs: RawFSInterface, F: RawFileInterfaceExt> SDSSFileIO<Fs, F> {
+    pub fn retrieve_cursor(&mut self) -> RuntimeResult<u64> {
+        self.f.fext_cursor()
+    }
     pub fn file_length(&self) -> RuntimeResult<u64> {
         self.f.fext_file_length()
     }
     pub fn seek_from_start(&mut self, by: u64) -> RuntimeResult<()> {
         self.f.fext_seek_ahead_from_start_by(by)
     }
-    pub fn retrieve_cursor(&mut self) -> RuntimeResult<u64> {
-        self.f.fext_cursor()
-    }
-    pub fn read_byte(&mut self) -> RuntimeResult<u8> {
-        let mut r = [0; 1];
-        self.read_to_buffer(&mut r).map(|_| r[0])
-    }
+}
+
+impl<Fs: RawFSInterface, F: RawFileInterfaceRead + RawFileInterfaceExt> SDSSFileIO<Fs, F> {
     pub fn load_remaining_into_buffer(&mut self) -> RuntimeResult<Vec<u8>> {
         let len = self.file_length()? - self.retrieve_cursor()?;
         let mut buf = vec![0; len as usize];
@@ -451,3 +498,20 @@ impl<Fs: RawFSInterface> SDSSFileIO<Fs> {
         Ok(buf)
     }
 }
+
+impl<Fs: RawFSInterface, F: RawFileInterfaceWrite> SDSSFileIO<Fs, F> {
+    pub fn unfsynced_write(&mut self, data: &[u8]) -> RuntimeResult<()> {
+        self.f.fw_write_all(data)
+    }
+}
+
+impl<Fs: RawFSInterface, F: RawFileInterfaceWrite + RawFileInterfaceWriteExt> SDSSFileIO<Fs, F> {
+    pub fn fsync_all(&mut self) -> RuntimeResult<()> {
+        self.f.fwext_fsync_all()?;
+        Ok(())
+    }
+    pub fn fsynced_write(&mut self, data: &[u8]) -> RuntimeResult<()> {
+        self.f.fw_write_all(data)?;
+        self.f.fwext_fsync_all()
+    }
+}
diff --git a/server/src/engine/storage/v1/tests/batch.rs b/server/src/engine/storage/v1/tests/batch.rs
index 4dd654e8..a0e6ad1e 100644
--- a/server/src/engine/storage/v1/tests/batch.rs
+++ b/server/src/engine/storage/v1/tests/batch.rs
@@ -67,7 +67,7 @@ fn open_batch_data(fpath: &str, mdl: &Model) -> DataBatchPersistDriver<VirtualFS
         FileOpen::Existing((f, _header)) => {
             let mut dbr = DataBatchRestoreDriver::new(f).unwrap();
             dbr.read_data_batch_into_model(mdl).unwrap();
-            DataBatchPersistDriver::new(dbr.into_file(), false)
+            DataBatchPersistDriver::new(dbr.into_file().unwrap(), false)
         }
     }
     .unwrap()
@@ -144,7 +144,7 @@ fn empty_multi_open_reopen() {
         ),
     );
     for _ in 0..100 {
-        let mut writer = open_batch_data("empty_multi_open_reopen.db-btlog", &mdl);
+        let writer = open_batch_data("empty_multi_open_reopen.db-btlog", &mdl);
         writer.close().unwrap();
     }
 }
diff --git a/server/src/engine/storage/v1/tests/tx.rs b/server/src/engine/storage/v1/tests/tx.rs
index 17dae70d..ba814378 100644
--- a/server/src/engine/storage/v1/tests/tx.rs
+++ b/server/src/engine/storage/v1/tests/tx.rs
@@ -142,7 +142,7 @@ fn first_boot_second_readonly() {
         let mut log = open_log("testtxn.log", &db1)?;
         db1.txn_set(0, 20, &mut log)?;
         db1.txn_set(9, 21, &mut log)?;
-        log.append_journal_close_and_close()
+        log.close()
     };
     x().unwrap();
     // backup original data
@@ -151,7 +151,7 @@ fn first_boot_second_readonly() {
     let empty_db2 = Database::new();
     open_log("testtxn.log", &empty_db2)
         .unwrap()
-        .append_journal_close_and_close()
+        .close()
         .unwrap();
     assert_eq!(original_data, empty_db2.copy_data());
 }
@@ -164,7 +164,7 @@ fn oneboot_mod_twoboot_mod_thirdboot_read() {
         for i in 0..10 {
             db1.txn_set(i, 1, &mut log)?;
         }
-        log.append_journal_close_and_close()
+        log.close()
     };
     x().unwrap();
     let bkp_db1 = db1.copy_data();
@@ -178,7 +178,7 @@ fn oneboot_mod_twoboot_mod_thirdboot_read() {
             let current_val = db2.data.borrow()[i];
             db2.txn_set(i, current_val + i as u8, &mut log)?;
         }
-        log.append_journal_close_and_close()
+        log.close()
     };
     x().unwrap();
     let bkp_db2 = db2.copy_data();
@@ -186,7 +186,7 @@ fn oneboot_mod_twoboot_mod_thirdboot_read() {
     // third boot
     let db3 = Database::new();
     let log = open_log("duatxn.db-tlog", &db3).unwrap();
-    log.append_journal_close_and_close().unwrap();
+    log.close().unwrap();
     assert_eq!(bkp_db2, db3.copy_data());
     assert_eq!(
         db3.copy_data(),
diff --git a/server/src/engine/txn/gns/mod.rs b/server/src/engine/txn/gns/mod.rs
index 9befe0a1..09f33f5c 100644
--- a/server/src/engine/txn/gns/mod.rs
+++ b/server/src/engine/txn/gns/mod.rs
@@ -65,6 +65,9 @@ impl<Fs: RawFSInterface> GNSTransactionDriverAnyFS<Fs> {
     pub fn new(journal: JournalWriter<Fs, GNSAdapter>) -> Self {
         Self { journal }
     }
+    pub fn into_inner(self) -> JournalWriter<Fs, GNSAdapter> {
+        self.journal
+    }
     pub fn __journal_mut(&mut self) -> &mut JournalWriter<Fs, GNSAdapter> {
         &mut self.journal
     }
diff --git a/server/src/main.rs b/server/src/main.rs
index c6dc512b..bed3a748 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -83,6 +83,7 @@ fn main() {
     match runtime.block_on(async move { engine::start(config, global).await }) {
         Ok(()) => {
             engine::finish(g);
+            info!("finished all pending tasks. Goodbye!");
         }
         Err(e) => {
             error!("{e}");

From cac33bf7c2b9a250d5b3c0c76e0b6a0ae6802c1d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 8 Nov 2023 17:38:49 +0000
Subject: [PATCH 276/310] Simplify DML path [skip ci]

---
 server/src/engine/core/mod.rs                 | 124 ++++---
 server/src/engine/core/model/alt.rs           |  20 +-
 server/src/engine/core/model/mod.rs           |  70 ++--
 server/src/engine/core/space.rs               | 303 ++++++++----------
 server/src/engine/core/tests/ddl_model/alt.rs |  20 +-
 server/src/engine/core/tests/ddl_model/mod.rs |  19 +-
 .../src/engine/core/tests/ddl_space/alter.rs  |  38 ++-
 .../src/engine/core/tests/ddl_space/create.rs |  17 +-
 server/src/engine/core/tests/ddl_space/mod.rs |   4 +-
 server/src/engine/core/tests/dml/mod.rs       |  10 +-
 server/src/engine/core/util.rs                | 131 +++++++-
 server/src/engine/fractal/mgr.rs              |  55 ++--
 server/src/engine/ql/ast/mod.rs               |  24 +-
 server/src/engine/storage/v1/inf/obj.rs       |   7 +-
 server/src/engine/storage/v1/inf/tests.rs     |   7 +-
 server/src/engine/storage/v1/loader.rs        |  11 +-
 server/src/engine/txn/gns/model.rs            |  84 +++--
 server/src/engine/txn/gns/space.rs            |  17 +-
 server/src/engine/txn/gns/tests/full_chain.rs |  37 +--
 server/src/engine/txn/gns/tests/io.rs         |  14 +-
 20 files changed, 572 insertions(+), 440 deletions(-)

diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 6a759f71..a3cff237 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -24,6 +24,9 @@
  *
 */
 
+use self::dml::QueryExecMeta;
+pub use self::util::{EntityID, EntityIDRef};
+use super::{fractal::GlobalInstanceLike, ql::ast::Entity};
 pub(in crate::engine) mod dml;
 pub mod exec;
 pub(in crate::engine) mod index;
@@ -36,86 +39,105 @@ mod util;
 pub(super) mod tests;
 // imports
 use {
-    self::{model::Model, util::EntityLocator},
+    self::model::Model,
     crate::engine::{
         core::space::Space,
         error::{QueryError, QueryResult},
-        fractal::GlobalInstanceLike,
-        idx::{IndexST, STIndex},
+        idx::IndexST,
     },
     parking_lot::RwLock,
+    std::collections::HashMap,
 };
 
 /// Use this for now since it substitutes for a file lock (and those syscalls are expensive),
 /// but something better is in the offing
 type RWLIdx<K, V> = RwLock<IndexST<K, V>>;
 
-// FIXME(@ohsayan): Make sure we update what all structures we're making use of here
-
 #[cfg_attr(test, derive(Debug))]
 pub struct GlobalNS {
-    index_space: RWLIdx<Box<str>, Space>,
-}
-
-pub(self) fn with_model_for_data_update<'a, E, F>(
-    global: &impl GlobalInstanceLike,
-    entity: E,
-    f: F,
-) -> QueryResult<()>
-where
-    F: FnOnce(&Model) -> QueryResult<dml::QueryExecMeta>,
-    E: 'a + EntityLocator<'a>,
-{
-    let (space_name, model_name) = entity.parse_entity()?;
-    global
-        .namespace()
-        .with_model((space_name, model_name), |mdl| {
-            let r = f(mdl);
-            match r {
-                Ok(dhint) => {
-                    model::DeltaState::guard_delta_overflow(
-                        global, space_name, model_name, mdl, dhint,
-                    );
-                    Ok(())
-                }
-                Err(e) => Err(e),
-            }
-        })
+    idx_mdl: RWLIdx<EntityID, Model>,
+    idx: RWLIdx<Box<str>, RwLock<Space>>,
 }
 
 impl GlobalNS {
-    pub fn spaces(&self) -> &RWLIdx<Box<str>, Space> {
-        &self.index_space
-    }
     pub fn empty() -> Self {
         Self {
-            index_space: RWLIdx::default(),
+            idx_mdl: RWLIdx::default(),
+            idx: RWLIdx::default(),
         }
     }
-    #[cfg(test)]
-    pub(self) fn test_new_empty_space(&self, space_id: &str) -> bool {
-        self.index_space
-            .write()
-            .st_insert(space_id.into(), Space::empty())
+    pub fn ddl_with_spaces_write<T>(
+        &self,
+        f: impl FnOnce(&mut HashMap<Box<str>, RwLock<Space>>) -> T,
+    ) -> T {
+        let mut spaces = self.idx.write();
+        f(&mut spaces)
     }
-    pub fn with_space<T>(
+    pub fn ddl_with_space_mut<T>(
         &self,
         space: &str,
-        f: impl FnOnce(&Space) -> QueryResult<T>,
+        f: impl FnOnce(&mut Space) -> QueryResult<T>,
     ) -> QueryResult<T> {
-        let sread = self.index_space.read();
-        let Some(space) = sread.st_get(space) else {
+        let spaces = self.idx.read();
+        let Some(space) = spaces.get(space) else {
             return Err(QueryError::QExecObjectNotFound);
         };
-        f(space)
+        let mut space = space.write();
+        f(&mut space)
     }
-    pub fn with_model<'a, T, E, F>(&self, entity: E, f: F) -> QueryResult<T>
+    pub fn with_model_space<'a, T, F>(&self, entity: Entity<'a>, f: F) -> QueryResult<T>
+    where
+        F: FnOnce(&Space, &Model) -> QueryResult<T>,
+    {
+        let (space, model_name) = entity.into_full_result()?;
+        let mdl_idx = self.idx_mdl.read();
+        let Some(model) = mdl_idx.get(&EntityIDRef::new(&space, &model_name)) else {
+            return Err(QueryError::QExecObjectNotFound);
+        };
+        let space_read = self.idx.read();
+        let space = space_read.get(space.as_str()).unwrap().read();
+        f(&space, model)
+    }
+    pub fn with_model<'a, T, F>(&self, entity: Entity<'a>, f: F) -> QueryResult<T>
     where
         F: FnOnce(&Model) -> QueryResult<T>,
-        E: 'a + EntityLocator<'a>,
     {
-        entity
-            .parse_entity()
-            .and_then(|(space, model)| self.with_space(space, |space| space.with_model(model, f)))
+        let (space, model_name) = entity.into_full_result()?;
+        let mdl_idx = self.idx_mdl.read();
+        let Some(model) = mdl_idx.get(&EntityIDRef::new(&space, &model_name)) else {
+            return Err(QueryError::QExecObjectNotFound);
+        };
+        f(model)
+    }
+    pub fn idx_models(&self) -> &RWLIdx<EntityID, Model> {
+        &self.idx_mdl
+    }
+    pub fn idx(&self) -> &RWLIdx<Box<str>, RwLock<Space>> {
+        &self.idx
+    }
+    #[cfg(test)]
+    pub fn create_empty_test_space(&self, space_name: &str) {
+        let _ = self
+            .idx()
+            .write()
+            .insert(space_name.into(), Space::new_auto_all().into());
     }
 }
+
+pub(self) fn with_model_for_data_update<'a, F>(
+    global: &impl GlobalInstanceLike,
+    entity: Entity<'a>,
+    f: F,
+) -> QueryResult<()>
+where
+    F: FnOnce(&Model) -> QueryResult<QueryExecMeta>,
+{
+    let (space, model_name) = entity.into_full_result()?;
+    let mdl_idx = global.namespace().idx_mdl.read();
+    let Some(model) = mdl_idx.get(&EntityIDRef::new(&space, &model_name)) else {
+        return Err(QueryError::QExecObjectNotFound);
+    };
+    let r = f(model)?;
+    model::DeltaState::guard_delta_overflow(global, &space, &model_name, model, r);
+    Ok(())
+}
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index fe2fdd79..ef1386cd 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -28,7 +28,6 @@ use {
     super::{Field, IWModel, Layer, Model},
     crate::{
         engine::{
-            core::util::EntityLocator,
             data::{
                 tag::{DataTag, TagClass},
                 DictEntryGeneric,
@@ -253,9 +252,10 @@ impl Model {
         global: &G,
         alter: AlterModel,
     ) -> QueryResult<()> {
-        let (space_name, model_name) = EntityLocator::parse_entity(alter.model)?;
-        global.namespace().with_space(space_name, |space| {
-            space.with_model(model_name, |model| {
+        let (space_name, model_name) = alter.model.into_full_result()?;
+        global
+            .namespace()
+            .with_model_space(alter.model, |space, model| {
                 // make intent
                 let iwm = model.intent_write_model();
                 // prepare plan
@@ -275,7 +275,12 @@ impl Model {
                         if G::FS_IS_NON_NULL {
                             // prepare txn
                             let txn = gnstxn::AlterModelAddTxn::new(
-                                gnstxn::ModelIDRef::new_ref(space_name, space, model_name, model),
+                                gnstxn::ModelIDRef::new_ref(
+                                    &space_name,
+                                    &space,
+                                    &model_name,
+                                    model,
+                                ),
                                 &new_fields,
                             );
                             // commit txn
@@ -296,7 +301,7 @@ impl Model {
                         if G::FS_IS_NON_NULL {
                             // prepare txn
                             let txn = gnstxn::AlterModelRemoveTxn::new(
-                                gnstxn::ModelIDRef::new_ref(space_name, space, model_name, model),
+                                gnstxn::ModelIDRef::new_ref(&space_name, space, &model_name, model),
                                 &removed,
                             );
                             // commit txn
@@ -314,7 +319,7 @@ impl Model {
                         if G::FS_IS_NON_NULL {
                             // prepare txn
                             let txn = gnstxn::AlterModelUpdateTxn::new(
-                                gnstxn::ModelIDRef::new_ref(space_name, space, model_name, model),
+                                gnstxn::ModelIDRef::new_ref(&space_name, space, &model_name, model),
                                 &updated,
                             );
                             // commit txn
@@ -327,6 +332,5 @@ impl Model {
                 }
                 Ok(())
             })
-        })
     }
 }
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index d2966c14..e1b222e0 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -32,7 +32,7 @@ use std::cell::RefCell;
 
 use {
     self::delta::{IRModel, IRModelSMData, ISyncMatrix, IWModel},
-    super::{index::PrimaryIndex, util::EntityLocator},
+    super::index::PrimaryIndex,
     crate::engine::{
         data::{
             cell::Datacell,
@@ -48,12 +48,14 @@ use {
             drop::DropModel,
             syn::{FieldSpec, LayerSpec},
         },
-        txn::gns as gnstxn,
+        txn::gns::{self as gnstxn, SpaceIDRef},
     },
     std::cell::UnsafeCell,
 };
 
 pub(in crate::engine::core) use self::delta::{DeltaState, DeltaVersion, SchemaDeltaKind};
+
+use super::util::{EntityID, EntityIDRef};
 pub(in crate::engine) type Fields = IndexSTSeqCns<Box<str>, Field>;
 
 #[derive(Debug)]
@@ -204,40 +206,41 @@ impl Model {
         global: &G,
         stmt: CreateModel,
     ) -> QueryResult<()> {
-        let (space_name, model_name) = stmt.model_name.parse_entity()?;
+        let (space_name, model_name) = stmt.model_name.into_full_result()?;
         let model = Self::process_create(stmt)?;
-        global.namespace().with_space(space_name, |space| {
-            let mut w_space = space.models().write();
-            if w_space.st_contains(model_name) {
+        global.namespace().ddl_with_space_mut(&space_name, |space| {
+            // TODO(@ohsayan): be extra cautious with post-transactional tasks (memck)
+            if space.models().contains(model_name.as_str()) {
                 return Err(QueryError::QExecDdlObjectAlreadyExists);
             }
+            // since we've locked this down, no one else can parallely create another model in the same space (or remove)
             if G::FS_IS_NON_NULL {
                 let irm = model.intent_read_model();
                 let mut txn_driver = global.namespace_txn_driver().lock();
                 // prepare txn
                 let txn = gnstxn::CreateModelTxn::new(
-                    gnstxn::SpaceIDRef::new(space_name, space),
-                    model_name,
+                    SpaceIDRef::new(&space_name, &space),
+                    &model_name,
                     &model,
                     &irm,
                 );
                 // attempt to initialize driver
                 global.initialize_model_driver(
-                    space_name,
+                    &space_name,
                     space.get_uuid(),
-                    model_name,
+                    &model_name,
                     model.get_uuid(),
                 )?;
                 // commit txn
                 match txn_driver.try_commit(txn) {
                     Ok(()) => {}
                     Err(e) => {
-                        // failed to commit, delete this
+                        // failed to commit, request cleanup
                         global.taskmgr_post_standard_priority(Task::new(
                             GenericTask::delete_model_dir(
-                                space_name,
+                                &space_name,
                                 space.get_uuid(),
-                                model_name,
+                                &model_name,
                                 model.get_uuid(),
                             ),
                         ));
@@ -246,7 +249,12 @@ impl Model {
                 }
             }
             // update global state
-            let _ = w_space.st_insert(model_name.into(), model);
+            let _ = space.models_mut().insert(model_name.as_str().into());
+            let _ = global
+                .namespace()
+                .idx_models()
+                .write()
+                .insert(EntityID::new(&space_name, &model_name), model);
             Ok(())
         })
     }
@@ -254,32 +262,44 @@ impl Model {
         global: &G,
         stmt: DropModel,
     ) -> QueryResult<()> {
-        let (space_name, model_name) = stmt.entity.parse_entity()?;
-        global.namespace().with_space(space_name, |space| {
-            let mut w_space = space.models().write();
-            let Some(model) = w_space.get(model_name) else {
+        let (space_name, model_name) = stmt.entity.into_full_result()?;
+        global.namespace().ddl_with_space_mut(&space_name, |space| {
+            if !space.models().contains(model_name.as_str()) {
+                // the model isn't even present
                 return Err(QueryError::QExecObjectNotFound);
-            };
+            }
+            // get exclusive lock on models
+            let mut models_idx = global.namespace().idx_models().write();
+            let model = models_idx
+                .get(&EntityIDRef::new(&space_name, &model_name))
+                .unwrap();
+            // the model must be empty for us to clean it up! (NB: consistent view + EX)
+            if model.primary_index().count() != 0 {
+                // nope, we can't drop this
+                return Err(QueryError::QExecDdlNotEmpty);
+            }
+            // okay this is looking good for us
             if G::FS_IS_NON_NULL {
                 // prepare txn
                 let txn = gnstxn::DropModelTxn::new(gnstxn::ModelIDRef::new(
-                    gnstxn::SpaceIDRef::new(space_name, space),
-                    model_name,
+                    SpaceIDRef::new(&space_name, &space),
+                    &model_name,
                     model.get_uuid(),
                     model.delta_state().schema_current_version().value_u64(),
                 ));
                 // commit txn
                 global.namespace_txn_driver().lock().try_commit(txn)?;
-                // ask for cleanup
+                // request cleanup
                 global.taskmgr_post_standard_priority(Task::new(GenericTask::delete_model_dir(
-                    space_name,
+                    &space_name,
                     space.get_uuid(),
-                    model_name,
+                    &model_name,
                     model.get_uuid(),
                 )));
             }
             // update global state
-            let _ = w_space.st_delete(model_name);
+            let _ = models_idx.remove(&EntityIDRef::new(&space_name, &model_name));
+            let _ = space.models_mut().remove(model_name.as_str());
             Ok(())
         })
     }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 05685d0e..8e5adb0f 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -26,58 +26,26 @@
 
 use {
     crate::engine::{
-        core::{model::Model, RWLIdx},
         data::{dict, uuid::Uuid, DictEntryGeneric, DictGeneric},
         error::{QueryError, QueryResult},
         fractal::{GenericTask, GlobalInstanceLike, Task},
-        idx::{IndexST, STIndex},
+        idx::STIndex,
         ql::ddl::{alt::AlterSpace, crt::CreateSpace, drop::DropSpace},
         storage::v1::{loader::SEInitState, RawFSInterface},
         txn::gns as gnstxn,
     },
     parking_lot::RwLock,
+    std::collections::HashSet,
 };
 
-#[derive(Debug)]
-/// A space with the model namespace
+#[derive(Debug, PartialEq)]
 pub struct Space {
     uuid: Uuid,
-    mns: RWLIdx<Box<str>, Model>,
-    pub(super) meta: SpaceMeta,
+    models: HashSet<Box<str>>,
+    props: DictGeneric,
 }
 
-#[derive(Debug, Default)]
-/// Space metadata
-pub struct SpaceMeta {
-    pub(super) props: RwLock<DictGeneric>,
-}
-
-impl SpaceMeta {
-    pub const KEY_ENV: &'static str = "env";
-    pub fn new_with_meta(props: DictGeneric) -> Self {
-        Self {
-            props: RwLock::new(props),
-        }
-    }
-    pub fn with_env(env: DictGeneric) -> Self {
-        Self {
-            props: RwLock::new(into_dict!("env" => DictEntryGeneric::Map(env))),
-        }
-    }
-    pub fn dict(&self) -> &RwLock<DictGeneric> {
-        &self.props
-    }
-    #[cfg(test)]
-    pub fn get_env<'a>(rwl: &'a parking_lot::RwLockReadGuard<'a, DictGeneric>) -> &'a DictGeneric {
-        match rwl.get(Self::KEY_ENV).unwrap() {
-            DictEntryGeneric::Data(_) => unreachable!(),
-            DictEntryGeneric::Map(m) => m,
-        }
-    }
-}
-
-#[derive(Debug)]
-#[cfg_attr(test, derive(PartialEq))]
+#[derive(Debug, PartialEq)]
 /// Procedure for `create space`
 struct ProcedureCreate {
     space_name: Box<str>,
@@ -85,66 +53,52 @@ struct ProcedureCreate {
 }
 
 impl Space {
-    pub fn _create_model(&self, name: &str, model: Model) -> QueryResult<()> {
-        if self
-            .mns
-            .write()
-            .st_insert(name.to_string().into_boxed_str(), model)
-        {
-            Ok(())
-        } else {
-            Err(QueryError::QExecDdlObjectAlreadyExists)
+    pub fn new(uuid: Uuid, models: HashSet<Box<str>>, props: DictGeneric) -> Self {
+        Self {
+            uuid,
+            models,
+            props,
         }
     }
+    #[cfg(test)]
+    pub fn new_auto_all() -> Self {
+        Self::new_auto(Default::default(), Default::default())
+    }
     pub fn get_uuid(&self) -> Uuid {
         self.uuid
     }
-    pub fn models(&self) -> &RWLIdx<Box<str>, Model> {
-        &self.mns
+    pub fn new_restore_empty(uuid: Uuid, props: DictGeneric) -> Self {
+        Self::new(uuid, Default::default(), props)
     }
-    pub fn metadata(&self) -> &SpaceMeta {
-        &self.meta
+    pub fn new_empty_auto(props: DictGeneric) -> Self {
+        Self::new_auto(Default::default(), props)
     }
-    pub fn with_model<T>(
-        &self,
-        model: &str,
-        f: impl FnOnce(&Model) -> QueryResult<T>,
-    ) -> QueryResult<T> {
-        let mread = self.mns.read();
-        let Some(model) = mread.st_get(model) else {
-            return Err(QueryError::QExecObjectNotFound);
-        };
-        f(model)
+    pub fn new_auto(models: HashSet<Box<str>>, props: DictGeneric) -> Self {
+        Self::new(Uuid::new(), models, props)
     }
-    pub(crate) fn new_restore_empty(meta: SpaceMeta, uuid: Uuid) -> Space {
-        Self::new_with_uuid(Default::default(), meta, uuid)
+    pub fn models(&self) -> &HashSet<Box<str>> {
+        &self.models
     }
-}
-
-impl Space {
-    #[cfg(test)]
-    pub fn empty() -> Self {
-        Space::new_auto(Default::default(), SpaceMeta::with_env(into_dict! {}))
+    pub fn models_mut(&mut self) -> &mut HashSet<Box<str>> {
+        &mut self.models
     }
-    #[cfg(test)]
-    pub fn empty_with_uuid(uuid: Uuid) -> Self {
-        Space::new_with_uuid(Default::default(), SpaceMeta::with_env(into_dict!()), uuid)
+    pub fn props(&self) -> &DictGeneric {
+        &self.props
     }
-    #[inline(always)]
-    pub fn new_auto(mns: IndexST<Box<str>, Model>, meta: SpaceMeta) -> Self {
-        Self {
-            uuid: Uuid::new(),
-            mns: RWLIdx::new(mns),
-            meta,
-        }
+    pub fn props_mut(&mut self) -> &mut DictGeneric {
+        &mut self.props
     }
-    pub fn new_with_uuid(mns: IndexST<Box<str>, Model>, meta: SpaceMeta, uuid: Uuid) -> Self {
-        Self {
-            uuid,
-            meta,
-            mns: RwLock::new(mns),
+    #[cfg(test)]
+    pub fn env(&self) -> &DictGeneric {
+        match self.props().get(Self::KEY_ENV).unwrap() {
+            DictEntryGeneric::Map(m) => m,
+            _ => panic!(),
         }
     }
+}
+
+impl Space {
+    const KEY_ENV: &'static str = "env";
     #[inline]
     /// Validate a `create` stmt
     fn process_create(
@@ -154,25 +108,34 @@ impl Space {
         }: CreateSpace,
     ) -> QueryResult<ProcedureCreate> {
         let space_name = space_name.to_string().into_boxed_str();
-        // check env
-        let env = match props.remove(SpaceMeta::KEY_ENV) {
-            Some(DictEntryGeneric::Map(m)) if props.is_empty() => m,
-            Some(DictEntryGeneric::Data(l)) if l.is_null() => IndexST::default(),
-            None if props.is_empty() => IndexST::default(),
+        // now let's check our props
+        match props.get(Self::KEY_ENV) {
+            Some(d) if props.len() == 1 => {
+                match d {
+                    DictEntryGeneric::Data(d) if d.is_init() => {
+                        // not the right type for a dict
+                        return Err(QueryError::QExecDdlInvalidProperties);
+                    }
+                    DictEntryGeneric::Data(_) => {
+                        // a null? make it empty
+                        let _ =
+                            props.insert(Self::KEY_ENV.into(), DictEntryGeneric::Map(into_dict!()));
+                    }
+                    DictEntryGeneric::Map(_) => {}
+                }
+            }
+            None if props.is_empty() => {
+                let _ = props.st_insert(Self::KEY_ENV.into(), DictEntryGeneric::Map(into_dict!()));
+            }
             _ => {
-                // unknown properties
+                // in all the other cases, we have illegal properties
+                // not the right type for a dict
                 return Err(QueryError::QExecDdlInvalidProperties);
             }
-        };
+        }
         Ok(ProcedureCreate {
             space_name,
-            space: Self::new_auto(
-                IndexST::default(),
-                SpaceMeta::with_env(
-                    // FIXME(@ohsayan): see this is bad. attempt to do it at AST build time
-                    dict::rflatten_metadata(env),
-                ),
-            ),
+            space: Space::new_empty_auto(dict::rflatten_metadata(props)),
         })
     }
 }
@@ -184,36 +147,36 @@ impl Space {
     ) -> QueryResult<()> {
         // process create
         let ProcedureCreate { space_name, space } = Self::process_create(space)?;
-        // acquire access
-        let mut wl = global.namespace().spaces().write();
-        if wl.st_contains(&space_name) {
-            return Err(QueryError::QExecDdlObjectAlreadyExists);
-        }
-        // commit txn
-        if G::FS_IS_NON_NULL {
-            // prepare txn
-            let s_read = space.metadata().dict().read();
-            let txn = gnstxn::CreateSpaceTxn::new(&s_read, &space_name, &space);
-            // try to create space for...the space
-            G::FileSystem::fs_create_dir_all(&SEInitState::space_dir(
-                &space_name,
-                space.get_uuid(),
-            ))?;
+        // lock the global namespace
+        global.namespace().ddl_with_spaces_write(|spaces| {
+            if spaces.st_contains(&space_name) {
+                return Err(QueryError::QExecDdlObjectAlreadyExists);
+            }
             // commit txn
-            match global.namespace_txn_driver().lock().try_commit(txn) {
-                Ok(()) => {}
-                Err(e) => {
-                    // tell fractal to clean it up sometime
-                    global.taskmgr_post_standard_priority(Task::new(
-                        GenericTask::delete_space_dir(&space_name, space.get_uuid()),
-                    ));
-                    return Err(e.into());
+            if G::FS_IS_NON_NULL {
+                // prepare txn
+                let txn = gnstxn::CreateSpaceTxn::new(space.props(), &space_name, &space);
+                // try to create space for...the space
+                G::FileSystem::fs_create_dir_all(&SEInitState::space_dir(
+                    &space_name,
+                    space.get_uuid(),
+                ))?;
+                // commit txn
+                match global.namespace_txn_driver().lock().try_commit(txn) {
+                    Ok(()) => {}
+                    Err(e) => {
+                        // tell fractal to clean it up sometime
+                        global.taskmgr_post_standard_priority(Task::new(
+                            GenericTask::delete_space_dir(&space_name, space.get_uuid()),
+                        ));
+                        return Err(e.into());
+                    }
                 }
             }
-        }
-        // update global state
-        let _ = wl.st_insert(space_name, space);
-        Ok(())
+            // update global state
+            let _ = spaces.st_insert(space_name, RwLock::new(space));
+            Ok(())
+        })
     }
     #[allow(unused)]
     pub fn transactional_exec_alter<G: GlobalInstanceLike>(
@@ -223,16 +186,15 @@ impl Space {
             updated_props,
         }: AlterSpace,
     ) -> QueryResult<()> {
-        global.namespace().with_space(&space_name, |space| {
-            match updated_props.get(SpaceMeta::KEY_ENV) {
+        global.namespace().ddl_with_space_mut(&space_name, |space| {
+            match updated_props.get(Self::KEY_ENV) {
                 Some(DictEntryGeneric::Map(_)) if updated_props.len() == 1 => {}
                 Some(DictEntryGeneric::Data(l)) if updated_props.len() == 1 && l.is_null() => {}
                 None if updated_props.is_empty() => return Ok(()),
                 _ => return Err(QueryError::QExecDdlInvalidProperties),
             }
-            let mut space_props = space.meta.dict().write();
             // create patch
-            let patch = match dict::rprepare_metadata_patch(&space_props, updated_props) {
+            let patch = match dict::rprepare_metadata_patch(space.props(), updated_props) {
                 Some(patch) => patch,
                 None => return Err(QueryError::QExecDdlInvalidProperties),
             };
@@ -244,64 +206,49 @@ impl Space {
                 global.namespace_txn_driver().lock().try_commit(txn)?;
             }
             // merge
-            dict::rmerge_data_with_patch(&mut space_props, patch);
+            dict::rmerge_data_with_patch(space.props_mut(), patch);
             // the `env` key may have been popped, so put it back (setting `env: null` removes the env key and we don't want to waste time enforcing this in the
             // merge algorithm)
-            let _ = space_props.st_insert(
-                SpaceMeta::KEY_ENV.into(),
-                DictEntryGeneric::Map(into_dict!()),
-            );
+            let _ = space
+                .props_mut()
+                .st_insert(Self::KEY_ENV.into(), DictEntryGeneric::Map(into_dict!()));
             Ok(())
         })
     }
     pub fn transactional_exec_drop<G: GlobalInstanceLike>(
         global: &G,
-        DropSpace { space, force: _ }: DropSpace,
+        DropSpace {
+            space: space_name,
+            force: _,
+        }: DropSpace,
     ) -> QueryResult<()> {
         // TODO(@ohsayan): force remove option
         // TODO(@ohsayan): should a drop space block the entire global table?
-        let space_name = space;
-        let mut wgns = global.namespace().spaces().write();
-        let space = match wgns.get(space_name.as_str()) {
-            Some(space) => space,
-            None => return Err(QueryError::QExecObjectNotFound),
-        };
-        let space_w = space.mns.write();
-        if space_w.st_len() != 0 {
-            return Err(QueryError::QExecDdlNotEmpty);
-        }
-        // we can remove this
-        if G::FS_IS_NON_NULL {
-            // prepare txn
-            let txn = gnstxn::DropSpaceTxn::new(gnstxn::SpaceIDRef::new(&space_name, space));
-            // commit txn
-            global.namespace_txn_driver().lock().try_commit(txn)?;
-            // ask for cleanup
-            global.taskmgr_post_standard_priority(Task::new(GenericTask::delete_space_dir(
-                &space_name,
-                space.get_uuid(),
-            )));
-        }
-        drop(space_w);
-        let _ = wgns.st_delete(space_name.as_str());
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-impl PartialEq for SpaceMeta {
-    fn eq(&self, other: &Self) -> bool {
-        let x = self.props.read();
-        let y = other.props.read();
-        *x == *y
-    }
-}
-
-#[cfg(test)]
-impl PartialEq for Space {
-    fn eq(&self, other: &Self) -> bool {
-        let self_mns = self.mns.read();
-        let other_mns = other.mns.read();
-        self.meta == other.meta && *self_mns == *other_mns && self.uuid == other.uuid
+        global.namespace().ddl_with_spaces_write(|spaces| {
+            let Some(space) = spaces.get(space_name.as_str()) else {
+                return Err(QueryError::QExecObjectNotFound);
+            };
+            let space = space.read();
+            if !space.models.is_empty() {
+                // nonempty, we can't do anything
+                return Err(QueryError::QExecDdlNotEmpty);
+            }
+            // okay, it's empty; good riddance
+            if G::FS_IS_NON_NULL {
+                // prepare txn
+                let txn = gnstxn::DropSpaceTxn::new(gnstxn::SpaceIDRef::new(&space_name, &space));
+                // commit txn
+                global.namespace_txn_driver().lock().try_commit(txn)?;
+                // request cleanup
+                global.taskmgr_post_standard_priority(Task::new(GenericTask::delete_space_dir(
+                    &space_name,
+                    space.get_uuid(),
+                )));
+            }
+            // good, we can get rid of this thing
+            drop(space);
+            let _ = spaces.st_delete(space_name.as_str());
+            Ok(())
+        })
     }
 }
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index 3d6246ef..5977b11e 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -28,10 +28,10 @@ use crate::engine::{
     core::{
         model::{alt::AlterPlan, Model},
         tests::ddl_model::{create, exec_create},
+        EntityIDRef,
     },
     error::QueryResult,
     fractal::GlobalInstanceLike,
-    idx::STIndex,
     ql::{ast::parse_ast_node_full, ddl::alt::AlterModel, tests::lex_insecure},
 };
 
@@ -55,19 +55,19 @@ fn exec_plan(
 ) -> QueryResult<()> {
     let mdl_name = exec_create(global, model, new_space)?;
     let prev_uuid = {
-        let gns = global.namespace().spaces().read();
-        let space = gns.get("myspace").unwrap();
-        let space_read = space.models().read();
-        space_read.get(mdl_name.as_str()).unwrap().get_uuid()
+        global
+            .namespace()
+            .idx_models()
+            .read()
+            .get(&EntityIDRef::new("myspace", &mdl_name))
+            .map(|mdl| mdl.get_uuid())
+            .unwrap()
     };
     let tok = lex_insecure(plan.as_bytes()).unwrap();
     let alter = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
-    let (_space, model_name) = alter.model.into_full().unwrap();
     Model::transactional_exec_alter(global, alter)?;
-    let gns_read = global.namespace().spaces().read();
-    let space = gns_read.st_get("myspace").unwrap();
-    let model = space.models().read();
-    let model = model.st_get(model_name.as_str()).unwrap();
+    let models = global.namespace().idx_models().read();
+    let model = models.get(&EntityIDRef::new("myspace", &mdl_name)).unwrap();
     assert_eq!(prev_uuid, model.get_uuid());
     f(model);
     Ok(())
diff --git a/server/src/engine/core/tests/ddl_model/mod.rs b/server/src/engine/core/tests/ddl_model/mod.rs
index 94df1dda..6d4352f6 100644
--- a/server/src/engine/core/tests/ddl_model/mod.rs
+++ b/server/src/engine/core/tests/ddl_model/mod.rs
@@ -29,10 +29,9 @@ mod crt;
 mod layer;
 
 use crate::engine::{
-    core::{model::Model, space::Space},
+    core::{model::Model, EntityIDRef},
     error::QueryResult,
     fractal::GlobalInstanceLike,
-    idx::STIndex,
     ql::{
         ast::{parse_ast_node_full, Entity},
         ddl::crt::CreateModel,
@@ -59,7 +58,7 @@ pub fn exec_create(
     if create_new_space {
         global
             .namespace()
-            .test_new_empty_space(&create_model.model_name.into_full().unwrap().0);
+            .create_empty_test_space(&create_model.model_name.into_full().unwrap().0)
     }
     Model::transactional_exec_create(global, create_model).map(|_| name)
 }
@@ -71,21 +70,13 @@ pub fn exec_create_new_space(
     exec_create(global, create_stmt, true).map(|_| ())
 }
 
-fn with_space(global: &impl GlobalInstanceLike, space_name: &str, f: impl Fn(&Space)) {
-    let rl = global.namespace().spaces().read();
-    let space = rl.st_get(space_name).unwrap();
-    f(space);
-}
-
 fn with_model(
     global: &impl GlobalInstanceLike,
     space_id: &str,
     model_name: &str,
     f: impl Fn(&Model),
 ) {
-    with_space(global, space_id, |space| {
-        let space_rl = space.models().read();
-        let model = space_rl.st_get(model_name).unwrap();
-        f(model)
-    })
+    let models = global.namespace().idx_models().read();
+    let model = models.get(&EntityIDRef::new(space_id, model_name)).unwrap();
+    f(model)
 }
diff --git a/server/src/engine/core/tests/ddl_space/alter.rs b/server/src/engine/core/tests/ddl_space/alter.rs
index 38640972..8973c3ff 100644
--- a/server/src/engine/core/tests/ddl_space/alter.rs
+++ b/server/src/engine/core/tests/ddl_space/alter.rs
@@ -25,8 +25,8 @@
 */
 
 use crate::engine::{
-    core::space::{Space, SpaceMeta},
-    data::cell::Datacell,
+    core::space::Space,
+    data::{cell::Datacell, DictEntryGeneric},
     error::QueryError,
     fractal::test_utils::TestGlobal,
 };
@@ -41,10 +41,9 @@ fn alter_add_prop_env_var() {
         |space| {
             assert_eq!(
                 space,
-                &Space::new_with_uuid(
-                    into_dict!(),
-                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint_default(100))),
-                    space.get_uuid()
+                &Space::new_restore_empty(
+                    space.get_uuid(),
+                    into_dict!("env" => DictEntryGeneric::Map(into_dict!("MY_NEW_PROP" => Datacell::new_uint_default(100)))),
                 )
             );
         },
@@ -59,9 +58,8 @@ fn alter_update_prop_env_var() {
         &global,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
-            let rl = space.meta.dict().read();
             assert_eq!(
-                SpaceMeta::get_env(&rl).get("MY_NEW_PROP").unwrap(),
+                space.env().get("MY_NEW_PROP").unwrap(),
                 &(Datacell::new_uint_default(100).into())
             )
         },
@@ -73,10 +71,9 @@ fn alter_update_prop_env_var() {
         |space| {
             assert_eq!(
                 space,
-                &Space::new_with_uuid(
-                    into_dict!(),
-                    SpaceMeta::with_env(into_dict! ("MY_NEW_PROP" => Datacell::new_uint_default(200))),
+                &Space::new_restore_empty(
                     uuid,
+                    into_dict! ("env" => DictEntryGeneric::Map(into_dict!("MY_NEW_PROP" => Datacell::new_uint_default(200)))),
                 )
             )
         },
@@ -91,9 +88,8 @@ fn alter_remove_prop_env_var() {
         &global,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
-            let rl = space.meta.dict().read();
             assert_eq!(
-                SpaceMeta::get_env(&rl).get("MY_NEW_PROP").unwrap(),
+                space.env().get("MY_NEW_PROP").unwrap(),
                 &(Datacell::new_uint_default(100).into())
             )
         },
@@ -105,7 +101,10 @@ fn alter_remove_prop_env_var() {
         |space| {
             assert_eq!(
                 space,
-                &Space::new_with_uuid(into_dict!(), SpaceMeta::with_env(into_dict!()), uuid)
+                &Space::new_restore_empty(
+                    uuid,
+                    into_dict!("env" => DictEntryGeneric::Map(into_dict!()))
+                )
             )
         },
     )
@@ -133,16 +132,21 @@ fn alter_remove_all_env() {
         &global,
         "create space myspace with { env: { MY_NEW_PROP: 100 } }",
         |space| {
-            let rl = space.meta.dict().read();
             assert_eq!(
-                SpaceMeta::get_env(&rl).get("MY_NEW_PROP").unwrap(),
+                space.env().get("MY_NEW_PROP").unwrap(),
                 &(Datacell::new_uint_default(100).into())
             )
         },
     )
     .unwrap();
     super::exec_alter(&global, "alter space myspace with { env: null }", |space| {
-        assert_eq!(space, &Space::empty_with_uuid(uuid))
+        assert_eq!(
+            space,
+            &Space::new_restore_empty(
+                uuid,
+                into_dict!("env" => DictEntryGeneric::Map(into_dict!()))
+            )
+        )
     })
     .unwrap();
 }
diff --git a/server/src/engine/core/tests/ddl_space/create.rs b/server/src/engine/core/tests/ddl_space/create.rs
index 787f8c67..1010b5d4 100644
--- a/server/src/engine/core/tests/ddl_space/create.rs
+++ b/server/src/engine/core/tests/ddl_space/create.rs
@@ -25,8 +25,8 @@
 */
 
 use crate::engine::{
-    core::space::{Space, SpaceMeta},
-    data::cell::Datacell,
+    core::space::Space,
+    data::{cell::Datacell, DictEntryGeneric},
     error::QueryError,
     fractal::test_utils::TestGlobal,
 };
@@ -35,7 +35,7 @@ use crate::engine::{
 fn exec_create_space_simple() {
     let global = TestGlobal::new_with_tmp_nullfs_driver();
     super::exec_create(&global, "create space myspace", |spc| {
-        assert!(spc.models().read().is_empty())
+        assert!(spc.models().is_empty())
     })
     .unwrap();
 }
@@ -55,12 +55,11 @@ fn exec_create_space_with_env() {
         |space| {
             assert_eq!(
                 space,
-                &Space::new_with_uuid(
-                    into_dict! {},
-                    SpaceMeta::with_env(into_dict! {
-                        "MAX_MODELS" => Datacell::new_uint_default(100)
-                    }),
-                    space.get_uuid()
+                &Space::new_restore_empty(
+                    space.get_uuid(),
+                    into_dict! {
+                        "env" => DictEntryGeneric::Map(into_dict!("MAX_MODELS" => Datacell::new_uint_default(100)))
+                    },
                 )
             );
         },
diff --git a/server/src/engine/core/tests/ddl_space/mod.rs b/server/src/engine/core/tests/ddl_space/mod.rs
index fb0c6799..77940719 100644
--- a/server/src/engine/core/tests/ddl_space/mod.rs
+++ b/server/src/engine/core/tests/ddl_space/mod.rs
@@ -48,7 +48,7 @@ fn exec_create(
         ast::parse_ast_node_full::<crate::engine::ql::ddl::crt::CreateSpace>(&tok[2..]).unwrap();
     let name = ast_node.space_name;
     Space::transactional_exec_create(gns, ast_node)?;
-    gns.namespace().with_space(&name, |space| {
+    gns.namespace().ddl_with_space_mut(&name, |space| {
         verify(space);
         Ok(space.get_uuid())
     })
@@ -64,7 +64,7 @@ fn exec_alter(
         ast::parse_ast_node_full::<crate::engine::ql::ddl::alt::AlterSpace>(&tok[2..]).unwrap();
     let name = ast_node.space_name;
     Space::transactional_exec_alter(gns, ast_node)?;
-    gns.namespace().with_space(&name, |space| {
+    gns.namespace().ddl_with_space_mut(&name, |space| {
         verify(space);
         Ok(space.get_uuid())
     })
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index 9d9fa230..f06779ae 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -30,7 +30,7 @@ mod select;
 mod update;
 
 use crate::engine::{
-    core::{dml, index::Row, model::Model},
+    core::{dml, index::Row, model::Model, space::Space},
     data::{cell::Datacell, lit::Lit},
     error::QueryResult,
     fractal::GlobalInstanceLike,
@@ -43,9 +43,11 @@ use crate::engine::{
 };
 
 fn _exec_only_create_space_model(global: &impl GlobalInstanceLike, model: &str) -> QueryResult<()> {
-    if !global.namespace().spaces().read().contains_key("myspace") {
-        global.namespace().test_new_empty_space("myspace");
-    }
+    let _ = global
+        .namespace()
+        .idx()
+        .write()
+        .insert("myspace".into(), Space::new_auto_all().into());
     let lex_create_model = lex_insecure(model.as_bytes()).unwrap();
     let stmt_create_model = parse_ast_node_full(&lex_create_model[2..]).unwrap();
     Model::transactional_exec_create(global, stmt_create_model)
diff --git a/server/src/engine/core/util.rs b/server/src/engine/core/util.rs
index f7f54895..fb5b7021 100644
--- a/server/src/engine/core/util.rs
+++ b/server/src/engine/core/util.rs
@@ -24,28 +24,127 @@
  *
 */
 
-use crate::engine::{
-    error::{QueryError, QueryResult},
-    ql::ast::Entity,
+use std::{
+    alloc::{dealloc, Layout},
+    borrow::Borrow,
+    fmt,
+    hash::Hash,
+    marker::PhantomData,
+    mem::ManuallyDrop,
+    slice, str,
 };
 
-pub trait EntityLocator<'a> {
-    fn parse_entity(self) -> QueryResult<(&'a str, &'a str)>
-    where
-        Self: 'a;
+pub struct EntityID {
+    sp: *mut u8,
+    sl: usize,
+    ep: *mut u8,
+    el: usize,
 }
 
-impl<'a> EntityLocator<'a> for (&'a str, &'a str) {
-    fn parse_entity(self) -> QueryResult<(&'a str, &'a str)> {
-        Ok(self)
+impl EntityID {
+    pub fn new(space: &str, entity: &str) -> Self {
+        let mut space = ManuallyDrop::new(space.to_owned().into_boxed_str().into_boxed_bytes());
+        let mut entity = ManuallyDrop::new(entity.to_owned().into_boxed_str().into_boxed_bytes());
+        Self {
+            sp: space.as_mut_ptr(),
+            sl: space.len(),
+            ep: entity.as_mut_ptr(),
+            el: entity.len(),
+        }
+    }
+    pub fn space(&self) -> &str {
+        unsafe { str::from_utf8_unchecked(slice::from_raw_parts(self.sp, self.sl)) }
+    }
+    pub fn entity(&self) -> &str {
+        unsafe { str::from_utf8_unchecked(slice::from_raw_parts(self.ep, self.el)) }
+    }
+}
+
+impl Drop for EntityID {
+    fn drop(&mut self) {
+        unsafe {
+            dealloc(self.sp, Layout::array::<u8>(self.sl).unwrap_unchecked());
+            dealloc(self.ep, Layout::array::<u8>(self.el).unwrap_unchecked());
+        }
+    }
+}
+
+impl PartialEq for EntityID {
+    fn eq(&self, other: &Self) -> bool {
+        self.space() == other.space() && self.entity() == other.entity()
+    }
+}
+
+impl Eq for EntityID {}
+
+impl Hash for EntityID {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.space().hash(state);
+        self.entity().hash(state);
+    }
+}
+
+impl fmt::Debug for EntityID {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("EntityID")
+            .field("space", &self.space())
+            .field("entity", &self.entity())
+            .finish()
+    }
+}
+
+pub struct EntityIDRef<'a> {
+    sp: *const u8,
+    sl: usize,
+    ep: *const u8,
+    el: usize,
+    _lt: PhantomData<(&'a str, &'a str)>,
+}
+
+impl<'a> EntityIDRef<'a> {
+    pub fn new(space: &'a str, entity: &'a str) -> Self {
+        Self {
+            sp: space.as_ptr(),
+            sl: space.len(),
+            ep: entity.as_ptr(),
+            el: entity.len(),
+            _lt: PhantomData,
+        }
+    }
+    pub fn space(&self) -> &'a str {
+        unsafe { str::from_utf8_unchecked(slice::from_raw_parts(self.sp, self.sl)) }
+    }
+    pub fn entity(&self) -> &'a str {
+        unsafe { str::from_utf8_unchecked(slice::from_raw_parts(self.ep, self.el)) }
+    }
+}
+
+impl<'a> PartialEq for EntityIDRef<'a> {
+    fn eq(&self, other: &Self) -> bool {
+        self.space() == other.space() && self.entity() == other.entity()
+    }
+}
+
+impl<'a> Eq for EntityIDRef<'a> {}
+
+impl<'a> Hash for EntityIDRef<'a> {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.space().hash(state);
+        self.entity().hash(state);
+    }
+}
+
+impl<'a> fmt::Debug for EntityIDRef<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("EntityIDRef")
+            .field("space", &self.space())
+            .field("entity", &self.entity())
+            .finish()
     }
 }
 
-impl<'a> EntityLocator<'a> for Entity<'a> {
-    fn parse_entity(self) -> QueryResult<(&'a str, &'a str)>
-    where
-        Self: 'a,
-    {
-        self.into_full_str().ok_or(QueryError::QLExpectedEntity)
+impl<'a> Borrow<EntityIDRef<'a>> for EntityID {
+    fn borrow(&self) -> &EntityIDRef<'a> {
+        unsafe { core::mem::transmute(self) }
     }
 }
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 0307c549..1cabe3eb 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -30,6 +30,7 @@ use {
         engine::{
             core::model::{delta::DataDelta, Model},
             data::uuid::Uuid,
+            ql::ast::Entity,
             storage::v1::LocalFS,
         },
         util::os,
@@ -279,17 +280,17 @@ impl FractalMgr {
                     // was way behind in the queue
                     return;
                 };
-                let res =
-                    global
-                        ._namespace()
-                        .with_model((model_id.space(), model_id.model()), |model| {
-                            if model.get_uuid() != model_id.uuid() {
-                                // once again, throughput maximization will lead to, in extremely rare cases, this
-                                // branch returning. but it is okay
-                                return Ok(());
-                            }
-                            Self::try_write_model_data_batch(model, observed_size, mdl_driver)
-                        });
+                let res = global._namespace().with_model(
+                    Entity::Full(model_id.space().into(), model_id.model().into()),
+                    |model| {
+                        if model.get_uuid() != model_id.uuid() {
+                            // once again, throughput maximization will lead to, in extremely rare cases, this
+                            // branch returning. but it is okay
+                            return Ok(());
+                        }
+                        Self::try_write_model_data_batch(model, observed_size, mdl_driver)
+                    },
+                );
                 match res {
                     Ok(()) => {
                         if observed_size != 0 {
@@ -298,7 +299,7 @@ impl FractalMgr {
                     }
                     Err(_) => {
                         log::error!(
-                            "fhp: error writing data batch for model {}. Retrying...",
+                            "fhp: error writing data batch for model {}. retrying ...",
                             model_id.uuid()
                         );
                         // enqueue again for retrying
@@ -369,21 +370,21 @@ impl FractalMgr {
         let mdl_drivers = global.get_state().get_mdl_drivers().read();
         for (model_id, driver) in mdl_drivers.iter() {
             let mut observed_len = 0;
-            let res =
-                global
-                    ._namespace()
-                    .with_model((model_id.space(), model_id.model()), |model| {
-                        if model.get_uuid() != model_id.uuid() {
-                            // once again, throughput maximization will lead to, in extremely rare cases, this
-                            // branch returning. but it is okay
-                            return Ok(());
-                        }
-                        // mark that we're taking these deltas
-                        observed_len = model
-                            .delta_state()
-                            .__fractal_take_full_from_data_delta(super::FractalToken::new());
-                        Self::try_write_model_data_batch(model, observed_len, driver)
-                    });
+            let res = global._namespace().with_model(
+                Entity::Full(model_id.space().into(), model_id.model().into()),
+                |model| {
+                    if model.get_uuid() != model_id.uuid() {
+                        // once again, throughput maximization will lead to, in extremely rare cases, this
+                        // branch returning. but it is okay
+                        return Ok(());
+                    }
+                    // mark that we're taking these deltas
+                    observed_len = model
+                        .delta_state()
+                        .__fractal_take_full_from_data_delta(super::FractalToken::new());
+                    Self::try_write_model_data_batch(model, observed_len, driver)
+                },
+            );
             match res {
                 Ok(()) => {
                     if observed_len != 0 {
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index fbc75463..e605ca67 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -324,6 +324,21 @@ pub enum Entity<'a> {
     Full(Ident<'a>, Ident<'a>),
 }
 
+impl<'a> Entity<'a> {
+    pub fn into_full_result(self) -> QueryResult<(Ident<'a>, Ident<'a>)> {
+        match self {
+            Self::Full(a, b) => Ok((a, b)),
+            _ => Err(QueryError::QLExpectedEntity),
+        }
+    }
+}
+
+impl<'a> From<(&'a str, &'a str)> for Entity<'a> {
+    fn from((s, e): (&'a str, &'a str)) -> Self {
+        Self::Full(s.into(), e.into())
+    }
+}
+
 impl<'a> Entity<'a> {
     #[cfg(test)]
     pub fn into_full(self) -> Option<(Ident<'a>, Ident<'a>)> {
@@ -333,13 +348,6 @@ impl<'a> Entity<'a> {
             None
         }
     }
-    pub fn into_full_str(self) -> Option<(&'a str, &'a str)> {
-        if let Self::Full(a, b) = self {
-            Some((a.as_str(), b.as_str()))
-        } else {
-            None
-        }
-    }
     #[inline(always)]
     /// Parse a full entity from the given slice
     ///
@@ -453,6 +461,7 @@ impl<'a> Entity<'a> {
 }
 
 #[derive(Debug, PartialEq)]
+#[allow(dead_code)] // TODO(@ohsayan): get rid of this
 /// A [`Statement`] is a fully BlueQL statement that can be executed by the query engine
 // TODO(@ohsayan): Determine whether we actually need this
 pub enum Statement<'a> {
@@ -496,6 +505,7 @@ pub enum Statement<'a> {
 
 #[inline(always)]
 #[cfg(debug_assertions)]
+#[allow(dead_code)] // TODO(@ohsayan): get rid of this
 pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResult<Statement<'a>> {
     if compiler::unlikely(tok.len() < 2) {
         return Err(QueryError::QLUnexpectedEndOfStatement);
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index c9876ec2..43c2d727 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -30,7 +30,7 @@ use {
         engine::{
             core::{
                 model::{delta::IRModel, Field, Layer, Model},
-                space::{Space, SpaceMeta},
+                space::Space,
             },
             data::{
                 tag::{DataTag, TagClass, TagSelector},
@@ -515,9 +515,6 @@ impl<'a> PersistObject for SpaceLayoutRef<'a> {
                 scanner,
                 super::map::MapIndexSizeMD(md.prop_c),
             )?;
-        Ok(Space::new_restore_empty(
-            SpaceMeta::new_with_meta(space_meta),
-            md.uuid,
-        ))
+        Ok(Space::new_restore_empty(md.uuid, space_meta))
     }
 }
diff --git a/server/src/engine/storage/v1/inf/tests.rs b/server/src/engine/storage/v1/inf/tests.rs
index 1ce59fb4..b402854e 100644
--- a/server/src/engine/storage/v1/inf/tests.rs
+++ b/server/src/engine/storage/v1/inf/tests.rs
@@ -29,7 +29,7 @@ use {
     crate::engine::{
         core::{
             model::{Field, Layer, Model},
-            space::{Space, SpaceMeta},
+            space::Space,
         },
         data::{
             cell::Datacell,
@@ -111,11 +111,10 @@ fn model() {
 #[test]
 fn space() {
     let uuid = Uuid::new();
-    let space = Space::new_with_uuid(Default::default(), SpaceMeta::default(), uuid);
-    let space_meta_read = space.metadata().dict().read();
+    let space = Space::new_restore_empty(uuid, Default::default());
     let enc = super::enc::enc_full::<obj::SpaceLayoutRef>(obj::SpaceLayoutRef::from((
         &space,
-        &*space_meta_read,
+        space.props(),
     )));
     let dec = super::dec::dec_full::<obj::SpaceLayoutRef>(&enc).unwrap();
     assert_eq!(space, dec);
diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs
index b44b2000..61fe064a 100644
--- a/server/src/engine/storage/v1/loader.rs
+++ b/server/src/engine/storage/v1/loader.rs
@@ -30,7 +30,7 @@ use crate::engine::storage::v1::{
     JournalWriter,
 };
 use crate::engine::{
-    core::GlobalNS,
+    core::{EntityIDRef, GlobalNS},
     data::uuid::Uuid,
     error::RuntimeResult,
     fractal::error::ErrorContext,
@@ -76,10 +76,15 @@ impl SEInitState {
                 std::fs::create_dir(DATA_DIR).inherit_set_dmsg("creating data directory")?;
             }
             if !is_new {
+                let models = gns.idx_models().read();
                 // this is an existing instance, so read in all data
-                for (space_name, space) in gns.spaces().read().iter() {
+                for (space_name, space) in gns.idx().read().iter() {
+                    let space = space.read();
                     let space_uuid = space.get_uuid();
-                    for (model_name, model) in space.models().read().iter() {
+                    for model_name in space.models().iter() {
+                        let model = models
+                            .get(&EntityIDRef::new(&space_name, &model_name))
+                            .unwrap();
                         let path =
                             Self::model_path(space_name, space_uuid, model_name, model.get_uuid());
                         let persist_driver = batch_jrnl::reinit(&path, model).inherit_set_dmsg(
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index 39bbfe0a..0dc8a482 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -31,7 +31,7 @@ use {
             core::{
                 model::{delta::IRModel, Field, Model},
                 space::Space,
-                GlobalNS,
+                GlobalNS, {EntityID, EntityIDRef},
             },
             data::uuid::Uuid,
             error::TransactionError,
@@ -160,14 +160,31 @@ fn with_space<T>(
     space_id: &super::SpaceIDRes,
     mut f: impl FnMut(&Space) -> RuntimeResult<T>,
 ) -> RuntimeResult<T> {
-    let spaces = gns.spaces().read();
+    let spaces = gns.idx().read();
     let Some(space) = spaces.st_get(&space_id.name) else {
         return Err(TransactionError::OnRestoreDataMissing.into());
     };
+    let space = space.read();
     if space.get_uuid() != space_id.uuid {
         return Err(TransactionError::OnRestoreDataConflictMismatch.into());
     }
-    f(space)
+    f(&space)
+}
+
+fn with_space_mut<T>(
+    gns: &GlobalNS,
+    space_id: &super::SpaceIDRes,
+    mut f: impl FnMut(&mut Space) -> RuntimeResult<T>,
+) -> RuntimeResult<T> {
+    let spaces = gns.idx().read();
+    let Some(space) = spaces.st_get(&space_id.name) else {
+        return Err(TransactionError::OnRestoreDataMissing.into());
+    };
+    let mut space = space.write();
+    if space.get_uuid() != space_id.uuid {
+        return Err(TransactionError::OnRestoreDataConflictMismatch.into());
+    }
+    f(&mut space)
 }
 
 fn with_model<T>(
@@ -176,9 +193,10 @@ fn with_model<T>(
     model_id: &ModelIDRes,
     mut f: impl FnMut(&Model) -> RuntimeResult<T>,
 ) -> RuntimeResult<T> {
-    with_space(gns, space_id, |space| {
-        let models = space.models().read();
-        let Some(model) = models.st_get(&model_id.model_name) else {
+    with_space(gns, space_id, |_| {
+        let models = gns.idx_models().read();
+        let Some(model) = models.get(&EntityIDRef::new(&space_id.name, &model_id.model_name))
+        else {
             return Err(TransactionError::OnRestoreDataMissing.into());
         };
         if model.get_uuid() != model_id.model_uuid {
@@ -302,26 +320,30 @@ impl<'a> GNSEvent for CreateModelTxn<'a> {
         }: Self::RestoreType,
         gns: &GlobalNS,
     ) -> RuntimeResult<()> {
-        let rgns = gns.spaces().read();
         /*
             NOTE(@ohsayan):
-            do note that this is a little interesting situation especially because we need to be able to handle
-            changes in the schema *and* be able to "sync" that (for consistency) with the model's primary index.
-
-            There is no evident way about how this is going to be handled, but the ideal way would be to keep
-            versioned index of schemas.
+            A jump to the second branch is practically impossible and should be caught long before we actually end up
+            here (due to mismatched checksums), but might be theoretically possible because the cosmic rays can be wild
+            (or well magnetic stuff arounding spinning disks). But we just want to be extra sure. Don't let the aliens (or
+            rather, radiation) from the cosmos deter us!
         */
-        match rgns.st_get(&space_id.name) {
-            Some(space) if space.get_uuid() == space_id.uuid => {
-                if space._create_model(&model_name, model).is_ok() {
-                    Ok(())
-                } else {
-                    Err(TransactionError::OnRestoreDataConflictAlreadyExists.into())
-                }
-            }
-            Some(_) => return Err(TransactionError::OnRestoreDataConflictMismatch.into()),
-            None => return Err(TransactionError::OnRestoreDataMissing.into()),
+        let spaces = gns.idx().write();
+        let mut models = gns.idx_models().write();
+        let Some(space) = spaces.get(&space_id.name) else {
+            return Err(TransactionError::OnRestoreDataMissing.into());
+        };
+        let mut space = space.write();
+        if space.models().contains(&model_name) {
+            return Err(TransactionError::OnRestoreDataConflictAlreadyExists.into());
         }
+        if models
+            .insert(EntityID::new(&space_id.name, &model_name), model)
+            .is_some()
+        {
+            return Err(TransactionError::OnRestoreDataConflictMismatch.into());
+        }
+        space.models_mut().insert(model_name);
+        Ok(())
     }
 }
 
@@ -724,13 +746,19 @@ impl<'a> GNSEvent for DropModelTxn<'a> {
         }: Self::RestoreType,
         gns: &GlobalNS,
     ) -> RuntimeResult<()> {
-        with_space(gns, &space_id, |space| {
-            let mut wgns = space.models().write();
-            match wgns.st_delete_if(&model_name, |mdl| mdl.get_uuid() == model_uuid) {
-                Some(true) => Ok(()),
-                Some(false) => return Err(TransactionError::OnRestoreDataConflictMismatch.into()),
-                None => Err(TransactionError::OnRestoreDataMissing.into()),
+        with_space_mut(gns, &space_id, |space| {
+            let mut models = gns.idx_models().write();
+            if !space.models_mut().remove(&model_name) {
+                return Err(TransactionError::OnRestoreDataMissing.into());
             }
+            let Some(removed_model) = models.remove(&EntityIDRef::new(&space_id.name, &model_name))
+            else {
+                return Err(TransactionError::OnRestoreDataMissing.into());
+            };
+            if removed_model.get_uuid() != model_uuid {
+                return Err(TransactionError::OnRestoreDataConflictMismatch.into());
+            }
+            Ok(())
         })
     }
 }
diff --git a/server/src/engine/txn/gns/space.rs b/server/src/engine/txn/gns/space.rs
index 5f9032b8..193b9344 100644
--- a/server/src/engine/txn/gns/space.rs
+++ b/server/src/engine/txn/gns/space.rs
@@ -120,8 +120,8 @@ impl<'a> GNSEvent for CreateSpaceTxn<'a> {
         CreateSpaceTxnRestorePL { space_name, space }: CreateSpaceTxnRestorePL,
         gns: &crate::engine::core::GlobalNS,
     ) -> RuntimeResult<()> {
-        let mut wgns = gns.spaces().write();
-        if wgns.st_insert(space_name, space) {
+        let mut spaces = gns.idx().write();
+        if spaces.st_insert(space_name, space.into()) {
             Ok(())
         } else {
             Err(TransactionError::OnRestoreDataConflictAlreadyExists.into())
@@ -215,11 +215,11 @@ impl<'a> GNSEvent for AlterSpaceTxn<'a> {
         }: Self::RestoreType,
         gns: &crate::engine::core::GlobalNS,
     ) -> RuntimeResult<()> {
-        let gns = gns.spaces().read();
+        let gns = gns.idx().read();
         match gns.st_get(&space_id.name) {
             Some(space) => {
-                let mut wmeta = space.metadata().dict().write();
-                if !crate::engine::data::dict::rmerge_metadata(&mut wmeta, space_meta) {
+                let mut space = space.write();
+                if !crate::engine::data::dict::rmerge_metadata(space.props_mut(), space_meta) {
                     return Err(TransactionError::OnRestoreDataConflictMismatch.into());
                 }
             }
@@ -278,10 +278,13 @@ impl<'a> GNSEvent for DropSpaceTxn<'a> {
         super::SpaceIDRes { uuid, name }: Self::RestoreType,
         gns: &GlobalNS,
     ) -> RuntimeResult<()> {
-        let mut wgns = gns.spaces().write();
+        let mut wgns = gns.idx().write();
         match wgns.entry(name) {
             std::collections::hash_map::Entry::Occupied(oe) => {
-                if oe.get().get_uuid() == uuid {
+                let space = oe.get().read();
+                if space.get_uuid() == uuid {
+                    // NB(@ohsayan): we do not need to remove models here since they must have been already removed for this query to have actually executed
+                    drop(space);
                     oe.remove_entry();
                     Ok(())
                 } else {
diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
index a23c351e..75559a7f 100644
--- a/server/src/engine/txn/gns/tests/full_chain.rs
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -27,7 +27,7 @@
 use crate::engine::{
     core::{
         model::{Field, Layer, Model},
-        space::{Space, SpaceMeta},
+        space::Space,
     },
     data::{cell::Datacell, tag::TagSelector, uuid::Uuid, DictEntryGeneric},
     error::QueryError,
@@ -58,10 +58,11 @@ fn init_space(global: &impl GlobalInstanceLike, space_name: &str, env: &str) ->
     Space::transactional_exec_create(global, stmt).unwrap();
     global
         .namespace()
-        .spaces()
+        .idx()
         .read()
         .get(name.as_str())
         .unwrap()
+        .read()
         .get_uuid()
 }
 
@@ -76,13 +77,13 @@ fn create_space() {
         }
         multirun(|| {
             let global = TestGlobal::new_with_vfs_driver(log_name);
+            let spaces = global.namespace().idx().read();
+            let space = spaces.get("myspace").unwrap().read();
             assert_eq!(
-                global.namespace().spaces().read().get("myspace").unwrap(),
+                &*space,
                 &Space::new_restore_empty(
-                    SpaceMeta::with_env(
-                        into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536)))
-                    ),
-                    uuid
+                    uuid,
+                    into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536)))
                 )
             );
         })
@@ -104,13 +105,13 @@ fn alter_space() {
         }
         multirun(|| {
             let global = TestGlobal::new_with_vfs_driver(log_name);
+            let spaces = global.namespace().idx().read();
+            let space = spaces.get("myspace").unwrap().read();
             assert_eq!(
-                global.namespace().spaces().read().get("myspace").unwrap(),
+                &*space,
                 &Space::new_restore_empty(
-                    SpaceMeta::with_env(
-                        into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536)))
-                    ),
-                    uuid
+                    uuid,
+                    into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536)))
                 )
             );
         })
@@ -129,7 +130,7 @@ fn drop_space() {
         }
         multirun(|| {
             let global = TestGlobal::new_with_vfs_driver(log_name);
-            assert_eq!(global.namespace().spaces().read().get("myspace"), None);
+            assert!(global.namespace().idx().read().get("myspace").is_none());
         })
     })
 }
@@ -174,7 +175,7 @@ fn create_model() {
             let global = TestGlobal::new_with_vfs_driver(log_name);
             global
                 .namespace()
-                .with_model(("myspace", "mymodel"), |model| {
+                .with_model(("myspace", "mymodel").into(), |model| {
                     assert_eq!(
                         model,
                         &Model::new_restore(
@@ -212,7 +213,7 @@ fn alter_model_add() {
             let global = TestGlobal::new_with_vfs_driver(log_name);
             global
                 .namespace()
-                .with_model(("myspace", "mymodel"), |model| {
+                .with_model(("myspace", "mymodel").into(), |model| {
                     assert_eq!(
                         model
                             .intent_read_model()
@@ -251,7 +252,7 @@ fn alter_model_remove() {
             let global = TestGlobal::new_with_vfs_driver(log_name);
             global
                 .namespace()
-                .with_model(("myspace", "mymodel"), |model| {
+                .with_model(("myspace", "mymodel").into(), |model| {
                     let irm = model.intent_read_model();
                     assert!(irm.fields().st_get("has_secure_key").is_none());
                     assert!(irm.fields().st_get("is_dumb").is_none());
@@ -284,7 +285,7 @@ fn alter_model_update() {
             let global = TestGlobal::new_with_vfs_driver(log_name);
             global
                 .namespace()
-                .with_model(("myspace", "mymodel"), |model| {
+                .with_model(("myspace", "mymodel").into(), |model| {
                     assert_eq!(
                         model
                             .intent_read_model()
@@ -316,7 +317,7 @@ fn drop_model() {
             assert_eq!(
                 global
                     .namespace()
-                    .with_model(("myspace", "mymodel"), |_| { Ok(()) })
+                    .with_model(("myspace", "mymodel").into(), |_| { Ok(()) })
                     .unwrap_err(),
                 QueryError::QExecObjectNotFound
             );
diff --git a/server/src/engine/txn/gns/tests/io.rs b/server/src/engine/txn/gns/tests/io.rs
index acffafd8..470aac63 100644
--- a/server/src/engine/txn/gns/tests/io.rs
+++ b/server/src/engine/txn/gns/tests/io.rs
@@ -46,23 +46,23 @@ mod space_tests {
     };
     #[test]
     fn create() {
-        let orig_space = Space::empty();
-        let space_r = orig_space.metadata().dict().read();
+        let orig_space = Space::new_auto_all();
+        let space_r = orig_space.props();
         let txn = CreateSpaceTxn::new(&space_r, "myspace", &orig_space);
         let encoded = enc::enc_full_self(txn);
         let decoded = dec::dec_full::<CreateSpaceTxn>(&encoded).unwrap();
         assert_eq!(
             CreateSpaceTxnRestorePL {
                 space_name: "myspace".into(),
-                space: Space::empty_with_uuid(orig_space.get_uuid())
+                space: Space::new_restore_empty(orig_space.get_uuid(), Default::default())
             },
             decoded
         );
     }
     #[test]
     fn alter() {
-        let space = Space::empty();
-        let space_r = space.metadata().dict().read();
+        let space = Space::new_auto_all();
+        let space_r = space.props();
         let txn = AlterSpaceTxn::new(SpaceIDRef::new("myspace", &space), &space_r);
         let encoded = enc::enc_full_self(txn);
         let decoded = dec::dec_full::<AlterSpaceTxn>(&encoded).unwrap();
@@ -76,7 +76,7 @@ mod space_tests {
     }
     #[test]
     fn drop() {
-        let space = Space::empty();
+        let space = Space::new_auto_all();
         let txn = DropSpaceTxn::new(super::SpaceIDRef::new("myspace", &space));
         let encoded = enc::enc_full_self(txn);
         let decoded = dec::dec_full::<DropSpaceTxn>(&encoded).unwrap();
@@ -103,7 +103,7 @@ mod model_tests {
         },
     };
     fn default_space_model() -> (Space, Model) {
-        let space = Space::empty();
+        let space = Space::new_auto_all();
         let model = Model::new_restore(
             Uuid::new(),
             "username".into(),

From d6eeb5cdf655e262d67cc9bb25d01e1e056f9d44 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 10 Nov 2023 01:46:35 +0530
Subject: [PATCH 277/310] Cleanup

---
 server/src/engine/core/dml/mod.rs             | 10 +++---
 server/src/engine/idx/mtchm/meta.rs           |  2 +-
 server/src/engine/mem/mod.rs                  |  3 +-
 server/src/engine/mod.rs                      | 12 ++++---
 server/src/engine/ql/ddl/drop.rs              |  1 +
 server/src/engine/ql/ddl/ins.rs               |  1 +
 server/src/engine/ql/lex/mod.rs               |  7 ++--
 .../engine/storage/v1/batch_jrnl/restore.rs   |  3 +-
 server/src/engine/storage/v1/mod.rs           |  3 +-
 server/src/main.rs                            | 35 +++++++++++--------
 10 files changed, 41 insertions(+), 36 deletions(-)

diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index 702ed855..ea01a691 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -40,13 +40,13 @@ use crate::{
 };
 
 #[cfg(test)]
-pub use upd::collect_trace_path as update_flow_trace;
 pub use {
-    del::{delete, delete_resp},
-    ins::{insert, insert_resp},
-    sel::{select_custom, select_resp},
-    upd::{update, update_resp},
+    del::delete,
+    ins::insert,
+    sel::select_custom,
+    upd::{collect_trace_path as update_flow_trace, update},
 };
+pub use {del::delete_resp, ins::insert_resp, sel::select_resp, upd::update_resp};
 
 impl Model {
     pub(self) fn resolve_where<'a>(
diff --git a/server/src/engine/idx/mtchm/meta.rs b/server/src/engine/idx/mtchm/meta.rs
index 874e18d2..d916bcb2 100644
--- a/server/src/engine/idx/mtchm/meta.rs
+++ b/server/src/engine/idx/mtchm/meta.rs
@@ -32,7 +32,7 @@ use {
     std::{collections::hash_map::RandomState, sync::Arc},
 };
 
-const LNODE_STACK: usize = 2;
+const LNODE_STACK: usize = 1;
 pub type DefConfig = Config2B<RandomState>;
 pub type LNode<T> = VInline<LNODE_STACK, T>;
 
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index eca293bc..abf66651 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -39,10 +39,9 @@ pub use {
     astr::AStr,
     ll::CachePadded,
     scanner::BufferedScanner,
-    stackop::ByteStack,
     uarray::UArray,
     vinline::VInline,
-    word::{DwordNN, DwordQN, QwordNNNN, TwordNNN, WordIO, ZERO_BLOCK},
+    word::{DwordNN, DwordQN, WordIO, ZERO_BLOCK},
 };
 // imports
 use std::alloc::{self, Layout};
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 8e367b2e..57fd0fb5 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -41,11 +41,12 @@ mod txn;
 // test
 #[cfg(test)]
 mod tests;
+// re-export
+pub use error::RuntimeResult;
 
 use {
     self::{
         config::{ConfigEndpoint, ConfigEndpointTls, ConfigMode, ConfigReturn, Configuration},
-        error::RuntimeResult,
         fractal::context::{self, Subsystem},
         storage::v1::{
             loader::{self, SEInitState},
@@ -58,6 +59,10 @@ use {
     tokio::sync::broadcast,
 };
 
+pub(super) fn set_context_init(msg: &'static str) {
+    context::set(Subsystem::Init, msg)
+}
+
 /// Initialize all drivers, load all data
 ///
 /// WARN: Must be in [`tokio::runtime::Runtime`] context!
@@ -137,12 +142,10 @@ impl EndpointListeners {
 }
 
 pub async fn start(
+    termsig: TerminationSignal,
     Configuration { endpoints, .. }: Configuration,
     fractal::GlobalStateStart { global, boot }: fractal::GlobalStateStart,
 ) -> RuntimeResult<()> {
-    // bind termination signal
-    context::set(Subsystem::Init, "binding system signals");
-    let termsig = TerminationSignal::init()?;
     // create our system-wide channel
     let (signal, _) = broadcast::channel::<()>(1);
     // start our services
@@ -212,7 +215,6 @@ pub async fn start(
         (_, Err(e)) => error!("error while terminating flp-executor: {e}"),
         _ => {}
     }
-    info!("all services have exited");
     Ok(())
 }
 
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index fed26836..cddef4c6 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -101,6 +101,7 @@ pub fn parse_drop<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResu
     }
 }
 
+#[cfg(test)]
 pub use impls::DropStatementAST;
 mod impls {
     use {
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
index ff07600d..323fb1b2 100644
--- a/server/src/engine/ql/ddl/ins.rs
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -70,6 +70,7 @@ pub fn parse_inspect<'a, Qd: QueryData<'a>>(
     }
 }
 
+#[cfg(test)]
 pub use impls::InspectStatementAST;
 mod impls {
     use crate::engine::{
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index ad9f3478..f25a3ad3 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -25,10 +25,9 @@
 */
 
 mod raw;
-pub use {
-    insecure_impl::InsecureLexer,
-    raw::{Ident, Keyword, KeywordMisc, KeywordStmt, Symbol, Token},
-};
+#[cfg(test)]
+pub use insecure_impl::InsecureLexer;
+pub use raw::{Ident, Keyword, KeywordMisc, KeywordStmt, Symbol, Token};
 
 use {
     crate::engine::{
diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs
index 963be39f..c5b1bace 100644
--- a/server/src/engine/storage/v1/batch_jrnl/restore.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs
@@ -44,7 +44,6 @@ use {
         idx::{MTIndex, STIndex, STIndexSeq},
         storage::v1::rw::{RawFSInterface, SDSSFileIO, SDSSFileTrackedReader},
     },
-    crossbeam_epoch::pin,
     std::{
         collections::{hash_map::Entry as HMEntry, HashMap},
         mem::ManuallyDrop,
@@ -217,7 +216,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
         // NOTE(@ohsayan): current complexity is O(n) which is good enough (in the future I might revise this to a fancier impl)
         // pin model
         let irm = m.intent_read_model();
-        let g = pin();
+        let g = unsafe { crossbeam_epoch::unprotected() };
         let mut pending_delete = HashMap::new();
         let p_index = m.primary_index().__raw_index();
         // scan rows
diff --git a/server/src/engine/storage/v1/mod.rs b/server/src/engine/storage/v1/mod.rs
index ac06e514..3a1bb406 100644
--- a/server/src/engine/storage/v1/mod.rs
+++ b/server/src/engine/storage/v1/mod.rs
@@ -41,9 +41,8 @@ mod tests;
 // re-exports
 pub use {
     journal::{JournalAdapter, JournalWriter},
-    memfs::NullFS,
     rw::{LocalFS, RawFSInterface, SDSSFileIO},
 };
 pub mod data_batch {
-    pub use super::batch_jrnl::{create, reinit, DataBatchPersistDriver, DataBatchRestoreDriver};
+    pub use super::batch_jrnl::{create, DataBatchPersistDriver};
 }
diff --git a/server/src/main.rs b/server/src/main.rs
index bed3a748..f2b4be21 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -67,23 +67,28 @@ fn main() {
         .parse_filters(&env::var("SKY_LOG").unwrap_or_else(|_| "info".to_owned()))
         .init();
     println!("{TEXT}\nSkytable v{VERSION} | {URL}\n");
-    let (config, global) = match engine::load_all() {
-        Ok(x) => x,
-        Err(e) => {
-            error!("{e}");
-            exit_error()
-        }
+    let run = || {
+        let runtime = tokio::runtime::Builder::new_multi_thread()
+            .thread_name("server")
+            .enable_all()
+            .build()
+            .unwrap();
+        runtime.block_on(async move {
+            engine::set_context_init("binding system signals");
+            let signal = util::os::TerminationSignal::init()?;
+            let (config, global) = tokio::task::spawn_blocking(|| engine::load_all())
+                .await
+                .unwrap()?;
+            let g = global.global.clone();
+            engine::start(signal, config, global).await?;
+            engine::RuntimeResult::Ok(g)
+        })
     };
-    let g = global.global.clone();
-    let runtime = tokio::runtime::Builder::new_multi_thread()
-        .thread_name("server")
-        .enable_all()
-        .build()
-        .unwrap();
-    match runtime.block_on(async move { engine::start(config, global).await }) {
-        Ok(()) => {
+    match run() {
+        Ok(g) => {
+            info!("completing cleanup before exit");
             engine::finish(g);
-            info!("finished all pending tasks. Goodbye!");
+            println!("Goodbye!");
         }
         Err(e) => {
             error!("{e}");

From 0fed52fe8aaee22fdf144abcb2415a1aaa824cab Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 14 Nov 2023 18:08:18 +0530
Subject: [PATCH 278/310] Sync system stores when config changes

---
 server/src/engine/core/dcl.rs                 |  61 +++
 server/src/engine/core/exec.rs                |  67 +++-
 server/src/engine/core/index/row.rs           |   2 +-
 server/src/engine/core/mod.rs                 |  14 +-
 server/src/engine/core/model/delta.rs         |  41 +-
 server/src/engine/data/cell.rs                |  15 +
 server/src/engine/error.rs                    |   2 +
 server/src/engine/fractal/mod.rs              |  13 +-
 .../fractal/{config.rs => sys_store.rs}       | 119 ++++--
 server/src/engine/fractal/test_utils.rs       |  10 +-
 server/src/engine/mod.rs                      |   9 +-
 server/src/engine/net/protocol/mod.rs         |  47 ++-
 server/src/engine/ql/dcl.rs                   |  12 +
 server/src/engine/storage/v1/sysdb.rs         | 350 +++++++++---------
 server/src/engine/storage/v1/tests.rs         |  59 ++-
 15 files changed, 499 insertions(+), 322 deletions(-)
 create mode 100644 server/src/engine/core/dcl.rs
 rename server/src/engine/fractal/{config.rs => sys_store.rs} (69%)

diff --git a/server/src/engine/core/dcl.rs b/server/src/engine/core/dcl.rs
new file mode 100644
index 00000000..127328d0
--- /dev/null
+++ b/server/src/engine/core/dcl.rs
@@ -0,0 +1,61 @@
+/*
+ * Created on Fri Nov 10 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    data::{tag::TagClass, DictEntryGeneric},
+    error::{QueryError, QueryResult},
+    fractal::GlobalInstanceLike,
+    net::protocol::ClientLocalState,
+    ql::dcl::{UserAdd, UserDel},
+};
+
+const KEY_PASSWORD: &str = "password";
+
+pub fn create_user(global: &impl GlobalInstanceLike, mut user_add: UserAdd<'_>) -> QueryResult<()> {
+    let username = user_add.username().to_owned();
+    let password = match user_add.options_mut().remove(KEY_PASSWORD) {
+        Some(DictEntryGeneric::Data(d))
+            if d.kind() == TagClass::Str && user_add.options().is_empty() =>
+        unsafe { d.into_str().unwrap_unchecked() },
+        None | Some(_) => {
+            // invalid properties
+            return Err(QueryError::QExecDdlInvalidProperties);
+        }
+    };
+    global.sys_store().create_new_user(username, password)
+}
+
+pub fn drop_user(
+    global: &impl GlobalInstanceLike,
+    cstate: &ClientLocalState,
+    user_del: UserDel<'_>,
+) -> QueryResult<()> {
+    if cstate.username() == user_del.username() {
+        // you can't delete yourself!
+        return Err(QueryError::SysAuthError);
+    }
+    global.sys_store().drop_user(user_del.username())
+}
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 2ae93770..c5c1104b 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -29,7 +29,7 @@ use {
         core::{dml, model::Model, space::Space},
         error::{QueryError, QueryResult},
         fractal::Global,
-        net::protocol::{Response, SQuery},
+        net::protocol::{ClientLocalState, Response, SQuery},
         ql::{
             ast::{traits::ASTNode, InplaceData, State},
             lex::{Keyword, KeywordStmt, Token},
@@ -38,8 +38,9 @@ use {
     core::ops::Deref,
 };
 
-pub async fn dispatch_to_executor<'a, 'b>(
-    global: &'b Global,
+pub async fn dispatch_to_executor<'a>(
+    global: &Global,
+    cstate: &ClientLocalState,
     query: SQuery<'a>,
 ) -> QueryResult<Response> {
     let tokens =
@@ -52,9 +53,9 @@ pub async fn dispatch_to_executor<'a, 'b>(
     };
     state.cursor_ahead();
     if stmt.is_blocking() {
-        run_blocking_stmt(state, stmt, global).await
+        run_blocking_stmt(global, cstate, state, stmt).await
     } else {
-        run_nb(global, state, stmt)
+        run_nb(global, cstate, state, stmt)
     }
 }
 
@@ -114,10 +115,15 @@ fn _call<A: ASTNode<'static> + core::fmt::Debug, T>(
 }
 
 async fn run_blocking_stmt(
+    global: &Global,
+    cstate: &ClientLocalState,
     mut state: State<'_, InplaceData>,
     stmt: KeywordStmt,
-    global: &Global,
 ) -> Result<Response, QueryError> {
+    if !cstate.is_root() {
+        // all the actions here need root permission
+        return Err(QueryError::SysPermissionDenied);
+    }
     let (a, b) = (&state.current()[0], &state.current()[1]);
     let sysctl = stmt == KeywordStmt::Sysctl;
     let create = stmt == KeywordStmt::Create;
@@ -132,24 +138,26 @@ async fn run_blocking_stmt(
     let d_m = (drop & Token![model].eq(a) & last_id) as u8 * 7;
     let fc = sysctl as u8 | c_s | c_m | a_s | a_m | d_s | d_m;
     state.cursor_ahead();
-    static BLK_EXEC: [fn(Global, RawSlice<Token<'static>>) -> QueryResult<()>; 8] = [
-        |_, _| Err(QueryError::QLUnknownStatement), // unknown
-        blocking_exec_sysctl,                       // sysctl
-        |g, t| call(g, t, Space::transactional_exec_create),
-        |g, t| call(g, t, Model::transactional_exec_create),
-        |g, t| call(g, t, Space::transactional_exec_alter),
-        |g, t| call(g, t, Model::transactional_exec_alter),
-        |g, t| call(g, t, Space::transactional_exec_drop),
-        |g, t| call(g, t, Model::transactional_exec_drop),
+    static BLK_EXEC: [fn(Global, &ClientLocalState, RawSlice<Token<'static>>) -> QueryResult<()>;
+        8] = [
+        |_, _, _| Err(QueryError::QLUnknownStatement), // unknown
+        blocking_exec_sysctl,                          // sysctl
+        |g, _, t| call(g, t, Space::transactional_exec_create),
+        |g, _, t| call(g, t, Model::transactional_exec_create),
+        |g, _, t| call(g, t, Space::transactional_exec_alter),
+        |g, _, t| call(g, t, Model::transactional_exec_alter),
+        |g, _, t| call(g, t, Space::transactional_exec_drop),
+        |g, _, t| call(g, t, Model::transactional_exec_drop),
     ];
     let r = unsafe {
         // UNSAFE(@ohsayan): the only await is within this block
         let c_glob = global.clone();
         let ptr = state.current().as_ptr() as usize;
         let len = state.current().len();
+        let cstate: &'static ClientLocalState = core::mem::transmute(cstate);
         tokio::task::spawn_blocking(move || {
             let tokens = RawSlice::new(ptr as *const Token, len);
-            BLK_EXEC[fc as usize](c_glob, tokens)?;
+            BLK_EXEC[fc as usize](c_glob, cstate, tokens)?;
             Ok(Response::Empty)
         })
         .await
@@ -157,8 +165,30 @@ async fn run_blocking_stmt(
     r.unwrap()
 }
 
-fn blocking_exec_sysctl(_: Global, _: RawSlice<Token<'static>>) -> QueryResult<()> {
-    todo!()
+fn blocking_exec_sysctl(
+    g: Global,
+    cstate: &ClientLocalState,
+    tokens: RawSlice<Token<'static>>,
+) -> QueryResult<()> {
+    let mut state = State::new_inplace(&tokens);
+    /*
+        currently supported: sysctl create user, sysctl drop user
+    */
+    if state.remaining() != 2 {
+        return Err(QueryError::QLInvalidSyntax);
+    }
+    let (a, b) = (state.fw_read(), state.fw_read());
+    match (a, b) {
+        (Token![create], Token::Ident(id)) if id.eq_ignore_ascii_case("user") => {
+            let useradd = ASTNode::from_state(&mut state)?;
+            super::dcl::create_user(&g, useradd)
+        }
+        (Token![drop], Token::Ident(id)) if id.eq_ignore_ascii_case("user") => {
+            let userdel = ASTNode::from_state(&mut state)?;
+            super::dcl::drop_user(&g, cstate, userdel)
+        }
+        _ => Err(QueryError::QLUnknownStatement),
+    }
 }
 
 /*
@@ -167,6 +197,7 @@ fn blocking_exec_sysctl(_: Global, _: RawSlice<Token<'static>>) -> QueryResult<(
 
 fn run_nb(
     global: &Global,
+    _cstate: &ClientLocalState,
     state: State<'_, InplaceData>,
     stmt: KeywordStmt,
 ) -> QueryResult<Response> {
diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
index c2e409bf..c44d8a4f 100644
--- a/server/src/engine/core/index/row.rs
+++ b/server/src/engine/core/index/row.rs
@@ -110,7 +110,7 @@ impl Row {
             data,
             schema_version,
             txn_revised_data,
-            DeltaVersion::__new(0),
+            DeltaVersion::genesis(),
         )
     }
     pub fn new_restored(
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index a3cff237..da929a4f 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -24,22 +24,24 @@
  *
 */
 
-use self::dml::QueryExecMeta;
-pub use self::util::{EntityID, EntityIDRef};
-use super::{fractal::GlobalInstanceLike, ql::ast::Entity};
+pub(in crate::engine) mod dcl;
 pub(in crate::engine) mod dml;
-pub mod exec;
+pub(in crate::engine) mod exec;
 pub(in crate::engine) mod index;
 pub(in crate::engine) mod model;
 pub(in crate::engine) mod query_meta;
-pub mod space;
+pub(in crate::engine) mod space;
+// util
 mod util;
 // test
 #[cfg(test)]
 pub(super) mod tests;
+// re-exports
+pub use self::util::{EntityID, EntityIDRef};
 // imports
 use {
-    self::model::Model,
+    self::{dml::QueryExecMeta, model::Model},
+    super::{fractal::GlobalInstanceLike, ql::ast::Entity},
     crate::engine::{
         core::space::Space,
         error::{QueryError, QueryResult},
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index 91b1434f..a3760b24 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -24,18 +24,13 @@
  *
 */
 
-#![allow(unused)]
-
 use {
     super::{Fields, Model},
-    crate::{
-        engine::{
-            core::{dml::QueryExecMeta, index::Row},
-            fractal::{FractalToken, GlobalInstanceLike},
-            sync::atm::Guard,
-            sync::queue::Queue,
-        },
-        util::compiler,
+    crate::engine::{
+        core::{dml::QueryExecMeta, index::Row},
+        fractal::{FractalToken, GlobalInstanceLike},
+        sync::atm::Guard,
+        sync::queue::Queue,
     },
     parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard},
     std::{
@@ -70,8 +65,8 @@ impl PartialEq for ISyncMatrix {
 #[derive(Debug)]
 /// Read model, write new data
 pub struct IRModelSMData<'a> {
-    rmodel: RwLockReadGuard<'a, ()>,
-    mdata: RwLockReadGuard<'a, ()>,
+    _rmodel: RwLockReadGuard<'a, ()>,
+    _mdata: RwLockReadGuard<'a, ()>,
     fields: &'a Fields,
 }
 
@@ -80,8 +75,8 @@ impl<'a> IRModelSMData<'a> {
         let rmodel = m.sync_matrix().v_priv_model_alter.read();
         let mdata = m.sync_matrix().v_priv_data_new_or_revise.read();
         Self {
-            rmodel,
-            mdata,
+            _rmodel: rmodel,
+            _mdata: mdata,
             fields: unsafe {
                 // UNSAFE(@ohsayan): we already have acquired this resource
                 m._read_fields()
@@ -96,14 +91,14 @@ impl<'a> IRModelSMData<'a> {
 #[derive(Debug)]
 /// Read model
 pub struct IRModel<'a> {
-    rmodel: RwLockReadGuard<'a, ()>,
+    _rmodel: RwLockReadGuard<'a, ()>,
     fields: &'a Fields,
 }
 
 impl<'a> IRModel<'a> {
     pub fn new(m: &'a Model) -> Self {
         Self {
-            rmodel: m.sync_matrix().v_priv_model_alter.read(),
+            _rmodel: m.sync_matrix().v_priv_model_alter.read(),
             fields: unsafe {
                 // UNSAFE(@ohsayan): we already have acquired this resource
                 m._read_fields()
@@ -118,14 +113,14 @@ impl<'a> IRModel<'a> {
 #[derive(Debug)]
 /// Write model
 pub struct IWModel<'a> {
-    wmodel: RwLockWriteGuard<'a, ()>,
+    _wmodel: RwLockWriteGuard<'a, ()>,
     fields: &'a mut Fields,
 }
 
 impl<'a> IWModel<'a> {
     pub fn new(m: &'a Model) -> Self {
         Self {
-            wmodel: m.sync_matrix().v_priv_model_alter.write(),
+            _wmodel: m.sync_matrix().v_priv_model_alter.write(),
             fields: unsafe {
                 // UNSAFE(@ohsayan): we have exclusive access to this resource
                 m._read_fields_mut()
@@ -211,9 +206,6 @@ impl DeltaState {
     pub fn create_new_data_delta_version(&self) -> DeltaVersion {
         DeltaVersion(self.__data_delta_step())
     }
-    pub fn get_data_delta_size(&self) -> usize {
-        self.data_deltas_size.load(Ordering::Acquire)
-    }
 }
 
 impl DeltaState {
@@ -356,7 +348,7 @@ impl<'a> SchemaDeltaIndexRGuard<'a> {
 
 #[derive(Debug, Clone)]
 pub struct DataDelta {
-    schema_version: DeltaVersion,
+    _schema_version: DeltaVersion,
     data_version: DeltaVersion,
     row: Row,
     change: DataDeltaKind,
@@ -370,15 +362,12 @@ impl DataDelta {
         change: DataDeltaKind,
     ) -> Self {
         Self {
-            schema_version,
+            _schema_version: schema_version,
             data_version,
             row,
             change,
         }
     }
-    pub fn schema_version(&self) -> DeltaVersion {
-        self.schema_version
-    }
     pub fn data_version(&self) -> DeltaVersion {
         self.data_version
     }
diff --git a/server/src/engine/data/cell.rs b/server/src/engine/data/cell.rs
index c23ba0a4..f8a3dfd3 100644
--- a/server/src/engine/data/cell.rs
+++ b/server/src/engine/data/cell.rs
@@ -220,6 +220,21 @@ impl Datacell {
     pub fn str(&self) -> &str {
         self.try_str().unwrap()
     }
+    pub fn into_str(self) -> Option<String> {
+        if self.kind() != TagClass::Str {
+            return None;
+        }
+        unsafe {
+            // UNSAFE(@ohsayan): no double free + tagck
+            let md = ManuallyDrop::new(self);
+            let (a, b) = md.data.word.dwordqn_load_qw_nw();
+            Some(String::from_raw_parts(
+                b as *const u8 as *mut u8,
+                a as usize,
+                a as usize,
+            ))
+        }
+    }
     // list
     pub fn new_list(l: Vec<Self>) -> Self {
         unsafe {
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index a9aeae83..6e41fc2d 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -45,6 +45,8 @@ pub enum QueryError {
     SysAuthError = 3,
     /// transactional error
     SysTransactionalError = 4,
+    /// insufficient permissions error
+    SysPermissionDenied = 5,
     // exchange
     NetworkSubsystemCorruptedPacket = 24,
     // QL
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index 6cc37f0f..74975e37 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -25,6 +25,7 @@
 */
 
 use {
+    self::sys_store::SystemStore,
     super::{
         core::{dml::QueryExecMeta, model::Model, GlobalNS},
         data::uuid::Uuid,
@@ -40,11 +41,11 @@ use {
     tokio::sync::mpsc::unbounded_channel,
 };
 
-pub mod config;
 pub mod context;
 mod drivers;
 pub mod error;
 mod mgr;
+pub mod sys_store;
 #[cfg(test)]
 pub mod test_utils;
 mod util;
@@ -74,7 +75,7 @@ pub struct GlobalStateStart {
 /// Must be called iff this is the only thread calling it
 pub unsafe fn load_and_enable_all(
     gns: GlobalNS,
-    config: config::SysConfig,
+    config: SystemStore<LocalFS>,
     gns_driver: GNSTransactionDriverAnyFS<LocalFS>,
     model_drivers: ModelDrivers<LocalFS>,
 ) -> GlobalStateStart {
@@ -145,7 +146,7 @@ pub trait GlobalInstanceLike {
         }
     }
     // config handle
-    fn sys_cfg(&self) -> &config::SysConfig;
+    fn sys_store(&self) -> &SystemStore<Self::FileSystem>;
 }
 
 impl GlobalInstanceLike for Global {
@@ -169,7 +170,7 @@ impl GlobalInstanceLike for Global {
         self._get_max_delta_size()
     }
     // sys
-    fn sys_cfg(&self) -> &config::SysConfig {
+    fn sys_store(&self) -> &SystemStore<Self::FileSystem> {
         &self.get_state().config
     }
     // model
@@ -264,7 +265,7 @@ struct GlobalState {
     gns_driver: drivers::FractalGNSDriver<LocalFS>,
     mdl_driver: RwLock<ModelDrivers<LocalFS>>,
     task_mgr: mgr::FractalMgr,
-    config: config::SysConfig,
+    config: SystemStore<LocalFS>,
 }
 
 impl GlobalState {
@@ -273,7 +274,7 @@ impl GlobalState {
         gns_driver: drivers::FractalGNSDriver<LocalFS>,
         mdl_driver: RwLock<ModelDrivers<LocalFS>>,
         task_mgr: mgr::FractalMgr,
-        config: config::SysConfig,
+        config: SystemStore<LocalFS>,
     ) -> Self {
         Self {
             gns,
diff --git a/server/src/engine/fractal/config.rs b/server/src/engine/fractal/sys_store.rs
similarity index 69%
rename from server/src/engine/fractal/config.rs
rename to server/src/engine/fractal/sys_store.rs
index 699b401a..625285d0 100644
--- a/server/src/engine/fractal/config.rs
+++ b/server/src/engine/fractal/sys_store.rs
@@ -24,17 +24,31 @@
  *
 */
 
-use crate::engine::config::ConfigAuth;
-
 use {
     crate::engine::{
-        config::ConfigMode,
+        config::{ConfigAuth, ConfigMode},
         error::{QueryError, QueryResult},
+        storage::v1::RawFSInterface,
     },
     parking_lot::RwLock,
-    std::collections::{hash_map::Entry, HashMap},
+    std::{
+        collections::{hash_map::Entry, HashMap},
+        marker::PhantomData,
+    },
 };
 
+#[derive(Debug)]
+pub struct SystemStore<Fs> {
+    syscfg: SysConfig,
+    _fs: PhantomData<Fs>,
+}
+
+impl<Fs> SystemStore<Fs> {
+    pub fn system_store(&self) -> &SysConfig {
+        &self.syscfg
+    }
+}
+
 #[derive(Debug)]
 /// The global system configuration
 pub struct SysConfig {
@@ -63,10 +77,10 @@ impl SysConfig {
     pub fn new_full(new_auth: ConfigAuth, host_data: SysHostData, run_mode: ConfigMode) -> Self {
         Self::new(
             RwLock::new(SysAuth::new(
-                rcrypt::hash(new_auth.root_key, rcrypt::DEFAULT_COST)
+                into_dict!(SysAuthUser::USER_ROOT => SysAuthUser::new(
+                rcrypt::hash(new_auth.root_key.as_str(), rcrypt::DEFAULT_COST)
                     .unwrap()
-                    .into_boxed_slice(),
-                Default::default(),
+                    .into_boxed_slice())),
             )),
             host_data,
             run_mode,
@@ -80,10 +94,10 @@ impl SysConfig {
     pub(super) fn test_default() -> Self {
         Self {
             auth_data: RwLock::new(SysAuth::new(
+                into_dict!(SysAuthUser::USER_ROOT => SysAuthUser::new(
                 rcrypt::hash("password12345678", rcrypt::DEFAULT_COST)
                     .unwrap()
-                    .into_boxed_slice(),
-                Default::default(),
+                    .into_boxed_slice())),
             )),
             host_data: SysHostData::new(0, 0),
             run_mode: ConfigMode::Dev,
@@ -133,6 +147,51 @@ impl SysHostData {
     }
 }
 
+impl<Fs: RawFSInterface> SystemStore<Fs> {
+    pub fn _new(syscfg: SysConfig) -> Self {
+        Self {
+            syscfg,
+            _fs: PhantomData,
+        }
+    }
+    /// Create a new user with the given details
+    pub fn create_new_user(&self, username: String, password: String) -> QueryResult<()> {
+        // TODO(@ohsayan): we want to be very careful with this
+        let _username = username.clone();
+        let mut auth = self.system_store().auth_data().write();
+        match auth.users.entry(username.into()) {
+            Entry::Vacant(ve) => {
+                ve.insert(SysAuthUser::new(
+                    rcrypt::hash(password, rcrypt::DEFAULT_COST)
+                        .unwrap()
+                        .into_boxed_slice(),
+                ));
+                self.sync_db_or_rollback(|| {
+                    auth.users.remove(_username.as_str());
+                })?;
+                Ok(())
+            }
+            Entry::Occupied(_) => Err(QueryError::SysAuthError),
+        }
+    }
+    pub fn drop_user(&self, username: &str) -> QueryResult<()> {
+        let mut auth = self.system_store().auth_data().write();
+        if username == SysAuthUser::USER_ROOT {
+            // you can't remove root!
+            return Err(QueryError::SysAuthError);
+        }
+        match auth.users.remove_entry(username) {
+            Some((username, user)) => {
+                self.sync_db_or_rollback(|| {
+                    let _ = auth.users.insert(username, user);
+                })?;
+                Ok(())
+            }
+            None => Err(QueryError::SysAuthError),
+        }
+    }
+}
+
 /*
     auth
 */
@@ -140,28 +199,24 @@ impl SysHostData {
 #[derive(Debug, PartialEq)]
 /// The auth data section (system.auth)
 pub struct SysAuth {
-    root_key: Box<[u8]>,
     users: HashMap<Box<str>, SysAuthUser>,
 }
 
 impl SysAuth {
     /// New [`SysAuth`] with the given settings
-    pub fn new(root_key: Box<[u8]>, users: HashMap<Box<str>, SysAuthUser>) -> Self {
-        Self { root_key, users }
+    pub fn new(users: HashMap<Box<str>, SysAuthUser>) -> Self {
+        Self { users }
     }
-    /// Create a new user with the given details
-    #[allow(unused)]
-    pub fn create_new_user(&mut self, username: &str, password: &str) -> QueryResult<()> {
-        match self.users.entry(username.into()) {
-            Entry::Vacant(ve) => {
-                ve.insert(SysAuthUser::new(
-                    rcrypt::hash(password, rcrypt::DEFAULT_COST)
-                        .unwrap()
-                        .into_boxed_slice(),
-                ));
-                Ok(())
+    pub fn verify_user_is_root<T: AsRef<[u8]> + ?Sized>(
+        &self,
+        username: &str,
+        password: &T,
+    ) -> QueryResult<bool> {
+        match self.users.get(username) {
+            Some(user) if rcrypt::verify(password, user.key()).unwrap() => {
+                Ok(username == SysAuthUser::USER_ROOT)
             }
-            Entry::Occupied(_) => Err(QueryError::SysAuthError),
+            Some(_) | None => Err(QueryError::SysAuthError),
         }
     }
     /// Verify the user with the given details
@@ -170,20 +225,7 @@ impl SysAuth {
         username: &str,
         password: &T,
     ) -> QueryResult<()> {
-        if username == "root" {
-            if rcrypt::verify(password, self.root_key()).unwrap() {
-                return Ok(());
-            } else {
-                return Err(QueryError::SysAuthError);
-            }
-        }
-        match self.users.get(username) {
-            Some(user) if rcrypt::verify(password, user.key()).unwrap() => Ok(()),
-            Some(_) | None => Err(QueryError::SysAuthError),
-        }
-    }
-    pub fn root_key(&self) -> &[u8] {
-        &self.root_key
+        self.verify_user_is_root(username, password).map(|_| ())
     }
     pub fn users(&self) -> &HashMap<Box<str>, SysAuthUser> {
         &self.users
@@ -197,6 +239,7 @@ pub struct SysAuthUser {
 }
 
 impl SysAuthUser {
+    pub const USER_ROOT: &'static str = "root";
     /// Create a new [`SysAuthUser`]
     pub fn new(key: Box<[u8]>) -> Self {
         Self { key }
diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
index d3211ee9..504a5773 100644
--- a/server/src/engine/fractal/test_utils.rs
+++ b/server/src/engine/fractal/test_utils.rs
@@ -26,8 +26,8 @@
 
 use {
     super::{
-        config::SysConfig, CriticalTask, FractalModelDriver, GenericTask, GlobalInstanceLike,
-        ModelUniqueID, Task,
+        sys_store::{SysConfig, SystemStore},
+        CriticalTask, FractalModelDriver, GenericTask, GlobalInstanceLike, ModelUniqueID, Task,
     },
     crate::engine::{
         core::GlobalNS,
@@ -54,7 +54,7 @@ pub struct TestGlobal<Fs: RawFSInterface = VirtualFS> {
     max_delta_size: usize,
     txn_driver: Mutex<GNSTransactionDriverAnyFS<Fs>>,
     model_drivers: RwLock<HashMap<ModelUniqueID, FractalModelDriver<Fs>>>,
-    sys_cfg: SysConfig,
+    sys_cfg: SystemStore<Fs>,
 }
 
 impl<Fs: RawFSInterface> TestGlobal<Fs> {
@@ -70,7 +70,7 @@ impl<Fs: RawFSInterface> TestGlobal<Fs> {
             max_delta_size,
             txn_driver: Mutex::new(txn_driver),
             model_drivers: RwLock::default(),
-            sys_cfg: SysConfig::test_default(),
+            sys_cfg: SystemStore::_new(SysConfig::test_default()),
         }
     }
 }
@@ -117,7 +117,7 @@ impl<Fs: RawFSInterface> GlobalInstanceLike for TestGlobal<Fs> {
     fn get_max_delta_size(&self) -> usize {
         100
     }
-    fn sys_cfg(&self) -> &super::config::SysConfig {
+    fn sys_store(&self) -> &SystemStore<Fs> {
         &self.sys_cfg
     }
     fn initialize_model_driver(
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index 57fd0fb5..bb8f7788 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -47,10 +47,12 @@ pub use error::RuntimeResult;
 use {
     self::{
         config::{ConfigEndpoint, ConfigEndpointTls, ConfigMode, ConfigReturn, Configuration},
-        fractal::context::{self, Subsystem},
+        fractal::{
+            context::{self, Subsystem},
+            sys_store::SystemStore,
+        },
         storage::v1::{
             loader::{self, SEInitState},
-            sysdb::{self, SystemStoreInit},
             LocalFS,
         },
     },
@@ -83,8 +85,7 @@ pub fn load_all() -> RuntimeResult<(Configuration, fractal::GlobalStateStart)> {
     // restore system database
     info!("loading system database ...");
     context::set_dmsg("loading system database");
-    let SystemStoreInit { store, state } =
-        sysdb::open_system_database::<LocalFS>(config.auth.clone(), config.mode)?;
+    let (store, state) = SystemStore::<LocalFS>::open_or_restore(config.auth.clone(), config.mode)?;
     let sysdb_is_new = state.is_created();
     if state.is_existing_updated_root() {
         warn!("the root account was updated");
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index cb3cdcaf..a553d5db 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -51,6 +51,25 @@ use {
     tokio::io::{AsyncReadExt, AsyncWriteExt, BufWriter},
 };
 
+#[derive(Debug, PartialEq)]
+pub struct ClientLocalState {
+    username: Box<str>,
+    root: bool,
+    hs: handshake::CHandshakeStatic,
+}
+
+impl ClientLocalState {
+    pub fn new(username: Box<str>, root: bool, hs: handshake::CHandshakeStatic) -> Self {
+        Self { username, root, hs }
+    }
+    pub fn is_root(&self) -> bool {
+        self.root
+    }
+    pub fn username(&self) -> &str {
+        &self.username
+    }
+}
+
 #[derive(Debug, PartialEq)]
 pub enum Response {
     Empty,
@@ -63,7 +82,7 @@ pub(super) async fn query_loop<S: Socket>(
     global: &Global,
 ) -> IoResult<QueryLoopResult> {
     // handshake
-    let _ = match do_handshake(con, buf, global).await? {
+    let client_state = match do_handshake(con, buf, global).await? {
         PostHandshake::Okay(hs) => hs,
         PostHandshake::ConnectionClosedFin => return Ok(QueryLoopResult::Fin),
         PostHandshake::ConnectionClosedRst => return Ok(QueryLoopResult::Rst),
@@ -116,7 +135,7 @@ pub(super) async fn query_loop<S: Socket>(
             }
         };
         // now execute query
-        match engine::core::exec::dispatch_to_executor(global, sq).await {
+        match engine::core::exec::dispatch_to_executor(global, &client_state, sq).await {
             Ok(Response::Empty) => {
                 con.write_all(&[0x12]).await?;
             }
@@ -137,7 +156,7 @@ pub(super) async fn query_loop<S: Socket>(
 
 #[derive(Debug, PartialEq)]
 enum PostHandshake {
-    Okay(handshake::CHandshakeStatic),
+    Okay(ClientLocalState),
     Error(ProtocolError),
     ConnectionClosedFin,
     ConnectionClosedRst,
@@ -197,14 +216,20 @@ async fn do_handshake<S: Socket>(
     }
     match core::str::from_utf8(handshake.hs_auth().username()) {
         Ok(uname) => {
-            let auth = global.sys_cfg().auth_data().read();
-            if auth
-                .verify_user(uname, handshake.hs_auth().password())
-                .is_ok()
-            {
-                let hs_static = handshake.hs_static();
-                buf.advance(cursor);
-                return Ok(PostHandshake::Okay(hs_static));
+            let auth = global.sys_store().system_store().auth_data().read();
+            let r = auth.verify_user_is_root(uname, handshake.hs_auth().password());
+            match r {
+                Ok(is_root) => {
+                    let hs = handshake.hs_static();
+                    let ret = Ok(PostHandshake::Okay(ClientLocalState::new(
+                        uname.into(),
+                        is_root,
+                        hs,
+                    )));
+                    buf.advance(cursor);
+                    return ret;
+                }
+                Err(_) => {}
             }
         }
         Err(_) => {}
diff --git a/server/src/engine/ql/dcl.rs b/server/src/engine/ql/dcl.rs
index 630e2f42..f624bab6 100644
--- a/server/src/engine/ql/dcl.rs
+++ b/server/src/engine/ql/dcl.rs
@@ -95,6 +95,15 @@ impl<'a> UserAdd<'a> {
     pub fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         parse(state).map(|UserMeta { username, options }: UserMeta| Self::new(username, options))
     }
+    pub fn username(&self) -> &str {
+        self.username
+    }
+    pub fn options_mut(&mut self) -> &mut DictGeneric {
+        &mut self.options
+    }
+    pub fn options(&self) -> &DictGeneric {
+        &self.options
+    }
 }
 
 impl<'a> traits::ASTNode<'a> for UserAdd<'a> {
@@ -131,6 +140,9 @@ impl<'a> UserDel<'a> {
         }
         Err(QueryError::QLInvalidSyntax)
     }
+    pub fn username(&self) -> &str {
+        self.username
+    }
 }
 
 impl<'a> traits::ASTNode<'a> for UserDel<'a> {
diff --git a/server/src/engine/storage/v1/sysdb.rs b/server/src/engine/storage/v1/sysdb.rs
index 71703c3d..12a8c63e 100644
--- a/server/src/engine/storage/v1/sysdb.rs
+++ b/server/src/engine/storage/v1/sysdb.rs
@@ -30,22 +30,13 @@ use {
         config::{ConfigAuth, ConfigMode},
         data::{cell::Datacell, DictEntryGeneric, DictGeneric},
         error::{RuntimeResult, StorageError},
-        fractal::config::{SysAuth, SysAuthUser, SysConfig, SysHostData},
+        fractal::sys_store::{SysAuth, SysAuthUser, SysConfig, SysHostData, SystemStore},
         storage::v1::{inf, spec, RawFSInterface, SDSSFileIO},
     },
     parking_lot::RwLock,
     std::collections::HashMap,
 };
 
-const SYSDB_PATH: &str = "sys.db";
-const SYSDB_COW_PATH: &str = "sys.db.cow";
-const SYS_KEY_AUTH: &str = "auth";
-const SYS_KEY_AUTH_ROOT: &str = "root";
-const SYS_KEY_AUTH_USERS: &str = "users";
-const SYS_KEY_SYS: &str = "sys";
-const SYS_KEY_SYS_STARTUP_COUNTER: &str = "sc";
-const SYS_KEY_SYS_SETTINGS_VERSION: &str = "sv";
-
 #[derive(Debug, PartialEq)]
 /// The system store init state
 pub enum SystemStoreInitState {
@@ -66,123 +57,6 @@ impl SystemStoreInitState {
     }
 }
 
-#[derive(Debug, PartialEq)]
-/// Result of initializing the system store (sysdb)
-pub struct SystemStoreInit {
-    pub store: SysConfig,
-    pub state: SystemStoreInitState,
-}
-
-impl SystemStoreInit {
-    pub fn new(store: SysConfig, state: SystemStoreInitState) -> Self {
-        Self { store, state }
-    }
-}
-
-/// Open the system database
-///
-/// - If it doesn't exist, create it
-/// - If it exists, look for config changes and sync them
-pub fn open_system_database<Fs: RawFSInterface>(
-    auth: ConfigAuth,
-    mode: ConfigMode,
-) -> RuntimeResult<SystemStoreInit> {
-    open_or_reinit_system_database::<Fs>(auth, mode, SYSDB_PATH, SYSDB_COW_PATH)
-}
-
-/// Open or re-initialize the system database
-pub fn open_or_reinit_system_database<Fs: RawFSInterface>(
-    auth: ConfigAuth,
-    run_mode: ConfigMode,
-    sysdb_path: &str,
-    sysdb_path_cow: &str,
-) -> RuntimeResult<SystemStoreInit> {
-    let sysdb_file = match SDSSFileIO::<Fs>::open_or_create_perm_rw::<spec::SysDBV1>(sysdb_path)? {
-        FileOpen::Created(new) => {
-            // init new syscfg
-            let new_syscfg = SysConfig::new_auth(auth, run_mode);
-            sync_system_database_to(&new_syscfg, new)?;
-            return Ok(SystemStoreInit::new(
-                new_syscfg,
-                SystemStoreInitState::Created,
-            ));
-        }
-        FileOpen::Existing((ex, _)) => ex,
-    };
-    let prev_sysdb = decode_system_database(sysdb_file, run_mode)?;
-    let state;
-    // see if settings have changed
-    if prev_sysdb
-        .auth_data()
-        .read()
-        .verify_user("root", &auth.root_key)
-        .is_ok()
-    {
-        state = SystemStoreInitState::Unchanged;
-    } else {
-        state = SystemStoreInitState::UpdatedRoot;
-    }
-    // create new config
-    let new_syscfg = SysConfig::new_full(
-        auth,
-        SysHostData::new(
-            prev_sysdb.host_data().startup_counter() + 1,
-            prev_sysdb.host_data().settings_version()
-                + !matches!(state, SystemStoreInitState::Unchanged) as u32,
-        ),
-        run_mode,
-    );
-    // sync
-    sync_system_database_to(
-        &new_syscfg,
-        SDSSFileIO::<Fs>::create::<spec::SysDBV1>(sysdb_path_cow)?,
-    )?;
-    Fs::fs_rename_file(sysdb_path_cow, sysdb_path)?;
-    Ok(SystemStoreInit::new(new_syscfg, state))
-}
-
-/// Sync the system database to the given file
-pub fn sync_system_database_to<Fs: RawFSInterface>(
-    cfg: &SysConfig,
-    mut f: SDSSFileIO<Fs>,
-) -> RuntimeResult<()> {
-    // prepare our flat file
-    let mut map: DictGeneric = into_dict!(
-        SYS_KEY_SYS => DictEntryGeneric::Map(into_dict!(
-            SYS_KEY_SYS_SETTINGS_VERSION => Datacell::new_uint_default(cfg.host_data().settings_version() as _),
-            SYS_KEY_SYS_STARTUP_COUNTER => Datacell::new_uint_default(cfg.host_data().startup_counter() as _),
-        )),
-        SYS_KEY_AUTH => DictGeneric::new(),
-    );
-    let auth_key = map.get_mut(SYS_KEY_AUTH).unwrap();
-    let auth = cfg.auth_data().read();
-    let auth_key = auth_key.as_dict_mut().unwrap();
-    auth_key.insert(
-        SYS_KEY_AUTH_ROOT.into(),
-        DictEntryGeneric::Data(Datacell::new_bin(auth.root_key().into())),
-    );
-    auth_key.insert(
-        SYS_KEY_AUTH_USERS.into(),
-        DictEntryGeneric::Map(
-            // username -> [..settings]
-            auth.users()
-                .iter()
-                .map(|(username, user)| {
-                    (
-                        username.to_owned(),
-                        DictEntryGeneric::Data(Datacell::new_list(vec![Datacell::new_bin(
-                            user.key().into(),
-                        )])),
-                    )
-                })
-                .collect(),
-        ),
-    );
-    // write
-    let buf = super::inf::enc::enc_dict_full::<super::inf::map::GenericDictSpec>(&map);
-    f.fsynced_write(&buf)
-}
-
 fn rkey<T>(
     d: &mut DictGeneric,
     key: &str,
@@ -194,55 +68,181 @@ fn rkey<T>(
     }
 }
 
-/// Decode the system database
-pub fn decode_system_database<Fs: RawFSInterface>(
-    mut f: SDSSFileIO<Fs>,
-    run_mode: ConfigMode,
-) -> RuntimeResult<SysConfig> {
-    let mut sysdb_data =
-        inf::dec::dec_dict_full::<inf::map::GenericDictSpec>(&f.load_remaining_into_buffer()?)?;
-    // get our auth and sys stores
-    let mut auth_store = rkey(&mut sysdb_data, SYS_KEY_AUTH, DictEntryGeneric::into_dict)?;
-    let mut sys_store = rkey(&mut sysdb_data, SYS_KEY_SYS, DictEntryGeneric::into_dict)?;
-    // load auth store
-    let root_key = rkey(&mut auth_store, SYS_KEY_AUTH_ROOT, |d| {
-        d.into_data()?.into_bin()
-    })?;
-    let users = rkey(
-        &mut auth_store,
-        SYS_KEY_AUTH_USERS,
-        DictEntryGeneric::into_dict,
-    )?;
-    // load users
-    let mut loaded_users = HashMap::new();
-    for (username, userdata) in users {
-        let mut userdata = userdata
-            .into_data()
-            .and_then(Datacell::into_list)
-            .ok_or(StorageError::SysDBCorrupted)?;
-        if userdata.len() != 1 {
-            return Err(StorageError::SysDBCorrupted.into());
+impl<Fs: RawFSInterface> SystemStore<Fs> {
+    const SYSDB_PATH: &'static str = "sys.db";
+    const SYSDB_COW_PATH: &'static str = "sys.db.cow";
+    const SYS_KEY_AUTH: &'static str = "auth";
+    const SYS_KEY_AUTH_USERS: &'static str = "users";
+    const SYS_KEY_SYS: &'static str = "sys";
+    const SYS_KEY_SYS_STARTUP_COUNTER: &'static str = "sc";
+    const SYS_KEY_SYS_SETTINGS_VERSION: &'static str = "sv";
+    pub fn open_or_restore(
+        auth: ConfigAuth,
+        run_mode: ConfigMode,
+    ) -> RuntimeResult<(Self, SystemStoreInitState)> {
+        Self::open_with_name(Self::SYSDB_PATH, Self::SYSDB_COW_PATH, auth, run_mode)
+    }
+    pub fn sync_db_or_rollback(&self, rb: impl FnOnce()) -> RuntimeResult<()> {
+        match self.sync_db() {
+            Ok(()) => Ok(()),
+            Err(e) => {
+                rb();
+                Err(e)
+            }
+        }
+    }
+    pub fn sync_db(&self) -> RuntimeResult<()> {
+        self._sync_with(Self::SYSDB_PATH, Self::SYSDB_COW_PATH)
+    }
+    pub fn open_with_name(
+        sysdb_name: &str,
+        sysdb_cow_path: &str,
+        auth: ConfigAuth,
+        run_mode: ConfigMode,
+    ) -> RuntimeResult<(Self, SystemStoreInitState)> {
+        match SDSSFileIO::open_or_create_perm_rw::<spec::SysDBV1>(sysdb_name)? {
+            FileOpen::Created(new) => {
+                let me = Self::_new(SysConfig::new_auth(auth, run_mode));
+                me._sync(new)?;
+                Ok((me, SystemStoreInitState::Created))
+            }
+            FileOpen::Existing((ex, _)) => {
+                Self::restore_and_sync(ex, auth, run_mode, sysdb_name, sysdb_cow_path)
+            }
+        }
+    }
+}
+
+impl<Fs: RawFSInterface> SystemStore<Fs> {
+    fn _sync(&self, mut f: SDSSFileIO<Fs>) -> RuntimeResult<()> {
+        let cfg = self.system_store();
+        // prepare our flat file
+        let mut map: DictGeneric = into_dict!(
+            Self::SYS_KEY_SYS => DictEntryGeneric::Map(into_dict!(
+                Self::SYS_KEY_SYS_SETTINGS_VERSION => Datacell::new_uint_default(cfg.host_data().settings_version() as _),
+                Self::SYS_KEY_SYS_STARTUP_COUNTER => Datacell::new_uint_default(cfg.host_data().startup_counter() as _),
+            )),
+            Self::SYS_KEY_AUTH => DictGeneric::new(),
+        );
+        let auth_key = map.get_mut(Self::SYS_KEY_AUTH).unwrap();
+        let auth = cfg.auth_data().read();
+        let auth_key = auth_key.as_dict_mut().unwrap();
+        auth_key.insert(
+            Self::SYS_KEY_AUTH_USERS.into(),
+            DictEntryGeneric::Map(
+                // username -> [..settings]
+                auth.users()
+                    .iter()
+                    .map(|(username, user)| {
+                        (
+                            username.to_owned(),
+                            DictEntryGeneric::Data(Datacell::new_list(vec![Datacell::new_bin(
+                                user.key().into(),
+                            )])),
+                        )
+                    })
+                    .collect(),
+            ),
+        );
+        // write
+        let buf = super::inf::enc::enc_dict_full::<super::inf::map::GenericDictSpec>(&map);
+        f.fsynced_write(&buf)
+    }
+    fn _sync_with(&self, target: &str, cow: &str) -> RuntimeResult<()> {
+        let f = SDSSFileIO::create::<spec::SysDBV1>(cow)?;
+        self._sync(f)?;
+        Fs::fs_rename_file(cow, target)
+    }
+    fn restore_and_sync(
+        f: SDSSFileIO<Fs>,
+        auth: ConfigAuth,
+        run_mode: ConfigMode,
+        fname: &str,
+        fcow_name: &str,
+    ) -> RuntimeResult<(Self, SystemStoreInitState)> {
+        let prev_sysdb = Self::_restore(f, run_mode)?;
+        let state;
+        // see if settings have changed
+        if prev_sysdb
+            .auth_data()
+            .read()
+            .verify_user(SysAuthUser::USER_ROOT, &auth.root_key)
+            .is_ok()
+        {
+            state = SystemStoreInitState::Unchanged;
+        } else {
+            state = SystemStoreInitState::UpdatedRoot;
         }
-        let user_password = userdata
-            .remove(0)
-            .into_bin()
-            .ok_or(StorageError::SysDBCorrupted)?;
-        loaded_users.insert(username, SysAuthUser::new(user_password.into_boxed_slice()));
+        // create new config
+        let new_syscfg = SysConfig::new_full(
+            auth,
+            SysHostData::new(
+                prev_sysdb.host_data().startup_counter() + 1,
+                prev_sysdb.host_data().settings_version()
+                    + !matches!(state, SystemStoreInitState::Unchanged) as u32,
+            ),
+            run_mode,
+        );
+        let slf = Self::_new(new_syscfg);
+        // now sync
+        slf._sync_with(fname, fcow_name)?;
+        Ok((slf, state))
     }
-    let sys_auth = SysAuth::new(root_key.into_boxed_slice(), loaded_users);
-    // load sys data
-    let sc = rkey(&mut sys_store, SYS_KEY_SYS_STARTUP_COUNTER, |d| {
-        d.into_data()?.into_uint()
-    })?;
-    let sv = rkey(&mut sys_store, SYS_KEY_SYS_SETTINGS_VERSION, |d| {
-        d.into_data()?.into_uint()
-    })?;
-    if !(sysdb_data.is_empty() & auth_store.is_empty() & sys_store.is_empty()) {
-        return Err(StorageError::SysDBCorrupted.into());
+    fn _restore(mut f: SDSSFileIO<Fs>, run_mode: ConfigMode) -> RuntimeResult<SysConfig> {
+        let mut sysdb_data =
+            inf::dec::dec_dict_full::<inf::map::GenericDictSpec>(&f.load_remaining_into_buffer()?)?;
+        // get our auth and sys stores
+        let mut auth_store = rkey(
+            &mut sysdb_data,
+            Self::SYS_KEY_AUTH,
+            DictEntryGeneric::into_dict,
+        )?;
+        let mut sys_store = rkey(
+            &mut sysdb_data,
+            Self::SYS_KEY_SYS,
+            DictEntryGeneric::into_dict,
+        )?;
+        // load auth store
+        let users = rkey(
+            &mut auth_store,
+            Self::SYS_KEY_AUTH_USERS,
+            DictEntryGeneric::into_dict,
+        )?;
+        // load users
+        let mut loaded_users = HashMap::new();
+        for (username, userdata) in users {
+            let mut userdata = userdata
+                .into_data()
+                .and_then(Datacell::into_list)
+                .ok_or(StorageError::SysDBCorrupted)?;
+            if userdata.len() != 1 {
+                return Err(StorageError::SysDBCorrupted.into());
+            }
+            let user_password = userdata
+                .remove(0)
+                .into_bin()
+                .ok_or(StorageError::SysDBCorrupted)?;
+            loaded_users.insert(username, SysAuthUser::new(user_password.into_boxed_slice()));
+        }
+        let sys_auth = SysAuth::new(loaded_users);
+        // load sys data
+        let sc = rkey(&mut sys_store, Self::SYS_KEY_SYS_STARTUP_COUNTER, |d| {
+            d.into_data()?.into_uint()
+        })?;
+        let sv = rkey(&mut sys_store, Self::SYS_KEY_SYS_SETTINGS_VERSION, |d| {
+            d.into_data()?.into_uint()
+        })?;
+        if !(sysdb_data.is_empty()
+            & auth_store.is_empty()
+            & sys_store.is_empty()
+            & sys_auth.users().contains_key(SysAuthUser::USER_ROOT))
+        {
+            return Err(StorageError::SysDBCorrupted.into());
+        }
+        Ok(SysConfig::new(
+            RwLock::new(sys_auth),
+            SysHostData::new(sc, sv as u32),
+            run_mode,
+        ))
     }
-    Ok(SysConfig::new(
-        RwLock::new(sys_auth),
-        SysHostData::new(sc, sv as u32),
-        run_mode,
-    ))
 }
diff --git a/server/src/engine/storage/v1/tests.rs b/server/src/engine/storage/v1/tests.rs
index c0d5fe7e..49700f1b 100644
--- a/server/src/engine/storage/v1/tests.rs
+++ b/server/src/engine/storage/v1/tests.rs
@@ -32,24 +32,19 @@ mod tx;
 
 mod sysdb {
     use {
-        super::{
-            super::sysdb::{self, SystemStoreInitState},
-            VirtualFS as VFS,
+        super::{super::sysdb::SystemStoreInitState, VirtualFS as VFS},
+        crate::engine::{
+            config::{AuthDriver, ConfigAuth, ConfigMode},
+            fractal::sys_store::SystemStore,
         },
-        crate::engine::config::{AuthDriver, ConfigAuth, ConfigMode},
     };
     fn open_sysdb(
         auth_config: ConfigAuth,
         sysdb_path: &str,
         sysdb_cow_path: &str,
-    ) -> sysdb::SystemStoreInit {
-        sysdb::open_or_reinit_system_database::<VFS>(
-            auth_config,
-            ConfigMode::Dev,
-            sysdb_path,
-            sysdb_cow_path,
-        )
-        .unwrap()
+    ) -> (SystemStore<VFS>, SystemStoreInitState) {
+        SystemStore::<VFS>::open_with_name(sysdb_path, sysdb_cow_path, auth_config, ConfigMode::Dev)
+            .unwrap()
     }
     #[test]
     fn open_close() {
@@ -62,28 +57,28 @@ mod sysdb {
         };
         let auth_config = ConfigAuth::new(AuthDriver::Pwd, "password12345678".into());
         {
-            let config = open(auth_config.clone());
-            assert_eq!(config.state, SystemStoreInitState::Created);
+            let (config, state) = open(auth_config.clone());
+            assert_eq!(state, SystemStoreInitState::Created);
             assert!(config
-                .store
+                .system_store()
                 .auth_data()
                 .read()
                 .verify_user("root", "password12345678")
                 .is_ok());
-            assert_eq!(config.store.host_data().settings_version(), 0);
-            assert_eq!(config.store.host_data().startup_counter(), 0);
+            assert_eq!(config.system_store().host_data().settings_version(), 0);
+            assert_eq!(config.system_store().host_data().startup_counter(), 0);
         }
         // reboot
-        let config = open(auth_config);
-        assert_eq!(config.state, SystemStoreInitState::Unchanged);
+        let (config, state) = open(auth_config);
+        assert_eq!(state, SystemStoreInitState::Unchanged);
         assert!(config
-            .store
+            .system_store()
             .auth_data()
             .read()
             .verify_user("root", "password12345678")
             .is_ok());
-        assert_eq!(config.store.host_data().settings_version(), 0);
-        assert_eq!(config.store.host_data().startup_counter(), 1);
+        assert_eq!(config.system_store().host_data().settings_version(), 0);
+        assert_eq!(config.system_store().host_data().startup_counter(), 1);
     }
     #[test]
     fn open_change_root_password() {
@@ -95,26 +90,26 @@ mod sysdb {
             )
         };
         {
-            let config = open(ConfigAuth::new(AuthDriver::Pwd, "password12345678".into()));
-            assert_eq!(config.state, SystemStoreInitState::Created);
+            let (config, state) = open(ConfigAuth::new(AuthDriver::Pwd, "password12345678".into()));
+            assert_eq!(state, SystemStoreInitState::Created);
             assert!(config
-                .store
+                .system_store()
                 .auth_data()
                 .read()
                 .verify_user("root", "password12345678")
                 .is_ok());
-            assert_eq!(config.store.host_data().settings_version(), 0);
-            assert_eq!(config.store.host_data().startup_counter(), 0);
+            assert_eq!(config.system_store().host_data().settings_version(), 0);
+            assert_eq!(config.system_store().host_data().startup_counter(), 0);
         }
-        let config = open(ConfigAuth::new(AuthDriver::Pwd, "password23456789".into()));
-        assert_eq!(config.state, SystemStoreInitState::UpdatedRoot);
+        let (config, state) = open(ConfigAuth::new(AuthDriver::Pwd, "password23456789".into()));
+        assert_eq!(state, SystemStoreInitState::UpdatedRoot);
         assert!(config
-            .store
+            .system_store()
             .auth_data()
             .read()
             .verify_user("root", "password23456789")
             .is_ok());
-        assert_eq!(config.store.host_data().settings_version(), 1);
-        assert_eq!(config.store.host_data().startup_counter(), 1);
+        assert_eq!(config.system_store().host_data().settings_version(), 1);
+        assert_eq!(config.system_store().host_data().startup_counter(), 1);
     }
 }

From fff5780586db1f70f7b726ba77a8551874b1b132 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 15 Nov 2023 17:48:10 +0530
Subject: [PATCH 279/310] Remove old tools

---
 .cargo/config.toml                            |   1 -
 .gitignore                                    |   3 +-
 Cargo.lock                                    | 448 ++++--------------
 Cargo.toml                                    |   2 -
 ci/server1.toml                               |   9 -
 ci/server1.yaml                               |  17 +
 ci/server2.toml                               |  12 -
 ci/server3.toml                               |  14 -
 cli/Cargo.toml                                |  11 +-
 cli/src/argparse.rs                           | 283 -----------
 cli/src/cli.yml                               |  55 ---
 cli/src/macros.rs                             | 122 -----
 cli/src/main.rs                               |  22 +-
 cli/src/runner.rs                             | 262 ----------
 cli/src/tests.rs                              | 196 --------
 cli/src/tokenizer.rs                          | 226 ---------
 examples/config-files/badcfg.toml             |   3 -
 examples/config-files/badcfg2.toml            |   8 -
 examples/config-files/bgsave-justenabled.toml |   6 -
 examples/config-files/bgsave-justevery.toml   |   6 -
 examples/config-files/docker.toml             |   5 -
 examples/config-files/ipv6.toml               |   4 -
 examples/config-files/secure-noart.toml       |   4 -
 examples/config-files/skyd.toml               |   5 -
 examples/config-files/snapshot.toml           |  17 -
 examples/config-files/ssl.toml                |  17 -
 examples/config-files/template.toml           |  40 --
 examples/config-files/template.yaml           |  22 +
 examples/config-files/withcustombgsave.toml   |   7 -
 harness/Cargo.toml                            |   4 +-
 harness/src/build.rs                          |   2 +-
 harness/src/test/mod.rs                       |  37 +-
 harness/src/test/svc.rs                       |  31 +-
 pkg/debian/description.txt                    |   4 +-
 server/Cargo.toml                             |   4 +-
 server/src/cli.yml                            | 123 -----
 server/src/engine/config.rs                   |  19 +-
 server/src/engine/core/exec.rs                |   6 +
 server/src/engine/fractal/mgr.rs              |  21 +-
 server/src/engine/fractal/mod.rs              |   2 +-
 server/src/engine/fractal/sys_store.rs        |   4 +-
 server/src/engine/mod.rs                      |   6 +-
 server/src/engine/net/protocol/mod.rs         |   2 +-
 server/src/engine/net/protocol/tests.rs       |   2 +-
 server/src/engine/txn/gns/tests/full_chain.rs |   8 +-
 sky-bench/Cargo.toml                          |  13 +-
 sky-bench/src/bench/benches.rs                | 245 ----------
 sky-bench/src/bench/mod.rs                    | 276 -----------
 sky-bench/src/bench/report.rs                 |  82 ----
 sky-bench/src/bench/validation.rs             |  39 --
 sky-bench/src/cli.yml                         |  72 ---
 sky-bench/src/config.rs                       | 148 ------
 sky-bench/src/error.rs                        |  71 ---
 sky-bench/src/main.rs                         |  42 +-
 sky-bench/src/util.rs                         | 143 ------
 sky-migrate/Cargo.toml                        |  14 -
 sky-migrate/README.md                         |  10 -
 sky-migrate/src/cli.yml                       |  30 --
 sky-migrate/src/main.rs                       | 129 -----
 stress-test/Cargo.toml                        |  21 -
 stress-test/README.md                         |   7 -
 stress-test/src/linearity_client.rs           | 264 -----------
 stress-test/src/main.rs                       |  90 ----
 stress-test/src/utils.rs                      |  62 ---
 64 files changed, 232 insertions(+), 3628 deletions(-)
 delete mode 100644 ci/server1.toml
 create mode 100644 ci/server1.yaml
 delete mode 100644 ci/server2.toml
 delete mode 100644 ci/server3.toml
 delete mode 100644 cli/src/argparse.rs
 delete mode 100644 cli/src/cli.yml
 delete mode 100644 cli/src/macros.rs
 delete mode 100644 cli/src/runner.rs
 delete mode 100644 cli/src/tests.rs
 delete mode 100644 cli/src/tokenizer.rs
 delete mode 100644 examples/config-files/badcfg.toml
 delete mode 100644 examples/config-files/badcfg2.toml
 delete mode 100644 examples/config-files/bgsave-justenabled.toml
 delete mode 100644 examples/config-files/bgsave-justevery.toml
 delete mode 100644 examples/config-files/docker.toml
 delete mode 100644 examples/config-files/ipv6.toml
 delete mode 100644 examples/config-files/secure-noart.toml
 delete mode 100644 examples/config-files/skyd.toml
 delete mode 100644 examples/config-files/snapshot.toml
 delete mode 100644 examples/config-files/ssl.toml
 delete mode 100644 examples/config-files/template.toml
 create mode 100644 examples/config-files/template.yaml
 delete mode 100644 examples/config-files/withcustombgsave.toml
 delete mode 100644 server/src/cli.yml
 delete mode 100644 sky-bench/src/bench/benches.rs
 delete mode 100644 sky-bench/src/bench/mod.rs
 delete mode 100644 sky-bench/src/bench/report.rs
 delete mode 100644 sky-bench/src/bench/validation.rs
 delete mode 100644 sky-bench/src/cli.yml
 delete mode 100644 sky-bench/src/config.rs
 delete mode 100644 sky-bench/src/error.rs
 delete mode 100644 sky-bench/src/util.rs
 delete mode 100644 sky-migrate/Cargo.toml
 delete mode 100644 sky-migrate/README.md
 delete mode 100644 sky-migrate/src/cli.yml
 delete mode 100644 sky-migrate/src/main.rs
 delete mode 100644 stress-test/Cargo.toml
 delete mode 100644 stress-test/README.md
 delete mode 100644 stress-test/src/linearity_client.rs
 delete mode 100644 stress-test/src/main.rs
 delete mode 100644 stress-test/src/utils.rs

diff --git a/.cargo/config.toml b/.cargo/config.toml
index ac33e0c8..7c9f7c37 100644
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@@ -1,3 +1,2 @@
 [env]
 ROOT_DIR = { value = "", relative = true }
-TEST_ORIGIN_KEY = "4527387f92a381cbe804593f33991d327d456a97"
diff --git a/.gitignore b/.gitignore
index 79cf9637..a40afb49 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,4 +14,5 @@ snapstore.partmap
 .devcontainer
 *.deb
 .skytest_*
-*.pem
\ No newline at end of file
+*.pem
+passphrase.txt
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index 48518883..e72855d6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -37,37 +37,17 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "ansi_term"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "async-trait"
-version = "0.1.72"
+version = "0.1.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
+checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
 dependencies = [
  "proc-macro2",
  "quote",
  "syn 2.0.28",
 ]
 
-[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi 0.1.19",
- "libc",
- "winapi",
-]
-
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -114,15 +94,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "bincode"
-version = "1.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -214,38 +185,21 @@ dependencies = [
 ]
 
 [[package]]
-name = "clap"
-version = "2.34.0"
+name = "constant_time_eq"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
-dependencies = [
- "ansi_term",
- "atty",
- "bitflags 1.3.2",
- "strsim",
- "textwrap",
- "unicode-width",
- "vec_map",
- "yaml-rust",
-]
+checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
 
 [[package]]
-name = "clipboard-win"
-version = "4.5.0"
+name = "core-foundation"
+version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362"
+checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"
 dependencies = [
- "error-code",
- "str-buf",
- "winapi",
+ "core-foundation-sys",
+ "libc",
 ]
 
-[[package]]
-name = "constant_time_eq"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
-
 [[package]]
 name = "core-foundation-sys"
 version = "0.8.4"
@@ -328,31 +282,6 @@ dependencies = [
  "cfg-if",
 ]
 
-[[package]]
-name = "crossterm"
-version = "0.26.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13"
-dependencies = [
- "bitflags 1.3.2",
- "crossterm_winapi",
- "libc",
- "mio",
- "parking_lot",
- "signal-hook",
- "signal-hook-mio",
- "winapi",
-]
-
-[[package]]
-name = "crossterm_winapi"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -369,12 +298,6 @@ version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929"
 
-[[package]]
-name = "devtimer"
-version = "4.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "907339959a92f6b98846570500c0a567c9aecbb3871cef00561eb5d20d47b7c1"
-
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -392,12 +315,6 @@ version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 
-[[package]]
-name = "endian-type"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
-
 [[package]]
 name = "env_logger"
 version = "0.10.0"
@@ -439,25 +356,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "error-code"
-version = "2.3.1"
+name = "fastrand"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21"
-dependencies = [
- "libc",
- "str-buf",
-]
-
-[[package]]
-name = "fd-lock"
-version = "3.0.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5"
-dependencies = [
- "cfg-if",
- "rustix",
- "windows-sys",
-]
+checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
 
 [[package]]
 name = "flate2"
@@ -555,7 +457,7 @@ dependencies = [
  "log",
  "openssl",
  "powershell_script",
- "skytable 0.8.0 (git+https://github.com/skytable/client-rust.git)",
+ "skytable",
  "zip",
 ]
 
@@ -565,15 +467,6 @@ version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
 
-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "hermit-abi"
 version = "0.3.2"
@@ -589,15 +482,6 @@ dependencies = [
  "digest",
 ]
 
-[[package]]
-name = "home"
-version = "0.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb"
-dependencies = [
- "windows-sys",
-]
-
 [[package]]
 name = "humantime"
 version = "2.1.0"
@@ -629,7 +513,7 @@ version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
 dependencies = [
- "hermit-abi 0.3.2",
+ "hermit-abi",
  "rustix",
  "windows-sys",
 ]
@@ -677,9 +561,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.147"
+version = "0.2.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
+checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
 
 [[package]]
 name = "libsky"
@@ -743,44 +627,31 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.8.8"
+version = "0.8.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"
+checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0"
 dependencies = [
  "libc",
- "log",
  "wasi",
  "windows-sys",
 ]
 
 [[package]]
-name = "nibble_vec"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
-dependencies = [
- "smallvec",
-]
-
-[[package]]
-name = "nix"
-version = "0.26.2"
+name = "native-tls"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a"
+checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
 dependencies = [
- "bitflags 1.3.2",
- "cfg-if",
+ "lazy_static",
  "libc",
- "static_assertions",
-]
-
-[[package]]
-name = "ntapi"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
-dependencies = [
- "winapi",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
 ]
 
 [[package]]
@@ -789,7 +660,7 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi 0.3.2",
+ "hermit-abi",
  "libc",
 ]
 
@@ -834,6 +705,12 @@ dependencies = [
  "syn 2.0.28",
 ]
 
+[[package]]
+name = "openssl-probe"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+
 [[package]]
 name = "openssl-src"
 version = "111.27.0+1.1.1v"
@@ -961,16 +838,6 @@ dependencies = [
  "scheduled-thread-pool",
 ]
 
-[[package]]
-name = "radix_trie"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
-dependencies = [
- "endian-type",
- "nibble_vec",
-]
-
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -1091,35 +958,21 @@ dependencies = [
  "windows-sys",
 ]
 
-[[package]]
-name = "rustyline"
-version = "12.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "994eca4bca05c87e86e15d90fc7a91d1be64b4482b38cb2d27474568fe7c9db9"
-dependencies = [
- "bitflags 2.3.3",
- "cfg-if",
- "clipboard-win",
- "fd-lock",
- "home",
- "libc",
- "log",
- "memchr",
- "nix",
- "radix_trie",
- "scopeguard",
- "unicode-segmentation",
- "unicode-width",
- "utf8parse",
- "winapi",
-]
-
 [[package]]
 name = "ryu"
 version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
 
+[[package]]
+name = "schannel"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88"
+dependencies = [
+ "windows-sys",
+]
+
 [[package]]
 name = "scheduled-thread-pool"
 version = "0.2.7"
@@ -1135,6 +988,29 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
+[[package]]
+name = "security-framework"
+version = "2.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de"
+dependencies = [
+ "bitflags 1.3.2",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "serde"
 version = "1.0.183"
@@ -1155,17 +1031,6 @@ dependencies = [
  "syn 2.0.28",
 ]
 
-[[package]]
-name = "serde_json"
-version = "1.0.104"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
-dependencies = [
- "itoa",
- "ryu",
- "serde",
-]
-
 [[package]]
 name = "serde_yaml"
 version = "0.9.25"
@@ -1201,27 +1066,6 @@ dependencies = [
  "digest",
 ]
 
-[[package]]
-name = "signal-hook"
-version = "0.3.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801"
-dependencies = [
- "libc",
- "signal-hook-registry",
-]
-
-[[package]]
-name = "signal-hook-mio"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
-dependencies = [
- "libc",
- "mio",
- "signal-hook",
-]
-
 [[package]]
 name = "signal-hook-registry"
 version = "1.4.1"
@@ -1235,26 +1079,8 @@ dependencies = [
 name = "sky-bench"
 version = "0.8.0"
 dependencies = [
- "clap",
- "devtimer",
- "env_logger",
  "libstress",
- "log",
- "rand",
- "serde",
- "serde_json",
- "skytable 0.8.0 (git+https://github.com/skytable/client-rust.git)",
-]
-
-[[package]]
-name = "sky-migrate"
-version = "0.8.0"
-dependencies = [
- "bincode",
- "clap",
- "env_logger",
- "log",
- "skytable 0.8.0 (git+https://github.com/skytable/client-rust.git)",
+ "skytable",
 ]
 
 [[package]]
@@ -1297,35 +1123,22 @@ dependencies = [
 name = "skysh"
 version = "0.8.0"
 dependencies = [
- "clap",
- "crossterm",
- "lazy_static",
  "libsky",
- "rustyline",
- "skytable 0.8.0 (git+https://github.com/skytable/client-rust?branch=next)",
- "tokio",
+ "skytable",
 ]
 
 [[package]]
 name = "skytable"
 version = "0.8.0"
-source = "git+https://github.com/skytable/client-rust?branch=next#a55fbdc964e34c75c99404ea2395d03fd302daee"
+source = "git+https://github.com/skytable/client-rust.git?branch=octave#b2b0ea7197d9a3425809ce269e30b74ddd3eb340"
 dependencies = [
  "async-trait",
  "bb8",
- "bytes",
- "openssl",
+ "native-tls",
  "r2d2",
+ "rand",
  "tokio",
- "tokio-openssl",
-]
-
-[[package]]
-name = "skytable"
-version = "0.8.0"
-source = "git+https://github.com/skytable/client-rust.git#a55fbdc964e34c75c99404ea2395d03fd302daee"
-dependencies = [
- "r2d2",
+ "tokio-native-tls",
 ]
 
 [[package]]
@@ -1345,46 +1158,14 @@ checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
 
 [[package]]
 name = "socket2"
-version = "0.5.3"
+version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877"
+checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9"
 dependencies = [
  "libc",
  "windows-sys",
 ]
 
-[[package]]
-name = "static_assertions"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
-
-[[package]]
-name = "str-buf"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0"
-
-[[package]]
-name = "stress-test"
-version = "0.1.0"
-dependencies = [
- "crossbeam-channel",
- "devtimer",
- "env_logger",
- "libstress",
- "log",
- "rand",
- "skytable 0.8.0 (git+https://github.com/skytable/client-rust?branch=next)",
- "sysinfo",
-]
-
-[[package]]
-name = "strsim"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
-
 [[package]]
 name = "subtle"
 version = "2.5.0"
@@ -1414,18 +1195,16 @@ dependencies = [
 ]
 
 [[package]]
-name = "sysinfo"
-version = "0.29.7"
+name = "tempfile"
+version = "3.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "165d6d8539689e3d3bc8b98ac59541e1f21c7de7c85d60dc80e43ae0ed2113db"
+checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
 dependencies = [
  "cfg-if",
- "core-foundation-sys",
- "libc",
- "ntapi",
- "once_cell",
- "rayon",
- "winapi",
+ "fastrand",
+ "redox_syscall",
+ "rustix",
+ "windows-sys",
 ]
 
 [[package]]
@@ -1437,15 +1216,6 @@ dependencies = [
  "winapi-util",
 ]
 
-[[package]]
-name = "textwrap"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
-dependencies = [
- "unicode-width",
-]
-
 [[package]]
 name = "time"
 version = "0.3.25"
@@ -1465,9 +1235,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
 
 [[package]]
 name = "tokio"
-version = "1.30.0"
+version = "1.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d3ce25f50619af8b0aec2eb23deebe84249e19e2ddd393a6e16e3300a6dadfd"
+checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9"
 dependencies = [
  "backtrace",
  "bytes",
@@ -1484,15 +1254,25 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
+checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
 dependencies = [
  "proc-macro2",
  "quote",
  "syn 2.0.28",
 ]
 
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-openssl"
 version = "0.6.3"
@@ -1517,30 +1297,12 @@ version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
 
-[[package]]
-name = "unicode-segmentation"
-version = "1.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
-
-[[package]]
-name = "unicode-width"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
-
 [[package]]
 name = "unsafe-libyaml"
 version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa"
 
-[[package]]
-name = "utf8parse"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
-
 [[package]]
 name = "uuid"
 version = "1.4.1"
@@ -1569,12 +1331,6 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
-[[package]]
-name = "vec_map"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
-
 [[package]]
 name = "version_check"
 version = "0.9.4"
@@ -1684,12 +1440,6 @@ version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
 
-[[package]]
-name = "yaml-rust"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992"
-
 [[package]]
 name = "zip"
 version = "0.6.6"
diff --git a/Cargo.toml b/Cargo.toml
index 448c2321..a07132a4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -7,8 +7,6 @@ members = [
     "sky-bench",
     "sky-macros",
     "libstress",
-    "stress-test",
-    "sky-migrate",
     "harness",
 ]
 
diff --git a/ci/server1.toml b/ci/server1.toml
deleted file mode 100644
index dad19e6c..00000000
--- a/ci/server1.toml
+++ /dev/null
@@ -1,9 +0,0 @@
-[server]
-host = "127.0.0.1"
-port = 2003
-noart = true
-
-[ssl]
-key="../key.pem"
-chain="../cert.pem"
-port = 2004
diff --git a/ci/server1.yaml b/ci/server1.yaml
new file mode 100644
index 00000000..fc3aaa18
--- /dev/null
+++ b/ci/server1.yaml
@@ -0,0 +1,17 @@
+system:
+  mode: prod
+
+auth:
+  plugin: pwd
+  root_pass: password12345678
+
+endpoints:
+  insecure:
+    host: 127.0.0.1
+    port: 2003
+  secure:
+    host: 127.0.0.1
+    port: 2004
+    cert: ../cert.pem
+    private_key: ../key.pem
+    pkey_passphrase: ../passphrase.txt
diff --git a/ci/server2.toml b/ci/server2.toml
deleted file mode 100644
index 05083faa..00000000
--- a/ci/server2.toml
+++ /dev/null
@@ -1,12 +0,0 @@
-[server]
-host = "127.0.0.1"
-port = 2005
-noart = true
-
-[auth]
-origin_key = "4527387f92a381cbe804593f33991d327d456a97"
-
-[ssl]
-key = "../key.pem"
-chain = "../cert.pem"
-port = 2006
diff --git a/ci/server3.toml b/ci/server3.toml
deleted file mode 100644
index 69d6b24b..00000000
--- a/ci/server3.toml
+++ /dev/null
@@ -1,14 +0,0 @@
-[server]
-host = "127.0.0.1"
-port = 2007
-noart = true
-
-[snapshot]
-every = 3600
-atmost = 4
-failsafe = true
-
-[ssl]
-key = "../key.pem"
-chain = "../cert.pem"
-port = 2008
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index 0402ce65..68312688 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -9,13 +9,4 @@ edition = "2021"
 [dependencies]
 # internal deps
 libsky = { path = "../libsky" }
-skytable = { git = "https://github.com/skytable/client-rust", branch = "next", features = [
-  "aio",
-  "aio-sslv",
-], default-features = false }
-# external deps
-tokio = { version = "1.29.1", features = ["full"] }
-clap = { version = "2", features = ["yaml"] }
-rustyline = "12.0.0"
-crossterm = "0.26.1"
-lazy_static = "1.4.0"
+skytable = { git = "https://github.com/skytable/client-rust.git", branch = "octave" }
diff --git a/cli/src/argparse.rs b/cli/src/argparse.rs
deleted file mode 100644
index 12ad8e80..00000000
--- a/cli/src/argparse.rs
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Created on Wed Jul 01 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{runner::Runner, tokenizer},
-    clap::{load_yaml, App},
-    crossterm::{
-        cursor, execute,
-        terminal::{Clear, ClearType},
-    },
-    libsky::{URL, VERSION},
-    rustyline::{config::Configurer, error::ReadlineError, DefaultEditor},
-    skytable::{Pipeline, Query},
-    std::{io::stdout, process},
-};
-
-const ADDR: &str = "127.0.0.1";
-const SKYSH_HISTORY_FILE: &str = ".sky_history";
-
-const HELP_TEXT: &str = r#"
-███████ ██   ██ ██    ██ ████████  █████  ██████  ██      ███████
-██      ██  ██   ██  ██     ██    ██   ██ ██   ██ ██      ██
-███████ █████     ████      ██    ███████ ██████  ██      █████
-     ██ ██  ██     ██       ██    ██   ██ ██   ██ ██      ██
-███████ ██   ██    ██       ██    ██   ██ ██████  ███████ ███████
-
-Welcome to Skytable's interactive shell (REPL) environment. Using the Skytable
-shell, you can create, read, update or delete data on your remote Skytable
-instance. When you connect to your database instance, you'll be connected to
-the `default` table in the `default` keyspace. This table has binary keys and
-binary values as the default data type. Here's a brief guide on doing some
-everyday tasks:
-
-(1) Running actions
-================================================================================
-An action is like a shell command: it starts with a name and contains arguments!
-To run actions, simply type them out, like "set x 100" or "inspect table mytbl"
-and hit enter.
-
-(2) Running shell commands
-================================================================================
-Shell commands are those which are provided by `skysh` and are not supported by
-the server. These enable you to do convenient things like:
-- "exit": exits the shell
-- "clear": clears the terminal screen
-
-Apart from these, you can use the following shell commands:
-- "!pipe": Lets you create a pipeline. Terminate with a semicolon (`;`)
-- "!help": Brings up this help menu
-- "?<command name>": Describes what the built-in shell command is for
-
-With Skytable in your hands, the sky is the only limit on what you can create!"#;
-
-const SKY_WELCOME: &str = "
-Welcome to Skytable's interactive shell (REPL) environment. For usage and help
-within the shell, you can run `!help` anytime. Now that you have Skytable in
-your hands, the sky is the only limit on what you can create!
-";
-
-/// This creates a REPL on the command line and also parses command-line arguments
-///
-/// Anything that is entered following a return, is parsed into a query and is
-/// written to the socket (which is either `localhost:2003` or it is determined by
-/// command line parameters)
-pub async fn start_repl() {
-    let mut skysh_blank: String = "                     > ".to_owned();
-    let mut skysh_prompt: String = "skysh@default:default> ".to_owned();
-    let mut did_swap = false;
-
-    macro_rules! readln {
-        ($editor:expr) => {
-            match $editor.readline(&skysh_blank) {
-                Ok(l) => l,
-                Err(ReadlineError::Interrupted | ReadlineError::Eof) => return,
-                Err(err) => fatal!("ERROR: Failed to read line with error: {}", err),
-            }
-        };
-    }
-
-    let cfg_layout = load_yaml!("./cli.yml");
-    let matches = App::from_yaml(cfg_layout).get_matches();
-    let host = libsky::option_unwrap_or!(matches.value_of("host"), ADDR);
-    let port = match matches.value_of("port") {
-        Some(p) => match p.parse::<u16>() {
-            Ok(p) => p,
-            Err(_) => fatal!("Invalid port"),
-        },
-        None => 2003,
-    };
-    let mut editor = match DefaultEditor::new() {
-        Ok(e) => e,
-        Err(e) => fatal!("Editor init error: {}", e),
-    };
-    editor.set_auto_add_history(true);
-    editor
-        .set_history_ignore_dups(true)
-        .unwrap_or_else(|_| fatal!("couldn't set up terminal"));
-    editor.bind_sequence(
-        rustyline::KeyEvent(
-            rustyline::KeyCode::BracketedPasteStart,
-            rustyline::Modifiers::NONE,
-        ),
-        rustyline::Cmd::Noop,
-    );
-    let con = match matches.value_of("cert") {
-        Some(cert) => Runner::new_secure(host, port, cert).await,
-        None => Runner::new_insecure(host, port).await,
-    };
-    let mut runner = match con {
-        Ok(c) => c,
-        Err(e) => fatal!("Failed to connect to server with error: {}", e),
-    };
-
-    macro_rules! checkswap {
-        () => {
-            if did_swap {
-                // noice, we need to poll for the location of the new entity
-                runner
-                    .check_entity(&mut skysh_blank, &mut skysh_prompt)
-                    .await;
-            }
-        };
-    }
-
-    if let Some(eval_expr) = matches.values_of("eval") {
-        for eval_expr in eval_expr {
-            if !eval_expr.is_empty() {
-                runner.run_query(eval_expr).await;
-            }
-        }
-        process::exit(0x00);
-    }
-    println!("Skytable v{} | {}", VERSION, URL);
-    match editor.load_history(SKYSH_HISTORY_FILE) {
-        Ok(_) => {}
-        Err(e) => match e {
-            ReadlineError::Io(e) if e.kind() == std::io::ErrorKind::NotFound => {
-                println!("{}", SKY_WELCOME)
-            }
-            _ => fatal!("Failed to read history file with error: {}", e),
-        },
-    }
-    loop {
-        match editor.readline(&skysh_prompt) {
-            Ok(mut line) => {
-                macro_rules! tokenize {
-                    ($inp:expr) => {
-                        match tokenizer::get_query($inp) {
-                            Ok(q) => q,
-                            Err(e) => {
-                                eskysh!(e);
-                                continue;
-                            }
-                        }
-                    };
-                    () => {
-                        tokenize!(line.as_bytes())
-                    };
-                }
-                match line.to_lowercase().as_str() {
-                    "exit" => break,
-                    "clear" => {
-                        clear_screen();
-                        continue;
-                    }
-                    "help" => {
-                        println!("To get help, run `!help`");
-                        continue;
-                    }
-                    _ => {
-                        if line.is_empty() {
-                            continue;
-                        }
-                        match line.as_bytes()[0] {
-                            b'#' => continue,
-                            b'!' => {
-                                match &line.as_bytes()[1..] {
-                                    b"" => eskysh!("Bad shell command"),
-                                    b"help" => println!("{}", HELP_TEXT),
-                                    b"pipe" => {
-                                        // so we need to handle a pipeline
-                                        let mut pipeline = Pipeline::new();
-                                        line = readln!(editor);
-                                        loop {
-                                            did_swap = line
-                                                .get(..3)
-                                                .map(|v| v.eq_ignore_ascii_case("use"))
-                                                .unwrap_or(did_swap);
-                                            if !line.is_empty() {
-                                                if *(line.as_bytes().last().unwrap()) == b';' {
-                                                    break;
-                                                } else {
-                                                    let q: Query = tokenize!();
-                                                    pipeline.push(q);
-                                                }
-                                            }
-                                            line = readln!(editor);
-                                        }
-                                        if line.len() > 1 {
-                                            line.drain(line.len() - 1..);
-                                            let q: Query = tokenize!();
-                                            pipeline.push(q);
-                                        }
-                                        runner.run_pipeline(pipeline).await;
-                                        checkswap!();
-                                    }
-                                    _ => eskysh!("Unknown shell command"),
-                                }
-                                continue;
-                            }
-                            b'?' => {
-                                // handle explanation for a shell command
-                                print_help(&line);
-                                continue;
-                            }
-                            _ => {}
-                        }
-                        while line.len() >= 2 && line[line.len() - 2..].as_bytes().eq(br#" \"#) {
-                            // continuation on next line
-                            let cl = readln!(editor);
-                            line.drain(line.len() - 2..);
-                            line.push_str(&cl);
-                        }
-                        did_swap = line
-                            .get(..3)
-                            .map(|v| v.eq_ignore_ascii_case("use"))
-                            .unwrap_or(did_swap);
-                        runner.run_query(&line).await;
-                        checkswap!();
-                    }
-                }
-            }
-            Err(ReadlineError::Interrupted | ReadlineError::Eof) => break,
-            Err(err) => fatal!("ERROR: Failed to read line with error: {}", err),
-        }
-    }
-    editor
-        .save_history(SKYSH_HISTORY_FILE)
-        .map_err(|e| {
-            fatal!("ERROR: Failed to save history with error: '{}'", e);
-        })
-        .unwrap();
-}
-
-fn print_help(line: &str) {
-    match &line.as_bytes()[1..] {
-        b"" => eskysh!("Bad shell command"),
-        b"help" => println!("`!help` shows the help menu"),
-        b"exit" => println!("`exit` ends the shell session"),
-        b"clear" => println!("`clear` clears the terminal screen"),
-        b"pipe" | b"!pipe" => println!("`!pipe` lets you run pipelines using the shell"),
-        _ => eskysh!("Unknown shell command"),
-    }
-}
-
-fn clear_screen() {
-    let mut stdout = stdout();
-    execute!(stdout, Clear(ClearType::All)).expect("Failed to clear screen");
-    execute!(stdout, cursor::MoveTo(0, 0)).expect("Failed to move cursor to origin");
-}
diff --git a/cli/src/cli.yml b/cli/src/cli.yml
deleted file mode 100644
index db562b18..00000000
--- a/cli/src/cli.yml
+++ /dev/null
@@ -1,55 +0,0 @@
-#
-# Created on Tue Nov 03 2020
-#
-# This file is a part of Skytable
-# Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-#
-
-name: Skytable Shell
-version: 0.8.0
-author: Sayan N. <ohsayan@outlook.com>
-about: The Skytable Shell (skysh)
-args:
-  - host:
-      short: h
-      required: false
-      long: host
-      value_name: host
-      help: Sets the remote host to connect to
-      takes_value: true
-  - port:
-      short: p
-      required: false
-      long: port
-      value_name: port
-      help: Sets the remote port to connect to
-      takes_value: true
-  - eval:
-      short: e
-      required: false
-      long: eval
-      multiple: true
-      value_name: expression
-      help: Run an expression without REPL
-      takes_value: true
-  - cert:
-      short: C
-      required: false
-      long: sslcert
-      value_name: cert
-      help: Sets the PEM certificate to use for SSL connections
-      takes_value: true
diff --git a/cli/src/macros.rs b/cli/src/macros.rs
deleted file mode 100644
index cdc809d6..00000000
--- a/cli/src/macros.rs
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Created on Wed Nov 03 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-macro_rules! write_str {
-    ($st:ident) => {
-        println!("\"{}\"", $st)
-    };
-    ($idx:ident, $st:ident) => {
-        println!("({}) \"{}\"", $idx, $st)
-    };
-}
-
-macro_rules! write_binstr {
-    ($st:ident) => {
-        println!("{}", BinaryData($st))
-    };
-    ($idx:ident, $st:ident) => {
-        println!("({}) {}", $idx, BinaryData($st))
-    };
-}
-
-macro_rules! write_int {
-    ($int:ident) => {
-        println!("{}", $int)
-    };
-    ($idx:ident, $st:ident) => {
-        println!("({}) \"{}\"", $idx, $st)
-    };
-}
-
-macro_rules! write_err {
-    ($idx:expr, $err:ident) => {
-        err!(if let Some(idx) = $idx {
-            format!("({}) ({})\n", idx, $err)
-        } else {
-            format!("({})\n", $err)
-        })
-    };
-    ($idx:ident, $err:literal) => {
-        err!(
-            (if let Some(idx) = $idx {
-                format!("({}) ({})\n", idx, $err)
-            } else {
-                format!("({})\n", $err)
-            })
-        )
-    };
-}
-
-macro_rules! write_okay {
-    () => {
-        crossterm::execute!(
-            std::io::stdout(),
-            SetForegroundColor(Color::Cyan),
-            Print("(Okay)\n".to_string()),
-            ResetColor
-        )
-        .expect("Failed to write to stdout")
-    };
-    ($idx:ident) => {
-        crossterm::execute!(
-            std::io::stdout(),
-            SetForegroundColor(Color::Cyan),
-            Print(format!("({}) (Okay)\n", $idx)),
-            ResetColor
-        )
-        .expect("Failed to write to stdout")
-    };
-}
-
-macro_rules! err {
-    ($input:expr) => {
-        crossterm::execute!(
-            std::io::stdout(),
-            ::crossterm::style::SetForegroundColor(::crossterm::style::Color::Red),
-            ::crossterm::style::Print($input),
-            ::crossterm::style::ResetColor
-        )
-        .expect("Failed to write to stdout")
-    };
-}
-
-macro_rules! eskysh {
-    ($e:expr) => {
-        err!(format!("[SKYSH ERROR] {}\n", $e))
-    };
-}
-
-macro_rules! fatal {
-    ($e:expr) => {{
-        err!($e);
-        ::std::process::exit(0x01);
-    }};
-    ($e:expr, $desc:expr) => {{
-        err!(format!($e, $desc));
-        println!();
-        ::std::process::exit(0x01)
-    }};
-}
diff --git a/cli/src/main.rs b/cli/src/main.rs
index fd90672e..57a21b8e 100644
--- a/cli/src/main.rs
+++ b/cli/src/main.rs
@@ -1,5 +1,5 @@
 /*
- * Created on Wed Jul 01 2020
+ * Created on Wed Nov 15 2023
  *
  * This file is a part of Skytable
  * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@@ -7,7 +7,7 @@
  * vision to provide flexibility in data modelling without compromising
  * on performance, queryability or scalability.
  *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published by
@@ -24,20 +24,4 @@
  *
 */
 
-#![deny(unused_crate_dependencies)]
-#![deny(unused_imports)]
-
-#[macro_use]
-mod macros;
-mod argparse;
-mod runner;
-mod tokenizer;
-// tests
-#[cfg(test)]
-mod tests;
-
-#[tokio::main]
-async fn main() {
-    argparse::start_repl().await;
-    println!("Goodbye!");
-}
+fn main() {}
diff --git a/cli/src/runner.rs b/cli/src/runner.rs
deleted file mode 100644
index f9ce45ac..00000000
--- a/cli/src/runner.rs
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Created on Wed May 12 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::tokenizer,
-    core::fmt,
-    crossterm::style::{Color, Print, ResetColor, SetForegroundColor},
-    skytable::{
-        aio, error::Error, types::Array, types::FlatElement, Element, Pipeline, Query, RespCode,
-    },
-};
-
-type SkyResult<T> = Result<T, Error>;
-
-pub enum Runner {
-    Insecure(aio::Connection),
-    Secure(aio::TlsConnection),
-}
-
-impl Runner {
-    pub async fn new_insecure(host: &str, port: u16) -> SkyResult<Self> {
-        let con = aio::Connection::new(host, port).await?;
-        Ok(Self::Insecure(con))
-    }
-    pub async fn new_secure(host: &str, port: u16, cert: &str) -> SkyResult<Self> {
-        let con = aio::TlsConnection::new(host, port, cert).await?;
-        Ok(Self::Secure(con))
-    }
-    pub async fn run_pipeline(&mut self, pipeline: Pipeline) {
-        let ret = match self {
-            Self::Insecure(con) => con.run_pipeline(pipeline).await,
-            Self::Secure(con) => con.run_pipeline(pipeline).await,
-        };
-        let retok = match ret {
-            Ok(r) => r,
-            Err(e) => fatal!("An I/O error occurred while querying: {}", e),
-        };
-        for (idx, resp) in retok
-            .into_iter()
-            .enumerate()
-            .map(|(idx, resp)| (idx + 1, resp))
-        {
-            println!("[Response {}]", idx);
-            print_element(resp);
-        }
-    }
-    pub async fn run_query(&mut self, unescaped: &str) {
-        let query: Query = match tokenizer::get_query(unescaped.as_bytes()) {
-            Ok(q) => q,
-            Err(e) => {
-                err!(format!("[Syntax Error: {}]\n", e));
-                return;
-            }
-        };
-        let ret = match self {
-            Self::Insecure(con) => con.run_query_raw(&query).await,
-            Self::Secure(con) => con.run_query_raw(&query).await,
-        };
-        match ret {
-            Ok(resp) => print_element(resp),
-            Err(e) => fatal!("An I/O error occurred while querying: {}", e),
-        }
-    }
-    pub async fn check_entity(&mut self, blank: &mut String, prompt: &mut String) {
-        let query: Query = tokenizer::get_query(b"whereami").unwrap();
-        let ret = match self {
-            Self::Insecure(con) => con.run_query_raw(&query).await,
-            Self::Secure(con) => con.run_query_raw(&query).await,
-        };
-        let ret = match ret {
-            Ok(resp) => resp,
-            Err(e) => fatal!("An I/O error occurred while querying: {}", e),
-        };
-        match ret {
-            Element::Array(Array::NonNullStr(srr)) => match srr.len() {
-                1 => {
-                    *blank = format!("      {blank}> ", blank = " ".repeat(srr[0].len()));
-                    *prompt = format!("skysh@{ks}> ", ks = srr[0]);
-                }
-                2 => {
-                    let ks = &srr[0];
-                    let tbl = &srr[1];
-                    *blank = format!(
-                        "      {blank}> ",
-                        blank = " ".repeat(ks.len() + tbl.len() + 1)
-                    );
-                    *prompt = format!("skysh@{ks}:{tbl}> ", ks = ks, tbl = tbl);
-                }
-                count => fatal!(
-                    "The server returned {} IDs while checking entity state",
-                    count
-                ),
-            },
-            _ => fatal!("The server returned the wrong data type for entity state check"),
-        }
-    }
-}
-
-fn print_float(float: f32, idx: Option<usize>) {
-    if let Some(idx) = idx {
-        println!("({idx}) {float}")
-    } else {
-        println!("{float}");
-    }
-}
-
-fn print_element(el: Element) {
-    match el {
-        Element::String(st) => write_str!(st),
-        Element::Binstr(st) => write_binstr!(st),
-        Element::Array(Array::Bin(brr)) => print_bin_array(brr),
-        Element::Array(Array::Str(srr)) => print_str_array(srr),
-        Element::RespCode(r) => print_rcode(r, None),
-        Element::UnsignedInt(int) => write_int!(int),
-        Element::Array(Array::Flat(frr)) => write_flat_array(frr),
-        Element::Array(Array::Recursive(a)) => print_array(a),
-        Element::Array(Array::NonNullBin(nbrr)) => print_array_nonnull_bin(nbrr),
-        Element::Array(Array::NonNullStr(nsrr)) => print_array_nonnull_str(nsrr),
-        Element::Float(float) => print_float(float, None),
-        _ => eskysh!("The server possibly sent a newer data type that we can't parse"),
-    }
-}
-
-fn print_rcode(rcode: RespCode, idx: Option<usize>) {
-    match rcode {
-        RespCode::Okay => write_okay!(),
-        RespCode::ActionError => write_err!(idx, "Action Error"),
-        RespCode::ErrorString(st) => write_err!(idx, st),
-        RespCode::OtherError => write_err!(idx, "Other Error"),
-        RespCode::NotFound => write_err!(idx, "Not Found"),
-        RespCode::OverwriteError => write_err!(idx, "Overwrite Error"),
-        RespCode::PacketError => write_err!(idx, "Packet Error"),
-        RespCode::ServerError => write_err!(idx, "Server Error"),
-        RespCode::UnknownDataType => write_err!(idx, "Unknown data type"),
-        RespCode::EncodingError => write_err!(idx, "Encoding error"),
-        RespCode::AuthBadCredentials => write_err!(idx, "auth bad credentials"),
-        RespCode::AuthPermissionError => write_err!(idx, "auth permission error"),
-        _ => write_err!(idx, "Unknown error"),
-    }
-}
-
-fn print_bin_array(bin_array: Vec<Option<Vec<u8>>>) {
-    bin_array.into_iter().enumerate().for_each(|(idx, elem)| {
-        let idx = idx + 1;
-        match elem {
-            Some(ele) => {
-                write_binstr!(idx, ele);
-            }
-            None => print_rcode(RespCode::NotFound, Some(idx)),
-        }
-    })
-}
-
-fn print_str_array(str_array: Vec<Option<String>>) {
-    str_array.into_iter().enumerate().for_each(|(idx, elem)| {
-        let idx = idx + 1;
-        match elem {
-            Some(ele) => {
-                write_str!(idx, ele);
-            }
-            None => print_rcode(RespCode::NotFound, Some(idx)),
-        }
-    })
-}
-
-fn print_array_nonnull_str(str_array: Vec<String>) {
-    str_array.into_iter().enumerate().for_each(|(idx, elem)| {
-        let idx = idx + 1;
-        write_str!(idx, elem)
-    })
-}
-
-fn print_array_nonnull_bin(str_array: Vec<Vec<u8>>) {
-    str_array.into_iter().enumerate().for_each(|(idx, elem)| {
-        let idx = idx + 1;
-        write_binstr!(idx, elem)
-    })
-}
-
-fn write_flat_array(flat_array: Vec<FlatElement>) {
-    for (idx, item) in flat_array.into_iter().enumerate() {
-        let idx = idx + 1;
-        match item {
-            FlatElement::String(st) => write_str!(idx, st),
-            FlatElement::Binstr(st) => {
-                write_binstr!(idx, st)
-            }
-            FlatElement::RespCode(rc) => print_rcode(rc, Some(idx)),
-            FlatElement::UnsignedInt(int) => write_int!(int, idx),
-            _ => eskysh!("Element typed cannot yet be parsed"),
-        }
-    }
-}
-
-fn print_array(array: Vec<Element>) {
-    for (idx, item) in array.into_iter().enumerate() {
-        let idx = idx + 1;
-        match item {
-            Element::String(st) => write_str!(idx, st),
-            Element::RespCode(rc) => print_rcode(rc, Some(idx)),
-            Element::UnsignedInt(int) => write_int!(idx, int),
-            Element::Array(Array::Bin(brr)) => print_bin_array(brr),
-            Element::Array(Array::Str(srr)) => print_str_array(srr),
-            _ => eskysh!("Nested arrays cannot be printed just yet"),
-        }
-    }
-}
-
-pub struct BinaryData(Vec<u8>);
-
-impl fmt::Display for BinaryData {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
-        write!(f, "b\"")?;
-        for b in self.0.iter() {
-            let b = *b;
-            // See this: https://doc.rust-lang.org/reference/tokens.html#byte-escapes
-            // this idea was borrowed from the Bytes crate
-            #[allow(clippy::manual_range_contains)]
-            if b == b'\n' {
-                write!(f, "\\n")?;
-            } else if b == b'\r' {
-                write!(f, "\\r")?;
-            } else if b == b'\t' {
-                write!(f, "\\t")?;
-            } else if b == b'\\' || b == b'"' {
-                write!(f, "\\{}", b as char)?;
-            } else if b == b'\0' {
-                write!(f, "\\0")?;
-            // ASCII printable
-            } else if b >= 0x20 && b < 0x7f {
-                write!(f, "{}", b as char)?;
-            } else {
-                write!(f, "\\x{:02x}", b)?;
-            }
-        }
-        write!(f, "\"")?;
-        Ok(())
-    }
-}
diff --git a/cli/src/tests.rs b/cli/src/tests.rs
deleted file mode 100644
index d527b1f8..00000000
--- a/cli/src/tests.rs
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Created on Sun Oct 10 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::tokenizer::{get_query, TokenizerError};
-
-fn query_from(input: &[u8]) -> Result<Vec<String>, TokenizerError> {
-    get_query(input)
-}
-
-#[test]
-fn test_basic_tokenization() {
-    let input = "set x 100".as_bytes();
-    let ret = query_from(input).unwrap();
-    assert_eq!(
-        ret,
-        vec!["set".to_owned(), "x".to_owned(), "100".to_owned()]
-    );
-}
-
-#[test]
-fn test_single_quote_tokens() {
-    let input = "set 'x with a whitespace' 100".as_bytes();
-    let ret = query_from(input).unwrap();
-    assert_eq!(
-        ret,
-        vec![
-            "set".to_owned(),
-            "x with a whitespace".to_owned(),
-            "100".to_owned()
-        ]
-    );
-}
-
-#[test]
-fn test_double_quote_tokens() {
-    let input = r#"set "x with a whitespace" 100"#.as_bytes();
-    let ret = query_from(input).unwrap();
-    assert_eq!(
-        ret,
-        vec![
-            "set".to_owned(),
-            "x with a whitespace".to_owned(),
-            "100".to_owned()
-        ]
-    );
-}
-
-#[test]
-fn test_single_and_double_quote_tokens() {
-    let input = r#"set "x with a whitespace" 'y with a whitespace'"#.as_bytes();
-    let ret = query_from(input).unwrap();
-    assert_eq!(
-        ret,
-        vec![
-            "set".to_owned(),
-            "x with a whitespace".to_owned(),
-            "y with a whitespace".to_owned()
-        ]
-    );
-}
-
-#[test]
-fn test_multiple_single_quote_tokens() {
-    let input = r#"'set' 'x with a whitespace' 'y with a whitespace'"#.as_bytes();
-    let ret = query_from(input).unwrap();
-    assert_eq!(
-        ret,
-        vec![
-            "set".to_owned(),
-            "x with a whitespace".to_owned(),
-            "y with a whitespace".to_owned()
-        ]
-    );
-}
-
-#[test]
-fn test_multiple_double_quote_tokens() {
-    let input = r#""set" "x with a whitespace" "y with a whitespace""#.as_bytes();
-    let ret = query_from(input).unwrap();
-    assert_eq!(
-        ret,
-        vec![
-            "set".to_owned(),
-            "x with a whitespace".to_owned(),
-            "y with a whitespace".to_owned()
-        ]
-    );
-}
-
-#[test]
-fn test_missing_single_quote() {
-    let input = r#"'get' 'x with a whitespace"#.as_bytes();
-    let ret = format!("{}", query_from(input).unwrap_err());
-    assert_eq!(ret, "mismatched quotes near end of: `x with a whitespace`");
-}
-
-#[test]
-fn test_missing_double_quote() {
-    let input = r#"'get' "x with a whitespace"#.as_bytes();
-    let ret = format!("{}", query_from(input).unwrap_err());
-    assert_eq!(ret, "mismatched quotes near end of: `x with a whitespace`");
-}
-
-#[test]
-fn test_extra_whitespace() {
-    let input = "set  x  '100'".as_bytes();
-    let ret = query_from(input).unwrap();
-    assert_eq!(
-        ret,
-        vec!["set".to_owned(), "x".to_owned(), "100".to_owned()]
-    );
-}
-
-#[test]
-fn test_singly_quoted() {
-    let input = "set tables' wth".as_bytes();
-    let ret = query_from(input).unwrap_err();
-    assert_eq!(ret, TokenizerError::ExpectedWhitespace("tables".to_owned()));
-}
-
-#[test]
-fn test_text_after_quote_nospace() {
-    let input = "get 'rust'ferris".as_bytes();
-    let ret = query_from(input).unwrap_err();
-    assert_eq!(ret, TokenizerError::ExpectedWhitespace("rust'".to_owned()));
-}
-
-#[test]
-fn test_text_after_double_quote_nospace() {
-    let input = r#"get "rust"ferris"#.as_bytes();
-    let ret = query_from(input).unwrap_err();
-    assert_eq!(ret, TokenizerError::ExpectedWhitespace("rust\"".to_owned()));
-}
-
-#[test]
-fn test_inline_comment() {
-    let input = "set x 100 # sets x to 100".as_bytes();
-    let ret = query_from(input).unwrap();
-    assert_eq!(
-        ret,
-        vec!["set".to_owned(), "x".to_owned(), "100".to_owned()]
-    )
-}
-
-#[test]
-fn test_full_comment() {
-    let input = "# what is going on?".as_bytes();
-    let ret = query_from(input).unwrap();
-    assert!(ret.is_empty());
-}
-
-#[test]
-fn test_ignore_comment() {
-    let input = "set x \"# ooh la la\"".as_bytes();
-    assert_eq!(
-        query_from(input).unwrap(),
-        vec!["set".to_owned(), "x".to_owned(), "# ooh la la".to_owned()]
-    );
-    let input = "set x \"#\"".as_bytes();
-    assert_eq!(
-        query_from(input).unwrap(),
-        vec!["set".to_owned(), "x".to_owned(), "#".to_owned()]
-    );
-}
-
-#[test]
-fn test_blueql_query() {
-    let input = b"create model mymodel(string, binary)";
-    assert_eq!(
-        query_from(input).unwrap(),
-        vec!["create model mymodel(string, binary)"]
-    );
-}
diff --git a/cli/src/tokenizer.rs b/cli/src/tokenizer.rs
deleted file mode 100644
index e6d6183f..00000000
--- a/cli/src/tokenizer.rs
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Created on Sat Oct 09 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! This module provides a simple way to avoid "the funk" with "funky input queries". It simply
-//! tokenizes char-by-char analyzing quotes et al as required
-//!
-
-use {
-    core::fmt,
-    skytable::{types::RawString, Query},
-    std::collections::HashSet,
-};
-
-lazy_static::lazy_static! {
-    static ref BLUEQL_KW: HashSet<&'static [u8]> = {
-        let mut hs = HashSet::new();
-        hs.insert("create".as_bytes());
-        hs.insert(b"inspect");
-        hs.insert(b"drop");
-        hs.insert(b"use");
-        hs
-    };
-}
-
-#[derive(Debug, PartialEq)]
-pub enum TokenizerError {
-    QuoteMismatch(String),
-    BacktickMismatch(String),
-    ExpectedWhitespace(String),
-}
-
-impl fmt::Display for TokenizerError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::QuoteMismatch(expr) => write!(f, "mismatched quotes near end of: `{}`", expr),
-            Self::ExpectedWhitespace(expr) => {
-                write!(f, "expected whitespace near end of: `{}`", expr)
-            }
-            Self::BacktickMismatch(expr) => {
-                write!(f, "mismatched backticks near end of: `{}`", expr)
-            }
-        }
-    }
-}
-
-pub trait SequentialQuery {
-    fn push(&mut self, input: &[u8]);
-    fn new() -> Self;
-    fn is_empty(&self) -> bool;
-}
-
-// #[cfg(test)]
-impl SequentialQuery for Vec<String> {
-    fn push(&mut self, input: &[u8]) {
-        Vec::push(self, String::from_utf8_lossy(input).to_string())
-    }
-    fn is_empty(&self) -> bool {
-        Vec::len(self) == 0
-    }
-    fn new() -> Self {
-        Vec::new()
-    }
-}
-
-impl SequentialQuery for Query {
-    fn push(&mut self, input: &[u8]) {
-        Query::push(self, RawString::from(input.to_owned()))
-    }
-    fn is_empty(&self) -> bool {
-        Query::len(self) == 0
-    }
-    fn new() -> Self {
-        Query::new()
-    }
-}
-
-// FIXME(@ohsayan): Fix this entire impl. At this point, it's almost like legacy code
-pub fn get_query<T: SequentialQuery>(inp: &[u8]) -> Result<T, TokenizerError> {
-    assert!(!inp.is_empty(), "Input is empty");
-    let mut query = T::new();
-    let mut it = inp.iter().peekable();
-    macro_rules! pos {
-        () => {
-            inp.len() - it.len()
-        };
-    }
-    macro_rules! expect_whitespace {
-        ($start:expr) => {
-            match it.peek() {
-                Some(b) => match **b {
-                    b' ' => {}
-                    _ => {
-                        return Err(TokenizerError::ExpectedWhitespace(
-                            String::from_utf8_lossy(&inp[$start..pos!()]).to_string(),
-                        ))
-                    }
-                },
-                None => {}
-            }
-        };
-    }
-    // skip useless starting whitespace
-    while let Some(b' ') = it.next() {}
-    let end_of_first = match it.position(|x| *x == b' ') {
-        Some(e) => e + 1,
-        None if it.len() == 0 => inp.len(),
-        None => {
-            return Err(TokenizerError::ExpectedWhitespace(
-                String::from_utf8_lossy(inp).to_string(),
-            ))
-        }
-    };
-    if BLUEQL_KW.contains(inp[..end_of_first].to_ascii_lowercase().as_slice()) {
-        query.push(inp);
-        return Ok(query);
-    } else {
-        it = inp.iter().peekable();
-    }
-    'outer: while let Some(tok) = it.next() {
-        match tok {
-            b'\'' => {
-                // hmm, quotes; let's see where it ends
-                let pos = pos!();
-                let qidx = it.position(|x| *x == b'\'');
-                match qidx {
-                    Some(idx) => query.push(&inp[pos..idx + pos]),
-                    None => {
-                        let end = pos!();
-                        return Err(TokenizerError::QuoteMismatch(
-                            String::from_utf8_lossy(&inp[pos..end]).to_string(),
-                        ));
-                    }
-                }
-                expect_whitespace!(pos);
-            }
-            b'"' => {
-                // hmm, quotes; let's see where it ends
-                let pos = pos!();
-                let qidx = it.position(|x| *x == b'"');
-                match qidx {
-                    Some(idx) => query.push(&inp[pos..idx + pos]),
-                    None => {
-                        let end = pos!();
-                        return Err(TokenizerError::QuoteMismatch(
-                            String::from_utf8_lossy(&inp[pos..end]).to_string(),
-                        ));
-                    }
-                }
-                expect_whitespace!(pos);
-            }
-            b'`' => {
-                // hmm, backtick? let's look for the end
-                let pos = pos!();
-                let qidx = it.position(|x| *x == b'`');
-                match qidx {
-                    Some(idx) => query.push(&inp[pos..idx + pos]),
-                    None => {
-                        let end = pos!();
-                        return Err(TokenizerError::BacktickMismatch(
-                            String::from_utf8_lossy(&inp[pos..end]).to_string(),
-                        ));
-                    }
-                }
-                expect_whitespace!(pos);
-            }
-            b' ' => {
-                // this just prevents control from being handed to the wildcard
-                continue;
-            }
-            b'#' => {
-                // so this is an inline comment; skip until newline
-                let _ = it.position(|x| *x == b'\n');
-            }
-            _ => {
-                let start = pos!() - 1;
-                let mut end = start;
-                // alpha? cool, go on
-                'inner: while let Some(tok) = it.peek() {
-                    match **tok {
-                        b' ' => {
-                            it.next();
-                            break 'inner;
-                        }
-                        b'\'' | b'"' => {
-                            return Err(TokenizerError::ExpectedWhitespace(
-                                String::from_utf8_lossy(&inp[start..pos!()]).to_string(),
-                            ))
-                        }
-                        b'#' => continue 'outer,
-                        _ => {
-                            end += 1;
-                            it.next();
-                            continue 'inner;
-                        }
-                    }
-                }
-                end += 1;
-                query.push(&inp[start..end]);
-            }
-        }
-    }
-    Ok(query)
-}
diff --git a/examples/config-files/badcfg.toml b/examples/config-files/badcfg.toml
deleted file mode 100644
index 4e7c7e47..00000000
--- a/examples/config-files/badcfg.toml
+++ /dev/null
@@ -1,3 +0,0 @@
-# This is a 'bad' configuration file since it contains an invalid port
-[server]
-port = 20033002
\ No newline at end of file
diff --git a/examples/config-files/badcfg2.toml b/examples/config-files/badcfg2.toml
deleted file mode 100644
index 066661b2..00000000
--- a/examples/config-files/badcfg2.toml
+++ /dev/null
@@ -1,8 +0,0 @@
-# This is a bad toml file since it contains an invalid value of `every` in BGSAVE
-[server]
-host = "127.0.0.1"
-port = 2003
-
-[bgsave]
-every = 0.5 # Not possible!
-enabled = true
\ No newline at end of file
diff --git a/examples/config-files/bgsave-justenabled.toml b/examples/config-files/bgsave-justenabled.toml
deleted file mode 100644
index b1dfbeb9..00000000
--- a/examples/config-files/bgsave-justenabled.toml
+++ /dev/null
@@ -1,6 +0,0 @@
-[server]
-host = "127.0.0.1"
-port = 2003
-
-[bgsave]
-enabled = true
\ No newline at end of file
diff --git a/examples/config-files/bgsave-justevery.toml b/examples/config-files/bgsave-justevery.toml
deleted file mode 100644
index eb9e69a7..00000000
--- a/examples/config-files/bgsave-justevery.toml
+++ /dev/null
@@ -1,6 +0,0 @@
-[server]
-host = "127.0.0.1"
-port = 2003
-
-[bgsave]
-every = 600
\ No newline at end of file
diff --git a/examples/config-files/docker.toml b/examples/config-files/docker.toml
deleted file mode 100644
index 3b5058bd..00000000
--- a/examples/config-files/docker.toml
+++ /dev/null
@@ -1,5 +0,0 @@
-# this is the default configuration file for Docker images. Modify it as required
-[server]
-host = "0.0.0.0"
-port = 2003
-noart = true
diff --git a/examples/config-files/ipv6.toml b/examples/config-files/ipv6.toml
deleted file mode 100644
index c98fe3bc..00000000
--- a/examples/config-files/ipv6.toml
+++ /dev/null
@@ -1,4 +0,0 @@
-# This makes use of an IPv6 address
-[server]
-host = "::1"
-port = 2003
diff --git a/examples/config-files/secure-noart.toml b/examples/config-files/secure-noart.toml
deleted file mode 100644
index 14adffca..00000000
--- a/examples/config-files/secure-noart.toml
+++ /dev/null
@@ -1,4 +0,0 @@
-[server]
-host = '127.0.0.1' # i.e localhost
-port = 2003 # The port to bind to
-noart = true # No terminal artwork
\ No newline at end of file
diff --git a/examples/config-files/skyd.toml b/examples/config-files/skyd.toml
deleted file mode 100644
index 7bf537d7..00000000
--- a/examples/config-files/skyd.toml
+++ /dev/null
@@ -1,5 +0,0 @@
-# This is a 'good' configuration file since it contains a valid port
-# and appropriate keys
-[server]
-host = '127.0.0.1'
-port = 2003 # Set the server's port to 2003
\ No newline at end of file
diff --git a/examples/config-files/snapshot.toml b/examples/config-files/snapshot.toml
deleted file mode 100644
index fd202c9f..00000000
--- a/examples/config-files/snapshot.toml
+++ /dev/null
@@ -1,17 +0,0 @@
-[server]
-host = "127.0.0.1" # The IP address to which you want sdb to bind to
-port = 2003 # The port to which you want sdb to bind to
-# Set `noart` to true if you want to disable terminal artwork
-noart = false
-
-[bgsave]
-# Run `BGSAVE` `every` seconds. For example, setting this to 60 will cause BGSAVE to run
-# after every 2 minutes
-enabled = true
-every = 120
-
-[snapshot]
-# Create a snapshot every hour (1 hour = 60 minutes = 60 * 60 seconds = 3600 seconds)
-every = 3600
-# How many of the snapshots to keep
-atmost = 4 # keep the four most recent snapshots
diff --git a/examples/config-files/ssl.toml b/examples/config-files/ssl.toml
deleted file mode 100644
index c4e951e3..00000000
--- a/examples/config-files/ssl.toml
+++ /dev/null
@@ -1,17 +0,0 @@
-[server]
-host = "127.0.0.1"
-port = 2003
-noart = false
-
-[ssl]
-key = "/path/to/keyfile.pem"
-chain = "/path/to/chain.pem"
-port = 2004
-
-[bgsave]
-enabled = true
-every = 120
-
-[snapshot]
-every = 3600
-atmost = 4
\ No newline at end of file
diff --git a/examples/config-files/template.toml b/examples/config-files/template.toml
deleted file mode 100644
index bc21d894..00000000
--- a/examples/config-files/template.toml
+++ /dev/null
@@ -1,40 +0,0 @@
-# This is a complete sdb configuration template which is always kept updated
-# to include all the configuration options. I encourage you to always use this
-# when you use a configuration file
-# Instead of deleting entire sections from this file, comment them out, so that you
-# now what you've kept enabled and what you've kept disabled. This helps avoid
-# configuration problems during production
-
-# This is a *REQUIRED* key
-[server]
-host = "127.0.0.1" # The IP address to which you want sdb to bind to
-port = 2003        # The port to which you want sdb to bind to
-noart = false      # Set `noart` to true if you want to disable terminal artwork
-maxcon = 50000     # set the maximum number of clients that the server can accept
-mode = "dev"       # Set this to `prod` when you're running in production and `dev` when in development
-
-# This is an optional key
-[auth]
-# the origin key to be used to claim the root account
-origin_key = "4527387f92a381cbe804593f33991d327d456a97"
-
-# This key is *OPTIONAL*
-[bgsave]
-# Run `BGSAVE` `every` seconds. For example, setting this to 60 will cause BGSAVE to run
-# after every 2 minutes
-enabled = true
-every = 120
-
-# This key is *OPTIONAL*
-[snapshot]
-every = 3600    # Make a snapshot after every 1 hour (60min * 60sec= 3600secs)
-atmost = 4      # Keep the 4 most recent snapshots
-failsafe = true # stops accepting writes if snapshotting fails
-
-# This key is *OPTIONAL*, used for TLS/SSL config
-[ssl]
-key = "/path/to/keyfile.pem"
-chain = "/path/to/chain.pem"
-port = 2004
-only = true                             # optional to enable SSL-only requests
-passin = "/path/to/cert/passphrase.txt" # optional to programmatically verify the TLS cert
diff --git a/examples/config-files/template.yaml b/examples/config-files/template.yaml
new file mode 100644
index 00000000..1e7f286c
--- /dev/null
+++ b/examples/config-files/template.yaml
@@ -0,0 +1,22 @@
+system:
+  mode: prod
+  rs_window: 600
+
+auth:
+  plugin: pwd
+  # replace with your root password of choice
+  root_pass: password
+
+endpoints:
+  secure:
+    host: 127.0.0.1
+    port: 2004
+    # replace `cert` with the path to your self-signed certificate
+    cert: cert.pem
+    # replace `private_key` with the path to your private key
+    private_key: private.key
+    # replace `passphrase.txt` with the path to your private key passphrase
+    pkey_passphrase: passphrase.txt
+  insecure:
+    host: 127.0.0.1
+    port: 2003
diff --git a/examples/config-files/withcustombgsave.toml b/examples/config-files/withcustombgsave.toml
deleted file mode 100644
index 34ee730b..00000000
--- a/examples/config-files/withcustombgsave.toml
+++ /dev/null
@@ -1,7 +0,0 @@
-[server]
-host = "127.0.0.1"
-port = 2003
-
-[bgsave]
-enabled = true
-every = 600 # Every 10 minutes
diff --git a/harness/Cargo.toml b/harness/Cargo.toml
index 246f9243..605e03c9 100644
--- a/harness/Cargo.toml
+++ b/harness/Cargo.toml
@@ -7,9 +7,7 @@ edition = "2021"
 
 [dependencies]
 # internal deps
-skytable = { git = "https://github.com/skytable/client-rust.git", features = [
-    "sync",
-], default-features = false }
+skytable = { git = "https://github.com/skytable/client-rust.git", branch = "octave" }
 libsky = { path = "../libsky" }
 # external deps
 env_logger = "0.10.0"
diff --git a/harness/src/build.rs b/harness/src/build.rs
index 2664e1ec..e897060a 100644
--- a/harness/src/build.rs
+++ b/harness/src/build.rs
@@ -34,7 +34,7 @@ use {
 };
 
 /// The binaries that will be present in a bundle
-pub const BINARIES: [&str; 4] = ["skyd", "sky-bench", "skysh", "sky-migrate"];
+pub const BINARIES: [&str; 3] = ["skyd", "sky-bench", "skysh"];
 
 /// The build mode
 #[derive(Copy, Clone, PartialEq)]
diff --git a/harness/src/test/mod.rs b/harness/src/test/mod.rs
index ae2afda1..2695e109 100644
--- a/harness/src/test/mod.rs
+++ b/harness/src/test/mod.rs
@@ -35,8 +35,9 @@ use {
         bn::{BigNum, MsbOption},
         error::ErrorStack,
         hash::MessageDigest,
-        pkey::{PKey, Private},
+        pkey::PKey,
         rsa::Rsa,
+        symm::Cipher,
         x509::{
             extension::{BasicConstraints, KeyUsage, SubjectKeyIdentifier},
             X509NameBuilder, X509,
@@ -89,16 +90,16 @@ fn append_target(args: &mut Vec<String>) {
 /// - The standard test suite
 /// - The persistence test suite
 fn run_test_inner() -> HarnessResult<()> {
+    const TEST_PASSWORD: &str = "xCqe4yuVM7l2MnHZOFZDDieqjqmmL3qvO5LOEOhpXPE=";
     // first create the TLS keys
     info!("Creating TLS key+cert");
-    let (cert, pkey) = mk_ca_cert().expect("Failed to create cert");
+    let (cert, pkey) = mk_ca_cert(TEST_PASSWORD.as_bytes()).expect("Failed to create cert");
+    let mut passfile = fs::File::create("passphrase.txt").unwrap();
+    passfile.write_all(TEST_PASSWORD.as_bytes()).unwrap();
     let mut certfile = fs::File::create("cert.pem").expect("failed to create cert.pem");
-    certfile.write_all(&cert.to_pem().unwrap()).unwrap();
+    certfile.write_all(&cert).unwrap();
     let mut pkeyfile = fs::File::create("key.pem").expect("failed to create key.pem");
-    pkeyfile
-        .write_all(&pkey.private_key_to_pem_pkcs8().unwrap())
-        .unwrap();
-
+    pkeyfile.write_all(&pkey).unwrap();
     // assemble commands
     let target_folder = util::get_target_folder(BuildMode::Debug);
     let mut standard_test_suite_args = vec!["cargo".to_owned(), "test".into()];
@@ -110,15 +111,9 @@ fn run_test_inner() -> HarnessResult<()> {
     ];
     append_target(&mut build_cmd_args);
     append_target(&mut standard_test_suite_args);
-    let persist_test_suite_args = [
-        standard_test_suite_args.as_slice(),
-        &["--features".to_owned(), "persist-suite".into()],
-    ]
-    .concat();
     // get cmd
     let build_cmd = util::assemble_command_from_slice(build_cmd_args);
     let standard_test_suite = util::assemble_command_from_slice(standard_test_suite_args);
-    let persist_test_suite = util::assemble_command_from_slice(persist_test_suite_args);
 
     // build skyd
     info!("Building server binary ...");
@@ -130,20 +125,11 @@ fn run_test_inner() -> HarnessResult<()> {
         util::handle_child("standard test suite", standard_test_suite)?;
         Ok(())
     })?;
-
-    // run persistence tests; don't kill the servers because if either of the tests fail
-    // then we can ensure that they will be killed by `run_test`
-    svc::run_with_servers(&target_folder, false, move || {
-        info!("Running persistence test suite ...");
-        util::handle_child("standard test suite", persist_test_suite)?;
-        Ok(())
-    })?;
-
     Ok(())
 }
 
 /// Generate certificates
-fn mk_ca_cert() -> Result<(X509, PKey<Private>), ErrorStack> {
+fn mk_ca_cert(password: &[u8]) -> Result<(Vec<u8>, Vec<u8>), ErrorStack> {
     let rsa = Rsa::generate(2048)?;
     let key_pair = PKey::from_rsa(rsa)?;
 
@@ -182,9 +168,8 @@ fn mk_ca_cert() -> Result<(X509, PKey<Private>), ErrorStack> {
     let subject_key_identifier =
         SubjectKeyIdentifier::new().build(&cert_builder.x509v3_context(None, None))?;
     cert_builder.append_extension(subject_key_identifier)?;
-
     cert_builder.sign(&key_pair, MessageDigest::sha256())?;
-    let cert = cert_builder.build();
-
+    let cert = cert_builder.build().to_pem().unwrap();
+    let key_pair = key_pair.private_key_to_pem_pkcs8_passphrase(Cipher::des_cbc(), password)?;
     Ok((cert, key_pair))
 }
diff --git a/harness/src/test/svc.rs b/harness/src/test/svc.rs
index 0c583969..433e0fcd 100644
--- a/harness/src/test/svc.rs
+++ b/harness/src/test/svc.rs
@@ -26,12 +26,16 @@
 
 #[cfg(windows)]
 use std::os::windows::process::CommandExt;
+
 use {
     crate::{
         util::{self},
         HarnessError, HarnessResult, ROOT_DIR,
     },
-    skytable::{error::Error, Connection, SkyResult},
+    skytable::{
+        error::{ClientResult, Error},
+        Config, Connection,
+    },
     std::{
         io::ErrorKind,
         path::Path,
@@ -45,16 +49,17 @@ const POWERSHELL_SCRIPT: &str = include_str!("../../../ci/windows/stop.ps1");
 #[cfg(windows)]
 /// Flag for new console Window
 const CREATE_NEW_CONSOLE: u32 = 0x00000010;
-pub(super) const SERVERS: [(&str, [u16; 2]); 3] = [
-    ("server1", [2003, 2004]),
-    ("server2", [2005, 2006]),
-    ("server3", [2007, 2008]),
-];
+pub(super) const SERVERS: [(&str, [u16; 2]); 1] = [("server1", [2003, 2004])];
 /// The test suite server host
 const TESTSUITE_SERVER_HOST: &str = "127.0.0.1";
 /// The workspace root
 const WORKSPACE_ROOT: &str = env!("ROOT_DIR");
 
+fn connect_db(host: &str, port: u16) -> ClientResult<Connection> {
+    let cfg = Config::new(host, port, "root", "password12345678");
+    cfg.connect()
+}
+
 /// Get the command to start the provided server1
 pub fn get_run_server_cmd(server_id: &'static str, target_folder: impl AsRef<Path>) -> Command {
     let args = vec![
@@ -63,8 +68,8 @@ pub fn get_run_server_cmd(server_id: &'static str, target_folder: impl AsRef<Pat
             .to_string_lossy()
             .to_string(),
         // config
-        "--withconfig".to_owned(),
-        format!("{WORKSPACE_ROOT}ci/{server_id}.toml"),
+        "--config".to_owned(),
+        format!("{WORKSPACE_ROOT}ci/{server_id}.yaml"),
     ];
     let mut cmd = util::assemble_command_from_slice(&args);
     cmd.current_dir(server_id);
@@ -94,7 +99,7 @@ pub(super) fn wait_for_server_exit() -> HarnessResult<()> {
     Ok(())
 }
 
-fn connection_refused<T>(input: SkyResult<T>) -> HarnessResult<bool> {
+fn connection_refused<T>(input: ClientResult<T>) -> HarnessResult<bool> {
     match input {
         Ok(_) => Ok(false),
         Err(Error::IoError(e))
@@ -118,7 +123,7 @@ fn wait_for_startup() -> HarnessResult<()> {
         for port in ports {
             let connection_string = format!("{TESTSUITE_SERVER_HOST}:{port}");
             let mut backoff = 1;
-            let mut con = Connection::new(TESTSUITE_SERVER_HOST, port);
+            let mut con = connect_db(TESTSUITE_SERVER_HOST, port);
             while connection_refused(con)? {
                 if backoff > 64 {
                     // enough sleeping, return an error
@@ -131,7 +136,7 @@ fn wait_for_startup() -> HarnessResult<()> {
                 "Server at {connection_string} not started. Sleeping for {backoff} second(s) ..."
             );
                 util::sleep_sec(backoff);
-                con = Connection::new(TESTSUITE_SERVER_HOST, port);
+                con = connect_db(TESTSUITE_SERVER_HOST, port);
                 backoff *= 2;
             }
             info!("Server at {connection_string} has started");
@@ -148,7 +153,7 @@ fn wait_for_shutdown() -> HarnessResult<()> {
         for port in ports {
             let connection_string = format!("{TESTSUITE_SERVER_HOST}:{port}");
             let mut backoff = 1;
-            let mut con = Connection::new(TESTSUITE_SERVER_HOST, port);
+            let mut con = connect_db(TESTSUITE_SERVER_HOST, port);
             while !connection_refused(con)? {
                 if backoff > 64 {
                     // enough sleeping, return an error
@@ -161,7 +166,7 @@ fn wait_for_shutdown() -> HarnessResult<()> {
                 "Server at {connection_string} still active. Sleeping for {backoff} second(s) ..."
             );
                 util::sleep_sec(backoff);
-                con = Connection::new(TESTSUITE_SERVER_HOST, port);
+                con = connect_db(TESTSUITE_SERVER_HOST, port);
                 backoff *= 2;
             }
             info!("Server at {connection_string} has stopped accepting connections");
diff --git a/pkg/debian/description.txt b/pkg/debian/description.txt
index 15a8dca7..ac8d681f 100644
--- a/pkg/debian/description.txt
+++ b/pkg/debian/description.txt
@@ -1,5 +1,5 @@
 Skytable is a free and open-source NoSQL database that aims
 to provide flexibility in data modeling at scale.
 The `skytable` package contains the database server (`skyd`),
-an interactive command-line client (`skysh`), a benchmarking
-tool (`sky-bench`) and a migration tool (`sky-migrate`).
\ No newline at end of file
+an interactive command-line client (`skysh`) and a benchmarking
+tool (`sky-bench`).
\ No newline at end of file
diff --git a/server/Cargo.toml b/server/Cargo.toml
index e218fcbc..d99e8c5a 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -18,7 +18,7 @@ openssl = { version = "0.10.56", features = ["vendored"] }
 crossbeam-epoch = { version = "0.9.15" }
 parking_lot = "0.12.1"
 serde = { version = "1.0.183", features = ["derive"] }
-tokio = { version = "1.30.0", features = ["full"] }
+tokio = { version = "1.34.0", features = ["full"] }
 tokio-openssl = "0.6.3"
 uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics"] }
 crc = "3.0.1"
@@ -42,7 +42,7 @@ cc = "1.0.82"
 [dev-dependencies]
 # external deps
 rand = "0.8.5"
-tokio = { version = "1.30.0", features = ["test-util"] }
+tokio = { version = "1.34.0", features = ["test-util"] }
 
 [features]
 nightly = []
diff --git a/server/src/cli.yml b/server/src/cli.yml
deleted file mode 100644
index c8a3024c..00000000
--- a/server/src/cli.yml
+++ /dev/null
@@ -1,123 +0,0 @@
-name: Skytable Server
-version: 0.8.0
-author: Sayan N. <ohsayan@outlook.com>
-about: The Skytable Database server
-args:
-  - config:
-      short: c
-      required: false
-      long: withconfig
-      value_name: cfgfile
-      help: Sets a configuration file to start skyd
-      takes_value: true
-  - restore:
-      short: r
-      required: false
-      long: restore
-      value_name: backupdir
-      help: Restores data from a previous snapshot made in the provided directory
-      takes_value: true
-  - host:
-      short: h
-      required: false
-      long: host
-      value_name: host
-      help: Sets the host to which the server will bind
-      takes_value: true
-  - port:
-      short: p
-      required: false
-      long: port
-      value_name: port
-      help: Sets the port to which the server will bind
-      takes_value: true
-  - noart:
-      required: false
-      long: noart
-      help: Disables terminal artwork
-      takes_value: false
-  - nosave:
-      required: false
-      long: nosave
-      help: Disables automated background saving
-      takes_value: false
-  - saveduration:
-      required: false
-      long: saveduration
-      value_name: duration
-      short: S
-      takes_value: true
-      help: Set the BGSAVE duration
-  - snapevery:
-      required: false
-      long: snapevery
-      value_name: duration
-      help: Set the periodic snapshot duration
-      takes_value: true
-  - snapkeep:
-      required: false
-      long: snapkeep
-      value_name: count
-      help: Sets the number of most recent snapshots to keep
-      takes_value: true
-  - sslkey:
-      required: false
-      long: sslkey
-      short: k
-      value_name: key
-      help: Sets the PEM key file to use for SSL/TLS
-      takes_value: true
-  - sslchain:
-      required: false
-      long: sslchain
-      short: z
-      value_name: chain
-      help: Sets the PEM chain file to use for SSL/TLS
-      takes_value: true
-  - sslonly:
-      required: false
-      long: sslonly
-      takes_value: false
-      help: Tells the server to only accept SSL connections and disables the non-SSL port
-  - sslport:
-      required: false
-      long: sslport
-      takes_value: true
-      value_name: sslport
-      help: Set a custom SSL port to bind to
-  - tlspassin:
-      required: false
-      long: tlspassin
-      takes_value: true
-      value_name: tlspassin
-      help: Path to the file containing the passphrase for the TLS certificate
-  - stopwriteonfail:
-      required: false
-      long: stop-write-on-fail
-      takes_value: true
-      help: Stop accepting writes if any persistence method except BGSAVE fails (defaults to true)
-  - maxcon:
-      required: false
-      long: maxcon
-      takes_value: true
-      help: Set the maximum number of connections
-      value_name: maxcon
-  - mode:
-      required: false
-      long: mode
-      takes_value: true
-      short: m
-      help: Sets the deployment type
-      value_name: mode
-  - authkey:
-      required: false
-      long: auth-origin-key
-      takes_value: true
-      help: Set the authentication origin key
-      value_name: origin_key
-  - protover:
-      required: false
-      long: protover
-      takes_value: true
-      help: Set the protocol version
-      value_name: protover
diff --git a/server/src/engine/config.rs b/server/src/engine/config.rs
index 93c44647..330d0639 100644
--- a/server/src/engine/config.rs
+++ b/server/src/engine/config.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    crate::engine::error::RuntimeResult,
+    crate::engine::{error::RuntimeResult, fractal},
     core::fmt,
     serde::Deserialize,
     std::{collections::HashMap, fs},
@@ -97,7 +97,6 @@ impl Configuration {
     }
     const DEFAULT_HOST: &'static str = "127.0.0.1";
     const DEFAULT_PORT_TCP: u16 = 2003;
-    const DEFAULT_RELIABILITY_SVC_PING: u64 = 5 * 60;
     pub fn default_dev_mode(auth: DecodedAuth) -> Self {
         Self {
             endpoints: ConfigEndpoint::Insecure(ConfigEndpointTcp {
@@ -105,9 +104,7 @@ impl Configuration {
                 port: Self::DEFAULT_PORT_TCP,
             }),
             mode: ConfigMode::Dev,
-            system: ConfigSystem {
-                reliability_system_window: Self::DEFAULT_RELIABILITY_SVC_PING,
-            },
+            system: ConfigSystem::new(fractal::GENERAL_EXECUTOR_WINDOW),
             auth: ConfigAuth::new(auth.plugin, auth.root_pass),
         }
     }
@@ -204,11 +201,10 @@ pub enum ConfigMode {
 /// System configuration settings
 pub struct ConfigSystem {
     /// time window in seconds for the reliability system to kick-in automatically
-    reliability_system_window: u64,
+    pub reliability_system_window: u64,
 }
 
 impl ConfigSystem {
-    #[cfg(test)]
     pub fn new(reliability_system_window: u64) -> Self {
         Self {
             reliability_system_window,
@@ -472,6 +468,7 @@ fn decode_tls_ep(
     host: &str,
     port: u16,
 ) -> RuntimeResult<DecodedEPSecureConfig> {
+    super::fractal::context::set_dmsg("loading TLS configuration from disk");
     let tls_key = fs::read_to_string(key_path)?;
     let tls_cert = fs::read_to_string(cert_path)?;
     let tls_priv_key_passphrase = fs::read_to_string(pkey_pass)?;
@@ -653,6 +650,7 @@ Flags:
   -v, --version              Display the version number and exit.
 
 Options:
+  --config <path>             Set configuration options using the config file
   --tlscert <path>            Specify the path to the TLS certificate.
   --tlskey <path>             Specify the path to the TLS private key.
   --endpoint <definition>     Designate an endpoint. Format: protocol@host:port.
@@ -1075,6 +1073,7 @@ fn get_file_from_store(filename: &str) -> RuntimeResult<String> {
     }
     #[cfg(not(test))]
     {
+        super::fractal::context::set_dmsg("loading configuration file from disk");
         f = Ok(fs::read_to_string(filename)?);
     }
     f
@@ -1202,6 +1201,12 @@ fn check_config_file(
         match config_from_file.endpoints.as_mut() {
             Some(ep) => match ep.secure.as_mut() {
                 Some(secure_ep) => {
+                    super::fractal::context::set_dmsg("loading TLS configuration from disk");
+                    dbg!(
+                        &secure_ep.cert,
+                        &secure_ep.private_key,
+                        &secure_ep.pkey_passphrase
+                    );
                     let cert = fs::read_to_string(&secure_ep.cert)?;
                     let private_key = fs::read_to_string(&secure_ep.private_key)?;
                     let private_key_passphrase = fs::read_to_string(&secure_ep.pkey_passphrase)?;
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index c5c1104b..4446a9e5 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -187,6 +187,12 @@ fn blocking_exec_sysctl(
             let userdel = ASTNode::from_state(&mut state)?;
             super::dcl::drop_user(&g, cstate, userdel)
         }
+        (Token::Ident(k1), Token::Ident(k2))
+            if k1.eq_ignore_ascii_case("report") && k2.eq_ignore_ascii_case("status") =>
+        {
+            // TODO(@ohsayan): replace dummy endpoint with actual `system report status` responses
+            Ok(())
+        }
         _ => Err(QueryError::QLUnknownStatement),
     }
 }
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 1cabe3eb..63906601 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -46,6 +46,8 @@ use {
     },
 };
 
+pub const GENERAL_EXECUTOR_WINDOW: u64 = 5 * 60;
+
 /// A task for the [`FractalMgr`] to perform
 pub struct Task<T> {
     threshold: usize,
@@ -190,13 +192,13 @@ impl FractalBoot {
             hp_recv,
         }
     }
-    pub fn boot(self, sigterm: &broadcast::Sender<()>) -> FractalHandle {
+    pub fn boot(self, sigterm: &broadcast::Sender<()>, rs_window: u64) -> FractalHandle {
         let Self {
             global,
             lp_recv: lp_receiver,
             hp_recv: hp_receiver,
         } = self;
-        FractalMgr::start_all(global, sigterm, lp_receiver, hp_receiver)
+        FractalMgr::start_all(global, sigterm, lp_receiver, hp_receiver, rs_window)
     }
 }
 
@@ -207,6 +209,7 @@ impl FractalMgr {
         sigterm: &broadcast::Sender<()>,
         lp_receiver: UnboundedReceiver<Task<GenericTask>>,
         hp_receiver: UnboundedReceiver<Task<CriticalTask>>,
+        rs_window: u64,
     ) -> FractalHandle {
         let fractal_mgr = global.get_state().fractal_mgr();
         let global_1 = global.clone();
@@ -217,7 +220,14 @@ impl FractalMgr {
         });
         let sigterm_rx = sigterm.subscribe();
         let lp_handle = tokio::spawn(async move {
-            FractalMgr::general_executor_svc(fractal_mgr, global_2, lp_receiver, sigterm_rx).await
+            FractalMgr::general_executor_svc(
+                fractal_mgr,
+                global_2,
+                lp_receiver,
+                sigterm_rx,
+                rs_window,
+            )
+            .await
         });
         FractalHandle {
             hp_handle,
@@ -228,7 +238,6 @@ impl FractalMgr {
 
 // services
 impl FractalMgr {
-    const GENERAL_EXECUTOR_WINDOW: u64 = 5 * 60;
     /// The high priority executor service runs in the background to take care of high priority tasks and take any
     /// appropriate action. It will exclusively own the high priority queue since it is the only broker that is
     /// allowed to perform HP tasks
@@ -323,7 +332,9 @@ impl FractalMgr {
         global: super::Global,
         mut lpq: UnboundedReceiver<Task<GenericTask>>,
         mut sigterm: broadcast::Receiver<()>,
+        rs_window: u64,
     ) {
+        let dur = std::time::Duration::from_secs(rs_window);
         loop {
             tokio::select! {
                 _ = sigterm.recv() => {
@@ -333,7 +344,7 @@ impl FractalMgr {
                     info!("flp: exited executor service");
                     break;
                 },
-                _ = tokio::time::sleep(std::time::Duration::from_secs(Self::GENERAL_EXECUTOR_WINDOW)) => {
+                _ = tokio::time::sleep(dur) => {
                     let global = global.clone();
                     tokio::task::spawn_blocking(|| self.general_executor_model_maintenance(global)).await.unwrap()
                 }
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index 74975e37..1645f2e6 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -51,7 +51,7 @@ pub mod test_utils;
 mod util;
 pub use {
     drivers::FractalModelDriver,
-    mgr::{CriticalTask, GenericTask, Task},
+    mgr::{CriticalTask, GenericTask, Task, GENERAL_EXECUTOR_WINDOW},
     util::FractalToken,
 };
 
diff --git a/server/src/engine/fractal/sys_store.rs b/server/src/engine/fractal/sys_store.rs
index 625285d0..7ec79546 100644
--- a/server/src/engine/fractal/sys_store.rs
+++ b/server/src/engine/fractal/sys_store.rs
@@ -207,7 +207,7 @@ impl SysAuth {
     pub fn new(users: HashMap<Box<str>, SysAuthUser>) -> Self {
         Self { users }
     }
-    pub fn verify_user_is_root<T: AsRef<[u8]> + ?Sized>(
+    pub fn verify_user_check_root<T: AsRef<[u8]> + ?Sized>(
         &self,
         username: &str,
         password: &T,
@@ -225,7 +225,7 @@ impl SysAuth {
         username: &str,
         password: &T,
     ) -> QueryResult<()> {
-        self.verify_user_is_root(username, password).map(|_| ())
+        self.verify_user_check_root(username, password).map(|_| ())
     }
     pub fn users(&self) -> &HashMap<Box<str>, SysAuthUser> {
         &self.users
diff --git a/server/src/engine/mod.rs b/server/src/engine/mod.rs
index bb8f7788..73f75835 100644
--- a/server/src/engine/mod.rs
+++ b/server/src/engine/mod.rs
@@ -144,14 +144,16 @@ impl EndpointListeners {
 
 pub async fn start(
     termsig: TerminationSignal,
-    Configuration { endpoints, .. }: Configuration,
+    Configuration {
+        endpoints, system, ..
+    }: Configuration,
     fractal::GlobalStateStart { global, boot }: fractal::GlobalStateStart,
 ) -> RuntimeResult<()> {
     // create our system-wide channel
     let (signal, _) = broadcast::channel::<()>(1);
     // start our services
     context::set_dmsg("starting fractal engine");
-    let fractal_handle = boot.boot(&signal);
+    let fractal_handle = boot.boot(&signal, system.reliability_system_window);
     // create our server
     context::set(Subsystem::Network, "initializing endpoints");
     let str;
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index a553d5db..c539309c 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -217,7 +217,7 @@ async fn do_handshake<S: Socket>(
     match core::str::from_utf8(handshake.hs_auth().username()) {
         Ok(uname) => {
             let auth = global.sys_store().system_store().auth_data().read();
-            let r = auth.verify_user_is_root(uname, handshake.hs_auth().password());
+            let r = auth.verify_user_check_root(uname, handshake.hs_auth().password());
             match r {
                 Ok(is_root) => {
                     let hs = handshake.hs_static();
diff --git a/server/src/engine/net/protocol/tests.rs b/server/src/engine/net/protocol/tests.rs
index b7f13946..c32ca03e 100644
--- a/server/src/engine/net/protocol/tests.rs
+++ b/server/src/engine/net/protocol/tests.rs
@@ -39,7 +39,7 @@ use crate::engine::{
     client handshake
 */
 
-const FULL_HANDSHAKE_WITH_AUTH: [u8; 23] = *b"H\0\0\0\0\x015\n8\nsayanpass1234";
+const FULL_HANDSHAKE_WITH_AUTH: [u8; 23] = *b"H\0\0\0\0\x005\n8\nsayanpass1234";
 
 const STATIC_HANDSHAKE_WITH_AUTH: CHandshakeStatic = CHandshakeStatic::new(
     HandshakeVersion::Original,
diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
index 75559a7f..7ec18033 100644
--- a/server/src/engine/txn/gns/tests/full_chain.rs
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -83,7 +83,9 @@ fn create_space() {
                 &*space,
                 &Space::new_restore_empty(
                     uuid,
-                    into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536)))
+                    into_dict!("env" => DictEntryGeneric::Map(
+                        into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536)))
+                    ))
                 )
             );
         })
@@ -111,7 +113,9 @@ fn alter_space() {
                 &*space,
                 &Space::new_restore_empty(
                     uuid,
-                    into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536)))
+                    into_dict!("env" => DictEntryGeneric::Map(
+                        into_dict!("SAYAN_MAX" => DictEntryGeneric::Data(Datacell::new_uint_default(65536))
+                    )))
                 )
             );
         })
diff --git a/sky-bench/Cargo.toml b/sky-bench/Cargo.toml
index 8726792b..19595da4 100644
--- a/sky-bench/Cargo.toml
+++ b/sky-bench/Cargo.toml
@@ -8,16 +8,5 @@ version = "0.8.0"
 
 [dependencies]
 # internal deps
-skytable = { git = "https://github.com/skytable/client-rust.git", features = [
-    "sync",
-    "dbg",
-] }
+skytable = { git = "https://github.com/skytable/client-rust.git", branch = "octave" }
 libstress = { path = "../libstress" }
-# external deps
-clap = { version = "2", features = ["yaml"] }
-log = "0.4.19"
-env_logger = "0.10.0"
-devtimer = "4.0.1"
-serde = { version = "1.0.183", features = ["derive"] }
-serde_json = "1.0.104"
-rand = "0.8.5"
diff --git a/sky-bench/src/bench/benches.rs b/sky-bench/src/bench/benches.rs
deleted file mode 100644
index 87dc93d4..00000000
--- a/sky-bench/src/bench/benches.rs
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Created on Sat Aug 13 2022
- *
- * This file is a part of S{
-    let ref this = loopmon;
-    this.current
-}le (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    super::{
-        report::{AggregateReport, SingleReport},
-        validation, vec_with_cap, BenchmarkConfig, LoopMonitor,
-    },
-    crate::error::BResult,
-    devtimer::SimpleTimer,
-    libstress::Workpool,
-    skytable::{types::RawString, Connection, Element, Query, RespCode},
-    std::{
-        io::{Read, Write},
-        net::{Shutdown, TcpStream},
-    },
-};
-
-/// Run a benchmark using the given pre-loop, in-loop and post-loop closures
-fn run_bench_custom<Inp, Lp, Lv, Ex>(
-    bench_config: BenchmarkConfig,
-    packets: Vec<Box<[u8]>>,
-    on_init: Lv,
-    on_loop: Lp,
-    on_loop_exit: Ex,
-    loopmon: LoopMonitor,
-    reports: &mut AggregateReport,
-) -> BResult<()>
-where
-    Ex: Clone + Fn(&mut Inp) + Send + Sync + 'static,
-    Inp: Sync + 'static,
-    Lp: Clone + Fn(&mut Inp, Box<[u8]>) + Send + Sync + 'static,
-    Lv: Clone + Fn() -> Inp + Send + 'static + Sync,
-{
-    // now do our runs
-    let mut loopmon = loopmon;
-
-    while loopmon.should_continue() {
-        // now create our connection pool
-        let pool = Workpool::new(
-            bench_config.server.connections(),
-            on_init.clone(),
-            on_loop.clone(),
-            on_loop_exit.clone(),
-            true,
-            Some(bench_config.query_count()),
-        )?;
-
-        // get our local copy
-        let this_packets = packets.clone();
-
-        // run and time our operations
-        let mut dt = SimpleTimer::new();
-        dt.start();
-        pool.execute_and_finish_iter(this_packets);
-        dt.stop();
-        loopmon.incr_time(&dt);
-
-        // cleanup
-        loopmon.cleanup()?;
-        loopmon.step();
-    }
-
-    // save time
-    reports.push(SingleReport::new(
-        loopmon.name(),
-        loopmon.sum() as f64 / bench_config.runs() as f64,
-    ));
-    Ok(())
-}
-
-#[inline(always)]
-/// Init connection and buffer
-fn init_connection_and_buf(
-    host: &str,
-    port: u16,
-    start_command: Vec<u8>,
-    bufsize: usize,
-) -> (TcpStream, Vec<u8>) {
-    let mut con = TcpStream::connect((host, port)).unwrap();
-    con.write_all(&start_command).unwrap();
-    let mut ret = [0u8; validation::RESPCODE_OKAY.len()];
-    con.read_exact(&mut ret).unwrap();
-    let readbuf = vec![0; bufsize];
-    (con, readbuf)
-}
-
-/// Benchmark SET
-pub fn bench_set(
-    keys: &[Vec<u8>],
-    values: &[Vec<u8>],
-    connection: &mut Connection,
-    bench_config: &BenchmarkConfig,
-    create_table: &[u8],
-    reports: &mut AggregateReport,
-) -> BResult<()> {
-    let bench_config = bench_config.clone();
-    let create_table = create_table.to_owned();
-    let loopmon = LoopMonitor::new_cleanup(
-        bench_config.runs(),
-        "set",
-        connection,
-        Query::from("FLUSHDB").arg("default.tmpbench"),
-        Element::RespCode(RespCode::Okay),
-        true,
-    );
-    let mut packets = vec_with_cap(bench_config.query_count())?;
-    (0..bench_config.query_count()).for_each(|i| {
-        packets.push(
-            Query::from("SET")
-                .arg(RawString::from(keys[i].to_owned()))
-                .arg(RawString::from(values[i].to_owned()))
-                .into_raw_query()
-                .into_boxed_slice(),
-        )
-    });
-    run_bench_custom(
-        bench_config.clone(),
-        packets,
-        move || {
-            init_connection_and_buf(
-                bench_config.server.host(),
-                bench_config.server.port(),
-                create_table.to_owned(),
-                validation::RESPCODE_OKAY.len(),
-            )
-        },
-        |(con, buf), packet| {
-            con.write_all(&packet).unwrap();
-            con.read_exact(buf).unwrap();
-            assert_eq!(buf, validation::RESPCODE_OKAY);
-        },
-        |(con, _)| con.shutdown(Shutdown::Both).unwrap(),
-        loopmon,
-        reports,
-    )
-}
-
-/// Benchmark UPDATE
-pub fn bench_update(
-    keys: &[Vec<u8>],
-    new_value: &[u8],
-    bench_config: &BenchmarkConfig,
-    create_table: &[u8],
-    reports: &mut AggregateReport,
-) -> BResult<()> {
-    let bench_config = bench_config.clone();
-    let create_table = create_table.to_owned();
-    let loopmon = LoopMonitor::new(bench_config.runs(), "update");
-    let mut packets = vec_with_cap(bench_config.query_count())?;
-    (0..bench_config.query_count()).for_each(|i| {
-        packets.push(
-            Query::from("update")
-                .arg(RawString::from(keys[i].clone()))
-                .arg(RawString::from(new_value.to_owned()))
-                .into_raw_query()
-                .into_boxed_slice(),
-        )
-    });
-    run_bench_custom(
-        bench_config.clone(),
-        packets,
-        move || {
-            init_connection_and_buf(
-                bench_config.server.host(),
-                bench_config.server.port(),
-                create_table.to_owned(),
-                validation::RESPCODE_OKAY.len(),
-            )
-        },
-        |(con, buf), packet| {
-            con.write_all(&packet).unwrap();
-            con.read_exact(buf).unwrap();
-            assert_eq!(buf, validation::RESPCODE_OKAY);
-        },
-        |(con, _)| con.shutdown(Shutdown::Both).unwrap(),
-        loopmon,
-        reports,
-    )
-}
-
-/// Benchmark GET
-pub fn bench_get(
-    keys: &[Vec<u8>],
-    bench_config: &BenchmarkConfig,
-    create_table: &[u8],
-    reports: &mut AggregateReport,
-) -> BResult<()> {
-    let bench_config = bench_config.clone();
-    let create_table = create_table.to_owned();
-    let loopmon = LoopMonitor::new(bench_config.runs(), "get");
-    let mut packets = vec_with_cap(bench_config.query_count())?;
-    (0..bench_config.query_count()).for_each(|i| {
-        packets.push(
-            Query::from("get")
-                .arg(RawString::from(keys[i].clone()))
-                .into_raw_query()
-                .into_boxed_slice(),
-        )
-    });
-    run_bench_custom(
-        bench_config.clone(),
-        packets,
-        move || {
-            init_connection_and_buf(
-                bench_config.server.host(),
-                bench_config.server.port(),
-                create_table.to_owned(),
-                validation::calculate_response_size(bench_config.kvsize()),
-            )
-        },
-        |(con, buf), packet| {
-            con.write_all(&packet).unwrap();
-            con.read_exact(buf).unwrap();
-        },
-        |(con, _)| con.shutdown(Shutdown::Both).unwrap(),
-        loopmon,
-        reports,
-    )
-}
diff --git a/sky-bench/src/bench/mod.rs b/sky-bench/src/bench/mod.rs
deleted file mode 100644
index 77cb2789..00000000
--- a/sky-bench/src/bench/mod.rs
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
- * Created on Tue Aug 09 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    self::report::AggregateReport,
-    crate::{
-        config,
-        config::{BenchmarkConfig, ServerConfig},
-        error::{BResult, Error},
-        util,
-    },
-    clap::ArgMatches,
-    devtimer::SimpleTimer,
-    libstress::utils::{generate_random_byte_vector, ran_bytes},
-    skytable::{Connection, Element, Query, RespCode},
-};
-
-mod benches;
-mod report;
-mod validation;
-
-macro_rules! binfo {
-    ($($arg:tt)+) => {
-        if $crate::config::should_output_messages() {
-            ::log::info!($($arg)+)
-        }
-    };
-}
-
-/// The loop monitor can be used for maintaining a loop for a given benchmark
-struct LoopMonitor<'a> {
-    /// cleanup instructions
-    inner: Option<CleanupInner<'a>>,
-    /// maximum iterations
-    max: usize,
-    /// current iteration
-    current: usize,
-    /// total time
-    time: u128,
-    /// name of test
-    name: &'static str,
-}
-
-impl<'a> LoopMonitor<'a> {
-    /// Create a benchmark loop monitor that doesn't need any cleanup
-    pub fn new(max: usize, name: &'static str) -> Self {
-        Self {
-            inner: None,
-            max,
-            current: 0,
-            time: 0,
-            name,
-        }
-    }
-    /// Create a new benchmark loop monitor that uses the given cleanup instructions:
-    /// - `max`: Total iterations
-    /// - `name`: Name of benchmark
-    /// - `connection`: A connection to use for cleanup instructions
-    /// - `query`: Query to run for cleanup
-    /// - `response`: Response expected when cleaned up
-    /// - `skip_on_last`: Skip running the cleanup instructions on the last loop
-    pub fn new_cleanup(
-        max: usize,
-        name: &'static str,
-        connection: &'a mut Connection,
-        query: Query,
-        response: Element,
-        skip_on_last: bool,
-    ) -> Self {
-        Self {
-            inner: Some(CleanupInner::new(query, response, connection, skip_on_last)),
-            max,
-            current: 0,
-            time: 0,
-            name,
-        }
-    }
-    /// Run cleanup
-    fn cleanup(&mut self) -> BResult<()> {
-        let last_iter = self.is_last_iter();
-        if let Some(ref mut cleanup) = self.inner {
-            let should_run_cleanup = !(last_iter && cleanup.skip_on_last);
-            if should_run_cleanup {
-                return cleanup.cleanup(self.name);
-            }
-        }
-        Ok(())
-    }
-    /// Check if this is the last iteration
-    fn is_last_iter(&self) -> bool {
-        (self.max - 1) == self.current
-    }
-    /// Step the counter ahead
-    fn step(&mut self) {
-        self.current += 1;
-    }
-    /// Determine if we should continue executing
-    fn should_continue(&self) -> bool {
-        self.current < self.max
-    }
-    /// Append a new time to the sum
-    fn incr_time(&mut self, dt: &SimpleTimer) {
-        self.time += dt.time_in_nanos().unwrap();
-    }
-    /// Return the sum
-    fn sum(&self) -> u128 {
-        self.time
-    }
-    /// Return the name of the benchmark
-    fn name(&self) -> &'static str {
-        self.name
-    }
-}
-
-/// Cleanup instructions
-struct CleanupInner<'a> {
-    /// the connection to use for cleanup processes
-    connection: &'a mut Connection,
-    /// the query to be run
-    query: Query,
-    /// the response to expect
-    response: Element,
-    /// whether we should skip on the last loop
-    skip_on_last: bool,
-}
-
-impl<'a> CleanupInner<'a> {
-    /// Init cleanup instructions
-    fn new(q: Query, r: Element, connection: &'a mut Connection, skip_on_last: bool) -> Self {
-        Self {
-            query: q,
-            response: r,
-            connection,
-            skip_on_last,
-        }
-    }
-    /// Run cleanup
-    fn cleanup(&mut self, name: &'static str) -> BResult<()> {
-        let r: Element = self.connection.run_query(&self.query)?;
-        if r.ne(&self.response) {
-            Err(Error::Runtime(format!(
-                "Failed to run cleanup for benchmark `{}`",
-                name
-            )))
-        } else {
-            Ok(())
-        }
-    }
-}
-
-#[inline(always)]
-/// Returns a vec with the given cap, ensuring that we don't overflow memory
-fn vec_with_cap<T>(cap: usize) -> BResult<Vec<T>> {
-    let mut v = Vec::new();
-    v.try_reserve_exact(cap)?;
-    Ok(v)
-}
-
-/// Run the actual benchmarks
-pub fn run_bench(servercfg: &ServerConfig, matches: ArgMatches) -> BResult<()> {
-    // init bench config
-    let bench_config = BenchmarkConfig::new(servercfg, matches)?;
-    // check if we have enough combinations for the given query count and key size
-    if !util::has_enough_ncr(bench_config.kvsize(), bench_config.query_count()) {
-        return Err(Error::Runtime(
-            "too low sample space for given query count. use larger kvsize".into(),
-        ));
-    }
-    // run sanity test; this will also set up the temporary table for benchmarking
-    binfo!("Running sanity test ...");
-    util::run_sanity_test(&bench_config.server)?;
-
-    // pool pre-exec setup
-    let servercfg = servercfg.clone();
-    let switch_table = Query::from("use default.tmpbench").into_raw_query();
-
-    // init pool config; side_connection is for cleanups
-    let mut misc_connection = Connection::new(servercfg.host(), servercfg.port())?;
-
-    // init timer and reports
-    let mut reports = AggregateReport::new(bench_config.query_count());
-
-    // init test data
-    binfo!("Initializing test data ...");
-    let mut rng = rand::thread_rng();
-    let keys = generate_random_byte_vector(
-        bench_config.query_count(),
-        bench_config.kvsize(),
-        &mut rng,
-        true,
-    )?;
-    let values = generate_random_byte_vector(
-        bench_config.query_count(),
-        bench_config.kvsize(),
-        &mut rng,
-        false,
-    )?;
-    let new_updated_key = ran_bytes(bench_config.kvsize(), &mut rng);
-
-    // run tests; the idea here is to run all tests one-by-one instead of generating all packets at once
-    // such an approach helps us keep memory usage low
-    // bench set
-    binfo!("Benchmarking SET ...");
-    benches::bench_set(
-        &keys,
-        &values,
-        &mut misc_connection,
-        &bench_config,
-        &switch_table,
-        &mut reports,
-    )?;
-
-    // bench update
-    binfo!("Benchmarking UPDATE ...");
-    benches::bench_update(
-        &keys,
-        &new_updated_key,
-        &bench_config,
-        &switch_table,
-        &mut reports,
-    )?;
-
-    // bench get
-    binfo!("Benchmarking GET ...");
-    benches::bench_get(&keys, &bench_config, &switch_table, &mut reports)?;
-
-    // remove all test data
-    binfo!("Finished benchmarks. Cleaning up ...");
-    let r: Element = misc_connection.run_query(Query::from("drop model default.tmpbench force"))?;
-    if r != Element::RespCode(RespCode::Okay) {
-        return Err(Error::Runtime("failed to clean up after benchmarks".into()));
-    }
-
-    if config::should_output_messages() {
-        // normal output
-        println!("===========RESULTS===========");
-        let (maxpad, reports) = reports.finish();
-        for report in reports {
-            let padding = " ".repeat(maxpad - report.name().len());
-            println!(
-                "{}{} {:.6}/sec",
-                report.name().to_uppercase(),
-                padding,
-                report.stat(),
-            );
-        }
-        println!("=============================");
-    } else {
-        // JSON
-        println!("{}", reports.into_json())
-    }
-    Ok(())
-}
diff --git a/sky-bench/src/bench/report.rs b/sky-bench/src/bench/report.rs
deleted file mode 100644
index 4d0edbab..00000000
--- a/sky-bench/src/bench/report.rs
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Created on Wed Aug 10 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use serde::Serialize;
-
-#[derive(Serialize)]
-pub struct SingleReport {
-    name: &'static str,
-    stat: f64,
-}
-
-impl SingleReport {
-    pub fn new(name: &'static str, stat: f64) -> Self {
-        Self { name, stat }
-    }
-
-    pub fn stat(&self) -> f64 {
-        self.stat
-    }
-
-    pub fn name(&self) -> &str {
-        self.name
-    }
-}
-
-pub struct AggregateReport {
-    names: Vec<SingleReport>,
-    query_count: usize,
-}
-
-impl AggregateReport {
-    pub fn new(query_count: usize) -> Self {
-        Self {
-            names: Vec::new(),
-            query_count,
-        }
-    }
-    pub fn push(&mut self, report: SingleReport) {
-        self.names.push(report)
-    }
-    pub(crate) fn into_json(self) -> String {
-        let (_, report) = self.finish();
-        serde_json::to_string(&report).unwrap()
-    }
-
-    pub(crate) fn finish(self) -> (usize, Vec<SingleReport>) {
-        let mut maxpad = self.names[0].name.len();
-        let mut reps = self.names;
-        reps.iter_mut().for_each(|rep| {
-            let total_time = rep.stat;
-            let qps = (self.query_count as f64 / total_time) * 1_000_000_000_f64;
-            rep.stat = qps;
-            if rep.name.len() > maxpad {
-                maxpad = rep.name.len();
-            }
-        });
-        (maxpad, reps)
-    }
-}
diff --git a/sky-bench/src/bench/validation.rs b/sky-bench/src/bench/validation.rs
deleted file mode 100644
index c93b35f0..00000000
--- a/sky-bench/src/bench/validation.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Created on Tue Aug 09 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-pub const RESPCODE_OKAY: &[u8] = b"*!0\n";
-
-pub fn calculate_response_size(keylen: usize) -> usize {
-    /*
-    *+5\n
-    hello
-    */
-    let mut size = 2; // simple query byte + tsymbol
-    size += keylen.to_string().len(); // bytes in length
-    size += 1; // LF
-    size += keylen; // payload
-    size
-}
diff --git a/sky-bench/src/cli.yml b/sky-bench/src/cli.yml
deleted file mode 100644
index 0afc683c..00000000
--- a/sky-bench/src/cli.yml
+++ /dev/null
@@ -1,72 +0,0 @@
-#
-# Created on Tue Nov 03 2020
-#
-# This file is a part of Skytable
-# Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-#
-
-name: Skytable Benchmark Tool
-version: 0.8.0
-author: Sayan N. <ohsayan@outlook.com>
-about: |
-  The Skytable benchmark tool can be used to benchmark Skytable installations. 
-  If you find any issues, then report one here: https://github.com/skytable/skytable
-args:
-  - connections:
-      short: c
-      long: connections
-      value_name: count
-      help: Sets the number of simultaneous clients
-      takes_value: true
-  - queries:
-      short: q
-      long: queries
-      value_name: number
-      help: Sets the number of queries to run
-      takes_value: true
-  - size:
-      short: s
-      long: kvsize
-      value_name: bytes
-      help: Sets the size of the key/value pairs
-      takes_value: true
-  - json:
-      required: false
-      long: json
-      help: Sets output type to JSON
-      takes_value: false
-  - host:
-      short: h
-      required: false
-      long: host
-      value_name: host
-      help: Sets the remote host to connect to
-      takes_value: true
-  - port:
-      short: p
-      required: false
-      long: port
-      value_name: port
-      help: Sets the remote port to connect to
-      takes_value: true
-  - runs:
-      short: r
-      required: false
-      long: runs
-      value_name: runs
-      takes_value: true
-      help: Sets the number of times the entire test should be run
diff --git a/sky-bench/src/config.rs b/sky-bench/src/config.rs
deleted file mode 100644
index f8e283a4..00000000
--- a/sky-bench/src/config.rs
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Created on Mon Aug 08 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::error::{BResult, Error},
-    crate::util,
-    clap::ArgMatches,
-    std::{fmt::Display, str::FromStr},
-};
-
-static mut OUTPUT_JSON: bool = false;
-
-#[derive(Clone)]
-pub struct ServerConfig {
-    /// host
-    host: Box<str>,
-    /// port
-    port: u16,
-    /// connection count for network pool
-    connections: usize,
-}
-
-#[inline(always)]
-fn try_update<T: FromStr, S: AsRef<str>>(input: Option<S>, target: &mut T) -> BResult<()>
-where
-    <T as FromStr>::Err: Display,
-{
-    if let Some(input) = input {
-        let parsed = input
-            .as_ref()
-            .parse::<T>()
-            .map_err(|e| Error::Config(format!("parse error: `{}`", e)))?;
-        *target = parsed;
-    }
-    Ok(())
-}
-
-impl ServerConfig {
-    const DEFAULT_HOST: &'static str = "127.0.0.1";
-    const DEFAULT_PORT: u16 = 2003;
-    const DEFAULT_CONNECTIONS: usize = 10;
-    /// Init the default server config
-    pub fn new(matches: &ArgMatches) -> BResult<Self> {
-        let mut slf = Self {
-            host: Self::DEFAULT_HOST.into(),
-            port: Self::DEFAULT_PORT,
-            connections: Self::DEFAULT_CONNECTIONS,
-        };
-        slf.try_host(matches.value_of_lossy("host"));
-        slf.try_port(matches.value_of_lossy("port"))?;
-        slf.try_connections(matches.value_of_lossy("connections"))?;
-        Ok(slf)
-    }
-    /// Update the host
-    pub fn try_host<T: AsRef<str>>(&mut self, host: Option<T>) {
-        if let Some(host) = host {
-            self.host = host.as_ref().into();
-        }
-    }
-    /// Attempt to update the port
-    pub fn try_port<T: AsRef<str>>(&mut self, port: Option<T>) -> BResult<()> {
-        try_update(port, &mut self.port)
-    }
-    /// Attempt to update the connections
-    pub fn try_connections<T: AsRef<str>>(&mut self, con: Option<T>) -> BResult<()> {
-        try_update(con, &mut self.connections)
-    }
-}
-
-impl ServerConfig {
-    pub fn host(&self) -> &str {
-        self.host.as_ref()
-    }
-    pub fn port(&self) -> u16 {
-        self.port
-    }
-    pub fn connections(&self) -> usize {
-        self.connections
-    }
-}
-
-/// Benchmark configuration
-#[derive(Clone)]
-pub struct BenchmarkConfig {
-    pub server: ServerConfig,
-    kvsize: usize,
-    queries: usize,
-    runs: usize,
-}
-
-impl BenchmarkConfig {
-    const DEFAULT_QUERIES: usize = 100_000;
-    const DEFAULT_KVSIZE: usize = 3;
-    const DEFAULT_RUNS: usize = 5;
-    pub fn new(server: &ServerConfig, matches: ArgMatches) -> BResult<Self> {
-        let mut slf = Self {
-            server: server.clone(),
-            queries: Self::DEFAULT_QUERIES,
-            kvsize: Self::DEFAULT_KVSIZE,
-            runs: Self::DEFAULT_RUNS,
-        };
-        try_update(matches.value_of_lossy("queries"), &mut slf.queries)?;
-        try_update(matches.value_of_lossy("size"), &mut slf.kvsize)?;
-        try_update(matches.value_of_lossy("runs"), &mut slf.runs)?;
-        util::ensure_main_thread();
-        unsafe {
-            OUTPUT_JSON = matches.is_present("json");
-        }
-        Ok(slf)
-    }
-    pub fn kvsize(&self) -> usize {
-        self.kvsize
-    }
-    pub fn query_count(&self) -> usize {
-        self.queries
-    }
-    pub fn runs(&self) -> usize {
-        self.runs
-    }
-}
-
-pub fn should_output_messages() -> bool {
-    util::ensure_main_thread();
-    unsafe { !OUTPUT_JSON }
-}
diff --git a/sky-bench/src/error.rs b/sky-bench/src/error.rs
deleted file mode 100644
index c2519702..00000000
--- a/sky-bench/src/error.rs
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Created on Mon Aug 08 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    libstress::WorkpoolError,
-    skytable::error::Error as SkyError,
-    std::{collections::TryReserveError, fmt::Display},
-};
-
-pub type BResult<T> = Result<T, Error>;
-
-/// Benchmark tool errors
-pub enum Error {
-    /// An error originating from the Skytable client
-    Client(SkyError),
-    /// An error originating from the benchmark/server configuration
-    Config(String),
-    /// A runtime error
-    Runtime(String),
-}
-
-impl From<SkyError> for Error {
-    fn from(e: SkyError) -> Self {
-        Self::Client(e)
-    }
-}
-
-impl Display for Error {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Error::Client(e) => write!(f, "client error: {}", e),
-            Error::Config(e) => write!(f, "config error: {}", e),
-            Error::Runtime(e) => write!(f, "runtime error: {}", e),
-        }
-    }
-}
-
-impl From<TryReserveError> for Error {
-    fn from(e: TryReserveError) -> Self {
-        Error::Runtime(format!("memory reserve error: {}", e))
-    }
-}
-
-impl From<WorkpoolError> for Error {
-    fn from(e: WorkpoolError) -> Self {
-        Error::Runtime(format!("threadpool error: {}", e))
-    }
-}
diff --git a/sky-bench/src/main.rs b/sky-bench/src/main.rs
index 76ea1b2b..57a21b8e 100644
--- a/sky-bench/src/main.rs
+++ b/sky-bench/src/main.rs
@@ -1,5 +1,5 @@
 /*
- * Created on Mon Aug 08 2022
+ * Created on Wed Nov 15 2023
  *
  * This file is a part of Skytable
  * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@@ -7,7 +7,7 @@
  * vision to provide flexibility in data modelling without compromising
  * on performance, queryability or scalability.
  *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published by
@@ -24,40 +24,4 @@
  *
 */
 
-use {
-    clap::{load_yaml, App},
-    config::ServerConfig,
-    env_logger::Builder,
-    std::{env, process},
-};
-#[macro_use]
-extern crate log;
-
-mod bench;
-mod config;
-mod error;
-mod util;
-
-fn main() {
-    Builder::new()
-        .parse_filters(&env::var("SKYBENCH_LOG").unwrap_or_else(|_| "info".to_owned()))
-        .init();
-    if let Err(e) = run() {
-        error!("sky-bench exited with error: {}", e);
-        process::exit(0x01);
-    }
-}
-
-fn run() -> error::BResult<()> {
-    // init CLI arg parser
-    let cli_args = load_yaml!("cli.yml");
-    let cli = App::from_yaml(cli_args);
-    let matches = cli.get_matches();
-
-    // parse args
-    let cfg = ServerConfig::new(&matches)?;
-
-    // run our task
-    bench::run_bench(&cfg, matches)?;
-    util::cleanup(&cfg)
-}
+fn main() {}
diff --git a/sky-bench/src/util.rs b/sky-bench/src/util.rs
deleted file mode 100644
index fc2ab9bb..00000000
--- a/sky-bench/src/util.rs
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Created on Tue Aug 09 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{
-        config::ServerConfig,
-        error::{BResult, Error},
-    },
-    skytable::{Connection, Element, Query, RespCode},
-    std::thread,
-};
-
-/// Check if the provided keysize has enough combinations to support the given `queries` count
-///
-/// This function is heavily optimized and should take Θ(1) time. The `ALWAYS_TRUE_FACTOR` is
-/// dependent on pointer width (more specifically the virtual address space size).
-/// - For 64-bit address spaces: `(256!)/r!(256-r!)`; for a value of r >= 12, we'll hit the maximum
-/// of the address space and hence this will always return true (because of the size of `usize`)
-///     > The value for r = 12 is `1.27309515e+20` which largely exceeds `1.8446744e+19`
-/// - For 32-bit address spaces: `(256!)/r!(256-r!)`; for a value of r >= 5, we'll hit the maximum
-/// of the address space and hence this will always return true (because of the size of `usize`)
-///     > The value for r = 5 is `8.81e+9` which largely exceeds `4.3e+9`
-pub const fn has_enough_ncr(keysize: usize, queries: usize) -> bool {
-    const LUT: [u64; 11] = [
-        // 1B
-        256,
-        // 2B
-        32640,
-        // 3B
-        2763520,
-        // 4B
-        174792640,
-        // 5B
-        8809549056,
-        // 6B
-        368532802176,
-        // 7B
-        13161885792000,
-        // 8B
-        409663695276000,
-        // 9B
-        11288510714272000,
-        // 10B
-        278826214642518400,
-        // 11B
-        6235568072914502400,
-    ];
-    #[cfg(target_pointer_width = "64")]
-    const ALWAYS_TRUE_FACTOR: usize = 12;
-    #[cfg(target_pointer_width = "32")]
-    const ALWAYS_TRUE_FACTOR: usize = 5;
-    keysize >= ALWAYS_TRUE_FACTOR || (LUT[keysize - 1] >= queries as _)
-}
-
-/// Run a sanity test, making sure that the server is ready for benchmarking. This function will do the
-/// following tests:
-/// - Connect to the instance
-/// - Run a `heya` as a preliminary test
-/// - Create a new table `tmpbench`. This is where we're supposed to run all the benchmarks.
-/// - Switch to the new table
-/// - Set a key, and get it checking the equality of the returned value
-pub fn run_sanity_test(server_config: &ServerConfig) -> BResult<()> {
-    let mut con = Connection::new(server_config.host(), server_config.port())?;
-    let tests: [(Query, Element, &str); 5] = [
-        (
-            Query::from("HEYA"),
-            Element::String("HEY!".to_owned()),
-            "heya",
-        ),
-        (
-            Query::from("CREATE MODEL default.tmpbench(binary, binary)"),
-            Element::RespCode(RespCode::Okay),
-            "create model",
-        ),
-        (
-            Query::from("use default.tmpbench"),
-            Element::RespCode(RespCode::Okay),
-            "use",
-        ),
-        (
-            Query::from("set").arg("x").arg("100"),
-            Element::RespCode(RespCode::Okay),
-            "set",
-        ),
-        (
-            Query::from("get").arg("x"),
-            Element::Binstr("100".as_bytes().to_owned()),
-            "get",
-        ),
-    ];
-    for (query, expected, test_kind) in tests {
-        let r: Element = con.run_query(query)?;
-        if r != expected {
-            return Err(Error::Runtime(format!(
-                "sanity test for `{test_kind}` failed"
-            )));
-        }
-    }
-    Ok(())
-}
-
-/// Ensures that the current thread is the main thread. If not, this function will panic
-pub fn ensure_main_thread() {
-    assert_eq!(
-        thread::current().name().unwrap(),
-        "main",
-        "unsafe function called from non-main thread"
-    )
-}
-
-/// Run a cleanup. This function attempts to remove the `default.tmpbench` entity
-pub fn cleanup(server_config: &ServerConfig) -> BResult<()> {
-    let mut c = Connection::new(server_config.host(), server_config.port())?;
-    let r: Element = c.run_query(Query::from("drop model default.tmpbench force"))?;
-    if r == Element::RespCode(RespCode::Okay) {
-        Err(Error::Runtime("failed to run cleanup".into()))
-    } else {
-        Ok(())
-    }
-}
diff --git a/sky-migrate/Cargo.toml b/sky-migrate/Cargo.toml
deleted file mode 100644
index 30dfd28f..00000000
--- a/sky-migrate/Cargo.toml
+++ /dev/null
@@ -1,14 +0,0 @@
-[package]
-name = "sky-migrate"
-version = "0.8.0"
-authors = ["Sayan Nandan <nandansayan@outlook.com>"]
-edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-skytable = { git = "https://github.com/skytable/client-rust.git" }
-env_logger = "0.10.0"
-bincode = "1.3.3"
-log = "0.4.19"
-clap = { version = "2", features = ["yaml"] }
diff --git a/sky-migrate/README.md b/sky-migrate/README.md
deleted file mode 100644
index 58b75930..00000000
--- a/sky-migrate/README.md
+++ /dev/null
@@ -1,10 +0,0 @@
-# Skytable migration tool
-
-The Skytable migration tool can be used to perform migrations between database versions. The basic idea is:
-"read all data from the older machine and send it over to the newer one."
-
-For migrating versions from 0.6 to 0.7, the database administrator has to launch the tool in the old data directory of the old instance and then pass the new instance host/port information. The tool will then read the data from the directory (this was possible because 0.6 used a very simple disk format than newer versions). This approach however has the advantage of not having to start the database server for the migration to happen.
-
-## License
-
-All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE).
diff --git a/sky-migrate/src/cli.yml b/sky-migrate/src/cli.yml
deleted file mode 100644
index 97bde94d..00000000
--- a/sky-migrate/src/cli.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-name: Skytable Migration Tool
-version: 0.7.0
-author: Sayan N. <ohsayan@outlook.com>
-about: |
-  The Skytable migration tool allows users coming from older versions (>=0.8.0)
-  to upgrade their datasets to the latest Skytable version. This tool currently
-  supports versions >= 0.8.0 and upgrading it to 0.7.0. To upgrade, on needs
-  to simply run:
-  sky-migrate --prevdir <lastpath> --new <host>:<port>
-  Where `<lastpath>` is the path to the last installation's data directory and
-  `<host>` and `<port>` is the hostname and port for the new server instance
-args:
-  - new:
-      long: new
-      takes_value: true
-      required: true
-      help: The <host>:<port> combo for the new instance
-      value_name: new
-  - prevdir:
-      long: prevdir
-      takes_value: true
-      required: true
-      help: Path to the previous installation location
-      value_name: prevdir
-  - serial:
-      long: serial
-      takes_value: false
-      required: false
-      help: |
-        Transfer entries one-by-one instead of all at once to save memory
diff --git a/sky-migrate/src/main.rs b/sky-migrate/src/main.rs
deleted file mode 100644
index f77bcf6a..00000000
--- a/sky-migrate/src/main.rs
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Created on Tue Aug 17 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#![allow(clippy::unit_arg)]
-
-use {
-    clap::{load_yaml, App},
-    core::hint::unreachable_unchecked,
-    env_logger::Builder,
-    log::{error as err, info},
-    skytable::{query, sync::Connection, Element, Query, RespCode},
-    std::{collections::HashMap, env, fs, path::PathBuf, process},
-};
-
-type Bytes = Vec<u8>;
-
-fn main() {
-    // first evaluate config
-    let cfg_layout = load_yaml!("cli.yml");
-    let matches = App::from_yaml(cfg_layout).get_matches();
-    Builder::new()
-        .parse_filters(&env::var("SKY_LOG").unwrap_or_else(|_| "info".to_owned()))
-        .init();
-    let new_host = matches
-        .value_of("new")
-        .map(|v| v.to_string())
-        .unwrap_or_else(|| unsafe { unreachable_unchecked() });
-    let serial = matches.is_present("serial");
-    let hostsplit: Vec<&str> = new_host.split(':').collect();
-    if hostsplit.len() != 2 {
-        err(err!("Bad value for --new"));
-    }
-    let (host, port) = unsafe { (hostsplit.get_unchecked(0), hostsplit.get_unchecked(1)) };
-    let port = match port.parse() {
-        Ok(p) => p,
-        Err(e) => err(err!("Bad value for port in --new: {}", e)),
-    };
-    let mut old_dir = matches
-        .value_of("prevdir")
-        .map(PathBuf::from)
-        .unwrap_or_else(|| unsafe { unreachable_unchecked() });
-    old_dir.push("data.bin");
-    // now connect
-    let mut con = match Connection::new(host, port) {
-        Ok(con) => con,
-        Err(e) => err(err!("Failed to connect to new instance with error: {}", e)),
-    };
-    // run sanity test
-    let q = query!("HEYA");
-    match con.run_query_raw(&q) {
-        Ok(Element::String(s)) if s.eq("HEY!") => {}
-        Ok(_) => err(err!("Unknown response from server")),
-        Err(e) => err(err!(
-            "An I/O error occurred while running sanity test: {}",
-            e
-        )),
-    }
-    info!("Sanity test complete");
-
-    // now de old file
-    let read = match fs::read(old_dir) {
-        Ok(r) => r,
-        Err(e) => err(err!(
-            "Failed to read data.bin file from old directory: {}",
-            e
-        )),
-    };
-    let de: HashMap<Bytes, Bytes> = match bincode::deserialize(&read) {
-        Ok(r) => r,
-        Err(e) => err(err!("Failed to unpack old file with: {}", e)),
-    };
-    unsafe {
-        if serial {
-            // transfer serially
-            for (key, value) in de.into_iter() {
-                let q = query!(
-                    "USET",
-                    String::from_utf8_unchecked(key),
-                    String::from_utf8_unchecked(value)
-                );
-                okay(&mut con, q)
-            }
-        } else {
-            // transfer all at once
-            let mut query = Query::from("USET");
-            for (key, value) in de.into_iter() {
-                query.push(String::from_utf8_unchecked(key));
-                query.push(String::from_utf8_unchecked(value));
-            }
-            okay(&mut con, query)
-        }
-    }
-    info!("Finished migration");
-}
-
-fn err(_i: ()) -> ! {
-    process::exit(0x01)
-}
-
-fn okay(con: &mut Connection, q: Query) {
-    match con.run_query_raw(&q) {
-        Ok(Element::RespCode(RespCode::Okay)) => {}
-        Err(e) => err(err!("An I/O error occurred while running query: {}", e)),
-        Ok(_) => err(err!("Unknown response from server")),
-    }
-}
diff --git a/stress-test/Cargo.toml b/stress-test/Cargo.toml
deleted file mode 100644
index f05bb998..00000000
--- a/stress-test/Cargo.toml
+++ /dev/null
@@ -1,21 +0,0 @@
-[package]
-name = "stress-test"
-version = "0.1.0"
-authors = ["Sayan Nandan <nandansayan@outlook.com>"]
-edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-# internal deps
-libstress = { path = "../libstress" }
-skytable = { git = "https://github.com/skytable/client-rust.git", branch = "next", features = [
-    "dbg",
-] }
-devtimer = "4.0.1"
-# external deps
-sysinfo = "0.29.7"
-env_logger = "0.10.0"
-log = "0.4.19"
-rand = "0.8.5"
-crossbeam-channel = "0.5.8"
diff --git a/stress-test/README.md b/stress-test/README.md
deleted file mode 100644
index 5008306b..00000000
--- a/stress-test/README.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# Skytable stress-test tool
-
-This is a tool in its infancy, but its ultimate goal is to provide a stress testing framework for Skytable. For now, it tests linearity (core scalability) with increasing clients.
-
-## License
-
-All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE).
diff --git a/stress-test/src/linearity_client.rs b/stress-test/src/linearity_client.rs
deleted file mode 100644
index d97e37b7..00000000
--- a/stress-test/src/linearity_client.rs
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Created on Fri Jun 18 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # Client linearity tests
-//!
-//! This module contains functions to test the linearity of the database with increasing number
-//! of clients, i.e how the number of queries scale with increasing clients. These functions
-//! however, DO NOT focus on benchmarking and instead focus on correctness under load from
-//! concurrent clients.
-//!
-
-use {
-    crate::{logstress, DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV},
-    crossbeam_channel::bounded,
-    devtimer::SimpleTimer,
-    libstress::{rayon::prelude::*, utils::generate_random_string_vector, Workpool},
-    skytable::{actions::Actions, query, Connection, Element, Query, RespCode},
-};
-
-macro_rules! log_client_linearity {
-    ($stressid:expr, $counter:expr, $what:expr) => {
-        log::info!(
-            "Stress ({}{}) [{}]: Clients: {}; K/V size: {}; Queries: {}",
-            $stressid,
-            $counter,
-            $what,
-            $counter,
-            DEFAULT_SIZE_KV,
-            DEFAULT_QUERY_COUNT
-        );
-    };
-}
-
-/// This object provides methods to measure the percentage change in the slope
-/// of a function that is expected to have linearity.
-///
-/// For example, we can think of it to work in the following way:
-/// Let h(x) be a function with linearity (not proportionality because that isn't
-/// applicable in our case). h(x) gives us the time taken to run a given number of
-/// queries (invariant; not plotted on axes), where x is the number of concurrent
-/// clients. As we would want our database to scale with increasing clients (and cores),
-/// we'd expect linearity, hence the gradient should continue to fall with increasing
-/// values in the +ve x-axis effectively producing a constantly decreasing slope, reflected
-/// by increasing values of abs(get_delta(h(x))).
-///
-/// TODO(@ohsayan): Of course, some unexpected kernel errors/scheduler hiccups et al can
-/// cause there to be a certain epsilon that must be tolerated with a tolerance factor
-///
-pub struct LinearityMeter {
-    init: Option<u128>,
-    measure: Vec<f32>,
-}
-
-impl LinearityMeter {
-    pub const fn new() -> Self {
-        Self {
-            init: None,
-            measure: Vec::new(),
-        }
-    }
-    pub fn get_delta(&mut self, current: u128) -> f32 {
-        if let Some(u) = self.init {
-            let cur = ((current as f32 - u as f32) / u as f32) * 100.00_f32;
-            self.measure.push(cur);
-            cur
-        } else {
-            // if init is not initialized, initialize it
-            self.init = Some(current);
-            // no change when at base
-            0.00
-        }
-    }
-}
-
-pub fn stress_linearity_concurrent_clients_set(
-    mut rng: &mut impl rand::Rng,
-    max_workers: usize,
-    temp_con: &mut Connection,
-) {
-    logstress!(
-        "A [SET]",
-        "Linearity test with monotonically increasing clients"
-    );
-    let mut current_thread_count = 1usize;
-
-    // generate the random k/v pairs
-    let keys = generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, true);
-    let values =
-        generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, false);
-    let (keys, values) = match (keys, values) {
-        (Ok(k), Ok(v)) => (k, v),
-        _ => {
-            eprintln!("Allocation error");
-            std::process::exit(0x01);
-        }
-    };
-    // make sure the database is empty
-    temp_con.flushdb().unwrap();
-
-    // initialize the linearity counter
-    let mut linearity = LinearityMeter::new();
-    while current_thread_count <= max_workers {
-        log_client_linearity!("A", current_thread_count, "SET");
-        // generate the set packets
-        let set_packs: Vec<Query> = keys
-            .par_iter()
-            .zip(values.par_iter())
-            .map(|(k, v)| query!("SET", k, v))
-            .collect();
-        let workpool = Workpool::new(
-            current_thread_count,
-            || Connection::new("127.0.0.1", 2003).unwrap(),
-            move |sock, query| {
-                assert_eq!(
-                    sock.run_query_raw(&query).unwrap(),
-                    Element::RespCode(RespCode::Okay)
-                );
-            },
-            |_| {},
-            true,
-            Some(DEFAULT_QUERY_COUNT),
-        )
-        .unwrap();
-        let mut timer = SimpleTimer::new();
-        timer.start();
-        workpool.execute_and_finish_iter(set_packs);
-        timer.stop();
-        log::info!(
-            "Delta: {}%",
-            linearity.get_delta(timer.time_in_nanos().unwrap())
-        );
-        // clean up the database
-        temp_con.flushdb().unwrap();
-        current_thread_count += 1;
-    }
-}
-
-pub fn stress_linearity_concurrent_clients_get(
-    mut rng: &mut impl rand::Rng,
-    max_workers: usize,
-    temp_con: &mut Connection,
-) {
-    logstress!(
-        "A [GET]",
-        "Linearity test with monotonically increasing clients"
-    );
-    let mut current_thread_count = 1usize;
-
-    // generate the random k/v pairs
-    let keys = generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, true);
-    let values =
-        generate_random_string_vector(DEFAULT_QUERY_COUNT, DEFAULT_SIZE_KV, &mut rng, false);
-    let (keys, values) = match (keys, values) {
-        (Ok(k), Ok(v)) => (k, v),
-        _ => {
-            eprintln!("Allocation error");
-            std::process::exit(0x01);
-        }
-    };
-
-    // Make sure that the database is empty
-    temp_con.flushdb().unwrap();
-
-    // First set the keys
-    let set_packs: Vec<Query> = keys
-        .par_iter()
-        .zip(values.par_iter())
-        .map(|(k, v)| query!("SET", k, v))
-        .collect();
-    let workpool = Workpool::new_default_threads(
-        || Connection::new("127.0.0.1", 2003).unwrap(),
-        move |sock, query| {
-            assert_eq!(
-                sock.run_query_raw(&query).unwrap(),
-                Element::RespCode(RespCode::Okay)
-            );
-        },
-        |_| {},
-        true,
-        Some(DEFAULT_QUERY_COUNT),
-    )
-    .unwrap();
-    workpool.execute_and_finish_iter(set_packs);
-
-    // initialize the linearity counter
-    let mut linearity = LinearityMeter::new();
-    while current_thread_count <= max_workers {
-        log_client_linearity!("A", current_thread_count, "GET");
-        /*
-         We create a  mpmc to receive the results returned. This avoids us using
-         any kind of locking on the surface which can slow down things
-        */
-        let (tx, rx) = bounded::<Element>(DEFAULT_QUERY_COUNT);
-
-        // generate the get packets
-        let get_packs: Vec<Query> = keys.iter().map(|k| query!("GET", k)).collect();
-        let wp = Workpool::new(
-            current_thread_count,
-            || Connection::new("127.0.0.1", 2003).unwrap(),
-            move |sock, query| {
-                let tx = tx.clone();
-                tx.send(sock.run_query_raw(&query).unwrap()).unwrap();
-            },
-            |_| {},
-            true,
-            Some(DEFAULT_QUERY_COUNT),
-        )
-        .unwrap();
-        let mut timer = SimpleTimer::new();
-        timer.start();
-        wp.execute_and_finish_iter(get_packs);
-        timer.stop();
-        log::info!(
-            "Delta: {}%",
-            linearity.get_delta(timer.time_in_nanos().unwrap())
-        );
-        let rets: Vec<String> = rx
-            .into_iter()
-            .map(|v| {
-                if let Element::String(val) = v {
-                    val
-                } else {
-                    panic!("Unexpected response from server");
-                }
-            })
-            .collect();
-        assert_eq!(
-            rets.len(),
-            values.len(),
-            "Incorrect number of values returned by server"
-        );
-
-        // now evaluate them
-        assert!(
-            rets.into_par_iter().all(|v| values.contains(&v)),
-            "Values returned by the server don't match what was sent"
-        );
-        current_thread_count += 1;
-    }
-    temp_con.flushdb().unwrap();
-}
diff --git a/stress-test/src/main.rs b/stress-test/src/main.rs
deleted file mode 100644
index e59045ef..00000000
--- a/stress-test/src/main.rs
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Created on Wed Jun 16 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-#![deny(unused_crate_dependencies)]
-#![deny(unused_imports)]
-
-use std::thread::available_parallelism;
-
-use {
-    libstress::traits::ExitError,
-    log::{info, trace, warn},
-    rand::thread_rng,
-    skytable::Connection,
-    std::env,
-    sysinfo::{RefreshKind, System, SystemExt},
-};
-mod linearity_client;
-mod utils;
-
-pub const DEFAULT_SIZE_KV: usize = 4;
-pub const DEFAULT_QUERY_COUNT: usize = 100_000_usize;
-
-#[macro_export]
-macro_rules! logstress {
-    ($stressid:expr, $extra:expr) => {
-        log::info!("Stress ({}): {}", $stressid, $extra);
-    };
-}
-
-fn main() {
-    // Build the logger
-    env_logger::Builder::new()
-        .parse_filters(&env::var("SKY_STRESS_LOG").unwrap_or_else(|_| "trace".to_owned()))
-        .init();
-    warn!("The stress test checks correctness under load and DOES NOT show the true throughput");
-
-    // get the rng and refresh sysinfo
-    let mut rng = thread_rng();
-    // we only need to refresh memory and CPU info; don't waste time syncing other things
-    let to_refresh = RefreshKind::new().with_memory();
-    let mut sys = System::new_with_specifics(to_refresh);
-    sys.refresh_specifics(to_refresh);
-    let core_count = available_parallelism().map_or(1, usize::from);
-    let max_workers = core_count * 2;
-    trace!(
-        "This host has {} logical cores. Will spawn a maximum of {} threads",
-        core_count,
-        max_workers * 2
-    );
-
-    // establish a connection to ensure sanity
-    let mut temp_con = Connection::new("127.0.0.1", 2003).exit_error("Failed to connect to server");
-
-    // calculate the maximum keylen
-    let max_keylen = utils::calculate_max_keylen(DEFAULT_QUERY_COUNT, &mut sys);
-    info!(
-        "This host can support a maximum theoretical keylen of: {}",
-        max_keylen
-    );
-
-    // run the actual stress tests
-    linearity_client::stress_linearity_concurrent_clients_set(&mut rng, max_workers, &mut temp_con);
-    linearity_client::stress_linearity_concurrent_clients_get(&mut rng, max_workers, &mut temp_con);
-
-    // done, exit
-    info!("SUCCESS. Stress test complete!");
-}
diff --git a/stress-test/src/utils.rs b/stress-test/src/utils.rs
deleted file mode 100644
index 76cbbc8f..00000000
--- a/stress-test/src/utils.rs
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Created on Fri Jun 18 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-use {
-    log::trace,
-    skytable::Query,
-    sysinfo::{System, SystemExt},
-};
-
-pub fn calculate_max_keylen(expected_queries: usize, sys: &mut System) -> usize {
-    let total_mem_in_bytes = (sys.total_memory() * 1024) as usize;
-    trace!(
-        "This host has a total memory of: {} Bytes",
-        total_mem_in_bytes
-    );
-    // av_mem gives us 90% of the memory size
-    let ninety_percent_of_memory = (0.90_f32 * total_mem_in_bytes as f32) as usize;
-    let mut highest_len = 1usize;
-    loop {
-        let set_pack_len = Query::array_packet_size_hint(vec![3, highest_len, highest_len]);
-        let get_pack_len = Query::array_packet_size_hint(vec![3, highest_len]);
-        let resulting_size = expected_queries
-            * (
-                // for the set packets
-                set_pack_len +
-                // for the get packets
-                get_pack_len +
-                // for the keys themselves
-                highest_len
-            );
-        if resulting_size >= ninety_percent_of_memory as usize {
-            break;
-        }
-        // increase the length by 5% every time to get the maximum possible length
-        // now this 5% increment is a tradeoff, but it's worth it to not wait for
-        // so long
-        highest_len = (highest_len as f32 * 1.05_f32).ceil() as usize;
-    }
-    highest_len
-}

From fddb24ad0b4b8a4bcb808581c03d7392dfac4f16 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 16 Nov 2023 02:02:32 +0530
Subject: [PATCH 280/310] Implement new REPL and remove old impls

---
 .gitignore                                |   4 +-
 Cargo.lock                                | 175 +++++++++++++-
 cli/Cargo.toml                            |   2 +
 cli/help_text/help                        |  30 +++
 cli/help_text/welcome                     |  16 ++
 cli/src/args.rs                           | 221 ++++++++++++++++++
 cli/src/error.rs                          |  60 +++++
 cli/src/main.rs                           |  30 ++-
 cli/src/query.rs                          | 266 ++++++++++++++++++++++
 cli/src/repl.rs                           | 137 +++++++++++
 cli/src/resp.rs                           | 144 ++++++++++++
 server/src/engine/mem/scanner.rs          |   3 +
 server/src/engine/ql/lex/mod.rs           |  32 ++-
 server/src/engine/ql/tests/lexer_tests.rs |   2 +-
 14 files changed, 1099 insertions(+), 23 deletions(-)
 create mode 100644 cli/help_text/help
 create mode 100644 cli/help_text/welcome
 create mode 100644 cli/src/args.rs
 create mode 100644 cli/src/error.rs
 create mode 100644 cli/src/query.rs
 create mode 100644 cli/src/repl.rs
 create mode 100644 cli/src/resp.rs

diff --git a/.gitignore b/.gitignore
index a40afb49..9a175c06 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,4 +15,6 @@ snapstore.partmap
 *.deb
 .skytest_*
 *.pem
-passphrase.txt
\ No newline at end of file
+passphrase.txt
+*.db-tlog
+*.db
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index e72855d6..c084f43a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -184,6 +184,17 @@ dependencies = [
  "inout",
 ]
 
+[[package]]
+name = "clipboard-win"
+version = "4.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362"
+dependencies = [
+ "error-code",
+ "str-buf",
+ "winapi",
+]
+
 [[package]]
 name = "constant_time_eq"
 version = "0.1.5"
@@ -282,6 +293,31 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crossterm"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df"
+dependencies = [
+ "bitflags 2.3.3",
+ "crossterm_winapi",
+ "libc",
+ "mio",
+ "parking_lot",
+ "signal-hook",
+ "signal-hook-mio",
+ "winapi",
+]
+
+[[package]]
+name = "crossterm_winapi"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -315,6 +351,12 @@ version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 
+[[package]]
+name = "endian-type"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
+
 [[package]]
 name = "env_logger"
 version = "0.10.0"
@@ -355,12 +397,33 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "error-code"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21"
+dependencies = [
+ "libc",
+ "str-buf",
+]
+
 [[package]]
 name = "fastrand"
 version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
 
+[[package]]
+name = "fd-lock"
+version = "3.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5"
+dependencies = [
+ "cfg-if",
+ "rustix",
+ "windows-sys",
+]
+
 [[package]]
 name = "flate2"
 version = "1.0.26"
@@ -482,6 +545,15 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "home"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb"
+dependencies = [
+ "windows-sys",
+]
+
 [[package]]
 name = "humantime"
 version = "2.1.0"
@@ -632,6 +704,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0"
 dependencies = [
  "libc",
+ "log",
  "wasi",
  "windows-sys",
 ]
@@ -654,6 +727,26 @@ dependencies = [
  "tempfile",
 ]
 
+[[package]]
+name = "nibble_vec"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
+dependencies = [
+ "smallvec",
+]
+
+[[package]]
+name = "nix"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
+dependencies = [
+ "bitflags 1.3.2",
+ "cfg-if",
+ "libc",
+]
+
 [[package]]
 name = "num_cpus"
 version = "1.16.0"
@@ -838,6 +931,16 @@ dependencies = [
  "scheduled-thread-pool",
 ]
 
+[[package]]
+name = "radix_trie"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
+dependencies = [
+ "endian-type",
+ "nibble_vec",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -958,6 +1061,29 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "rustyline"
+version = "12.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "994eca4bca05c87e86e15d90fc7a91d1be64b4482b38cb2d27474568fe7c9db9"
+dependencies = [
+ "bitflags 2.3.3",
+ "cfg-if",
+ "clipboard-win",
+ "fd-lock",
+ "home",
+ "libc",
+ "log",
+ "memchr",
+ "nix",
+ "radix_trie",
+ "scopeguard",
+ "unicode-segmentation",
+ "unicode-width",
+ "utf8parse",
+ "winapi",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.15"
@@ -1066,6 +1192,27 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "signal-hook"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801"
+dependencies = [
+ "libc",
+ "signal-hook-registry",
+]
+
+[[package]]
+name = "signal-hook-mio"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
+dependencies = [
+ "libc",
+ "mio",
+ "signal-hook",
+]
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.4.1"
@@ -1123,14 +1270,16 @@ dependencies = [
 name = "skysh"
 version = "0.8.0"
 dependencies = [
+ "crossterm",
  "libsky",
+ "rustyline",
  "skytable",
 ]
 
 [[package]]
 name = "skytable"
 version = "0.8.0"
-source = "git+https://github.com/skytable/client-rust.git?branch=octave#b2b0ea7197d9a3425809ce269e30b74ddd3eb340"
+source = "git+https://github.com/skytable/client-rust.git?branch=octave#ce16be88204044cd8358a5aa48521bd30e60ae33"
 dependencies = [
  "async-trait",
  "bb8",
@@ -1166,6 +1315,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "str-buf"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0"
+
 [[package]]
 name = "subtle"
 version = "2.5.0"
@@ -1297,12 +1452,30 @@ version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"
+
 [[package]]
 name = "unsafe-libyaml"
 version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa"
 
+[[package]]
+name = "utf8parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
+
 [[package]]
 name = "uuid"
 version = "1.4.1"
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index 68312688..e27538bd 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -10,3 +10,5 @@ edition = "2021"
 # internal deps
 libsky = { path = "../libsky" }
 skytable = { git = "https://github.com/skytable/client-rust.git", branch = "octave" }
+crossterm = "0.27.0"
+rustyline = "12.0.0"
diff --git a/cli/help_text/help b/cli/help_text/help
new file mode 100644
index 00000000..8a4d6ab6
--- /dev/null
+++ b/cli/help_text/help
@@ -0,0 +1,30 @@
+skysh 0.8.0
+Sayan N. <ohsayan@outlook.com>
+The Skytable interactive shell (skysh)
+
+USAGE:
+    skysh [OPTIONS]
+
+FLAGS:
+    --help          Diplays this help message
+    --version       Displays the shell version
+
+OPTIONS:
+    --endpoint      Set the endpoint for the connection
+    --user          Set the user for this client session
+    --password      Set the password for this client session
+    --tls-cert      Set the TLS certificate to use (for TLS endpoints)
+
+NOTES:
+    - When no endpoint is specified, skysh will attempt to connect to the default
+    TCP endpoint `tcp@127.0.0.1:2003`
+    - When no user is specified, skysh will attempt to authenticate as root
+    - All connections need an username and password. If this is not provided
+    via arguments, it will be asked for interactively
+    - Endpoints are specified using the Skytable endpoint syntax. For example,
+    the default TCP endpoint is `tcp@127.0.0.1:2003` while the default TLS
+    endpoint is `tls@127.0.0.1:2004`
+    - If you choose to use a TLS endpoint, you must provide a certificate.
+    Failing to do so will throw an error, as expected
+    - All history is stored in the `.sky_history` file. If you wish to delete
+    it, simply remove the file
\ No newline at end of file
diff --git a/cli/help_text/welcome b/cli/help_text/welcome
new file mode 100644
index 00000000..27c14bbd
--- /dev/null
+++ b/cli/help_text/welcome
@@ -0,0 +1,16 @@
+Welcome to the Skytable Interactive shell (REPL environment).
+
+Here are a few tips to help you get started:
+- Skytable uses its own query language called BlueQL. It is mostly like SQL
+but with some important changes for security
+- Since BlueQL doesn't need a query terminator ';' you do not need to use it
+here for running queries
+- You might be surprised to see that you can use literals in this REPL while
+Skytable does not allow the use of literals for security concerns. This is
+because whenever you run a query, the REPL turns it into a parameterized query.
+- You can also run some `skysh` specific commands:
+    - `!help` displays this help message
+    - `clear` clears the terminal screen
+    - `exit` exits the REPL session
+
+Now, it's time to get querying!
diff --git a/cli/src/args.rs b/cli/src/args.rs
new file mode 100644
index 00000000..29923f8d
--- /dev/null
+++ b/cli/src/args.rs
@@ -0,0 +1,221 @@
+/*
+ * Created on Wed Nov 15 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::error::{CliError, CliResult},
+    crossterm::{
+        event::{self, Event, KeyCode, KeyEvent},
+        terminal,
+    },
+    std::{
+        collections::{hash_map::Entry, HashMap},
+        env, fs,
+        io::{self, Write},
+        process::exit,
+    },
+};
+
+const TXT_HELP: &str = include_str!("../help_text/help");
+
+#[derive(Debug)]
+pub struct ClientConfig {
+    pub kind: ClientConfigKind,
+    pub username: String,
+    pub password: String,
+}
+
+impl ClientConfig {
+    pub fn new(kind: ClientConfigKind, username: String, password: String) -> Self {
+        Self {
+            kind,
+            username,
+            password,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum ClientConfigKind {
+    Tcp(String, u16),
+    Tls(String, u16, String),
+}
+
+#[derive(Debug)]
+pub enum Task {
+    HelpMessage(String),
+    OpenShell(ClientConfig),
+}
+
+enum TaskInner {
+    HelpMsg(String),
+    OpenShell(HashMap<String, String>),
+}
+
+fn load_env() -> CliResult<TaskInner> {
+    let mut args = HashMap::new();
+    let mut it = env::args().skip(1).into_iter();
+    while let Some(arg) = it.next() {
+        let (arg, arg_val) = match arg.as_str() {
+            "--help" => return Ok(TaskInner::HelpMsg(TXT_HELP.into())),
+            "--version" => return Ok(TaskInner::HelpMsg(format!("skysh v{}", libsky::VERSION))),
+            _ if arg.starts_with("--") => match it.next() {
+                Some(arg_val) => (arg, arg_val),
+                None => {
+                    // self contained?
+                    let split: Vec<&str> = arg.split("=").collect();
+                    if split.len() != 2 {
+                        return Err(CliError::ArgsErr(format!("expected value for {arg}")));
+                    }
+                    (split[0].into(), split[1].into())
+                }
+            },
+            unknown_arg => {
+                return Err(CliError::ArgsErr(format!(
+                    "unknown argument: {unknown_arg}"
+                )))
+            }
+        };
+        match args.entry(arg) {
+            Entry::Occupied(oe) => {
+                return Err(CliError::ArgsErr(format!(
+                    "found duplicate values for {}",
+                    oe.key()
+                )))
+            }
+            Entry::Vacant(ve) => {
+                ve.insert(arg_val);
+            }
+        }
+    }
+    Ok(TaskInner::OpenShell(args))
+}
+
+pub fn parse() -> CliResult<Task> {
+    let mut args = match load_env()? {
+        TaskInner::HelpMsg(msg) => return Ok(Task::HelpMessage(msg)),
+        TaskInner::OpenShell(args) => args,
+    };
+    let endpoint = match args.remove("--endpoint") {
+        None => ClientConfigKind::Tcp("127.0.0.1".into(), 2003),
+        Some(ep) => {
+            // should be in the format protocol@host:port
+            let proto_host_port: Vec<&str> = ep.split("@").collect();
+            if proto_host_port.len() != 2 {
+                return Err(CliError::ArgsErr("invalid value for --endpoint".into()));
+            }
+            let (protocol, host_port) = (proto_host_port[0], proto_host_port[1]);
+            let host_port: Vec<&str> = host_port.split(":").collect();
+            if host_port.len() != 2 {
+                return Err(CliError::ArgsErr("invalid value for --endpoint".into()));
+            }
+            let (host, port) = (host_port[0], host_port[1]);
+            let port = match port.parse::<u16>() {
+                Ok(port) => port,
+                Err(e) => {
+                    return Err(CliError::ArgsErr(format!(
+                        "invalid value for endpoint port. {e}"
+                    )))
+                }
+            };
+            let tls_cert = args.remove("--tls-cert");
+            match protocol {
+                "tcp" => {
+                    // TODO(@ohsayan): warn!
+                    ClientConfigKind::Tcp(host.into(), port)
+                }
+                "tls" => {
+                    // we need a TLS cert
+                    match tls_cert {
+                        Some(path) => {
+                            let cert = fs::read_to_string(path)?;
+                            ClientConfigKind::Tls(host.into(), port, cert)
+                        }
+                        None => {
+                            return Err(CliError::ArgsErr(format!(
+                                "must provide TLS cert when using TLS endpoint"
+                            )))
+                        }
+                    }
+                }
+                _ => {
+                    return Err(CliError::ArgsErr(format!(
+                        "unknown protocol scheme `{protocol}`"
+                    )))
+                }
+            }
+        }
+    };
+    let username = match args.remove("--user") {
+        Some(u) => u,
+        None => {
+            // default
+            "root".into()
+        }
+    };
+    let password = match args.remove("--password") {
+        Some(p) => p,
+        None => read_password("Enter password: ")?,
+    };
+    if args.is_empty() {
+        Ok(Task::OpenShell(ClientConfig::new(
+            endpoint, username, password,
+        )))
+    } else {
+        Err(CliError::ArgsErr(format!("found unknown arguments")))
+    }
+}
+
+fn read_password(prompt: &str) -> Result<String, std::io::Error> {
+    terminal::enable_raw_mode()?;
+    print!("{prompt}");
+    io::stdout().flush()?;
+    let mut password = String::new();
+    loop {
+        match event::read()? {
+            Event::Key(KeyEvent {
+                code: KeyCode::Char('c'),
+                modifiers: event::KeyModifiers::CONTROL,
+                ..
+            }) => {
+                terminal::disable_raw_mode()?;
+                println!();
+                exit(0x00)
+            }
+            Event::Key(KeyEvent { code, .. }) => match code {
+                KeyCode::Backspace => {
+                    let _ = password.pop();
+                }
+                KeyCode::Char(c) => password.push(c),
+                KeyCode::Enter => break,
+                _ => {}
+            },
+            _ => {}
+        }
+    }
+    terminal::disable_raw_mode()?;
+    println!();
+    Ok(password)
+}
diff --git a/cli/src/error.rs b/cli/src/error.rs
new file mode 100644
index 00000000..89878bac
--- /dev/null
+++ b/cli/src/error.rs
@@ -0,0 +1,60 @@
+/*
+ * Created on Wed Nov 15 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use core::fmt;
+
+pub type CliResult<T> = Result<T, CliError>;
+
+#[derive(Debug)]
+pub enum CliError {
+    QueryError(String),
+    ArgsErr(String),
+    ClientError(skytable::error::Error),
+    IoError(std::io::Error),
+}
+
+impl From<skytable::error::Error> for CliError {
+    fn from(cle: skytable::error::Error) -> Self {
+        Self::ClientError(cle)
+    }
+}
+
+impl From<std::io::Error> for CliError {
+    fn from(ioe: std::io::Error) -> Self {
+        Self::IoError(ioe)
+    }
+}
+
+impl fmt::Display for CliError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::ArgsErr(e) => write!(f, "incorrect arguments. {e}"),
+            Self::ClientError(e) => write!(f, "client error. {e}"),
+            Self::IoError(e) => write!(f, "i/o error. {e}"),
+            Self::QueryError(e) => write!(f, "invalid query. {e}"),
+        }
+    }
+}
diff --git a/cli/src/main.rs b/cli/src/main.rs
index 57a21b8e..c23a09b6 100644
--- a/cli/src/main.rs
+++ b/cli/src/main.rs
@@ -24,4 +24,32 @@
  *
 */
 
-fn main() {}
+macro_rules! fatal {
+    ($($arg:tt)*) => {{
+        eprintln!($($arg)*);
+        std::process::exit(0x01);
+    }}
+}
+
+mod args;
+mod error;
+mod query;
+mod repl;
+mod resp;
+
+use args::Task;
+
+fn main() {
+    match run() {
+        Ok(()) => {}
+        Err(e) => fatal!("cli error: {e}"),
+    }
+}
+
+fn run() -> error::CliResult<()> {
+    match args::parse()? {
+        Task::HelpMessage(msg) => println!("{msg}"),
+        Task::OpenShell(cfg) => repl::start(cfg)?,
+    }
+    Ok(())
+}
diff --git a/cli/src/query.rs b/cli/src/query.rs
new file mode 100644
index 00000000..db272493
--- /dev/null
+++ b/cli/src/query.rs
@@ -0,0 +1,266 @@
+/*
+ * Created on Thu Nov 16 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::error::{CliError, CliResult},
+    skytable::{
+        error::ClientResult, query::SQParam, response::Response, Connection, ConnectionTls, Query,
+    },
+};
+
+pub trait IsConnection {
+    fn execute_query(&mut self, q: Query) -> ClientResult<Response>;
+}
+
+impl IsConnection for Connection {
+    fn execute_query(&mut self, q: Query) -> ClientResult<Response> {
+        self.query(&q)
+    }
+}
+
+impl IsConnection for ConnectionTls {
+    fn execute_query(&mut self, q: Query) -> ClientResult<Response> {
+        self.query(&q)
+    }
+}
+
+#[derive(Debug, PartialEq)]
+enum Item {
+    UInt(u64),
+    SInt(i64),
+    Float(f64),
+    String(String),
+    Bin(Vec<u8>),
+}
+
+impl SQParam for Item {
+    fn push(self, buf: &mut Vec<u8>) {
+        match self {
+            Item::UInt(u) => u.push(buf),
+            Item::SInt(s) => s.push(buf),
+            Item::Float(f) => f.push(buf),
+            Item::String(s) => s.push(buf),
+            Item::Bin(b) => SQParam::push(&*b, buf),
+        }
+    }
+}
+
+pub struct Parameterizer {
+    buf: Vec<u8>,
+    i: usize,
+    params: Vec<Item>,
+    query: Vec<u8>,
+}
+
+impl Parameterizer {
+    pub fn new(q: String) -> Self {
+        Self {
+            buf: q.into_bytes(),
+            i: 0,
+            params: vec![],
+            query: vec![],
+        }
+    }
+    pub fn parameterize(mut self) -> CliResult<Query> {
+        while self.not_eof() {
+            match self.buf[self.i] {
+                b if b.is_ascii_alphabetic() || b == b'_' => self.read_ident(),
+                b if b.is_ascii_digit() => self.read_unsigned_integer(),
+                b'-' => self.read_signed_integer(),
+                quote_style @ (b'"' | b'\'') => {
+                    self.i += 1;
+                    self.read_string(quote_style)
+                }
+                b'`' => {
+                    self.i += 1;
+                    self.read_binary()
+                }
+                sym => {
+                    self.i += 1;
+                    self.query.push(sym);
+                    Ok(())
+                }
+            }?
+        }
+        match String::from_utf8(self.query) {
+            Ok(qstr) => {
+                let mut q = Query::new(&qstr);
+                self.params.into_iter().for_each(|p| {
+                    q.push_param(p);
+                });
+                Ok(q)
+            }
+            Err(_) => Err(CliError::QueryError("query is not valid UTF-8".into())),
+        }
+    }
+    fn read_string(&mut self, quote_style: u8) -> CliResult<()> {
+        self.query.push(b'?');
+        let mut string = Vec::new();
+        let mut terminated = false;
+        while self.not_eof() && !terminated {
+            let b = self.buf[self.i];
+            if b == b'\\' {
+                self.i += 1;
+                // escape sequence
+                if self.i == self.buf.len() {
+                    // string was not terminated
+                    return Err(CliError::QueryError("string not terminated".into()));
+                }
+                match self.buf[self.i] {
+                    b'\\' => {
+                        // escaped \
+                        string.push(b'\\');
+                    }
+                    b if b == quote_style => {
+                        // escape quote
+                        string.push(quote_style);
+                    }
+                    _ => return Err(CliError::QueryError("unknown escape sequence".into())),
+                }
+            }
+            if b == quote_style {
+                terminated = true;
+            } else {
+                string.push(b);
+            }
+            self.i += 1;
+        }
+        if terminated {
+            match String::from_utf8(string) {
+                Ok(s) => self.params.push(Item::String(s)),
+                Err(_) => return Err(CliError::QueryError("invalid UTF-8 string".into())),
+            }
+            Ok(())
+        } else {
+            return Err(CliError::QueryError("string not terminated".into()));
+        }
+    }
+    fn read_ident(&mut self) -> CliResult<()> {
+        // we're looking at an ident
+        let start = self.i;
+        self.i += 1;
+        while self.not_eof() {
+            if self.buf[self.i].is_ascii_alphanumeric() || self.buf[self.i] == b'_' {
+                self.i += 1;
+            } else {
+                break;
+            }
+        }
+        let stop = self.i;
+        self.query.extend(&self.buf[start..stop]);
+        Ok(())
+    }
+    fn read_float(&mut self, start: usize) -> CliResult<()> {
+        self.read_until_number_escape();
+        let stop = self.i;
+        match core::str::from_utf8(&self.buf[start..stop]).map(|v| v.parse()) {
+            Ok(Ok(num)) => self.params.push(Item::Float(num)),
+            _ => {
+                return Err(CliError::QueryError(
+                    "invalid floating point literal".into(),
+                ))
+            }
+        }
+        Ok(())
+    }
+    fn read_signed_integer(&mut self) -> CliResult<()> {
+        self.query.push(b'?');
+        // we must have encountered a `-`
+        let start = self.i;
+        self.read_until_number_escape();
+        let stop = self.i;
+        match core::str::from_utf8(&self.buf[start..stop]).map(|v| v.parse()) {
+            Ok(Ok(s)) => self.params.push(Item::SInt(s)),
+            _ => {
+                return Err(CliError::QueryError(
+                    "invalid signed integer literal".into(),
+                ))
+            }
+        }
+        Ok(())
+    }
+    fn read_unsigned_integer(&mut self) -> CliResult<()> {
+        self.query.push(b'?');
+        let start = self.i;
+        let mut ret = 0u64;
+        while self.not_eof() {
+            match self.buf[self.i] {
+                b if b.is_ascii_digit() => {
+                    self.i += 1;
+                    ret = match ret
+                        .checked_mul(10)
+                        .map(|v| v.checked_add((b & 0x0f) as u64))
+                    {
+                        Some(Some(r)) => r,
+                        _ => return Err(CliError::QueryError("bad value for integer".into())),
+                    };
+                }
+                b'.' => {
+                    self.i += 1;
+                    // uh oh, that's a float
+                    return self.read_float(start);
+                }
+                b if b == b' ' || b == b'\t' || b.is_ascii_punctuation() => {
+                    break;
+                }
+                _ => {
+                    // nothing else is valid here
+                    return Err(CliError::QueryError(
+                        "invalid unsigned integer literal".into(),
+                    ));
+                }
+            }
+        }
+        self.params.push(Item::UInt(ret));
+        Ok(())
+    }
+    fn read_until_number_escape(&mut self) {
+        while self.not_eof() {
+            let b = self.buf[self.i];
+            if b == b'\n' || b == b'\t' || b.is_ascii_punctuation() {
+                break;
+            }
+            self.i += 1;
+        }
+    }
+    fn read_binary(&mut self) -> CliResult<()> {
+        self.query.push(b'?');
+        let start = self.i;
+        while self.not_eof() {
+            let b = self.buf[self.i];
+            self.i += 1;
+            if b == b'`' {
+                self.params
+                    .push(Item::Bin(self.buf[start..self.i].to_vec()));
+                return Ok(());
+            }
+        }
+        Err(CliError::QueryError("binary literal not terminated".into()))
+    }
+    fn not_eof(&self) -> bool {
+        self.i < self.buf.len()
+    }
+}
diff --git a/cli/src/repl.rs b/cli/src/repl.rs
new file mode 100644
index 00000000..48ea52b6
--- /dev/null
+++ b/cli/src/repl.rs
@@ -0,0 +1,137 @@
+/*
+ * Created on Thu Nov 16 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::{
+        args::{ClientConfig, ClientConfigKind},
+        error::{CliError, CliResult},
+        query::{self, IsConnection},
+        resp,
+    },
+    crossterm::{cursor, execute, terminal},
+    rustyline::{config::Configurer, error::ReadlineError, DefaultEditor},
+    skytable::Config,
+    std::io::{stdout, ErrorKind},
+};
+
+const SKYSH_HISTORY_FILE: &str = ".sky_history";
+const TXT_WELCOME: &str = include_str!("../help_text/welcome");
+
+pub fn start(cfg: ClientConfig) -> CliResult<()> {
+    match cfg.kind {
+        ClientConfigKind::Tcp(host, port) => {
+            let c = Config::new(&host, port, &cfg.username, &cfg.password).connect()?;
+            println!(
+                "Authenticated as '{}' on {}:{} over Skyhash/TCP\n---",
+                &cfg.username, &host, &port
+            );
+            repl(c)
+        }
+        ClientConfigKind::Tls(host, port, cert) => {
+            let c = Config::new(&host, port, &cfg.username, &cfg.password).connect_tls(&cert)?;
+            println!(
+                "Authenticated as '{}' on {}:{} over Skyhash/TLS\n---",
+                &cfg.username, &host, &port
+            );
+            repl(c)
+        }
+    }
+}
+
+fn repl<C: IsConnection>(mut con: C) -> CliResult<()> {
+    let init_editor = || {
+        let mut editor = DefaultEditor::new()?;
+        editor.set_auto_add_history(true);
+        editor.set_history_ignore_dups(true)?;
+        editor.bind_sequence(
+            rustyline::KeyEvent(
+                rustyline::KeyCode::BracketedPasteStart,
+                rustyline::Modifiers::NONE,
+            ),
+            rustyline::Cmd::Noop,
+        );
+        match editor.load_history(SKYSH_HISTORY_FILE) {
+            Ok(()) => {}
+            Err(e) => match e {
+                ReadlineError::Io(ref ioe) => match ioe.kind() {
+                    ErrorKind::NotFound => {
+                        println!("{TXT_WELCOME}");
+                    }
+                    _ => return Err(e),
+                },
+                e => return Err(e),
+            },
+        }
+        rustyline::Result::Ok(editor)
+    };
+    let mut editor = match init_editor() {
+        Ok(e) => e,
+        Err(e) => fatal!("error: failed to init REPL. {e}"),
+    };
+    loop {
+        match editor.readline("> ") {
+            Ok(line) => match line.as_str() {
+                "!help" => println!("{TXT_WELCOME}"),
+                "exit" => break,
+                "clear" => clear_screen()?,
+                _ => {
+                    if line.is_empty() {
+                        continue;
+                    }
+                    match query::Parameterizer::new(line).parameterize() {
+                        Ok(q) => resp::format_response(con.execute_query(q)?)?,
+                        Err(e) => match e {
+                            CliError::QueryError(e) => {
+                                eprintln!("[skysh error]: bad query. {e}");
+                                continue;
+                            }
+                            _ => return Err(e),
+                        },
+                    };
+                }
+            },
+            Err(e) => match e {
+                ReadlineError::Interrupted | ReadlineError::Eof => {
+                    // done
+                    break;
+                }
+                ReadlineError::WindowResized => {}
+                e => fatal!("error: failed to read line REPL. {e}"),
+            },
+        }
+    }
+    editor
+        .save_history(SKYSH_HISTORY_FILE)
+        .expect("failed to save history");
+    println!("Goodbye!");
+    Ok(())
+}
+
+fn clear_screen() -> std::io::Result<()> {
+    let mut stdout = stdout();
+    execute!(stdout, terminal::Clear(terminal::ClearType::All))?;
+    execute!(stdout, cursor::MoveTo(0, 0))
+}
diff --git a/cli/src/resp.rs b/cli/src/resp.rs
new file mode 100644
index 00000000..c35f3374
--- /dev/null
+++ b/cli/src/resp.rs
@@ -0,0 +1,144 @@
+/*
+ * Created on Thu Nov 16 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::error::CliResult,
+    crossterm::{
+        style::{Color, ResetColor, SetForegroundColor},
+        ExecutableCommand,
+    },
+    skytable::response::{Response, Row, Value},
+    std::io::{self, Write},
+};
+
+pub fn format_response(resp: Response) -> CliResult<()> {
+    match resp {
+        Response::Empty => print_cyan("(Okay)\n")?,
+        Response::Error(e) => print_red(&format!("(server error code: {e})\n"))?,
+        Response::Value(v) => {
+            print_value(v)?;
+            println!();
+        }
+        Response::Row(r) => {
+            print_row(r)?;
+            println!();
+        }
+    };
+    Ok(())
+}
+
+fn print_row(r: Row) -> CliResult<()> {
+    print!("(");
+    let mut columns = r.into_values().into_iter().peekable();
+    while let Some(cell) = columns.next() {
+        print_value(cell)?;
+        if columns.peek().is_some() {
+            print!(", ");
+        }
+    }
+    print!(")");
+    Ok(())
+}
+
+fn print_value(v: Value) -> CliResult<()> {
+    match v {
+        Value::Null => print_gray("null")?,
+        Value::String(s) => print_string(&s),
+        Value::Binary(b) => print_binary(&b),
+        Value::Bool(b) => print!("{b}"),
+        Value::UInt8(i) => print!("{i}"),
+        Value::UInt16(i) => print!("{i}"),
+        Value::UInt32(i) => print!("{i}"),
+        Value::UInt64(i) => print!("{i}"),
+        Value::SInt8(i) => print!("{i}"),
+        Value::SInt16(i) => print!("{i}"),
+        Value::SInt32(i) => print!("{i}"),
+        Value::SInt64(i) => print!("{i}"),
+        Value::Float32(f) => print!("{f}"),
+        Value::Float64(f) => print!("{f}"),
+        Value::List(items) => {
+            print!("[");
+            let mut items = items.into_iter().peekable();
+            while let Some(item) = items.next() {
+                print_value(item)?;
+                if items.peek().is_some() {
+                    print!(", ");
+                }
+            }
+            print!("]");
+        }
+    }
+    Ok(())
+}
+
+fn print_binary(b: &[u8]) {
+    let mut it = b.into_iter().peekable();
+    print!("[");
+    while let Some(byte) = it.next() {
+        print!("{byte}");
+        if it.peek().is_some() {
+            print!(", ");
+        }
+    }
+    print!("]");
+}
+
+fn print_string(s: &str) {
+    print!("\"");
+    for ch in s.chars() {
+        if ch == '"' || ch == '\'' {
+            print!("\\{ch}");
+        } else if ch == '\t' {
+            print!("\\t");
+        } else if ch == '\n' {
+            print!("\\n");
+        } else {
+            print!("{ch}");
+        }
+    }
+    print!("\"");
+}
+
+fn print_gray(s: &str) -> std::io::Result<()> {
+    print_colored_text(s, Color::White)
+}
+
+fn print_red(s: &str) -> std::io::Result<()> {
+    print_colored_text(s, Color::Red)
+}
+
+fn print_cyan(s: &str) -> std::io::Result<()> {
+    print_colored_text(s, Color::Cyan)
+}
+
+fn print_colored_text(text: &str, color: Color) -> std::io::Result<()> {
+    let mut stdout = io::stdout();
+    stdout.execute(SetForegroundColor(color))?;
+    print!("{text}");
+    stdout.flush()?;
+    stdout.execute(ResetColor)?;
+    Ok(())
+}
diff --git a/server/src/engine/mem/scanner.rs b/server/src/engine/mem/scanner.rs
index e6fb1bfb..48332b64 100644
--- a/server/src/engine/mem/scanner.rs
+++ b/server/src/engine/mem/scanner.rs
@@ -104,6 +104,9 @@ impl<'a, T> Scanner<'a, T> {
 }
 
 impl<'a, T> Scanner<'a, T> {
+    pub fn inner_buffer(&self) -> &'a [T] {
+        &self.d
+    }
     /// Manually set the cursor position
     ///
     /// ## Safety
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index f25a3ad3..3ca0b428 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -442,32 +442,26 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             if okay {
                 slf.l.push_token(Lit::new_sint(int))
             } else {
-                slf.l.set_error(QueryError::LexInvalidLiteral)
+                slf.l.set_error(QueryError::LexInvalidParameter)
             }
         },
         // float
         |slf| {
-            let Some(size_of_body) = slf
-                .param_buffer
-                .try_next_ascii_u64_lf_separated_or_restore_cursor()
-            else {
-                slf.l.set_error(QueryError::LexInvalidParameter);
-                return;
-            };
-            let body = match slf
-                .param_buffer
-                .try_next_variable_block(size_of_body as usize)
-            {
-                Some(body) => body,
-                None => {
-                    slf.l.set_error(QueryError::LexInvalidParameter);
+            let start = slf.param_buffer.cursor();
+            while !slf.param_buffer.eof() {
+                let cursor = slf.param_buffer.cursor();
+                let byte = slf.param_buffer.next_byte();
+                if byte == b'\n' {
+                    match core::str::from_utf8(&slf.param_buffer.inner_buffer()[start..cursor])
+                        .map(core::str::FromStr::from_str)
+                    {
+                        Ok(Ok(f)) => slf.l.push_token(Lit::new_float(f)),
+                        _ => slf.l.set_error(QueryError::LexInvalidParameter),
+                    }
                     return;
                 }
-            };
-            match core::str::from_utf8(body).map(core::str::FromStr::from_str) {
-                Ok(Ok(fp)) => slf.l.push_token(Lit::new_float(fp)),
-                _ => slf.l.set_error(QueryError::LexInvalidParameter),
             }
+            slf.l.set_error(QueryError::LexInvalidParameter)
         },
         // binary
         |slf| {
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index d5e3e121..0985e101 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -195,7 +195,7 @@ fn make_safe_query(a: &[u8], b: &[u8]) -> (Vec<u8>, usize) {
 fn safe_query_all_literals() {
     let (query, query_window) = make_safe_query(
         b"? ? ? ? ? ? ?",
-        b"\x00\x01\x01\x021234\n\x03-1234\n\x049\n1234.5678\x0513\nbinarywithlf\n\x065\nsayan",
+        b"\x00\x01\x01\x021234\n\x03-1234\n\x041234.5678\n\x0513\nbinarywithlf\n\x065\nsayan",
     );
     let ret = lex_secure(&query, query_window).unwrap();
     assert_eq!(

From f923a4fc399a3c8f10813634f39e20b853e07911 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 16 Nov 2023 22:37:38 +0530
Subject: [PATCH 281/310] Fix test harness

Don't try to initiate an actual connection (right now). Instead,
try to see if the TCP port is live and can be connected to.

Also added some misc `rustfmt` fixes.
---
 Cargo.lock                                |  1 -
 harness/Cargo.toml                        |  1 -
 harness/src/test/svc.rs                   | 14 +++++---------
 server/src/engine/fractal/mgr.rs          |  2 +-
 server/src/engine/net/mod.rs              | 10 +++++-----
 server/src/engine/ql/tests/lexer_tests.rs |  2 +-
 server/src/engine/storage/v1/tests/tx.rs  |  6 ++++--
 server/src/engine/sync/smart.rs           |  1 -
 8 files changed, 16 insertions(+), 21 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index c084f43a..046992a4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -520,7 +520,6 @@ dependencies = [
  "log",
  "openssl",
  "powershell_script",
- "skytable",
  "zip",
 ]
 
diff --git a/harness/Cargo.toml b/harness/Cargo.toml
index 605e03c9..1bd479a9 100644
--- a/harness/Cargo.toml
+++ b/harness/Cargo.toml
@@ -7,7 +7,6 @@ edition = "2021"
 
 [dependencies]
 # internal deps
-skytable = { git = "https://github.com/skytable/client-rust.git", branch = "octave" }
 libsky = { path = "../libsky" }
 # external deps
 env_logger = "0.10.0"
diff --git a/harness/src/test/svc.rs b/harness/src/test/svc.rs
index 433e0fcd..1203f81a 100644
--- a/harness/src/test/svc.rs
+++ b/harness/src/test/svc.rs
@@ -32,10 +32,6 @@ use {
         util::{self},
         HarnessError, HarnessResult, ROOT_DIR,
     },
-    skytable::{
-        error::{ClientResult, Error},
-        Config, Connection,
-    },
     std::{
         io::ErrorKind,
         path::Path,
@@ -55,9 +51,9 @@ const TESTSUITE_SERVER_HOST: &str = "127.0.0.1";
 /// The workspace root
 const WORKSPACE_ROOT: &str = env!("ROOT_DIR");
 
-fn connect_db(host: &str, port: u16) -> ClientResult<Connection> {
-    let cfg = Config::new(host, port, "root", "password12345678");
-    cfg.connect()
+fn connect_db(host: &str, port: u16) -> std::io::Result<std::net::TcpStream> {
+    let tcp_stream = std::net::TcpStream::connect((host, port))?;
+    Ok(tcp_stream)
 }
 
 /// Get the command to start the provided server1
@@ -99,10 +95,10 @@ pub(super) fn wait_for_server_exit() -> HarnessResult<()> {
     Ok(())
 }
 
-fn connection_refused<T>(input: ClientResult<T>) -> HarnessResult<bool> {
+fn connection_refused<T>(input: std::io::Result<T>) -> HarnessResult<bool> {
     match input {
         Ok(_) => Ok(false),
-        Err(Error::IoError(e))
+        Err(e)
             if matches!(
                 e.kind(),
                 ErrorKind::ConnectionRefused | ErrorKind::ConnectionReset
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 63906601..2abc9b2d 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -307,7 +307,7 @@ impl FractalMgr {
                         }
                     }
                     Err(_) => {
-                        log::error!(
+                        error!(
                             "fhp: error writing data batch for model {}. retrying ...",
                             model_id.uuid()
                         );
diff --git a/server/src/engine/net/mod.rs b/server/src/engine/net/mod.rs
index 2e1977c2..4467fd8e 100644
--- a/server/src/engine/net/mod.rs
+++ b/server/src/engine/net/mod.rs
@@ -40,7 +40,7 @@ use {
     },
     std::{cell::Cell, net::SocketAddr, pin::Pin, time::Duration},
     tokio::{
-        io::{AsyncRead, AsyncWrite, BufWriter, AsyncWriteExt},
+        io::{AsyncRead, AsyncWrite, AsyncWriteExt, BufWriter},
         net::{TcpListener, TcpStream},
         sync::{broadcast, mpsc, Semaphore},
     },
@@ -220,7 +220,7 @@ impl Listener {
                     /*
                         SECURITY: IGNORE THIS ERROR
                     */
-                    log::error!("failed to accept connection on TCP socket: `{e}`");
+                    warn!("failed to accept connection on TCP socket: `{e}`");
                     continue;
                 }
             };
@@ -232,7 +232,7 @@ impl Listener {
             );
             tokio::spawn(async move {
                 if let Err(e) = handler.run().await {
-                    log::error!("error handling client connection: `{e}`");
+                    warn!("error handling client connection: `{e}`");
                 }
             });
             // return the permit
@@ -274,7 +274,7 @@ impl Listener {
                     /*
                         SECURITY: Once again, ignore this error
                     */
-                    log::error!("failed to accept connection on TLS socket: `{e:#?}`");
+                    warn!("failed to accept connection on TLS socket: `{e}`");
                     continue;
                 }
             };
@@ -286,7 +286,7 @@ impl Listener {
             );
             tokio::spawn(async move {
                 if let Err(e) = handler.run().await {
-                    log::error!("error handling client TLS connection: `{e}`");
+                    warn!("error handling client TLS connection: `{e}`");
                 }
             });
         }
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index 0985e101..558bfc5c 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -217,7 +217,7 @@ const SFQ_BOOL_FALSE: &[u8] = b"\x01\0";
 const SFQ_BOOL_TRUE: &[u8] = b"\x01\x01";
 const SFQ_UINT: &[u8] = b"\x0218446744073709551615\n";
 const SFQ_SINT: &[u8] = b"\x03-9223372036854775808\n";
-const SFQ_FLOAT: &[u8] = b"\x0411\n3.141592654";
+const SFQ_FLOAT: &[u8] = b"\x043.141592654\n";
 const SFQ_BINARY: &[u8] = "\x0546\ncringe😃😄😁😆😅😂🤣😊😸😺".as_bytes();
 const SFQ_STRING: &[u8] = "\x0646\ncringe😃😄😁😆😅😂🤣😊😸😺".as_bytes();
 
diff --git a/server/src/engine/storage/v1/tests/tx.rs b/server/src/engine/storage/v1/tests/tx.rs
index ba814378..6258b40c 100644
--- a/server/src/engine/storage/v1/tests/tx.rs
+++ b/server/src/engine/storage/v1/tests/tx.rs
@@ -130,8 +130,10 @@ fn open_log(
     log_name: &str,
     db: &Database,
 ) -> RuntimeResult<JournalWriter<super::VirtualFS, DatabaseTxnAdapter>> {
-    journal::open_or_create_journal::<DatabaseTxnAdapter, super::VirtualFS, spec::TestFile>(log_name, db)
-        .map(|v| v.into_inner())
+    journal::open_or_create_journal::<DatabaseTxnAdapter, super::VirtualFS, spec::TestFile>(
+        log_name, db,
+    )
+    .map(|v| v.into_inner())
 }
 
 #[test]
diff --git a/server/src/engine/sync/smart.rs b/server/src/engine/sync/smart.rs
index 3d2b79cc..59c710fb 100644
--- a/server/src/engine/sync/smart.rs
+++ b/server/src/engine/sync/smart.rs
@@ -24,7 +24,6 @@
  *
 */
 
-
 use {
     super::atm::{ORD_ACQ, ORD_REL, ORD_RLX},
     std::{

From 3cdd8140670ff4ea5494ea6c8da546f6efac50a2 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 16 Nov 2023 23:39:06 +0530
Subject: [PATCH 282/310] Ensure PID locks prevent corruption

---
 Cargo.lock                             |   1 -
 scripts/unicode.pl                     | 227 -------------------------
 server/Cargo.toml                      |   5 -
 server/build.rs                        |   9 -
 server/native/flock-posix.c            |  60 -------
 server/src/engine/core/exec.rs         |   4 +-
 server/src/engine/fractal/sys_store.rs |  24 ++-
 server/src/engine/ql/dcl.rs            |   8 +-
 server/src/engine/storage/v1/sysdb.rs  |  24 +--
 server/src/main.rs                     |  11 +-
 server/src/util/compiler.rs            |  14 --
 server/src/util/os/flock.rs            |  24 +--
 server/src/util/test_utils.rs          |   6 +-
 13 files changed, 50 insertions(+), 367 deletions(-)
 delete mode 100644 scripts/unicode.pl
 delete mode 100644 server/build.rs
 delete mode 100644 server/native/flock-posix.c

diff --git a/Cargo.lock b/Cargo.lock
index 046992a4..eb537c47 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1244,7 +1244,6 @@ name = "skyd"
 version = "0.8.0"
 dependencies = [
  "bytes",
- "cc",
  "crc",
  "crossbeam-epoch",
  "env_logger",
diff --git a/scripts/unicode.pl b/scripts/unicode.pl
deleted file mode 100644
index 9457f1e1..00000000
--- a/scripts/unicode.pl
+++ /dev/null
@@ -1,227 +0,0 @@
-#!/usr/bin/perl -w
-=pod
-All credits for the random unicode string generation logic go to Paul Sarena who released
-the original version here: https://github.com/bits/UTF-8-Unicode-Test-Documents and released
-it under the BSD 3-Clause "New" or "Revised" License 
-=cut
-use strict;
-use warnings qw( FATAL utf8 );
-use utf8;  # tell Perl parser there are non-ASCII characters in this lexical scope
-use open qw( :encoding(UTF-8) :std );  # Declare that anything that opens a filehandles within this lexical scope is to assume that that stream is encoded in UTF-8 unless you tell it otherwise
-
-use Encode;
-use HTML::Entities;
-
-my $html_pre = q|<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-	<title>UTF-8 Codepoint Sequence</title>
-</head>
-<body>|;
-
-my $html_post = q|</body>
-</html>|;
-
-my $output_directory = './utf8/';
-
-my $utf8_seq;
-
-#    0000–​FFFF Plane 0:      Basic Multilingual Plane
-#  10000–​1FFFF Plane 1:      Supplementary Multilingual Plane
-#  20000–​2FFFF Plane 2:      Supplementary Ideographic Plane
-#  30000–​DFFFF Planes 3–13:  Unassigned
-#  E0000–​EFFFF Plane 14:     Supplement­ary Special-purpose Plane
-# F0000–​10FFFF Planes 15–16: Supplement­ary Private Use Area
-
-foreach my $separator ('', ' ') {
-	foreach my $end (0xFF, 0xFFF, 0xFFFF, 0x1FFFF, 0x2FFFF, 0x10FFFF) {
-
-		# UTF-8 codepoint sequence of assigned, printable codepoints
-		$utf8_seq = gen_seq({
-			start => 0x00,
-			end => $end,
-			separator => $separator,
-			skip_unprintable => 1,
-				replace_unprintable => 1,
-			skip_unassigned => 1,
-			writefiles => ($separator ? 'txt,html' : 'txt')
-		});
-
-
-		# UTF-8 codepoint sequence of assigned, printable and unprintable codepoints as-is
-		$utf8_seq = gen_seq({
-			start => 0x00,
-			end => $end,
-			separator => $separator,
-			skip_unprintable => 0,
-				replace_unprintable => 0,
-			skip_unassigned => 1,
-			writefiles => ($separator ? 'txt,html' : 'txt')
-		});
-		# UTF-8 codepoint sequence of assigned, printable and unprintable codepoints replaced
-		$utf8_seq = gen_seq({
-			start => 0x00,
-			end => $end,
-			separator => $separator,
-			skip_unprintable => 0,
-				replace_unprintable => 1,
-			skip_unassigned => 1,
-			writefiles => ($separator ? 'txt,html' : 'txt')
-		});
-
-
-		# UTF-8 codepoint sequence of assinged and unassigned, printable and unprintable codepoints as-is
-		$utf8_seq = gen_seq({
-			start => 0x00,
-			end => $end,
-			separator => $separator,
-			skip_unprintable => 0,
-				replace_unprintable => 0,
-			skip_unassigned => 0,
-			writefiles => ($separator ? 'txt,html' : 'txt')
-		});
-		# UTF-8 codepoint sequence of assinged and unassigned, printable and unprintable codepoints replaced
-		$utf8_seq = gen_seq({
-			start => 0x00,
-			end => $end,
-			separator => $separator,
-			skip_unprintable => 0,
-				replace_unprintable => 1,
-			skip_unassigned => 0,
-			writefiles => ($separator ? 'txt,html' : 'txt')
-		});
-
-	}
-}
-
-# print Encode::encode('UTF-8', $utf8_seq), "\n";
-
-
-
-sub gen_seq{
-	my $config = shift;
-
-	$config->{start}               = 0x00        unless defined $config->{start};
-	$config->{end}                 = 0x10FFFF    unless defined $config->{end};
-	$config->{skip_unassigned}     = 1           unless defined $config->{skip_unassigned};
-	$config->{skip_unprintable}    = 1           unless defined $config->{skip_unprintable};
-	$config->{replace_unprintable} = 1           unless defined $config->{replace_unprintable};
-	$config->{separator}           = ' '         unless defined $config->{separator};
-	$config->{newlines_every}      = 50          unless defined $config->{newlines_every};
-	$config->{writefiles}          = 'text,html' unless defined $config->{writefiles};
-
-	my $utf8_seq;
-	my $codepoints_this_line = 0;
-	my $codepoints_printed = 0;
-
-	for my $i ($config->{start} .. $config->{end}) {
-
-		next if ($i >= 0xD800 && $i <= 0xDFFF); # high and low surrogate halves used by UTF-16 (U+D800 through U+DFFF) are not legal Unicode values, and the UTF-8 encoding of them is an invalid byte sequence
-		next if ($i >= 0xFDD0 && $i <= 0xFDEF); # Non-characters
-		next if ( # Non-characters
-			$i ==   0xFFFE || $i ==   0xFFFF ||
-			$i ==  0x1FFFE || $i ==  0x1FFFF ||
-			$i ==  0x2FFFE || $i ==  0x2FFFF ||
-			$i ==  0x3FFFE || $i ==  0x3FFFF ||
-			$i ==  0x4FFFE || $i ==  0x4FFFF ||
-			$i ==  0x5FFFE || $i ==  0x5FFFF ||
-			$i ==  0x6FFFE || $i ==  0x6FFFF ||
-			$i ==  0x7FFFE || $i ==  0x7FFFF ||
-			$i ==  0x8FFFE || $i ==  0x8FFFF ||
-			$i ==  0x9FFFE || $i ==  0x9FFFF ||
-			$i ==  0xaFFFE || $i ==  0xAFFFF ||
-			$i ==  0xbFFFE || $i ==  0xBFFFF ||
-			$i ==  0xcFFFE || $i ==  0xCFFFF ||
-			$i ==  0xdFFFE || $i ==  0xDFFFF ||
-			$i ==  0xeFFFE || $i ==  0xEFFFF ||
-			$i ==  0xfFFFE || $i ==  0xFFFFF ||
-			$i == 0x10FFFE || $i == 0x10FFFF
-		);
-
-		my $codepoint = chr($i);
-
-		# skip unassiggned codepoints
-		next if $config->{skip_unassigned} && $codepoint !~ /^\p{Assigned}/o;
-
-		if ( $codepoint =~ /^\p{IsPrint}/o ) {
-			$utf8_seq .= $codepoint;
-		} else { # not printable
-			next if $config->{skip_unprintable};
-			# include unprintable or replace it
-			$utf8_seq .= $config->{replace_unprintable} ? '�' : $codepoint;
-		}
-
-		$codepoints_printed++;
-
-		if ($config->{separator}) {
-			if ($config->{newlines_every} && $codepoints_this_line++ == $config->{newlines_every}) {
-				$utf8_seq .= "\n";
-				$codepoints_this_line = 0;
-			} else {
-				$utf8_seq .= $config->{separator};
-			}
-		}
-	}
-
-	utf8::upgrade($utf8_seq);
-
-
-	if ($config->{writefiles}) {
-
-		my $filebasename = 'utf8_sequence_' .
-			(sprintf '%#x', $config->{start}) .
-			'-' .
-			(sprintf '%#x', $config->{end}) .
-			($config->{skip_unassigned} ? '_assigned' : '_including-unassigned') .
-			($config->{skip_unprintable} ? '_printable' : '_including-unprintable') .
-			(!$config->{skip_unprintable} ?
-				($config->{replace_unprintable} ? '-replaced' : '-asis') :
-				''
-			) .
-			($config->{separator} ?
-				($config->{newlines_every} ? '' : '_without-newlines') :
-				'_unseparated'
-			);
-
-
-		my $title = 'UTF-8 codepoint sequence' .
-			($config->{skip_unassigned} ? ' of assigned' : ' of assinged and unassigned') .
-			($config->{skip_unprintable} ? ', printable' : ', with unprintable') .
-			(!$config->{skip_unprintable} ?
-				($config->{replace_unprintable} ? ' codepoints replaced' : ' codepoints as-is') :
-				' codepoints'
-			) .
-			' in the range ' .
-			(sprintf '%#x', $config->{start}) .
-			'-' .
-			(sprintf '%#x', $config->{end}) .
-			($config->{newlines_every} ? '' : ', as a long string without newlines');
-
-		my $html_pre_custom = $html_pre;
-		$html_pre_custom =~ s|UTF\-8 codepoint sequence|$title|;
-
-
-		my $filename = ${output_directory} . ($config->{separator} ? '' : 'un') . 'separated/' . ${filebasename};
-
-		if ($config->{writefiles} =~ /te?xt/) {
-			open FH, ">${filename}.txt" or die "cannot open $filename: $!";
-			print FH $utf8_seq;
-			close FH;
-		}
-
-		if ($config->{writefiles} =~ /html/) {
-			open FH, ">${filename}_unescaped.html" or die "cannot open $filename: $!";
-			print FH $html_pre_custom, $utf8_seq, $html_post;
-			close FH;
-		}
-
-		# open FH, ">${output_directory}${filebasename}_escaped.html";
-		# print FH $html_pre_custom, HTML::Entities::encode_entities($utf8_seq), $html_post;
-		# close FH;
-
-		print "Output $title ($codepoints_printed codepoints)\n";
-	}
-
-	return $utf8_seq;
-}
\ No newline at end of file
diff --git a/server/Cargo.toml b/server/Cargo.toml
index d99e8c5a..51e235bb 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -1,6 +1,5 @@
 [package]
 authors = ["Sayan Nandan <ohsayan@outlook.com>"]
-build = "build.rs"
 edition = "2021"
 name = "skyd"
 version = "0.8.0"
@@ -35,10 +34,6 @@ winapi = { version = "0.3.9", features = ["fileapi"] }
 # external deps
 libc = "0.2.147"
 
-[target.'cfg(unix)'.build-dependencies]
-# external deps
-cc = "1.0.82"
-
 [dev-dependencies]
 # external deps
 rand = "0.8.5"
diff --git a/server/build.rs b/server/build.rs
deleted file mode 100644
index 666a620b..00000000
--- a/server/build.rs
+++ /dev/null
@@ -1,9 +0,0 @@
-fn main() {
-    #[cfg(unix)]
-    {
-        println!("cargo:rerun-if-changed=native/flock-posix.c");
-        cc::Build::new()
-            .file("native/flock-posix.c")
-            .compile("libflock-posix.a");
-    }
-}
diff --git a/server/native/flock-posix.c b/server/native/flock-posix.c
deleted file mode 100644
index f4e33a9c..00000000
--- a/server/native/flock-posix.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Created on Fri Aug 07 2020
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- */
-
-#include <errno.h>
-#include <sys/file.h>
-
-/* Acquire an exclusive lock for a file with the given descriptor */
-int lock_exclusive(int descriptor) {
-  if (descriptor < 0) {
-    return EBADF;
-  }
-  if (flock(descriptor, LOCK_EX) == -1) {
-    return errno;
-  }
-  return 0;
-}
-
-int try_lock_exclusive(int descriptor) {
-  if (descriptor < 0) {
-    return EBADF;
-  }
-  if (flock(descriptor, LOCK_EX | LOCK_NB) == -1) {
-    return errno;
-  }
-  return 0;
-}
-
-/* Unlock a file with the given descriptor */
-int unlock(int descriptor) {
-  if (descriptor < 0) {
-    return EBADF;
-  }
-  if (flock(descriptor, LOCK_UN) == -1) {
-    return errno;
-  }
-  return 0;
-}
\ No newline at end of file
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 4446a9e5..6c4b69b8 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -137,7 +137,7 @@ async fn run_blocking_stmt(
     let d_s = (drop & Token![space].eq(a) & last_id) as u8 * 6;
     let d_m = (drop & Token![model].eq(a) & last_id) as u8 * 7;
     let fc = sysctl as u8 | c_s | c_m | a_s | a_m | d_s | d_m;
-    state.cursor_ahead();
+    state.cursor_ahead_if(!sysctl);
     static BLK_EXEC: [fn(Global, &ClientLocalState, RawSlice<Token<'static>>) -> QueryResult<()>;
         8] = [
         |_, _, _| Err(QueryError::QLUnknownStatement), // unknown
@@ -174,7 +174,7 @@ fn blocking_exec_sysctl(
     /*
         currently supported: sysctl create user, sysctl drop user
     */
-    if state.remaining() != 2 {
+    if state.remaining() < 2 {
         return Err(QueryError::QLInvalidSyntax);
     }
     let (a, b) = (state.fw_read(), state.fw_read());
diff --git a/server/src/engine/fractal/sys_store.rs b/server/src/engine/fractal/sys_store.rs
index 7ec79546..5dd7ac44 100644
--- a/server/src/engine/fractal/sys_store.rs
+++ b/server/src/engine/fractal/sys_store.rs
@@ -154,6 +154,16 @@ impl<Fs: RawFSInterface> SystemStore<Fs> {
             _fs: PhantomData,
         }
     }
+    fn _try_sync_or(&self, auth: &mut SysAuth, rb: impl FnOnce(&mut SysAuth)) -> QueryResult<()> {
+        match self.sync_db(auth) {
+            Ok(()) => Ok(()),
+            Err(e) => {
+                error!("failed to sync system store: {e}");
+                rb(auth);
+                Err(e.into())
+            }
+        }
+    }
     /// Create a new user with the given details
     pub fn create_new_user(&self, username: String, password: String) -> QueryResult<()> {
         // TODO(@ohsayan): we want to be very careful with this
@@ -166,10 +176,9 @@ impl<Fs: RawFSInterface> SystemStore<Fs> {
                         .unwrap()
                         .into_boxed_slice(),
                 ));
-                self.sync_db_or_rollback(|| {
+                self._try_sync_or(&mut auth, |auth| {
                     auth.users.remove(_username.as_str());
-                })?;
-                Ok(())
+                })
             }
             Entry::Occupied(_) => Err(QueryError::SysAuthError),
         }
@@ -181,12 +190,9 @@ impl<Fs: RawFSInterface> SystemStore<Fs> {
             return Err(QueryError::SysAuthError);
         }
         match auth.users.remove_entry(username) {
-            Some((username, user)) => {
-                self.sync_db_or_rollback(|| {
-                    let _ = auth.users.insert(username, user);
-                })?;
-                Ok(())
-            }
+            Some((username, user)) => self._try_sync_or(&mut auth, |auth| {
+                let _ = auth.users.insert(username, user);
+            }),
             None => Err(QueryError::SysAuthError),
         }
     }
diff --git a/server/src/engine/ql/dcl.rs b/server/src/engine/ql/dcl.rs
index f624bab6..1f97cfa0 100644
--- a/server/src/engine/ql/dcl.rs
+++ b/server/src/engine/ql/dcl.rs
@@ -38,11 +38,11 @@ use crate::engine::{
 
 fn parse<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<UserMeta<'a>> {
     /*
-        user add [username] with { password: [password], ... }
-                 ^cursor
+        [username] with { password: [password], ... }
+        ^cursor
         7 tokens
     */
-    if state.remaining() < 7 {
+        if state.remaining() < 7 {
         return Err(QueryError::QLInvalidSyntax);
     }
     let token_buffer = state.current();
@@ -61,7 +61,7 @@ fn parse<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<UserMe
         token_buffer[0].uck_read_lit()
     };
     state.poison_if_not(maybe_username.kind().tag_class() == TagClass::Str);
-    if state.not_exhausted() & !state.okay() {
+    if state.not_exhausted() | !state.okay() {
         // we shouldn't have more tokens
         return Err(QueryError::QLInvalidSyntax);
     }
diff --git a/server/src/engine/storage/v1/sysdb.rs b/server/src/engine/storage/v1/sysdb.rs
index 12a8c63e..a0a4eace 100644
--- a/server/src/engine/storage/v1/sysdb.rs
+++ b/server/src/engine/storage/v1/sysdb.rs
@@ -82,17 +82,8 @@ impl<Fs: RawFSInterface> SystemStore<Fs> {
     ) -> RuntimeResult<(Self, SystemStoreInitState)> {
         Self::open_with_name(Self::SYSDB_PATH, Self::SYSDB_COW_PATH, auth, run_mode)
     }
-    pub fn sync_db_or_rollback(&self, rb: impl FnOnce()) -> RuntimeResult<()> {
-        match self.sync_db() {
-            Ok(()) => Ok(()),
-            Err(e) => {
-                rb();
-                Err(e)
-            }
-        }
-    }
-    pub fn sync_db(&self) -> RuntimeResult<()> {
-        self._sync_with(Self::SYSDB_PATH, Self::SYSDB_COW_PATH)
+    pub fn sync_db(&self, auth: &SysAuth) -> RuntimeResult<()> {
+        self._sync_with(Self::SYSDB_PATH, Self::SYSDB_COW_PATH, auth)
     }
     pub fn open_with_name(
         sysdb_name: &str,
@@ -103,7 +94,7 @@ impl<Fs: RawFSInterface> SystemStore<Fs> {
         match SDSSFileIO::open_or_create_perm_rw::<spec::SysDBV1>(sysdb_name)? {
             FileOpen::Created(new) => {
                 let me = Self::_new(SysConfig::new_auth(auth, run_mode));
-                me._sync(new)?;
+                me._sync(new, &me.system_store().auth_data().read())?;
                 Ok((me, SystemStoreInitState::Created))
             }
             FileOpen::Existing((ex, _)) => {
@@ -114,7 +105,7 @@ impl<Fs: RawFSInterface> SystemStore<Fs> {
 }
 
 impl<Fs: RawFSInterface> SystemStore<Fs> {
-    fn _sync(&self, mut f: SDSSFileIO<Fs>) -> RuntimeResult<()> {
+    fn _sync(&self, mut f: SDSSFileIO<Fs>, auth: &SysAuth) -> RuntimeResult<()> {
         let cfg = self.system_store();
         // prepare our flat file
         let mut map: DictGeneric = into_dict!(
@@ -125,7 +116,6 @@ impl<Fs: RawFSInterface> SystemStore<Fs> {
             Self::SYS_KEY_AUTH => DictGeneric::new(),
         );
         let auth_key = map.get_mut(Self::SYS_KEY_AUTH).unwrap();
-        let auth = cfg.auth_data().read();
         let auth_key = auth_key.as_dict_mut().unwrap();
         auth_key.insert(
             Self::SYS_KEY_AUTH_USERS.into(),
@@ -148,9 +138,9 @@ impl<Fs: RawFSInterface> SystemStore<Fs> {
         let buf = super::inf::enc::enc_dict_full::<super::inf::map::GenericDictSpec>(&map);
         f.fsynced_write(&buf)
     }
-    fn _sync_with(&self, target: &str, cow: &str) -> RuntimeResult<()> {
+    fn _sync_with(&self, target: &str, cow: &str, auth: &SysAuth) -> RuntimeResult<()> {
         let f = SDSSFileIO::create::<spec::SysDBV1>(cow)?;
-        self._sync(f)?;
+        self._sync(f, auth)?;
         Fs::fs_rename_file(cow, target)
     }
     fn restore_and_sync(
@@ -185,7 +175,7 @@ impl<Fs: RawFSInterface> SystemStore<Fs> {
         );
         let slf = Self::_new(new_syscfg);
         // now sync
-        slf._sync_with(fname, fcow_name)?;
+        slf._sync_with(fname, fcow_name, &slf.system_store().auth_data().read())?;
         Ok((slf, state))
     }
     fn _restore(mut f: SDSSFileIO<Fs>, run_mode: ConfigMode) -> RuntimeResult<SysConfig> {
diff --git a/server/src/main.rs b/server/src/main.rs
index f2b4be21..cd1de692 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -61,6 +61,7 @@ const TEXT: &str = "
 ";
 
 type IoResult<T> = std::io::Result<T>;
+const SKY_PID_FILE: &str = ".sky_pid";
 
 fn main() {
     Builder::new()
@@ -68,12 +69,14 @@ fn main() {
         .init();
     println!("{TEXT}\nSkytable v{VERSION} | {URL}\n");
     let run = || {
+        engine::set_context_init("locking PID file");
+        let pid_file = util::os::FileLock::new(SKY_PID_FILE)?;
         let runtime = tokio::runtime::Builder::new_multi_thread()
             .thread_name("server")
             .enable_all()
             .build()
             .unwrap();
-        runtime.block_on(async move {
+        let g = runtime.block_on(async move {
             engine::set_context_init("binding system signals");
             let signal = util::os::TerminationSignal::init()?;
             let (config, global) = tokio::task::spawn_blocking(|| engine::load_all())
@@ -82,12 +85,14 @@ fn main() {
             let g = global.global.clone();
             engine::start(signal, config, global).await?;
             engine::RuntimeResult::Ok(g)
-        })
+        })?;
+        engine::RuntimeResult::Ok((pid_file, g))
     };
     match run() {
-        Ok(g) => {
+        Ok((_, g)) => {
             info!("completing cleanup before exit");
             engine::finish(g);
+            std::fs::remove_file(SKY_PID_FILE).expect("failed to remove PID file");
             println!("Goodbye!");
         }
         Err(e) => {
diff --git a/server/src/util/compiler.rs b/server/src/util/compiler.rs
index a2b85dd0..1479d446 100644
--- a/server/src/util/compiler.rs
+++ b/server/src/util/compiler.rs
@@ -27,8 +27,6 @@
 //! Dark compiler arts and hackery to defy the normal. Use at your own
 //! risk
 
-use core::mem;
-
 #[cold]
 #[inline(never)]
 pub const fn cold() {}
@@ -67,18 +65,6 @@ pub const fn hot<T>(v: T) -> T {
     v
 }
 
-/// # Safety
-/// The caller is responsible for ensuring lifetime validity
-pub const unsafe fn extend_lifetime<'a, 'b, T>(inp: &'a T) -> &'b T {
-    mem::transmute(inp)
-}
-
-/// # Safety
-/// The caller is responsible for ensuring lifetime validity
-pub unsafe fn extend_lifetime_mut<'a, 'b, T>(inp: &'a mut T) -> &'b mut T {
-    mem::transmute(inp)
-}
-
 #[cold]
 #[inline(never)]
 pub fn cold_rerr<T, E>(e: E) -> Result<T, E> {
diff --git a/server/src/util/os/flock.rs b/server/src/util/os/flock.rs
index da0b2083..84f32585 100644
--- a/server/src/util/os/flock.rs
+++ b/server/src/util/os/flock.rs
@@ -48,39 +48,39 @@ impl FileLock {
         {
             use winapi::um::{
                 fileapi::LockFileEx,
-                minwinbase::{LOCKFILE_EXCLUSIVE_LOCK, OVERLAPPED},
+                minwinbase::{LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY, OVERLAPPED},
                 winnt::HANDLE,
             };
-
             let handle = file.as_raw_handle();
             let mut overlapped = OVERLAPPED::default();
             let result = unsafe {
                 LockFileEx(
                     handle as HANDLE,
-                    LOCKFILE_EXCLUSIVE_LOCK,
+                    LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY,
                     0,
                     u32::MAX,
                     u32::MAX,
                     &mut overlapped,
                 )
             };
-
             if result == 0 {
-                return Err(io::Error::last_os_error());
+                return Err(io::Error::new(
+                    io::ErrorKind::AlreadyExists,
+                    "file is already locked",
+                ));
             }
-
             return Ok(Self { file, handle });
         }
         #[cfg(unix)]
         {
-            use libc::{flock, LOCK_EX};
-
-            let result = unsafe { flock(file.as_raw_fd(), LOCK_EX) };
-
+            use libc::{flock, LOCK_EX, LOCK_NB};
+            let result = unsafe { flock(file.as_raw_fd(), LOCK_EX | LOCK_NB) };
             if result != 0 {
-                return Err(io::Error::last_os_error());
+                return Err(io::Error::new(
+                    io::ErrorKind::AlreadyExists,
+                    "file is already locked",
+                ));
             }
-
             return Ok(Self { file });
         }
     }
diff --git a/server/src/util/test_utils.rs b/server/src/util/test_utils.rs
index 962431fd..e3cf35ab 100644
--- a/server/src/util/test_utils.rs
+++ b/server/src/util/test_utils.rs
@@ -24,19 +24,17 @@
  *
 */
 
-use std::io::Read;
-
-use rand::seq::SliceRandom;
-
 use {
     rand::{
         distributions::{uniform::SampleUniform, Alphanumeric},
         rngs::ThreadRng,
+        seq::SliceRandom,
         Rng,
     },
     std::{
         collections::hash_map::RandomState,
         hash::{BuildHasher, Hash, Hasher},
+        io::Read,
     },
 };
 

From daf0f32c30cd4ec87341adb601cc66129b6a1ce2 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sat, 18 Nov 2023 13:25:40 +0530
Subject: [PATCH 283/310] Implement new benchmark tool

---
 Cargo.lock               |  63 +-----
 Cargo.toml               |  10 +-
 libstress/Cargo.toml     |  14 --
 libstress/src/lib.rs     | 469 ---------------------------------------
 libstress/src/traits.rs  |  69 ------
 sky-bench/Cargo.toml     |   7 +-
 sky-bench/help_text/help |  25 +++
 sky-bench/src/args.rs    | 217 ++++++++++++++++++
 sky-bench/src/bench.rs   | 216 ++++++++++++++++++
 sky-bench/src/error.rs   |  85 +++++++
 sky-bench/src/main.rs    |  29 ++-
 sky-bench/src/pool.rs    | 279 +++++++++++++++++++++++
 12 files changed, 866 insertions(+), 617 deletions(-)
 delete mode 100644 libstress/Cargo.toml
 delete mode 100644 libstress/src/lib.rs
 delete mode 100644 libstress/src/traits.rs
 create mode 100644 sky-bench/help_text/help
 create mode 100644 sky-bench/src/args.rs
 create mode 100644 sky-bench/src/bench.rs
 create mode 100644 sky-bench/src/error.rs
 create mode 100644 sky-bench/src/pool.rs

diff --git a/Cargo.lock b/Cargo.lock
index eb537c47..4dc37767 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -260,17 +260,6 @@ dependencies = [
  "crossbeam-utils",
 ]
 
-[[package]]
-name = "crossbeam-deque"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
-dependencies = [
- "cfg-if",
- "crossbeam-epoch",
- "crossbeam-utils",
-]
-
 [[package]]
 name = "crossbeam-epoch"
 version = "0.9.15"
@@ -345,12 +334,6 @@ dependencies = [
  "subtle",
 ]
 
-[[package]]
-name = "either"
-version = "1.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
-
 [[package]]
 name = "endian-type"
 version = "0.1.2"
@@ -359,9 +342,9 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
 
 [[package]]
 name = "env_logger"
-version = "0.10.0"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"
+checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece"
 dependencies = [
  "humantime",
  "is-terminal",
@@ -640,16 +623,6 @@ checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
 name = "libsky"
 version = "0.8.0"
 
-[[package]]
-name = "libstress"
-version = "0.8.0"
-dependencies = [
- "crossbeam-channel",
- "log",
- "rand",
- "rayon",
-]
-
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.5"
@@ -668,9 +641,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.19"
+version = "0.4.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
+checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
 
 [[package]]
 name = "memchr"
@@ -970,28 +943,6 @@ dependencies = [
  "getrandom",
 ]
 
-[[package]]
-name = "rayon"
-version = "1.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
-dependencies = [
- "either",
- "rayon-core",
-]
-
-[[package]]
-name = "rayon-core"
-version = "1.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
-dependencies = [
- "crossbeam-channel",
- "crossbeam-deque",
- "crossbeam-utils",
- "num_cpus",
-]
-
 [[package]]
 name = "rcrypt"
 version = "0.4.0"
@@ -1225,7 +1176,11 @@ dependencies = [
 name = "sky-bench"
 version = "0.8.0"
 dependencies = [
- "libstress",
+ "crossbeam-channel",
+ "env_logger",
+ "libsky",
+ "log",
+ "num_cpus",
  "skytable",
 ]
 
diff --git a/Cargo.toml b/Cargo.toml
index a07132a4..66bc3478 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,14 +1,6 @@
 [workspace]
 resolver = "1"
-members = [
-    "cli",
-    "server",
-    "libsky",
-    "sky-bench",
-    "sky-macros",
-    "libstress",
-    "harness",
-]
+members = ["cli", "server", "libsky", "sky-bench", "sky-macros", "harness"]
 
 [profile.release]
 opt-level = 3
diff --git a/libstress/Cargo.toml b/libstress/Cargo.toml
deleted file mode 100644
index dc66438b..00000000
--- a/libstress/Cargo.toml
+++ /dev/null
@@ -1,14 +0,0 @@
-[package]
-name = "libstress"
-version = "0.8.0"
-authors = ["Sayan Nandan <nandansayan@outlook.com>"]
-edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-# external deps
-crossbeam-channel = "0.5.8"
-rayon = "1.7.0"
-log = "0.4.19"
-rand = "0.8.5"
diff --git a/libstress/src/lib.rs b/libstress/src/lib.rs
deleted file mode 100644
index 4a922773..00000000
--- a/libstress/src/lib.rs
+++ /dev/null
@@ -1,469 +0,0 @@
-/*
- * Created on Wed Jun 16 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-//! # libstress
-//!
-//! Tools for emulating concurrent query behavior to _stress test_ the database server.
-//! As of now, this crate provides a [`Workpool`] which is a generic synchronous threadpool
-//! for doing multiple operations. But Workpool is a little different from standard threadpool
-//! implementations in it categorizing a job to be made up of three parts, namely:
-//!
-//! - The init_pre_loop_var (the pre-loop stage)
-//! - The on_loop (the in-loop stage)
-//! - The on_exit (the post-loop stage)
-//!
-//! These stages form a part of the event loop.
-//!
-//! ## The event loop
-//!
-//! A task runs in a loop with the `on_loop` routine to which the a reference of the result of
-//! the `init_pre_loop_var` is sent that is initialized. The loop proceeds whenever a worker
-//! receives a task or else it blocks the current thread, waiting for a task. Hence the loop
-//! cannot be terminated by an execute call. Instead, the _event loop_ is terminated when the
-//! Workpool is dropped, either by scoping out, or by using the provided finish-like methods
-//! (that call the destructor).
-//!
-//! ## Worker lifetime
-//!
-//! If a runtime panic occurs in the pre-loop stage, then the entire worker just terminates. Hence
-//! this worker is no longer able to perform any tasks. Similarly, if a runtime panic occurs in
-//! the in-loop stage, the worker terminates and is no longer available to do any work. This will
-//! be reflected when the workpool attempts to terminate in entirety, i.e when the threads are joined
-//! to the parent thread
-//!
-
-#![deny(unused_crate_dependencies)]
-#![deny(unused_imports)]
-
-pub mod traits;
-pub use rayon;
-
-use {
-    core::marker::PhantomData,
-    crossbeam_channel::{bounded, unbounded, Receiver as CReceiver, Sender as CSender},
-    rayon::prelude::{IntoParallelIterator, ParallelIterator},
-    std::{fmt::Display, thread},
-};
-
-#[derive(Debug)]
-pub enum WorkpoolError {
-    ThreadStartFailure(usize, usize),
-}
-
-impl Display for WorkpoolError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            WorkpoolError::ThreadStartFailure(expected, started) => {
-                write!(
-                    f,
-                    "couldn't start all threads. expected {expected} but started {started}"
-                )
-            }
-        }
-    }
-}
-
-pub type WorkpoolResult<T> = Result<T, WorkpoolError>;
-
-/// A Job. The UIn type parameter is the type that will be used to execute the action
-/// Nothing is a variant used by the drop implementation to terminate all the workers
-/// and call the exit_loop function
-enum JobType<UIn> {
-    Task(UIn),
-    Nothing,
-}
-
-/// A worker
-///
-/// The only reason we use option is to reduce the effort needed to implement [`Drop`] for the
-/// [`Workpool`]
-struct Worker {
-    thread: Option<thread::JoinHandle<()>>,
-}
-
-impl Worker {
-    /// Initialize a new worker
-    fn new<Inp: 'static, UIn, Lv, Lp, Ex>(
-        id: usize,
-        job_receiver: CReceiver<JobType<UIn>>,
-        init_pre_loop_var: Lv,
-        on_exit: Ex,
-        on_loop: Lp,
-        wgtx: CSender<()>,
-    ) -> Self
-    where
-        UIn: Send + Sync + 'static,
-        Lv: Fn() -> Inp + 'static + Send,
-        Lp: Fn(&mut Inp, UIn) + Send + Sync + 'static,
-        Ex: Fn(&mut Inp) + Send + 'static,
-    {
-        let thread = thread::Builder::new()
-            .name(format!("worker-{id}"))
-            .spawn(move || {
-                let on_loop = on_loop;
-                let mut pre_loop_var = init_pre_loop_var();
-                wgtx.send(()).unwrap();
-                drop(wgtx);
-                loop {
-                    let action = job_receiver.recv().unwrap();
-                    match action {
-                        JobType::Task(tsk) => on_loop(&mut pre_loop_var, tsk),
-                        JobType::Nothing => {
-                            on_exit(&mut pre_loop_var);
-                            break;
-                        }
-                    }
-                }
-            })
-            .unwrap();
-        Self {
-            thread: Some(thread),
-        }
-    }
-}
-
-/// A pool configuration setting to easily generate [`Workpool`]s without
-/// having to clone an entire pool and its threads upfront
-pub struct PoolConfig<Inp, UIn, Lv, Lp, Ex> {
-    /// the pool size
-    count: usize,
-    /// the function that sets the pre-loop variable
-    init_pre_loop_var: Lv,
-    /// the function to be executed on worker termination
-    on_exit: Ex,
-    /// the function to be executed on loop
-    on_loop: Lp,
-    /// a marker for `Inp` since no parameters use it directly
-    _marker: PhantomData<(Inp, UIn)>,
-    /// check if self needs a pool for parallel iterators
-    needs_iterator_pool: bool,
-    /// expected maximum number of jobs
-    expected_max_sends: Option<usize>,
-}
-
-impl<Inp: 'static, UIn, Lv, Lp, Ex> PoolConfig<Inp, UIn, Lv, Lp, Ex>
-where
-    UIn: Send + Sync + 'static,
-    Inp: Sync,
-    Ex: Fn(&mut Inp) + Send + Sync + 'static + Clone,
-    Lv: Fn() -> Inp + Send + Sync + 'static + Clone,
-    Lp: Fn(&mut Inp, UIn) + Clone + Send + Sync + 'static,
-{
-    /// Create a new pool config
-    pub fn new(
-        count: usize,
-        init_pre_loop_var: Lv,
-        on_loop: Lp,
-        on_exit: Ex,
-        needs_iterator_pool: bool,
-        expected_max_sends: Option<usize>,
-    ) -> Self {
-        Self {
-            count,
-            init_pre_loop_var,
-            on_loop,
-            on_exit,
-            needs_iterator_pool,
-            _marker: PhantomData,
-            expected_max_sends,
-        }
-    }
-    /// Get a new [`Workpool`] from the current config
-    pub fn get_pool(&self) -> WorkpoolResult<Workpool<Inp, UIn, Lv, Lp, Ex>> {
-        self.get_pool_with_workers(self.count)
-    }
-    /// Get a [`Workpool`] with the base config but with a different number of workers
-    pub fn get_pool_with_workers(
-        &self,
-        count: usize,
-    ) -> WorkpoolResult<Workpool<Inp, UIn, Lv, Lp, Ex>> {
-        Workpool::new(
-            count,
-            self.init_pre_loop_var.clone(),
-            self.on_loop.clone(),
-            self.on_exit.clone(),
-            self.needs_iterator_pool,
-            self.expected_max_sends,
-        )
-    }
-    /// Get a [`Workpool`] with the base config but with a custom loop-stage closure
-    pub fn with_loop_closure<Dlp>(&self, lp: Dlp) -> WorkpoolResult<Workpool<Inp, UIn, Lv, Dlp, Ex>>
-    where
-        Dlp: Fn(&mut Inp, UIn) + Clone + Send + Sync + 'static,
-    {
-        Workpool::new(
-            self.count,
-            self.init_pre_loop_var.clone(),
-            lp,
-            self.on_exit.clone(),
-            self.needs_iterator_pool,
-            self.expected_max_sends,
-        )
-    }
-}
-
-/// # Workpool
-///
-/// A Workpool is a generic synchronous thread pool that can be used to perform, well, anything.
-/// A workpool has to be initialized with the number of workers, the pre_loop_variable (set this
-/// to None if there isn't any). what to do on loop and what to do on exit of each worker. The
-/// closures are kept as `Clone`able types just to reduce complexity with copy (we were lazy).
-///
-/// ## Clones
-///
-/// Workpool clones simply create a new workpool with the same on_exit, on_loop and init_pre_loop_var
-/// configurations. This provides a very convenient interface if one desires to use multiple workpools
-/// to do the _same kind of thing_
-///
-/// ## Actual thread count
-///
-/// The actual thread count will depend on whether the caller requests the initialization of an
-/// iterator pool or not. If the caller does request for an iterator pool, then the number of threads
-/// spawned will be twice the number of the set workers. Else, the number of spawned threads is equal
-/// to the number of workers.
-pub struct Workpool<Inp, UIn, Lv, Lp, Ex> {
-    /// the workers
-    workers: Vec<Worker>,
-    /// the sender that sends jobs
-    job_distributor: CSender<JobType<UIn>>,
-    /// the function that sets the pre-loop variable
-    init_pre_loop_var: Lv,
-    /// the function to be executed on worker termination
-    on_exit: Ex,
-    /// the function to be executed on loop
-    on_loop: Lp,
-    /// a marker for `Inp` since no parameters use it directly
-    _marker: PhantomData<Inp>,
-    /// check if self needs a pool for parallel iterators
-    needs_iterator_pool: bool,
-    /// expected maximum number of sends
-    expected_max_sends: Option<usize>,
-}
-
-impl<Inp: 'static, UIn, Lv, Ex, Lp> Workpool<Inp, UIn, Lv, Lp, Ex>
-where
-    UIn: Send + Sync + 'static,
-    Ex: Fn(&mut Inp) + Send + Sync + 'static + Clone,
-    Lv: Fn() -> Inp + Send + Sync + 'static + Clone,
-    Lp: Fn(&mut Inp, UIn) + Send + Sync + 'static + Clone,
-    Inp: Sync,
-{
-    /// Create a new workpool
-    pub fn new(
-        count: usize,
-        init_pre_loop_var: Lv,
-        on_loop: Lp,
-        on_exit: Ex,
-        needs_iterator_pool: bool,
-        expected_max_sends: Option<usize>,
-    ) -> WorkpoolResult<Self> {
-        // init threadpool for iterator
-        if needs_iterator_pool {
-            // initialize a global threadpool for parallel iterators
-            let _ = rayon::ThreadPoolBuilder::new()
-                .num_threads(count)
-                .build_global();
-        }
-        assert!(count != 0, "Runtime panic: Bad value `0` for thread count");
-        let (sender, receiver) = match expected_max_sends {
-            Some(limit) => bounded(limit),
-            None => unbounded(),
-        };
-        let (wgtx, wgrx) = bounded::<()>(count);
-        let mut workers = Vec::with_capacity(count);
-        for i in 0..count {
-            workers.push(Worker::new(
-                i,
-                receiver.clone(),
-                init_pre_loop_var.clone(),
-                on_exit.clone(),
-                on_loop.clone(),
-                wgtx.clone(),
-            ));
-        }
-        drop(wgtx);
-        let sum: usize = wgrx.iter().map(|_| 1usize).sum();
-        if sum == count {
-            Ok(Self {
-                workers,
-                job_distributor: sender,
-                init_pre_loop_var,
-                on_exit,
-                on_loop,
-                _marker: PhantomData,
-                needs_iterator_pool,
-                expected_max_sends,
-            })
-        } else {
-            Err(WorkpoolError::ThreadStartFailure(count, sum))
-        }
-    }
-    pub fn clone_pool(&self) -> WorkpoolResult<Self> {
-        Self::new(
-            self.workers.len(),
-            self.init_pre_loop_var.clone(),
-            self.on_loop.clone(),
-            self.on_exit.clone(),
-            self.needs_iterator_pool,
-            self.expected_max_sends,
-        )
-    }
-    /// Execute something
-    pub fn execute(&self, inp: UIn) {
-        self.job_distributor
-            .send(JobType::Task(inp))
-            .expect("Worker thread crashed")
-    }
-    /// Execute something that can be executed as a parallel iterator
-    /// For the best performance, it is recommended that you pass true for `needs_iterator_pool`
-    /// on initialization of the [`Workpool`]
-    pub fn execute_iter(&self, iter: impl IntoParallelIterator<Item = UIn>) {
-        iter.into_par_iter().for_each(|inp| self.execute(inp))
-    }
-    /// Does the same thing as [`execute_iter`] but drops self ensuring that all the
-    /// workers actually finish their tasks
-    pub fn execute_and_finish_iter(self, iter: impl IntoParallelIterator<Item = UIn>) {
-        self.execute_iter(iter);
-        drop(self);
-    }
-    /// Initialize a new [`Workpool`] with the default count of threads. This is equal
-    /// to 2 * the number of logical cores.
-    pub fn new_default_threads(
-        init_pre_loop_var: Lv,
-        on_loop: Lp,
-        on_exit: Ex,
-        needs_iterator_pool: bool,
-        expected_max_sends: Option<usize>,
-    ) -> WorkpoolResult<Self> {
-        // we'll naively use the number of CPUs present on the system times 2 to determine
-        // the number of workers (sure the scheduler does tricks all the time)
-        let worker_count = thread::available_parallelism().map_or(1, usize::from) * 2;
-        Self::new(
-            worker_count,
-            init_pre_loop_var,
-            on_loop,
-            on_exit,
-            needs_iterator_pool,
-            expected_max_sends,
-        )
-    }
-}
-
-impl<Inp, UIn, Lv, Lp, Ex> Drop for Workpool<Inp, UIn, Lp, Lv, Ex> {
-    fn drop(&mut self) {
-        for _ in &self.workers {
-            self.job_distributor.send(JobType::Nothing).unwrap();
-        }
-        for worker in &mut self.workers {
-            if let Some(thread) = worker.thread.take() {
-                thread.join().unwrap()
-            }
-        }
-    }
-}
-
-pub mod utils {
-    const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
-    use rand::distributions::{Alphanumeric, Standard};
-    use std::collections::HashSet;
-    use std::collections::TryReserveError;
-
-    /// Generate a random UTF-8 string
-    pub fn ran_string(len: usize, rand: impl rand::Rng) -> String {
-        let rand_string: String = rand
-            .sample_iter(&Alphanumeric)
-            .take(len)
-            .map(char::from)
-            .collect();
-        rand_string
-    }
-    /// Generate a vector of random bytes
-    pub fn ran_bytes(len: usize, rand: impl rand::Rng) -> Vec<u8> {
-        rand.sample_iter(&Standard).take(len).collect()
-    }
-    /// Generate multiple vectors of random bytes
-    pub fn generate_random_byte_vector(
-        count: usize,
-        size: usize,
-        mut rng: impl rand::Rng,
-        unique: bool,
-    ) -> Result<Vec<Vec<u8>>, TryReserveError> {
-        if unique {
-            let mut keys = HashSet::new();
-            keys.try_reserve(size)?;
-            (0..count).into_iter().for_each(|_| {
-                let mut ran = ran_bytes(size, &mut rng);
-                while keys.contains(&ran) {
-                    ran = ran_bytes(size, &mut rng);
-                }
-                keys.insert(ran);
-            });
-            Ok(keys.into_iter().collect())
-        } else {
-            let mut keys = Vec::new();
-            keys.try_reserve_exact(size)?;
-            let ran_byte_key = ran_bytes(size, &mut rng);
-            (0..count).for_each(|_| keys.push(ran_byte_key.clone()));
-            Ok(keys)
-        }
-    }
-    /// Generate a vector of random UTF-8 valid strings
-    pub fn generate_random_string_vector(
-        count: usize,
-        size: usize,
-        mut rng: impl rand::Rng,
-        unique: bool,
-    ) -> Result<Vec<String>, TryReserveError> {
-        if unique {
-            let mut keys = HashSet::new();
-            keys.try_reserve(size)?;
-            (0..count).into_iter().for_each(|_| {
-                let mut ran = ran_string(size, &mut rng);
-                while keys.contains(&ran) {
-                    ran = ran_string(size, &mut rng);
-                }
-                keys.insert(ran);
-            });
-            Ok(keys.into_iter().collect())
-        } else {
-            let mut keys = Vec::new();
-            keys.try_reserve_exact(size)?;
-            (0..count)
-                .into_iter()
-                .map(|_| ran_string(size, &mut rng))
-                .for_each(|bytes| keys.push(bytes));
-            Ok(keys)
-        }
-    }
-    pub fn rand_alphastring(len: usize, rng: &mut impl rand::Rng) -> String {
-        (0..len)
-            .map(|_| {
-                let idx = rng.gen_range(0..CHARSET.len());
-                CHARSET[idx] as char
-            })
-            .collect()
-    }
-}
diff --git a/libstress/src/traits.rs b/libstress/src/traits.rs
deleted file mode 100644
index 12d7c343..00000000
--- a/libstress/src/traits.rs
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Created on Fri Jun 18 2021
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2021, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use std::fmt;
-
-/// A trait for aggresive erroring
-pub trait ExitError<T> {
-    /// Abort the process if the type errors with an error code or
-    /// return the type
-    fn exit_error<Ms>(self, msg: Ms) -> T
-    where
-        Ms: ToString;
-}
-
-impl<T, E> ExitError<T> for Result<T, E>
-where
-    E: fmt::Display,
-{
-    fn exit_error<Ms>(self, msg: Ms) -> T
-    where
-        Ms: ToString,
-    {
-        match self {
-            Self::Err(e) => {
-                log::error!("{} : '{}'", msg.to_string(), e);
-                std::process::exit(0x01);
-            }
-            Self::Ok(v) => v,
-        }
-    }
-}
-
-impl<T> ExitError<T> for Option<T> {
-    fn exit_error<Ms>(self, msg: Ms) -> T
-    where
-        Ms: ToString,
-    {
-        match self {
-            Self::None => {
-                log::error!("{}", msg.to_string());
-                std::process::exit(0x01);
-            }
-            Self::Some(v) => v,
-        }
-    }
-}
diff --git a/sky-bench/Cargo.toml b/sky-bench/Cargo.toml
index 19595da4..b04e8c17 100644
--- a/sky-bench/Cargo.toml
+++ b/sky-bench/Cargo.toml
@@ -9,4 +9,9 @@ version = "0.8.0"
 [dependencies]
 # internal deps
 skytable = { git = "https://github.com/skytable/client-rust.git", branch = "octave" }
-libstress = { path = "../libstress" }
+libsky = { path = "../libsky" }
+# external deps
+crossbeam-channel = "0.5.8"
+num_cpus = "1.16.0"
+env_logger = "0.10.1"
+log = "0.4.20"
diff --git a/sky-bench/help_text/help b/sky-bench/help_text/help
new file mode 100644
index 00000000..508f43f3
--- /dev/null
+++ b/sky-bench/help_text/help
@@ -0,0 +1,25 @@
+sky-bench 0.8.0
+Sayan N. <ohsayan@outlook.com>
+Skytable benchmark tool
+
+USAGE:
+    sky-bench [OPTIONS]
+
+FLAGS:
+    --help      Displays this help message
+    --version   Displays the benchmark tool version
+
+REQUIRED OPTIONS:
+    --password  Provide the password
+
+OPTIONS:
+    --endpoint  Set the endpoint (defaults to tcp@127.0.0.1:2003)
+    --threads   Set the number of threads to be used (defaults to number of physical CPU cores)
+    --keysize   Set the default primary key size. defaults to 7
+    --rowcount  Set the number of rows to be manipulated for the benchmark
+
+NOTES:
+    - The user for auth will be 'root' since only 'root' accounts allow the creation and deletion of spaces and models
+    - A space called 'benchmark_[random 8B string]' will be created
+    - A model called 'benchmark_[random 8B string]' will be created in the space created above. The created model has the structure {name: string, pass: string}
+    - The model and space will be removed once the benchmark is complete
\ No newline at end of file
diff --git a/sky-bench/src/args.rs b/sky-bench/src/args.rs
new file mode 100644
index 00000000..1a8a63aa
--- /dev/null
+++ b/sky-bench/src/args.rs
@@ -0,0 +1,217 @@
+/*
+ * Created on Sat Nov 18 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::error::{BenchError, BenchResult},
+    std::{
+        collections::hash_map::{Entry, HashMap},
+        env,
+    },
+};
+
+const TXT_HELP: &str = include_str!("../help_text/help");
+
+#[derive(Debug)]
+enum TaskInner {
+    HelpMsg(String),
+    CheckConfig(HashMap<String, String>),
+}
+
+#[derive(Debug)]
+pub enum Task {
+    HelpMsg(String),
+    BenchConfig(BenchConfig),
+}
+
+#[derive(Debug)]
+pub struct BenchConfig {
+    pub host: String,
+    pub port: u16,
+    pub root_pass: String,
+    pub threads: usize,
+    pub key_size: usize,
+    pub query_count: usize,
+}
+
+impl BenchConfig {
+    pub fn new(
+        host: String,
+        port: u16,
+        root_pass: String,
+        threads: usize,
+        key_size: usize,
+        query_count: usize,
+    ) -> Self {
+        Self {
+            host,
+            port,
+            root_pass,
+            threads,
+            key_size,
+            query_count,
+        }
+    }
+}
+
+fn load_env() -> BenchResult<TaskInner> {
+    let mut args = HashMap::new();
+    let mut it = env::args().skip(1).into_iter();
+    while let Some(arg) = it.next() {
+        let (arg, arg_val) = match arg.as_str() {
+            "--help" => return Ok(TaskInner::HelpMsg(TXT_HELP.into())),
+            "--version" => {
+                return Ok(TaskInner::HelpMsg(format!(
+                    "sky-bench v{}",
+                    libsky::VERSION
+                )))
+            }
+            _ if arg.starts_with("--") => match it.next() {
+                Some(arg_val) => (arg, arg_val),
+                None => {
+                    // self contained?
+                    let split: Vec<&str> = arg.split("=").collect();
+                    if split.len() != 2 {
+                        return Err(BenchError::ArgsErr(format!("expected value for {arg}")));
+                    }
+                    (split[0].into(), split[1].into())
+                }
+            },
+            unknown_arg => {
+                return Err(BenchError::ArgsErr(format!(
+                    "unknown argument: {unknown_arg}"
+                )))
+            }
+        };
+        match args.entry(arg) {
+            Entry::Occupied(oe) => {
+                return Err(BenchError::ArgsErr(format!(
+                    "found duplicate values for {}",
+                    oe.key()
+                )))
+            }
+            Entry::Vacant(ve) => {
+                ve.insert(arg_val);
+            }
+        }
+    }
+    Ok(TaskInner::CheckConfig(args))
+}
+
+fn cdig(n: usize) -> usize {
+    if n == 0 {
+        1
+    } else {
+        (n as f64).log10().floor() as usize + 1
+    }
+}
+
+pub fn parse() -> BenchResult<Task> {
+    let mut args = match load_env()? {
+        TaskInner::HelpMsg(msg) => return Ok(Task::HelpMsg(msg)),
+        TaskInner::CheckConfig(args) => args,
+    };
+    // endpoint
+    let (host, port) = match args.remove("--endpoint") {
+        None => ("127.0.0.1".to_owned(), 2003),
+        Some(ep) => {
+            // proto@host:port
+            let ep: Vec<&str> = ep.split("@").collect();
+            if ep.len() != 2 {
+                return Err(BenchError::ArgsErr(
+                    "value for --endpoint must be in the form `[protocol]@[host]:[port]`".into(),
+                ));
+            }
+            let protocol = ep[0];
+            let host_port: Vec<&str> = ep[1].split(":").collect();
+            if host_port.len() != 2 {
+                return Err(BenchError::ArgsErr(
+                    "value for --endpoint must be in the form `[protocol]@[host]:[port]`".into(),
+                ));
+            }
+            let (host, port) = (host_port[0], host_port[1]);
+            let Ok(port) = port.parse::<u16>() else {
+                return Err(BenchError::ArgsErr(
+                    "the value for port must be an integer in the range 0-65535".into(),
+                ));
+            };
+            if protocol != "tcp" {
+                return Err(BenchError::ArgsErr(
+                    "only TCP endpoints can be benchmarked at the moment".into(),
+                ));
+            }
+            (host.to_owned(), port)
+        }
+    };
+    // password
+    let passsword = match args.remove("--password") {
+        Some(p) => p,
+        None => {
+            return Err(BenchError::ArgsErr(
+                "you must provide a value for `--password`".into(),
+            ))
+        }
+    };
+    // threads
+    let thread_count = match args.remove("--threads") {
+        None => num_cpus::get(),
+        Some(tc) => match tc.parse() {
+            Ok(tc) if tc > 0 => tc,
+            Err(_) | Ok(_) => {
+                return Err(BenchError::ArgsErr(
+                    "incorrect value for `--threads`. must be a nonzero value".into(),
+                ))
+            }
+        },
+    };
+    // query count
+    let query_count = match args.remove("--rowcount") {
+        None => 1_000_000_usize,
+        Some(rc) => match rc.parse() {
+            Ok(rc) if rc != 0 => rc,
+            Err(_) | Ok(_) => {
+                return Err(BenchError::ArgsErr(format!(
+                    "bad value for `--rowcount` must be a nonzero value"
+                )))
+            }
+        },
+    };
+    let need_atleast = cdig(query_count);
+    let key_size = match args.remove("--keysize") {
+        None => need_atleast,
+        Some(ks) => match ks.parse() {
+            Ok(s) if s >= need_atleast => s,
+            Err(_) | Ok(_) => return Err(BenchError::ArgsErr(format!("incorrect value for `--keysize`. must be set to a value that can be used to generate atleast {query_count} unique primary keys"))),
+        }
+    };
+    Ok(Task::BenchConfig(BenchConfig::new(
+        host,
+        port,
+        passsword,
+        thread_count,
+        key_size,
+        query_count,
+    )))
+}
diff --git a/sky-bench/src/bench.rs b/sky-bench/src/bench.rs
new file mode 100644
index 00000000..651b30c6
--- /dev/null
+++ b/sky-bench/src/bench.rs
@@ -0,0 +1,216 @@
+/*
+ * Created on Sat Nov 18 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::error::BenchResult;
+
+use {
+    crate::{
+        args::BenchConfig,
+        error::{self, BenchmarkTaskWorkerError},
+        pool::{RuntimeStats, Taskpool, ThreadedTask},
+    },
+    skytable::{query, response::Response, Config, Connection, Query},
+    std::time::Instant,
+};
+
+#[derive(Debug)]
+pub struct BenchmarkTask {
+    cfg: Config,
+}
+
+impl BenchmarkTask {
+    pub fn new(host: &str, port: u16, username: &str, password: &str) -> Self {
+        Self {
+            cfg: Config::new(host, port, username, password),
+        }
+    }
+}
+
+impl ThreadedTask for BenchmarkTask {
+    type TaskWorker = Connection;
+    type TaskWorkerInitError = BenchmarkTaskWorkerError;
+    type TaskWorkerTerminateError = BenchmarkTaskWorkerError;
+    type TaskWorkerWorkError = BenchmarkTaskWorkerError;
+    type TaskInput = (Query, Response);
+    fn initialize_worker(&self) -> Result<Self::TaskWorker, Self::TaskWorkerInitError> {
+        self.cfg.connect().map_err(Into::into)
+    }
+    fn drive_worker_timed(
+        worker: &mut Self::TaskWorker,
+        (query, expected_resp): Self::TaskInput,
+    ) -> Result<(Instant, Instant), Self::TaskWorkerWorkError> {
+        let start = Instant::now();
+        let resp = worker.query(&query)?;
+        let stop = Instant::now();
+        if resp == expected_resp {
+            Ok((start, stop))
+        } else {
+            Err(BenchmarkTaskWorkerError::Error(format!(
+                "response from server did not match expected response: {:?}",
+                resp
+            )))
+        }
+    }
+    fn terminate_worker(
+        &self,
+        _: &mut Self::TaskWorker,
+    ) -> Result<(), Self::TaskWorkerTerminateError> {
+        Ok(())
+    }
+}
+
+pub fn run(bench: BenchConfig) -> error::BenchResult<()> {
+    let bench_config = BenchmarkTask::new(&bench.host, bench.port, "root", &bench.root_pass);
+    info!("running preliminary checks and creating model `bench.bench` with definition: `{{un: string, pw: uint8}}`");
+    let mut main_thread_db = bench_config.cfg.connect()?;
+    main_thread_db.query_parse::<()>(&query!("create space bench"))?;
+    main_thread_db.query_parse::<()>(&query!("create model bench.bench(un: string, pw: uint8)"))?;
+    info!(
+        "initializing connection pool with {} connections",
+        bench.threads
+    );
+    let mut p = Taskpool::new(bench.threads, bench_config)?;
+    info!(
+        "pool initialized successfully. preparing {} `INSERT` queries with primary key size={} bytes",
+        bench.query_count, bench.key_size
+    );
+    let mut insert_stats = Default::default();
+    let mut update_stats = Default::default();
+    let mut delete_stats = Default::default();
+    match || -> BenchResult<()> {
+        // bench insert
+        let insert_queries: Vec<(Query, Response)> = (0..bench.query_count)
+            .into_iter()
+            .map(|i| {
+                (
+                    query!(
+                        "insert into bench.bench(?, ?)",
+                        format!("{:0>width$}", i, width = bench.key_size),
+                        0u64
+                    ),
+                    Response::Empty,
+                )
+            })
+            .collect();
+        info!("benchmarking `INSERT` queries");
+        insert_stats = p.blocking_bombard(insert_queries)?;
+        // bench update
+        info!("completed benchmarking `INSERT`. preparing `UPDATE` queries");
+        let update_queries: Vec<(Query, Response)> = (0..bench.query_count)
+            .into_iter()
+            .map(|i| {
+                (
+                    query!(
+                        "update bench.bench set pw += ? where un = ?",
+                        1u64,
+                        format!("{:0>width$}", i, width = bench.key_size),
+                    ),
+                    Response::Empty,
+                )
+            })
+            .collect();
+        info!("benchmarking `UPDATE` queries");
+        update_stats = p.blocking_bombard(update_queries)?;
+        // bench delete
+        info!("completed benchmarking `UPDATE`. preparing `DELETE` queries");
+        let delete_queries: Vec<(Query, Response)> = (0..bench.query_count)
+            .into_iter()
+            .map(|i| {
+                (
+                    query!(
+                        "delete from bench.bench where un = ?",
+                        format!("{:0>width$}", i, width = bench.key_size),
+                    ),
+                    Response::Empty,
+                )
+            })
+            .collect();
+        info!("benchmarking `DELETE` queries");
+        delete_stats = p.blocking_bombard(delete_queries)?;
+        info!("completed benchmarking `DELETE` queries");
+        Ok(())
+    }() {
+        Ok(()) => {}
+        Err(e) => {
+            error!("benchmarking failed. attempting to clean up");
+            match cleanup(main_thread_db) {
+                Ok(()) => return Err(e),
+                Err(e_cleanup) => {
+                    error!("failed to clean up db: {e_cleanup}. please remove model `bench.bench` manually");
+                    return Err(e);
+                }
+            }
+        }
+    }
+    drop(p);
+    warn!("benchmarks might appear to be slower. this tool is currently experimental");
+    // print results
+    info!("results:");
+    print_table(vec![
+        ("INSERT", insert_stats),
+        ("UPDATE", update_stats),
+        ("DELETE", delete_stats),
+    ]);
+    cleanup(main_thread_db)?;
+    Ok(())
+}
+
+fn cleanup(mut main_thread_db: Connection) -> Result<(), error::BenchError> {
+    trace!("dropping space and table");
+    main_thread_db.query_parse::<()>(&query!("drop model bench.bench"))?;
+    main_thread_db.query_parse::<()>(&query!("drop space bench"))?;
+    Ok(())
+}
+
+fn print_table(data: Vec<(&'static str, RuntimeStats)>) {
+    println!(
+        "+---------+--------------------------+-----------------------+------------------------+"
+    );
+    println!(
+        "| Query   | Effective real-world QPS | Slowest Query (nanos) | Fastest Query (nanos)  |"
+    );
+    println!(
+        "+---------+--------------------------+-----------------------+------------------------+"
+    );
+    for (
+        query,
+        RuntimeStats {
+            qps,
+            avg_per_query_ns: _,
+            head_ns,
+            tail_ns,
+        },
+    ) in data
+    {
+        println!(
+            "| {:<7} | {:>24.2} | {:>21} | {:>22} |",
+            query, qps, tail_ns, head_ns
+        );
+    }
+    println!(
+        "+---------+--------------------------+-----------------------+------------------------+"
+    );
+}
diff --git a/sky-bench/src/error.rs b/sky-bench/src/error.rs
new file mode 100644
index 00000000..0fe3fcb6
--- /dev/null
+++ b/sky-bench/src/error.rs
@@ -0,0 +1,85 @@
+/*
+ * Created on Sat Nov 18 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crate::{bench::BenchmarkTask, pool::TaskpoolError},
+    core::fmt,
+    skytable::error::Error,
+};
+
+pub type BenchResult<T> = Result<T, BenchError>;
+
+#[derive(Debug)]
+pub enum BenchError {
+    ArgsErr(String),
+    BenchError(TaskpoolError<BenchmarkTask>),
+    DirectDbError(Error),
+}
+
+impl From<TaskpoolError<BenchmarkTask>> for BenchError {
+    fn from(e: TaskpoolError<BenchmarkTask>) -> Self {
+        Self::BenchError(e)
+    }
+}
+
+impl From<Error> for BenchError {
+    fn from(e: Error) -> Self {
+        Self::DirectDbError(e)
+    }
+}
+
+impl fmt::Display for BenchError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::ArgsErr(e) => write!(f, "args error: {e}"),
+            Self::BenchError(e) => write!(f, "benchmark system error: {e}"),
+            Self::DirectDbError(e) => write!(f, "direct operation on db failed. {e}"),
+        }
+    }
+}
+
+impl std::error::Error for BenchError {}
+
+#[derive(Debug)]
+pub enum BenchmarkTaskWorkerError {
+    DbError(Error),
+    Error(String),
+}
+
+impl From<Error> for BenchmarkTaskWorkerError {
+    fn from(e: Error) -> Self {
+        Self::DbError(e)
+    }
+}
+
+impl fmt::Display for BenchmarkTaskWorkerError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::DbError(e) => write!(f, "worker failed due to DB error. {e}"),
+            Self::Error(e) => write!(f, "worker failed. {e}"),
+        }
+    }
+}
diff --git a/sky-bench/src/main.rs b/sky-bench/src/main.rs
index 57a21b8e..27dfc5ae 100644
--- a/sky-bench/src/main.rs
+++ b/sky-bench/src/main.rs
@@ -24,4 +24,31 @@
  *
 */
 
-fn main() {}
+#[macro_use]
+extern crate log;
+mod args;
+mod bench;
+mod error;
+mod pool;
+
+fn main() {
+    env_logger::Builder::new()
+        .parse_filters(&std::env::var("SKYBENCH_LOG").unwrap_or_else(|_| "info".to_owned()))
+        .init();
+    match run() {
+        Ok(()) => {}
+        Err(e) => {
+            error!("bench error: {e}");
+            std::process::exit(0x01);
+        }
+    }
+}
+
+fn run() -> error::BenchResult<()> {
+    let task = args::parse()?;
+    match task {
+        args::Task::HelpMsg(msg) => println!("{msg}"),
+        args::Task::BenchConfig(bench) => bench::run(bench)?,
+    }
+    Ok(())
+}
diff --git a/sky-bench/src/pool.rs b/sky-bench/src/pool.rs
new file mode 100644
index 00000000..0b6e13e4
--- /dev/null
+++ b/sky-bench/src/pool.rs
@@ -0,0 +1,279 @@
+/*
+ * Created on Fri Nov 17 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crossbeam_channel::{unbounded, Receiver, Sender},
+    std::{
+        fmt,
+        marker::PhantomData,
+        thread::{self, JoinHandle},
+        time::Instant,
+    },
+};
+
+pub type TaskPoolResult<T, Th> = Result<T, TaskpoolError<Th>>;
+
+#[derive(Debug)]
+pub enum TaskpoolError<Th: ThreadedTask> {
+    InitError(Th::TaskWorkerInitError),
+    BombardError(&'static str),
+    WorkerError(Th::TaskWorkerWorkError),
+}
+
+impl<Th: ThreadedTask> fmt::Display for TaskpoolError<Th>
+where
+    Th::TaskWorkerInitError: fmt::Display,
+    Th::TaskWorkerWorkError: fmt::Display,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::InitError(e) => write!(f, "failed to init worker pool. {e}"),
+            Self::BombardError(e) => write!(f, "failed to post work to pool. {e}"),
+            Self::WorkerError(e) => write!(f, "failed running worker task. {e}"),
+        }
+    }
+}
+
+pub trait ThreadedTask: Send + Sync + 'static {
+    /// the per-thread item that does the actual work
+    ///
+    /// NB: this is not to be confused with the actual underlying thread pool worker
+    type TaskWorker: Send + Sync;
+    /// when attempting initialization of the per-thread task worker, if an error is thrown, this is the type
+    /// you're looking for
+    type TaskWorkerInitError: Send + Sync;
+    /// when attempting to run a single unit of work, if any error occurs this is the error type that is to be returned
+    type TaskWorkerWorkError: Send + Sync;
+    /// when attempting to close a worker, if an error occurs this is the error type that is returned
+    type TaskWorkerTerminateError: Send + Sync;
+    /// the task that is sent to each worker
+    type TaskInput: Send + Sync;
+    // fn
+    /// initialize the worker
+    fn initialize_worker(&self) -> Result<Self::TaskWorker, Self::TaskWorkerInitError>;
+    /// drive the worker to complete a task and return the time
+    fn drive_worker_timed(
+        worker: &mut Self::TaskWorker,
+        task: Self::TaskInput,
+    ) -> Result<(Instant, Instant), Self::TaskWorkerWorkError>;
+    fn terminate_worker(
+        &self,
+        worker: &mut Self::TaskWorker,
+    ) -> Result<(), Self::TaskWorkerTerminateError>;
+}
+
+#[derive(Debug)]
+struct ThreadWorker<Th> {
+    handle: JoinHandle<()>,
+    _m: PhantomData<Th>,
+}
+
+#[derive(Debug)]
+enum WorkerTask<Th: ThreadedTask> {
+    Task(Th::TaskInput),
+    Exit,
+}
+
+impl<Th: ThreadedTask> ThreadWorker<Th> {
+    fn new(
+        hl_worker: Th::TaskWorker,
+        task_rx: Receiver<WorkerTask<Th>>,
+        res_tx: Sender<Result<(Instant, Instant), Th::TaskWorkerWorkError>>,
+    ) -> Self {
+        Self {
+            handle: thread::spawn(move || {
+                let mut worker = hl_worker;
+                loop {
+                    let task = match task_rx.recv().unwrap() {
+                        WorkerTask::Exit => {
+                            drop(task_rx);
+                            return;
+                        }
+                        WorkerTask::Task(t) => t,
+                    };
+                    res_tx
+                        .send(Th::drive_worker_timed(&mut worker, task))
+                        .unwrap();
+                }
+            }),
+            _m: PhantomData,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Taskpool<Th: ThreadedTask> {
+    workers: Vec<ThreadWorker<Th>>,
+    _config: Th,
+    task_tx: Sender<WorkerTask<Th>>,
+    res_rx: Receiver<Result<(Instant, Instant), Th::TaskWorkerWorkError>>,
+    record_real_start: Instant,
+    record_real_stop: Instant,
+    stat_run_avg_ns: f64,
+    stat_run_tail_ns: u128,
+    stat_run_head_ns: u128,
+}
+
+// TODO(@ohsayan): prepare histogram for report; for now there's no use of the head and tail latencies
+#[derive(Default, Debug)]
+pub struct RuntimeStats {
+    pub qps: f64,
+    pub avg_per_query_ns: f64,
+    pub head_ns: u128,
+    pub tail_ns: u128,
+}
+
+impl<Th: ThreadedTask> Taskpool<Th> {
+    pub fn stat_avg(&self) -> f64 {
+        self.stat_run_avg_ns
+    }
+    pub fn stat_tail(&self) -> u128 {
+        self.stat_run_tail_ns
+    }
+    pub fn stat_head(&self) -> u128 {
+        self.stat_run_head_ns
+    }
+    pub fn stat_elapsed(&self) -> u128 {
+        self.record_real_stop
+            .duration_since(self.record_real_start)
+            .as_nanos()
+    }
+}
+
+fn qps(query_count: usize, time_taken_in_nanos: u128) -> f64 {
+    const NANOS_PER_SECOND: u128 = 1_000_000_000;
+    let time_taken_in_nanos_f64 = time_taken_in_nanos as f64;
+    let query_count_f64 = query_count as f64;
+    (query_count_f64 / time_taken_in_nanos_f64) * NANOS_PER_SECOND as f64
+}
+
+impl<Th: ThreadedTask> Taskpool<Th> {
+    pub fn new(size: usize, config: Th) -> TaskPoolResult<Self, Th> {
+        let (task_tx, task_rx) = unbounded();
+        let (res_tx, res_rx) = unbounded();
+        let mut workers = Vec::with_capacity(size);
+        for _ in 0..size {
+            let con = config
+                .initialize_worker()
+                .map_err(TaskpoolError::InitError)?;
+            workers.push(ThreadWorker::new(con, task_rx.clone(), res_tx.clone()));
+        }
+        Ok(Self {
+            workers,
+            _config: config,
+            task_tx,
+            res_rx,
+            stat_run_avg_ns: 0.0,
+            record_real_start: Instant::now(),
+            record_real_stop: Instant::now(),
+            stat_run_head_ns: u128::MAX,
+            stat_run_tail_ns: u128::MIN,
+        })
+    }
+    pub fn blocking_bombard(
+        &mut self,
+        vec: Vec<Th::TaskInput>,
+    ) -> TaskPoolResult<RuntimeStats, Th> {
+        let expected = vec.len();
+        let mut received = 0usize;
+        for task in vec {
+            match self.task_tx.send(WorkerTask::Task(task)) {
+                Ok(()) => {}
+                Err(_) => {
+                    // stop bombarding, we hit an error
+                    return Err(TaskpoolError::BombardError(
+                        "all worker threads exited. this indicates a catastrophic failure",
+                    ));
+                }
+            }
+        }
+        while received != expected {
+            match self.res_rx.recv() {
+                Err(_) => {
+                    // all workers exited. that is catastrophic
+                    return Err(TaskpoolError::BombardError(
+                        "detected all worker threads crashed during run check",
+                    ));
+                }
+                Ok(r) => self.recompute_stats(&mut received, r)?,
+            };
+        }
+        // compute stats
+        let ret = Ok(RuntimeStats {
+            qps: qps(received, self.stat_elapsed()),
+            avg_per_query_ns: self.stat_avg(),
+            head_ns: self.stat_head(),
+            tail_ns: self.stat_tail(),
+        });
+        // reset stats
+        self.stat_run_avg_ns = 0.0;
+        self.record_real_start = Instant::now();
+        self.record_real_stop = Instant::now();
+        self.stat_run_head_ns = u128::MAX;
+        self.stat_run_tail_ns = u128::MIN;
+        // return
+        ret
+    }
+    fn recompute_stats(
+        &mut self,
+        received: &mut usize,
+        result: Result<(Instant, Instant), <Th as ThreadedTask>::TaskWorkerWorkError>,
+    ) -> Result<(), TaskpoolError<Th>> {
+        *received += 1;
+        let (start, stop) = match result {
+            Ok(time) => time,
+            Err(e) => return Err(TaskpoolError::WorkerError(e)),
+        };
+        // adjust real start
+        if start < self.record_real_start {
+            self.record_real_start = start;
+        }
+        if stop > self.record_real_stop {
+            self.record_real_stop = stop;
+        }
+        let current_time = stop.duration_since(start).as_nanos();
+        self.stat_run_avg_ns = self.stat_run_avg_ns
+            + ((current_time as f64 - self.stat_run_avg_ns) / *received as f64);
+        if current_time > self.stat_run_tail_ns {
+            self.stat_run_tail_ns = current_time;
+        }
+        if current_time < self.stat_run_head_ns {
+            self.stat_run_head_ns = current_time;
+        }
+        Ok(())
+    }
+}
+
+impl<Th: ThreadedTask> Drop for Taskpool<Th> {
+    fn drop(&mut self) {
+        for _ in 0..self.workers.len() {
+            self.task_tx.send(WorkerTask::Exit).unwrap();
+        }
+        for worker in self.workers.drain(..) {
+            worker.handle.join().unwrap()
+        }
+    }
+}

From 250a2b3c163f565d5ebe6569850d4b9d7cc44aeb Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 19 Nov 2023 23:45:44 +0530
Subject: [PATCH 284/310] Implement new benchmark engine

---
 cli/Cargo.toml           |   1 +
 cli/src/args.rs          |  45 +----
 cli/src/error.rs         |  13 ++
 libsky/src/lib.rs        | 136 +++++++++++--
 sky-bench/README.md      |  20 +-
 sky-bench/src/args.rs    |  53 +----
 sky-bench/src/bench.rs   | 294 ++++++++++++++++------------
 sky-bench/src/error.rs   |  27 ++-
 sky-bench/src/main.rs    |   2 +-
 sky-bench/src/pool.rs    | 279 ---------------------------
 sky-bench/src/runtime.rs | 406 +++++++++++++++++++++++++++++++++++++++
 11 files changed, 758 insertions(+), 518 deletions(-)
 delete mode 100644 sky-bench/src/pool.rs
 create mode 100644 sky-bench/src/runtime.rs

diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index e27538bd..acd78d13 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -10,5 +10,6 @@ edition = "2021"
 # internal deps
 libsky = { path = "../libsky" }
 skytable = { git = "https://github.com/skytable/client-rust.git", branch = "octave" }
+# external deps
 crossterm = "0.27.0"
 rustyline = "12.0.0"
diff --git a/cli/src/args.rs b/cli/src/args.rs
index 29923f8d..804afc16 100644
--- a/cli/src/args.rs
+++ b/cli/src/args.rs
@@ -30,9 +30,10 @@ use {
         event::{self, Event, KeyCode, KeyEvent},
         terminal,
     },
+    libsky::CliAction,
     std::{
-        collections::{hash_map::Entry, HashMap},
-        env, fs,
+        collections::HashMap,
+        fs,
         io::{self, Write},
         process::exit,
     },
@@ -75,42 +76,12 @@ enum TaskInner {
 }
 
 fn load_env() -> CliResult<TaskInner> {
-    let mut args = HashMap::new();
-    let mut it = env::args().skip(1).into_iter();
-    while let Some(arg) = it.next() {
-        let (arg, arg_val) = match arg.as_str() {
-            "--help" => return Ok(TaskInner::HelpMsg(TXT_HELP.into())),
-            "--version" => return Ok(TaskInner::HelpMsg(format!("skysh v{}", libsky::VERSION))),
-            _ if arg.starts_with("--") => match it.next() {
-                Some(arg_val) => (arg, arg_val),
-                None => {
-                    // self contained?
-                    let split: Vec<&str> = arg.split("=").collect();
-                    if split.len() != 2 {
-                        return Err(CliError::ArgsErr(format!("expected value for {arg}")));
-                    }
-                    (split[0].into(), split[1].into())
-                }
-            },
-            unknown_arg => {
-                return Err(CliError::ArgsErr(format!(
-                    "unknown argument: {unknown_arg}"
-                )))
-            }
-        };
-        match args.entry(arg) {
-            Entry::Occupied(oe) => {
-                return Err(CliError::ArgsErr(format!(
-                    "found duplicate values for {}",
-                    oe.key()
-                )))
-            }
-            Entry::Vacant(ve) => {
-                ve.insert(arg_val);
-            }
-        }
+    let action = libsky::parse_cli_args_disallow_duplicate()?;
+    match action {
+        CliAction::Help => Ok(TaskInner::HelpMsg(TXT_HELP.into())),
+        CliAction::Version => Ok(TaskInner::HelpMsg(libsky::version_msg("skysh"))),
+        CliAction::Action(a) => Ok(TaskInner::OpenShell(a)),
     }
-    Ok(TaskInner::OpenShell(args))
 }
 
 pub fn parse() -> CliResult<Task> {
diff --git a/cli/src/error.rs b/cli/src/error.rs
index 89878bac..913c2e9f 100644
--- a/cli/src/error.rs
+++ b/cli/src/error.rs
@@ -36,6 +36,19 @@ pub enum CliError {
     IoError(std::io::Error),
 }
 
+impl From<libsky::ArgParseError> for CliError {
+    fn from(e: libsky::ArgParseError) -> Self {
+        match e {
+            libsky::ArgParseError::Duplicate(d) => {
+                Self::ArgsErr(format!("duplicate value for `{d}`"))
+            }
+            libsky::ArgParseError::MissingValue(m) => {
+                Self::ArgsErr(format!("missing value for `{m}`"))
+            }
+        }
+    }
+}
+
 impl From<skytable::error::Error> for CliError {
     fn from(cle: skytable::error::Error) -> Self {
         Self::ClientError(cle)
diff --git a/libsky/src/lib.rs b/libsky/src/lib.rs
index 15ee9247..e5c93208 100644
--- a/libsky/src/lib.rs
+++ b/libsky/src/lib.rs
@@ -31,9 +31,6 @@
 //!
 //! This contains modules which are shared by both the `cli` and the `server` modules
 
-use std::error::Error;
-/// A generic result
-pub type TResult<T> = Result<T, Box<dyn Error>>;
 /// The size of the read buffer in bytes
 pub const BUF_CAP: usize = 8 * 1024; // 8 KB per-connection
 /// The current version
@@ -41,15 +38,130 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 /// The URL
 pub const URL: &str = "https://github.com/skytable/skytable";
 
-#[macro_export]
-/// Don't use unwrap_or but use this macro as the optimizer fails to optimize away usages
-/// of unwrap_or and creates a lot of LLVM IR bloat. use
-// FIXME(@ohsayan): Fix this when https://github.com/rust-lang/rust/issues/68667 is addressed
-macro_rules! option_unwrap_or {
-    ($try:expr, $fallback:expr) => {
-        match $try {
-            Some(t) => t,
-            None => $fallback,
+use std::{
+    collections::{hash_map::Entry, HashMap},
+    env,
+};
+
+/// Returns a formatted version message `{binary} vx.y.z`
+pub fn version_msg(binary: &str) -> String {
+    format!("{binary} v{VERSION}")
+}
+
+#[derive(Debug, PartialEq)]
+/// The CLI action that is expected to be performed
+pub enum CliAction<A> {
+    /// Display the `--help` message
+    Help,
+    /// Dipslay the `--version`
+    Version,
+    /// Perform an action using the given args
+    Action(A),
+}
+
+pub type CliActionMulti = CliAction<HashMap<String, Vec<String>>>;
+pub type CliActionSingle = CliAction<HashMap<String, String>>;
+
+/*
+    generic cli arg parser
+*/
+
+#[derive(Debug, PartialEq)]
+/// Argument parse error
+pub enum AnyArgsParseError {
+    /// The value for the given argument was either incorrectly formatted or missing
+    MissingValue(String),
+}
+/// Parse CLI args, allowing duplicates (bucketing them)
+pub fn parse_cli_args_allow_duplicate() -> Result<CliActionMulti, AnyArgsParseError> {
+    parse_args(env::args())
+}
+/// Parse args allowing and bucketing any duplicates
+pub fn parse_args(
+    args: impl IntoIterator<Item = String>,
+) -> Result<CliActionMulti, AnyArgsParseError> {
+    let mut ret: HashMap<String, Vec<String>> = HashMap::new();
+    let mut args = args.into_iter().skip(1).peekable();
+    while let Some(arg) = args.next() {
+        if arg == "--help" {
+            return Ok(CliAction::Help);
+        }
+        if arg == "--version" {
+            return Ok(CliAction::Version);
+        }
+        let (arg, value) = extract_arg(arg, &mut args).map_err(AnyArgsParseError::MissingValue)?;
+        match ret.get_mut(&arg) {
+            Some(values) => {
+                values.push(value);
+            }
+            None => {
+                ret.insert(arg, vec![value]);
+            }
+        }
+    }
+    Ok(CliAction::Action(ret))
+}
+
+/*
+    no duplicate arg parser
+*/
+
+#[derive(Debug, PartialEq)]
+/// Argument parse error
+pub enum ArgParseError {
+    /// The given argument had a duplicate value
+    Duplicate(String),
+    /// The given argument did not have an appropriate value
+    MissingValue(String),
+}
+/// Parse all non-repeating CLI arguments
+pub fn parse_cli_args_disallow_duplicate() -> Result<CliActionSingle, ArgParseError> {
+    parse_args_deny_duplicate(env::args())
+}
+/// Parse all arguments but deny any duplicates
+pub fn parse_args_deny_duplicate(
+    args: impl IntoIterator<Item = String>,
+) -> Result<CliActionSingle, ArgParseError> {
+    let mut ret: HashMap<String, String> = HashMap::new();
+    let mut args = args.into_iter().skip(1).peekable();
+    while let Some(arg) = args.next() {
+        if arg == "--help" {
+            return Ok(CliAction::Help);
+        }
+        if arg == "--version" {
+            return Ok(CliAction::Version);
+        }
+        let (arg, value) = extract_arg(arg, &mut args).map_err(ArgParseError::MissingValue)?;
+        match ret.entry(arg) {
+            Entry::Vacant(v) => {
+                v.insert(value);
+            }
+            Entry::Occupied(oe) => return Err(ArgParseError::Duplicate(oe.key().into())),
+        }
+    }
+    Ok(CliAction::Action(ret))
+}
+
+/// Extract an argument:
+/// - `--arg=value`
+/// - `--arg value`
+fn extract_arg(
+    arg: String,
+    args: &mut impl Iterator<Item = String>,
+) -> Result<(String, String), String> {
+    let this_args: Vec<&str> = arg.split("=").collect();
+    let (arg, value) = if this_args.len() == 2 {
+        // self contained arg
+        (this_args[0].to_owned(), this_args[1].to_owned())
+    } else {
+        if this_args.len() == 1 {
+            match args.next() {
+                None => return Err(arg),
+                Some(val) => (arg, val),
+            }
+        } else {
+            return Err(arg);
         }
     };
+    Ok((arg, value))
 }
diff --git a/sky-bench/README.md b/sky-bench/README.md
index 69869217..f9327f13 100644
--- a/sky-bench/README.md
+++ b/sky-bench/README.md
@@ -5,16 +5,10 @@ tool doesn't do anything "fancy" to make benchmarks appear better than they are.
 
 Here's how the benchmark tool works (it's dead simple):
 
-1. Depending on the configuration it launches "network pools" which are just thread pools where each worker
-   holds a persistent connection to the database (something like a connection pool)
-2. A collection of unique, random keys are generated using a PRNG provided by the `rand` library that is
-   seeded using the OS' source of randomness. The values are allowed to repeat
-3. The [Skytable Rust driver](https://github.com/skytable/client-rust) is used to generate _raw query packets_. To put it simply, the keys and values are turned into `Query` objects and then into the raw bytes that will be sent over the network. (This just makes it simpler to design the network pool interface)
-4. For every type of benchmark (GET,SET,...) we use the network pool to send all the bytes and wait until we receive the expected response. We time how long it takes to send and receive the response for all the queries for the given test (aggregate)
-5. We repeat this for all the remaining tests
-6. We repeat the entire set of tests 5 times (by default, this can be changed).
-7. We do the calculations and output the results.
-
-## License
-
-All files in this directory are distributed under the [AGPL-3.0 License](../LICENSE).
+1. We start up some threads with each having a thread local connection to the database
+2. Each thread attempts to keep running queries until the target number of queries is reached.
+   - This sort of simulates a real-world scenario where these threads are like your application servers sending requests to the database
+   - Also there is no ideal distribution and the number of queries each worker runs is unspecified (but owing to low latencies from the database, that should be even)
+   - We do this to ensure that the distribution of queries executed by each "server" is skewed as it would be in the real world.
+3. Once the target number of queries are reached, the workers notify that the task is complete. Each worker keeps track of how long it spent processing queries and this is also notified to the benchmark engine
+4. The benchmark engine then computes relevant statistics
diff --git a/sky-bench/src/args.rs b/sky-bench/src/args.rs
index 1a8a63aa..31828ce2 100644
--- a/sky-bench/src/args.rs
+++ b/sky-bench/src/args.rs
@@ -26,10 +26,8 @@
 
 use {
     crate::error::{BenchError, BenchResult},
-    std::{
-        collections::hash_map::{Entry, HashMap},
-        env,
-    },
+    libsky::CliAction,
+    std::collections::hash_map::HashMap,
 };
 
 const TXT_HELP: &str = include_str!("../help_text/help");
@@ -77,47 +75,12 @@ impl BenchConfig {
 }
 
 fn load_env() -> BenchResult<TaskInner> {
-    let mut args = HashMap::new();
-    let mut it = env::args().skip(1).into_iter();
-    while let Some(arg) = it.next() {
-        let (arg, arg_val) = match arg.as_str() {
-            "--help" => return Ok(TaskInner::HelpMsg(TXT_HELP.into())),
-            "--version" => {
-                return Ok(TaskInner::HelpMsg(format!(
-                    "sky-bench v{}",
-                    libsky::VERSION
-                )))
-            }
-            _ if arg.starts_with("--") => match it.next() {
-                Some(arg_val) => (arg, arg_val),
-                None => {
-                    // self contained?
-                    let split: Vec<&str> = arg.split("=").collect();
-                    if split.len() != 2 {
-                        return Err(BenchError::ArgsErr(format!("expected value for {arg}")));
-                    }
-                    (split[0].into(), split[1].into())
-                }
-            },
-            unknown_arg => {
-                return Err(BenchError::ArgsErr(format!(
-                    "unknown argument: {unknown_arg}"
-                )))
-            }
-        };
-        match args.entry(arg) {
-            Entry::Occupied(oe) => {
-                return Err(BenchError::ArgsErr(format!(
-                    "found duplicate values for {}",
-                    oe.key()
-                )))
-            }
-            Entry::Vacant(ve) => {
-                ve.insert(arg_val);
-            }
-        }
+    let action = libsky::parse_cli_args_disallow_duplicate()?;
+    match action {
+        CliAction::Help => Ok(TaskInner::HelpMsg(TXT_HELP.into())),
+        CliAction::Version => Ok(TaskInner::HelpMsg(libsky::version_msg("sky-bench"))),
+        CliAction::Action(a) => Ok(TaskInner::CheckConfig(a)),
     }
-    Ok(TaskInner::CheckConfig(args))
 }
 
 fn cdig(n: usize) -> usize {
@@ -176,7 +139,7 @@ pub fn parse() -> BenchResult<Task> {
     };
     // threads
     let thread_count = match args.remove("--threads") {
-        None => num_cpus::get(),
+        None => num_cpus::get_physical(),
         Some(tc) => match tc.parse() {
             Ok(tc) if tc > 0 => tc,
             Err(_) | Ok(_) => {
diff --git a/sky-bench/src/bench.rs b/sky-bench/src/bench.rs
index 651b30c6..7325734c 100644
--- a/sky-bench/src/bench.rs
+++ b/sky-bench/src/bench.rs
@@ -24,136 +24,157 @@
  *
 */
 
-use crate::error::BenchResult;
-
 use {
     crate::{
         args::BenchConfig,
-        error::{self, BenchmarkTaskWorkerError},
-        pool::{RuntimeStats, Taskpool, ThreadedTask},
+        error::{self, BenchResult},
+        runtime::{BombardPool, RuntimeStats, ThreadedBombardTask},
     },
-    skytable::{query, response::Response, Config, Connection, Query},
-    std::time::Instant,
+    skytable::{error::Error, query, response::Response, Config, Connection, Query},
+    std::{fmt, time::Instant},
 };
 
+/*
+    task impl
+*/
+
+/// A bombard task used for benchmarking
+
 #[derive(Debug)]
-pub struct BenchmarkTask {
-    cfg: Config,
+pub struct BombardTask {
+    config: Config,
 }
 
-impl BenchmarkTask {
-    pub fn new(host: &str, port: u16, username: &str, password: &str) -> Self {
+impl BombardTask {
+    pub fn new(config: Config) -> Self {
+        Self { config }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum BombardTaskKind {
+    Insert(u8),
+    Update,
+    Delete,
+}
+
+#[derive(Debug, Clone)]
+pub struct BombardTaskSpec {
+    kind: BombardTaskKind,
+    base_query: String,
+    pk_len: usize,
+}
+
+impl BombardTaskSpec {
+    pub fn insert(base_query: String, pk_len: usize, second_column: u8) -> Self {
         Self {
-            cfg: Config::new(host, port, username, password),
+            kind: BombardTaskKind::Insert(second_column),
+            base_query,
+            pk_len,
+        }
+    }
+    pub fn update(base_query: String, pk_len: usize) -> Self {
+        Self {
+            kind: BombardTaskKind::Update,
+            base_query,
+            pk_len,
+        }
+    }
+    pub fn delete(base_query: String, pk_len: usize) -> Self {
+        Self {
+            kind: BombardTaskKind::Delete,
+            base_query,
+            pk_len,
+        }
+    }
+    fn generate(&self, current: u64) -> (Query, Response) {
+        let mut q = query!(&self.base_query);
+        let resp = match self.kind {
+            BombardTaskKind::Insert(second_column) => {
+                q.push_param(format!("{:0>width$}", current, width = self.pk_len));
+                q.push_param(second_column);
+                Response::Empty
+            }
+            BombardTaskKind::Update => {
+                q.push_param(1u64);
+                q.push_param(format!("{:0>width$}", current, width = self.pk_len));
+                Response::Empty
+            }
+            BombardTaskKind::Delete => {
+                q.push_param(format!("{:0>width$}", current, width = self.pk_len));
+                Response::Empty
+            }
+        };
+        (q, resp)
+    }
+}
+
+/// Errors while running a bombard
+#[derive(Debug)]
+pub enum BombardTaskError {
+    DbError(Error),
+    Mismatch,
+}
+
+impl fmt::Display for BombardTaskError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::DbError(e) => write!(f, "a bombard subtask failed with {e}"),
+            Self::Mismatch => write!(f, "got unexpected response for bombard subtask"),
         }
     }
 }
 
-impl ThreadedTask for BenchmarkTask {
-    type TaskWorker = Connection;
-    type TaskWorkerInitError = BenchmarkTaskWorkerError;
-    type TaskWorkerTerminateError = BenchmarkTaskWorkerError;
-    type TaskWorkerWorkError = BenchmarkTaskWorkerError;
-    type TaskInput = (Query, Response);
-    fn initialize_worker(&self) -> Result<Self::TaskWorker, Self::TaskWorkerInitError> {
-        self.cfg.connect().map_err(Into::into)
+impl From<Error> for BombardTaskError {
+    fn from(dbe: Error) -> Self {
+        Self::DbError(dbe)
+    }
+}
+
+impl ThreadedBombardTask for BombardTask {
+    type Worker = Connection;
+    type WorkerTask = (Query, Response);
+    type WorkerTaskSpec = BombardTaskSpec;
+    type WorkerInitError = Error;
+    type WorkerTaskError = BombardTaskError;
+    fn worker_init(&self) -> Result<Self::Worker, Self::WorkerInitError> {
+        self.config.connect()
     }
-    fn drive_worker_timed(
-        worker: &mut Self::TaskWorker,
-        (query, expected_resp): Self::TaskInput,
-    ) -> Result<(Instant, Instant), Self::TaskWorkerWorkError> {
+    fn generate_task(spec: &Self::WorkerTaskSpec, current: u64) -> Self::WorkerTask {
+        spec.generate(current)
+    }
+    fn worker_drive_timed(
+        worker: &mut Self::Worker,
+        (query, response): Self::WorkerTask,
+    ) -> Result<u128, Self::WorkerTaskError> {
         let start = Instant::now();
-        let resp = worker.query(&query)?;
+        let ret = worker.query(&query)?;
         let stop = Instant::now();
-        if resp == expected_resp {
-            Ok((start, stop))
+        if ret == response {
+            Ok(stop.duration_since(start).as_nanos())
         } else {
-            Err(BenchmarkTaskWorkerError::Error(format!(
-                "response from server did not match expected response: {:?}",
-                resp
-            )))
+            Err(BombardTaskError::Mismatch)
         }
     }
-    fn terminate_worker(
-        &self,
-        _: &mut Self::TaskWorker,
-    ) -> Result<(), Self::TaskWorkerTerminateError> {
-        Ok(())
-    }
 }
 
+/*
+    runner
+*/
+
 pub fn run(bench: BenchConfig) -> error::BenchResult<()> {
-    let bench_config = BenchmarkTask::new(&bench.host, bench.port, "root", &bench.root_pass);
+    let bench_config = BombardTask::new(Config::new(
+        &bench.host,
+        bench.port,
+        "root",
+        &bench.root_pass,
+    ));
     info!("running preliminary checks and creating model `bench.bench` with definition: `{{un: string, pw: uint8}}`");
-    let mut main_thread_db = bench_config.cfg.connect()?;
+    let mut main_thread_db = bench_config.config.connect()?;
     main_thread_db.query_parse::<()>(&query!("create space bench"))?;
     main_thread_db.query_parse::<()>(&query!("create model bench.bench(un: string, pw: uint8)"))?;
-    info!(
-        "initializing connection pool with {} connections",
-        bench.threads
-    );
-    let mut p = Taskpool::new(bench.threads, bench_config)?;
-    info!(
-        "pool initialized successfully. preparing {} `INSERT` queries with primary key size={} bytes",
-        bench.query_count, bench.key_size
-    );
-    let mut insert_stats = Default::default();
-    let mut update_stats = Default::default();
-    let mut delete_stats = Default::default();
-    match || -> BenchResult<()> {
-        // bench insert
-        let insert_queries: Vec<(Query, Response)> = (0..bench.query_count)
-            .into_iter()
-            .map(|i| {
-                (
-                    query!(
-                        "insert into bench.bench(?, ?)",
-                        format!("{:0>width$}", i, width = bench.key_size),
-                        0u64
-                    ),
-                    Response::Empty,
-                )
-            })
-            .collect();
-        info!("benchmarking `INSERT` queries");
-        insert_stats = p.blocking_bombard(insert_queries)?;
-        // bench update
-        info!("completed benchmarking `INSERT`. preparing `UPDATE` queries");
-        let update_queries: Vec<(Query, Response)> = (0..bench.query_count)
-            .into_iter()
-            .map(|i| {
-                (
-                    query!(
-                        "update bench.bench set pw += ? where un = ?",
-                        1u64,
-                        format!("{:0>width$}", i, width = bench.key_size),
-                    ),
-                    Response::Empty,
-                )
-            })
-            .collect();
-        info!("benchmarking `UPDATE` queries");
-        update_stats = p.blocking_bombard(update_queries)?;
-        // bench delete
-        info!("completed benchmarking `UPDATE`. preparing `DELETE` queries");
-        let delete_queries: Vec<(Query, Response)> = (0..bench.query_count)
-            .into_iter()
-            .map(|i| {
-                (
-                    query!(
-                        "delete from bench.bench where un = ?",
-                        format!("{:0>width$}", i, width = bench.key_size),
-                    ),
-                    Response::Empty,
-                )
-            })
-            .collect();
-        info!("benchmarking `DELETE` queries");
-        delete_stats = p.blocking_bombard(delete_queries)?;
-        info!("completed benchmarking `DELETE` queries");
-        Ok(())
-    }() {
-        Ok(()) => {}
+    let stats = match bench_internal(bench_config, bench) {
+        Ok(stats) => stats,
         Err(e) => {
             error!("benchmarking failed. attempting to clean up");
             match cleanup(main_thread_db) {
@@ -164,20 +185,18 @@ pub fn run(bench: BenchConfig) -> error::BenchResult<()> {
                 }
             }
         }
-    }
-    drop(p);
+    };
     warn!("benchmarks might appear to be slower. this tool is currently experimental");
     // print results
-    info!("results:");
-    print_table(vec![
-        ("INSERT", insert_stats),
-        ("UPDATE", update_stats),
-        ("DELETE", delete_stats),
-    ]);
+    print_table(stats);
     cleanup(main_thread_db)?;
     Ok(())
 }
 
+/*
+    util
+*/
+
 fn cleanup(mut main_thread_db: Connection) -> Result<(), error::BenchError> {
     trace!("dropping space and table");
     main_thread_db.query_parse::<()>(&query!("drop model bench.bench"))?;
@@ -195,22 +214,51 @@ fn print_table(data: Vec<(&'static str, RuntimeStats)>) {
     println!(
         "+---------+--------------------------+-----------------------+------------------------+"
     );
-    for (
-        query,
-        RuntimeStats {
-            qps,
-            avg_per_query_ns: _,
-            head_ns,
-            tail_ns,
-        },
-    ) in data
-    {
+    for (query, RuntimeStats { qps, head, tail }) in data {
         println!(
             "| {:<7} | {:>24.2} | {:>21} | {:>22} |",
-            query, qps, tail_ns, head_ns
+            query, qps, tail, head
         );
     }
     println!(
         "+---------+--------------------------+-----------------------+------------------------+"
     );
 }
+
+/*
+    bench runner
+*/
+
+fn bench_internal(
+    config: BombardTask,
+    bench: BenchConfig,
+) -> BenchResult<Vec<(&'static str, RuntimeStats)>> {
+    let mut ret = vec![];
+    // initialize pool
+    info!("initializing connection pool");
+    let mut pool = BombardPool::new(bench.threads, config)?;
+    // bench INSERT
+    info!("benchmarking `INSERT`");
+    let insert = BombardTaskSpec::insert("insert into bench.bench(?, ?)".into(), bench.key_size, 0);
+    let insert_stats = pool.blocking_bombard(insert, bench.query_count)?;
+    ret.push(("INSERT", insert_stats));
+    // bench UPDATE
+    info!("benchmarking `UPDATE`");
+    let update = BombardTaskSpec::update(
+        "update bench.bench set pw += ? where un = ?".into(),
+        bench.key_size,
+    );
+    let update_stats = pool.blocking_bombard(update, bench.query_count)?;
+    ret.push(("UPDATE", update_stats));
+    // bench DELETE
+    info!("benchmarking `DELETE`");
+    let delete = BombardTaskSpec::delete(
+        "delete from bench.bench where un = ?".into(),
+        bench.key_size,
+    );
+    let delete_stats = pool.blocking_bombard(delete, bench.query_count)?;
+    ret.push(("DELETE", delete_stats));
+    info!("completed benchmarks. closing pool");
+    drop(pool);
+    Ok(ret)
+}
diff --git a/sky-bench/src/error.rs b/sky-bench/src/error.rs
index 0fe3fcb6..6cf5caf9 100644
--- a/sky-bench/src/error.rs
+++ b/sky-bench/src/error.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    crate::{bench::BenchmarkTask, pool::TaskpoolError},
+    crate::{bench::BombardTask, runtime::BombardError},
     core::fmt,
     skytable::error::Error,
 };
@@ -35,13 +35,20 @@ pub type BenchResult<T> = Result<T, BenchError>;
 #[derive(Debug)]
 pub enum BenchError {
     ArgsErr(String),
-    BenchError(TaskpoolError<BenchmarkTask>),
+    BenchBombardError(BombardError<BombardTask>),
     DirectDbError(Error),
 }
 
-impl From<TaskpoolError<BenchmarkTask>> for BenchError {
-    fn from(e: TaskpoolError<BenchmarkTask>) -> Self {
-        Self::BenchError(e)
+impl From<libsky::ArgParseError> for BenchError {
+    fn from(e: libsky::ArgParseError) -> Self {
+        match e {
+            libsky::ArgParseError::Duplicate(d) => {
+                Self::ArgsErr(format!("duplicate value for `{d}`"))
+            }
+            libsky::ArgParseError::MissingValue(m) => {
+                Self::ArgsErr(format!("missing value for `{m}`"))
+            }
+        }
     }
 }
 
@@ -51,12 +58,18 @@ impl From<Error> for BenchError {
     }
 }
 
+impl From<BombardError<BombardTask>> for BenchError {
+    fn from(e: BombardError<BombardTask>) -> Self {
+        Self::BenchBombardError(e)
+    }
+}
+
 impl fmt::Display for BenchError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             Self::ArgsErr(e) => write!(f, "args error: {e}"),
-            Self::BenchError(e) => write!(f, "benchmark system error: {e}"),
             Self::DirectDbError(e) => write!(f, "direct operation on db failed. {e}"),
+            Self::BenchBombardError(e) => write!(f, "benchmark failed: {e}"),
         }
     }
 }
@@ -66,7 +79,6 @@ impl std::error::Error for BenchError {}
 #[derive(Debug)]
 pub enum BenchmarkTaskWorkerError {
     DbError(Error),
-    Error(String),
 }
 
 impl From<Error> for BenchmarkTaskWorkerError {
@@ -79,7 +91,6 @@ impl fmt::Display for BenchmarkTaskWorkerError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             Self::DbError(e) => write!(f, "worker failed due to DB error. {e}"),
-            Self::Error(e) => write!(f, "worker failed. {e}"),
         }
     }
 }
diff --git a/sky-bench/src/main.rs b/sky-bench/src/main.rs
index 27dfc5ae..07e5f522 100644
--- a/sky-bench/src/main.rs
+++ b/sky-bench/src/main.rs
@@ -29,7 +29,7 @@ extern crate log;
 mod args;
 mod bench;
 mod error;
-mod pool;
+mod runtime;
 
 fn main() {
     env_logger::Builder::new()
diff --git a/sky-bench/src/pool.rs b/sky-bench/src/pool.rs
deleted file mode 100644
index 0b6e13e4..00000000
--- a/sky-bench/src/pool.rs
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Created on Fri Nov 17 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crossbeam_channel::{unbounded, Receiver, Sender},
-    std::{
-        fmt,
-        marker::PhantomData,
-        thread::{self, JoinHandle},
-        time::Instant,
-    },
-};
-
-pub type TaskPoolResult<T, Th> = Result<T, TaskpoolError<Th>>;
-
-#[derive(Debug)]
-pub enum TaskpoolError<Th: ThreadedTask> {
-    InitError(Th::TaskWorkerInitError),
-    BombardError(&'static str),
-    WorkerError(Th::TaskWorkerWorkError),
-}
-
-impl<Th: ThreadedTask> fmt::Display for TaskpoolError<Th>
-where
-    Th::TaskWorkerInitError: fmt::Display,
-    Th::TaskWorkerWorkError: fmt::Display,
-{
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::InitError(e) => write!(f, "failed to init worker pool. {e}"),
-            Self::BombardError(e) => write!(f, "failed to post work to pool. {e}"),
-            Self::WorkerError(e) => write!(f, "failed running worker task. {e}"),
-        }
-    }
-}
-
-pub trait ThreadedTask: Send + Sync + 'static {
-    /// the per-thread item that does the actual work
-    ///
-    /// NB: this is not to be confused with the actual underlying thread pool worker
-    type TaskWorker: Send + Sync;
-    /// when attempting initialization of the per-thread task worker, if an error is thrown, this is the type
-    /// you're looking for
-    type TaskWorkerInitError: Send + Sync;
-    /// when attempting to run a single unit of work, if any error occurs this is the error type that is to be returned
-    type TaskWorkerWorkError: Send + Sync;
-    /// when attempting to close a worker, if an error occurs this is the error type that is returned
-    type TaskWorkerTerminateError: Send + Sync;
-    /// the task that is sent to each worker
-    type TaskInput: Send + Sync;
-    // fn
-    /// initialize the worker
-    fn initialize_worker(&self) -> Result<Self::TaskWorker, Self::TaskWorkerInitError>;
-    /// drive the worker to complete a task and return the time
-    fn drive_worker_timed(
-        worker: &mut Self::TaskWorker,
-        task: Self::TaskInput,
-    ) -> Result<(Instant, Instant), Self::TaskWorkerWorkError>;
-    fn terminate_worker(
-        &self,
-        worker: &mut Self::TaskWorker,
-    ) -> Result<(), Self::TaskWorkerTerminateError>;
-}
-
-#[derive(Debug)]
-struct ThreadWorker<Th> {
-    handle: JoinHandle<()>,
-    _m: PhantomData<Th>,
-}
-
-#[derive(Debug)]
-enum WorkerTask<Th: ThreadedTask> {
-    Task(Th::TaskInput),
-    Exit,
-}
-
-impl<Th: ThreadedTask> ThreadWorker<Th> {
-    fn new(
-        hl_worker: Th::TaskWorker,
-        task_rx: Receiver<WorkerTask<Th>>,
-        res_tx: Sender<Result<(Instant, Instant), Th::TaskWorkerWorkError>>,
-    ) -> Self {
-        Self {
-            handle: thread::spawn(move || {
-                let mut worker = hl_worker;
-                loop {
-                    let task = match task_rx.recv().unwrap() {
-                        WorkerTask::Exit => {
-                            drop(task_rx);
-                            return;
-                        }
-                        WorkerTask::Task(t) => t,
-                    };
-                    res_tx
-                        .send(Th::drive_worker_timed(&mut worker, task))
-                        .unwrap();
-                }
-            }),
-            _m: PhantomData,
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct Taskpool<Th: ThreadedTask> {
-    workers: Vec<ThreadWorker<Th>>,
-    _config: Th,
-    task_tx: Sender<WorkerTask<Th>>,
-    res_rx: Receiver<Result<(Instant, Instant), Th::TaskWorkerWorkError>>,
-    record_real_start: Instant,
-    record_real_stop: Instant,
-    stat_run_avg_ns: f64,
-    stat_run_tail_ns: u128,
-    stat_run_head_ns: u128,
-}
-
-// TODO(@ohsayan): prepare histogram for report; for now there's no use of the head and tail latencies
-#[derive(Default, Debug)]
-pub struct RuntimeStats {
-    pub qps: f64,
-    pub avg_per_query_ns: f64,
-    pub head_ns: u128,
-    pub tail_ns: u128,
-}
-
-impl<Th: ThreadedTask> Taskpool<Th> {
-    pub fn stat_avg(&self) -> f64 {
-        self.stat_run_avg_ns
-    }
-    pub fn stat_tail(&self) -> u128 {
-        self.stat_run_tail_ns
-    }
-    pub fn stat_head(&self) -> u128 {
-        self.stat_run_head_ns
-    }
-    pub fn stat_elapsed(&self) -> u128 {
-        self.record_real_stop
-            .duration_since(self.record_real_start)
-            .as_nanos()
-    }
-}
-
-fn qps(query_count: usize, time_taken_in_nanos: u128) -> f64 {
-    const NANOS_PER_SECOND: u128 = 1_000_000_000;
-    let time_taken_in_nanos_f64 = time_taken_in_nanos as f64;
-    let query_count_f64 = query_count as f64;
-    (query_count_f64 / time_taken_in_nanos_f64) * NANOS_PER_SECOND as f64
-}
-
-impl<Th: ThreadedTask> Taskpool<Th> {
-    pub fn new(size: usize, config: Th) -> TaskPoolResult<Self, Th> {
-        let (task_tx, task_rx) = unbounded();
-        let (res_tx, res_rx) = unbounded();
-        let mut workers = Vec::with_capacity(size);
-        for _ in 0..size {
-            let con = config
-                .initialize_worker()
-                .map_err(TaskpoolError::InitError)?;
-            workers.push(ThreadWorker::new(con, task_rx.clone(), res_tx.clone()));
-        }
-        Ok(Self {
-            workers,
-            _config: config,
-            task_tx,
-            res_rx,
-            stat_run_avg_ns: 0.0,
-            record_real_start: Instant::now(),
-            record_real_stop: Instant::now(),
-            stat_run_head_ns: u128::MAX,
-            stat_run_tail_ns: u128::MIN,
-        })
-    }
-    pub fn blocking_bombard(
-        &mut self,
-        vec: Vec<Th::TaskInput>,
-    ) -> TaskPoolResult<RuntimeStats, Th> {
-        let expected = vec.len();
-        let mut received = 0usize;
-        for task in vec {
-            match self.task_tx.send(WorkerTask::Task(task)) {
-                Ok(()) => {}
-                Err(_) => {
-                    // stop bombarding, we hit an error
-                    return Err(TaskpoolError::BombardError(
-                        "all worker threads exited. this indicates a catastrophic failure",
-                    ));
-                }
-            }
-        }
-        while received != expected {
-            match self.res_rx.recv() {
-                Err(_) => {
-                    // all workers exited. that is catastrophic
-                    return Err(TaskpoolError::BombardError(
-                        "detected all worker threads crashed during run check",
-                    ));
-                }
-                Ok(r) => self.recompute_stats(&mut received, r)?,
-            };
-        }
-        // compute stats
-        let ret = Ok(RuntimeStats {
-            qps: qps(received, self.stat_elapsed()),
-            avg_per_query_ns: self.stat_avg(),
-            head_ns: self.stat_head(),
-            tail_ns: self.stat_tail(),
-        });
-        // reset stats
-        self.stat_run_avg_ns = 0.0;
-        self.record_real_start = Instant::now();
-        self.record_real_stop = Instant::now();
-        self.stat_run_head_ns = u128::MAX;
-        self.stat_run_tail_ns = u128::MIN;
-        // return
-        ret
-    }
-    fn recompute_stats(
-        &mut self,
-        received: &mut usize,
-        result: Result<(Instant, Instant), <Th as ThreadedTask>::TaskWorkerWorkError>,
-    ) -> Result<(), TaskpoolError<Th>> {
-        *received += 1;
-        let (start, stop) = match result {
-            Ok(time) => time,
-            Err(e) => return Err(TaskpoolError::WorkerError(e)),
-        };
-        // adjust real start
-        if start < self.record_real_start {
-            self.record_real_start = start;
-        }
-        if stop > self.record_real_stop {
-            self.record_real_stop = stop;
-        }
-        let current_time = stop.duration_since(start).as_nanos();
-        self.stat_run_avg_ns = self.stat_run_avg_ns
-            + ((current_time as f64 - self.stat_run_avg_ns) / *received as f64);
-        if current_time > self.stat_run_tail_ns {
-            self.stat_run_tail_ns = current_time;
-        }
-        if current_time < self.stat_run_head_ns {
-            self.stat_run_head_ns = current_time;
-        }
-        Ok(())
-    }
-}
-
-impl<Th: ThreadedTask> Drop for Taskpool<Th> {
-    fn drop(&mut self) {
-        for _ in 0..self.workers.len() {
-            self.task_tx.send(WorkerTask::Exit).unwrap();
-        }
-        for worker in self.workers.drain(..) {
-            worker.handle.join().unwrap()
-        }
-    }
-}
diff --git a/sky-bench/src/runtime.rs b/sky-bench/src/runtime.rs
new file mode 100644
index 00000000..4386e1a9
--- /dev/null
+++ b/sky-bench/src/runtime.rs
@@ -0,0 +1,406 @@
+/*
+ * Created on Sun Nov 19 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    crossbeam_channel::{unbounded, Receiver, Sender},
+    std::{
+        fmt::{self, Display},
+        sync::atomic::{AtomicBool, AtomicU64, Ordering},
+        thread::{self, JoinHandle},
+        time::{Duration, Instant},
+    },
+};
+
+pub type BombardResult<T, Bt> = Result<T, BombardError<Bt>>;
+
+/*
+    state mgmt
+*/
+
+#[derive(Debug)]
+/// The pool state. Be warned **ONLY ONE POOL AT A TIME!**
+struct GPState {
+    current: AtomicU64,
+    state: AtomicBool,
+    occupied: AtomicBool,
+}
+
+impl GPState {
+    #[inline(always)]
+    fn get() -> &'static Self {
+        static STATE: GPState = GPState::zero();
+        &STATE
+    }
+    const fn zero() -> Self {
+        Self {
+            current: AtomicU64::new(0),
+            state: AtomicBool::new(true),
+            occupied: AtomicBool::new(false),
+        }
+    }
+    fn occupy(&self) {
+        assert!(!self.occupied.swap(true, Ordering::Release));
+    }
+    fn vacate(&self) {
+        assert!(self.occupied.swap(false, Ordering::Release));
+    }
+    fn guard<T>(f: impl FnOnce() -> T) -> T {
+        let slf = Self::get();
+        slf.occupy();
+        let ret = f();
+        slf.vacate();
+        ret
+    }
+    fn post_failure(&self) {
+        self.state.store(false, Ordering::Release)
+    }
+    fn post_target(&self, target: u64) {
+        self.current.store(target, Ordering::Release)
+    }
+    /// WARNING: this is not atomic! only sensible to run a quiescent state
+    fn post_reset(&self) {
+        self.current.store(0, Ordering::Release);
+        self.state.store(true, Ordering::Release);
+    }
+    fn update_target(&self) -> u64 {
+        let mut current = self.current.load(Ordering::Acquire);
+        loop {
+            if current == 0 {
+                return 0;
+            }
+            match self.current.compare_exchange(
+                current,
+                current - 1,
+                Ordering::Release,
+                Ordering::Acquire,
+            ) {
+                Ok(last) => {
+                    return last;
+                }
+                Err(new) => {
+                    current = new;
+                }
+            }
+        }
+    }
+    fn load_okay(&self) -> bool {
+        self.state.load(Ordering::Acquire)
+    }
+}
+
+/*
+    task spec
+*/
+
+/// A threaded bombard task specification which drives a global pool of threads towards a common goal
+pub trait ThreadedBombardTask: Send + Sync + 'static {
+    /// The per-task worker that is initialized once in every thread (not to be confused with the actual thread worker!)
+    type Worker: Send + Sync;
+    /// The task that the [`ThreadedBombardTask::TaskWorker`] performs
+    type WorkerTask: Send + Sync;
+    type WorkerTaskSpec: Clone + Send + Sync + 'static;
+    /// Errors while running a task
+    type WorkerTaskError: Send + Sync;
+    /// Errors while initializing a task worker
+    type WorkerInitError: Send + Sync;
+    /// Initialize a task worker
+    fn worker_init(&self) -> Result<Self::Worker, Self::WorkerInitError>;
+    fn generate_task(spec: &Self::WorkerTaskSpec, current: u64) -> Self::WorkerTask;
+    /// Drive a single subtask
+    fn worker_drive_timed(
+        worker: &mut Self::Worker,
+        task: Self::WorkerTask,
+    ) -> Result<u128, Self::WorkerTaskError>;
+}
+
+/*
+    worker
+*/
+
+#[derive(Debug)]
+enum WorkerResult<Bt: ThreadedBombardTask> {
+    Completed(WorkerLocalStats),
+    Errored(Bt::WorkerTaskError),
+}
+
+#[derive(Debug)]
+struct WorkerLocalStats {
+    start: Instant,
+    elapsed: u128,
+    head: u128,
+    tail: u128,
+}
+
+impl WorkerLocalStats {
+    fn new(start: Instant, elapsed: u128, head: u128, tail: u128) -> Self {
+        Self {
+            start,
+            elapsed,
+            head,
+            tail,
+        }
+    }
+}
+
+#[derive(Debug)]
+enum WorkerTask<Bt: ThreadedBombardTask> {
+    Task(Bt::WorkerTaskSpec),
+    Exit,
+}
+
+#[derive(Debug)]
+struct Worker {
+    handle: JoinHandle<()>,
+}
+
+impl Worker {
+    fn start<Bt: ThreadedBombardTask>(
+        id: usize,
+        driver: Bt::Worker,
+        rx_work: Receiver<WorkerTask<Bt>>,
+        tx_res: Sender<WorkerResult<Bt>>,
+    ) -> Self {
+        Self {
+            handle: thread::Builder::new()
+                .name(format!("worker-{id}"))
+                .spawn(move || {
+                    let mut worker_driver = driver;
+                    'blocking_wait: loop {
+                        let task = match rx_work.recv().unwrap() {
+                            WorkerTask::Exit => return,
+                            WorkerTask::Task(spec) => spec,
+                        };
+                        // check global state
+                        let mut global_okay = GPState::get().load_okay();
+                        let mut global_position = GPState::get().update_target();
+                        // init local state
+                        let mut local_start = None;
+                        let mut local_elapsed = 0u128;
+                        let mut local_head = u128::MAX;
+                        let mut local_tail = 0;
+                        // bombard
+                        while (global_position != 0) & global_okay {
+                            let task = Bt::generate_task(&task, global_position);
+                            if local_start.is_none() {
+                                local_start = Some(Instant::now());
+                            }
+                            let this_elapsed =
+                                match Bt::worker_drive_timed(&mut worker_driver, task) {
+                                    Ok(elapsed) => elapsed,
+                                    Err(e) => {
+                                        GPState::get().post_failure();
+                                        tx_res.send(WorkerResult::Errored(e)).unwrap();
+                                        continue 'blocking_wait;
+                                    }
+                                };
+                            local_elapsed += this_elapsed;
+                            if this_elapsed < local_head {
+                                local_head = this_elapsed;
+                            }
+                            if this_elapsed > local_tail {
+                                local_tail = this_elapsed;
+                            }
+                            global_position = GPState::get().update_target();
+                            global_okay = GPState::get().load_okay();
+                        }
+                        if global_okay {
+                            // we're done
+                            tx_res
+                                .send(WorkerResult::Completed(WorkerLocalStats::new(
+                                    local_start.unwrap(),
+                                    local_elapsed,
+                                    local_head,
+                                    local_tail,
+                                )))
+                                .unwrap();
+                        }
+                    }
+                })
+                .expect("failed to start thread"),
+        }
+    }
+}
+
+/*
+    pool
+*/
+
+#[derive(Debug)]
+pub enum BombardError<Bt: ThreadedBombardTask> {
+    InitError(Bt::WorkerInitError),
+    WorkerTaskError(Bt::WorkerTaskError),
+    AllWorkersOffline,
+}
+
+impl<Bt: ThreadedBombardTask> fmt::Display for BombardError<Bt>
+where
+    Bt::WorkerInitError: fmt::Display,
+    Bt::WorkerTaskError: Display,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::AllWorkersOffline => write!(
+                f,
+                "bombard failed because all workers went offline indicating catastrophic failure"
+            ),
+            Self::WorkerTaskError(e) => write!(f, "worker task failed. {e}"),
+            Self::InitError(e) => write!(f, "worker init failed. {e}"),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct RuntimeStats {
+    pub qps: f64,
+    pub head: u128,
+    pub tail: u128,
+}
+
+#[derive(Debug)]
+pub struct BombardPool<Bt: ThreadedBombardTask> {
+    workers: Vec<(Worker, Sender<WorkerTask<Bt>>)>,
+    rx_res: Receiver<WorkerResult<Bt>>,
+    _config: Bt,
+}
+
+impl<Bt: ThreadedBombardTask> BombardPool<Bt> {
+    fn qps(query_count: usize, time_taken_in_nanos: u128) -> f64 {
+        const NANOS_PER_SECOND: u128 = 1_000_000_000;
+        let time_taken_in_nanos_f64 = time_taken_in_nanos as f64;
+        let query_count_f64 = query_count as f64;
+        (query_count_f64 / time_taken_in_nanos_f64) * NANOS_PER_SECOND as f64
+    }
+    pub fn new(size: usize, config: Bt) -> BombardResult<Self, Bt> {
+        assert_ne!(size, 0, "pool can't be empty");
+        let mut workers = Vec::with_capacity(size);
+        let (tx_res, rx_res) = unbounded();
+        for id in 0..size {
+            let (tx_work, rx_work) = unbounded();
+            let driver = config.worker_init().map_err(BombardError::InitError)?;
+            workers.push((Worker::start(id, driver, rx_work, tx_res.clone()), tx_work));
+        }
+        Ok(Self {
+            workers,
+            rx_res,
+            _config: config,
+        })
+    }
+    /// Bombard queries to the workers
+    pub fn blocking_bombard(
+        &mut self,
+        task_description: Bt::WorkerTaskSpec,
+        count: usize,
+    ) -> BombardResult<RuntimeStats, Bt> {
+        GPState::guard(|| {
+            GPState::get().post_target(count as _);
+            let mut global_start = None;
+            let mut global_stop = None;
+            let mut global_head = u128::MAX;
+            let mut global_tail = 0u128;
+            let messages: Vec<<Bt as ThreadedBombardTask>::WorkerTaskSpec> =
+                (0..self.workers.len())
+                    .into_iter()
+                    .map(|_| task_description.clone())
+                    .collect();
+            for ((_, sender), msg) in self.workers.iter().zip(messages) {
+                sender.send(WorkerTask::Task(msg)).unwrap();
+            }
+            // wait for all workers to complete
+            let mut received = 0;
+            while received != self.workers.len() {
+                let results = match self.rx_res.recv() {
+                    Err(_) => return Err(BombardError::AllWorkersOffline),
+                    Ok(r) => r,
+                };
+                let WorkerLocalStats {
+                    start: this_start,
+                    elapsed,
+                    head,
+                    tail,
+                } = match results {
+                    WorkerResult::Completed(r) => r,
+                    WorkerResult::Errored(e) => return Err(BombardError::WorkerTaskError(e)),
+                };
+                // update start if required
+                match global_start.as_mut() {
+                    None => {
+                        global_start = Some(this_start);
+                    }
+                    Some(start) => {
+                        if this_start < *start {
+                            *start = this_start;
+                        }
+                    }
+                }
+                let this_task_stopped_at =
+                    this_start + Duration::from_nanos(elapsed.try_into().unwrap());
+                match global_stop.as_mut() {
+                    None => {
+                        global_stop = Some(this_task_stopped_at);
+                    }
+                    Some(stop) => {
+                        if this_task_stopped_at > *stop {
+                            // this task stopped later than the previous one
+                            *stop = this_task_stopped_at;
+                        }
+                    }
+                }
+                if head < global_head {
+                    global_head = head;
+                }
+                if tail > global_tail {
+                    global_tail = tail;
+                }
+                received += 1;
+            }
+            // reset global pool state
+            GPState::get().post_reset();
+            // compute results
+            let global_elapsed = global_stop
+                .unwrap()
+                .duration_since(global_start.unwrap())
+                .as_nanos();
+            Ok(RuntimeStats {
+                qps: Self::qps(count, global_elapsed),
+                head: global_head,
+                tail: global_tail,
+            })
+        })
+    }
+}
+
+impl<Bt: ThreadedBombardTask> Drop for BombardPool<Bt> {
+    fn drop(&mut self) {
+        info!("taking all workers offline");
+        for (_, sender) in self.workers.iter() {
+            sender.send(WorkerTask::Exit).unwrap();
+        }
+        for (worker, _) in self.workers.drain(..) {
+            worker.handle.join().unwrap();
+        }
+        info!("all workers now offline");
+    }
+}

From a3d87e48506bfe151e2a11aa87e6f3b8f014529b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 20 Nov 2023 16:47:13 +0530
Subject: [PATCH 285/310] Make sure that model drops are synchronized across
 the system

---
 Cargo.lock                                  |   2 +-
 server/src/engine/core/dml/del.rs           |   2 -
 server/src/engine/core/dml/ins.rs           |   8 +-
 server/src/engine/core/dml/upd.rs           |   9 +-
 server/src/engine/core/index/row.rs         |   2 -
 server/src/engine/core/model/delta.rs       |  12 +-
 server/src/engine/core/model/mod.rs         |   8 +-
 server/src/engine/fractal/mgr.rs            |   6 +-
 server/src/engine/fractal/mod.rs            |  24 ++++
 server/src/engine/fractal/test_utils.rs     |  16 +++
 server/src/engine/storage/v1/tests/batch.rs |  22 ++--
 sky-bench/src/bench.rs                      | 115 +++++++++++++++-----
 sky-bench/src/runtime.rs                    |  11 +-
 13 files changed, 152 insertions(+), 85 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 4dc37767..b49ca2a3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1232,7 +1232,7 @@ dependencies = [
 [[package]]
 name = "skytable"
 version = "0.8.0"
-source = "git+https://github.com/skytable/client-rust.git?branch=octave#ce16be88204044cd8358a5aa48521bd30e60ae33"
+source = "git+https://github.com/skytable/client-rust.git?branch=octave#18aad43142fbe013ace64d92c3ffd71ab1394a18"
 dependencies = [
  "async-trait",
  "bb8",
diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs
index 0cea6aaa..119d1c92 100644
--- a/server/src/engine/core/dml/del.rs
+++ b/server/src/engine/core/dml/del.rs
@@ -44,7 +44,6 @@ pub fn delete_resp(
 pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) -> QueryResult<()> {
     core::with_model_for_data_update(global, delete.entity(), |model| {
         let g = sync::atm::cpin();
-        let schema_version = model.delta_state().schema_current_version();
         let delta_state = model.delta_state();
         // create new version
         let new_version = delta_state.create_new_data_delta_version();
@@ -57,7 +56,6 @@ pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) ->
                 let dp = delta_state.append_new_data_delta_with(
                     DataDeltaKind::Delete,
                     row.clone(),
-                    schema_version,
                     new_version,
                     &g,
                 );
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index 33242746..2ff9723b 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -57,13 +57,7 @@ pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> Quer
         let row = Row::new(pk, data, ds.schema_current_version(), new_version);
         if mdl.primary_index().__raw_index().mt_insert(row.clone(), &g) {
             // append delta for new version
-            let dp = ds.append_new_data_delta_with(
-                DataDeltaKind::Insert,
-                row,
-                ds.schema_current_version(),
-                new_version,
-                &g,
-            );
+            let dp = ds.append_new_data_delta_with(DataDeltaKind::Insert, row, new_version, &g);
             Ok(QueryExecMeta::new(dp))
         } else {
             Err(QueryError::QExecDmlDuplicate)
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index 296a227e..b3810a5d 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -372,13 +372,8 @@ pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) ->
             // update revised tag
             row_data_wl.set_txn_revised(new_version);
             // publish delta
-            let dp = ds.append_new_data_delta_with(
-                DataDeltaKind::Update,
-                row.clone(),
-                ds.schema_current_version(),
-                new_version,
-                &g,
-            );
+            let dp =
+                ds.append_new_data_delta_with(DataDeltaKind::Update, row.clone(), new_version, &g);
             ret = Ok(QueryExecMeta::new(dp))
         }
         ret
diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
index c44d8a4f..b667edd4 100644
--- a/server/src/engine/core/index/row.rs
+++ b/server/src/engine/core/index/row.rs
@@ -43,7 +43,6 @@ pub type DcFieldIndex = IndexST<Box<str>, Datacell, HasherNativeFx>;
 
 #[derive(Debug)]
 pub struct Row {
-    __genesis_schema_version: DeltaVersion,
     __pk: ManuallyDrop<PrimaryIndexKey>,
     __rc: RawRC<RwLock<RowData>>,
 }
@@ -121,7 +120,6 @@ impl Row {
         restore_txn_id: DeltaVersion,
     ) -> Self {
         Self {
-            __genesis_schema_version: schema_version,
             __pk: ManuallyDrop::new(pk),
             __rc: unsafe {
                 // UNSAFE(@ohsayan): we free this up later
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index a3760b24..a09c5514 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -193,11 +193,10 @@ impl DeltaState {
         &self,
         kind: DataDeltaKind,
         row: Row,
-        schema_version: DeltaVersion,
         data_version: DeltaVersion,
         g: &Guard,
     ) -> usize {
-        self.append_new_data_delta(DataDelta::new(schema_version, data_version, row, kind), g)
+        self.append_new_data_delta(DataDelta::new(data_version, row, kind), g)
     }
     pub fn append_new_data_delta(&self, delta: DataDelta, g: &Guard) -> usize {
         self.data_deltas.blocking_enqueue(delta, g);
@@ -348,21 +347,14 @@ impl<'a> SchemaDeltaIndexRGuard<'a> {
 
 #[derive(Debug, Clone)]
 pub struct DataDelta {
-    _schema_version: DeltaVersion,
     data_version: DeltaVersion,
     row: Row,
     change: DataDeltaKind,
 }
 
 impl DataDelta {
-    pub const fn new(
-        schema_version: DeltaVersion,
-        data_version: DeltaVersion,
-        row: Row,
-        change: DataDeltaKind,
-    ) -> Self {
+    pub const fn new(data_version: DeltaVersion, row: Row, change: DataDeltaKind) -> Self {
         Self {
-            _schema_version: schema_version,
             data_version,
             row,
             change,
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index e1b222e0..71203879 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -290,12 +290,12 @@ impl Model {
                 // commit txn
                 global.namespace_txn_driver().lock().try_commit(txn)?;
                 // request cleanup
-                global.taskmgr_post_standard_priority(Task::new(GenericTask::delete_model_dir(
-                    &space_name,
+                global.purge_model_driver(
+                    space_name.as_str(),
                     space.get_uuid(),
-                    &model_name,
+                    model_name.as_str(),
                     model.get_uuid(),
-                )));
+                );
             }
             // update global state
             let _ = models_idx.remove(&EntityIDRef::new(&space_name, &model_name));
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 2abc9b2d..3440dbf7 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -340,13 +340,13 @@ impl FractalMgr {
                 _ = sigterm.recv() => {
                     info!("flp: finishing any pending maintenance tasks");
                     let global = global.clone();
-                    tokio::task::spawn_blocking(|| self.general_executor_model_maintenance(global)).await.unwrap();
+                    tokio::task::spawn_blocking(|| self.general_executor(global)).await.unwrap();
                     info!("flp: exited executor service");
                     break;
                 },
                 _ = tokio::time::sleep(dur) => {
                     let global = global.clone();
-                    tokio::task::spawn_blocking(|| self.general_executor_model_maintenance(global)).await.unwrap()
+                    tokio::task::spawn_blocking(|| self.general_executor(global)).await.unwrap()
                 }
                 task = lpq.recv() => {
                     let Task { threshold, task } = match task {
@@ -377,7 +377,7 @@ impl FractalMgr {
             }
         }
     }
-    fn general_executor_model_maintenance(&'static self, global: super::Global) {
+    fn general_executor(&'static self, global: super::Global) {
         let mdl_drivers = global.get_state().get_mdl_drivers().read();
         for (model_id, driver) in mdl_drivers.iter() {
             let mut observed_len = 0;
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index 1645f2e6..168593cb 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -120,6 +120,13 @@ pub trait GlobalInstanceLike {
         model_name: &str,
         model_uuid: Uuid,
     ) -> RuntimeResult<()>;
+    fn purge_model_driver(
+        &self,
+        space_name: &str,
+        space_uuid: Uuid,
+        model_name: &str,
+        model_uuid: Uuid,
+    );
     // taskmgr
     fn taskmgr_post_high_priority(&self, task: Task<CriticalTask>);
     fn taskmgr_post_standard_priority(&self, task: Task<GenericTask>);
@@ -174,6 +181,23 @@ impl GlobalInstanceLike for Global {
         &self.get_state().config
     }
     // model
+    fn purge_model_driver(
+        &self,
+        space_name: &str,
+        space_uuid: Uuid,
+        model_name: &str,
+        model_uuid: Uuid,
+    ) {
+        let id = ModelUniqueID::new(space_name, model_name, model_uuid);
+        self.get_state()
+            .mdl_driver
+            .write()
+            .remove(&id)
+            .expect("tried to remove non existent driver");
+        self.taskmgr_post_standard_priority(Task::new(GenericTask::delete_model_dir(
+            space_name, space_uuid, model_name, model_uuid,
+        )));
+    }
     fn initialize_model_driver(
         &self,
         space_name: &str,
diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
index 504a5773..91e28852 100644
--- a/server/src/engine/fractal/test_utils.rs
+++ b/server/src/engine/fractal/test_utils.rs
@@ -120,6 +120,22 @@ impl<Fs: RawFSInterface> GlobalInstanceLike for TestGlobal<Fs> {
     fn sys_store(&self) -> &SystemStore<Fs> {
         &self.sys_cfg
     }
+    fn purge_model_driver(
+        &self,
+        space_name: &str,
+        space_uuid: Uuid,
+        model_name: &str,
+        model_uuid: Uuid,
+    ) {
+        let id = ModelUniqueID::new(space_name, model_name, model_uuid);
+        self.model_drivers
+            .write()
+            .remove(&id)
+            .expect("tried to remove non-existent model");
+        self.taskmgr_post_standard_priority(Task::new(GenericTask::delete_model_dir(
+            space_name, space_uuid, model_name, model_uuid,
+        )));
+    }
     fn initialize_model_driver(
         &self,
         space_name: &str,
diff --git a/server/src/engine/storage/v1/tests/batch.rs b/server/src/engine/storage/v1/tests/batch.rs
index a0e6ad1e..0cb4d24c 100644
--- a/server/src/engine/storage/v1/tests/batch.rs
+++ b/server/src/engine/storage/v1/tests/batch.rs
@@ -81,7 +81,6 @@ fn new_delta(
     change: DataDeltaKind,
 ) -> DataDelta {
     new_delta_with_row(
-        schema,
         txnid,
         Row::new(
             pkey(pk),
@@ -93,13 +92,8 @@ fn new_delta(
     )
 }
 
-fn new_delta_with_row(schema: u64, txnid: u64, row: Row, change: DataDeltaKind) -> DataDelta {
-    DataDelta::new(
-        DeltaVersion::__new(schema),
-        DeltaVersion::__new(txnid),
-        row,
-        change,
-    )
+fn new_delta_with_row(txnid: u64, row: Row, change: DataDeltaKind) -> DataDelta {
+    DataDelta::new(DeltaVersion::__new(txnid), row, change)
 }
 
 fn flush_deltas_and_re_read<const N: usize>(
@@ -247,7 +241,7 @@ fn skewed_delta() {
     // prepare deltas
     let deltas = [
         // insert catname: Schrödinger's cat, is_good: true
-        new_delta_with_row(0, 0, row.clone(), DataDeltaKind::Insert),
+        new_delta_with_row(0, row.clone(), DataDeltaKind::Insert),
         // insert catname: good cat, is_good: true, magical: false
         new_delta(
             0,
@@ -265,7 +259,7 @@ fn skewed_delta() {
             DataDeltaKind::Insert,
         ),
         // update catname: Schrödinger's cat, is_good: true, magical: true
-        new_delta_with_row(0, 3, row.clone(), DataDeltaKind::Update),
+        new_delta_with_row(3, row.clone(), DataDeltaKind::Update),
     ];
     let batch = flush_deltas_and_re_read(&mdl, deltas, "skewed_delta.db-btlog");
     assert_eq!(
@@ -352,10 +346,10 @@ fn skewed_shuffled_persist_restore() {
             into_dict!("password" => "pwd456789"),
             DataDeltaKind::Insert,
         ),
-        new_delta_with_row(0, 4, mongobongo.clone(), DataDeltaKind::Insert),
-        new_delta_with_row(0, 5, rds.clone(), DataDeltaKind::Insert),
-        new_delta_with_row(0, 6, mongobongo.clone(), DataDeltaKind::Delete),
-        new_delta_with_row(0, 7, rds.clone(), DataDeltaKind::Delete),
+        new_delta_with_row(4, mongobongo.clone(), DataDeltaKind::Insert),
+        new_delta_with_row(5, rds.clone(), DataDeltaKind::Insert),
+        new_delta_with_row(6, mongobongo.clone(), DataDeltaKind::Delete),
+        new_delta_with_row(7, rds.clone(), DataDeltaKind::Delete),
     ];
     for i in 0..deltas.len() {
         // prepare pretest
diff --git a/sky-bench/src/bench.rs b/sky-bench/src/bench.rs
index 7325734c..dc725c66 100644
--- a/sky-bench/src/bench.rs
+++ b/sky-bench/src/bench.rs
@@ -91,22 +91,28 @@ impl BombardTaskSpec {
         let mut q = query!(&self.base_query);
         let resp = match self.kind {
             BombardTaskKind::Insert(second_column) => {
-                q.push_param(format!("{:0>width$}", current, width = self.pk_len));
+                self.push_pk(&mut q, current);
                 q.push_param(second_column);
                 Response::Empty
             }
             BombardTaskKind::Update => {
                 q.push_param(1u64);
-                q.push_param(format!("{:0>width$}", current, width = self.pk_len));
+                self.push_pk(&mut q, current);
                 Response::Empty
             }
             BombardTaskKind::Delete => {
-                q.push_param(format!("{:0>width$}", current, width = self.pk_len));
+                self.push_pk(&mut q, current);
                 Response::Empty
             }
         };
         (q, resp)
     }
+    fn push_pk(&self, q: &mut Query, current: u64) {
+        q.push_param(self.get_primary_key(current));
+    }
+    fn get_primary_key(&self, current: u64) -> String {
+        format!("{:0>width$}", current, width = self.pk_len)
+    }
 }
 
 /// Errors while running a bombard
@@ -229,36 +235,89 @@ fn print_table(data: Vec<(&'static str, RuntimeStats)>) {
     bench runner
 */
 
+struct BenchItem {
+    name: &'static str,
+    spec: BombardTaskSpec,
+    count: usize,
+}
+
+impl BenchItem {
+    fn new(name: &'static str, spec: BombardTaskSpec, count: usize) -> Self {
+        Self { name, spec, count }
+    }
+    fn print_log_start(&self) {
+        info!(
+            "benchmarking `{}`. average payload size = {} bytes. queries = {}",
+            self.name,
+            self.spec.generate(0).0.debug_encode_packet().len(),
+            self.count
+        )
+    }
+    fn run(self, pool: &mut BombardPool<BombardTask>) -> BenchResult<RuntimeStats> {
+        pool.blocking_bombard(self.spec, self.count)
+            .map_err(From::from)
+    }
+}
+
 fn bench_internal(
     config: BombardTask,
     bench: BenchConfig,
 ) -> BenchResult<Vec<(&'static str, RuntimeStats)>> {
-    let mut ret = vec![];
     // initialize pool
-    info!("initializing connection pool");
-    let mut pool = BombardPool::new(bench.threads, config)?;
-    // bench INSERT
-    info!("benchmarking `INSERT`");
-    let insert = BombardTaskSpec::insert("insert into bench.bench(?, ?)".into(), bench.key_size, 0);
-    let insert_stats = pool.blocking_bombard(insert, bench.query_count)?;
-    ret.push(("INSERT", insert_stats));
-    // bench UPDATE
-    info!("benchmarking `UPDATE`");
-    let update = BombardTaskSpec::update(
-        "update bench.bench set pw += ? where un = ?".into(),
-        bench.key_size,
+    info!(
+        "initializing connections. threads={}, primary key size ={} bytes",
+        bench.threads, bench.key_size
     );
-    let update_stats = pool.blocking_bombard(update, bench.query_count)?;
-    ret.push(("UPDATE", update_stats));
-    // bench DELETE
-    info!("benchmarking `DELETE`");
-    let delete = BombardTaskSpec::delete(
-        "delete from bench.bench where un = ?".into(),
-        bench.key_size,
+    let mut pool = BombardPool::new(bench.threads, config)?;
+    // prepare benches
+    let benches = vec![
+        BenchItem::new(
+            "INSERT",
+            BombardTaskSpec::insert("insert into bench.bench(?, ?)".into(), bench.key_size, 0),
+            bench.query_count,
+        ),
+        BenchItem::new(
+            "UPDATE",
+            BombardTaskSpec::update(
+                "update bench.bench set pw += ? where un = ?".into(),
+                bench.key_size,
+            ),
+            bench.query_count,
+        ),
+        BenchItem::new(
+            "DELETE",
+            BombardTaskSpec::delete(
+                "delete from bench.bench where un = ?".into(),
+                bench.key_size,
+            ),
+            bench.query_count,
+        ),
+    ];
+    // bench
+    let total_queries = bench.query_count as u64 * benches.len() as u64;
+    let mut results = vec![];
+    for task in benches {
+        let name = task.name;
+        task.print_log_start();
+        let this_result = task.run(&mut pool)?;
+        results.push((name, this_result));
+    }
+    info!(
+        "benchmark complete. finished executing {} queries",
+        fmt_u64(total_queries)
     );
-    let delete_stats = pool.blocking_bombard(delete, bench.query_count)?;
-    ret.push(("DELETE", delete_stats));
-    info!("completed benchmarks. closing pool");
-    drop(pool);
-    Ok(ret)
+    Ok(results)
+}
+
+fn fmt_u64(n: u64) -> String {
+    let num_str = n.to_string();
+    let mut result = String::new();
+    let chars_rev: Vec<_> = num_str.chars().rev().collect();
+    for (i, ch) in chars_rev.iter().enumerate() {
+        if i % 3 == 0 && i != 0 {
+            result.push(',');
+        }
+        result.push(*ch);
+    }
+    result.chars().rev().collect()
 }
diff --git a/sky-bench/src/runtime.rs b/sky-bench/src/runtime.rs
index 4386e1a9..5675c11e 100644
--- a/sky-bench/src/runtime.rs
+++ b/sky-bench/src/runtime.rs
@@ -320,13 +320,10 @@ impl<Bt: ThreadedBombardTask> BombardPool<Bt> {
             let mut global_stop = None;
             let mut global_head = u128::MAX;
             let mut global_tail = 0u128;
-            let messages: Vec<<Bt as ThreadedBombardTask>::WorkerTaskSpec> =
-                (0..self.workers.len())
-                    .into_iter()
-                    .map(|_| task_description.clone())
-                    .collect();
-            for ((_, sender), msg) in self.workers.iter().zip(messages) {
-                sender.send(WorkerTask::Task(msg)).unwrap();
+            for (_, sender) in self.workers.iter() {
+                sender
+                    .send(WorkerTask::Task(task_description.clone()))
+                    .unwrap();
             }
             // wait for all workers to complete
             let mut received = 0;

From f5866bc22e390e36c570f2ac0ee219d6dadc3d6b Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 22 Nov 2023 01:36:29 +0530
Subject: [PATCH 286/310] Add new fury benchmark engine

---
 Cargo.lock                      |   3 +-
 sky-bench/Cargo.toml            |   1 +
 sky-bench/help_text/help        |  22 +-
 sky-bench/src/args.rs           |  69 +++++-
 sky-bench/src/bench.rs          | 106 ++++++---
 sky-bench/src/error.rs          |  19 +-
 sky-bench/src/runtime.rs        | 366 ++-----------------------------
 sky-bench/src/runtime/fury.rs   | 315 ++++++++++++++++++++++++++
 sky-bench/src/runtime/rookie.rs | 378 ++++++++++++++++++++++++++++++++
 9 files changed, 880 insertions(+), 399 deletions(-)
 create mode 100644 sky-bench/src/runtime/fury.rs
 create mode 100644 sky-bench/src/runtime/rookie.rs

diff --git a/Cargo.lock b/Cargo.lock
index b49ca2a3..1dfd1815 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1182,6 +1182,7 @@ dependencies = [
  "log",
  "num_cpus",
  "skytable",
+ "tokio",
 ]
 
 [[package]]
@@ -1232,7 +1233,7 @@ dependencies = [
 [[package]]
 name = "skytable"
 version = "0.8.0"
-source = "git+https://github.com/skytable/client-rust.git?branch=octave#18aad43142fbe013ace64d92c3ffd71ab1394a18"
+source = "git+https://github.com/skytable/client-rust.git?branch=octave#691e15578c1a09a5d2b6b85c8752e6eee31fbda2"
 dependencies = [
  "async-trait",
  "bb8",
diff --git a/sky-bench/Cargo.toml b/sky-bench/Cargo.toml
index b04e8c17..04a857a8 100644
--- a/sky-bench/Cargo.toml
+++ b/sky-bench/Cargo.toml
@@ -15,3 +15,4 @@ crossbeam-channel = "0.5.8"
 num_cpus = "1.16.0"
 env_logger = "0.10.1"
 log = "0.4.20"
+tokio = { version = "1.34.0", features = ["full"] }
diff --git a/sky-bench/help_text/help b/sky-bench/help_text/help
index 508f43f3..6af1bf3a 100644
--- a/sky-bench/help_text/help
+++ b/sky-bench/help_text/help
@@ -13,13 +13,21 @@ REQUIRED OPTIONS:
     --password  Provide the password
 
 OPTIONS:
-    --endpoint  Set the endpoint (defaults to tcp@127.0.0.1:2003)
-    --threads   Set the number of threads to be used (defaults to number of physical CPU cores)
-    --keysize   Set the default primary key size. defaults to 7
-    --rowcount  Set the number of rows to be manipulated for the benchmark
+    --endpoint    Set the endpoint (defaults to tcp@127.0.0.1:2003)
+    --threads     Set the number of threads to be used (defaults to logical
+                  CPU count)
+    --connections Set the number of connections. Defaults to twice the logical CPU
+                  count but is only supported by the `fury` engine
+    --keysize     Set the default primary key size. defaults to 7
+    --rowcount    Set the number of rows to be manipulated for the benchmark
+                  Defaults to 1,000,000 rows.
+    --engine      Set the engine for benchmarking. `rookie` is the stable engine
+                  and `fury` is the experimental engine
 
 NOTES:
-    - The user for auth will be 'root' since only 'root' accounts allow the creation and deletion of spaces and models
+    - The user for auth will be 'root' since only 'root' accounts allow the
+      creation and deletion of spaces and models
     - A space called 'benchmark_[random 8B string]' will be created
-    - A model called 'benchmark_[random 8B string]' will be created in the space created above. The created model has the structure {name: string, pass: string}
-    - The model and space will be removed once the benchmark is complete
\ No newline at end of file
+    - A model called 'benchmark_[random 8B string]' will be created in the space
+      created above. The created model has the structure {name: string, pass: string}
+    - The model and space will be removed once the benchmark is complete
diff --git a/sky-bench/src/args.rs b/sky-bench/src/args.rs
index 31828ce2..84ace127 100644
--- a/sky-bench/src/args.rs
+++ b/sky-bench/src/args.rs
@@ -44,6 +44,12 @@ pub enum Task {
     BenchConfig(BenchConfig),
 }
 
+#[derive(Debug, PartialEq)]
+pub enum BenchEngine {
+    Rookie,
+    Fury,
+}
+
 #[derive(Debug)]
 pub struct BenchConfig {
     pub host: String,
@@ -52,6 +58,8 @@ pub struct BenchConfig {
     pub threads: usize,
     pub key_size: usize,
     pub query_count: usize,
+    pub engine: BenchEngine,
+    pub connections: usize,
 }
 
 impl BenchConfig {
@@ -62,6 +70,8 @@ impl BenchConfig {
         threads: usize,
         key_size: usize,
         query_count: usize,
+        engine: BenchEngine,
+        connections: usize,
     ) -> Self {
         Self {
             host,
@@ -70,6 +80,8 @@ impl BenchConfig {
             threads,
             key_size,
             query_count,
+            engine,
+            connections,
         }
     }
 }
@@ -139,7 +151,7 @@ pub fn parse() -> BenchResult<Task> {
     };
     // threads
     let thread_count = match args.remove("--threads") {
-        None => num_cpus::get_physical(),
+        None => num_cpus::get(),
         Some(tc) => match tc.parse() {
             Ok(tc) if tc > 0 => tc,
             Err(_) | Ok(_) => {
@@ -169,12 +181,51 @@ pub fn parse() -> BenchResult<Task> {
             Err(_) | Ok(_) => return Err(BenchError::ArgsErr(format!("incorrect value for `--keysize`. must be set to a value that can be used to generate atleast {query_count} unique primary keys"))),
         }
     };
-    Ok(Task::BenchConfig(BenchConfig::new(
-        host,
-        port,
-        passsword,
-        thread_count,
-        key_size,
-        query_count,
-    )))
+    let engine = match args.remove("--engine") {
+        None => {
+            warn!("engine unspecified. choosing 'fury'");
+            BenchEngine::Fury
+        }
+        Some(engine) => match engine.as_str() {
+            "rookie" => BenchEngine::Rookie,
+            "fury" => BenchEngine::Fury,
+            _ => {
+                return Err(BenchError::ArgsErr(format!(
+                    "bad value for `--engine`. got `{engine}` but expected warp or rookie"
+                )))
+            }
+        },
+    };
+    let connections = match args.remove("--connections") {
+        None => num_cpus::get() * 2,
+        Some(c) => match c.parse::<usize>() {
+            Ok(s) if s != 0 => {
+                if engine == BenchEngine::Rookie {
+                    return Err(BenchError::ArgsErr(format!(
+                        "the 'rookie' engine does not support explicit connection count. the number of threads is the connection count"
+                    )));
+                }
+                s
+            }
+            _ => {
+                return Err(BenchError::ArgsErr(format!(
+                    "bad value for `--connections`. must be a nonzero value"
+                )))
+            }
+        },
+    };
+    if args.is_empty() {
+        Ok(Task::BenchConfig(BenchConfig::new(
+            host,
+            port,
+            passsword,
+            thread_count,
+            key_size,
+            query_count,
+            engine,
+            connections,
+        )))
+    } else {
+        Err(BenchError::ArgsErr(format!("unrecognized arguments")))
+    }
 }
diff --git a/sky-bench/src/bench.rs b/sky-bench/src/bench.rs
index dc725c66..9d93f062 100644
--- a/sky-bench/src/bench.rs
+++ b/sky-bench/src/bench.rs
@@ -24,11 +24,13 @@
  *
 */
 
+use crate::args::BenchEngine;
+
 use {
     crate::{
         args::BenchConfig,
         error::{self, BenchResult},
-        runtime::{BombardPool, RuntimeStats, ThreadedBombardTask},
+        runtime::{fury, rookie, RuntimeStats},
     },
     skytable::{error::Error, query, response::Response, Config, Connection, Query},
     std::{fmt, time::Instant},
@@ -87,7 +89,7 @@ impl BombardTaskSpec {
             pk_len,
         }
     }
-    fn generate(&self, current: u64) -> (Query, Response) {
+    pub fn generate(&self, current: u64) -> (Query, Response) {
         let mut q = query!(&self.base_query);
         let resp = match self.kind {
             BombardTaskKind::Insert(second_column) => {
@@ -137,7 +139,7 @@ impl From<Error> for BombardTaskError {
     }
 }
 
-impl ThreadedBombardTask for BombardTask {
+impl rookie::ThreadedBombardTask for BombardTask {
     type Worker = Connection;
     type WorkerTask = (Query, Response);
     type WorkerTaskSpec = BombardTaskSpec;
@@ -179,7 +181,11 @@ pub fn run(bench: BenchConfig) -> error::BenchResult<()> {
     let mut main_thread_db = bench_config.config.connect()?;
     main_thread_db.query_parse::<()>(&query!("create space bench"))?;
     main_thread_db.query_parse::<()>(&query!("create model bench.bench(un: string, pw: uint8)"))?;
-    let stats = match bench_internal(bench_config, bench) {
+    let stats = match bench.engine {
+        BenchEngine::Rookie => bench_rookie(bench_config, bench),
+        BenchEngine::Fury => bench_fury(bench),
+    };
+    let stats = match stats {
         Ok(stats) => stats,
         Err(e) => {
             error!("benchmarking failed. attempting to clean up");
@@ -253,24 +259,19 @@ impl BenchItem {
             self.count
         )
     }
-    fn run(self, pool: &mut BombardPool<BombardTask>) -> BenchResult<RuntimeStats> {
+    fn run(self, pool: &mut rookie::BombardPool<BombardTask>) -> BenchResult<RuntimeStats> {
         pool.blocking_bombard(self.spec, self.count)
             .map_err(From::from)
     }
+    async fn run_async(self, pool: &mut fury::Fury) -> BenchResult<RuntimeStats> {
+        pool.bombard(self.count, self.spec)
+            .await
+            .map_err(From::from)
+    }
 }
 
-fn bench_internal(
-    config: BombardTask,
-    bench: BenchConfig,
-) -> BenchResult<Vec<(&'static str, RuntimeStats)>> {
-    // initialize pool
-    info!(
-        "initializing connections. threads={}, primary key size ={} bytes",
-        bench.threads, bench.key_size
-    );
-    let mut pool = BombardPool::new(bench.threads, config)?;
-    // prepare benches
-    let benches = vec![
+fn prepare_bench_spec(bench: &BenchConfig) -> Vec<BenchItem> {
+    vec![
         BenchItem::new(
             "INSERT",
             BombardTaskSpec::insert("insert into bench.bench(?, ?)".into(), bench.key_size, 0),
@@ -292,7 +293,34 @@ fn bench_internal(
             ),
             bench.query_count,
         ),
-    ];
+    ]
+}
+
+fn fmt_u64(n: u64) -> String {
+    let num_str = n.to_string();
+    let mut result = String::new();
+    let chars_rev: Vec<_> = num_str.chars().rev().collect();
+    for (i, ch) in chars_rev.iter().enumerate() {
+        if i % 3 == 0 && i != 0 {
+            result.push(',');
+        }
+        result.push(*ch);
+    }
+    result.chars().rev().collect()
+}
+
+fn bench_rookie(
+    task: BombardTask,
+    bench: BenchConfig,
+) -> BenchResult<Vec<(&'static str, RuntimeStats)>> {
+    // initialize pool
+    info!(
+        "initializing connections. engine=rookie, threads={}, primary key size ={} bytes",
+        bench.threads, bench.key_size
+    );
+    let mut pool = rookie::BombardPool::new(bench.threads, task)?;
+    // prepare benches
+    let benches = prepare_bench_spec(&bench);
     // bench
     let total_queries = bench.query_count as u64 * benches.len() as u64;
     let mut results = vec![];
@@ -309,15 +337,37 @@ fn bench_internal(
     Ok(results)
 }
 
-fn fmt_u64(n: u64) -> String {
-    let num_str = n.to_string();
-    let mut result = String::new();
-    let chars_rev: Vec<_> = num_str.chars().rev().collect();
-    for (i, ch) in chars_rev.iter().enumerate() {
-        if i % 3 == 0 && i != 0 {
-            result.push(',');
+fn bench_fury(bench: BenchConfig) -> BenchResult<Vec<(&'static str, RuntimeStats)>> {
+    let rt = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(bench.threads)
+        .enable_all()
+        .build()
+        .unwrap();
+    rt.block_on(async move {
+        info!(
+            "initializing connections. engine=fury, threads={}, connections={}, primary key size ={} bytes",
+            bench.threads, bench.connections, bench.key_size
+        );
+        let mut pool = fury::Fury::new(
+            bench.connections,
+            Config::new(&bench.host, bench.port, "root", &bench.root_pass),
+        )
+        .await?;
+        // prepare benches
+        let benches = prepare_bench_spec(&bench);
+        // bench
+        let total_queries = bench.query_count as u64 * benches.len() as u64;
+        let mut results = vec![];
+        for task in benches {
+            let name = task.name;
+            task.print_log_start();
+            let this_result = task.run_async(&mut pool).await?;
+            results.push((name, this_result));
         }
-        result.push(*ch);
-    }
-    result.chars().rev().collect()
+        info!(
+            "benchmark complete. finished executing {} queries",
+            fmt_u64(total_queries)
+        );
+        Ok(results)
+    })
 }
diff --git a/sky-bench/src/error.rs b/sky-bench/src/error.rs
index 6cf5caf9..1f012149 100644
--- a/sky-bench/src/error.rs
+++ b/sky-bench/src/error.rs
@@ -25,7 +25,10 @@
 */
 
 use {
-    crate::{bench::BombardTask, runtime::BombardError},
+    crate::{
+        bench::BombardTask,
+        runtime::{fury, rookie::BombardError},
+    },
     core::fmt,
     skytable::error::Error,
 };
@@ -35,10 +38,17 @@ pub type BenchResult<T> = Result<T, BenchError>;
 #[derive(Debug)]
 pub enum BenchError {
     ArgsErr(String),
-    BenchBombardError(BombardError<BombardTask>),
+    RookieEngineError(BombardError<BombardTask>),
+    FuryEngineError(fury::FuryError),
     DirectDbError(Error),
 }
 
+impl From<fury::FuryError> for BenchError {
+    fn from(e: fury::FuryError) -> Self {
+        Self::FuryEngineError(e)
+    }
+}
+
 impl From<libsky::ArgParseError> for BenchError {
     fn from(e: libsky::ArgParseError) -> Self {
         match e {
@@ -60,7 +70,7 @@ impl From<Error> for BenchError {
 
 impl From<BombardError<BombardTask>> for BenchError {
     fn from(e: BombardError<BombardTask>) -> Self {
-        Self::BenchBombardError(e)
+        Self::RookieEngineError(e)
     }
 }
 
@@ -69,7 +79,8 @@ impl fmt::Display for BenchError {
         match self {
             Self::ArgsErr(e) => write!(f, "args error: {e}"),
             Self::DirectDbError(e) => write!(f, "direct operation on db failed. {e}"),
-            Self::BenchBombardError(e) => write!(f, "benchmark failed: {e}"),
+            Self::RookieEngineError(e) => write!(f, "benchmark failed (rookie engine): {e}"),
+            Self::FuryEngineError(e) => write!(f, "benchmark failed (fury engine): {e}"),
         }
     }
 }
diff --git a/sky-bench/src/runtime.rs b/sky-bench/src/runtime.rs
index 5675c11e..2077d6a7 100644
--- a/sky-bench/src/runtime.rs
+++ b/sky-bench/src/runtime.rs
@@ -24,126 +24,29 @@
  *
 */
 
-use {
-    crossbeam_channel::{unbounded, Receiver, Sender},
-    std::{
-        fmt::{self, Display},
-        sync::atomic::{AtomicBool, AtomicU64, Ordering},
-        thread::{self, JoinHandle},
-        time::{Duration, Instant},
-    },
-};
+pub mod fury;
+pub mod rookie;
 
-pub type BombardResult<T, Bt> = Result<T, BombardError<Bt>>;
+use std::time::Instant;
 
-/*
-    state mgmt
-*/
-
-#[derive(Debug)]
-/// The pool state. Be warned **ONLY ONE POOL AT A TIME!**
-struct GPState {
-    current: AtomicU64,
-    state: AtomicBool,
-    occupied: AtomicBool,
-}
-
-impl GPState {
-    #[inline(always)]
-    fn get() -> &'static Self {
-        static STATE: GPState = GPState::zero();
-        &STATE
-    }
-    const fn zero() -> Self {
-        Self {
-            current: AtomicU64::new(0),
-            state: AtomicBool::new(true),
-            occupied: AtomicBool::new(false),
-        }
-    }
-    fn occupy(&self) {
-        assert!(!self.occupied.swap(true, Ordering::Release));
-    }
-    fn vacate(&self) {
-        assert!(self.occupied.swap(false, Ordering::Release));
-    }
-    fn guard<T>(f: impl FnOnce() -> T) -> T {
-        let slf = Self::get();
-        slf.occupy();
-        let ret = f();
-        slf.vacate();
-        ret
-    }
-    fn post_failure(&self) {
-        self.state.store(false, Ordering::Release)
-    }
-    fn post_target(&self, target: u64) {
-        self.current.store(target, Ordering::Release)
-    }
-    /// WARNING: this is not atomic! only sensible to run a quiescent state
-    fn post_reset(&self) {
-        self.current.store(0, Ordering::Release);
-        self.state.store(true, Ordering::Release);
-    }
-    fn update_target(&self) -> u64 {
-        let mut current = self.current.load(Ordering::Acquire);
-        loop {
-            if current == 0 {
-                return 0;
-            }
-            match self.current.compare_exchange(
-                current,
-                current - 1,
-                Ordering::Release,
-                Ordering::Acquire,
-            ) {
-                Ok(last) => {
-                    return last;
-                }
-                Err(new) => {
-                    current = new;
-                }
-            }
-        }
-    }
-    fn load_okay(&self) -> bool {
-        self.state.load(Ordering::Acquire)
-    }
+fn qps(query_count: usize, time_taken_in_nanos: u128) -> f64 {
+    const NANOS_PER_SECOND: u128 = 1_000_000_000;
+    let time_taken_in_nanos_f64 = time_taken_in_nanos as f64;
+    let query_count_f64 = query_count as f64;
+    (query_count_f64 / time_taken_in_nanos_f64) * NANOS_PER_SECOND as f64
 }
 
-/*
-    task spec
-*/
-
-/// A threaded bombard task specification which drives a global pool of threads towards a common goal
-pub trait ThreadedBombardTask: Send + Sync + 'static {
-    /// The per-task worker that is initialized once in every thread (not to be confused with the actual thread worker!)
-    type Worker: Send + Sync;
-    /// The task that the [`ThreadedBombardTask::TaskWorker`] performs
-    type WorkerTask: Send + Sync;
-    type WorkerTaskSpec: Clone + Send + Sync + 'static;
-    /// Errors while running a task
-    type WorkerTaskError: Send + Sync;
-    /// Errors while initializing a task worker
-    type WorkerInitError: Send + Sync;
-    /// Initialize a task worker
-    fn worker_init(&self) -> Result<Self::Worker, Self::WorkerInitError>;
-    fn generate_task(spec: &Self::WorkerTaskSpec, current: u64) -> Self::WorkerTask;
-    /// Drive a single subtask
-    fn worker_drive_timed(
-        worker: &mut Self::Worker,
-        task: Self::WorkerTask,
-    ) -> Result<u128, Self::WorkerTaskError>;
+#[derive(Debug, Clone)]
+pub(self) enum WorkerTask<T> {
+    Task(T),
+    Exit,
 }
 
-/*
-    worker
-*/
-
 #[derive(Debug)]
-enum WorkerResult<Bt: ThreadedBombardTask> {
-    Completed(WorkerLocalStats),
-    Errored(Bt::WorkerTaskError),
+pub struct RuntimeStats {
+    pub qps: f64,
+    pub head: u128,
+    pub tail: u128,
 }
 
 #[derive(Debug)]
@@ -164,240 +67,3 @@ impl WorkerLocalStats {
         }
     }
 }
-
-#[derive(Debug)]
-enum WorkerTask<Bt: ThreadedBombardTask> {
-    Task(Bt::WorkerTaskSpec),
-    Exit,
-}
-
-#[derive(Debug)]
-struct Worker {
-    handle: JoinHandle<()>,
-}
-
-impl Worker {
-    fn start<Bt: ThreadedBombardTask>(
-        id: usize,
-        driver: Bt::Worker,
-        rx_work: Receiver<WorkerTask<Bt>>,
-        tx_res: Sender<WorkerResult<Bt>>,
-    ) -> Self {
-        Self {
-            handle: thread::Builder::new()
-                .name(format!("worker-{id}"))
-                .spawn(move || {
-                    let mut worker_driver = driver;
-                    'blocking_wait: loop {
-                        let task = match rx_work.recv().unwrap() {
-                            WorkerTask::Exit => return,
-                            WorkerTask::Task(spec) => spec,
-                        };
-                        // check global state
-                        let mut global_okay = GPState::get().load_okay();
-                        let mut global_position = GPState::get().update_target();
-                        // init local state
-                        let mut local_start = None;
-                        let mut local_elapsed = 0u128;
-                        let mut local_head = u128::MAX;
-                        let mut local_tail = 0;
-                        // bombard
-                        while (global_position != 0) & global_okay {
-                            let task = Bt::generate_task(&task, global_position);
-                            if local_start.is_none() {
-                                local_start = Some(Instant::now());
-                            }
-                            let this_elapsed =
-                                match Bt::worker_drive_timed(&mut worker_driver, task) {
-                                    Ok(elapsed) => elapsed,
-                                    Err(e) => {
-                                        GPState::get().post_failure();
-                                        tx_res.send(WorkerResult::Errored(e)).unwrap();
-                                        continue 'blocking_wait;
-                                    }
-                                };
-                            local_elapsed += this_elapsed;
-                            if this_elapsed < local_head {
-                                local_head = this_elapsed;
-                            }
-                            if this_elapsed > local_tail {
-                                local_tail = this_elapsed;
-                            }
-                            global_position = GPState::get().update_target();
-                            global_okay = GPState::get().load_okay();
-                        }
-                        if global_okay {
-                            // we're done
-                            tx_res
-                                .send(WorkerResult::Completed(WorkerLocalStats::new(
-                                    local_start.unwrap(),
-                                    local_elapsed,
-                                    local_head,
-                                    local_tail,
-                                )))
-                                .unwrap();
-                        }
-                    }
-                })
-                .expect("failed to start thread"),
-        }
-    }
-}
-
-/*
-    pool
-*/
-
-#[derive(Debug)]
-pub enum BombardError<Bt: ThreadedBombardTask> {
-    InitError(Bt::WorkerInitError),
-    WorkerTaskError(Bt::WorkerTaskError),
-    AllWorkersOffline,
-}
-
-impl<Bt: ThreadedBombardTask> fmt::Display for BombardError<Bt>
-where
-    Bt::WorkerInitError: fmt::Display,
-    Bt::WorkerTaskError: Display,
-{
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::AllWorkersOffline => write!(
-                f,
-                "bombard failed because all workers went offline indicating catastrophic failure"
-            ),
-            Self::WorkerTaskError(e) => write!(f, "worker task failed. {e}"),
-            Self::InitError(e) => write!(f, "worker init failed. {e}"),
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct RuntimeStats {
-    pub qps: f64,
-    pub head: u128,
-    pub tail: u128,
-}
-
-#[derive(Debug)]
-pub struct BombardPool<Bt: ThreadedBombardTask> {
-    workers: Vec<(Worker, Sender<WorkerTask<Bt>>)>,
-    rx_res: Receiver<WorkerResult<Bt>>,
-    _config: Bt,
-}
-
-impl<Bt: ThreadedBombardTask> BombardPool<Bt> {
-    fn qps(query_count: usize, time_taken_in_nanos: u128) -> f64 {
-        const NANOS_PER_SECOND: u128 = 1_000_000_000;
-        let time_taken_in_nanos_f64 = time_taken_in_nanos as f64;
-        let query_count_f64 = query_count as f64;
-        (query_count_f64 / time_taken_in_nanos_f64) * NANOS_PER_SECOND as f64
-    }
-    pub fn new(size: usize, config: Bt) -> BombardResult<Self, Bt> {
-        assert_ne!(size, 0, "pool can't be empty");
-        let mut workers = Vec::with_capacity(size);
-        let (tx_res, rx_res) = unbounded();
-        for id in 0..size {
-            let (tx_work, rx_work) = unbounded();
-            let driver = config.worker_init().map_err(BombardError::InitError)?;
-            workers.push((Worker::start(id, driver, rx_work, tx_res.clone()), tx_work));
-        }
-        Ok(Self {
-            workers,
-            rx_res,
-            _config: config,
-        })
-    }
-    /// Bombard queries to the workers
-    pub fn blocking_bombard(
-        &mut self,
-        task_description: Bt::WorkerTaskSpec,
-        count: usize,
-    ) -> BombardResult<RuntimeStats, Bt> {
-        GPState::guard(|| {
-            GPState::get().post_target(count as _);
-            let mut global_start = None;
-            let mut global_stop = None;
-            let mut global_head = u128::MAX;
-            let mut global_tail = 0u128;
-            for (_, sender) in self.workers.iter() {
-                sender
-                    .send(WorkerTask::Task(task_description.clone()))
-                    .unwrap();
-            }
-            // wait for all workers to complete
-            let mut received = 0;
-            while received != self.workers.len() {
-                let results = match self.rx_res.recv() {
-                    Err(_) => return Err(BombardError::AllWorkersOffline),
-                    Ok(r) => r,
-                };
-                let WorkerLocalStats {
-                    start: this_start,
-                    elapsed,
-                    head,
-                    tail,
-                } = match results {
-                    WorkerResult::Completed(r) => r,
-                    WorkerResult::Errored(e) => return Err(BombardError::WorkerTaskError(e)),
-                };
-                // update start if required
-                match global_start.as_mut() {
-                    None => {
-                        global_start = Some(this_start);
-                    }
-                    Some(start) => {
-                        if this_start < *start {
-                            *start = this_start;
-                        }
-                    }
-                }
-                let this_task_stopped_at =
-                    this_start + Duration::from_nanos(elapsed.try_into().unwrap());
-                match global_stop.as_mut() {
-                    None => {
-                        global_stop = Some(this_task_stopped_at);
-                    }
-                    Some(stop) => {
-                        if this_task_stopped_at > *stop {
-                            // this task stopped later than the previous one
-                            *stop = this_task_stopped_at;
-                        }
-                    }
-                }
-                if head < global_head {
-                    global_head = head;
-                }
-                if tail > global_tail {
-                    global_tail = tail;
-                }
-                received += 1;
-            }
-            // reset global pool state
-            GPState::get().post_reset();
-            // compute results
-            let global_elapsed = global_stop
-                .unwrap()
-                .duration_since(global_start.unwrap())
-                .as_nanos();
-            Ok(RuntimeStats {
-                qps: Self::qps(count, global_elapsed),
-                head: global_head,
-                tail: global_tail,
-            })
-        })
-    }
-}
-
-impl<Bt: ThreadedBombardTask> Drop for BombardPool<Bt> {
-    fn drop(&mut self) {
-        info!("taking all workers offline");
-        for (_, sender) in self.workers.iter() {
-            sender.send(WorkerTask::Exit).unwrap();
-        }
-        for (worker, _) in self.workers.drain(..) {
-            worker.handle.join().unwrap();
-        }
-        info!("all workers now offline");
-    }
-}
diff --git a/sky-bench/src/runtime/fury.rs b/sky-bench/src/runtime/fury.rs
new file mode 100644
index 00000000..b6677ca7
--- /dev/null
+++ b/sky-bench/src/runtime/fury.rs
@@ -0,0 +1,315 @@
+/*
+ * Created on Wed Nov 22 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{RuntimeStats, WorkerLocalStats, WorkerTask},
+    crate::bench::BombardTaskSpec,
+    skytable::Config,
+    std::{
+        fmt,
+        sync::atomic::{AtomicBool, AtomicUsize, Ordering},
+        time::{Duration, Instant},
+    },
+    tokio::sync::{broadcast, mpsc, RwLock},
+};
+
+/*
+    state
+*/
+
+static GLOBAL_START: RwLock<()> = RwLock::const_new(());
+static GLOBAL_TARGET: AtomicUsize = AtomicUsize::new(0);
+static GLOBAL_EXIT: AtomicBool = AtomicBool::new(false);
+
+fn gset_target(target: usize) {
+    GLOBAL_TARGET.store(target, Ordering::Release)
+}
+fn gset_exit() {
+    GLOBAL_EXIT.store(true, Ordering::Release)
+}
+fn grefresh_target() -> usize {
+    let mut current = GLOBAL_TARGET.load(Ordering::Acquire);
+    loop {
+        if current == 0 {
+            return 0;
+        }
+        match GLOBAL_TARGET.compare_exchange(
+            current,
+            current - 1,
+            Ordering::Release,
+            Ordering::Acquire,
+        ) {
+            Ok(prev) => return prev,
+            Err(new) => current = new,
+        }
+    }
+}
+fn grefresh_early_exit() -> bool {
+    GLOBAL_EXIT.load(Ordering::Acquire)
+}
+
+/*
+    errors
+*/
+
+pub type FuryResult<T> = Result<T, FuryError>;
+
+#[derive(Debug)]
+pub enum FuryError {
+    Init(skytable::error::Error),
+    Worker(FuryWorkerError),
+    Dead,
+}
+
+impl fmt::Display for FuryError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Init(e) => write!(f, "fury init failed. {e}"),
+            Self::Worker(e) => write!(f, "worker failed. {e}"),
+            Self::Dead => write!(f, "all workers offline"),
+        }
+    }
+}
+
+impl From<FuryWorkerError> for FuryError {
+    fn from(e: FuryWorkerError) -> Self {
+        Self::Worker(e)
+    }
+}
+
+#[derive(Debug)]
+pub enum FuryWorkerError {
+    DbError(skytable::error::Error),
+    Mismatch,
+}
+
+impl fmt::Display for FuryWorkerError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::DbError(e) => write!(f, "client errored. {e}"),
+            Self::Mismatch => write!(f, "server response did not match expected response"),
+        }
+    }
+}
+
+/*
+    impl
+*/
+
+#[derive(Debug)]
+pub struct Fury {
+    tx_task: broadcast::Sender<WorkerTask<BombardTaskSpec>>,
+    rx_task_result: mpsc::Receiver<FuryResult<WorkerLocalStats>>,
+    client_count: usize,
+}
+
+impl Fury {
+    pub async fn new(client_count: usize, config: Config) -> FuryResult<Self> {
+        let (tx_task, rx_task) = broadcast::channel(1);
+        let (tx_task_result, rx_task_result) = mpsc::channel(client_count);
+        let (tx_ack, mut rx_ack) = mpsc::channel(1);
+        for _ in 0..client_count {
+            let rx_task = tx_task.subscribe();
+            let tx_task_result = tx_task_result.clone();
+            let tx_ack = tx_ack.clone();
+            let config = config.clone();
+            tokio::spawn(async move { worker_svc(rx_task, tx_task_result, tx_ack, config).await });
+        }
+        drop((tx_ack, rx_task));
+        match rx_ack.recv().await {
+            None => {}
+            Some(e) => return Err(FuryError::Init(e)),
+        }
+        info!("all workers online. ready for event loop");
+        Ok(Self {
+            tx_task,
+            rx_task_result,
+            client_count,
+        })
+    }
+    pub async fn bombard(
+        &mut self,
+        count: usize,
+        task: BombardTaskSpec,
+    ) -> FuryResult<RuntimeStats> {
+        // pause workers and set target
+        let start_guard = GLOBAL_START.write().await;
+        gset_target(count);
+        // send tasks
+        self.tx_task.send(WorkerTask::Task(task)).unwrap();
+        // begin work
+        drop(start_guard);
+        // init stats
+        let mut global_start = None;
+        let mut global_stop = None;
+        let mut global_head = u128::MAX;
+        let mut global_tail = 0u128;
+        let mut remaining = self.client_count;
+        while remaining != 0 {
+            let WorkerLocalStats {
+                start: this_start,
+                elapsed: this_elapsed,
+                head: this_head,
+                tail: this_tail,
+            } = match self.rx_task_result.recv().await {
+                None => {
+                    return Err(FuryError::Dead);
+                }
+                Some(res) => res,
+            }?;
+            match global_start.as_mut() {
+                None => global_start = Some(this_start),
+                Some(current_start) => {
+                    if this_start < *current_start {
+                        *current_start = this_start;
+                    }
+                }
+            }
+            let this_stop = this_start + Duration::from_nanos(this_elapsed.try_into().unwrap());
+            match global_stop.as_mut() {
+                None => global_stop = Some(this_stop),
+                Some(current_gstop) => {
+                    if this_stop > *current_gstop {
+                        *current_gstop = this_stop;
+                    }
+                }
+            }
+            if this_head < global_head {
+                global_head = this_head;
+            }
+            if this_tail > global_tail {
+                global_tail = this_tail;
+            }
+            remaining -= 1;
+        }
+        Ok(RuntimeStats {
+            qps: super::qps(
+                count,
+                global_stop
+                    .unwrap()
+                    .duration_since(global_start.unwrap())
+                    .as_nanos(),
+            ),
+            head: global_head,
+            tail: global_tail,
+        })
+    }
+}
+
+async fn worker_svc(
+    mut rx_task: broadcast::Receiver<WorkerTask<BombardTaskSpec>>,
+    tx_task_result: mpsc::Sender<FuryResult<WorkerLocalStats>>,
+    tx_ack: mpsc::Sender<skytable::error::Error>,
+    connection_cfg: Config,
+) {
+    let mut db = match connection_cfg.connect_async().await {
+        Ok(c) => c,
+        Err(e) => {
+            tx_ack.send(e).await.unwrap();
+            return;
+        }
+    };
+    // we're connected and ready to server
+    drop(tx_ack);
+    'wait: loop {
+        let task = match rx_task.recv().await.unwrap() {
+            WorkerTask::Exit => return,
+            WorkerTask::Task(t) => t,
+        };
+        // received a task; ready to roll; wait for begin signal
+        let permit = GLOBAL_START.read().await;
+        // off to the races
+        let mut current = grefresh_target();
+        let mut exit_now = grefresh_early_exit();
+        // init local stats
+        let mut local_start = None;
+        let mut local_elapsed = 0u128;
+        let mut local_head = u128::MAX;
+        let mut local_tail = 0u128;
+        while (current != 0) && !exit_now {
+            // prepare query
+            let (query, response) = task.generate(current as u64);
+            // execute timed
+            let start = Instant::now();
+            let ret = db.query(&query).await;
+            let stop = Instant::now();
+            // check response
+            let resp = match ret {
+                Ok(resp) => resp,
+                Err(e) => {
+                    tx_task_result
+                        .send(Err(FuryError::Worker(FuryWorkerError::DbError(e))))
+                        .await
+                        .unwrap();
+                    gset_exit();
+                    continue 'wait;
+                }
+            };
+            if resp != response {
+                tx_task_result
+                    .send(Err(FuryError::Worker(FuryWorkerError::Mismatch)))
+                    .await
+                    .unwrap();
+                gset_exit();
+                continue 'wait;
+            }
+            // update stats
+            if local_start.is_none() {
+                local_start = Some(start);
+            }
+            let elapsed = stop.duration_since(start).as_nanos();
+            local_elapsed += elapsed;
+            if elapsed > local_tail {
+                local_tail = elapsed;
+            }
+            if elapsed < local_head {
+                local_head = elapsed;
+            }
+            current = grefresh_target();
+            exit_now = grefresh_early_exit();
+        }
+        if exit_now {
+            continue 'wait;
+        }
+        // good! send these results
+        tx_task_result
+            .send(Ok(WorkerLocalStats::new(
+                local_start.unwrap(),
+                local_elapsed,
+                local_head,
+                local_tail,
+            )))
+            .await
+            .unwrap();
+        drop(permit);
+    }
+}
+
+impl Drop for Fury {
+    fn drop(&mut self) {
+        self.tx_task.send(WorkerTask::Exit).unwrap();
+    }
+}
diff --git a/sky-bench/src/runtime/rookie.rs b/sky-bench/src/runtime/rookie.rs
new file mode 100644
index 00000000..000891f3
--- /dev/null
+++ b/sky-bench/src/runtime/rookie.rs
@@ -0,0 +1,378 @@
+/*
+ * Created on Tue Nov 21 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{RuntimeStats, WorkerLocalStats, WorkerTask},
+    crossbeam_channel::{unbounded, Receiver, Sender},
+    std::{
+        fmt::{self, Display},
+        sync::{
+            atomic::{AtomicBool, AtomicU64, Ordering},
+            RwLock, RwLockReadGuard, RwLockWriteGuard,
+        },
+        thread::{self, JoinHandle},
+        time::{Duration, Instant},
+    },
+};
+
+pub type BombardResult<T, Bt> = Result<T, BombardError<Bt>>;
+
+/*
+    state mgmt
+*/
+
+#[derive(Debug)]
+/// The pool state. Be warned **ONLY ONE POOL AT A TIME!**
+struct GPState {
+    current: AtomicU64,
+    state: AtomicBool,
+    occupied: AtomicBool,
+    start_sig: RwLock<()>,
+}
+
+impl GPState {
+    #[inline(always)]
+    fn get() -> &'static Self {
+        static STATE: GPState = GPState::zero();
+        &STATE
+    }
+    const fn zero() -> Self {
+        Self {
+            current: AtomicU64::new(0),
+            state: AtomicBool::new(true),
+            occupied: AtomicBool::new(false),
+            start_sig: RwLock::new(()),
+        }
+    }
+    fn wait_for_global_begin(&self) -> RwLockReadGuard<'_, ()> {
+        self.start_sig.read().unwrap()
+    }
+    fn occupy(&self) {
+        assert!(!self.occupied.swap(true, Ordering::Release));
+    }
+    fn vacate(&self) {
+        assert!(self.occupied.swap(false, Ordering::Release));
+    }
+    fn guard<T>(f: impl FnOnce(RwLockWriteGuard<'static, ()>) -> T) -> T {
+        let slf = Self::get();
+        slf.occupy();
+        let ret = f(slf.start_sig.write().unwrap());
+        slf.vacate();
+        ret
+    }
+    fn post_failure(&self) {
+        self.state.store(false, Ordering::Release)
+    }
+    fn post_target(&self, target: u64) {
+        self.current.store(target, Ordering::Release)
+    }
+    /// WARNING: this is not atomic! only sensible to run a quiescent state
+    fn post_reset(&self) {
+        self.current.store(0, Ordering::Release);
+        self.state.store(true, Ordering::Release);
+    }
+    fn update_target(&self) -> u64 {
+        let mut current = self.current.load(Ordering::Acquire);
+        loop {
+            if current == 0 {
+                return 0;
+            }
+            match self.current.compare_exchange(
+                current,
+                current - 1,
+                Ordering::Release,
+                Ordering::Acquire,
+            ) {
+                Ok(last) => {
+                    return last;
+                }
+                Err(new) => {
+                    current = new;
+                }
+            }
+        }
+    }
+    fn load_okay(&self) -> bool {
+        self.state.load(Ordering::Acquire)
+    }
+}
+
+/*
+    task spec
+*/
+
+/// A threaded bombard task specification which drives a global pool of threads towards a common goal
+pub trait ThreadedBombardTask: Send + Sync + 'static {
+    /// The per-task worker that is initialized once in every thread (not to be confused with the actual thread worker!)
+    type Worker: Send + Sync;
+    /// The task that the [`ThreadedBombardTask::TaskWorker`] performs
+    type WorkerTask: Send + Sync;
+    type WorkerTaskSpec: Clone + Send + Sync + 'static;
+    /// Errors while running a task
+    type WorkerTaskError: Send + Sync;
+    /// Errors while initializing a task worker
+    type WorkerInitError: Send + Sync;
+    /// Initialize a task worker
+    fn worker_init(&self) -> Result<Self::Worker, Self::WorkerInitError>;
+    fn generate_task(spec: &Self::WorkerTaskSpec, current: u64) -> Self::WorkerTask;
+    /// Drive a single subtask
+    fn worker_drive_timed(
+        worker: &mut Self::Worker,
+        task: Self::WorkerTask,
+    ) -> Result<u128, Self::WorkerTaskError>;
+}
+
+/*
+    worker
+*/
+
+#[derive(Debug)]
+enum WorkerResult<Bt: ThreadedBombardTask> {
+    Completed(WorkerLocalStats),
+    Errored(Bt::WorkerTaskError),
+}
+
+#[derive(Debug)]
+struct Worker {
+    handle: JoinHandle<()>,
+}
+
+impl Worker {
+    fn start<Bt: ThreadedBombardTask>(
+        id: usize,
+        driver: Bt::Worker,
+        rx_work: Receiver<WorkerTask<Bt::WorkerTaskSpec>>,
+        tx_res: Sender<WorkerResult<Bt>>,
+    ) -> Self {
+        Self {
+            handle: thread::Builder::new()
+                .name(format!("worker-{id}"))
+                .spawn(move || {
+                    let mut worker_driver = driver;
+                    'blocking_wait: loop {
+                        let task = match rx_work.recv().unwrap() {
+                            WorkerTask::Exit => return,
+                            WorkerTask::Task(spec) => spec,
+                        };
+                        let guard = GPState::get().wait_for_global_begin();
+                        // check global state
+                        let mut global_okay = GPState::get().load_okay();
+                        let mut global_position = GPState::get().update_target();
+                        // init local state
+                        let mut local_start = None;
+                        let mut local_elapsed = 0u128;
+                        let mut local_head = u128::MAX;
+                        let mut local_tail = 0;
+                        // bombard
+                        while (global_position != 0) & global_okay {
+                            let task = Bt::generate_task(&task, global_position);
+                            if local_start.is_none() {
+                                local_start = Some(Instant::now());
+                            }
+                            let this_elapsed =
+                                match Bt::worker_drive_timed(&mut worker_driver, task) {
+                                    Ok(elapsed) => elapsed,
+                                    Err(e) => {
+                                        GPState::get().post_failure();
+                                        tx_res.send(WorkerResult::Errored(e)).unwrap();
+                                        continue 'blocking_wait;
+                                    }
+                                };
+                            local_elapsed += this_elapsed;
+                            if this_elapsed < local_head {
+                                local_head = this_elapsed;
+                            }
+                            if this_elapsed > local_tail {
+                                local_tail = this_elapsed;
+                            }
+                            global_position = GPState::get().update_target();
+                            global_okay = GPState::get().load_okay();
+                        }
+                        if global_okay {
+                            // we're done
+                            tx_res
+                                .send(WorkerResult::Completed(WorkerLocalStats::new(
+                                    local_start.unwrap(),
+                                    local_elapsed,
+                                    local_head,
+                                    local_tail,
+                                )))
+                                .unwrap();
+                        }
+                        drop(guard);
+                    }
+                })
+                .expect("failed to start thread"),
+        }
+    }
+}
+
+/*
+    pool
+*/
+
+#[derive(Debug)]
+pub enum BombardError<Bt: ThreadedBombardTask> {
+    InitError(Bt::WorkerInitError),
+    WorkerTaskError(Bt::WorkerTaskError),
+    AllWorkersOffline,
+}
+
+impl<Bt: ThreadedBombardTask> fmt::Display for BombardError<Bt>
+where
+    Bt::WorkerInitError: fmt::Display,
+    Bt::WorkerTaskError: Display,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::AllWorkersOffline => write!(
+                f,
+                "bombard failed because all workers went offline indicating catastrophic failure"
+            ),
+            Self::WorkerTaskError(e) => write!(f, "worker task failed. {e}"),
+            Self::InitError(e) => write!(f, "worker init failed. {e}"),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct BombardPool<Bt: ThreadedBombardTask> {
+    workers: Vec<(Worker, Sender<WorkerTask<Bt::WorkerTaskSpec>>)>,
+    rx_res: Receiver<WorkerResult<Bt>>,
+    _config: Bt,
+}
+
+impl<Bt: ThreadedBombardTask> BombardPool<Bt> {
+    pub fn new(size: usize, config: Bt) -> BombardResult<Self, Bt> {
+        assert_ne!(size, 0, "pool can't be empty");
+        let mut workers = Vec::with_capacity(size);
+        let (tx_res, rx_res) = unbounded();
+        for id in 0..size {
+            let (tx_work, rx_work) = unbounded();
+            let driver = config.worker_init().map_err(BombardError::InitError)?;
+            workers.push((Worker::start(id, driver, rx_work, tx_res.clone()), tx_work));
+        }
+        Ok(Self {
+            workers,
+            rx_res,
+            _config: config,
+        })
+    }
+    /// Bombard queries to the workers
+    pub fn blocking_bombard(
+        &mut self,
+        task_description: Bt::WorkerTaskSpec,
+        count: usize,
+    ) -> BombardResult<RuntimeStats, Bt> {
+        GPState::guard(|paused| {
+            GPState::get().post_target(count as _);
+            let mut global_start = None;
+            let mut global_stop = None;
+            let mut global_head = u128::MAX;
+            let mut global_tail = 0u128;
+            for (_, sender) in self.workers.iter() {
+                sender
+                    .send(WorkerTask::Task(task_description.clone()))
+                    .unwrap();
+            }
+            // now let them begin!
+            drop(paused);
+            // wait for all workers to complete
+            let mut received = 0;
+            while received != self.workers.len() {
+                let results = match self.rx_res.recv() {
+                    Err(_) => return Err(BombardError::AllWorkersOffline),
+                    Ok(r) => r,
+                };
+                let WorkerLocalStats {
+                    start: this_start,
+                    elapsed,
+                    head,
+                    tail,
+                } = match results {
+                    WorkerResult::Completed(r) => r,
+                    WorkerResult::Errored(e) => return Err(BombardError::WorkerTaskError(e)),
+                };
+                // update start if required
+                match global_start.as_mut() {
+                    None => {
+                        global_start = Some(this_start);
+                    }
+                    Some(start) => {
+                        if this_start < *start {
+                            *start = this_start;
+                        }
+                    }
+                }
+                let this_task_stopped_at =
+                    this_start + Duration::from_nanos(elapsed.try_into().unwrap());
+                match global_stop.as_mut() {
+                    None => {
+                        global_stop = Some(this_task_stopped_at);
+                    }
+                    Some(stop) => {
+                        if this_task_stopped_at > *stop {
+                            // this task stopped later than the previous one
+                            *stop = this_task_stopped_at;
+                        }
+                    }
+                }
+                if head < global_head {
+                    global_head = head;
+                }
+                if tail > global_tail {
+                    global_tail = tail;
+                }
+                received += 1;
+            }
+            // reset global pool state
+            GPState::get().post_reset();
+            // compute results
+            let global_elapsed = global_stop
+                .unwrap()
+                .duration_since(global_start.unwrap())
+                .as_nanos();
+            Ok(RuntimeStats {
+                qps: super::qps(count, global_elapsed),
+                head: global_head,
+                tail: global_tail,
+            })
+        })
+    }
+}
+
+impl<Bt: ThreadedBombardTask> Drop for BombardPool<Bt> {
+    fn drop(&mut self) {
+        info!("taking all workers offline");
+        for (_, sender) in self.workers.iter() {
+            sender.send(WorkerTask::Exit).unwrap();
+        }
+        for (worker, _) in self.workers.drain(..) {
+            worker.handle.join().unwrap();
+        }
+        info!("all workers now offline");
+    }
+}

From 4ba7b97ac7e0ef6be083079e9e76bbe42fde1605 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 22 Nov 2023 01:55:33 +0530
Subject: [PATCH 287/310] Fix windows builds

Also added changelog entries
---
 CHANGELOG.md                  | 88 +++++++++++++++++++++++++++++------
 server/Cargo.toml             |  6 ++-
 server/src/util/os/flock.rs   | 45 ++++++++++++------
 sky-bench/README.md           | 33 ++++++++++---
 sky-bench/help_text/help      | 12 ++---
 sky-bench/src/args.rs         |  2 +-
 sky-bench/src/bench.rs        | 24 ++++------
 sky-bench/src/runtime/fury.rs | 11 +++++
 8 files changed, 164 insertions(+), 57 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4582ce6b..221b6142 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,23 +2,85 @@
 
 All changes in this project will be noted in this file.
 
-## Unreleased
+## Version 0.8.0
+
+> This is the first release of Skytable Octave, and it changes the query API entirely making all previous versions incompatible 
+> excluding the data files which are automatically upgraded per our backwards compatibility guarantees
 
 ### Additions
 
+#### BlueQL query language
+- DDL:
+  - `space`s are the equivalent of the `keyspace` from previous versions
+  - `model`s are the equivalent of `table`s from previous version
+  - The following queries were added:
+    - `CREATE SPACE ...`
+    - `CREATE MODEL ...`
+      - Nested lists are now supported
+      - Type definitions are now supported
+      - Multiple fields are now supported
+    - `ALTER SPACE ...`
+    - `ALTER MODEL ...`
+    - `DROP SPACE ...`
+    - `DROP MODEL ...`
+- DML:
+  - **All actions removed**: All the prior `SET`, `GET` and other actions have been removed in favor of the new query language
+  - The following queries were added:
+    - `INSERT INTO <space>.<model>(col, col2, col3, ...)`
+    - `SELECT field1, field2, ... FROM <space>.<model> WHERE <primary_key_column> = <value>`
+    - New data manipulation via `UPDATE` allows arithmetic operations, string manipulation and more!:
+      - `UPDATE <space>.<model> SET col_num += 1 WHERE <primary_key_column> = <value>`
+    - `DELETE FROM <space>.<model> WHERE <primary_key_column> = <value>`
+- DCL:
+  - `SYSCTL CREATE USER <name> WITH { password: <password> }`
+  - `SYSCTL DROP USER <name>`
+
+#### Fractal engine
+
+- The fractal engine is the start of the development of advanced internal state management in Skytable
+- Effectively balances performance and reliability tasks
+
+#### Skyhash 2 protocol
+- The `Skyhash-2` protocol now uses a multi-stage connection sequence for improved security and performance
+- Seamless auth
+- More types
+- Fewer retransmissions
+
+#### Storage engines
+- **New `deltax` storage engine for data**:
+  - The `deltax` storage engine monitors the database for changes and only records the changes in an append only file
+  - The interval can be adjusted per requirements of reliability
+  - Faster and hugely more reliable than the previous engine
+- **New DDL ACID transactions with with the `logx` engine**:
+  - DDL queries are now fully transactional which means that if they are executed, you can be sure that they were complete and synced to disk
+  - This largely improves reliability
+
+#### New shell
+
+- The new client shell easily authenticates based on the Skyhash-2 protocol
+- More reliable
+
+#### Benchmark tool
+
+- New benchmark engines to enable more efficient load testing
+- New benchmark engines use far lesser memory
+- New engines can handle midway benchmark crashes
+
+### Breaking changes
+
 - `skyd`:
-  - New protocol: Skyhash 2.0
-    - Reduced bandwidth usage (as much as 50%)
-    - Even simpler client implementations
-  - Backward compatibility with Skyhash 1.0:
-    - Simply set the protocol version you want to use in the config file, env vars or pass it as a CLI
-      argument
-    - Even faster implementation, even for Skyhash 1.0
-  - New query language: BlueQL
-    - `create keyspace` is now `create space`
-    - `create table` is now `create model`
-    - Similary, all `inspect` queries have been changed
-    - Entities are now of the form `space.model` instead of `ks:tbl`
+  - **The entire query API has changed as actions have been removed**
+  - **Spaces and models**: replace keyspaces and models respectively
+  - **Configuration**:
+    - The configuration system now uses YAML instead of TOML for better readability
+    - The configuration options have changed across CLI, ENV and the config file
+    - Authentication **must be enabled** irrespective of `dev`/`prod` mode
+- `sky-bench`:
+  - New benchmark engines are completely different from the previous engines (see above)
+  - Configuration options have changed because of how the new Skytable engine works
+- `skysh`:
+  - Configuration options have changed because of how the new Skytable engine works
+- `sky-migrate`: **This tool is deprecated and has been removed**. The Skytable engine will now automatically manage data upgrades. (please see [this issue](https://github.com/skytable/skytable/issues/320) for discussion on the same)
 
 ## Version 0.7.6
 
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 51e235bb..15e2d358 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -28,7 +28,11 @@ serde_yaml = "0.9"
 jemallocator = "0.5.4"
 [target.'cfg(target_os = "windows")'.dependencies]
 # external deps
-winapi = { version = "0.3.9", features = ["fileapi"] }
+winapi = { version = "0.3.9", features = [
+  "fileapi",
+  "sysinfoapi",
+  "minwinbase",
+] }
 
 [target.'cfg(unix)'.dependencies]
 # external deps
diff --git a/server/src/util/os/flock.rs b/server/src/util/os/flock.rs
index 84f32585..8af608ca 100644
--- a/server/src/util/os/flock.rs
+++ b/server/src/util/os/flock.rs
@@ -33,12 +33,12 @@ extern crate winapi;
 #[cfg(windows)]
 use std::os::windows::io::AsRawHandle;
 
-use std::{fs::File, io, os::unix::io::AsRawFd, path::Path};
+use std::{fs::File, io, path::Path};
 
 pub struct FileLock {
-    file: File,
+    _file: File,
     #[cfg(windows)]
-    handle: winapi::um::handleapi::HANDLE,
+    handle: winapi::um::winnt::HANDLE,
 }
 
 impl FileLock {
@@ -46,13 +46,16 @@ impl FileLock {
         let file = File::create(path)?;
         #[cfg(windows)]
         {
-            use winapi::um::{
-                fileapi::LockFileEx,
-                minwinbase::{LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY, OVERLAPPED},
-                winnt::HANDLE,
+            use {
+                std::mem,
+                winapi::um::{
+                    fileapi::LockFileEx,
+                    minwinbase::{LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY},
+                    winnt::HANDLE,
+                },
             };
             let handle = file.as_raw_handle();
-            let mut overlapped = OVERLAPPED::default();
+            let mut overlapped = unsafe { mem::zeroed() };
             let result = unsafe {
                 LockFileEx(
                     handle as HANDLE,
@@ -69,11 +72,17 @@ impl FileLock {
                     "file is already locked",
                 ));
             }
-            return Ok(Self { file, handle });
+            return Ok(Self {
+                _file: file,
+                handle,
+            });
         }
         #[cfg(unix)]
         {
-            use libc::{flock, LOCK_EX, LOCK_NB};
+            use {
+                libc::{flock, LOCK_EX, LOCK_NB},
+                std::os::unix::io::AsRawFd,
+            };
             let result = unsafe { flock(file.as_raw_fd(), LOCK_EX | LOCK_NB) };
             if result != 0 {
                 return Err(io::Error::new(
@@ -81,15 +90,18 @@ impl FileLock {
                     "file is already locked",
                 ));
             }
-            return Ok(Self { file });
+            return Ok(Self { _file: file });
         }
     }
     pub fn release(self) -> io::Result<()> {
         #[cfg(windows)]
         {
-            use winapi::um::{fileapi::UnlockFileEx, minwinbase::OVERLAPPED, winnt::HANDLE};
+            use {
+                std::mem,
+                winapi::um::{fileapi::UnlockFileEx, winnt::HANDLE},
+            };
 
-            let mut overlapped = OVERLAPPED::default();
+            let mut overlapped = unsafe { mem::zeroed() };
             let result = unsafe {
                 UnlockFileEx(
                     self.handle as HANDLE,
@@ -106,8 +118,11 @@ impl FileLock {
         }
         #[cfg(unix)]
         {
-            use libc::{flock, LOCK_UN};
-            let result = unsafe { flock(self.file.as_raw_fd(), LOCK_UN) };
+            use {
+                libc::{flock, LOCK_UN},
+                std::os::unix::io::AsRawFd,
+            };
+            let result = unsafe { flock(self._file.as_raw_fd(), LOCK_UN) };
             if result != 0 {
                 return Err(io::Error::last_os_error());
             }
diff --git a/sky-bench/README.md b/sky-bench/README.md
index f9327f13..1464d400 100644
--- a/sky-bench/README.md
+++ b/sky-bench/README.md
@@ -3,12 +3,31 @@
 `sky-bench` is Skytable's benchmarking tool. Unlike most other benchmarking tools, Skytable's benchmark
 tool doesn't do anything "fancy" to make benchmarks appear better than they are. As it happens, the benchmark tool might show Skytable to be slower!
 
+> **The benchmarking engine is currently experimental!** You can indirectly compare it to the working of `redis-benchmark` but one
+> important note: **Skytable has a full fledged query language.** *Even then, you will probably enjoy the benchmarks!*
+>
+> Other tools like `memtier_benchmark` are far more sophisticated and use several strategies that can hugely affect benchmark 
+> numbers.
+>
+> We will upgrade the benchmark engine from time to time to improve the reporting of statistics. For example, right now the 
+> engine only outputs the slowest and fastest query speeds **in nanoseconds** but we plan to provide a overall distribution of 
+> latencies.
+
+## Working
+
 Here's how the benchmark tool works (it's dead simple):
 
-1. We start up some threads with each having a thread local connection to the database
-2. Each thread attempts to keep running queries until the target number of queries is reached.
-   - This sort of simulates a real-world scenario where these threads are like your application servers sending requests to the database
-   - Also there is no ideal distribution and the number of queries each worker runs is unspecified (but owing to low latencies from the database, that should be even)
-   - We do this to ensure that the distribution of queries executed by each "server" is skewed as it would be in the real world.
-3. Once the target number of queries are reached, the workers notify that the task is complete. Each worker keeps track of how long it spent processing queries and this is also notified to the benchmark engine
-4. The benchmark engine then computes relevant statistics
+1. We spawn up multiple client tasks (or what you can call "green threads") that can each handle tasks. These tasks are run on a threadpool which has multiple worker threads, kind of simulating multiple "application server instances"
+   - You can use `--connections` to set the number of client connections
+   - You can use `--threads` to set the number of threads to be used
+2. An overall target is sent to these tasks and all tasks start executing queries until the overall target is reached
+   > The distribution of tasks across clients is generally unspecified, but because of Skytable's extremely low average latencies, in most common scenarios, the distribution is even.
+3. Once the overall target is reached, each task relays its local execution statistics to the monitoring task
+4. The monitoring task then prepares the final results, and this is returned
+
+### Engines
+
+There are two benchmark engines:
+
+- `fury`: this is the new experimental engine, but also set as the default engine. It is generally more efficient and tracks statistics more effectively. At the same time, it is capable of generating larger consistent loads without crashing or blowing up CPU usage
+- `rookie`: this is the old engine that's still available but is not used by default. it still uses lesser memory than prior versions (which used a very inefficient and memory hungry algorithm) but is not as resource efficient as the `fury` engine.
diff --git a/sky-bench/help_text/help b/sky-bench/help_text/help
index 6af1bf3a..a191d8ba 100644
--- a/sky-bench/help_text/help
+++ b/sky-bench/help_text/help
@@ -16,18 +16,18 @@ OPTIONS:
     --endpoint    Set the endpoint (defaults to tcp@127.0.0.1:2003)
     --threads     Set the number of threads to be used (defaults to logical
                   CPU count)
-    --connections Set the number of connections. Defaults to twice the logical CPU
-                  count but is only supported by the `fury` engine
+    --connections Set the number of connections. Defaults to 8 x logical CPU
+                  count. Only supported by the `fury` engine.
     --keysize     Set the default primary key size. defaults to 7
     --rowcount    Set the number of rows to be manipulated for the benchmark
                   Defaults to 1,000,000 rows.
     --engine      Set the engine for benchmarking. `rookie` is the stable engine
-                  and `fury` is the experimental engine
+                  and `fury` is the new experimental engine. Defaults to `fury`
 
 NOTES:
     - The user for auth will be 'root' since only 'root' accounts allow the
       creation and deletion of spaces and models
-    - A space called 'benchmark_[random 8B string]' will be created
-    - A model called 'benchmark_[random 8B string]' will be created in the space
-      created above. The created model has the structure {name: string, pass: string}
+    - A space called 'bench' will be created
+    - A model called 'bench' will be created in the space
+      created above. The created model has the structure {un: string, pw: uint8}
     - The model and space will be removed once the benchmark is complete
diff --git a/sky-bench/src/args.rs b/sky-bench/src/args.rs
index 84ace127..2aa5336d 100644
--- a/sky-bench/src/args.rs
+++ b/sky-bench/src/args.rs
@@ -197,7 +197,7 @@ pub fn parse() -> BenchResult<Task> {
         },
     };
     let connections = match args.remove("--connections") {
-        None => num_cpus::get() * 2,
+        None => num_cpus::get() * 8,
         Some(c) => match c.parse::<usize>() {
             Ok(s) if s != 0 => {
                 if engine == BenchEngine::Rookie {
diff --git a/sky-bench/src/bench.rs b/sky-bench/src/bench.rs
index 9d93f062..4b9d435f 100644
--- a/sky-bench/src/bench.rs
+++ b/sky-bench/src/bench.rs
@@ -185,8 +185,8 @@ pub fn run(bench: BenchConfig) -> error::BenchResult<()> {
         BenchEngine::Rookie => bench_rookie(bench_config, bench),
         BenchEngine::Fury => bench_fury(bench),
     };
-    let stats = match stats {
-        Ok(stats) => stats,
+    let (total_queries, stats) = match stats {
+        Ok(ret) => ret,
         Err(e) => {
             error!("benchmarking failed. attempting to clean up");
             match cleanup(main_thread_db) {
@@ -198,6 +198,10 @@ pub fn run(bench: BenchConfig) -> error::BenchResult<()> {
             }
         }
     };
+    info!(
+        "{} queries executed. benchmark complete.",
+        fmt_u64(total_queries)
+    );
     warn!("benchmarks might appear to be slower. this tool is currently experimental");
     // print results
     print_table(stats);
@@ -312,7 +316,7 @@ fn fmt_u64(n: u64) -> String {
 fn bench_rookie(
     task: BombardTask,
     bench: BenchConfig,
-) -> BenchResult<Vec<(&'static str, RuntimeStats)>> {
+) -> BenchResult<(u64, Vec<(&'static str, RuntimeStats)>)> {
     // initialize pool
     info!(
         "initializing connections. engine=rookie, threads={}, primary key size ={} bytes",
@@ -330,14 +334,10 @@ fn bench_rookie(
         let this_result = task.run(&mut pool)?;
         results.push((name, this_result));
     }
-    info!(
-        "benchmark complete. finished executing {} queries",
-        fmt_u64(total_queries)
-    );
-    Ok(results)
+    Ok((total_queries, results))
 }
 
-fn bench_fury(bench: BenchConfig) -> BenchResult<Vec<(&'static str, RuntimeStats)>> {
+fn bench_fury(bench: BenchConfig) -> BenchResult<(u64, Vec<(&'static str, RuntimeStats)>)> {
     let rt = tokio::runtime::Builder::new_multi_thread()
         .worker_threads(bench.threads)
         .enable_all()
@@ -364,10 +364,6 @@ fn bench_fury(bench: BenchConfig) -> BenchResult<Vec<(&'static str, RuntimeStats
             let this_result = task.run_async(&mut pool).await?;
             results.push((name, this_result));
         }
-        info!(
-            "benchmark complete. finished executing {} queries",
-            fmt_u64(total_queries)
-        );
-        Ok(results)
+        Ok((total_queries,results))
     })
 }
diff --git a/sky-bench/src/runtime/fury.rs b/sky-bench/src/runtime/fury.rs
index b6677ca7..53f0b2df 100644
--- a/sky-bench/src/runtime/fury.rs
+++ b/sky-bench/src/runtime/fury.rs
@@ -232,6 +232,17 @@ async fn worker_svc(
             return;
         }
     };
+    // warm up connection
+    match db
+        .query_parse::<()>(&skytable::query!("sysctl report status"))
+        .await
+    {
+        Ok(()) => {}
+        Err(e) => {
+            tx_ack.send(e).await.unwrap();
+            return;
+        }
+    }
     // we're connected and ready to server
     drop(tx_ack);
     'wait: loop {

From 3e49971efc56ae47c841c81981eb3e78482aa76f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 23 Nov 2023 00:12:26 +0530
Subject: [PATCH 288/310] Fix inconsistent SE state due to incorrect
 propagation

---
 server/src/engine/storage/v1/batch_jrnl/persist.rs | 7 +++++--
 server/src/engine/storage/v1/inf/map.rs            | 2 +-
 server/src/engine/storage/v1/journal.rs            | 6 +++++-
 server/src/util/compiler.rs                        | 2 +-
 4 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs
index ee355690..ec0d3613 100644
--- a/server/src/engine/storage/v1/batch_jrnl/persist.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs
@@ -125,13 +125,16 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         };
         match exec() {
             Ok(()) => Ok(()),
-            Err(_) => {
+            Err(e) => {
                 // republish changes since we failed to commit
                 restore_list.into_iter().for_each(|delta| {
                     model.delta_state().append_new_data_delta(delta, &g);
                 });
                 // now attempt to fix the file
-                return self.attempt_fix_data_batchfile();
+                self.attempt_fix_data_batchfile()?;
+                // IMPORTANT: return an error because even though we recovered the journal we still didn't succeed in
+                // writing the batch
+                return Err(e);
             }
         }
     }
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index bd4ba03f..aab7a1b8 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -142,8 +142,8 @@ pub struct GenericDictSpec;
 
 /// generic dict entry metadata
 pub struct GenericDictEntryMD {
-    pub(crate) dscr: u8,
     pub(crate) klen: usize,
+    pub(crate) dscr: u8,
 }
 
 impl GenericDictEntryMD {
diff --git a/server/src/engine/storage/v1/journal.rs b/server/src/engine/storage/v1/journal.rs
index 0f28ecef..a1815b6c 100644
--- a/server/src/engine/storage/v1/journal.rs
+++ b/server/src/engine/storage/v1/journal.rs
@@ -402,7 +402,11 @@ impl<Fs: RawFSInterface, TA: JournalAdapter> JournalWriter<Fs, TA> {
         debug_assert!(TA::RECOVERY_PLUGIN);
         match self.append_event(event) {
             Ok(()) => Ok(()),
-            Err(_) => compiler::cold_call(|| return self.appendrec_journal_reverse_entry()),
+            Err(e) => compiler::cold_call(move || {
+                // IMPORTANT: we still need to return an error so that the caller can retry if deemed appropriate
+                self.appendrec_journal_reverse_entry()?;
+                Err(e)
+            }),
         }
     }
 }
diff --git a/server/src/util/compiler.rs b/server/src/util/compiler.rs
index 1479d446..92567d44 100644
--- a/server/src/util/compiler.rs
+++ b/server/src/util/compiler.rs
@@ -47,7 +47,7 @@ pub const fn unlikely(b: bool) -> bool {
 
 #[cold]
 #[inline(never)]
-pub fn cold_call<U>(mut v: impl FnMut() -> U) -> U {
+pub fn cold_call<U>(v: impl FnOnce() -> U) -> U {
     v()
 }
 

From 341a896c37ead1bbc70b28bcf170124f4f9219d5 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 23 Nov 2023 12:15:45 +0530
Subject: [PATCH 289/310] Simplify schema changes

---
 server/src/engine/core/dml/ins.rs             |   7 +-
 server/src/engine/core/dml/sel.rs             |   3 +-
 server/src/engine/core/dml/upd.rs             |   4 +-
 server/src/engine/core/index/row.rs           |   3 +-
 server/src/engine/core/mod.rs                 |   8 +-
 server/src/engine/core/model/alt.rs           |  45 ++---
 server/src/engine/core/model/delta.rs         | 185 ++----------------
 server/src/engine/core/model/mod.rs           |  51 ++---
 server/src/engine/core/tests/ddl_model/alt.rs |  12 +-
 server/src/engine/core/tests/ddl_model/crt.rs |   3 -
 server/src/engine/core/tests/dml/mod.rs       |   2 -
 .../engine/storage/v1/batch_jrnl/persist.rs   |  18 +-
 .../engine/storage/v1/batch_jrnl/restore.rs   |   3 +-
 server/src/engine/storage/v1/inf/obj.rs       |  26 +--
 server/src/engine/storage/v1/inf/tests.rs     |   3 +-
 server/src/engine/txn/gns/model.rs            |  52 ++---
 server/src/engine/txn/gns/tests/full_chain.rs |  17 +-
 server/src/engine/txn/gns/tests/io.rs         |   9 +-
 18 files changed, 116 insertions(+), 335 deletions(-)

diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index 2ff9723b..1bc3d183 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -29,7 +29,7 @@ use crate::engine::{
         self,
         dml::QueryExecMeta,
         index::{DcFieldIndex, PrimaryIndexKey, Row},
-        model::{delta::DataDeltaKind, Fields, Model},
+        model::{delta::DataDeltaKind, Model},
     },
     error::{QueryError, QueryResult},
     fractal::GlobalInstanceLike,
@@ -48,8 +48,7 @@ pub fn insert_resp(
 
 pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> QueryResult<()> {
     core::with_model_for_data_update(global, insert.entity(), |mdl| {
-        let irmwd = mdl.intent_write_new_data();
-        let (pk, data) = prepare_insert(mdl, irmwd.fields(), insert.data())?;
+        let (pk, data) = prepare_insert(mdl, insert.data())?;
         let g = cpin();
         let ds = mdl.delta_state();
         // create new version
@@ -68,9 +67,9 @@ pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> Quer
 // TODO(@ohsayan): optimize null case
 fn prepare_insert(
     model: &Model,
-    fields: &Fields,
     insert: InsertData,
 ) -> QueryResult<(PrimaryIndexKey, DcFieldIndex)> {
+    let fields = model.fields();
     let mut okay = fields.len() == insert.column_count();
     let mut prepared_data = DcFieldIndex::idx_init_cap(fields.len());
     match insert {
diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
index 8c2d0f1f..146850cf 100644
--- a/server/src/engine/core/dml/sel.rs
+++ b/server/src/engine/core/dml/sel.rs
@@ -102,7 +102,6 @@ where
     F: FnMut(&Datacell),
 {
     global.namespace().with_model(select.entity(), |mdl| {
-        let irm = mdl.intent_read_model();
         let target_key = mdl.resolve_where(select.clauses_mut())?;
         let pkdc = VirtualDatacell::new(target_key.clone());
         let g = sync::atm::cpin();
@@ -118,7 +117,7 @@ where
             Some(row) => {
                 let r = row.resolve_schema_deltas_and_freeze(mdl.delta_state());
                 if select.is_wildcard() {
-                    for key in irm.fields().stseq_ord_key() {
+                    for key in mdl.fields().stseq_ord_key() {
                         read_field(key.as_ref(), r.fields())?;
                     }
                 } else {
diff --git a/server/src/engine/core/dml/upd.rs b/server/src/engine/core/dml/upd.rs
index b3810a5d..d665a741 100644
--- a/server/src/engine/core/dml/upd.rs
+++ b/server/src/engine/core/dml/upd.rs
@@ -266,8 +266,6 @@ pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) ->
         let mut ret = Ok(QueryExecMeta::zero());
         // prepare row fetch
         let key = mdl.resolve_where(update.clauses_mut())?;
-        // freeze schema
-        let irm = mdl.intent_read_model();
         // fetch row
         let g = sync::atm::cpin();
         let Some(row) = mdl.primary_index().select(key, &g) else {
@@ -298,7 +296,7 @@ pub fn update(global: &impl GlobalInstanceLike, mut update: UpdateStatement) ->
             let field_definition;
             let field_data;
             match (
-                irm.fields().st_get(lhs.as_str()),
+                mdl.fields().st_get(lhs.as_str()),
                 row_data_wl.fields_mut().st_get_mut(lhs.as_str()),
             ) {
                 (Some(fdef), Some(fdata)) => {
diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
index b667edd4..7cde70a6 100644
--- a/server/src/engine/core/index/row.rs
+++ b/server/src/engine/core/index/row.rs
@@ -167,9 +167,8 @@ impl Row {
         }
         // we have deltas to apply
         let mut wl = RwLockUpgradableReadGuard::upgrade(rwl_ug);
-        let delta_read = delta_state.schema_delta_read();
         let mut max_delta = wl.txn_revised_schema_version;
-        for (delta_id, delta) in delta_read.resolve_iter_since(wl.txn_revised_schema_version) {
+        for (delta_id, delta) in delta_state.resolve_iter_since(wl.txn_revised_schema_version) {
             match delta.kind() {
                 SchemaDeltaKind::FieldAdd(f) => {
                     wl.fields.st_insert(f.clone(), Datacell::null());
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index da929a4f..69bfd829 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -87,13 +87,13 @@ impl GlobalNS {
         let mut space = space.write();
         f(&mut space)
     }
-    pub fn with_model_space<'a, T, F>(&self, entity: Entity<'a>, f: F) -> QueryResult<T>
+    pub fn with_model_space_mut_for_ddl<'a, T, F>(&self, entity: Entity<'a>, f: F) -> QueryResult<T>
     where
-        F: FnOnce(&Space, &Model) -> QueryResult<T>,
+        F: FnOnce(&Space, &mut Model) -> QueryResult<T>,
     {
         let (space, model_name) = entity.into_full_result()?;
-        let mdl_idx = self.idx_mdl.read();
-        let Some(model) = mdl_idx.get(&EntityIDRef::new(&space, &model_name)) else {
+        let mut mdl_idx = self.idx_mdl.write();
+        let Some(model) = mdl_idx.get_mut(&EntityIDRef::new(&space, &model_name)) else {
             return Err(QueryError::QExecObjectNotFound);
         };
         let space_read = self.idx.read();
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index ef1386cd..ad24f554 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -25,7 +25,7 @@
 */
 
 use {
-    super::{Field, IWModel, Layer, Model},
+    super::{Field, Layer, Model},
     crate::{
         engine::{
             data::{
@@ -76,7 +76,7 @@ macro_rules! can_ignore {
 }
 
 #[inline(always)]
-fn no_field(mr: &IWModel, new: &str) -> bool {
+fn no_field(mr: &Model, new: &str) -> bool {
     !mr.fields().st_contains(new)
 }
 
@@ -90,8 +90,7 @@ fn check_nullable(props: &mut HashMap<Box<str>, DictEntryGeneric>) -> QueryResul
 
 impl<'a> AlterPlan<'a> {
     pub fn fdeltas(
-        mv: &Model,
-        wm: &IWModel,
+        mdl: &Model,
         AlterModel { model, kind }: AlterModel<'a>,
     ) -> QueryResult<AlterPlan<'a>> {
         let mut no_lock = true;
@@ -104,8 +103,8 @@ impl<'a> AlterPlan<'a> {
                 }
                 let mut not_found = false;
                 if r.iter().all(|id| {
-                    let not_pk = mv.not_pk(id);
-                    let exists = !no_field(wm, id.as_str());
+                    let not_pk = mdl.not_pk(id);
+                    let exists = !no_field(mdl, id.as_str());
                     not_found = !exists;
                     not_pk & exists
                 }) {
@@ -125,7 +124,7 @@ impl<'a> AlterPlan<'a> {
                         layers,
                         mut props,
                     } = fields.next().unwrap();
-                    okay &= no_field(wm, &field_name) & mv.not_pk(&field_name);
+                    okay &= no_field(mdl, &field_name) & mdl.not_pk(&field_name);
                     let is_nullable = check_nullable(&mut props)?;
                     let layers = Field::parse_layers(layers, is_nullable)?;
                     okay &= add.st_insert(field_name.to_string().into_boxed_str(), layers);
@@ -144,9 +143,9 @@ impl<'a> AlterPlan<'a> {
                         mut props,
                     } = updated_fields.next().unwrap();
                     // enforce pk
-                    mv.guard_pk(&field_name)?;
+                    mdl.guard_pk(&field_name)?;
                     // get the current field
-                    let Some(current_field) = wm.fields().st_get(field_name.as_str()) else {
+                    let Some(current_field) = mdl.fields().st_get(field_name.as_str()) else {
                         return Err(QueryError::QExecUnknownField);
                     };
                     // check props
@@ -255,22 +254,18 @@ impl Model {
         let (space_name, model_name) = alter.model.into_full_result()?;
         global
             .namespace()
-            .with_model_space(alter.model, |space, model| {
-                // make intent
-                let iwm = model.intent_write_model();
+            .with_model_space_mut_for_ddl(alter.model, |space, model| {
                 // prepare plan
-                let plan = AlterPlan::fdeltas(model, &iwm, alter)?;
+                let plan = AlterPlan::fdeltas(model, alter)?;
                 // we have a legal plan; acquire exclusive if we need it
                 if !plan.no_lock {
                     // TODO(@ohsayan): allow this later on, once we define the syntax
                     return Err(QueryError::QExecNeedLock);
                 }
                 // fine, we're good
-                let mut iwm = iwm;
                 match plan.action {
-                    AlterAction::Ignore => drop(iwm),
+                    AlterAction::Ignore => {}
                     AlterAction::Add(new_fields) => {
-                        let mut guard = model.delta_state().schema_delta_write();
                         // TODO(@ohsayan): this impacts lockdown duration; fix it
                         if G::FS_IS_NON_NULL {
                             // prepare txn
@@ -291,13 +286,12 @@ impl Model {
                             .map(|(x, y)| (x.clone(), y.clone()))
                             .for_each(|(field_id, field)| {
                                 model
-                                    .delta_state()
-                                    .schema_append_unresolved_wl_field_add(&mut guard, &field_id);
-                                iwm.fields_mut().st_insert(field_id, field);
+                                    .delta_state_mut()
+                                    .schema_append_unresolved_wl_field_add(&field_id);
+                                model.fields_mut().st_insert(field_id, field);
                             });
                     }
                     AlterAction::Remove(removed) => {
-                        let mut guard = model.delta_state().schema_delta_write();
                         if G::FS_IS_NON_NULL {
                             // prepare txn
                             let txn = gnstxn::AlterModelRemoveTxn::new(
@@ -308,11 +302,10 @@ impl Model {
                             global.namespace_txn_driver().lock().try_commit(txn)?;
                         }
                         removed.iter().for_each(|field_id| {
-                            model.delta_state().schema_append_unresolved_wl_field_rem(
-                                &mut guard,
-                                field_id.as_str(),
-                            );
-                            iwm.fields_mut().st_delete(field_id.as_str());
+                            model
+                                .delta_state_mut()
+                                .schema_append_unresolved_wl_field_rem(field_id.as_str());
+                            model.fields_mut().st_delete(field_id.as_str());
                         });
                     }
                     AlterAction::Update(updated) => {
@@ -326,7 +319,7 @@ impl Model {
                             global.namespace_txn_driver().lock().try_commit(txn)?;
                         }
                         updated.into_iter().for_each(|(field_id, field)| {
-                            iwm.fields_mut().st_update(&field_id, field);
+                            model.fields_mut().st_update(&field_id, field);
                         });
                     }
                 }
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index a09c5514..2feb6cf0 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -25,136 +25,25 @@
 */
 
 use {
-    super::{Fields, Model},
+    super::Model,
     crate::engine::{
         core::{dml::QueryExecMeta, index::Row},
         fractal::{FractalToken, GlobalInstanceLike},
         sync::atm::Guard,
         sync::queue::Queue,
     },
-    parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard},
     std::{
         collections::btree_map::{BTreeMap, Range},
         sync::atomic::{AtomicU64, AtomicUsize, Ordering},
     },
 };
 
-/*
-    sync matrix
-*/
-
-// FIXME(@ohsayan): This an inefficient repr of the matrix; replace it with my other design
-#[derive(Debug)]
-/// A sync matrix enables different queries to have different access permissions on the data model, and the data in the
-/// index
-pub struct ISyncMatrix {
-    // virtual privileges
-    /// read/write model
-    v_priv_model_alter: RwLock<()>,
-    /// RW data/block all
-    v_priv_data_new_or_revise: RwLock<()>,
-}
-
-#[cfg(test)]
-impl PartialEq for ISyncMatrix {
-    fn eq(&self, _: &Self) -> bool {
-        true
-    }
-}
-
-#[derive(Debug)]
-/// Read model, write new data
-pub struct IRModelSMData<'a> {
-    _rmodel: RwLockReadGuard<'a, ()>,
-    _mdata: RwLockReadGuard<'a, ()>,
-    fields: &'a Fields,
-}
-
-impl<'a> IRModelSMData<'a> {
-    pub fn new(m: &'a Model) -> Self {
-        let rmodel = m.sync_matrix().v_priv_model_alter.read();
-        let mdata = m.sync_matrix().v_priv_data_new_or_revise.read();
-        Self {
-            _rmodel: rmodel,
-            _mdata: mdata,
-            fields: unsafe {
-                // UNSAFE(@ohsayan): we already have acquired this resource
-                m._read_fields()
-            },
-        }
-    }
-    pub fn fields(&'a self) -> &'a Fields {
-        self.fields
-    }
-}
-
-#[derive(Debug)]
-/// Read model
-pub struct IRModel<'a> {
-    _rmodel: RwLockReadGuard<'a, ()>,
-    fields: &'a Fields,
-}
-
-impl<'a> IRModel<'a> {
-    pub fn new(m: &'a Model) -> Self {
-        Self {
-            _rmodel: m.sync_matrix().v_priv_model_alter.read(),
-            fields: unsafe {
-                // UNSAFE(@ohsayan): we already have acquired this resource
-                m._read_fields()
-            },
-        }
-    }
-    pub fn fields(&'a self) -> &'a Fields {
-        self.fields
-    }
-}
-
-#[derive(Debug)]
-/// Write model
-pub struct IWModel<'a> {
-    _wmodel: RwLockWriteGuard<'a, ()>,
-    fields: &'a mut Fields,
-}
-
-impl<'a> IWModel<'a> {
-    pub fn new(m: &'a Model) -> Self {
-        Self {
-            _wmodel: m.sync_matrix().v_priv_model_alter.write(),
-            fields: unsafe {
-                // UNSAFE(@ohsayan): we have exclusive access to this resource
-                m._read_fields_mut()
-            },
-        }
-    }
-    pub fn fields(&'a self) -> &'a Fields {
-        self.fields
-    }
-    // ALIASING
-    pub fn fields_mut(&mut self) -> &mut Fields {
-        self.fields
-    }
-}
-
-impl ISyncMatrix {
-    pub const fn new() -> Self {
-        Self {
-            v_priv_model_alter: RwLock::new(()),
-            v_priv_data_new_or_revise: RwLock::new(()),
-        }
-    }
-}
-
-/*
-    delta
-*/
-
 #[derive(Debug)]
 /// A delta state for the model
 pub struct DeltaState {
     // schema
-    schema_current_version: AtomicU64,
-    schema_deltas: RwLock<BTreeMap<DeltaVersion, SchemaDeltaPart>>,
+    schema_current_version: u64,
+    schema_deltas: BTreeMap<DeltaVersion, SchemaDeltaPart>,
     // data
     data_current_version: AtomicU64,
     data_deltas: Queue<DataDelta>,
@@ -165,8 +54,8 @@ impl DeltaState {
     /// A new, fully resolved delta state with version counters set to 0
     pub fn new_resolved() -> Self {
         Self {
-            schema_current_version: AtomicU64::new(0),
-            schema_deltas: RwLock::new(BTreeMap::new()),
+            schema_current_version: 0,
+            schema_deltas: BTreeMap::new(),
             data_current_version: AtomicU64::new(0),
             data_deltas: Queue::new(),
             data_deltas_size: AtomicUsize::new(0),
@@ -218,51 +107,32 @@ impl DeltaState {
 
 // schema
 impl DeltaState {
-    pub fn schema_delta_write<'a>(&'a self) -> SchemaDeltaIndexWGuard<'a> {
-        SchemaDeltaIndexWGuard(self.schema_deltas.write())
-    }
-    pub fn schema_delta_read<'a>(&'a self) -> SchemaDeltaIndexRGuard<'a> {
-        SchemaDeltaIndexRGuard(self.schema_deltas.read())
-    }
-    pub fn schema_current_version(&self) -> DeltaVersion {
-        self.__schema_delta_current()
-    }
-    pub fn schema_append_unresolved_wl_field_add(
+    pub fn resolve_iter_since(
         &self,
-        guard: &mut SchemaDeltaIndexWGuard,
-        field_name: &str,
-    ) {
-        self.__schema_append_unresolved_delta(&mut guard.0, SchemaDeltaPart::field_add(field_name));
+        current_version: DeltaVersion,
+    ) -> Range<DeltaVersion, SchemaDeltaPart> {
+        self.schema_deltas.range(current_version.step()..)
     }
-    pub fn schema_append_unresolved_wl_field_rem(
-        &self,
-        guard: &mut SchemaDeltaIndexWGuard,
-        field_name: &str,
-    ) {
-        self.__schema_append_unresolved_delta(&mut guard.0, SchemaDeltaPart::field_rem(field_name));
+    pub fn schema_current_version(&self) -> DeltaVersion {
+        DeltaVersion(self.schema_current_version)
     }
-    pub fn schema_append_unresolved_field_add(&self, field_name: &str) {
-        self.schema_append_unresolved_wl_field_add(&mut self.schema_delta_write(), field_name);
+    pub fn schema_append_unresolved_wl_field_add(&mut self, field_name: &str) {
+        self.__schema_append_unresolved_delta(SchemaDeltaPart::field_add(field_name));
     }
-    pub fn schema_append_unresolved_field_rem(&self, field_name: &str) {
-        self.schema_append_unresolved_wl_field_rem(&mut self.schema_delta_write(), field_name);
+    pub fn schema_append_unresolved_wl_field_rem(&mut self, field_name: &str) {
+        self.__schema_append_unresolved_delta(SchemaDeltaPart::field_rem(field_name));
     }
 }
 
 impl DeltaState {
-    fn __schema_delta_step(&self) -> DeltaVersion {
-        DeltaVersion(self.schema_current_version.fetch_add(1, Ordering::AcqRel))
-    }
-    fn __schema_delta_current(&self) -> DeltaVersion {
-        DeltaVersion(self.schema_current_version.load(Ordering::Acquire))
+    fn __schema_delta_step(&mut self) -> DeltaVersion {
+        let current = self.schema_current_version;
+        self.schema_current_version += 1;
+        DeltaVersion(current)
     }
-    fn __schema_append_unresolved_delta(
-        &self,
-        w: &mut BTreeMap<DeltaVersion, SchemaDeltaPart>,
-        part: SchemaDeltaPart,
-    ) -> DeltaVersion {
+    fn __schema_append_unresolved_delta(&mut self, part: SchemaDeltaPart) -> DeltaVersion {
         let v = self.__schema_delta_step();
-        w.insert(v, part);
+        self.schema_deltas.insert(v, part);
         v
     }
 }
@@ -328,19 +198,6 @@ impl SchemaDeltaPart {
     }
 }
 
-pub struct SchemaDeltaIndexWGuard<'a>(
-    RwLockWriteGuard<'a, BTreeMap<DeltaVersion, SchemaDeltaPart>>,
-);
-pub struct SchemaDeltaIndexRGuard<'a>(RwLockReadGuard<'a, BTreeMap<DeltaVersion, SchemaDeltaPart>>);
-impl<'a> SchemaDeltaIndexRGuard<'a> {
-    pub fn resolve_iter_since(
-        &self,
-        current_version: DeltaVersion,
-    ) -> Range<DeltaVersion, SchemaDeltaPart> {
-        self.0.range(current_version.step()..)
-    }
-}
-
 /*
     data delta
 */
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 71203879..d965787c 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -31,7 +31,6 @@ pub(in crate::engine) mod delta;
 use std::cell::RefCell;
 
 use {
-    self::delta::{IRModel, IRModelSMData, ISyncMatrix, IWModel},
     super::index::PrimaryIndex,
     crate::engine::{
         data::{
@@ -50,7 +49,6 @@ use {
         },
         txn::gns::{self as gnstxn, SpaceIDRef},
     },
-    std::cell::UnsafeCell,
 };
 
 pub(in crate::engine::core) use self::delta::{DeltaState, DeltaVersion, SchemaDeltaKind};
@@ -63,8 +61,7 @@ pub struct Model {
     uuid: Uuid,
     p_key: Box<str>,
     p_tag: FullTag,
-    fields: UnsafeCell<Fields>,
-    sync_matrix: ISyncMatrix,
+    fields: Fields,
     data: PrimaryIndex,
     delta: DeltaState,
 }
@@ -72,11 +69,10 @@ pub struct Model {
 #[cfg(test)]
 impl PartialEq for Model {
     fn eq(&self, m: &Self) -> bool {
-        let mdl1 = self.intent_read_model();
-        let mdl2 = m.intent_read_model();
-        ((self.p_key == m.p_key) & (self.p_tag == m.p_tag))
-            && self.uuid == m.uuid
-            && mdl1.fields() == mdl2.fields()
+        self.uuid == m.uuid
+            && self.p_key == m.p_key
+            && self.p_tag == m.p_tag
+            && self.fields == m.fields
     }
 }
 
@@ -85,8 +81,7 @@ impl Model {
         uuid: Uuid,
         p_key: Box<str>,
         p_tag: FullTag,
-        fields: UnsafeCell<Fields>,
-        sync_matrix: ISyncMatrix,
+        fields: Fields,
         data: PrimaryIndex,
         delta: DeltaState,
     ) -> Self {
@@ -95,12 +90,10 @@ impl Model {
             p_key,
             p_tag,
             fields,
-            sync_matrix,
             data,
             delta,
         }
     }
-
     pub fn get_uuid(&self) -> Uuid {
         self.uuid
     }
@@ -110,24 +103,6 @@ impl Model {
     pub fn p_tag(&self) -> FullTag {
         self.p_tag
     }
-    pub fn sync_matrix(&self) -> &ISyncMatrix {
-        &self.sync_matrix
-    }
-    unsafe fn _read_fields<'a>(&'a self) -> &'a Fields {
-        &*self.fields.get().cast_const()
-    }
-    unsafe fn _read_fields_mut<'a>(&'a self) -> &'a mut Fields {
-        &mut *self.fields.get()
-    }
-    pub fn intent_read_model<'a>(&'a self) -> IRModel<'a> {
-        IRModel::new(self)
-    }
-    pub fn intent_write_model<'a>(&'a self) -> IWModel<'a> {
-        IWModel::new(self)
-    }
-    pub fn intent_write_new_data<'a>(&'a self) -> IRModelSMData<'a> {
-        IRModelSMData::new(self)
-    }
     fn is_pk(&self, new: &str) -> bool {
         self.p_key.as_bytes() == new.as_bytes()
     }
@@ -147,6 +122,15 @@ impl Model {
     pub fn delta_state(&self) -> &DeltaState {
         &self.delta
     }
+    pub fn delta_state_mut(&mut self) -> &mut DeltaState {
+        &mut self.delta
+    }
+    pub fn fields_mut(&mut self) -> &mut Fields {
+        &mut self.fields
+    }
+    pub fn fields(&self) -> &Fields {
+        &self.fields
+    }
 }
 
 impl Model {
@@ -155,8 +139,7 @@ impl Model {
             uuid,
             p_key,
             p_tag,
-            UnsafeCell::new(fields),
-            ISyncMatrix::new(),
+            fields,
             PrimaryIndex::new_empty(),
             DeltaState::new_resolved(),
         )
@@ -215,14 +198,12 @@ impl Model {
             }
             // since we've locked this down, no one else can parallely create another model in the same space (or remove)
             if G::FS_IS_NON_NULL {
-                let irm = model.intent_read_model();
                 let mut txn_driver = global.namespace_txn_driver().lock();
                 // prepare txn
                 let txn = gnstxn::CreateModelTxn::new(
                     SpaceIDRef::new(&space_name, &space),
                     &model_name,
                     &model,
-                    &irm,
                 );
                 // attempt to initialize driver
                 global.initialize_model_driver(
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index 5977b11e..5ec493f6 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -39,8 +39,7 @@ fn with_plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) -> QueryResult<()>
     let model = create(model)?;
     let tok = lex_insecure(plan.as_bytes()).unwrap();
     let alter = parse_ast_node_full(&tok[2..]).unwrap();
-    let model_write = model.intent_write_model();
-    let mv = AlterPlan::fdeltas(&model, &model_write, alter)?;
+    let mv = AlterPlan::fdeltas(&model, alter)?;
     Ok(f(mv))
 }
 fn plan(model: &str, plan: &str, f: impl Fn(AlterPlan)) {
@@ -366,9 +365,8 @@ mod exec {
             "create model myspace.mymodel(username: string, col1: uint64)",
             "alter model myspace.mymodel add (col2 { type: uint32, nullable: true }, col3 { type: uint16, nullable: true })",
             |model| {
-                let schema = model.intent_read_model();
                 assert_eq!(
-                    schema
+                    model
                         .fields()
                         .stseq_ord_kv()
                         .rev()
@@ -397,9 +395,8 @@ mod exec {
             "create model myspace.mymodel(username: string, col1: uint64, col2: uint32, col3: uint16, col4: uint8)",
             "alter model myspace.mymodel remove (col1, col2, col3, col4)",
             |mdl| {
-                let schema = mdl.intent_read_model();
                 assert_eq!(
-                    schema
+                    mdl
                         .fields()
                         .stseq_ord_kv()
                         .rev()
@@ -423,8 +420,7 @@ mod exec {
             "create model myspace.mymodel(username: string, password: binary)",
             "alter model myspace.mymodel update password { nullable: true }",
             |model| {
-                let schema = model.intent_read_model();
-                assert!(schema.fields().st_get("password").unwrap().is_nullable());
+                assert!(model.fields().st_get("password").unwrap().is_nullable());
                 assert_eq!(
                     model.delta_state().schema_current_version(),
                     DeltaVersion::genesis()
diff --git a/server/src/engine/core/tests/ddl_model/crt.rs b/server/src/engine/core/tests/ddl_model/crt.rs
index fa52a40d..909582c2 100644
--- a/server/src/engine/core/tests/ddl_model/crt.rs
+++ b/server/src/engine/core/tests/ddl_model/crt.rs
@@ -42,7 +42,6 @@ mod validation {
         assert_eq!(model.p_tag(), FullTag::STR);
         assert_eq!(
             model
-                .intent_read_model()
                 .fields()
                 .stseq_ord_value()
                 .cloned()
@@ -66,7 +65,6 @@ mod validation {
         assert_eq!(model.p_tag(), FullTag::STR);
         assert_eq!(
             model
-                .intent_read_model()
                 .fields()
                 .stseq_ord_value()
                 .cloned()
@@ -153,7 +151,6 @@ mod exec {
         .unwrap();
         with_model(&global, SPACE, "mymodel", |model| {
             let models: Vec<(String, Field)> = model
-                .intent_read_model()
                 .fields()
                 .stseq_ord_kv()
                 .map(|(k, v)| (k.to_string(), v.clone()))
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index f06779ae..41dd7854 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -74,7 +74,6 @@ fn _exec_only_read_key_and_then<T>(
 ) -> QueryResult<T> {
     let guard = sync::atm::cpin();
     global.namespace().with_model(entity, |mdl| {
-        let _irm = mdl.intent_read_model();
         let row = mdl
             .primary_index()
             .select(Lit::from(key_name), &guard)
@@ -92,7 +91,6 @@ fn _exec_delete_only(global: &impl GlobalInstanceLike, delete: &str, key: &str)
     dml::delete(global, delete)?;
     assert_eq!(
         global.namespace().with_model(entity, |model| {
-            let _ = model.intent_read_model();
             let g = sync::atm::cpin();
             Ok(model.primary_index().select(key.into(), &g).is_none())
         }),
diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs
index ec0d3613..d0b8905c 100644
--- a/server/src/engine/storage/v1/batch_jrnl/persist.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs
@@ -36,7 +36,7 @@ use {
             core::{
                 index::{PrimaryIndexKey, RowData},
                 model::{
-                    delta::{DataDelta, DataDeltaKind, DeltaVersion, IRModel},
+                    delta::{DataDelta, DataDeltaKind, DeltaVersion},
                     Model,
                 },
             },
@@ -76,7 +76,6 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
     }
     pub fn write_new_batch(&mut self, model: &Model, observed_len: usize) -> RuntimeResult<()> {
         // pin model
-        let irm = model.intent_read_model();
         let schema_version = model.delta_state().schema_current_version();
         let g = pin();
         // init restore list
@@ -90,7 +89,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
                 observed_len,
                 schema_version,
                 model.p_tag().tag_unique(),
-                irm.fields().len() - 1,
+                model.fields().len() - 1,
             )?;
             while i < observed_len {
                 let delta = model.delta_state().__data_delta_dequeue(&g).unwrap();
@@ -116,7 +115,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
                         self.write_batch_item_common_row_data(&delta)?;
                         // encode data
                         self.encode_pk_only(delta.row().d_key())?;
-                        self.encode_row_data(model, &irm, &row_data)?;
+                        self.encode_row_data(model, &row_data)?;
                     }
                 }
                 i += 1;
@@ -227,18 +226,13 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         Ok(())
     }
     /// Encode row data
-    fn encode_row_data(
-        &mut self,
-        mdl: &Model,
-        irm: &IRModel,
-        row_data: &RowData,
-    ) -> RuntimeResult<()> {
-        for field_name in irm.fields().stseq_ord_key() {
+    fn encode_row_data(&mut self, model: &Model, row_data: &RowData) -> RuntimeResult<()> {
+        for field_name in model.fields().stseq_ord_key() {
             match row_data.fields().get(field_name) {
                 Some(cell) => {
                     self.encode_cell(cell)?;
                 }
-                None if field_name.as_ref() == mdl.p_key() => {}
+                None if field_name.as_ref() == model.p_key() => {}
                 None => self.f.write_unfsynced(&[0])?,
             }
         }
diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs
index c5b1bace..5851d044 100644
--- a/server/src/engine/storage/v1/batch_jrnl/restore.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs
@@ -215,7 +215,6 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
     ) -> RuntimeResult<()> {
         // NOTE(@ohsayan): current complexity is O(n) which is good enough (in the future I might revise this to a fancier impl)
         // pin model
-        let irm = m.intent_read_model();
         let g = unsafe { crossbeam_epoch::unprotected() };
         let mut pending_delete = HashMap::new();
         let p_index = m.primary_index().__raw_index();
@@ -235,7 +234,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                             // new row (logically)
                             let _ = p_index.mt_delete(&pk, &g);
                             let mut data = DcFieldIndex::default();
-                            for (field_name, new_data) in irm
+                            for (field_name, new_data) in m
                                 .fields()
                                 .stseq_ord_key()
                                 .filter(|key| key.as_ref() != m.p_key())
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index 43c2d727..f2ed1372 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -29,7 +29,7 @@ use {
     crate::{
         engine::{
             core::{
-                model::{delta::IRModel, Field, Layer, Model},
+                model::{Field, Layer, Model},
                 space::Space,
             },
             data::{
@@ -405,10 +405,10 @@ impl ModelLayoutMD {
 }
 
 #[derive(Clone, Copy)]
-pub struct ModelLayoutRef<'a>(pub(super) &'a Model, pub(super) &'a IRModel<'a>);
-impl<'a> From<(&'a Model, &'a IRModel<'a>)> for ModelLayoutRef<'a> {
-    fn from((mdl, irm): (&'a Model, &'a IRModel<'a>)) -> Self {
-        Self(mdl, irm)
+pub struct ModelLayoutRef<'a>(pub(super) &'a Model);
+impl<'a> From<&'a Model> for ModelLayoutRef<'a> {
+    fn from(mdl: &'a Model) -> Self {
+        Self(mdl)
     }
 }
 impl<'a> PersistObject for ModelLayoutRef<'a> {
@@ -419,11 +419,11 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
     fn pretest_can_dec_object(scanner: &BufferedScanner, md: &Self::Metadata) -> bool {
         scanner.has_left(md.p_key_len as usize)
     }
-    fn meta_enc(buf: &mut VecU8, ModelLayoutRef(v, irm): Self::InputType) {
-        buf.extend(v.get_uuid().to_le_bytes());
-        buf.extend(v.p_key().len().u64_bytes_le());
-        buf.extend(v.p_tag().tag_selector().value_qword().to_le_bytes());
-        buf.extend(irm.fields().len().u64_bytes_le());
+    fn meta_enc(buf: &mut VecU8, ModelLayoutRef(model_def): Self::InputType) {
+        buf.extend(model_def.get_uuid().to_le_bytes());
+        buf.extend(model_def.p_key().len().u64_bytes_le());
+        buf.extend(model_def.p_tag().tag_selector().value_qword().to_le_bytes());
+        buf.extend(model_def.fields().len().u64_bytes_le());
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         Ok(ModelLayoutMD::new(
@@ -433,11 +433,11 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
             scanner.next_u64_le(),
         ))
     }
-    fn obj_enc(buf: &mut VecU8, ModelLayoutRef(mdl, irm): Self::InputType) {
-        buf.extend(mdl.p_key().as_bytes());
+    fn obj_enc(buf: &mut VecU8, ModelLayoutRef(model_definition): Self::InputType) {
+        buf.extend(model_definition.p_key().as_bytes());
         <super::map::PersistMapImpl<super::map::FieldMapSpec> as PersistObject>::obj_enc(
             buf,
-            irm.fields(),
+            model_definition.fields(),
         )
     }
     unsafe fn obj_dec(
diff --git a/server/src/engine/storage/v1/inf/tests.rs b/server/src/engine/storage/v1/inf/tests.rs
index b402854e..a942652a 100644
--- a/server/src/engine/storage/v1/inf/tests.rs
+++ b/server/src/engine/storage/v1/inf/tests.rs
@@ -102,8 +102,7 @@ fn model() {
             "profile_pic" => Field::new([Layer::bin()].into(), true),
         },
     );
-    let model_irm = model.intent_read_model();
-    let enc = super::enc::enc_full::<obj::ModelLayoutRef>(obj::ModelLayoutRef(&model, &model_irm));
+    let enc = super::enc::enc_full::<obj::ModelLayoutRef>(obj::ModelLayoutRef(&model));
     let dec = super::dec::dec_full::<obj::ModelLayoutRef>(&enc).unwrap();
     assert_eq!(model, dec);
 }
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index 0dc8a482..629778e3 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -29,7 +29,7 @@ use {
     crate::{
         engine::{
             core::{
-                model::{delta::IRModel, Field, Model},
+                model::{Field, Model},
                 space::Space,
                 GlobalNS, {EntityID, EntityIDRef},
             },
@@ -187,15 +187,15 @@ fn with_space_mut<T>(
     f(&mut space)
 }
 
-fn with_model<T>(
+fn with_model_mut<T>(
     gns: &GlobalNS,
     space_id: &super::SpaceIDRes,
     model_id: &ModelIDRes,
-    mut f: impl FnMut(&Model) -> RuntimeResult<T>,
+    mut f: impl FnMut(&mut Model) -> RuntimeResult<T>,
 ) -> RuntimeResult<T> {
     with_space(gns, space_id, |_| {
-        let models = gns.idx_models().read();
-        let Some(model) = models.get(&EntityIDRef::new(&space_id.name, &model_id.model_name))
+        let mut models = gns.idx_models().write();
+        let Some(model) = models.get_mut(&EntityIDRef::new(&space_id.name, &model_id.model_name))
         else {
             return Err(TransactionError::OnRestoreDataMissing.into());
         };
@@ -217,7 +217,6 @@ pub struct CreateModelTxn<'a> {
     space_id: super::SpaceIDRef<'a>,
     model_name: &'a str,
     model: &'a Model,
-    model_read: &'a IRModel<'a>,
 }
 
 impl<'a> CreateModelTxn<'a> {
@@ -225,13 +224,11 @@ impl<'a> CreateModelTxn<'a> {
         space_id: super::SpaceIDRef<'a>,
         model_name: &'a str,
         model: &'a Model,
-        model_read: &'a IRModel<'a>,
     ) -> Self {
         Self {
             space_id,
             model_name,
             model,
-            model_read,
         }
     }
 }
@@ -266,10 +263,7 @@ impl<'a> PersistObject for CreateModelTxn<'a> {
         // model name
         buf.extend(data.model_name.len().u64_bytes_le());
         // model meta dump
-        <obj::ModelLayoutRef as PersistObject>::meta_enc(
-            buf,
-            obj::ModelLayoutRef::from((data.model, data.model_read)),
-        )
+        <obj::ModelLayoutRef as PersistObject>::meta_enc(buf, obj::ModelLayoutRef::from(data.model))
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         let space_id = <super::SpaceID as PersistObject>::meta_dec(scanner)?;
@@ -287,10 +281,7 @@ impl<'a> PersistObject for CreateModelTxn<'a> {
         // model name
         buf.extend(data.model_name.as_bytes());
         // model dump
-        <obj::ModelLayoutRef as PersistObject>::obj_enc(
-            buf,
-            obj::ModelLayoutRef::from((data.model, data.model_read)),
-        )
+        <obj::ModelLayoutRef as PersistObject>::obj_enc(buf, obj::ModelLayoutRef::from(data.model))
     }
     unsafe fn obj_dec(
         s: &mut BufferedScanner,
@@ -432,16 +423,15 @@ impl<'a> GNSEvent for AlterModelAddTxn<'a> {
         }: Self::RestoreType,
         gns: &GlobalNS,
     ) -> RuntimeResult<()> {
-        with_model(gns, &model_id.space_id, &model_id, |model| {
-            let mut wmodel = model.intent_write_model();
+        with_model_mut(gns, &model_id.space_id, &model_id, |model| {
             for (i, (field_name, field)) in new_fields.stseq_ord_kv().enumerate() {
-                if !wmodel
+                if !model
                     .fields_mut()
                     .st_insert(field_name.to_owned(), field.clone())
                 {
                     // rollback; corrupted
                     new_fields.stseq_ord_key().take(i).for_each(|field_id| {
-                        let _ = wmodel.fields_mut().st_delete(field_id);
+                        let _ = model.fields_mut().st_delete(field_id);
                     });
                     return Err(TransactionError::OnRestoreDataConflictMismatch.into());
                 }
@@ -450,8 +440,8 @@ impl<'a> GNSEvent for AlterModelAddTxn<'a> {
             // publish deltas
             for field_name in new_fields.stseq_ord_key() {
                 model
-                    .delta_state()
-                    .schema_append_unresolved_field_add(field_name);
+                    .delta_state_mut()
+                    .schema_append_unresolved_wl_field_add(field_name);
             }
             Ok(())
         })
@@ -551,18 +541,17 @@ impl<'a> GNSEvent for AlterModelRemoveTxn<'a> {
         }: Self::RestoreType,
         gns: &GlobalNS,
     ) -> RuntimeResult<()> {
-        with_model(gns, &model_id.space_id, &model_id, |model| {
-            let mut iwm = model.intent_write_model();
+        with_model_mut(gns, &model_id.space_id, &model_id, |model| {
             let mut removed_fields_rb = vec![];
             for removed_field in removed_fields.iter() {
-                match iwm.fields_mut().st_delete_return(removed_field) {
+                match model.fields_mut().st_delete_return(removed_field) {
                     Some(field) => {
                         removed_fields_rb.push((removed_field as &str, field));
                     }
                     None => {
                         // rollback
                         removed_fields_rb.into_iter().for_each(|(field_id, field)| {
-                            let _ = iwm.fields_mut().st_insert(field_id.into(), field);
+                            let _ = model.fields_mut().st_insert(field_id.into(), field);
                         });
                         return Err(TransactionError::OnRestoreDataConflictMismatch.into());
                     }
@@ -572,8 +561,8 @@ impl<'a> GNSEvent for AlterModelRemoveTxn<'a> {
             // publish deltas
             for field_name in removed_fields.iter() {
                 model
-                    .delta_state()
-                    .schema_append_unresolved_field_rem(field_name);
+                    .delta_state_mut()
+                    .schema_append_unresolved_wl_field_rem(field_name);
             }
             Ok(())
         })
@@ -667,16 +656,15 @@ impl<'a> GNSEvent for AlterModelUpdateTxn<'a> {
         }: Self::RestoreType,
         gns: &GlobalNS,
     ) -> RuntimeResult<()> {
-        with_model(gns, &model_id.space_id, &model_id, |model| {
-            let mut iwm = model.intent_write_model();
+        with_model_mut(gns, &model_id.space_id, &model_id, |model| {
             let mut fields_rb = vec![];
             for (field_id, field) in updated_fields.iter() {
-                match iwm.fields_mut().st_update_return(field_id, field.clone()) {
+                match model.fields_mut().st_update_return(field_id, field.clone()) {
                     Some(f) => fields_rb.push((field_id as &str, f)),
                     None => {
                         // rollback
                         fields_rb.into_iter().for_each(|(field_id, field)| {
-                            let _ = iwm.fields_mut().st_update(field_id, field);
+                            let _ = model.fields_mut().st_update(field_id, field);
                         });
                         return Err(TransactionError::OnRestoreDataConflictMismatch.into());
                     }
diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
index 7ec18033..191d663f 100644
--- a/server/src/engine/txn/gns/tests/full_chain.rs
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -219,11 +219,7 @@ fn alter_model_add() {
                 .namespace()
                 .with_model(("myspace", "mymodel").into(), |model| {
                     assert_eq!(
-                        model
-                            .intent_read_model()
-                            .fields()
-                            .st_get("profile_pic")
-                            .unwrap(),
+                        model.fields().st_get("profile_pic").unwrap(),
                         &Field::new([Layer::bin()].into(), true)
                     );
                     Ok(())
@@ -257,9 +253,8 @@ fn alter_model_remove() {
             global
                 .namespace()
                 .with_model(("myspace", "mymodel").into(), |model| {
-                    let irm = model.intent_read_model();
-                    assert!(irm.fields().st_get("has_secure_key").is_none());
-                    assert!(irm.fields().st_get("is_dumb").is_none());
+                    assert!(model.fields().st_get("has_secure_key").is_none());
+                    assert!(model.fields().st_get("is_dumb").is_none());
                     Ok(())
                 })
                 .unwrap();
@@ -291,11 +286,7 @@ fn alter_model_update() {
                 .namespace()
                 .with_model(("myspace", "mymodel").into(), |model| {
                     assert_eq!(
-                        model
-                            .intent_read_model()
-                            .fields()
-                            .st_get("profile_pic")
-                            .unwrap(),
+                        model.fields().st_get("profile_pic").unwrap(),
                         &Field::new([Layer::bin()].into(), true)
                     );
                     Ok(())
diff --git a/server/src/engine/txn/gns/tests/io.rs b/server/src/engine/txn/gns/tests/io.rs
index 470aac63..fd6b6dcc 100644
--- a/server/src/engine/txn/gns/tests/io.rs
+++ b/server/src/engine/txn/gns/tests/io.rs
@@ -118,15 +118,8 @@ mod model_tests {
     #[test]
     fn create() {
         let (space, model) = default_space_model();
-        let irm = model.intent_read_model();
-        let txn = CreateModelTxn::new(
-            super::SpaceIDRef::new("myspace", &space),
-            "mymodel",
-            &model,
-            &irm,
-        );
+        let txn = CreateModelTxn::new(super::SpaceIDRef::new("myspace", &space), "mymodel", &model);
         let encoded = super::enc::enc_full_self(txn);
-        core::mem::drop(irm);
         let decoded = super::dec::dec_full::<CreateModelTxn>(&encoded).unwrap();
         assert_eq!(
             CreateModelTxnRestorePL {

From 2b4d9efb1b44de7f0109b471bff84eb0f30c16a5 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 23 Nov 2023 16:21:19 +0530
Subject: [PATCH 290/310] Simplify response handling

---
 server/src/engine/core/dml/sel.rs     |  25 ++--
 server/src/engine/core/model/mod.rs   |   2 +-
 server/src/engine/mem/mod.rs          |   2 +
 server/src/engine/mem/numbuf.rs       | 174 ++++++++++++++++++++++++++
 server/src/engine/net/protocol/mod.rs |  13 +-
 5 files changed, 194 insertions(+), 22 deletions(-)
 create mode 100644 server/src/engine/mem/numbuf.rs

diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
index 146850cf..bb586677 100644
--- a/server/src/engine/core/dml/sel.rs
+++ b/server/src/engine/core/dml/sel.rs
@@ -33,6 +33,7 @@ use crate::engine::{
     error::{QueryError, QueryResult},
     fractal::GlobalInstanceLike,
     idx::{STIndex, STIndexSeq},
+    mem::IntegerRepr,
     net::protocol::Response,
     ql::dml::sel::SelectStatement,
     sync,
@@ -42,20 +43,13 @@ pub fn select_resp(
     global: &impl GlobalInstanceLike,
     select: SelectStatement,
 ) -> QueryResult<Response> {
-    let mut resp_b = vec![];
-    let mut resp_a = vec![];
-    let mut i = 0u64;
+    let mut data = vec![];
+    let mut i = 0usize;
     self::select_custom(global, select, |item| {
-        encode_cell(&mut resp_b, item);
+        encode_cell(&mut data, item);
         i += 1;
     })?;
-    resp_a.push(0x11);
-    resp_a.extend(i.to_string().as_bytes());
-    resp_a.push(b'\n');
-    Ok(Response::EncodedAB(
-        resp_a.into_boxed_slice(),
-        resp_b.into_boxed_slice(),
-    ))
+    Ok(Response::Row { size: i, data })
 }
 
 fn encode_cell(resp: &mut Vec<u8>, item: &Datacell) {
@@ -65,15 +59,14 @@ fn encode_cell(resp: &mut Vec<u8>, item: &Datacell) {
     }
     unsafe {
         // UNSAFE(@ohsayan): +tagck
-        // NOTE(@ohsayan): optimize out unwanted alloc
         match item.tag().tag_class() {
             TagClass::Bool => resp.push(item.read_bool() as _),
-            TagClass::UnsignedInt => resp.extend(item.read_uint().to_string().as_bytes()),
-            TagClass::SignedInt => resp.extend(item.read_sint().to_string().as_bytes()),
+            TagClass::UnsignedInt => IntegerRepr::scoped(item.read_uint(), |b| resp.extend(b)),
+            TagClass::SignedInt => IntegerRepr::scoped(item.read_sint(), |b| resp.extend(b)),
             TagClass::Float => resp.extend(item.read_float().to_string().as_bytes()),
             TagClass::Bin | TagClass::Str => {
                 let slc = item.read_bin();
-                resp.extend(slc.len().to_string().as_bytes());
+                IntegerRepr::scoped(slc.len() as u64, |b| resp.extend(b));
                 resp.push(b'\n');
                 resp.extend(slc);
                 return;
@@ -81,7 +74,7 @@ fn encode_cell(resp: &mut Vec<u8>, item: &Datacell) {
             TagClass::List => {
                 let list = item.read_list();
                 let ls = list.read();
-                resp.extend(ls.len().to_string().as_bytes());
+                IntegerRepr::scoped(ls.len() as u64, |b| resp.extend(b));
                 resp.push(b'\n');
                 for item in ls.iter() {
                     encode_cell(resp, item);
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index d965787c..85a52365 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -54,7 +54,7 @@ use {
 pub(in crate::engine::core) use self::delta::{DeltaState, DeltaVersion, SchemaDeltaKind};
 
 use super::util::{EntityID, EntityIDRef};
-pub(in crate::engine) type Fields = IndexSTSeqCns<Box<str>, Field>;
+type Fields = IndexSTSeqCns<Box<str>, Field>;
 
 #[derive(Debug)]
 pub struct Model {
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index abf66651..6d7de7ef 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -26,6 +26,7 @@
 
 mod astr;
 mod ll;
+mod numbuf;
 pub mod scanner;
 mod stackop;
 mod uarray;
@@ -42,6 +43,7 @@ pub use {
     uarray::UArray,
     vinline::VInline,
     word::{DwordNN, DwordQN, WordIO, ZERO_BLOCK},
+    numbuf::IntegerRepr,
 };
 // imports
 use std::alloc::{self, Layout};
diff --git a/server/src/engine/mem/numbuf.rs b/server/src/engine/mem/numbuf.rs
new file mode 100644
index 00000000..7f0dc7e3
--- /dev/null
+++ b/server/src/engine/mem/numbuf.rs
@@ -0,0 +1,174 @@
+/*
+ * Created on Thu Nov 23 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+/*
+    derived from the implementation in libcore
+*/
+
+use core::{mem, ptr, slice};
+
+pub trait Int {
+    type Buffer: Default;
+    fn init_buf() -> Self::Buffer {
+        Self::Buffer::default()
+    }
+    fn init(self, buf: &mut Self::Buffer) -> &[u8];
+}
+
+pub struct IntegerRepr<I: Int> {
+    b: I::Buffer,
+}
+
+impl<I: Int> IntegerRepr<I> {
+    pub fn new() -> Self {
+        Self { b: I::init_buf() }
+    }
+    pub fn as_bytes(&mut self, i: I) -> &[u8] {
+        i.init(&mut self.b)
+    }
+    pub fn scoped<T>(i: I, mut f: impl FnMut(&[u8]) -> T) -> T {
+        let mut slf = Self::new();
+        f(slf.as_bytes(i))
+    }
+    #[cfg(test)]
+    pub fn as_str(&mut self, i: I) -> &str {
+        unsafe { core::mem::transmute(self.as_bytes(i)) }
+    }
+}
+
+const DEC_DIGITS_LUT: &[u8] = b"\
+      0001020304050607080910111213141516171819\
+      2021222324252627282930313233343536373839\
+      4041424344454647484950515253545556575859\
+      6061626364656667686970717273747576777879\
+      8081828384858687888990919293949596979899";
+
+macro_rules! impl_int {
+    ($($($int:ty => $max:literal),* as $cast:ty),*) => {
+        $($(impl Int for $int {
+            type Buffer = [u8; $max];
+            fn init(self, buf: &mut Self::Buffer) -> &[u8] {
+                #[allow(unused_comparisons)]
+                let negative = self < 0;
+                let mut n = if negative {
+                    // two's complement (invert, add 1)
+                    ((!(self as $cast)).wrapping_add(1))
+                } else {
+                    self as $cast
+                };
+                let mut curr_idx = buf.len() as isize;
+                let buf_ptr = buf.as_mut_ptr();
+                let lut_ptr = DEC_DIGITS_LUT.as_ptr();
+                unsafe {
+                    if mem::size_of::<Self>() >= 2 {
+                        while n >= 10_000 {
+                            let rem = (n % 10_000) as isize;
+                            n /= 10_000;
+                            let d1 = (rem / 100) << 1;
+                            let d2 = (rem % 100) << 1;
+                            curr_idx -= 4;
+                            ptr::copy_nonoverlapping(lut_ptr.offset(d1), buf_ptr.offset(curr_idx), 2);
+                            ptr::copy_nonoverlapping(lut_ptr.offset(d2), buf_ptr.offset(curr_idx + 2), 2);
+                        }
+                    }
+                    // 4 chars left
+                    let mut n = n as isize;
+                    // 2 chars
+                    if n >= 100 {
+                        let d1 = (n % 100) << 1;
+                        n /= 100;
+                        curr_idx -= 2;
+                        ptr::copy_nonoverlapping(lut_ptr.offset(d1), buf_ptr.offset(curr_idx), 2);
+                    }
+                    // 1 or 2 left
+                    if n < 10 {
+                        curr_idx -= 1;
+                        *buf_ptr.offset(curr_idx) = (n as u8) + b'0';
+                    } else {
+                        let d1 = n << 1;
+                        curr_idx -= 2;
+                        ptr::copy_nonoverlapping(lut_ptr.offset(d1), buf_ptr.offset(curr_idx), 2);
+                    }
+                    if negative {
+                        curr_idx -= 1;
+                        *buf_ptr.offset(curr_idx) = b'-';
+                    }
+                    slice::from_raw_parts(buf_ptr.offset(curr_idx), buf.len() - curr_idx as usize)
+                }
+            }
+        })*)*
+    };
+}
+
+impl_int!(u8 => 3, i8 => 4, u16 => 5, i16 => 6, u32 => 10, i32 => 11 as u32, u64 => 20, i64 => 20 as u64);
+
+#[cfg(test)]
+mod tests {
+    fn ibufeq<I: super::Int + ToString + Copy>(v: I) {
+        let mut buf = super::IntegerRepr::new();
+        assert_eq!(buf.as_str(v), v.to_string());
+    }
+    #[test]
+    fn u8() {
+        ibufeq(u8::MIN);
+        ibufeq(u8::MAX);
+    }
+    #[test]
+    fn i8() {
+        ibufeq(i8::MIN);
+        ibufeq(i8::MAX);
+    }
+    #[test]
+    fn u16() {
+        ibufeq(u16::MIN);
+        ibufeq(u16::MAX);
+    }
+    #[test]
+    fn i16() {
+        ibufeq(i16::MIN);
+        ibufeq(i16::MAX);
+    }
+    #[test]
+    fn u32() {
+        ibufeq(u32::MIN);
+        ibufeq(u32::MAX);
+    }
+    #[test]
+    fn i32() {
+        ibufeq(i32::MIN);
+        ibufeq(i32::MAX);
+    }
+    #[test]
+    fn u64() {
+        ibufeq(u64::MIN);
+        ibufeq(u64::MAX);
+    }
+    #[test]
+    fn i64() {
+        ibufeq(i64::MIN);
+        ibufeq(i64::MAX);
+    }
+}
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index c539309c..da804566 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -45,7 +45,7 @@ use {
         self,
         error::QueryError,
         fractal::{Global, GlobalInstanceLike},
-        mem::BufferedScanner,
+        mem::{BufferedScanner, IntegerRepr},
     },
     bytes::{Buf, BytesMut},
     tokio::io::{AsyncReadExt, AsyncWriteExt, BufWriter},
@@ -73,7 +73,7 @@ impl ClientLocalState {
 #[derive(Debug, PartialEq)]
 pub enum Response {
     Empty,
-    EncodedAB(Box<[u8]>, Box<[u8]>),
+    Row { size: usize, data: Vec<u8> },
 }
 
 pub(super) async fn query_loop<S: Socket>(
@@ -139,9 +139,12 @@ pub(super) async fn query_loop<S: Socket>(
             Ok(Response::Empty) => {
                 con.write_all(&[0x12]).await?;
             }
-            Ok(Response::EncodedAB(a, b)) => {
-                con.write_all(&a).await?;
-                con.write_all(&b).await?;
+            Ok(Response::Row { size, data }) => {
+                con.write_u8(0x11).await?;
+                let mut irep = IntegerRepr::new();
+                con.write_all(irep.as_bytes(size as u64)).await?;
+                con.write_u8(b'\n').await?;
+                con.write_all(&data).await?;
             }
             Err(e) => {
                 let [a, b] = (e.value_u8() as u16).to_le_bytes();

From 23f42969826d2999b31e1840af46151b71bcff0d Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 23 Nov 2023 23:27:31 +0530
Subject: [PATCH 291/310] Pass references for model data

Also fixed a bug that reported false positives of data corruption
resulting from incorrect checksum tracking.
---
 server/src/engine/config.rs                   |   5 -
 server/src/engine/core/dml/ins.rs             |  34 +-
 server/src/engine/core/exec.rs                |  46 +--
 server/src/engine/core/index/row.rs           |  15 +-
 server/src/engine/core/model/alt.rs           |  20 +-
 server/src/engine/core/model/delta.rs         |  21 +-
 server/src/engine/core/model/mod.rs           | 194 ++++++++--
 server/src/engine/core/tests/ddl_model/alt.rs |   4 +-
 server/src/engine/idx/mod.rs                  |  13 +-
 server/src/engine/idx/stord/config.rs         |   4 +-
 server/src/engine/idx/stord/iter.rs           | 118 ++++++-
 server/src/engine/idx/stord/mod.rs            |  23 +-
 server/src/engine/mem/mod.rs                  |   4 +-
 server/src/engine/mem/rawslice.rs             | 158 +++++++++
 server/src/engine/ql/dcl.rs                   |   2 +-
 .../engine/storage/v1/batch_jrnl/persist.rs   |  28 +-
 .../engine/storage/v1/batch_jrnl/restore.rs   |  11 +-
 server/src/engine/storage/v1/inf/map.rs       | 333 +++++++++---------
 server/src/engine/storage/v1/inf/mod.rs       | 126 ++++---
 server/src/engine/storage/v1/inf/obj.rs       |  14 +-
 server/src/engine/storage/v1/inf/tests.rs     |   7 +-
 server/src/engine/storage/v1/loader.rs        |   8 +-
 server/src/engine/storage/v1/rw.rs            |   5 +-
 server/src/engine/txn/gns/model.rs            |  77 ++--
 server/src/engine/txn/gns/tests/io.rs         |  10 +-
 25 files changed, 838 insertions(+), 442 deletions(-)
 create mode 100644 server/src/engine/mem/rawslice.rs

diff --git a/server/src/engine/config.rs b/server/src/engine/config.rs
index 330d0639..17c48494 100644
--- a/server/src/engine/config.rs
+++ b/server/src/engine/config.rs
@@ -1202,11 +1202,6 @@ fn check_config_file(
             Some(ep) => match ep.secure.as_mut() {
                 Some(secure_ep) => {
                     super::fractal::context::set_dmsg("loading TLS configuration from disk");
-                    dbg!(
-                        &secure_ep.cert,
-                        &secure_ep.private_key,
-                        &secure_ep.pkey_passphrase
-                    );
                     let cert = fs::read_to_string(&secure_ep.cert)?;
                     let private_key = fs::read_to_string(&secure_ep.private_key)?;
                     let private_key_passphrase = fs::read_to_string(&secure_ep.pkey_passphrase)?;
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index 1bc3d183..24cc73b8 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -87,23 +87,35 @@ fn prepare_insert(
                 }
                 let (field_id, field) = field;
                 okay &= field.vt_data_fpath(&mut data);
-                okay &= prepared_data.st_insert(field_id.clone(), data);
+                okay &= prepared_data.st_insert(
+                    unsafe {
+                        // UNSAFE(@ohsayan): the model is right here, so we're good
+                        field_id.clone()
+                    },
+                    data,
+                );
             }
         }
         InsertData::Map(map) => {
-            let mut map = map.into_iter();
-            while (map.len() != 0) & okay {
-                let (field_id, mut field_data) = unsafe {
+            let mut inserted = 0;
+            let mut iter = fields.st_iter_kv().zip(map.into_iter());
+            while (iter.len() != 0) & (okay) {
+                let ((model_field_key, model_field_spec), (this_field_key, mut this_field_data)) = unsafe {
                     // UNSAFE(@ohsayan): safe because of loop invariant
-                    map.next().unwrap_unchecked()
+                    iter.next().unwrap_unchecked()
                 };
-                let Some(field) = fields.st_get_cloned(field_id.as_str()) else {
-                    okay = false;
-                    break;
-                };
-                okay &= field.vt_data_fpath(&mut field_data);
-                prepared_data.st_insert(field_id.boxed_str(), field_data);
+                okay &= model_field_spec.vt_data_fpath(&mut this_field_data);
+                okay &= model_field_key.as_str() == this_field_key.as_str();
+                prepared_data.st_insert(
+                    unsafe {
+                        // UNSAFE(@ohsayan): the model is right here. it saves us the work!
+                        model_field_key.clone()
+                    },
+                    this_field_data,
+                );
+                inserted += 1;
             }
+            okay &= inserted == fields.len();
         }
     }
     let primary_key = prepared_data.remove(model.p_key());
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 6c4b69b8..8436ff65 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -24,18 +24,16 @@
  *
 */
 
-use {
-    crate::engine::{
-        core::{dml, model::Model, space::Space},
-        error::{QueryError, QueryResult},
-        fractal::Global,
-        net::protocol::{ClientLocalState, Response, SQuery},
-        ql::{
-            ast::{traits::ASTNode, InplaceData, State},
-            lex::{Keyword, KeywordStmt, Token},
-        },
+use crate::engine::{
+    core::{dml, model::Model, space::Space},
+    error::{QueryError, QueryResult},
+    fractal::Global,
+    mem::RawSlice,
+    net::protocol::{ClientLocalState, Response, SQuery},
+    ql::{
+        ast::{traits::ASTNode, InplaceData, State},
+        lex::{Keyword, KeywordStmt, Token},
     },
-    core::ops::Deref,
 };
 
 pub async fn dispatch_to_executor<'a>(
@@ -65,32 +63,6 @@ pub async fn dispatch_to_executor<'a>(
     trigger warning: disgusting hacks below (why can't async play nice with lifetimes :|)
 */
 
-struct RawSlice<T> {
-    t: *const T,
-    l: usize,
-}
-
-unsafe impl<T: Send> Send for RawSlice<T> {}
-unsafe impl<T: Sync> Sync for RawSlice<T> {}
-
-impl<T> RawSlice<T> {
-    #[inline(always)]
-    unsafe fn new(t: *const T, l: usize) -> Self {
-        Self { t, l }
-    }
-}
-
-impl<T> Deref for RawSlice<T> {
-    type Target = [T];
-    #[inline(always)]
-    fn deref(&self) -> &Self::Target {
-        unsafe {
-            // UNSAFE(@ohsayan): the caller MUST guarantee that this remains valid throughout the usage of the slice
-            core::slice::from_raw_parts(self.t, self.l)
-        }
-    }
-}
-
 #[inline(always)]
 fn call<A: ASTNode<'static> + core::fmt::Debug, T>(
     g: Global,
diff --git a/server/src/engine/core/index/row.rs b/server/src/engine/core/index/row.rs
index 7cde70a6..dab50a7b 100644
--- a/server/src/engine/core/index/row.rs
+++ b/server/src/engine/core/index/row.rs
@@ -31,6 +31,7 @@ use {
             core::model::{DeltaState, DeltaVersion, SchemaDeltaKind},
             data::cell::Datacell,
             idx::{meta::hash::HasherNativeFx, mtchm::meta::TreeElement, IndexST, STIndex},
+            mem::RawStr,
             sync::smart::RawRC,
         },
         util::compiler,
@@ -39,7 +40,7 @@ use {
     std::mem::ManuallyDrop,
 };
 
-pub type DcFieldIndex = IndexST<Box<str>, Datacell, HasherNativeFx>;
+pub type DcFieldIndex = IndexST<RawStr, Datacell, HasherNativeFx>;
 
 #[derive(Debug)]
 pub struct Row {
@@ -145,7 +146,7 @@ impl Row {
             .read()
             .fields()
             .st_iter_kv()
-            .map(|(id, data)| (id.clone(), data.clone()))
+            .map(|(id, data)| (id.as_str().to_owned().into_boxed_str(), data.clone()))
             .collect()
     }
 }
@@ -171,7 +172,15 @@ impl Row {
         for (delta_id, delta) in delta_state.resolve_iter_since(wl.txn_revised_schema_version) {
             match delta.kind() {
                 SchemaDeltaKind::FieldAdd(f) => {
-                    wl.fields.st_insert(f.clone(), Datacell::null());
+                    wl.fields.st_insert(
+                        unsafe {
+                            // UNSAFE(@ohsayan): a row is inside a model and is valid as long as it is in there!
+                            // even if the model was chucked and the row was lying around it won't cause any harm because it
+                            // neither frees anything nor allocates
+                            f.clone()
+                        },
+                        Datacell::null(),
+                    );
                 }
                 SchemaDeltaKind::FieldRem(f) => {
                     wl.fields.st_delete(f);
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index ad24f554..d9b7a51d 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -127,7 +127,7 @@ impl<'a> AlterPlan<'a> {
                     okay &= no_field(mdl, &field_name) & mdl.not_pk(&field_name);
                     let is_nullable = check_nullable(&mut props)?;
                     let layers = Field::parse_layers(layers, is_nullable)?;
-                    okay &= add.st_insert(field_name.to_string().into_boxed_str(), layers);
+                    okay &= add.st_insert(field_name.as_str().into(), layers);
                 }
                 can_ignore!(AlterAction::Add(add))
             }
@@ -155,8 +155,7 @@ impl<'a> AlterPlan<'a> {
                     let (anydelta, new_field) =
                         Self::ldeltas(current_field, layers, is_nullable, &mut no_lock, &mut okay)?;
                     any_delta += anydelta as usize;
-                    okay &=
-                        new_fields.st_insert(field_name.to_string().into_boxed_str(), new_field);
+                    okay &= new_fields.st_insert(field_name.as_str().into(), new_field);
                 }
                 if any_delta == 0 {
                     AlterAction::Ignore
@@ -281,14 +280,12 @@ impl Model {
                             // commit txn
                             global.namespace_txn_driver().lock().try_commit(txn)?;
                         }
+                        let mut mutator = model.model_mutator();
                         new_fields
                             .stseq_ord_kv()
                             .map(|(x, y)| (x.clone(), y.clone()))
                             .for_each(|(field_id, field)| {
-                                model
-                                    .delta_state_mut()
-                                    .schema_append_unresolved_wl_field_add(&field_id);
-                                model.fields_mut().st_insert(field_id, field);
+                                mutator.add_field(field_id, field);
                             });
                     }
                     AlterAction::Remove(removed) => {
@@ -301,11 +298,9 @@ impl Model {
                             // commit txn
                             global.namespace_txn_driver().lock().try_commit(txn)?;
                         }
+                        let mut mutator = model.model_mutator();
                         removed.iter().for_each(|field_id| {
-                            model
-                                .delta_state_mut()
-                                .schema_append_unresolved_wl_field_rem(field_id.as_str());
-                            model.fields_mut().st_delete(field_id.as_str());
+                            mutator.remove_field(field_id.as_str());
                         });
                     }
                     AlterAction::Update(updated) => {
@@ -318,8 +313,9 @@ impl Model {
                             // commit txn
                             global.namespace_txn_driver().lock().try_commit(txn)?;
                         }
+                        let mut mutator = model.model_mutator();
                         updated.into_iter().for_each(|(field_id, field)| {
-                            model.fields_mut().st_update(&field_id, field);
+                            mutator.update_field(field_id.as_ref(), field);
                         });
                     }
                 }
diff --git a/server/src/engine/core/model/delta.rs b/server/src/engine/core/model/delta.rs
index 2feb6cf0..e902f073 100644
--- a/server/src/engine/core/model/delta.rs
+++ b/server/src/engine/core/model/delta.rs
@@ -29,6 +29,7 @@ use {
     crate::engine::{
         core::{dml::QueryExecMeta, index::Row},
         fractal::{FractalToken, GlobalInstanceLike},
+        mem::RawStr,
         sync::atm::Guard,
         sync::queue::Queue,
     },
@@ -116,10 +117,10 @@ impl DeltaState {
     pub fn schema_current_version(&self) -> DeltaVersion {
         DeltaVersion(self.schema_current_version)
     }
-    pub fn schema_append_unresolved_wl_field_add(&mut self, field_name: &str) {
+    pub fn unresolved_append_field_add(&mut self, field_name: RawStr) {
         self.__schema_append_unresolved_delta(SchemaDeltaPart::field_add(field_name));
     }
-    pub fn schema_append_unresolved_wl_field_rem(&mut self, field_name: &str) {
+    pub fn unresolved_append_field_rem(&mut self, field_name: RawStr) {
         self.__schema_append_unresolved_delta(SchemaDeltaPart::field_rem(field_name));
     }
 }
@@ -178,23 +179,19 @@ impl SchemaDeltaPart {
 
 #[derive(Debug)]
 pub enum SchemaDeltaKind {
-    FieldAdd(Box<str>),
-    FieldRem(Box<str>),
+    FieldAdd(RawStr),
+    FieldRem(RawStr),
 }
 
 impl SchemaDeltaPart {
     fn new(kind: SchemaDeltaKind) -> Self {
         Self { kind }
     }
-    fn field_add(field_name: &str) -> Self {
-        Self::new(SchemaDeltaKind::FieldAdd(
-            field_name.to_owned().into_boxed_str(),
-        ))
+    fn field_add(field_name: RawStr) -> Self {
+        Self::new(SchemaDeltaKind::FieldAdd(field_name))
     }
-    fn field_rem(field_name: &str) -> Self {
-        Self::new(SchemaDeltaKind::FieldRem(
-            field_name.to_owned().into_boxed_str(),
-        ))
+    fn field_rem(field_name: RawStr) -> Self {
+        Self::new(SchemaDeltaKind::FieldRem(field_name))
     }
 }
 
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 85a52365..13f39cb6 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -40,8 +40,8 @@ use {
         },
         error::{QueryError, QueryResult},
         fractal::{GenericTask, GlobalInstanceLike, Task},
-        idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
-        mem::VInline,
+        idx::{self, IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
+        mem::{RawStr, VInline},
         ql::ddl::{
             crt::CreateModel,
             drop::DropModel,
@@ -49,21 +49,23 @@ use {
         },
         txn::gns::{self as gnstxn, SpaceIDRef},
     },
+    std::collections::hash_map::{Entry, HashMap},
 };
 
 pub(in crate::engine::core) use self::delta::{DeltaState, DeltaVersion, SchemaDeltaKind};
 
 use super::util::{EntityID, EntityIDRef};
-type Fields = IndexSTSeqCns<Box<str>, Field>;
+type Fields = IndexSTSeqCns<RawStr, Field>;
 
 #[derive(Debug)]
 pub struct Model {
     uuid: Uuid,
-    p_key: Box<str>,
+    p_key: RawStr,
     p_tag: FullTag,
     fields: Fields,
     data: PrimaryIndex,
     delta: DeltaState,
+    private: ModelPrivate,
 }
 
 #[cfg(test)]
@@ -77,23 +79,6 @@ impl PartialEq for Model {
 }
 
 impl Model {
-    pub fn new(
-        uuid: Uuid,
-        p_key: Box<str>,
-        p_tag: FullTag,
-        fields: Fields,
-        data: PrimaryIndex,
-        delta: DeltaState,
-    ) -> Self {
-        Self {
-            uuid,
-            p_key,
-            p_tag,
-            fields,
-            data,
-            delta,
-        }
-    }
     pub fn get_uuid(&self) -> Uuid {
         self.uuid
     }
@@ -122,27 +107,59 @@ impl Model {
     pub fn delta_state(&self) -> &DeltaState {
         &self.delta
     }
-    pub fn delta_state_mut(&mut self) -> &mut DeltaState {
-        &mut self.delta
-    }
-    pub fn fields_mut(&mut self) -> &mut Fields {
-        &mut self.fields
-    }
     pub fn fields(&self) -> &Fields {
         &self.fields
     }
+    pub fn model_mutator<'a>(&'a mut self) -> ModelMutator<'a> {
+        ModelMutator { model: self }
+    }
 }
 
 impl Model {
-    pub fn new_restore(uuid: Uuid, p_key: Box<str>, p_tag: FullTag, fields: Fields) -> Self {
-        Self::new(
+    fn new_with_private(
+        uuid: Uuid,
+        p_key: RawStr,
+        p_tag: FullTag,
+        fields: Fields,
+        private: ModelPrivate,
+    ) -> Self {
+        Self {
             uuid,
             p_key,
             p_tag,
             fields,
-            PrimaryIndex::new_empty(),
-            DeltaState::new_resolved(),
-        )
+            data: PrimaryIndex::new_empty(),
+            delta: DeltaState::new_resolved(),
+            private,
+        }
+    }
+    pub fn new_restore(
+        uuid: Uuid,
+        p_key: Box<str>,
+        p_tag: FullTag,
+        decl_fields: IndexSTSeqCns<Box<str>, Field>,
+    ) -> Self {
+        let mut private = ModelPrivate::empty();
+        let p_key = unsafe {
+            // UNSAFE(@ohsayan): once again, all cool since we maintain the allocation
+            private.push_allocated(p_key)
+        };
+        let mut fields = IndexSTSeqCns::idx_init();
+        decl_fields
+            .stseq_owned_kv()
+            .map(|(field_key, field)| {
+                (
+                    unsafe {
+                        // UNSAFE(@ohsayan): we ensure that priv is dropped iff model is dropped
+                        private.push_allocated(field_key)
+                    },
+                    field,
+                )
+            })
+            .for_each(|(field_key, field)| {
+                fields.st_insert(field_key, field);
+            });
+        Self::new_with_private(uuid, p_key, p_tag, fields, private)
     }
     pub fn process_create(
         CreateModel {
@@ -151,6 +168,7 @@ impl Model {
             props,
         }: CreateModel,
     ) -> QueryResult<Self> {
+        let mut private = ModelPrivate::empty();
         let mut okay = props.is_empty() & !fields.is_empty();
         // validate fields
         let mut field_spec = fields.into_iter();
@@ -164,20 +182,36 @@ impl Model {
                 null,
                 primary,
             } = field_spec.next().unwrap();
+            let this_field_ptr = unsafe {
+                // UNSAFE(@ohsayan): this is going to go with our alloc, so we're good! if we fail too, the dtor for private will run
+                private.allocate_or_recycle(field_name.as_str())
+            };
             if primary {
                 pk_cnt += 1usize;
-                last_pk = Some(field_name.as_str());
+                last_pk = Some(unsafe {
+                    // UNSAFE(@ohsayan): totally cool, it's all allocated
+                    this_field_ptr.clone()
+                });
                 okay &= !null;
             }
             let layer = Field::parse_layers(layers, null)?;
-            okay &= fields.st_insert(field_name.as_str().to_string().into_boxed_str(), layer);
+            okay &= fields.st_insert(this_field_ptr, layer);
         }
         okay &= pk_cnt <= 1;
         if okay {
-            let last_pk = last_pk.unwrap_or(fields.stseq_ord_key().next().unwrap());
-            let tag = fields.st_get(last_pk).unwrap().layers()[0].tag;
+            let last_pk = last_pk.unwrap_or(unsafe {
+                // UNSAFE(@ohsayan): once again, all of this is allocated
+                fields.stseq_ord_key().next().unwrap().clone()
+            });
+            let tag = fields.st_get(&last_pk).unwrap().layers()[0].tag;
             if tag.tag_unique().is_unique() {
-                return Ok(Self::new_restore(Uuid::new(), last_pk.into(), tag, fields));
+                return Ok(Self::new_with_private(
+                    Uuid::new(),
+                    last_pk,
+                    tag,
+                    fields,
+                    private,
+                ));
             }
         }
         Err(QueryError::QExecDdlModelBadDefinition)
@@ -286,6 +320,92 @@ impl Model {
     }
 }
 
+#[derive(Debug, PartialEq)]
+struct ModelPrivate {
+    alloc: HashMap<Box<str>, bool, idx::meta::hash::HasherNativeFx>,
+}
+
+impl ModelPrivate {
+    fn empty() -> Self {
+        Self {
+            alloc: HashMap::with_hasher(Default::default()),
+        }
+    }
+    pub(self) unsafe fn allocate_or_recycle(&mut self, new: &str) -> RawStr {
+        match self.alloc.get_key_value(new) {
+            Some((prev_alloc, _)) => {
+                // already allocated this
+                let ret = RawStr::new(prev_alloc.as_ptr(), prev_alloc.len());
+                // set live!
+                *self.alloc.get_mut(ret.as_str()).unwrap() = false;
+                return ret;
+            }
+            None => {
+                // need to allocate
+                let alloc = new.to_owned().into_boxed_str();
+                let ret = RawStr::new(alloc.as_ptr(), alloc.len());
+                let _ = self.alloc.insert(alloc, false);
+                return ret;
+            }
+        }
+    }
+    pub(self) unsafe fn mark_pending_remove(&mut self, v: &str) -> RawStr {
+        let ret = self.alloc.get_key_value(v).unwrap().0;
+        let ret = RawStr::new(ret.as_ptr(), ret.len());
+        *self.alloc.get_mut(v).unwrap() = true;
+        ret
+    }
+    pub(self) unsafe fn vacuum_marked(&mut self) {
+        self.alloc.retain(|_, dead| !*dead)
+    }
+    pub(self) unsafe fn push_allocated(&mut self, alloc: Box<str>) -> RawStr {
+        match self.alloc.entry(alloc) {
+            Entry::Occupied(mut oe) => {
+                oe.insert(false);
+                RawStr::new(oe.key().as_ptr(), oe.key().len())
+            }
+            Entry::Vacant(ve) => {
+                let ret = RawStr::new(ve.key().as_ptr(), ve.key().len());
+                ve.insert(false);
+                return ret;
+            }
+        }
+    }
+}
+
+pub struct ModelMutator<'a> {
+    model: &'a mut Model,
+}
+
+impl<'a> ModelMutator<'a> {
+    pub unsafe fn vacuum_stashed(&mut self) {
+        self.model.private.vacuum_marked()
+    }
+    pub fn remove_field(&mut self, name: &str) -> bool {
+        // remove
+        let r = self.model.fields.st_delete(name);
+        // recycle
+        let ptr = unsafe { self.model.private.mark_pending_remove(name) };
+        // publish delta
+        self.model.delta.unresolved_append_field_rem(ptr);
+        r
+    }
+    pub fn add_field(&mut self, name: Box<str>, field: Field) -> bool {
+        unsafe {
+            // allocate
+            let fkeyptr = self.model.private.push_allocated(name);
+            // add
+            let r = self.model.fields.st_insert(fkeyptr.clone(), field);
+            // delta
+            self.model.delta.unresolved_append_field_add(fkeyptr);
+            r
+        }
+    }
+    pub fn update_field(&mut self, name: &str, field: Field) -> bool {
+        self.model.fields.st_update(name, field)
+    }
+}
+
 /*
     Layer
 */
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index 5ec493f6..fc8b3c0c 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -371,7 +371,7 @@ mod exec {
                         .stseq_ord_kv()
                         .rev()
                         .take(2)
-                        .map(|(id, f)| (id.clone(), f.clone()))
+                        .map(|(id, f)| (id.as_str().to_owned(), f.clone()))
                         .collect::<Vec<_>>(),
                     [
                         ("col3".into(), Field::new([Layer::uint16()].into(), true)),
@@ -400,7 +400,7 @@ mod exec {
                         .fields()
                         .stseq_ord_kv()
                         .rev()
-                        .map(|(a, b)| (a.clone(), b.clone()))
+                        .map(|(a, b)| (a.as_str().to_owned(), b.clone()))
                         .collect::<Vec<_>>(),
                     [("username".into(), Field::new([Layer::str()].into(), false))]
                 );
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 197881d5..8543112b 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -39,7 +39,9 @@ use {
     core::{borrow::Borrow, hash::Hash},
 };
 
-pub use stord::iter::IndexSTSeqDllIterOrdKV;
+pub mod stdord_iter {
+    pub use super::stord::iter::IndexSTSeqDllIterOrdKV;
+}
 
 // re-exports
 pub type IndexSTSeqCns<K, V> = stord::IndexSTSeqDll<K, V, stord::config::ConservativeConfig<K, V>>;
@@ -308,7 +310,7 @@ pub trait STIndex<K: ?Sized, V>: IndexBaseSpec {
     fn st_iter_value<'a>(&'a self) -> Self::IterValue<'a>;
 }
 
-pub trait STIndexSeq<K: ?Sized, V>: STIndex<K, V> {
+pub trait STIndexSeq<K, V>: STIndex<K, V> {
     /// An ordered iterator over the keys and values
     type IterOrdKV<'a>: Iterator<Item = (&'a K, &'a V)> + DoubleEndedIterator<Item = (&'a K, &'a V)>
     where
@@ -325,10 +327,17 @@ pub trait STIndexSeq<K: ?Sized, V>: STIndex<K, V> {
     where
         Self: 'a,
         V: 'a;
+    type OwnedIterKV: Iterator<Item = (K, V)> + DoubleEndedIterator<Item = (K, V)>;
+    type OwnedIterKeys: Iterator<Item = K> + DoubleEndedIterator<Item = K>;
+    type OwnedIterValues: Iterator<Item = V> + DoubleEndedIterator<Item = V>;
     /// Returns an ordered iterator over the KV pairs
     fn stseq_ord_kv<'a>(&'a self) -> Self::IterOrdKV<'a>;
     /// Returns an ordered iterator over the keys
     fn stseq_ord_key<'a>(&'a self) -> Self::IterOrdKey<'a>;
     /// Returns an ordered iterator over the values
     fn stseq_ord_value<'a>(&'a self) -> Self::IterOrdValue<'a>;
+    // owned
+    fn stseq_owned_kv(self) -> Self::OwnedIterKV;
+    fn stseq_owned_keys(self) -> Self::OwnedIterKeys;
+    fn stseq_owned_values(self) -> Self::OwnedIterValues;
 }
diff --git a/server/src/engine/idx/stord/config.rs b/server/src/engine/idx/stord/config.rs
index 211a2041..757f33e8 100644
--- a/server/src/engine/idx/stord/config.rs
+++ b/server/src/engine/idx/stord/config.rs
@@ -115,7 +115,7 @@ pub trait Config<K, V> {
     type AllocStrategy: AllocStrategy<K, V>;
 }
 
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct ConservativeConfig<K, V>(PhantomData<super::IndexSTSeqDll<K, V, Self>>);
 
 impl<K, V> Config<K, V> for ConservativeConfig<K, V> {
@@ -123,7 +123,7 @@ impl<K, V> Config<K, V> for ConservativeConfig<K, V> {
     type AllocStrategy = ConservativeStrategy<K, V>;
 }
 
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct LiberalConfig<K, V>(PhantomData<super::IndexSTSeqDll<K, V, Self>>);
 
 impl<K, V> Config<K, V> for LiberalConfig<K, V> {
diff --git a/server/src/engine/idx/stord/iter.rs b/server/src/engine/idx/stord/iter.rs
index ea0ea5e2..c70d5850 100644
--- a/server/src/engine/idx/stord/iter.rs
+++ b/server/src/engine/idx/stord/iter.rs
@@ -28,14 +28,16 @@ use {
     super::{
         config::Config, IndexSTSeqDll, IndexSTSeqDllKeyptr, IndexSTSeqDllNode, IndexSTSeqDllNodePtr,
     },
+    crate::engine::idx::{AsKey, AsValue},
     std::{
         collections::{
-            hash_map::{Iter, Keys as StdMapIterKey, Values as StdMapIterVal},
+            hash_map::{Iter as StdMapIter, Keys as StdMapIterKey, Values as StdMapIterVal},
             HashMap as StdMap,
         },
         fmt::{self, Debug},
         iter::FusedIterator,
         marker::PhantomData,
+        mem::ManuallyDrop,
         ptr::{self, NonNull},
     },
 };
@@ -48,7 +50,7 @@ macro_rules! unsafe_marker_impl {
 }
 
 pub struct IndexSTSeqDllIterUnordKV<'a, K: 'a, V: 'a> {
-    i: Iter<'a, IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>>,
+    i: StdMapIter<'a, IndexSTSeqDllKeyptr<K>, IndexSTSeqDllNodePtr<K, V>>,
 }
 
 // UNSAFE(@ohsayan): aliasing guarantees correctness
@@ -263,6 +265,118 @@ impl<K, V> IndexSTSeqDllIterOrdConfig<K, V> for IndexSTSeqDllIterOrdConfigValue
     }
 }
 
+pub(super) struct OrderedOwnedIteratorRaw<K, V> {
+    h: *mut IndexSTSeqDllNode<K, V>,
+    t: *mut IndexSTSeqDllNode<K, V>,
+    r: usize,
+}
+
+impl<K: AsKey, V: AsValue> OrderedOwnedIteratorRaw<K, V> {
+    pub(super) fn new<Mc: Config<K, V>>(mut idx: IndexSTSeqDll<K, V, Mc>) -> Self {
+        // clean up if needed
+        idx.vacuum_full();
+        let mut idx = ManuallyDrop::new(idx);
+        // chuck the map
+        drop(unsafe { ptr::read((&mut idx.m) as *mut _) });
+        // we own everything now
+        unsafe {
+            Self {
+                h: if idx.h.is_null() {
+                    ptr::null_mut()
+                } else {
+                    (*idx.h).p
+                },
+                t: idx.h,
+                r: idx.len(),
+            }
+        }
+    }
+}
+
+impl<K, V> OrderedOwnedIteratorRaw<K, V> {
+    #[inline(always)]
+    fn _next(&mut self) -> Option<(K, V)> {
+        if self.h == self.t {
+            None
+        } else {
+            self.r -= 1;
+            unsafe {
+                // UNSAFE(@ohsayan): +nullck
+                let this = ptr::read(self.h);
+                // destroy this node
+                IndexSTSeqDllNode::dealloc_headless(self.h);
+                self.h = (*self.h).p;
+                Some((this.k, this.v))
+            }
+        }
+    }
+    #[inline(always)]
+    fn _next_back(&mut self) -> Option<(K, V)> {
+        if self.h == self.t {
+            None
+        } else {
+            self.r -= 1;
+            unsafe {
+                // UNSAFE(@ohsayan): +nullck
+                self.t = (*self.t).n;
+                let this = ptr::read(self.t);
+                IndexSTSeqDllNode::dealloc_headless(self.t);
+                Some((this.k, this.v))
+            }
+        }
+    }
+}
+
+impl<K, V> Drop for OrderedOwnedIteratorRaw<K, V> {
+    fn drop(&mut self) {
+        // clean up what's left
+        while let Some(_) = self._next() {}
+    }
+}
+
+pub struct OrderedOwnedIteratorKV<K, V>(pub(super) OrderedOwnedIteratorRaw<K, V>);
+
+impl<K: AsKey, V: AsValue> Iterator for OrderedOwnedIteratorKV<K, V> {
+    type Item = (K, V);
+    fn next(&mut self) -> Option<Self::Item> {
+        self.0._next()
+    }
+}
+
+impl<K: AsKey, V: AsValue> DoubleEndedIterator for OrderedOwnedIteratorKV<K, V> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.0._next_back()
+    }
+}
+
+pub struct OrderedOwnedIteratorKey<K, V>(pub(super) OrderedOwnedIteratorRaw<K, V>);
+
+impl<K: AsKey, V: AsValue> Iterator for OrderedOwnedIteratorKey<K, V> {
+    type Item = K;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.0._next().map(|(k, _)| k)
+    }
+}
+impl<K: AsKey, V: AsValue> DoubleEndedIterator for OrderedOwnedIteratorKey<K, V> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.0._next_back().map(|(k, _)| k)
+    }
+}
+
+pub struct OrderedOwnedIteratorValue<K, V>(pub(super) OrderedOwnedIteratorRaw<K, V>);
+
+impl<K: AsKey, V: AsValue> Iterator for OrderedOwnedIteratorValue<K, V> {
+    type Item = V;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.0._next().map(|(_, v)| v)
+    }
+}
+impl<K: AsKey, V: AsValue> DoubleEndedIterator for OrderedOwnedIteratorValue<K, V> {
+    fn next_back(&mut self) -> Option<Self::Item> {
+        self.0._next_back().map(|(_, v)| v)
+    }
+}
+
 struct IndexSTSeqDllIterOrdBase<'a, K: 'a, V: 'a, C: IndexSTSeqDllIterOrdConfig<K, V>> {
     h: *const IndexSTSeqDllNode<K, V>,
     t: *const IndexSTSeqDllNode<K, V>,
diff --git a/server/src/engine/idx/stord/mod.rs b/server/src/engine/idx/stord/mod.rs
index b2bf424b..6357cd26 100644
--- a/server/src/engine/idx/stord/mod.rs
+++ b/server/src/engine/idx/stord/mod.rs
@@ -275,6 +275,12 @@ impl<K, V, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
     }
 }
 
+impl<K, V, C: Config<K, V> + Default> Default for IndexSTSeqDll<K, V, C> {
+    fn default() -> Self {
+        Self::with_hasher(C::Hasher::default())
+    }
+}
+
 impl<K, V, C: Config<K, V>> IndexSTSeqDll<K, V, C> {
     #[inline(always)]
     fn metrics_update_f_decr(&mut self) {
@@ -707,28 +713,35 @@ where
         Self: 'a,
         K: 'a,
         V: 'a;
-
     type IterOrdKey<'a> = IndexSTSeqDllIterOrdKey<'a, K, V>
     where
         Self: 'a,
         K: 'a;
-
     type IterOrdValue<'a> = IndexSTSeqDllIterOrdValue<'a, K, V>
     where
         Self: 'a,
         V: 'a;
-
+    type OwnedIterKV = iter::OrderedOwnedIteratorKV<K, V>;
+    type OwnedIterKeys = iter::OrderedOwnedIteratorKey<K, V>;
+    type OwnedIterValues = iter::OrderedOwnedIteratorValue<K, V>;
     fn stseq_ord_kv<'a>(&'a self) -> Self::IterOrdKV<'a> {
         self._iter_ord_kv()
     }
-
     fn stseq_ord_key<'a>(&'a self) -> Self::IterOrdKey<'a> {
         self._iter_ord_k()
     }
-
     fn stseq_ord_value<'a>(&'a self) -> Self::IterOrdValue<'a> {
         self._iter_ord_v()
     }
+    fn stseq_owned_keys(self) -> Self::OwnedIterKeys {
+        iter::OrderedOwnedIteratorKey(iter::OrderedOwnedIteratorRaw::new(self))
+    }
+    fn stseq_owned_values(self) -> Self::OwnedIterValues {
+        iter::OrderedOwnedIteratorValue(iter::OrderedOwnedIteratorRaw::new(self))
+    }
+    fn stseq_owned_kv(self) -> Self::OwnedIterKV {
+        iter::OrderedOwnedIteratorKV(iter::OrderedOwnedIteratorRaw::new(self))
+    }
 }
 
 impl<K: AsKeyClone, V: AsValueClone, C: Config<K, V>> Clone for IndexSTSeqDll<K, V, C> {
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index 6d7de7ef..a823287c 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -27,6 +27,7 @@
 mod astr;
 mod ll;
 mod numbuf;
+mod rawslice;
 pub mod scanner;
 mod stackop;
 mod uarray;
@@ -39,11 +40,12 @@ mod tests;
 pub use {
     astr::AStr,
     ll::CachePadded,
+    numbuf::IntegerRepr,
+    rawslice::{RawSlice, RawStr},
     scanner::BufferedScanner,
     uarray::UArray,
     vinline::VInline,
     word::{DwordNN, DwordQN, WordIO, ZERO_BLOCK},
-    numbuf::IntegerRepr,
 };
 // imports
 use std::alloc::{self, Layout};
diff --git a/server/src/engine/mem/rawslice.rs b/server/src/engine/mem/rawslice.rs
new file mode 100644
index 00000000..f0faf014
--- /dev/null
+++ b/server/src/engine/mem/rawslice.rs
@@ -0,0 +1,158 @@
+/*
+ * Created on Thu Nov 23 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use core::{
+    borrow::Borrow,
+    fmt,
+    hash::{Hash, Hasher},
+    ops::Deref,
+    slice, str,
+};
+
+#[derive(PartialEq, Eq, PartialOrd, Ord)]
+pub struct RawStr {
+    base: RawSlice<u8>,
+}
+
+impl RawStr {
+    pub unsafe fn new(p: *const u8, l: usize) -> Self {
+        Self {
+            base: RawSlice::new(p, l),
+        }
+    }
+    pub unsafe fn clone(&self) -> Self {
+        Self {
+            base: self.base.clone(),
+        }
+    }
+    pub fn as_str(&self) -> &str {
+        unsafe {
+            // UNSAFE(@ohsayan): up to caller to ensure proper pointers
+            str::from_utf8_unchecked(self.base.as_slice())
+        }
+    }
+}
+
+impl From<&'static str> for RawStr {
+    fn from(s: &'static str) -> Self {
+        unsafe { Self::new(s.as_ptr(), s.len()) }
+    }
+}
+
+impl Borrow<str> for RawStr {
+    fn borrow(&self) -> &str {
+        unsafe { core::mem::transmute(self.clone()) }
+    }
+}
+
+impl Deref for RawStr {
+    type Target = str;
+    fn deref(&self) -> &Self::Target {
+        self.as_str()
+    }
+}
+
+impl fmt::Debug for RawStr {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        <str as fmt::Debug>::fmt(self.as_str(), f)
+    }
+}
+
+impl fmt::Display for RawStr {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        <str as fmt::Display>::fmt(self.as_str(), f)
+    }
+}
+
+impl Hash for RawStr {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.as_str().hash(state)
+    }
+}
+
+pub struct RawSlice<T> {
+    t: *const T,
+    l: usize,
+}
+
+unsafe impl<T: Send> Send for RawSlice<T> {}
+unsafe impl<T: Sync> Sync for RawSlice<T> {}
+
+impl<T> RawSlice<T> {
+    #[inline(always)]
+    pub unsafe fn new(t: *const T, l: usize) -> Self {
+        Self { t, l }
+    }
+    pub fn as_slice(&self) -> &[T] {
+        unsafe {
+            // UNSAFE(@ohsayan): the caller MUST guarantee that this remains valid throughout the usage of the slice
+            slice::from_raw_parts(self.t, self.l)
+        }
+    }
+    pub unsafe fn clone(&self) -> Self {
+        Self { ..*self }
+    }
+}
+
+impl<T: Hash> Hash for RawSlice<T> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.as_slice().hash(state)
+    }
+}
+
+impl<T: PartialEq> PartialEq for RawSlice<T> {
+    fn eq(&self, other: &Self) -> bool {
+        self.as_slice() == other.as_slice()
+    }
+}
+
+impl<T: Eq> Eq for RawSlice<T> {}
+
+impl<T: PartialOrd> PartialOrd for RawSlice<T> {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        self.as_slice().partial_cmp(other.as_slice())
+    }
+}
+
+impl<T: Ord> Ord for RawSlice<T> {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.as_slice().cmp(other.as_slice())
+    }
+}
+
+impl<T: fmt::Debug> fmt::Debug for RawSlice<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_list().entries(self.as_slice()).finish()
+    }
+}
+
+impl<T> Deref for RawSlice<T> {
+    type Target = [T];
+    #[inline(always)]
+    fn deref(&self) -> &Self::Target {
+        self.as_slice()
+    }
+}
diff --git a/server/src/engine/ql/dcl.rs b/server/src/engine/ql/dcl.rs
index 1f97cfa0..17520174 100644
--- a/server/src/engine/ql/dcl.rs
+++ b/server/src/engine/ql/dcl.rs
@@ -42,7 +42,7 @@ fn parse<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<UserMe
         ^cursor
         7 tokens
     */
-        if state.remaining() < 7 {
+    if state.remaining() < 7 {
         return Err(QueryError::QLInvalidSyntax);
     }
     let token_buffer = state.current();
diff --git a/server/src/engine/storage/v1/batch_jrnl/persist.rs b/server/src/engine/storage/v1/batch_jrnl/persist.rs
index d0b8905c..852411d1 100644
--- a/server/src/engine/storage/v1/batch_jrnl/persist.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/persist.rs
@@ -151,12 +151,12 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         col_cnt: usize,
     ) -> RuntimeResult<()> {
         self.f
-            .write_unfsynced(&[MARKER_ACTUAL_BATCH_EVENT, pk_tag.value_u8()])?;
+            .tracked_write_unfsynced(&[MARKER_ACTUAL_BATCH_EVENT, pk_tag.value_u8()])?;
         let observed_len_bytes = observed_len.u64_bytes_le();
-        self.f.write_unfsynced(&observed_len_bytes)?;
+        self.f.tracked_write_unfsynced(&observed_len_bytes)?;
         self.f
-            .write_unfsynced(&schema_version.value_u64().to_le_bytes())?;
-        self.f.write_unfsynced(&col_cnt.u64_bytes_le())?;
+            .tracked_write_unfsynced(&schema_version.value_u64().to_le_bytes())?;
+        self.f.tracked_write_unfsynced(&col_cnt.u64_bytes_le())?;
         Ok(())
     }
     /// Append a summary of this batch and most importantly, **sync everything to disk**
@@ -166,9 +166,9 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
         inconsistent_reads: usize,
     ) -> RuntimeResult<()> {
         // [0xFD][actual_commit][checksum]
-        self.f.write_unfsynced(&[MARKER_END_OF_BATCH])?;
+        self.f.tracked_write_unfsynced(&[MARKER_END_OF_BATCH])?;
         let actual_commit = (observed_len - inconsistent_reads).u64_bytes_le();
-        self.f.write_unfsynced(&actual_commit)?;
+        self.f.tracked_write_unfsynced(&actual_commit)?;
         let cs = self.f.reset_and_finish_checksum().to_le_bytes();
         self.f.untracked_write(&cs)?;
         // IMPORTANT: now that all data has been written, we need to actually ensure that the writes pass through the cache
@@ -200,7 +200,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
                     pk.read_uint()
                 }
                 .to_le_bytes();
-                buf.write_unfsynced(&data)?;
+                buf.tracked_write_unfsynced(&data)?;
             }
             TagUnique::Str | TagUnique::Bin => {
                 let slice = unsafe {
@@ -208,8 +208,8 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
                     pk.read_bin()
                 };
                 let slice_l = slice.len().u64_bytes_le();
-                buf.write_unfsynced(&slice_l)?;
-                buf.write_unfsynced(slice)?;
+                buf.tracked_write_unfsynced(&slice_l)?;
+                buf.tracked_write_unfsynced(slice)?;
             }
             TagUnique::Illegal => unsafe {
                 // UNSAFE(@ohsayan): a pk can't be constructed with illegal
@@ -222,7 +222,7 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
     fn encode_cell(&mut self, value: &Datacell) -> RuntimeResult<()> {
         let mut buf = vec![];
         cell::encode(&mut buf, value);
-        self.f.write_unfsynced(&buf)?;
+        self.f.tracked_write_unfsynced(&buf)?;
         Ok(())
     }
     /// Encode row data
@@ -232,8 +232,8 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
                 Some(cell) => {
                     self.encode_cell(cell)?;
                 }
-                None if field_name.as_ref() == model.p_key() => {}
-                None => self.f.write_unfsynced(&[0])?,
+                None if field_name.as_str() == model.p_key() => {}
+                None => self.f.tracked_write_unfsynced(&[0])?,
             }
         }
         Ok(())
@@ -241,9 +241,9 @@ impl<Fs: RawFSInterface> DataBatchPersistDriver<Fs> {
     /// Write the change type and txnid
     fn write_batch_item_common_row_data(&mut self, delta: &DataDelta) -> RuntimeResult<()> {
         let change_type = [delta.change().value_u8()];
-        self.f.write_unfsynced(&change_type)?;
+        self.f.tracked_write_unfsynced(&change_type)?;
         let txn_id = delta.data_version().value_u64().to_le_bytes();
-        self.f.write_unfsynced(&txn_id)?;
+        self.f.tracked_write_unfsynced(&txn_id)?;
         Ok(())
     }
 }
diff --git a/server/src/engine/storage/v1/batch_jrnl/restore.rs b/server/src/engine/storage/v1/batch_jrnl/restore.rs
index 5851d044..bcc99534 100644
--- a/server/src/engine/storage/v1/batch_jrnl/restore.rs
+++ b/server/src/engine/storage/v1/batch_jrnl/restore.rs
@@ -146,6 +146,7 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
         // begin
         let mut closed = false;
         while !self.f.is_eof() && !closed {
+            self.f.__reset_checksum();
             // try to decode this batch
             let Ok(batch) = self.read_batch() else {
                 self.attempt_recover_data_batch()?;
@@ -237,10 +238,16 @@ impl<F: RawFSInterface> DataBatchRestoreDriver<F> {
                             for (field_name, new_data) in m
                                 .fields()
                                 .stseq_ord_key()
-                                .filter(|key| key.as_ref() != m.p_key())
+                                .filter(|key| key.as_str() != m.p_key())
                                 .zip(new_row)
                             {
-                                data.st_insert(field_name.clone(), new_data);
+                                data.st_insert(
+                                    unsafe {
+                                        // UNSAFE(@ohsayan): model in scope, we're good
+                                        field_name.clone()
+                                    },
+                                    new_data,
+                                );
                             }
                             let row = Row::new_restored(
                                 pk,
diff --git a/server/src/engine/storage/v1/inf/map.rs b/server/src/engine/storage/v1/inf/map.rs
index aab7a1b8..1914eee0 100644
--- a/server/src/engine/storage/v1/inf/map.rs
+++ b/server/src/engine/storage/v1/inf/map.rs
@@ -30,56 +30,53 @@ use {
             cell::{self, CanYieldDict, StorageCellTypeID},
             FieldMD,
         },
-        PersistMapSpec, PersistObject, VecU8,
+        AbstractMap, MapStorageSpec, PersistObject, VecU8,
     },
     crate::{
         engine::{
             core::model::Field,
-            data::{dict::DictEntryGeneric, DictGeneric},
+            data::dict::DictEntryGeneric,
             error::{RuntimeResult, StorageError},
-            idx::{IndexBaseSpec, IndexSTSeqCns, STIndex, STIndexSeq},
-            mem::BufferedScanner,
+            idx::{IndexSTSeqCns, STIndexSeq},
+            mem::{BufferedScanner, StatelessLen},
             storage::v1::inf,
         },
         util::{copy_slice_to_array as memcpy, EndianQW},
     },
-    core::marker::PhantomData,
+    std::{collections::HashMap, marker::PhantomData},
 };
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord)]
 pub struct MapIndexSizeMD(pub usize);
 
 /// This is more of a lazy hack than anything sensible. Just implement a spec and then use this wrapper for any enc/dec operations
-pub struct PersistMapImpl<'a, M: PersistMapSpec>(PhantomData<&'a M::MapType>);
+pub struct PersistMapImpl<'a, M: MapStorageSpec>(PhantomData<&'a M::InMemoryMap>);
 
-impl<'a, M: PersistMapSpec> PersistObject for PersistMapImpl<'a, M>
-where
-    M::MapType: 'a + STIndex<M::Key, M::Value>,
-{
+impl<'a, M: MapStorageSpec> PersistObject for PersistMapImpl<'a, M> {
     const METADATA_SIZE: usize = sizeof!(u64);
-    type InputType = &'a M::MapType;
-    type OutputType = M::MapType;
+    type InputType = &'a M::InMemoryMap;
+    type OutputType = M::RestoredMap;
     type Metadata = MapIndexSizeMD;
     fn pretest_can_dec_object(
         s: &BufferedScanner,
         MapIndexSizeMD(dict_size): &Self::Metadata,
     ) -> bool {
-        M::pretest_collection_using_size(s, *dict_size)
+        M::decode_pretest_for_map(s, *dict_size)
     }
     fn meta_enc(buf: &mut VecU8, data: Self::InputType) {
-        buf.extend(data.st_len().u64_bytes_le());
+        buf.extend(data.stateless_len().u64_bytes_le());
     }
     unsafe fn meta_dec(scanner: &mut BufferedScanner) -> RuntimeResult<Self::Metadata> {
         Ok(MapIndexSizeMD(scanner.next_u64_le() as usize))
     }
     fn obj_enc(buf: &mut VecU8, map: Self::InputType) {
-        for (key, val) in M::_get_iter(map) {
-            M::entry_md_enc(buf, key, val);
-            if M::ENC_COUPLED {
-                M::enc_entry(buf, key, val);
+        for (key, val) in M::get_iter_from_memory(map) {
+            M::encode_entry_meta(buf, key, val);
+            if M::ENC_AS_ENTRY {
+                M::encode_entry_data(buf, key, val);
             } else {
-                M::enc_key(buf, key);
-                M::enc_val(buf, val);
+                M::encode_entry_key(buf, key);
+                M::encode_entry_val(buf, val);
             }
         }
     }
@@ -87,22 +84,23 @@ where
         scanner: &mut BufferedScanner,
         MapIndexSizeMD(dict_size): Self::Metadata,
     ) -> RuntimeResult<Self::OutputType> {
-        let mut dict = M::MapType::idx_init();
-        while M::pretest_entry_metadata(scanner) & (dict.st_len() != dict_size) {
+        let mut dict = M::RestoredMap::map_new();
+        let decode_pretest_for_entry_meta = M::decode_pretest_for_entry_meta(scanner);
+        while decode_pretest_for_entry_meta & (dict.map_length() != dict_size) {
             let md = unsafe {
                 // UNSAFE(@ohsayan): +pretest
-                M::entry_md_dec(scanner).ok_or::<StorageError>(
+                M::decode_entry_meta(scanner).ok_or::<StorageError>(
                     StorageError::InternalDecodeStructureCorruptedPayload.into(),
                 )?
             };
-            if !M::pretest_entry_data(scanner, &md) {
+            if !M::decode_pretest_for_entry_data(scanner, &md) {
                 return Err(StorageError::InternalDecodeStructureCorruptedPayload.into());
             }
             let key;
             let val;
             unsafe {
-                if M::DEC_COUPLED {
-                    match M::dec_entry(scanner, md) {
+                if M::DEC_AS_ENTRY {
+                    match M::decode_entry_data(scanner, md) {
                         Some((_k, _v)) => {
                             key = _k;
                             val = _v;
@@ -112,8 +110,8 @@ where
                         }
                     }
                 } else {
-                    let _k = M::dec_key(scanner, &md);
-                    let _v = M::dec_val(scanner, &md);
+                    let _k = M::decode_entry_key(scanner, &md);
+                    let _v = M::decode_entry_val(scanner, &md);
                     match (_k, _v) {
                         (Some(_k), Some(_v)) => {
                             key = _k;
@@ -125,11 +123,11 @@ where
                     }
                 }
             }
-            if !dict.st_insert(key, val) {
+            if !dict.map_insert(key, val) {
                 return Err(StorageError::InternalDecodeStructureIllegalData.into());
             }
         }
-        if dict.st_len() == dict_size {
+        if dict.map_length() == dict_size {
             Ok(dict)
         } else {
             Err(StorageError::InternalDecodeStructureIllegalData.into())
@@ -141,12 +139,12 @@ where
 pub struct GenericDictSpec;
 
 /// generic dict entry metadata
-pub struct GenericDictEntryMD {
+pub struct GenericDictEntryMetadata {
     pub(crate) klen: usize,
     pub(crate) dscr: u8,
 }
 
-impl GenericDictEntryMD {
+impl GenericDictEntryMetadata {
     /// decode md (no need for any validation since that has to be handled later and can only produce incorrect results
     /// if unsafe code is used to translate an incorrect dscr)
     pub(crate) fn decode(data: [u8; 9]) -> Self {
@@ -157,32 +155,25 @@ impl GenericDictEntryMD {
     }
 }
 
-impl PersistMapSpec for GenericDictSpec {
-    type MapIter<'a> = std::collections::hash_map::Iter<'a, Box<str>, DictEntryGeneric>;
-    type MapType = DictGeneric;
-    type Key = Box<str>;
-    type Value = DictEntryGeneric;
-    type EntryMD = GenericDictEntryMD;
-    const DEC_COUPLED: bool = false;
-    const ENC_COUPLED: bool = true;
-    fn _get_iter<'a>(map: &'a Self::MapType) -> Self::MapIter<'a> {
+impl MapStorageSpec for GenericDictSpec {
+    type InMemoryMap = HashMap<Self::InMemoryKey, Self::InMemoryVal>;
+    type InMemoryKey = Box<str>;
+    type InMemoryVal = DictEntryGeneric;
+    type InMemoryMapIter<'a> =
+        std::collections::hash_map::Iter<'a, Self::InMemoryKey, Self::InMemoryVal>;
+    type RestoredKey = Self::InMemoryKey;
+    type RestoredMap = Self::InMemoryMap;
+    type RestoredVal = Self::InMemoryVal;
+    type EntryMetadata = GenericDictEntryMetadata;
+    const DEC_AS_ENTRY: bool = false;
+    const ENC_AS_ENTRY: bool = true;
+    fn get_iter_from_memory<'a>(map: &'a Self::InMemoryMap) -> Self::InMemoryMapIter<'a> {
         map.iter()
     }
-    fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool {
-        // we just need to see if we can decode the entry metadata
-        scanner.has_left(9)
-    }
-    fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool {
-        StorageCellTypeID::is_valid(md.dscr)
-            & scanner.has_left(StorageCellTypeID::expect_atleast(md.dscr))
-    }
-    fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, _: &Self::Value) {
+    fn encode_entry_meta(buf: &mut VecU8, key: &Self::InMemoryKey, _: &Self::InMemoryVal) {
         buf.extend(key.len().u64_bytes_le());
     }
-    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::EntryMD> {
-        Some(Self::EntryMD::decode(scanner.next_chunk()))
-    }
-    fn enc_entry(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
+    fn encode_entry_data(buf: &mut VecU8, key: &Self::InMemoryKey, val: &Self::InMemoryVal) {
         match val {
             DictEntryGeneric::Map(map) => {
                 buf.push(StorageCellTypeID::Dict.value_u8());
@@ -196,12 +187,41 @@ impl PersistMapSpec for GenericDictSpec {
             }
         }
     }
-    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Key> {
-        inf::dec::utils::decode_string(scanner, md.klen as usize)
+    fn encode_entry_key(_: &mut VecU8, _: &Self::InMemoryKey) {
+        unimplemented!()
+    }
+    fn encode_entry_val(_: &mut VecU8, _: &Self::InMemoryVal) {
+        unimplemented!()
+    }
+    fn decode_pretest_for_entry_meta(scanner: &mut BufferedScanner) -> bool {
+        // we just need to see if we can decode the entry metadata
+        scanner.has_left(9)
+    }
+    fn decode_pretest_for_entry_data(s: &mut BufferedScanner, md: &Self::EntryMetadata) -> bool {
+        StorageCellTypeID::is_valid(md.dscr)
+            & s.has_left(StorageCellTypeID::expect_atleast(md.dscr))
+    }
+    unsafe fn decode_entry_meta(s: &mut BufferedScanner) -> Option<Self::EntryMetadata> {
+        Some(Self::EntryMetadata::decode(s.next_chunk()))
+    }
+    unsafe fn decode_entry_data(
+        _: &mut BufferedScanner,
+        _: Self::EntryMetadata,
+    ) -> Option<(Self::RestoredKey, Self::RestoredVal)> {
+        unimplemented!()
+    }
+    unsafe fn decode_entry_key(
+        s: &mut BufferedScanner,
+        md: &Self::EntryMetadata,
+    ) -> Option<Self::RestoredKey> {
+        inf::dec::utils::decode_string(s, md.klen as usize)
             .map(|s| s.into_boxed_str())
             .ok()
     }
-    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value> {
+    unsafe fn decode_entry_val(
+        scanner: &mut BufferedScanner,
+        md: &Self::EntryMetadata,
+    ) -> Option<Self::RestoredVal> {
         Some(
             match cell::decode_element::<CanYieldDict, BufferedScanner>(
                 scanner,
@@ -217,30 +237,16 @@ impl PersistMapSpec for GenericDictSpec {
             },
         )
     }
-    // not implemented
-    fn enc_key(_: &mut VecU8, _: &Self::Key) {
-        unimplemented!()
-    }
-    fn enc_val(_: &mut VecU8, _: &Self::Value) {
-        unimplemented!()
-    }
-    unsafe fn dec_entry(
-        _: &mut BufferedScanner,
-        _: Self::EntryMD,
-    ) -> Option<(Self::Key, Self::Value)> {
-        unimplemented!()
-    }
 }
 
-pub struct FieldMapSpec;
-pub struct FieldMapEntryMD {
+pub struct FieldMapEntryMetadata {
     field_id_l: u64,
     field_prop_c: u64,
     field_layer_c: u64,
     null: u8,
 }
 
-impl FieldMapEntryMD {
+impl FieldMapEntryMetadata {
     const fn new(field_id_l: u64, field_prop_c: u64, field_layer_c: u64, null: u8) -> Self {
         Self {
             field_id_l,
@@ -251,130 +257,127 @@ impl FieldMapEntryMD {
     }
 }
 
-impl PersistMapSpec for FieldMapSpec {
-    type MapIter<'a> = crate::engine::idx::IndexSTSeqDllIterOrdKV<'a, Box<str>, Field>;
-    type MapType = IndexSTSeqCns<Self::Key, Self::Value>;
-    type EntryMD = FieldMapEntryMD;
-    type Key = Box<str>;
-    type Value = Field;
-    const ENC_COUPLED: bool = false;
-    const DEC_COUPLED: bool = false;
-    fn _get_iter<'a>(m: &'a Self::MapType) -> Self::MapIter<'a> {
-        m.stseq_ord_kv()
-    }
-    fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool {
-        scanner.has_left(sizeof!(u64, 3) + 1)
+pub trait FieldMapAny: StatelessLen {
+    type Iterator<'a>: Iterator<Item = (&'a str, &'a Field)>
+    where
+        Self: 'a;
+    fn get_iter<'a>(&'a self) -> Self::Iterator<'a>
+    where
+        Self: 'a;
+}
+
+impl FieldMapAny for HashMap<Box<str>, Field> {
+    type Iterator<'a> = std::iter::Map<
+        std::collections::hash_map::Iter<'a, Box<str>, Field>,
+        fn((&Box<str>, &Field)) -> (&'a str, &'a Field),
+    >;
+    fn get_iter<'a>(&'a self) -> Self::Iterator<'a>
+    where
+        Self: 'a,
+    {
+        self.iter()
+            .map(|(a, b)| unsafe { core::mem::transmute((a.as_ref(), b)) })
     }
-    fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool {
-        scanner.has_left(md.field_id_l as usize) // TODO(@ohsayan): we can enforce way more here such as atleast one field etc
+}
+impl FieldMapAny for IndexSTSeqCns<crate::engine::mem::RawStr, Field> {
+    type Iterator<'a> = std::iter::Map<
+    crate::engine::idx::stdord_iter::IndexSTSeqDllIterOrdKV<'a, crate::engine::mem::RawStr, Field>,
+    fn((&crate::engine::mem::RawStr, &Field)) -> (&'a str, &'a Field)>
+    where
+        Self: 'a;
+
+    fn get_iter<'a>(&'a self) -> Self::Iterator<'a>
+    where
+        Self: 'a,
+    {
+        self.stseq_ord_kv()
+            .map(|(k, v)| unsafe { core::mem::transmute((k.as_str(), v)) })
+    }
+}
+impl FieldMapAny for IndexSTSeqCns<Box<str>, Field> {
+    type Iterator<'a> = std::iter::Map<
+    crate::engine::idx::stdord_iter::IndexSTSeqDllIterOrdKV<'a, Box<str>, Field>,
+    fn((&Box<str>, &Field)) -> (&'a str, &'a Field)>
+    where
+        Self: 'a;
+
+    fn get_iter<'a>(&'a self) -> Self::Iterator<'a>
+    where
+        Self: 'a,
+    {
+        self.stseq_ord_kv()
+            .map(|(k, v)| unsafe { core::mem::transmute((k.as_ref(), v)) })
     }
-    fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
+}
+
+pub struct FieldMapSpec<FM>(PhantomData<FM>);
+impl<FM: FieldMapAny> MapStorageSpec for FieldMapSpec<FM> {
+    type InMemoryMap = FM;
+    type InMemoryKey = str;
+    type InMemoryVal = Field;
+    type InMemoryMapIter<'a> = FM::Iterator<'a> where FM: 'a;
+    type RestoredKey = Box<str>;
+    type RestoredVal = Field;
+    type RestoredMap = IndexSTSeqCns<Box<str>, Field>;
+    type EntryMetadata = FieldMapEntryMetadata;
+    const ENC_AS_ENTRY: bool = false;
+    const DEC_AS_ENTRY: bool = false;
+    fn get_iter_from_memory<'a>(map: &'a Self::InMemoryMap) -> Self::InMemoryMapIter<'a> {
+        map.get_iter()
+    }
+    fn encode_entry_meta(buf: &mut VecU8, key: &Self::InMemoryKey, val: &Self::InMemoryVal) {
         buf.extend(key.len().u64_bytes_le());
         buf.extend(0u64.to_le_bytes()); // TODO(@ohsayan): props
         buf.extend(val.layers().len().u64_bytes_le());
         buf.push(val.is_nullable() as u8);
     }
-    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::EntryMD> {
-        Some(FieldMapEntryMD::new(
-            scanner.next_u64_le(),
-            scanner.next_u64_le(),
-            scanner.next_u64_le(),
-            scanner.next_byte(),
-        ))
+    fn encode_entry_data(_: &mut VecU8, _: &Self::InMemoryKey, _: &Self::InMemoryVal) {
+        unimplemented!()
     }
-    fn enc_key(buf: &mut VecU8, key: &Self::Key) {
+    fn encode_entry_key(buf: &mut VecU8, key: &Self::InMemoryKey) {
         buf.extend(key.as_bytes());
     }
-    fn enc_val(buf: &mut VecU8, val: &Self::Value) {
+    fn encode_entry_val(buf: &mut VecU8, val: &Self::InMemoryVal) {
         for layer in val.layers() {
             super::obj::LayerRef::default_full_enc(buf, super::obj::LayerRef(layer))
         }
     }
-    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Key> {
-        inf::dec::utils::decode_string(scanner, md.field_id_l as usize)
-            .map(|s| s.into_boxed_str())
-            .ok()
-    }
-    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value> {
-        super::obj::FieldRef::obj_dec(
-            scanner,
-            FieldMD::new(md.field_prop_c, md.field_layer_c, md.null),
-        )
-        .ok()
-    }
-    // unimplemented
-    fn enc_entry(_: &mut VecU8, _: &Self::Key, _: &Self::Value) {
-        unimplemented!()
-    }
-    unsafe fn dec_entry(
-        _: &mut BufferedScanner,
-        _: Self::EntryMD,
-    ) -> Option<(Self::Key, Self::Value)> {
-        unimplemented!()
-    }
-}
-
-// TODO(@ohsayan): common trait for k/v associations, independent of underlying maptype
-pub struct FieldMapSpecST;
-impl PersistMapSpec for FieldMapSpecST {
-    type MapIter<'a> = std::collections::hash_map::Iter<'a, Box<str>, Field>;
-    type MapType = std::collections::HashMap<Box<str>, Field>;
-    type EntryMD = FieldMapEntryMD;
-    type Key = Box<str>;
-    type Value = Field;
-    const ENC_COUPLED: bool = false;
-    const DEC_COUPLED: bool = false;
-    fn _get_iter<'a>(m: &'a Self::MapType) -> Self::MapIter<'a> {
-        m.iter()
-    }
-    fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool {
+    fn decode_pretest_for_entry_meta(scanner: &mut BufferedScanner) -> bool {
         scanner.has_left(sizeof!(u64, 3) + 1)
     }
-    fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool {
-        scanner.has_left(md.field_id_l as usize) // TODO(@ohsayan): we can enforce way more here such as atleast one field etc
+    fn decode_pretest_for_entry_data(s: &mut BufferedScanner, md: &Self::EntryMetadata) -> bool {
+        s.has_left(md.field_id_l as usize) // TODO(@ohsayan): we can enforce way more here such as atleast one field etc
     }
-    fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, val: &Self::Value) {
-        buf.extend(key.len().u64_bytes_le());
-        buf.extend(0u64.to_le_bytes()); // TODO(@ohsayan): props
-        buf.extend(val.layers().len().u64_bytes_le());
-        buf.push(val.is_nullable() as u8);
-    }
-    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::EntryMD> {
-        Some(FieldMapEntryMD::new(
+    unsafe fn decode_entry_meta(scanner: &mut BufferedScanner) -> Option<Self::EntryMetadata> {
+        Some(FieldMapEntryMetadata::new(
             scanner.next_u64_le(),
             scanner.next_u64_le(),
             scanner.next_u64_le(),
             scanner.next_byte(),
         ))
     }
-    fn enc_key(buf: &mut VecU8, key: &Self::Key) {
-        buf.extend(key.as_bytes());
-    }
-    fn enc_val(buf: &mut VecU8, val: &Self::Value) {
-        for layer in val.layers() {
-            super::obj::LayerRef::default_full_enc(buf, super::obj::LayerRef(layer))
-        }
+    unsafe fn decode_entry_data(
+        _: &mut BufferedScanner,
+        _: Self::EntryMetadata,
+    ) -> Option<(Self::RestoredKey, Self::RestoredVal)> {
+        unimplemented!()
     }
-    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Key> {
+    unsafe fn decode_entry_key(
+        scanner: &mut BufferedScanner,
+        md: &Self::EntryMetadata,
+    ) -> Option<Self::RestoredKey> {
         inf::dec::utils::decode_string(scanner, md.field_id_l as usize)
             .map(|s| s.into_boxed_str())
             .ok()
     }
-    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value> {
+    unsafe fn decode_entry_val(
+        scanner: &mut BufferedScanner,
+        md: &Self::EntryMetadata,
+    ) -> Option<Self::RestoredVal> {
         super::obj::FieldRef::obj_dec(
             scanner,
             FieldMD::new(md.field_prop_c, md.field_layer_c, md.null),
         )
         .ok()
     }
-    // unimplemented
-    fn enc_entry(_: &mut VecU8, _: &Self::Key, _: &Self::Value) {
-        unimplemented!()
-    }
-    unsafe fn dec_entry(
-        _: &mut BufferedScanner,
-        _: Self::EntryMD,
-    ) -> Option<(Self::Key, Self::Value)> {
-        unimplemented!()
-    }
 }
diff --git a/server/src/engine/storage/v1/inf/mod.rs b/server/src/engine/storage/v1/inf/mod.rs
index 24926db7..8a367b90 100644
--- a/server/src/engine/storage/v1/inf/mod.rs
+++ b/server/src/engine/storage/v1/inf/mod.rs
@@ -35,7 +35,7 @@ mod tests;
 use crate::engine::{
     error::{RuntimeResult, StorageError},
     idx::{AsKey, AsValue, STIndex},
-    mem::BufferedScanner,
+    mem::{BufferedScanner, StatelessLen},
 };
 
 type VecU8 = Vec<u8>;
@@ -142,64 +142,74 @@ pub trait PersistObject {
     map spec
 */
 
-/// specification for a persist map
-pub trait PersistMapSpec {
-    /// map type
-    type MapType: STIndex<Self::Key, Self::Value>;
-    /// map iter
-    type MapIter<'a>: Iterator<Item = (&'a Self::Key, &'a Self::Value)>
+pub trait AbstractMap<K, V> {
+    fn map_new() -> Self;
+    fn map_insert(&mut self, k: K, v: V) -> bool;
+    fn map_length(&self) -> usize;
+}
+
+impl<K: AsKey, V: AsValue, M: STIndex<K, V>> AbstractMap<K, V> for M {
+    fn map_new() -> Self {
+        Self::idx_init()
+    }
+    fn map_insert(&mut self, k: K, v: V) -> bool {
+        self.st_insert(k, v)
+    }
+    fn map_length(&self) -> usize {
+        self.st_len()
+    }
+}
+
+pub trait MapStorageSpec {
+    // in memory
+    type InMemoryMap: StatelessLen;
+    type InMemoryKey: ?Sized;
+    type InMemoryVal;
+    type InMemoryMapIter<'a>: Iterator<Item = (&'a Self::InMemoryKey, &'a Self::InMemoryVal)>
     where
-        Self: 'a;
-    /// metadata type
-    type EntryMD;
-    /// key type (NOTE: set this to the true key type; handle any differences using the spec unless you have an entirely different
-    /// wrapper type)
-    type Key: AsKey;
-    /// value type (NOTE: see [`PersistMapSpec::Key`])
-    type Value: AsValue;
-    /// coupled enc
-    const ENC_COUPLED: bool;
-    /// coupled dec
-    const DEC_COUPLED: bool;
-    // collection misc
-    fn _get_iter<'a>(map: &'a Self::MapType) -> Self::MapIter<'a>;
-    // collection meta
-    /// pretest before jmp to routine for entire collection
-    fn pretest_collection_using_size(_: &BufferedScanner, _: usize) -> bool {
+        Self: 'a,
+        Self::InMemoryKey: 'a,
+        Self::InMemoryVal: 'a;
+    // from disk
+    type RestoredKey: AsKey;
+    type RestoredVal: AsValue;
+    type RestoredMap: AbstractMap<Self::RestoredKey, Self::RestoredVal>;
+    // metadata
+    type EntryMetadata;
+    // settings
+    const ENC_AS_ENTRY: bool;
+    const DEC_AS_ENTRY: bool;
+    // iterator
+    fn get_iter_from_memory<'a>(map: &'a Self::InMemoryMap) -> Self::InMemoryMapIter<'a>;
+    // encode
+    fn encode_entry_meta(buf: &mut VecU8, key: &Self::InMemoryKey, val: &Self::InMemoryVal);
+    fn encode_entry_data(buf: &mut VecU8, key: &Self::InMemoryKey, val: &Self::InMemoryVal);
+    fn encode_entry_key(buf: &mut VecU8, key: &Self::InMemoryKey);
+    fn encode_entry_val(buf: &mut VecU8, val: &Self::InMemoryVal);
+    // decode
+    fn decode_pretest_for_map(_: &BufferedScanner, _: usize) -> bool {
         true
     }
-    /// pretest before jmp to entry dec routine
-    fn pretest_entry_metadata(scanner: &BufferedScanner) -> bool;
-    /// pretest the src before jmp to entry data dec routine
-    fn pretest_entry_data(scanner: &BufferedScanner, md: &Self::EntryMD) -> bool;
-    // entry meta
-    /// enc the entry meta
-    fn entry_md_enc(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);
-    /// dec the entry meta
-    /// SAFETY: ensure that all pretests have passed (we expect the caller to not be stupid)
-    unsafe fn entry_md_dec(scanner: &mut BufferedScanner) -> Option<Self::EntryMD>;
-    // independent packing
-    /// enc key (non-packed)
-    fn enc_key(buf: &mut VecU8, key: &Self::Key);
-    /// enc val (non-packed)
-    fn enc_val(buf: &mut VecU8, key: &Self::Value);
-    /// dec key (non-packed)
-    unsafe fn dec_key(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Key>;
-    /// dec val (non-packed)
-    unsafe fn dec_val(scanner: &mut BufferedScanner, md: &Self::EntryMD) -> Option<Self::Value>;
-    // coupled packing
-    /// entry packed enc
-    fn enc_entry(buf: &mut VecU8, key: &Self::Key, val: &Self::Value);
-    /// entry packed dec
-    unsafe fn dec_entry(
-        scanner: &mut BufferedScanner,
-        md: Self::EntryMD,
-    ) -> Option<(Self::Key, Self::Value)>;
+    fn decode_pretest_for_entry_meta(scanner: &mut BufferedScanner) -> bool;
+    fn decode_pretest_for_entry_data(s: &mut BufferedScanner, md: &Self::EntryMetadata) -> bool;
+    unsafe fn decode_entry_meta(s: &mut BufferedScanner) -> Option<Self::EntryMetadata>;
+    unsafe fn decode_entry_data(
+        s: &mut BufferedScanner,
+        md: Self::EntryMetadata,
+    ) -> Option<(Self::RestoredKey, Self::RestoredVal)>;
+    unsafe fn decode_entry_key(
+        s: &mut BufferedScanner,
+        md: &Self::EntryMetadata,
+    ) -> Option<Self::RestoredKey>;
+    unsafe fn decode_entry_val(
+        s: &mut BufferedScanner,
+        md: &Self::EntryMetadata,
+    ) -> Option<Self::RestoredVal>;
 }
 
 // enc
 pub mod enc {
-    use super::{map, PersistMapSpec, PersistObject, VecU8};
+    use super::{map, MapStorageSpec, PersistObject, VecU8};
     // obj
     #[cfg(test)]
     pub fn enc_full<Obj: PersistObject>(obj: Obj::InputType) -> Vec<u8> {
@@ -215,12 +225,12 @@ pub mod enc {
         enc_full::<Obj>(obj)
     }
     // dict
-    pub fn enc_dict_full<PM: PersistMapSpec>(dict: &PM::MapType) -> Vec<u8> {
+    pub fn enc_dict_full<PM: MapStorageSpec>(dict: &PM::InMemoryMap) -> Vec<u8> {
         let mut v = vec![];
         enc_dict_full_into_buffer::<PM>(&mut v, dict);
         v
     }
-    pub fn enc_dict_full_into_buffer<PM: PersistMapSpec>(buf: &mut VecU8, dict: &PM::MapType) {
+    pub fn enc_dict_full_into_buffer<PM: MapStorageSpec>(buf: &mut VecU8, dict: &PM::InMemoryMap) {
         <map::PersistMapImpl<PM> as PersistObject>::default_full_enc(buf, dict)
     }
 }
@@ -228,7 +238,7 @@ pub mod enc {
 // dec
 pub mod dec {
     use {
-        super::{map, PersistMapSpec, PersistObject},
+        super::{map, MapStorageSpec, PersistObject},
         crate::engine::{error::RuntimeResult, mem::BufferedScanner},
     };
     // obj
@@ -243,13 +253,13 @@ pub mod dec {
         Obj::default_full_dec(scanner)
     }
     // dec
-    pub fn dec_dict_full<PM: PersistMapSpec>(data: &[u8]) -> RuntimeResult<PM::MapType> {
+    pub fn dec_dict_full<PM: MapStorageSpec>(data: &[u8]) -> RuntimeResult<PM::RestoredMap> {
         let mut scanner = BufferedScanner::new(data);
         dec_dict_full_from_scanner::<PM>(&mut scanner)
     }
-    fn dec_dict_full_from_scanner<PM: PersistMapSpec>(
+    fn dec_dict_full_from_scanner<PM: MapStorageSpec>(
         scanner: &mut BufferedScanner,
-    ) -> RuntimeResult<PM::MapType> {
+    ) -> RuntimeResult<PM::RestoredMap> {
         <map::PersistMapImpl<PM> as PersistObject>::default_full_dec(scanner)
     }
     pub mod utils {
diff --git a/server/src/engine/storage/v1/inf/obj.rs b/server/src/engine/storage/v1/inf/obj.rs
index f2ed1372..62cb7679 100644
--- a/server/src/engine/storage/v1/inf/obj.rs
+++ b/server/src/engine/storage/v1/inf/obj.rs
@@ -38,6 +38,7 @@ use {
                 DictGeneric,
             },
             error::{RuntimeResult, StorageError},
+            idx::IndexSTSeqCns,
             mem::{BufferedScanner, VInline},
             storage::v1::inf,
         },
@@ -378,6 +379,7 @@ impl<'a> PersistObject for FieldRef<'a> {
     }
 }
 
+#[derive(Debug)]
 pub struct ModelLayoutMD {
     model_uuid: Uuid,
     p_key_len: u64,
@@ -435,7 +437,7 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
     }
     fn obj_enc(buf: &mut VecU8, ModelLayoutRef(model_definition): Self::InputType) {
         buf.extend(model_definition.p_key().as_bytes());
-        <super::map::PersistMapImpl<super::map::FieldMapSpec> as PersistObject>::obj_enc(
+        <super::map::PersistMapImpl<super::map::FieldMapSpec<_>> as PersistObject>::obj_enc(
             buf,
             model_definition.fields(),
         )
@@ -445,11 +447,11 @@ impl<'a> PersistObject for ModelLayoutRef<'a> {
         md: Self::Metadata,
     ) -> RuntimeResult<Self::OutputType> {
         let key = inf::dec::utils::decode_string(scanner, md.p_key_len as usize)?;
-        let fieldmap =
-            <super::map::PersistMapImpl<super::map::FieldMapSpec> as PersistObject>::obj_dec(
-                scanner,
-                super::map::MapIndexSizeMD(md.field_c as usize),
-            )?;
+        let fieldmap = <super::map::PersistMapImpl<
+            super::map::FieldMapSpec<IndexSTSeqCns<Box<str>, _>>,
+        > as PersistObject>::obj_dec(
+            scanner, super::map::MapIndexSizeMD(md.field_c as usize)
+        )?;
         let ptag = if md.p_key_tag > TagSelector::MAX as u64 {
             return Err(StorageError::InternalDecodeStructureCorruptedPayload.into());
         } else {
diff --git a/server/src/engine/storage/v1/inf/tests.rs b/server/src/engine/storage/v1/inf/tests.rs
index a942652a..9e1ce4fc 100644
--- a/server/src/engine/storage/v1/inf/tests.rs
+++ b/server/src/engine/storage/v1/inf/tests.rs
@@ -80,8 +80,11 @@ fn fieldmap() {
         "profile_pic".into(),
         Field::new([Layer::bin()].into(), true),
     );
-    let enc = super::enc::enc_dict_full::<super::map::FieldMapSpec>(&fields);
-    let dec = super::dec::dec_dict_full::<super::map::FieldMapSpec>(&enc).unwrap();
+    let enc = super::enc::enc_dict_full::<super::map::FieldMapSpec<_>>(&fields);
+    let dec = super::dec::dec_dict_full::<
+        super::map::FieldMapSpec<crate::engine::idx::IndexSTSeqCns<Box<str>, _>>,
+    >(&enc)
+    .unwrap();
     for ((orig_field_id, orig_field), (restored_field_id, restored_field)) in
         fields.stseq_ord_kv().zip(dec.stseq_ord_kv())
     {
diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs
index 61fe064a..e4340b64 100644
--- a/server/src/engine/storage/v1/loader.rs
+++ b/server/src/engine/storage/v1/loader.rs
@@ -76,20 +76,24 @@ impl SEInitState {
                 std::fs::create_dir(DATA_DIR).inherit_set_dmsg("creating data directory")?;
             }
             if !is_new {
-                let models = gns.idx_models().read();
+                let mut models = gns.idx_models().write();
                 // this is an existing instance, so read in all data
                 for (space_name, space) in gns.idx().read().iter() {
                     let space = space.read();
                     let space_uuid = space.get_uuid();
                     for model_name in space.models().iter() {
                         let model = models
-                            .get(&EntityIDRef::new(&space_name, &model_name))
+                            .get_mut(&EntityIDRef::new(&space_name, &model_name))
                             .unwrap();
                         let path =
                             Self::model_path(space_name, space_uuid, model_name, model.get_uuid());
                         let persist_driver = batch_jrnl::reinit(&path, model).inherit_set_dmsg(
                             format!("failed to restore model data from journal in `{path}`"),
                         )?;
+                        unsafe {
+                            // UNSAFE(@ohsayan): all pieces of data are upgraded by now, so vacuum
+                            model.model_mutator().vacuum_stashed();
+                        }
                         let _ = model_drivers.insert(
                             ModelUniqueID::new(space_name, model_name, model.get_uuid()),
                             FractalModelDriver::init(persist_driver),
diff --git a/server/src/engine/storage/v1/rw.rs b/server/src/engine/storage/v1/rw.rs
index 5f29b828..83b6d087 100644
--- a/server/src/engine/storage/v1/rw.rs
+++ b/server/src/engine/storage/v1/rw.rs
@@ -308,7 +308,7 @@ impl<Fs: RawFSInterface> SDSSFileTrackedWriter<Fs> {
             cs: SCrc::new(),
         })
     }
-    pub fn write_unfsynced(&mut self, block: &[u8]) -> RuntimeResult<()> {
+    pub fn tracked_write_unfsynced(&mut self, block: &[u8]) -> RuntimeResult<()> {
         self.untracked_write(block)
             .map(|_| self.cs.recompute_with_new_var_block(block))
     }
@@ -322,8 +322,7 @@ impl<Fs: RawFSInterface> SDSSFileTrackedWriter<Fs> {
         self.f.f.sync_write_cache()
     }
     pub fn reset_and_finish_checksum(&mut self) -> u64 {
-        let mut scrc = SCrc::new();
-        core::mem::swap(&mut self.cs, &mut scrc);
+        let scrc = core::mem::replace(&mut self.cs, SCrc::new());
         scrc.finish()
     }
     pub fn into_inner_file(self) -> RuntimeResult<SDSSFileIO<Fs>> {
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index 629778e3..0ddbd4cb 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -158,7 +158,7 @@ impl<'a> PersistObject for ModelID<'a> {
 fn with_space<T>(
     gns: &GlobalNS,
     space_id: &super::SpaceIDRes,
-    mut f: impl FnMut(&Space) -> RuntimeResult<T>,
+    f: impl FnOnce(&Space) -> RuntimeResult<T>,
 ) -> RuntimeResult<T> {
     let spaces = gns.idx().read();
     let Some(space) = spaces.st_get(&space_id.name) else {
@@ -191,7 +191,7 @@ fn with_model_mut<T>(
     gns: &GlobalNS,
     space_id: &super::SpaceIDRes,
     model_id: &ModelIDRes,
-    mut f: impl FnMut(&mut Model) -> RuntimeResult<T>,
+    f: impl FnOnce(&mut Model) -> RuntimeResult<T>,
 ) -> RuntimeResult<T> {
     with_space(gns, space_id, |_| {
         let mut models = gns.idx_models().write();
@@ -394,14 +394,14 @@ impl<'a> PersistObject for AlterModelAddTxn<'a> {
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
         <ModelID as PersistObject>::obj_enc(buf, data.model_id);
-        <map::PersistMapImpl<map::FieldMapSpec> as PersistObject>::obj_enc(buf, data.new_fields);
+        <map::PersistMapImpl<map::FieldMapSpec<_>> as PersistObject>::obj_enc(buf, data.new_fields);
     }
     unsafe fn obj_dec(
         s: &mut BufferedScanner,
         md: Self::Metadata,
     ) -> RuntimeResult<Self::OutputType> {
         let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_meta)?;
-        let new_fields = <map::PersistMapImpl<map::FieldMapSpec> as PersistObject>::obj_dec(
+        let new_fields = <map::PersistMapImpl<map::FieldMapSpec<IndexSTSeqCns<Box<str>, _>>> as PersistObject>::obj_dec(
             s,
             map::MapIndexSizeMD(md.new_field_c as usize),
         )?;
@@ -424,25 +424,12 @@ impl<'a> GNSEvent for AlterModelAddTxn<'a> {
         gns: &GlobalNS,
     ) -> RuntimeResult<()> {
         with_model_mut(gns, &model_id.space_id, &model_id, |model| {
-            for (i, (field_name, field)) in new_fields.stseq_ord_kv().enumerate() {
-                if !model
-                    .fields_mut()
-                    .st_insert(field_name.to_owned(), field.clone())
-                {
-                    // rollback; corrupted
-                    new_fields.stseq_ord_key().take(i).for_each(|field_id| {
-                        let _ = model.fields_mut().st_delete(field_id);
-                    });
+            let mut mutator = model.model_mutator();
+            for (field_name, field) in new_fields.stseq_owned_kv() {
+                if !mutator.add_field(field_name, field) {
                     return Err(TransactionError::OnRestoreDataConflictMismatch.into());
                 }
             }
-            // TODO(@ohsayan): avoid double iteration
-            // publish deltas
-            for field_name in new_fields.stseq_ord_key() {
-                model
-                    .delta_state_mut()
-                    .schema_append_unresolved_wl_field_add(field_name);
-            }
             Ok(())
         })
     }
@@ -542,28 +529,12 @@ impl<'a> GNSEvent for AlterModelRemoveTxn<'a> {
         gns: &GlobalNS,
     ) -> RuntimeResult<()> {
         with_model_mut(gns, &model_id.space_id, &model_id, |model| {
-            let mut removed_fields_rb = vec![];
+            let mut mutator = model.model_mutator();
             for removed_field in removed_fields.iter() {
-                match model.fields_mut().st_delete_return(removed_field) {
-                    Some(field) => {
-                        removed_fields_rb.push((removed_field as &str, field));
-                    }
-                    None => {
-                        // rollback
-                        removed_fields_rb.into_iter().for_each(|(field_id, field)| {
-                            let _ = model.fields_mut().st_insert(field_id.into(), field);
-                        });
-                        return Err(TransactionError::OnRestoreDataConflictMismatch.into());
-                    }
+                if !mutator.remove_field(&removed_field) {
+                    return Err(TransactionError::OnRestoreDataConflictMismatch.into());
                 }
             }
-            // TODO(@ohsayan): avoid double iteration
-            // publish deltas
-            for field_name in removed_fields.iter() {
-                model
-                    .delta_state_mut()
-                    .schema_append_unresolved_wl_field_rem(field_name);
-            }
             Ok(())
         })
     }
@@ -598,7 +569,7 @@ pub struct AlterModelUpdateTxnMD {
 #[derive(Debug, PartialEq)]
 pub struct AlterModelUpdateTxnRestorePL {
     pub(super) model_id: ModelIDRes,
-    pub(super) updated_fields: IndexST<Box<str>, Field>,
+    pub(super) updated_fields: IndexSTSeqCns<Box<str>, Field>,
 }
 
 impl<'a> PersistObject for AlterModelUpdateTxn<'a> {
@@ -624,7 +595,7 @@ impl<'a> PersistObject for AlterModelUpdateTxn<'a> {
     }
     fn obj_enc(buf: &mut Vec<u8>, data: Self::InputType) {
         <ModelID as PersistObject>::obj_enc(buf, data.model_id);
-        <map::PersistMapImpl<map::FieldMapSpecST> as PersistObject>::obj_enc(
+        <map::PersistMapImpl<map::FieldMapSpec<_>> as PersistObject>::obj_enc(
             buf,
             data.updated_fields,
         );
@@ -634,10 +605,11 @@ impl<'a> PersistObject for AlterModelUpdateTxn<'a> {
         md: Self::Metadata,
     ) -> RuntimeResult<Self::OutputType> {
         let model_id = <ModelID as PersistObject>::obj_dec(s, md.model_id_md)?;
-        let updated_fields = <map::PersistMapImpl<map::FieldMapSpecST> as PersistObject>::obj_dec(
-            s,
-            map::MapIndexSizeMD(md.updated_field_c as usize),
-        )?;
+        let updated_fields =
+            <map::PersistMapImpl<map::FieldMapSpec<IndexSTSeqCns<Box<str>, _>>> as PersistObject>::obj_dec(
+                s,
+                map::MapIndexSizeMD(md.updated_field_c as usize),
+            )?;
         Ok(AlterModelUpdateTxnRestorePL {
             model_id,
             updated_fields,
@@ -657,17 +629,10 @@ impl<'a> GNSEvent for AlterModelUpdateTxn<'a> {
         gns: &GlobalNS,
     ) -> RuntimeResult<()> {
         with_model_mut(gns, &model_id.space_id, &model_id, |model| {
-            let mut fields_rb = vec![];
-            for (field_id, field) in updated_fields.iter() {
-                match model.fields_mut().st_update_return(field_id, field.clone()) {
-                    Some(f) => fields_rb.push((field_id as &str, f)),
-                    None => {
-                        // rollback
-                        fields_rb.into_iter().for_each(|(field_id, field)| {
-                            let _ = model.fields_mut().st_update(field_id, field);
-                        });
-                        return Err(TransactionError::OnRestoreDataConflictMismatch.into());
-                    }
+            let mut mutator = model.model_mutator();
+            for (field_id, field) in updated_fields.stseq_owned_kv() {
+                if !mutator.update_field(&field_id, field) {
+                    return Err(TransactionError::OnRestoreDataConflictMismatch.into());
                 }
             }
             Ok(())
diff --git a/server/src/engine/txn/gns/tests/io.rs b/server/src/engine/txn/gns/tests/io.rs
index fd6b6dcc..99ffc200 100644
--- a/server/src/engine/txn/gns/tests/io.rs
+++ b/server/src/engine/txn/gns/tests/io.rs
@@ -155,7 +155,9 @@ mod model_tests {
                     model.get_uuid(),
                     model.delta_state().schema_current_version().value_u64()
                 ),
-                new_fields
+                new_fields: into_dict! {
+                    "auth_2fa" => Field::new([Layer::bool()].into(), true),
+                }
             },
             decoded
         );
@@ -191,6 +193,10 @@ mod model_tests {
     #[test]
     fn alter_update() {
         let (space, model) = default_space_model();
+        let updated_fields_copy = into_dict! {
+            // people of your social app will hate this, but hehe
+            "profile_pic" => Field::new([Layer::bin()].into(), false)
+        };
         let updated_fields = into_dict! {
             // people of your social app will hate this, but hehe
             "profile_pic" => Field::new([Layer::bin()].into(), false)
@@ -214,7 +220,7 @@ mod model_tests {
                     model.get_uuid(),
                     model.delta_state().schema_current_version().value_u64()
                 ),
-                updated_fields
+                updated_fields: updated_fields_copy
             },
             decoded
         );

From 6ea97587147ada6b950c6f5bf9565c7be119bff2 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Fri, 24 Nov 2023 23:17:58 +0530
Subject: [PATCH 292/310] Ensure queries are evaluated completely

---
 CHANGELOG.md                               | 18 +++++++-
 server/src/engine/core/dcl.rs              | 18 ++++++--
 server/src/engine/core/exec.rs             | 28 ++----------
 server/src/engine/ql/ast/mod.rs            | 21 ++++-----
 server/src/engine/ql/ast/traits.rs         | 44 +++++++++++++++---
 server/src/engine/ql/dcl.rs                | 53 +++++++++++++++++-----
 server/src/engine/ql/ddl/alt.rs            | 12 ++++-
 server/src/engine/ql/ddl/crt.rs            | 12 ++++-
 server/src/engine/ql/ddl/drop.rs           | 40 +++++++++-------
 server/src/engine/ql/ddl/ins.rs            |  6 ++-
 server/src/engine/ql/ddl/syn.rs            | 44 ++++++++++++++----
 server/src/engine/ql/dml/del.rs            |  6 ++-
 server/src/engine/ql/dml/ins.rs            | 30 +++++++++---
 server/src/engine/ql/dml/mod.rs            | 14 ++++--
 server/src/engine/ql/dml/sel.rs            |  6 ++-
 server/src/engine/ql/dml/upd.rs            | 14 ++++--
 server/src/engine/ql/lex/raw.rs            |  3 ++
 server/src/engine/ql/tests/dcl.rs          | 28 +++++++++---
 server/src/engine/ql/tests/schema_tests.rs |  8 ++--
 19 files changed, 288 insertions(+), 117 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 221b6142..e0f9e9dc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -27,9 +27,25 @@ All changes in this project will be noted in this file.
   - **All actions removed**: All the prior `SET`, `GET` and other actions have been removed in favor of the new query language
   - The following queries were added:
     - `INSERT INTO <space>.<model>(col, col2, col3, ...)`
+      - Insert queries can use several ways of insertion including:
+        - a standard order of declared columns like this:
+          ```sql
+          INSERT INTO myspace.mymodel(col1, col2, col3, ...)
+          ```
+        - using a map:
+          ```sql
+          INSERT INTO myspace.mymodel { col1: val1, col2: val2, col4: val4, col3: val3 }
+          ```
+      - Inserts can also make use of function calls during inserts:
+        - It can be called like this: `INSERT INTO myspace.mymodel(@uuidstr, ...)`
+        - The following functions are available:
+          - `@uuidstr`: returns a string with a randomly generated v4 UUID
+          - `@uuidbin`: same as `@uuidstr` but returns it as a blob
+          - `@timesec`: returns a 64-bit integer with the current time in seconds
     - `SELECT field1, field2, ... FROM <space>.<model> WHERE <primary_key_column> = <value>`
-    - New data manipulation via `UPDATE` allows arithmetic operations, string manipulation and more!:
+    - New data manipulation via `UPDATE` allows arithmetic operations, string manipulation and more! Examples:
       - `UPDATE <space>.<model> SET col_num += 1 WHERE <primary_key_column> = <value>`
+      - `UPDATE <space>.<model> SET mystring += " last part of string" WHERE ...`
     - `DELETE FROM <space>.<model> WHERE <primary_key_column> = <value>`
 - DCL:
   - `SYSCTL CREATE USER <name> WITH { password: <password> }`
diff --git a/server/src/engine/core/dcl.rs b/server/src/engine/core/dcl.rs
index 127328d0..7bf0d6c8 100644
--- a/server/src/engine/core/dcl.rs
+++ b/server/src/engine/core/dcl.rs
@@ -29,12 +29,24 @@ use crate::engine::{
     error::{QueryError, QueryResult},
     fractal::GlobalInstanceLike,
     net::protocol::ClientLocalState,
-    ql::dcl::{UserAdd, UserDel},
+    ql::dcl::{SysctlCommand, UserAdd, UserDel},
 };
 
 const KEY_PASSWORD: &str = "password";
 
-pub fn create_user(global: &impl GlobalInstanceLike, mut user_add: UserAdd<'_>) -> QueryResult<()> {
+pub fn exec<G: GlobalInstanceLike>(
+    g: G,
+    current_user: &ClientLocalState,
+    cmd: SysctlCommand,
+) -> QueryResult<()> {
+    match cmd {
+        SysctlCommand::CreateUser(new) => create_user(&g, new),
+        SysctlCommand::DropUser(drop) => drop_user(&g, current_user, drop),
+        SysctlCommand::ReportStatus => Ok(()),
+    }
+}
+
+fn create_user(global: &impl GlobalInstanceLike, mut user_add: UserAdd<'_>) -> QueryResult<()> {
     let username = user_add.username().to_owned();
     let password = match user_add.options_mut().remove(KEY_PASSWORD) {
         Some(DictEntryGeneric::Data(d))
@@ -48,7 +60,7 @@ pub fn create_user(global: &impl GlobalInstanceLike, mut user_add: UserAdd<'_>)
     global.sys_store().create_new_user(username, password)
 }
 
-pub fn drop_user(
+fn drop_user(
     global: &impl GlobalInstanceLike,
     cstate: &ClientLocalState,
     user_del: UserDel<'_>,
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 8436ff65..1d10113b 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -82,7 +82,7 @@ fn _call<A: ASTNode<'static> + core::fmt::Debug, T>(
     state: &mut State<'static, InplaceData>,
     f: impl FnOnce(&Global, A) -> Result<T, QueryError>,
 ) -> QueryResult<T> {
-    let cs = ASTNode::from_state(state)?;
+    let cs = ASTNode::parse_from_state_hardened(state)?;
     f(&g, cs)
 }
 
@@ -143,30 +143,8 @@ fn blocking_exec_sysctl(
     tokens: RawSlice<Token<'static>>,
 ) -> QueryResult<()> {
     let mut state = State::new_inplace(&tokens);
-    /*
-        currently supported: sysctl create user, sysctl drop user
-    */
-    if state.remaining() < 2 {
-        return Err(QueryError::QLInvalidSyntax);
-    }
-    let (a, b) = (state.fw_read(), state.fw_read());
-    match (a, b) {
-        (Token![create], Token::Ident(id)) if id.eq_ignore_ascii_case("user") => {
-            let useradd = ASTNode::from_state(&mut state)?;
-            super::dcl::create_user(&g, useradd)
-        }
-        (Token![drop], Token::Ident(id)) if id.eq_ignore_ascii_case("user") => {
-            let userdel = ASTNode::from_state(&mut state)?;
-            super::dcl::drop_user(&g, cstate, userdel)
-        }
-        (Token::Ident(k1), Token::Ident(k2))
-            if k1.eq_ignore_ascii_case("report") && k2.eq_ignore_ascii_case("status") =>
-        {
-            // TODO(@ohsayan): replace dummy endpoint with actual `system report status` responses
-            Ok(())
-        }
-        _ => Err(QueryError::QLUnknownStatement),
-    }
+    let r = ASTNode::parse_from_state_hardened(&mut state)?;
+    super::dcl::exec(g, cstate, r)
 }
 
 /*
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index e605ca67..b4b61306 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -26,8 +26,6 @@
 
 pub mod traits;
 
-#[cfg(debug_assertions)]
-use self::traits::ASTNode;
 #[cfg(test)]
 pub use traits::{parse_ast_node_full, parse_ast_node_multiple_full};
 use {
@@ -504,9 +502,10 @@ pub enum Statement<'a> {
 }
 
 #[inline(always)]
-#[cfg(debug_assertions)]
+#[cfg(test)]
 #[allow(dead_code)] // TODO(@ohsayan): get rid of this
 pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResult<Statement<'a>> {
+    use self::traits::ASTNode;
     if compiler::unlikely(tok.len() < 2) {
         return Err(QueryError::QLUnexpectedEndOfStatement);
     }
@@ -515,13 +514,13 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResul
         // DDL
         Token![use] => Entity::parse_from_state_rounded_result(&mut state).map(Statement::Use),
         Token![create] => match state.fw_read() {
-            Token![model] => ASTNode::from_state(&mut state).map(Statement::CreateModel),
-            Token![space] => ASTNode::from_state(&mut state).map(Statement::CreateSpace),
+            Token![model] => ASTNode::test_parse_from_state(&mut state).map(Statement::CreateModel),
+            Token![space] => ASTNode::test_parse_from_state(&mut state).map(Statement::CreateSpace),
             _ => compiler::cold_rerr(QueryError::QLUnknownStatement),
         },
         Token![alter] => match state.fw_read() {
-            Token![model] => ASTNode::from_state(&mut state).map(Statement::AlterModel),
-            Token![space] => ASTNode::from_state(&mut state).map(Statement::AlterSpace),
+            Token![model] => ASTNode::test_parse_from_state(&mut state).map(Statement::AlterModel),
+            Token![space] => ASTNode::test_parse_from_state(&mut state).map(Statement::AlterSpace),
             _ => compiler::cold_rerr(QueryError::QLUnknownStatement),
         },
         Token![drop] if state.remaining() >= 2 => ddl::drop::parse_drop(&mut state),
@@ -529,10 +528,10 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResul
             ddl::ins::parse_inspect(&mut state)
         }
         // DML
-        Token![insert] => ASTNode::from_state(&mut state).map(Statement::Insert),
-        Token![select] => ASTNode::from_state(&mut state).map(Statement::Select),
-        Token![update] => ASTNode::from_state(&mut state).map(Statement::Update),
-        Token![delete] => ASTNode::from_state(&mut state).map(Statement::Delete),
+        Token![insert] => ASTNode::test_parse_from_state(&mut state).map(Statement::Insert),
+        Token![select] => ASTNode::test_parse_from_state(&mut state).map(Statement::Select),
+        Token![update] => ASTNode::test_parse_from_state(&mut state).map(Statement::Update),
+        Token![delete] => ASTNode::test_parse_from_state(&mut state).map(Statement::Delete),
         _ => compiler::cold_rerr(QueryError::QLUnknownStatement),
     }
 }
diff --git a/server/src/engine/ql/ast/traits.rs b/server/src/engine/ql/ast/traits.rs
index 79a919ff..925a4da5 100644
--- a/server/src/engine/ql/ast/traits.rs
+++ b/server/src/engine/ql/ast/traits.rs
@@ -33,17 +33,47 @@ use crate::engine::{
 
 /// An AST node
 pub trait ASTNode<'a>: Sized {
-    const VERIFY: bool = false;
+    /// This AST node MUST use the full token range
+    const MUST_USE_FULL_TOKEN_RANGE: bool;
+    /// This AST node MUST use the full token range, and it also verifies that this is the case
+    const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool;
+    /// This AST node doesn't handle "deep errors" (for example, recursive collections)
+    const VERIFY_STATE_BEFORE_RETURN: bool = false;
+    /// A hardened parse that guarantees:
+    /// - The result is verified (even if it is a deep error)
+    /// - The result utilizes the full token range
+    fn parse_from_state_hardened<Qd: QueryData<'a>>(
+        state: &mut State<'a, Qd>,
+    ) -> QueryResult<Self> {
+        let r = Self::__base_impl_parse_from_state(state)?;
+        if Self::VERIFY_STATE_BEFORE_RETURN {
+            // must verify
+            if !state.okay() {
+                return Err(QueryError::QLInvalidSyntax);
+            }
+        }
+        if Self::MUST_USE_FULL_TOKEN_RANGE {
+            if !Self::VERIFIES_FULL_TOKEN_RANGE_USAGE {
+                if state.not_exhausted() {
+                    return Err(QueryError::QLInvalidSyntax);
+                }
+            }
+        }
+        Ok(r)
+    }
     /// Parse this AST node from the given state
     ///
     /// Note to implementors:
     /// - If the implementor uses a cow style parse, then set [`ASTNode::VERIFY`] to
     /// true
     /// - Try to propagate errors via [`State`] if possible
-    fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self>;
-    fn from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
-        let r = <Self as ASTNode>::_from_state(state);
-        if Self::VERIFY {
+    fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+        state: &mut State<'a, Qd>,
+    ) -> QueryResult<Self>;
+    #[cfg(test)]
+    fn test_parse_from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        let r = <Self as ASTNode>::__base_impl_parse_from_state(state);
+        if Self::VERIFY_STATE_BEFORE_RETURN {
             return if state.okay() {
                 r
             } else {
@@ -60,7 +90,7 @@ pub trait ASTNode<'a>: Sized {
     #[cfg(test)]
     fn multiple_from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Vec<Self>> {
         let r = <Self as ASTNode>::_multiple_from_state(state);
-        if Self::VERIFY {
+        if Self::VERIFY_STATE_BEFORE_RETURN {
             return if state.okay() {
                 r
             } else {
@@ -73,7 +103,7 @@ pub trait ASTNode<'a>: Sized {
     /// Parse this AST node utilizing the full token-stream. Intended for the test suite.
     fn from_insecure_tokens_full(tok: &'a [Token<'a>]) -> QueryResult<Self> {
         let mut state = State::new(tok, InplaceData::new());
-        let r = <Self as ASTNode>::from_state(&mut state)?;
+        let r = <Self as ASTNode>::test_parse_from_state(&mut state)?;
         assert!(state.exhausted());
         Ok(r)
     }
diff --git a/server/src/engine/ql/dcl.rs b/server/src/engine/ql/dcl.rs
index 17520174..5d6b9d31 100644
--- a/server/src/engine/ql/dcl.rs
+++ b/server/src/engine/ql/dcl.rs
@@ -36,6 +36,47 @@ use crate::engine::{
     },
 };
 
+#[derive(Debug, PartialEq)]
+pub enum SysctlCommand<'a> {
+    /// `sysctl create user ...`
+    CreateUser(UserAdd<'a>),
+    /// `sysctl drop user ...`
+    DropUser(UserDel<'a>),
+    /// `sysctl status`
+    ReportStatus,
+}
+
+impl<'a> traits::ASTNode<'a> for SysctlCommand<'a> {
+    const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+    const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+    fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+        state: &mut State<'a, Qd>,
+    ) -> QueryResult<Self> {
+        if state.remaining() < 1 {
+            return Err(QueryError::QLUnexpectedEndOfStatement);
+        }
+        let token = state.fw_read();
+        let create = Token![create].eq(token);
+        let drop = Token![drop].eq(token);
+        let status = token.ident_eq("status");
+        if status {
+            return Ok(SysctlCommand::ReportStatus);
+        }
+        if state.exhausted() {
+            return Err(QueryError::QLUnexpectedEndOfStatement);
+        }
+        let create_or_drop = state.fw_read();
+        if !create_or_drop.ident_eq("user") & !(create | drop) {
+            return Err(QueryError::QLUnknownStatement);
+        }
+        if create {
+            UserAdd::parse(state).map(SysctlCommand::CreateUser)
+        } else {
+            UserDel::parse(state).map(SysctlCommand::DropUser)
+        }
+    }
+}
+
 fn parse<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<UserMeta<'a>> {
     /*
         [username] with { password: [password], ... }
@@ -106,12 +147,6 @@ impl<'a> UserAdd<'a> {
     }
 }
 
-impl<'a> traits::ASTNode<'a> for UserAdd<'a> {
-    fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
-        Self::parse(state)
-    }
-}
-
 #[derive(Debug, PartialEq)]
 pub struct UserDel<'a> {
     username: &'a str,
@@ -144,9 +179,3 @@ impl<'a> UserDel<'a> {
         self.username
     }
 }
-
-impl<'a> traits::ASTNode<'a> for UserDel<'a> {
-    fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
-        Self::parse(state)
-    }
-}
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 4f833695..702dcdc8 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -196,12 +196,20 @@ mod impls {
         },
     };
     impl<'a> ASTNode<'a> for AlterModel<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
     impl<'a> ASTNode<'a> for AlterSpace<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index a728f8ba..edab417c 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -162,12 +162,20 @@ mod impls {
         },
     };
     impl<'a> ASTNode<'a> for CreateSpace<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
     impl<'a> ASTNode<'a> for CreateModel<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index cddef4c6..15634118 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -52,15 +52,13 @@ impl<'a> DropSpace<'a> {
             let force = state.cursor_rounded_eq(Token::Ident(Ident::from("force")));
             state.cursor_ahead_if(force);
             // either `force` or nothing
-            if state.exhausted() {
-                return Ok(DropSpace::new(
-                    unsafe {
-                        // UNSAFE(@ohsayan): Safe because the if predicate ensures that tok[0] (relative) is indeed an ident
-                        ident.uck_read_ident()
-                    },
-                    force,
-                ));
-            }
+            return Ok(DropSpace::new(
+                unsafe {
+                    // UNSAFE(@ohsayan): Safe because the if predicate ensures that tok[0] (relative) is indeed an ident
+                    ident.uck_read_ident()
+                },
+                force,
+            ));
         }
         Err(QueryError::QLInvalidSyntax)
     }
@@ -81,11 +79,7 @@ impl<'a> DropModel<'a> {
         let e = Entity::parse_from_state_rounded_result(state)?;
         let force = state.cursor_rounded_eq(Token::Ident(Ident::from("force")));
         state.cursor_ahead_if(force);
-        if state.exhausted() {
-            return Ok(DropModel::new(e, force));
-        } else {
-            Err(QueryError::QLInvalidSyntax)
-        }
+        Ok(DropModel::new(e, force))
     }
 }
 
@@ -112,19 +106,31 @@ mod impls {
         },
     };
     impl<'a> ASTNode<'a> for DropModel<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
     impl<'a> ASTNode<'a> for DropSpace<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
     #[derive(sky_macros::Wrapper, Debug)]
     pub struct DropStatementAST<'a>(Statement<'a>);
     impl<'a> ASTNode<'a> for DropStatementAST<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             super::parse_drop(state).map(Self)
         }
     }
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
index 323fb1b2..fc69e888 100644
--- a/server/src/engine/ql/ddl/ins.rs
+++ b/server/src/engine/ql/ddl/ins.rs
@@ -80,7 +80,11 @@ mod impls {
     #[derive(sky_macros::Wrapper, Debug)]
     pub struct InspectStatementAST<'a>(Statement<'a>);
     impl<'a> ASTNode<'a> for InspectStatementAST<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             super::parse_inspect(state).map(Self)
         }
     }
diff --git a/server/src/engine/ql/ddl/syn.rs b/server/src/engine/ql/ddl/syn.rs
index b2fcffe6..1c1a9388 100644
--- a/server/src/engine/ql/ddl/syn.rs
+++ b/server/src/engine/ql/ddl/syn.rs
@@ -535,7 +535,11 @@ mod impls {
         },
     };
     impl<'a> ASTNode<'a> for ExpandedField<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = false;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             Self::parse(state)
         }
         fn _multiple_from_state<Qd: QueryData<'a>>(
@@ -545,9 +549,13 @@ mod impls {
         }
     }
     impl<'a> ASTNode<'a> for LayerSpec<'a> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = false;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
         // important: upstream must verify this
-        const VERIFY: bool = true;
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const VERIFY_STATE_BEFORE_RETURN: bool = true;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             let mut layers = Vec::new();
             rfold_layers(state, &mut layers);
             assert!(layers.len() == 1);
@@ -564,9 +572,13 @@ mod impls {
     #[derive(sky_macros::Wrapper, Debug)]
     pub struct DictBasic(DictGeneric);
     impl<'a> ASTNode<'a> for DictBasic {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = false;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
         // important: upstream must verify this
-        const VERIFY: bool = true;
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const VERIFY_STATE_BEFORE_RETURN: bool = true;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             let mut dict = DictGeneric::new();
             rfold_dict(DictFoldState::OB, state, &mut dict);
             Ok(Self(dict))
@@ -575,9 +587,13 @@ mod impls {
     #[derive(sky_macros::Wrapper, Debug)]
     pub struct DictTypeMetaSplit(DictGeneric);
     impl<'a> ASTNode<'a> for DictTypeMetaSplit {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = false;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
         // important: upstream must verify this
-        const VERIFY: bool = true;
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const VERIFY_STATE_BEFORE_RETURN: bool = true;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             let mut dict = DictGeneric::new();
             rfold_tymeta(DictFoldState::CB_OR_IDENT, state, &mut dict);
             Ok(Self(dict))
@@ -586,16 +602,24 @@ mod impls {
     #[derive(sky_macros::Wrapper, Debug)]
     pub struct DictTypeMeta(DictGeneric);
     impl<'a> ASTNode<'a> for DictTypeMeta {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = false;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
         // important: upstream must verify this
-        const VERIFY: bool = true;
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const VERIFY_STATE_BEFORE_RETURN: bool = true;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             let mut dict = DictGeneric::new();
             rfold_tymeta(DictFoldState::OB, state, &mut dict);
             Ok(Self(dict))
         }
     }
     impl<'a> ASTNode<'a> for FieldSpec<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = false;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             Self::parse(state)
         }
     }
diff --git a/server/src/engine/ql/dml/del.rs b/server/src/engine/ql/dml/del.rs
index b1ca021d..ed38ca56 100644
--- a/server/src/engine/ql/dml/del.rs
+++ b/server/src/engine/ql/dml/del.rs
@@ -113,7 +113,11 @@ mod impls {
         },
     };
     impl<'a> ASTNode<'a> for DeleteStatement<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             Self::parse_delete(state)
         }
     }
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index 4265b71a..eaf70aba 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -400,7 +400,11 @@ mod impls {
         },
     };
     impl<'a> ASTNode<'a> for InsertStatement<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             Self::parse_insert(state)
         }
     }
@@ -418,9 +422,13 @@ mod impls {
         #[derive(sky_macros::Wrapper, Debug)]
         pub struct List(Vec<Datacell>);
         impl<'a> ASTNode<'a> for List {
+            const MUST_USE_FULL_TOKEN_RANGE: bool = false;
+            const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
             // important: upstream must verify this
-            const VERIFY: bool = true;
-            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+            const VERIFY_STATE_BEFORE_RETURN: bool = true;
+            fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+                state: &mut State<'a, Qd>,
+            ) -> QueryResult<Self> {
                 let mut l = Vec::new();
                 parse_list(state, &mut l);
                 Ok(List(l))
@@ -429,9 +437,13 @@ mod impls {
         #[derive(sky_macros::Wrapper, Debug)]
         pub struct DataTuple(Vec<Datacell>);
         impl<'a> ASTNode<'a> for DataTuple {
+            const MUST_USE_FULL_TOKEN_RANGE: bool = false;
+            const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
             // important: upstream must verify this
-            const VERIFY: bool = true;
-            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+            const VERIFY_STATE_BEFORE_RETURN: bool = true;
+            fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+                state: &mut State<'a, Qd>,
+            ) -> QueryResult<Self> {
                 let r = parse_data_tuple_syntax(state);
                 Ok(Self(r))
             }
@@ -439,9 +451,13 @@ mod impls {
         #[derive(sky_macros::Wrapper, Debug)]
         pub struct DataMap(HashMap<Box<str>, Datacell>);
         impl<'a> ASTNode<'a> for DataMap {
+            const MUST_USE_FULL_TOKEN_RANGE: bool = false;
+            const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
             // important: upstream must verify this
-            const VERIFY: bool = true;
-            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+            const VERIFY_STATE_BEFORE_RETURN: bool = true;
+            fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+                state: &mut State<'a, Qd>,
+            ) -> QueryResult<Self> {
                 let r = parse_data_map_syntax(state);
                 Ok(Self(
                     r.into_iter()
diff --git a/server/src/engine/ql/dml/mod.rs b/server/src/engine/ql/dml/mod.rs
index d8bb50d1..2c23a825 100644
--- a/server/src/engine/ql/dml/mod.rs
+++ b/server/src/engine/ql/dml/mod.rs
@@ -170,15 +170,23 @@ mod impls {
         },
     };
     impl<'a> ASTNode<'a> for WhereClause<'a> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = false;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
         // important: upstream must verify this
-        const VERIFY: bool = true;
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const VERIFY_STATE_BEFORE_RETURN: bool = true;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             let wh = Self::parse_where(state);
             Ok(wh)
         }
     }
     impl<'a> ASTNode<'a> for RelationalExpr<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = false;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             Self::try_parse(state).ok_or(QueryError::QLInvalidSyntax)
         }
     }
diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs
index 6cdc2209..a17d58d6 100644
--- a/server/src/engine/ql/dml/sel.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -161,7 +161,11 @@ mod impls {
         },
     };
     impl<'a> ASTNode<'a> for SelectStatement<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             Self::parse_select(state)
         }
     }
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index f2f4a225..9252ba06 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -219,7 +219,11 @@ mod impls {
         },
     };
     impl<'a> ASTNode<'a> for UpdateStatement<'a> {
-        fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
             Self::parse_update(state)
         }
     }
@@ -227,9 +231,13 @@ mod impls {
     mod test {
         use super::{super::AssignmentExpression, ASTNode, QueryData, QueryResult, State};
         impl<'a> ASTNode<'a> for AssignmentExpression<'a> {
+            const MUST_USE_FULL_TOKEN_RANGE: bool = false;
+            const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
             // important: upstream must verify this
-            const VERIFY: bool = true;
-            fn _from_state<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+            const VERIFY_STATE_BEFORE_RETURN: bool = true;
+            fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+                state: &mut State<'a, Qd>,
+            ) -> QueryResult<Self> {
                 let mut expr = Vec::new();
                 AssignmentExpression::parse_and_append_expression(state, &mut expr);
                 state.poison_if_not(expr.len() == 1);
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index d64cbafb..959116dd 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -135,6 +135,9 @@ impl<'a> Token<'a> {
     pub unsafe fn uck_read_lit(&self) -> &Lit<'a> {
         extract!(self, Self::Lit(l) => l)
     }
+    pub fn ident_eq(&self, ident: &str) -> bool {
+        matches!(self, Token::Ident(id) if id.eq_ignore_ascii_case(ident))
+    }
 }
 
 impl<'a> ToString for Token<'a> {
diff --git a/server/src/engine/ql/tests/dcl.rs b/server/src/engine/ql/tests/dcl.rs
index 35db503c..5e15170a 100644
--- a/server/src/engine/ql/tests/dcl.rs
+++ b/server/src/engine/ql/tests/dcl.rs
@@ -24,21 +24,35 @@
  *
 */
 
-use crate::engine::ql::{ast, dcl, tests::lex_insecure};
+use crate::engine::ql::{
+    ast,
+    dcl::{self, SysctlCommand},
+    tests::lex_insecure,
+};
+
+#[test]
+fn report_status_simple() {
+    let query = lex_insecure(b"sysctl status").unwrap();
+    let q = ast::parse_ast_node_full::<dcl::SysctlCommand>(&query[1..]).unwrap();
+    assert_eq!(q, SysctlCommand::ReportStatus)
+}
 
 #[test]
 fn create_user_simple() {
-    let query = lex_insecure(b"create user 'sayan' with { password: 'mypass123' }").unwrap();
-    let q = ast::parse_ast_node_full::<dcl::UserAdd>(&query[2..]).unwrap();
+    let query = lex_insecure(b"sysctl create user 'sayan' with { password: 'mypass123' }").unwrap();
+    let q = ast::parse_ast_node_full::<dcl::SysctlCommand>(&query[1..]).unwrap();
     assert_eq!(
         q,
-        dcl::UserAdd::new("sayan", into_dict!("password" => lit!("mypass123")))
+        SysctlCommand::CreateUser(dcl::UserAdd::new(
+            "sayan",
+            into_dict!("password" => lit!("mypass123"))
+        ))
     )
 }
 
 #[test]
 fn delete_user_simple() {
-    let query = lex_insecure(b"delete user 'monster'").unwrap();
-    let q = ast::parse_ast_node_full::<dcl::UserDel>(&query[2..]).unwrap();
-    assert_eq!(q, dcl::UserDel::new("monster"))
+    let query = lex_insecure(b"sysctl drop user 'monster'").unwrap();
+    let q = ast::parse_ast_node_full::<dcl::SysctlCommand>(&query[1..]).unwrap();
+    assert_eq!(q, SysctlCommand::DropUser(dcl::UserDel::new("monster")));
 }
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index d9927fe6..bb0e1acf 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -142,12 +142,12 @@ mod tymeta {
         //        ^^^^^^^^^^^^^^^^^^ cursor should be at string
         let tok = lex_insecure(br#"{maxlen: 100, type: string, unique: true }"#).unwrap();
         let mut state = State::new_inplace(&tok);
-        let tymeta: DictTypeMeta = ASTNode::from_state(&mut state).unwrap();
+        let tymeta: DictTypeMeta = ASTNode::test_parse_from_state(&mut state).unwrap();
         assert_eq!(state.cursor(), 6);
         assert!(Token![:].eq(state.fw_read()));
         assert!(Token::Ident(Ident::from("string")).eq(state.fw_read()));
         assert!(Token![,].eq(state.fw_read()));
-        let tymeta2: DictTypeMetaSplit = ASTNode::from_state(&mut state).unwrap();
+        let tymeta2: DictTypeMetaSplit = ASTNode::test_parse_from_state(&mut state).unwrap();
         assert!(state.exhausted());
         let mut final_ret = tymeta.into_inner();
         final_ret.extend(tymeta2.into_inner());
@@ -167,12 +167,12 @@ mod tymeta {
             lex_insecure(br#"{maxlen: 100, this: { is: "cool" }, type: string, unique: true }"#)
                 .unwrap();
         let mut state = State::new_inplace(&tok);
-        let tymeta: DictTypeMeta = ASTNode::from_state(&mut state).unwrap();
+        let tymeta: DictTypeMeta = ASTNode::test_parse_from_state(&mut state).unwrap();
         assert_eq!(state.cursor(), 14);
         assert!(Token![:].eq(state.fw_read()));
         assert!(Token::Ident(Ident::from("string")).eq(state.fw_read()));
         assert!(Token![,].eq(state.fw_read()));
-        let tymeta2: DictTypeMetaSplit = ASTNode::from_state(&mut state).unwrap();
+        let tymeta2: DictTypeMetaSplit = ASTNode::test_parse_from_state(&mut state).unwrap();
         assert!(state.exhausted());
         let mut final_ret = tymeta.into_inner();
         final_ret.extend(tymeta2.into_inner());

From 0fdd615665537bb0a0c3ac45a327a83f14138838 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Sun, 26 Nov 2023 20:12:35 +0530
Subject: [PATCH 293/310] Simplify entity handling, impl describe and use

---
 CHANGELOG.md                                  |   3 +
 Cargo.lock                                    |   3 +-
 cli/src/query.rs                              |  14 +-
 server/src/engine/core/dml/sel.rs             |   8 +-
 server/src/engine/core/exec.rs                | 131 ++++----
 server/src/engine/core/mod.rs                 |  28 +-
 server/src/engine/core/model/alt.rs           |   6 +-
 server/src/engine/core/model/mod.rs           |  73 +++-
 server/src/engine/core/tests/ddl_model/alt.rs |   8 +-
 server/src/engine/core/tests/ddl_model/crt.rs |  17 +-
 server/src/engine/core/tests/ddl_model/mod.rs |  12 +-
 server/src/engine/core/tests/dml/mod.rs       |   8 +-
 server/src/engine/core/util.rs                |   7 +
 server/src/engine/data/tag.rs                 |  49 +--
 server/src/engine/fractal/mgr.rs              |  10 +-
 server/src/engine/mem/mod.rs                  |   2 +-
 server/src/engine/net/protocol/mod.rs         |  59 +++-
 server/src/engine/ql/ast/mod.rs               | 313 +++++++-----------
 server/src/engine/ql/ast/traits.rs            |  18 +
 server/src/engine/ql/benches.rs               | 255 --------------
 server/src/engine/ql/dcl.rs                   |  23 +-
 server/src/engine/ql/ddl/alt.rs               |   9 +-
 server/src/engine/ql/ddl/crt.rs               |  16 +-
 server/src/engine/ql/ddl/drop.rs              |   9 +-
 server/src/engine/ql/ddl/ins.rs               |  91 -----
 server/src/engine/ql/ddl/mod.rs               |  35 +-
 server/src/engine/ql/dml/del.rs               |  16 +-
 server/src/engine/ql/dml/ins.rs               |  14 +-
 server/src/engine/ql/dml/sel.rs               |  16 +-
 server/src/engine/ql/dml/upd.rs               |  15 +-
 server/src/engine/ql/mod.rs                   |   3 -
 server/src/engine/ql/tests.rs                 |   2 +-
 server/src/engine/ql/tests/dcl.rs             |   2 +-
 server/src/engine/ql/tests/dml_tests.rs       |  48 +--
 .../engine/ql/tests/{entity.rs => misc.rs}    |  44 ++-
 server/src/engine/ql/tests/schema_tests.rs    | 123 +++----
 server/src/engine/storage/v1/inf/tests.rs     |   2 +-
 server/src/engine/storage/v1/tests/batch.rs   |   8 +-
 server/src/engine/txn/gns/tests/full_chain.rs |   2 +-
 server/src/engine/txn/gns/tests/io.rs         |   2 +-
 sky-bench/src/bench.rs                        |  22 +-
 sky-bench/src/runtime/fury.rs                 |   6 +-
 42 files changed, 641 insertions(+), 891 deletions(-)
 delete mode 100644 server/src/engine/ql/benches.rs
 delete mode 100644 server/src/engine/ql/ddl/ins.rs
 rename server/src/engine/ql/tests/{entity.rs => misc.rs} (59%)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0f9e9dc..35ea6ac1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,9 @@ All changes in this project will be noted in this file.
     - `ALTER MODEL ...`
     - `DROP SPACE ...`
     - `DROP MODEL ...`
+    - `USE <space>`:
+      - works just like SQL
+      - **does not work with DDL queries**: the reason it works in this way is to prevent accidental deletes
 - DML:
   - **All actions removed**: All the prior `SET`, `GET` and other actions have been removed in favor of the new query language
   - The following queries were added:
diff --git a/Cargo.lock b/Cargo.lock
index 1dfd1815..58bda0d7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1233,10 +1233,11 @@ dependencies = [
 [[package]]
 name = "skytable"
 version = "0.8.0"
-source = "git+https://github.com/skytable/client-rust.git?branch=octave#691e15578c1a09a5d2b6b85c8752e6eee31fbda2"
+source = "git+https://github.com/skytable/client-rust.git?branch=octave#62e4f3152d56127e406b9d5eeea60696acd77031"
 dependencies = [
  "async-trait",
  "bb8",
+ "itoa",
  "native-tls",
  "r2d2",
  "rand",
diff --git a/cli/src/query.rs b/cli/src/query.rs
index db272493..f4204131 100644
--- a/cli/src/query.rs
+++ b/cli/src/query.rs
@@ -57,13 +57,13 @@ enum Item {
 }
 
 impl SQParam for Item {
-    fn push(self, buf: &mut Vec<u8>) {
+    fn append_param(self, buf: &mut Vec<u8>) {
         match self {
-            Item::UInt(u) => u.push(buf),
-            Item::SInt(s) => s.push(buf),
-            Item::Float(f) => f.push(buf),
-            Item::String(s) => s.push(buf),
-            Item::Bin(b) => SQParam::push(&*b, buf),
+            Item::UInt(u) => u.append_param(buf),
+            Item::SInt(s) => s.append_param(buf),
+            Item::Float(f) => f.append_param(buf),
+            Item::String(s) => s.append_param(buf),
+            Item::Bin(b) => SQParam::append_param(&*b, buf),
         }
     }
 }
@@ -100,7 +100,7 @@ impl Parameterizer {
                 }
                 sym => {
                     self.i += 1;
-                    self.query.push(sym);
+                    Vec::push(&mut self.query, sym);
                     Ok(())
                 }
             }?
diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
index bb586677..7199f404 100644
--- a/server/src/engine/core/dml/sel.rs
+++ b/server/src/engine/core/dml/sel.rs
@@ -34,7 +34,7 @@ use crate::engine::{
     fractal::GlobalInstanceLike,
     idx::{STIndex, STIndexSeq},
     mem::IntegerRepr,
-    net::protocol::Response,
+    net::protocol::{Response, ResponseType},
     ql::dml::sel::SelectStatement,
     sync,
 };
@@ -49,7 +49,11 @@ pub fn select_resp(
         encode_cell(&mut data, item);
         i += 1;
     })?;
-    Ok(Response::Row { size: i, data })
+    Ok(Response::Serialized {
+        ty: ResponseType::Row,
+        size: i,
+        data,
+    })
 }
 
 fn encode_cell(resp: &mut Vec<u8>, item: &Datacell) {
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 1d10113b..0ad1c5ca 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -27,29 +27,34 @@
 use crate::engine::{
     core::{dml, model::Model, space::Space},
     error::{QueryError, QueryResult},
-    fractal::Global,
-    mem::RawSlice,
+    fractal::{Global, GlobalInstanceLike},
     net::protocol::{ClientLocalState, Response, SQuery},
     ql::{
         ast::{traits::ASTNode, InplaceData, State},
-        lex::{Keyword, KeywordStmt, Token},
+        ddl::Use,
+        lex::KeywordStmt,
     },
 };
 
+/*
+    ---
+    trigger warning: disgusting hacks below owing to token lifetimes
+*/
+
 pub async fn dispatch_to_executor<'a>(
     global: &Global,
-    cstate: &ClientLocalState,
+    cstate: &mut ClientLocalState,
     query: SQuery<'a>,
 ) -> QueryResult<Response> {
     let tokens =
         crate::engine::ql::lex::SecureLexer::new_with_segments(query.query(), query.params())
             .lex()?;
     let mut state = State::new_inplace(&tokens);
-    let stmt = match state.read() {
-        Token::Keyword(Keyword::Statement(stmt)) if state.remaining() >= 3 => *stmt,
-        _ => return Err(QueryError::QLExpectedStatement),
-    };
-    state.cursor_ahead();
+    state.set_space_maybe(unsafe {
+        // UNSAFE(@ohsayan): exclusively used within this scope
+        core::mem::transmute(cstate.get_cs())
+    });
+    let stmt = state.try_statement()?;
     if stmt.is_blocking() {
         run_blocking_stmt(global, cstate, state, stmt).await
     } else {
@@ -57,25 +62,6 @@ pub async fn dispatch_to_executor<'a>(
     }
 }
 
-/*
-    blocking exec
-    ---
-    trigger warning: disgusting hacks below (why can't async play nice with lifetimes :|)
-*/
-
-#[inline(always)]
-fn call<A: ASTNode<'static> + core::fmt::Debug, T>(
-    g: Global,
-    tokens: RawSlice<Token<'static>>,
-    f: impl FnOnce(&Global, A) -> QueryResult<T>,
-) -> QueryResult<T> {
-    let mut state = State::new_inplace(unsafe {
-        // UNSAFE(@ohsayan): nothing to drop. all cool
-        core::mem::transmute(tokens)
-    });
-    _call(&g, &mut state, f)
-}
-
 #[inline(always)]
 fn _call<A: ASTNode<'static> + core::fmt::Debug, T>(
     g: &Global,
@@ -88,7 +74,7 @@ fn _call<A: ASTNode<'static> + core::fmt::Debug, T>(
 
 async fn run_blocking_stmt(
     global: &Global,
-    cstate: &ClientLocalState,
+    cstate: &mut ClientLocalState,
     mut state: State<'_, InplaceData>,
     stmt: KeywordStmt,
 ) -> Result<Response, QueryError> {
@@ -96,6 +82,12 @@ async fn run_blocking_stmt(
         // all the actions here need root permission
         return Err(QueryError::SysPermissionDenied);
     }
+    state.ensure_minimum_for_blocking_stmt()?;
+    /*
+        IMPORTANT: DDL queries will NOT pick up the currently set space. instead EVERY DDL query must manually fully specify the entity that
+        they want to manipulate. this prevents a whole set of exciting errors like dropping a model with the same model name from another space
+    */
+    state.unset_space();
     let (a, b) = (&state.current()[0], &state.current()[1]);
     let sysctl = stmt == KeywordStmt::Sysctl;
     let create = stmt == KeywordStmt::Create;
@@ -110,26 +102,28 @@ async fn run_blocking_stmt(
     let d_m = (drop & Token![model].eq(a) & last_id) as u8 * 7;
     let fc = sysctl as u8 | c_s | c_m | a_s | a_m | d_s | d_m;
     state.cursor_ahead_if(!sysctl);
-    static BLK_EXEC: [fn(Global, &ClientLocalState, RawSlice<Token<'static>>) -> QueryResult<()>;
-        8] = [
+    static BLK_EXEC: [fn(
+        Global,
+        &ClientLocalState,
+        &mut State<'static, InplaceData>,
+    ) -> QueryResult<()>; 8] = [
         |_, _, _| Err(QueryError::QLUnknownStatement), // unknown
         blocking_exec_sysctl,                          // sysctl
-        |g, _, t| call(g, t, Space::transactional_exec_create),
-        |g, _, t| call(g, t, Model::transactional_exec_create),
-        |g, _, t| call(g, t, Space::transactional_exec_alter),
-        |g, _, t| call(g, t, Model::transactional_exec_alter),
-        |g, _, t| call(g, t, Space::transactional_exec_drop),
-        |g, _, t| call(g, t, Model::transactional_exec_drop),
+        |g, _, t| _call(&g, t, Space::transactional_exec_create),
+        |g, _, t| _call(&g, t, Model::transactional_exec_create),
+        |g, _, t| _call(&g, t, Space::transactional_exec_alter),
+        |g, _, t| _call(&g, t, Model::transactional_exec_alter),
+        |g, _, t| _call(&g, t, Space::transactional_exec_drop),
+        |g, _, t| _call(&g, t, Model::transactional_exec_drop),
     ];
     let r = unsafe {
         // UNSAFE(@ohsayan): the only await is within this block
         let c_glob = global.clone();
-        let ptr = state.current().as_ptr() as usize;
-        let len = state.current().len();
-        let cstate: &'static ClientLocalState = core::mem::transmute(cstate);
+        let static_cstate: &'static ClientLocalState = core::mem::transmute(cstate);
+        let static_state: &'static mut State<'static, InplaceData> =
+            core::mem::transmute(&mut state);
         tokio::task::spawn_blocking(move || {
-            let tokens = RawSlice::new(ptr as *const Token, len);
-            BLK_EXEC[fc as usize](c_glob, cstate, tokens)?;
+            BLK_EXEC[fc as usize](c_glob, static_cstate, static_state)?;
             Ok(Response::Empty)
         })
         .await
@@ -140,10 +134,9 @@ async fn run_blocking_stmt(
 fn blocking_exec_sysctl(
     g: Global,
     cstate: &ClientLocalState,
-    tokens: RawSlice<Token<'static>>,
+    state: &mut State<'static, InplaceData>,
 ) -> QueryResult<()> {
-    let mut state = State::new_inplace(&tokens);
-    let r = ASTNode::parse_from_state_hardened(&mut state)?;
+    let r = ASTNode::parse_from_state_hardened(state)?;
     super::dcl::exec(g, cstate, r)
 }
 
@@ -151,28 +144,54 @@ fn blocking_exec_sysctl(
     nb exec
 */
 
+fn cstate_use(
+    global: &Global,
+    cstate: &mut ClientLocalState,
+    state: &mut State<'static, InplaceData>,
+) -> QueryResult<Response> {
+    let use_c = Use::parse_from_state_hardened(state)?;
+    match use_c {
+        Use::Null => cstate.unset_cs(),
+        Use::Space(new_space) => {
+            /*
+                NB: just like SQL, we don't really care about what this is set to as it's basically a shorthand.
+                so we do a simple vanity check
+            */
+            if !global.namespace().contains_space(new_space.as_str()) {
+                return Err(QueryError::QExecObjectNotFound);
+            }
+            cstate.set_cs(new_space.boxed_str());
+        }
+    }
+    Ok(Response::Empty)
+}
+
 fn run_nb(
     global: &Global,
-    _cstate: &ClientLocalState,
+    cstate: &mut ClientLocalState,
     state: State<'_, InplaceData>,
     stmt: KeywordStmt,
 ) -> QueryResult<Response> {
     let stmt = stmt.value_u8() - KeywordStmt::Use.value_u8();
-    static F: [fn(&Global, &mut State<'static, InplaceData>) -> QueryResult<Response>; 8] = [
-        |_, _| Err(QueryError::QLUnknownStatement), // use
-        |_, _| Err(QueryError::QLUnknownStatement), // inspect
-        |_, _| Err(QueryError::QLUnknownStatement), // describe
-        |g, s| _call(g, s, dml::insert_resp),
-        |g, s| _call(g, s, dml::select_resp),
-        |g, s| _call(g, s, dml::update_resp),
-        |g, s| _call(g, s, dml::delete_resp),
-        |_, _| Err(QueryError::QLUnknownStatement), // exists
+    static F: [fn(
+        &Global,
+        &mut ClientLocalState,
+        &mut State<'static, InplaceData>,
+    ) -> QueryResult<Response>; 8] = [
+        cstate_use,                                    // use
+        |_, _, _| Err(QueryError::QLUnknownStatement), // inspect
+        |_, _, _| Err(QueryError::QLUnknownStatement), // describe
+        |g, _, s| _call(g, s, dml::insert_resp),
+        |g, _, s| _call(g, s, dml::select_resp),
+        |g, _, s| _call(g, s, dml::update_resp),
+        |g, _, s| _call(g, s, dml::delete_resp),
+        |_, _, _| Err(QueryError::QLUnknownStatement), // exists
     ];
     {
         let mut state = unsafe {
             // UNSAFE(@ohsayan): this is a lifetime issue with the token handle
             core::mem::transmute(state)
         };
-        F[stmt as usize](global, &mut state)
+        F[stmt as usize](global, cstate, &mut state)
     }
 }
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 69bfd829..8e416efc 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -41,7 +41,7 @@ pub use self::util::{EntityID, EntityIDRef};
 // imports
 use {
     self::{dml::QueryExecMeta, model::Model},
-    super::{fractal::GlobalInstanceLike, ql::ast::Entity},
+    super::fractal::GlobalInstanceLike,
     crate::engine::{
         core::space::Space,
         error::{QueryError, QueryResult},
@@ -87,26 +87,28 @@ impl GlobalNS {
         let mut space = space.write();
         f(&mut space)
     }
-    pub fn with_model_space_mut_for_ddl<'a, T, F>(&self, entity: Entity<'a>, f: F) -> QueryResult<T>
+    pub fn with_model_space_mut_for_ddl<'a, T, F>(
+        &self,
+        entity: EntityIDRef<'a>,
+        f: F,
+    ) -> QueryResult<T>
     where
         F: FnOnce(&Space, &mut Model) -> QueryResult<T>,
     {
-        let (space, model_name) = entity.into_full_result()?;
         let mut mdl_idx = self.idx_mdl.write();
-        let Some(model) = mdl_idx.get_mut(&EntityIDRef::new(&space, &model_name)) else {
+        let Some(model) = mdl_idx.get_mut(&entity) else {
             return Err(QueryError::QExecObjectNotFound);
         };
         let space_read = self.idx.read();
-        let space = space_read.get(space.as_str()).unwrap().read();
+        let space = space_read.get(entity.space()).unwrap().read();
         f(&space, model)
     }
-    pub fn with_model<'a, T, F>(&self, entity: Entity<'a>, f: F) -> QueryResult<T>
+    pub fn with_model<'a, T, F>(&self, entity: EntityIDRef<'a>, f: F) -> QueryResult<T>
     where
         F: FnOnce(&Model) -> QueryResult<T>,
     {
-        let (space, model_name) = entity.into_full_result()?;
         let mdl_idx = self.idx_mdl.read();
-        let Some(model) = mdl_idx.get(&EntityIDRef::new(&space, &model_name)) else {
+        let Some(model) = mdl_idx.get(&entity) else {
             return Err(QueryError::QExecObjectNotFound);
         };
         f(model)
@@ -124,22 +126,24 @@ impl GlobalNS {
             .write()
             .insert(space_name.into(), Space::new_auto_all().into());
     }
+    pub fn contains_space(&self, name: &str) -> bool {
+        self.idx.read().contains_key(name)
+    }
 }
 
 pub(self) fn with_model_for_data_update<'a, F>(
     global: &impl GlobalInstanceLike,
-    entity: Entity<'a>,
+    entity: EntityIDRef<'a>,
     f: F,
 ) -> QueryResult<()>
 where
     F: FnOnce(&Model) -> QueryResult<QueryExecMeta>,
 {
-    let (space, model_name) = entity.into_full_result()?;
     let mdl_idx = global.namespace().idx_mdl.read();
-    let Some(model) = mdl_idx.get(&EntityIDRef::new(&space, &model_name)) else {
+    let Some(model) = mdl_idx.get(&entity) else {
         return Err(QueryError::QExecObjectNotFound);
     };
     let r = f(model)?;
-    model::DeltaState::guard_delta_overflow(global, &space, &model_name, model, r);
+    model::DeltaState::guard_delta_overflow(global, entity.space(), entity.entity(), model, r);
     Ok(())
 }
diff --git a/server/src/engine/core/model/alt.rs b/server/src/engine/core/model/alt.rs
index d9b7a51d..3a8dbabc 100644
--- a/server/src/engine/core/model/alt.rs
+++ b/server/src/engine/core/model/alt.rs
@@ -28,6 +28,7 @@ use {
     super::{Field, Layer, Model},
     crate::{
         engine::{
+            core::EntityIDRef,
             data::{
                 tag::{DataTag, TagClass},
                 DictEntryGeneric,
@@ -36,7 +37,6 @@ use {
             fractal::GlobalInstanceLike,
             idx::{IndexST, IndexSTSeqCns, STIndex, STIndexSeq},
             ql::{
-                ast::Entity,
                 ddl::{
                     alt::{AlterKind, AlterModel},
                     syn::{ExpandedField, LayerSpec},
@@ -52,7 +52,7 @@ use {
 
 #[derive(Debug, PartialEq)]
 pub(in crate::engine::core) struct AlterPlan<'a> {
-    pub(in crate::engine::core) model: Entity<'a>,
+    pub(in crate::engine::core) model: EntityIDRef<'a>,
     pub(in crate::engine::core) no_lock: bool,
     pub(in crate::engine::core) action: AlterAction<'a>,
 }
@@ -250,7 +250,7 @@ impl Model {
         global: &G,
         alter: AlterModel,
     ) -> QueryResult<()> {
-        let (space_name, model_name) = alter.model.into_full_result()?;
+        let (space_name, model_name) = (alter.model.space(), alter.model.entity());
         global
             .namespace()
             .with_model_space_mut_for_ddl(alter.model, |space, model| {
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 13f39cb6..6c67b55b 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -66,6 +66,7 @@ pub struct Model {
     data: PrimaryIndex,
     delta: DeltaState,
     private: ModelPrivate,
+    decl: String,
 }
 
 #[cfg(test)]
@@ -113,6 +114,49 @@ impl Model {
     pub fn model_mutator<'a>(&'a mut self) -> ModelMutator<'a> {
         ModelMutator { model: self }
     }
+    fn sync_decl(&mut self) {
+        self.decl = self.redescribe();
+    }
+    pub fn describe(&self) -> &str {
+        &self.decl
+    }
+    fn redescribe(&self) -> String {
+        let mut ret = format!("{{ ");
+        let mut it = self.fields().stseq_ord_kv().peekable();
+        while let Some((field_name, field_decl)) = it.next() {
+            // legend: * -> primary, ! -> not null, ? -> null
+            if self.is_pk(&field_name) {
+                ret.push('*');
+            } else if field_decl.is_nullable() {
+                ret.push('?');
+            } else {
+                ret.push('!');
+            }
+            ret.push_str(&field_name);
+            ret.push(':');
+            ret.push(' ');
+            // TODO(@ohsayan): it's all lists right now, so this is okay but fix it later
+            if field_decl.layers().len() == 1 {
+                ret.push_str(field_decl.layers()[0].tag().tag_selector().name_str());
+            } else {
+                ret.push_str(&"[".repeat(field_decl.layers().len() - 1));
+                ret.push_str(
+                    field_decl.layers()[field_decl.layers().len() - 1]
+                        .tag()
+                        .tag_selector()
+                        .name_str(),
+                );
+                ret.push_str(&"]".repeat(field_decl.layers().len() - 1))
+            }
+            if it.peek().is_some() {
+                ret.push(',');
+                ret.push(' ');
+            }
+        }
+        ret.push(' ');
+        ret.push('}');
+        ret
+    }
 }
 
 impl Model {
@@ -123,7 +167,7 @@ impl Model {
         fields: Fields,
         private: ModelPrivate,
     ) -> Self {
-        Self {
+        let mut slf = Self {
             uuid,
             p_key,
             p_tag,
@@ -131,7 +175,10 @@ impl Model {
             data: PrimaryIndex::new_empty(),
             delta: DeltaState::new_resolved(),
             private,
-        }
+            decl: String::new(),
+        };
+        slf.sync_decl();
+        slf
     }
     pub fn new_restore(
         uuid: Uuid,
@@ -223,11 +270,11 @@ impl Model {
         global: &G,
         stmt: CreateModel,
     ) -> QueryResult<()> {
-        let (space_name, model_name) = stmt.model_name.into_full_result()?;
+        let (space_name, model_name) = (stmt.model_name.space(), stmt.model_name.entity());
         let model = Self::process_create(stmt)?;
         global.namespace().ddl_with_space_mut(&space_name, |space| {
             // TODO(@ohsayan): be extra cautious with post-transactional tasks (memck)
-            if space.models().contains(model_name.as_str()) {
+            if space.models().contains(model_name) {
                 return Err(QueryError::QExecDdlObjectAlreadyExists);
             }
             // since we've locked this down, no one else can parallely create another model in the same space (or remove)
@@ -264,7 +311,7 @@ impl Model {
                 }
             }
             // update global state
-            let _ = space.models_mut().insert(model_name.as_str().into());
+            let _ = space.models_mut().insert(model_name.into());
             let _ = global
                 .namespace()
                 .idx_models()
@@ -277,9 +324,9 @@ impl Model {
         global: &G,
         stmt: DropModel,
     ) -> QueryResult<()> {
-        let (space_name, model_name) = stmt.entity.into_full_result()?;
+        let (space_name, model_name) = (stmt.entity.space(), stmt.entity.entity());
         global.namespace().ddl_with_space_mut(&space_name, |space| {
-            if !space.models().contains(model_name.as_str()) {
+            if !space.models().contains(model_name) {
                 // the model isn't even present
                 return Err(QueryError::QExecObjectNotFound);
             }
@@ -306,15 +353,15 @@ impl Model {
                 global.namespace_txn_driver().lock().try_commit(txn)?;
                 // request cleanup
                 global.purge_model_driver(
-                    space_name.as_str(),
+                    space_name,
                     space.get_uuid(),
-                    model_name.as_str(),
+                    model_name,
                     model.get_uuid(),
                 );
             }
             // update global state
             let _ = models_idx.remove(&EntityIDRef::new(&space_name, &model_name));
-            let _ = space.models_mut().remove(model_name.as_str());
+            let _ = space.models_mut().remove(model_name);
             Ok(())
         })
     }
@@ -406,6 +453,12 @@ impl<'a> ModelMutator<'a> {
     }
 }
 
+impl<'a> Drop for ModelMutator<'a> {
+    fn drop(&mut self) {
+        self.model.sync_decl();
+    }
+}
+
 /*
     Layer
 */
diff --git a/server/src/engine/core/tests/ddl_model/alt.rs b/server/src/engine/core/tests/ddl_model/alt.rs
index fc8b3c0c..d154cabc 100644
--- a/server/src/engine/core/tests/ddl_model/alt.rs
+++ b/server/src/engine/core/tests/ddl_model/alt.rs
@@ -89,7 +89,7 @@ mod plan {
             "create model myspace.mymodel(username: string, password: binary)",
             "alter model myspace.mymodel add myfield { type: string, nullable: true }",
             |plan| {
-                assert_eq!(plan.model.into_full().unwrap().1.as_str(), "mymodel");
+                assert_eq!(plan.model.entity(), "mymodel");
                 assert!(plan.no_lock);
                 assert_eq!(
                     plan.action,
@@ -106,7 +106,7 @@ mod plan {
             "create model myspace.mymodel(username: string, password: binary, useless_field: uint8)",
             "alter model myspace.mymodel remove useless_field",
             |plan| {
-                assert_eq!(plan.model.into_full().unwrap().1.as_str(), "mymodel");
+                assert_eq!(plan.model.entity(), "mymodel");
                 assert!(plan.no_lock);
                 assert_eq!(
                     plan.action,
@@ -122,7 +122,7 @@ mod plan {
             "create model myspace.mymodel(username: string, password: binary)",
             "alter model myspace.mymodel update password { nullable: true }",
             |plan| {
-                assert_eq!(plan.model.into_full().unwrap().1.as_str(), "mymodel");
+                assert_eq!(plan.model.entity(), "mymodel");
                 assert!(plan.no_lock);
                 assert_eq!(
                     plan.action,
@@ -140,7 +140,7 @@ mod plan {
             "create model myspace.mymodel(username: string, null password: binary)",
             "alter model myspace.mymodel update password { nullable: false }",
             |plan| {
-                assert_eq!(plan.model.into_full().unwrap().1.as_str(), "mymodel");
+                assert_eq!(plan.model.entity(), "mymodel");
                 assert!(!plan.no_lock);
                 assert_eq!(
                     plan.action,
diff --git a/server/src/engine/core/tests/ddl_model/crt.rs b/server/src/engine/core/tests/ddl_model/crt.rs
index 909582c2..d519a365 100644
--- a/server/src/engine/core/tests/ddl_model/crt.rs
+++ b/server/src/engine/core/tests/ddl_model/crt.rs
@@ -37,7 +37,8 @@ mod validation {
 
     #[test]
     fn simple() {
-        let model = create("create model mymodel(username: string, password: binary)").unwrap();
+        let model =
+            create("create model myspace.mymodel(username: string, password: binary)").unwrap();
         assert_eq!(model.p_key(), "username");
         assert_eq!(model.p_tag(), FullTag::STR);
         assert_eq!(
@@ -60,7 +61,8 @@ mod validation {
     #[test]
     fn idiotic_order() {
         let model =
-            create("create model mymodel(password: binary, primary username: string)").unwrap();
+            create("create model myspace.mymodel(password: binary, primary username: string)")
+                .unwrap();
         assert_eq!(model.p_key(), "username");
         assert_eq!(model.p_tag(), FullTag::STR);
         assert_eq!(
@@ -84,7 +86,7 @@ mod validation {
     fn duplicate_primary_key() {
         assert_eq!(
             create(
-                "create model mymodel(primary username: string, primary contract_location: binary)"
+                "create model myspace.mymodel(primary username: string, primary contract_location: binary)"
             )
             .unwrap_err(),
             QueryError::QExecDdlModelBadDefinition
@@ -94,7 +96,8 @@ mod validation {
     #[test]
     fn duplicate_fields() {
         assert_eq!(
-            create("create model mymodel(primary username: string, username: binary)").unwrap_err(),
+            create("create model myspace.mymodel(primary username: string, username: binary)")
+                .unwrap_err(),
             QueryError::QExecDdlModelBadDefinition
         );
     }
@@ -102,7 +105,7 @@ mod validation {
     #[test]
     fn illegal_props() {
         assert_eq!(
-        create("create model mymodel(primary username: string, password: binary) with { lol_prop: false }").unwrap_err(),
+        create("create model myspace.mymodel(primary username: string, password: binary) with { lol_prop: false }").unwrap_err(),
         QueryError::QExecDdlModelBadDefinition
     );
     }
@@ -111,13 +114,13 @@ mod validation {
     fn illegal_pk() {
         assert_eq!(
         create(
-            "create model mymodel(primary username_bytes: list { type: uint8 }, password: binary)"
+            "create model myspace.mymodel(primary username_bytes: list { type: uint8 }, password: binary)"
         )
         .unwrap_err(),
         QueryError::QExecDdlModelBadDefinition
     );
         assert_eq!(
-            create("create model mymodel(primary username: float32, password: binary)")
+            create("create model myspace.mymodel(primary username: float32, password: binary)")
                 .unwrap_err(),
             QueryError::QExecDdlModelBadDefinition
         );
diff --git a/server/src/engine/core/tests/ddl_model/mod.rs b/server/src/engine/core/tests/ddl_model/mod.rs
index 6d4352f6..b58cdc5e 100644
--- a/server/src/engine/core/tests/ddl_model/mod.rs
+++ b/server/src/engine/core/tests/ddl_model/mod.rs
@@ -32,11 +32,7 @@ use crate::engine::{
     core::{model::Model, EntityIDRef},
     error::QueryResult,
     fractal::GlobalInstanceLike,
-    ql::{
-        ast::{parse_ast_node_full, Entity},
-        ddl::crt::CreateModel,
-        tests::lex_insecure,
-    },
+    ql::{ast::parse_ast_node_full, ddl::crt::CreateModel, tests::lex_insecure},
 };
 
 fn create(s: &str) -> QueryResult<Model> {
@@ -52,13 +48,11 @@ pub fn exec_create(
 ) -> QueryResult<String> {
     let tok = lex_insecure(create_stmt.as_bytes()).unwrap();
     let create_model = parse_ast_node_full::<CreateModel>(&tok[2..]).unwrap();
-    let name = match create_model.model_name {
-        Entity::Single(tbl) | Entity::Full(_, tbl) => tbl.to_string(),
-    };
+    let name = create_model.model_name.entity().to_owned();
     if create_new_space {
         global
             .namespace()
-            .create_empty_test_space(&create_model.model_name.into_full().unwrap().0)
+            .create_empty_test_space(create_model.model_name.space())
     }
     Model::transactional_exec_create(global, create_model).map(|_| name)
 }
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index 41dd7854..4418a943 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -30,12 +30,12 @@ mod select;
 mod update;
 
 use crate::engine::{
-    core::{dml, index::Row, model::Model, space::Space},
+    core::{dml, index::Row, model::Model, space::Space, EntityIDRef},
     data::{cell::Datacell, lit::Lit},
     error::QueryResult,
     fractal::GlobalInstanceLike,
     ql::{
-        ast::{parse_ast_node_full, Entity},
+        ast::parse_ast_node_full,
         dml::{del::DeleteStatement, ins::InsertStatement},
         tests::lex_insecure,
     },
@@ -56,7 +56,7 @@ fn _exec_only_create_space_model(global: &impl GlobalInstanceLike, model: &str)
 fn _exec_only_insert<T>(
     global: &impl GlobalInstanceLike,
     insert: &str,
-    and_then: impl Fn(Entity) -> T,
+    and_then: impl Fn(EntityIDRef) -> T,
 ) -> QueryResult<T> {
     let lex_insert = lex_insecure(insert.as_bytes()).unwrap();
     let stmt_insert = parse_ast_node_full::<InsertStatement>(&lex_insert[1..]).unwrap();
@@ -68,7 +68,7 @@ fn _exec_only_insert<T>(
 
 fn _exec_only_read_key_and_then<T>(
     global: &impl GlobalInstanceLike,
-    entity: Entity,
+    entity: EntityIDRef,
     key_name: &str,
     and_then: impl Fn(Row) -> T,
 ) -> QueryResult<T> {
diff --git a/server/src/engine/core/util.rs b/server/src/engine/core/util.rs
index fb5b7021..02afa907 100644
--- a/server/src/engine/core/util.rs
+++ b/server/src/engine/core/util.rs
@@ -93,6 +93,7 @@ impl fmt::Debug for EntityID {
     }
 }
 
+#[derive(Clone, Copy)]
 pub struct EntityIDRef<'a> {
     sp: *const u8,
     sl: usize,
@@ -148,3 +149,9 @@ impl<'a> Borrow<EntityIDRef<'a>> for EntityID {
         unsafe { core::mem::transmute(self) }
     }
 }
+
+impl From<(&'static str, &'static str)> for EntityIDRef<'static> {
+    fn from((s, e): (&'static str, &'static str)) -> Self {
+        Self::new(s, e)
+    }
+}
diff --git a/server/src/engine/data/tag.rs b/server/src/engine/data/tag.rs
index 68b66d50..62e6f73a 100644
--- a/server/src/engine/data/tag.rs
+++ b/server/src/engine/data/tag.rs
@@ -24,6 +24,15 @@
  *
 */
 
+macro_rules! strid {
+    ($(#[$attr:meta])*$vis:vis enum $enum:ident {$($(#[$var_attr:meta])* $variant:ident $(= $dscr:expr)?),* $(,)?}) => {
+        $(#[$attr])* $vis enum $enum { $($(#[$var_attr])* $variant $(= $dscr)?),*}
+        impl $enum {
+            pub const fn name_str(&self) -> &'static str { match self { $(Self::$variant => stringify!($variant),)* } }
+        }
+    }
+}
+
 #[repr(u8)]
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, sky_macros::EnumMethods)]
 pub enum TagClass {
@@ -36,23 +45,25 @@ pub enum TagClass {
     List = 6,
 }
 
-#[repr(u8)]
-#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, sky_macros::EnumMethods)]
-pub enum TagSelector {
-    Bool = 0,
-    UInt8 = 1,
-    UInt16 = 2,
-    UInt32 = 3,
-    UInt64 = 4,
-    SInt8 = 5,
-    SInt16 = 6,
-    SInt32 = 7,
-    SInt64 = 8,
-    Float32 = 9,
-    Float64 = 10,
-    Bin = 11,
-    Str = 12,
-    List = 13,
+strid! {
+    #[repr(u8)]
+    #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, sky_macros::EnumMethods)]
+    pub enum TagSelector {
+        Bool = 0,
+        UInt8 = 1,
+        UInt16 = 2,
+        UInt32 = 3,
+        UInt64 = 4,
+        SInt8 = 5,
+        SInt16 = 6,
+        SInt32 = 7,
+        SInt64 = 8,
+        Float32 = 9,
+        Float64 = 10,
+        Binary = 11,
+        String = 12,
+        List = 13,
+    }
 }
 
 impl TagSelector {
@@ -175,8 +186,8 @@ impl DataTag for FullTag {
     const UINT: Self = fulltag!(UnsignedInt, UInt64, UnsignedInt);
     const SINT: Self = fulltag!(SignedInt, SInt64, SignedInt);
     const FLOAT: Self = fulltag!(Float, Float64);
-    const BIN: Self = fulltag!(Bin, Bin, Bin);
-    const STR: Self = fulltag!(Str, Str, Str);
+    const BIN: Self = fulltag!(Bin, Binary, Bin);
+    const STR: Self = fulltag!(Str, String, Str);
     const LIST: Self = fulltag!(List, List);
     fn tag_class(&self) -> TagClass {
         self.class
diff --git a/server/src/engine/fractal/mgr.rs b/server/src/engine/fractal/mgr.rs
index 3440dbf7..ea9332ab 100644
--- a/server/src/engine/fractal/mgr.rs
+++ b/server/src/engine/fractal/mgr.rs
@@ -28,9 +28,11 @@ use {
     super::ModelUniqueID,
     crate::{
         engine::{
-            core::model::{delta::DataDelta, Model},
+            core::{
+                model::{delta::DataDelta, Model},
+                EntityIDRef,
+            },
             data::uuid::Uuid,
-            ql::ast::Entity,
             storage::v1::LocalFS,
         },
         util::os,
@@ -290,7 +292,7 @@ impl FractalMgr {
                     return;
                 };
                 let res = global._namespace().with_model(
-                    Entity::Full(model_id.space().into(), model_id.model().into()),
+                    EntityIDRef::new(model_id.space().into(), model_id.model().into()),
                     |model| {
                         if model.get_uuid() != model_id.uuid() {
                             // once again, throughput maximization will lead to, in extremely rare cases, this
@@ -382,7 +384,7 @@ impl FractalMgr {
         for (model_id, driver) in mdl_drivers.iter() {
             let mut observed_len = 0;
             let res = global._namespace().with_model(
-                Entity::Full(model_id.space().into(), model_id.model().into()),
+                EntityIDRef::new(model_id.space().into(), model_id.model().into()),
                 |model| {
                     if model.get_uuid() != model_id.uuid() {
                         // once again, throughput maximization will lead to, in extremely rare cases, this
diff --git a/server/src/engine/mem/mod.rs b/server/src/engine/mem/mod.rs
index a823287c..46ce25c2 100644
--- a/server/src/engine/mem/mod.rs
+++ b/server/src/engine/mem/mod.rs
@@ -41,7 +41,7 @@ pub use {
     astr::AStr,
     ll::CachePadded,
     numbuf::IntegerRepr,
-    rawslice::{RawSlice, RawStr},
+    rawslice::RawStr,
     scanner::BufferedScanner,
     uarray::UArray,
     vinline::VInline,
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index da804566..cc2c5eec 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -51,16 +51,42 @@ use {
     tokio::io::{AsyncReadExt, AsyncWriteExt, BufWriter},
 };
 
+#[repr(u8)]
+#[derive(sky_macros::EnumMethods, Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[allow(unused)]
+pub enum ResponseType {
+    Null = 0x00,
+    Bool = 0x01,
+    UInt8 = 0x02,
+    UInt16 = 0x03,
+    UInt32 = 0x04,
+    UInt64 = 0x05,
+    SInt8 = 0x06,
+    SInt16 = 0x07,
+    SInt32 = 0x08,
+    SInt64 = 0x09,
+    Float32 = 0x0A,
+    Float64 = 0x0B,
+    Binary = 0x0C,
+    String = 0x0D,
+    List = 0x0E,
+    Dict = 0x0F,
+    Error = 0x10,
+    Row = 0x11,
+    Empty = 0x12,
+}
+
 #[derive(Debug, PartialEq)]
 pub struct ClientLocalState {
     username: Box<str>,
     root: bool,
     hs: handshake::CHandshakeStatic,
+    cs: Option<Box<str>>,
 }
 
 impl ClientLocalState {
     pub fn new(username: Box<str>, root: bool, hs: handshake::CHandshakeStatic) -> Self {
-        Self { username, root, hs }
+        Self { username, root, hs, cs: None }
     }
     pub fn is_root(&self) -> bool {
         self.root
@@ -68,12 +94,25 @@ impl ClientLocalState {
     pub fn username(&self) -> &str {
         &self.username
     }
+    pub fn set_cs(&mut self, new: Box<str>) {
+        self.cs = Some(new);
+    }
+    pub fn unset_cs(&mut self) {
+        self.cs = None;
+    }
+    pub fn get_cs(&self) -> Option<&str> {
+        self.cs.as_deref()
+    }
 }
 
 #[derive(Debug, PartialEq)]
 pub enum Response {
     Empty,
-    Row { size: usize, data: Vec<u8> },
+    Serialized {
+        ty: ResponseType,
+        size: usize,
+        data: Vec<u8>,
+    },
 }
 
 pub(super) async fn query_loop<S: Socket>(
@@ -82,7 +121,7 @@ pub(super) async fn query_loop<S: Socket>(
     global: &Global,
 ) -> IoResult<QueryLoopResult> {
     // handshake
-    let client_state = match do_handshake(con, buf, global).await? {
+    let mut client_state = match do_handshake(con, buf, global).await? {
         PostHandshake::Okay(hs) => hs,
         PostHandshake::ConnectionClosedFin => return Ok(QueryLoopResult::Fin),
         PostHandshake::ConnectionClosedRst => return Ok(QueryLoopResult::Rst),
@@ -127,7 +166,8 @@ pub(super) async fn query_loop<S: Socket>(
             QueryTimeExchangeResult::Error => {
                 let [a, b] =
                     (QueryError::NetworkSubsystemCorruptedPacket.value_u8() as u16).to_le_bytes();
-                con.write_all(&[0x10, a, b]).await?;
+                con.write_all(&[ResponseType::Error.value_u8(), a, b])
+                    .await?;
                 con.flush().await?;
                 buf.clear();
                 exchg_state = QueryTimeExchangeState::default();
@@ -135,12 +175,12 @@ pub(super) async fn query_loop<S: Socket>(
             }
         };
         // now execute query
-        match engine::core::exec::dispatch_to_executor(global, &client_state, sq).await {
+        match engine::core::exec::dispatch_to_executor(global, &mut client_state, sq).await {
             Ok(Response::Empty) => {
-                con.write_all(&[0x12]).await?;
+                con.write_all(&[ResponseType::Empty.value_u8()]).await?;
             }
-            Ok(Response::Row { size, data }) => {
-                con.write_u8(0x11).await?;
+            Ok(Response::Serialized { ty, size, data }) => {
+                con.write_u8(ty.value_u8()).await?;
                 let mut irep = IntegerRepr::new();
                 con.write_all(irep.as_bytes(size as u64)).await?;
                 con.write_u8(b'\n').await?;
@@ -148,7 +188,8 @@ pub(super) async fn query_loop<S: Socket>(
             }
             Err(e) => {
                 let [a, b] = (e.value_u8() as u16).to_le_bytes();
-                con.write_all(&[0x10, a, b]).await?;
+                con.write_all(&[ResponseType::Error.value_u8(), a, b])
+                    .await?;
             }
         }
         con.flush().await?;
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index b4b61306..c2c29c99 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -27,26 +27,25 @@
 pub mod traits;
 
 #[cfg(test)]
-pub use traits::{parse_ast_node_full, parse_ast_node_multiple_full};
+pub use traits::{
+    parse_ast_node_full, parse_ast_node_full_with_space, parse_ast_node_multiple_full,
+};
+
 use {
     super::{
         ddl, dml,
-        lex::{Ident, Token},
+        lex::{Ident, Keyword, KeywordStmt, Token},
     },
     crate::{
         engine::{
+            core::EntityIDRef,
             data::{cell::Datacell, lit::Lit},
             error::{QueryError, QueryResult},
         },
-        util::{compiler, MaybeInit},
+        util::MaybeInit,
     },
 };
 
-#[inline(always)]
-pub fn minidx<T>(src: &[T], index: usize) -> usize {
-    (src.len() - 1).min(index)
-}
-
 #[derive(Debug, PartialEq)]
 /// Query parse state
 pub struct State<'a, Qd> {
@@ -54,6 +53,7 @@ pub struct State<'a, Qd> {
     d: Qd,
     i: usize,
     f: bool,
+    cs: Option<&'static str>,
 }
 
 impl<'a> State<'a, InplaceData> {
@@ -62,6 +62,79 @@ impl<'a> State<'a, InplaceData> {
     }
 }
 
+impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
+    fn _entity_signature_match_self_full(a: &Token<'a>, b: &Token<'a>, c: &Token<'a>) -> bool {
+        a.is_ident() & Token![.].eq(b) & c.is_ident()
+    }
+    fn _entity_signature_match_cs(&self, a: &Token<'a>) -> bool {
+        a.is_ident() & self.cs.is_some()
+    }
+    unsafe fn _entity_new_from_tokens(&mut self) -> EntityIDRef<'a> {
+        let space = self.fw_read().uck_read_ident();
+        self.cursor_ahead();
+        let entity = self.fw_read().uck_read_ident();
+        EntityIDRef::new(space.as_str(), entity.as_str())
+    }
+    unsafe fn _entity_new_from_cs(&mut self) -> EntityIDRef<'a> {
+        let entity = self.fw_read().uck_read_ident();
+        EntityIDRef::new(self.cs.unwrap_unchecked(), entity.as_str())
+    }
+    pub fn set_space_maybe(&mut self, maybe: Option<&'static str>) {
+        self.cs = maybe;
+    }
+    pub fn unset_space(&mut self) {
+        self.set_space_maybe(None)
+    }
+    #[cfg(test)]
+    pub fn set_space(&mut self, s: &'static str) {
+        self.set_space_maybe(Some(s));
+    }
+    pub fn try_entity_buffered_into_state_uninit(&mut self) -> MaybeInit<EntityIDRef<'a>> {
+        let mut ret = MaybeInit::uninit();
+        let self_has_full = Self::_entity_signature_match_self_full(
+            &self.t[self.cursor()],
+            &self.t[self.cursor() + 1],
+            &self.t[self.cursor() + 2],
+        );
+        let self_has_full_cs = self._entity_signature_match_cs(&self.t[self.cursor()]);
+        unsafe {
+            if self_has_full {
+                ret = MaybeInit::new(self._entity_new_from_tokens());
+            } else if self_has_full_cs {
+                ret = MaybeInit::new(self._entity_new_from_cs());
+            }
+        }
+        self.poison_if_not(self_has_full | self_has_full_cs);
+        ret
+    }
+    pub fn try_entity_ref(&mut self) -> Option<EntityIDRef<'a>> {
+        let self_has_full = Self::_entity_signature_match_self_full(
+            &self.t[self.round_cursor()],
+            &self.t[self.round_cursor_up(1)],
+            &self.t[self.round_cursor_up(2)],
+        );
+        let self_has_pre_full = self._entity_signature_match_cs(&self.t[self.round_cursor()]);
+        if self_has_full {
+            unsafe {
+                // UNSAFE(@ohsayan): +branch condition
+                Some(self._entity_new_from_tokens())
+            }
+        } else {
+            if self_has_pre_full {
+                unsafe {
+                    // UNSAFE(@ohsayan): +branch condition
+                    Some(self._entity_new_from_cs())
+                }
+            } else {
+                None
+            }
+        }
+    }
+    pub fn try_entity_ref_result(&mut self) -> QueryResult<EntityIDRef<'a>> {
+        self.try_entity_ref().ok_or(QueryError::QLExpectedEntity)
+    }
+}
+
 impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     #[inline(always)]
     /// Create a new [`State`] instance using the given tokens and data
@@ -71,6 +144,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
             f: true,
             t,
             d,
+            cs: None,
         }
     }
     #[inline(always)]
@@ -148,7 +222,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     #[inline(always)]
     /// Check if the current cursor can read a lit (with context from the data source); rounded
     pub fn can_read_lit_rounded(&self) -> bool {
-        let mx = minidx(self.t, self.i);
+        let mx = self.round_cursor();
         Qd::can_read_lit_from(&self.d, &self.t[mx]) && mx == self.i
     }
     #[inline(always)]
@@ -176,7 +250,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     #[inline(always)]
     /// Check if the cursor equals the given token; rounded
     pub fn cursor_rounded_eq(&self, tok: Token<'a>) -> bool {
-        let mx = minidx(self.t, self.i);
+        let mx = self.round_cursor();
         self.t[mx] == tok && mx == self.i
     }
     #[inline(always)]
@@ -196,33 +270,17 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     }
     #[inline(always)]
     pub(crate) fn cursor_has_ident_rounded(&self) -> bool {
-        self.t[minidx(self.t, self.i)].is_ident() && self.not_exhausted()
+        self.t[self.round_cursor()].is_ident() && self.not_exhausted()
     }
     #[inline(always)]
     /// Check if the current token stream matches the signature of an arity(0) fn; rounded
     ///
     /// NOTE: Consider using a direct comparison without rounding
     pub(crate) fn cursor_signature_match_fn_arity0_rounded(&self) -> bool {
-        let rem = self.has_remaining(3);
-        let idx_a = self.i * rem as usize;
-        let idx_b = (self.i + 1) * rem as usize;
-        let idx_c = (self.i + 2) * rem as usize;
-        (self.t[idx_a].is_ident())
-            & (self.t[idx_b] == Token![() open])
-            & (self.t[idx_c] == Token![() close])
-            & rem
-    }
-    #[inline(always)]
-    /// Check if the current token stream matches the signature of a full entity; rounded
-    ///
-    /// NOTE: Consider using a direct comparison without rounding; rounding is always slower
-    pub(crate) fn cursor_signature_match_entity_full_rounded(&self) -> bool {
-        let rem = self.has_remaining(3);
-        let rem_u = rem as usize;
-        let idx_a = self.i * rem_u;
-        let idx_b = (self.i + 1) * rem_u;
-        let idx_c = (self.i + 2) * rem_u;
-        (self.t[idx_a].is_ident()) & (self.t[idx_b] == Token![.]) & (self.t[idx_c].is_ident()) & rem
+        (self.t[self.round_cursor()].is_ident())
+            & (self.t[self.round_cursor_up(1)] == Token![() open])
+            & (self.t[self.round_cursor_up(2)] == Token![() close])
+            & self.has_remaining(3)
     }
     #[inline(always)]
     /// Reads a lit using the given token and the internal data source and return a data type
@@ -240,7 +298,6 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
         self.not_exhausted() && self.okay()
     }
     #[inline(always)]
-    #[cfg(test)]
     /// Returns the position of the cursor
     pub(crate) fn cursor(&self) -> usize {
         self.i
@@ -250,6 +307,27 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     pub(crate) fn cursor_is_ident(&self) -> bool {
         self.read().is_ident()
     }
+    #[inline(always)]
+    fn round_cursor_up(&self, up: usize) -> usize {
+        core::cmp::min(self.t.len() - 1, self.i + up)
+    }
+    #[inline(always)]
+    fn round_cursor(&self) -> usize {
+        self.round_cursor_up(0)
+    }
+    pub fn try_statement(&mut self) -> QueryResult<KeywordStmt> {
+        match self.fw_read() {
+            Token::Keyword(Keyword::Statement(stmt)) => Ok(*stmt),
+            _ => Err(QueryError::QLExpectedStatement),
+        }
+    }
+    pub fn ensure_minimum_for_blocking_stmt(&self) -> QueryResult<()> {
+        if self.remaining() < 2 {
+            return Err(QueryError::QLExpectedStatement);
+        } else {
+            Ok(())
+        }
+    }
 }
 
 pub trait QueryData<'a> {
@@ -297,174 +375,11 @@ impl<'a> QueryData<'a> for InplaceData {
     }
 }
 
-/*
-    AST
-*/
-
-#[derive(Debug, PartialEq, Clone, Copy)]
-/// An [`Entity`] represents the location for a specific structure, such as a model
-pub enum Entity<'a> {
-    /// A single entity is used when switching to a model wrt the currently set space (commonly used
-    /// when running DML queries)
-    ///
-    /// syntax:
-    /// ```sql
-    /// model
-    /// ```
-    Single(Ident<'a>),
-    /// A full entity is a complete definition to a model wrt to the given space (commonly used with
-    /// DML queries)
-    ///
-    /// syntax:
-    /// ```sql
-    /// space.model
-    /// ```
-    Full(Ident<'a>, Ident<'a>),
-}
-
-impl<'a> Entity<'a> {
-    pub fn into_full_result(self) -> QueryResult<(Ident<'a>, Ident<'a>)> {
-        match self {
-            Self::Full(a, b) => Ok((a, b)),
-            _ => Err(QueryError::QLExpectedEntity),
-        }
-    }
-}
-
-impl<'a> From<(&'a str, &'a str)> for Entity<'a> {
-    fn from((s, e): (&'a str, &'a str)) -> Self {
-        Self::Full(s.into(), e.into())
-    }
-}
-
-impl<'a> Entity<'a> {
-    #[cfg(test)]
-    pub fn into_full(self) -> Option<(Ident<'a>, Ident<'a>)> {
-        if let Self::Full(a, b) = self {
-            Some((a, b))
-        } else {
-            None
-        }
-    }
-    #[inline(always)]
-    /// Parse a full entity from the given slice
-    ///
-    /// ## Safety
-    ///
-    /// Caller guarantees that the token stream matches the exact stream of tokens
-    /// expected for a full entity
-    pub(super) unsafe fn parse_uck_tokens_full(sl: &'a [Token]) -> Self {
-        Entity::Full(sl[0].uck_read_ident(), sl[2].uck_read_ident())
-    }
-    #[inline(always)]
-    /// Parse a single entity from the given slice
-    ///
-    /// ## Safety
-    ///
-    /// Caller guarantees that the token stream matches the exact stream of tokens
-    /// expected for a single entity
-    pub(super) unsafe fn parse_uck_tokens_single(sl: &'a [Token]) -> Self {
-        Entity::Single(sl[0].uck_read_ident())
-    }
-    #[inline(always)]
-    #[cfg(test)]
-    /// Returns true if the given token stream matches the signature of single entity syntax
-    ///
-    /// ⚠ WARNING: This will pass for full and single
-    pub(super) fn signature_matches_single_len_checked(tok: &[Token]) -> bool {
-        !tok.is_empty() && tok[0].is_ident()
-    }
-    #[inline(always)]
-    #[cfg(test)]
-    /// Returns true if the given token stream matches the signature of full entity syntax
-    pub(super) fn signature_matches_full_len_checked(tok: &[Token]) -> bool {
-        tok.len() > 2 && tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident()
-    }
-    #[inline(always)]
-    #[cfg(test)]
-    /// Attempt to parse an entity using the given token stream. It also accepts a counter
-    /// argument to forward the cursor
-    pub fn parse_from_tokens_len_checked(tok: &'a [Token], c: &mut usize) -> QueryResult<Self> {
-        let is_current = Self::signature_matches_single_len_checked(tok);
-        let is_full = Self::signature_matches_full_len_checked(tok);
-        let r = match () {
-            _ if is_full => unsafe {
-                // UNSAFE(@ohsayan): just verified signature
-                *c += 3;
-                Self::parse_uck_tokens_full(tok)
-            },
-            _ if is_current => unsafe {
-                // UNSAFE(@ohsayan): just verified signature
-                *c += 1;
-                Self::parse_uck_tokens_single(tok)
-            },
-            _ => return Err(QueryError::QLExpectedEntity),
-        };
-        Ok(r)
-    }
-    #[inline(always)]
-    pub fn parse_from_state_rounded_result<Qd: QueryData<'a>>(
-        state: &mut State<'a, Qd>,
-    ) -> QueryResult<Self> {
-        let mut e = MaybeInit::uninit();
-        Self::parse_from_state_rounded(state, &mut e);
-        if compiler::likely(state.okay()) {
-            unsafe {
-                // UNSAFE(@ohsayan): just checked if okay
-                Ok(e.assume_init())
-            }
-        } else {
-            Err(QueryError::QLExpectedEntity)
-        }
-    }
-    #[inline(always)]
-    pub fn parse_from_state_rounded<Qd: QueryData<'a>>(
-        state: &mut State<'a, Qd>,
-        d: &mut MaybeInit<Entity<'a>>,
-    ) {
-        let tok = state.current();
-        let is_full = state.cursor_signature_match_entity_full_rounded();
-        let is_single = state.cursor_has_ident_rounded();
-        unsafe {
-            // UNSAFE(@ohsayan): verified signatures
-            if is_full {
-                state.cursor_ahead_by(3);
-                *d = MaybeInit::new(Entity::parse_uck_tokens_full(tok));
-            } else if is_single {
-                state.cursor_ahead();
-                *d = MaybeInit::new(Entity::parse_uck_tokens_single(tok));
-            }
-        }
-        state.poison_if_not(is_full | is_single);
-    }
-    pub fn parse_from_state_len_unchecked<Qd: QueryData<'a>>(
-        state: &mut State<'a, Qd>,
-        d: &mut MaybeInit<Entity<'a>>,
-    ) {
-        let tok = state.current();
-        let is_full = tok[0].is_ident() && tok[1] == Token![.] && tok[2].is_ident();
-        let is_single = tok[0].is_ident();
-        unsafe {
-            // UNSAFE(@ohsayan): verified signatures
-            if is_full {
-                state.cursor_ahead_by(3);
-                *d = MaybeInit::new(Entity::parse_uck_tokens_full(tok));
-            } else if is_single {
-                state.cursor_ahead();
-                *d = MaybeInit::new(Entity::parse_uck_tokens_single(tok));
-            }
-        }
-        state.poison_if_not(is_full | is_single);
-    }
-}
-
 #[derive(Debug, PartialEq)]
 #[allow(dead_code)] // TODO(@ohsayan): get rid of this
 /// A [`Statement`] is a fully BlueQL statement that can be executed by the query engine
 // TODO(@ohsayan): Determine whether we actually need this
 pub enum Statement<'a> {
-    /// DDL query to switch between spaces and models
-    Use(Entity<'a>),
     /// DDL query to create a model
     CreateModel(ddl::crt::CreateModel<'a>),
     /// DDL query to create a space
@@ -487,8 +402,6 @@ pub enum Statement<'a> {
     DropSpace(ddl::drop::DropSpace<'a>),
     /// DDL query to inspect a space (returns a list of models in the space)
     InspectSpace(Ident<'a>),
-    /// DDL query to inspect a model (returns the model definition)
-    InspectModel(Entity<'a>),
     /// DDL query to inspect all spaces (returns a list of spaces in the database)
     InspectSpaces,
     /// DML insert
@@ -505,14 +418,13 @@ pub enum Statement<'a> {
 #[cfg(test)]
 #[allow(dead_code)] // TODO(@ohsayan): get rid of this
 pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResult<Statement<'a>> {
-    use self::traits::ASTNode;
+    use {self::traits::ASTNode, crate::util::compiler};
     if compiler::unlikely(tok.len() < 2) {
         return Err(QueryError::QLUnexpectedEndOfStatement);
     }
     let mut state = State::new(tok, d);
     match state.fw_read() {
         // DDL
-        Token![use] => Entity::parse_from_state_rounded_result(&mut state).map(Statement::Use),
         Token![create] => match state.fw_read() {
             Token![model] => ASTNode::test_parse_from_state(&mut state).map(Statement::CreateModel),
             Token![space] => ASTNode::test_parse_from_state(&mut state).map(Statement::CreateSpace),
@@ -524,9 +436,6 @@ pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResul
             _ => compiler::cold_rerr(QueryError::QLUnknownStatement),
         },
         Token![drop] if state.remaining() >= 2 => ddl::drop::parse_drop(&mut state),
-        Token::Ident(id) if id.eq_ignore_ascii_case("inspect") => {
-            ddl::ins::parse_inspect(&mut state)
-        }
         // DML
         Token![insert] => ASTNode::test_parse_from_state(&mut state).map(Statement::Insert),
         Token![select] => ASTNode::test_parse_from_state(&mut state).map(Statement::Select),
diff --git a/server/src/engine/ql/ast/traits.rs b/server/src/engine/ql/ast/traits.rs
index 925a4da5..5ccb37e8 100644
--- a/server/src/engine/ql/ast/traits.rs
+++ b/server/src/engine/ql/ast/traits.rs
@@ -108,6 +108,17 @@ pub trait ASTNode<'a>: Sized {
         Ok(r)
     }
     #[cfg(test)]
+    fn from_insecure_tokens_full_with_space(
+        tok: &'a [Token<'a>],
+        space_name: &'static str,
+    ) -> QueryResult<Self> {
+        let mut state = State::new(tok, InplaceData::new());
+        state.set_space(space_name);
+        let r = <Self as ASTNode>::test_parse_from_state(&mut state)?;
+        assert!(state.exhausted());
+        Ok(r)
+    }
+    #[cfg(test)]
     /// Parse multiple nodes of this AST node type, utilizing the full token stream.
     /// Intended for the test suite.
     fn multiple_from_insecure_tokens_full(tok: &'a [Token<'a>]) -> QueryResult<Vec<Self>> {
@@ -126,6 +137,13 @@ pub fn parse_ast_node_full<'a, N: ASTNode<'a>>(tok: &'a [Token<'a>]) -> QueryRes
     N::from_insecure_tokens_full(tok)
 }
 #[cfg(test)]
+pub fn parse_ast_node_full_with_space<'a, N: ASTNode<'a>>(
+    tok: &'a [Token<'a>],
+    space_name: &'static str,
+) -> QueryResult<N> {
+    N::from_insecure_tokens_full_with_space(tok, space_name)
+}
+#[cfg(test)]
 pub fn parse_ast_node_multiple_full<'a, N: ASTNode<'a>>(
     tok: &'a [Token<'a>],
 ) -> QueryResult<Vec<N>> {
diff --git a/server/src/engine/ql/benches.rs b/server/src/engine/ql/benches.rs
deleted file mode 100644
index 5bd97a3e..00000000
--- a/server/src/engine/ql/benches.rs
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Created on Wed Nov 16 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-/*
-    All benches should be aggregate costs of full execution. This means that when for example
-    you're writing a benchmark for something like parsing a `select` statement, you should calculate
-    the total time of execution including lexing, parsing and allocating. Hopefully in the future we can
-    implement a testing framework that enables us to find the total tiered cost of execution for each stage
-    and hence enable us to iterate on the weakness and fix it. Maybe even visualize it? That'd be amazing
-    and maybe would be something I'll work on around 0.9.
-
-    -- Sayan (@ohsayan)
-*/
-
-extern crate test;
-
-use {
-    crate::engine::ql::{lex::Ident, tests::lex_insecure},
-    test::Bencher,
-};
-
-mod lexer {
-    use {
-        super::*,
-        crate::engine::{data::lit::Lit, ql::lex::Token},
-    };
-    #[bench]
-    fn lex_number(b: &mut Bencher) {
-        let src = b"1234567890";
-        let expected = vec![Token::Lit(1234567890_u64.into())];
-        b.iter(|| assert_eq!(lex_insecure(src).unwrap(), expected));
-    }
-    #[bench]
-    fn lex_bool(b: &mut Bencher) {
-        let s = b"true";
-        let e = vec![Token::Lit(true.into())];
-        b.iter(|| assert_eq!(lex_insecure(s).unwrap(), e));
-    }
-    #[bench]
-    fn lex_string_noescapes(b: &mut Bencher) {
-        let s = br#"'hello, world!'"#;
-        let e = vec![Token::Lit("hello, world!".into())];
-        b.iter(|| assert_eq!(lex_insecure(s).unwrap(), e));
-    }
-    #[bench]
-    fn lex_string_with_escapes(b: &mut Bencher) {
-        let s = br#"'hello, world! this is within a \'quote\''"#;
-        let e = vec![Token::Lit("hello, world! this is within a 'quote'".into())];
-        b.iter(|| assert_eq!(lex_insecure(s).unwrap(), e));
-    }
-    #[bench]
-    fn lex_raw_literal(b: &mut Bencher) {
-        let src = b"\r44\ne69b10ffcc250ae5091dec6f299072e23b0b41d6a739";
-        let expected = vec![Token::Lit(Lit::new_bin(
-            b"e69b10ffcc250ae5091dec6f299072e23b0b41d6a739",
-        ))];
-        b.iter(|| assert_eq!(lex_insecure(src).unwrap(), expected));
-    }
-}
-
-mod ast {
-    use {
-        super::*,
-        crate::engine::ql::ast::{Entity, InplaceData, State},
-    };
-    #[bench]
-    fn parse_entity_single(b: &mut Bencher) {
-        let e = Entity::Single(Ident::from("user"));
-        b.iter(|| {
-            let src = lex_insecure(b"user").unwrap();
-            let mut state = State::new(&src, InplaceData::new());
-            let re = Entity::parse_from_state_rounded_result(&mut state).unwrap();
-            assert_eq!(e, re);
-            assert!(state.exhausted());
-        })
-    }
-    #[bench]
-    fn parse_entity_double(b: &mut Bencher) {
-        let e = Entity::Full(Ident::from("tweeter"), Ident::from("user"));
-        b.iter(|| {
-            let src = lex_insecure(b"tweeter.user").unwrap();
-            let mut state = State::new(&src, InplaceData::new());
-            let re = Entity::parse_from_state_rounded_result(&mut state).unwrap();
-            assert_eq!(e, re);
-            assert!(state.exhausted());
-        })
-    }
-}
-
-mod ddl_queries {
-    use {
-        super::*,
-        crate::engine::ql::{
-            ast::{compile, Entity, InplaceData, Statement},
-            lex::InsecureLexer,
-        },
-    };
-    mod use_stmt {
-        use super::*;
-        #[bench]
-        fn use_space(b: &mut Bencher) {
-            let src = b"use myspace";
-            let expected = Statement::Use(Entity::Single(Ident::from("myspace")));
-            b.iter(|| {
-                let lexed = InsecureLexer::lex(src).unwrap();
-                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
-            });
-        }
-        #[bench]
-        fn use_model(b: &mut Bencher) {
-            let src = b"use myspace.mymodel";
-            let expected =
-                Statement::Use(Entity::Full(Ident::from("myspace"), Ident::from("mymodel")));
-            b.iter(|| {
-                let lexed = InsecureLexer::lex(src).unwrap();
-                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
-            });
-        }
-    }
-    mod inspect_stmt {
-        use super::*;
-        #[bench]
-        fn inspect_space(b: &mut Bencher) {
-            let src = b"inspect space myspace";
-            let expected = Statement::InspectSpace(Ident::from("myspace"));
-            b.iter(|| {
-                let lexed = InsecureLexer::lex(src).unwrap();
-                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
-            });
-        }
-        #[bench]
-        fn inspect_model_single_entity(b: &mut Bencher) {
-            let src = b"inspect model mymodel";
-            let expected = Statement::InspectModel(Entity::Single(Ident::from("mymodel")));
-            b.iter(|| {
-                let lexed = InsecureLexer::lex(src).unwrap();
-                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
-            });
-        }
-        #[bench]
-        fn inspect_model_full_entity(b: &mut Bencher) {
-            let src = b"inspect model myspace.mymodel";
-            let expected = Statement::InspectModel(Entity::Full(
-                Ident::from("myspace"),
-                Ident::from("mymodel"),
-            ));
-            b.iter(|| {
-                let lexed = InsecureLexer::lex(src).unwrap();
-                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
-            });
-        }
-        #[bench]
-        fn inspect_spaces(b: &mut Bencher) {
-            let src = b"inspect spaces";
-            let expected = Statement::InspectSpaces;
-            b.iter(|| {
-                let lexed = InsecureLexer::lex(src).unwrap();
-                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
-            });
-        }
-    }
-    mod drop_stmt {
-        use {
-            super::*,
-            crate::engine::ql::ddl::drop::{DropModel, DropSpace},
-        };
-        #[bench]
-        fn drop_space(b: &mut Bencher) {
-            let src = b"drop space myspace";
-            let expected = Statement::DropSpace(DropSpace::new(Ident::from("myspace"), false));
-            b.iter(|| {
-                let lexed = InsecureLexer::lex(src).unwrap();
-                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
-            });
-        }
-        #[bench]
-        fn drop_space_force(b: &mut Bencher) {
-            let src = b"drop space myspace force";
-            let expected = Statement::DropSpace(DropSpace::new(Ident::from("myspace"), true));
-            b.iter(|| {
-                let lexed = InsecureLexer::lex(src).unwrap();
-                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
-            });
-        }
-        #[bench]
-        fn drop_model_single(b: &mut Bencher) {
-            let src = b"drop model mymodel";
-            let expected = Statement::DropModel(DropModel::new(
-                Entity::Single(Ident::from("mymodel")),
-                false,
-            ));
-            b.iter(|| {
-                let lexed = InsecureLexer::lex(src).unwrap();
-                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
-            });
-        }
-        #[bench]
-        fn drop_model_single_force(b: &mut Bencher) {
-            let src = b"drop model mymodel force";
-            let expected =
-                Statement::DropModel(DropModel::new(Entity::Single(Ident::from("mymodel")), true));
-            b.iter(|| {
-                let lexed = InsecureLexer::lex(src).unwrap();
-                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
-            });
-        }
-        #[bench]
-        fn drop_model_full(b: &mut Bencher) {
-            let src = b"drop model myspace.mymodel";
-            let expected = Statement::DropModel(DropModel::new(
-                Entity::Full(Ident::from("myspace"), Ident::from("mymodel")),
-                false,
-            ));
-            b.iter(|| {
-                let lexed = InsecureLexer::lex(src).unwrap();
-                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
-            });
-        }
-        #[bench]
-        fn drop_model_full_force(b: &mut Bencher) {
-            let src = b"drop model myspace.mymodel force";
-            let expected = Statement::DropModel(DropModel::new(
-                Entity::Full(Ident::from("myspace"), Ident::from("mymodel")),
-                true,
-            ));
-            b.iter(|| {
-                let lexed = InsecureLexer::lex(src).unwrap();
-                assert_eq!(compile(&lexed, InplaceData::new()).unwrap(), expected);
-            });
-        }
-    }
-}
diff --git a/server/src/engine/ql/dcl.rs b/server/src/engine/ql/dcl.rs
index 5d6b9d31..ec4be180 100644
--- a/server/src/engine/ql/dcl.rs
+++ b/server/src/engine/ql/dcl.rs
@@ -52,27 +52,22 @@ impl<'a> traits::ASTNode<'a> for SysctlCommand<'a> {
     fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
         state: &mut State<'a, Qd>,
     ) -> QueryResult<Self> {
-        if state.remaining() < 1 {
+        if state.remaining() < 2 {
             return Err(QueryError::QLUnexpectedEndOfStatement);
         }
-        let token = state.fw_read();
-        let create = Token![create].eq(token);
-        let drop = Token![drop].eq(token);
-        let status = token.ident_eq("status");
-        if status {
-            return Ok(SysctlCommand::ReportStatus);
-        }
-        if state.exhausted() {
-            return Err(QueryError::QLUnexpectedEndOfStatement);
-        }
-        let create_or_drop = state.fw_read();
-        if !create_or_drop.ident_eq("user") & !(create | drop) {
+        let (a, b) = (state.fw_read(), state.fw_read());
+        let create = Token![create].eq(a) & b.ident_eq("user");
+        let drop = Token![drop].eq(a) & b.ident_eq("user");
+        let status = a.ident_eq("report") & b.ident_eq("status");
+        if !(create | drop | status) {
             return Err(QueryError::QLUnknownStatement);
         }
         if create {
             UserAdd::parse(state).map(SysctlCommand::CreateUser)
-        } else {
+        } else if drop {
             UserDel::parse(state).map(SysctlCommand::DropUser)
+        } else {
+            Ok(SysctlCommand::ReportStatus)
         }
     }
 }
diff --git a/server/src/engine/ql/ddl/alt.rs b/server/src/engine/ql/ddl/alt.rs
index 702dcdc8..b6069a05 100644
--- a/server/src/engine/ql/ddl/alt.rs
+++ b/server/src/engine/ql/ddl/alt.rs
@@ -28,10 +28,11 @@ use {
     super::syn::{self, DictFoldState, ExpandedField},
     crate::{
         engine::{
+            core::EntityIDRef,
             data::DictGeneric,
             error::{QueryError, QueryResult},
             ql::{
-                ast::{Entity, QueryData, State},
+                ast::{QueryData, State},
                 lex::{Ident, Token},
             },
         },
@@ -90,13 +91,13 @@ impl<'a> AlterSpace<'a> {
 
 #[derive(Debug, PartialEq)]
 pub struct AlterModel<'a> {
-    pub(in crate::engine) model: Entity<'a>,
+    pub(in crate::engine) model: EntityIDRef<'a>,
     pub(in crate::engine) kind: AlterKind<'a>,
 }
 
 impl<'a> AlterModel<'a> {
     #[inline(always)]
-    pub fn new(model: Entity<'a>, kind: AlterKind<'a>) -> Self {
+    pub fn new(model: EntityIDRef<'a>, kind: AlterKind<'a>) -> Self {
         Self { model, kind }
     }
 }
@@ -118,7 +119,7 @@ impl<'a> AlterModel<'a> {
             return compiler::cold_rerr(QueryError::QLInvalidSyntax);
             // FIXME(@ohsayan): bad because no specificity
         }
-        let model_name = Entity::parse_from_state_rounded_result(state)?;
+        let model_name = state.try_entity_ref_result()?;
         let kind = match state.fw_read() {
             Token![add] => AlterKind::alter_add(state),
             Token![remove] => AlterKind::alter_remove(state),
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index edab417c..4827146f 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -28,14 +28,15 @@ use {
     super::syn::{self, DictFoldState, FieldSpec},
     crate::{
         engine::{
+            core::EntityIDRef,
             data::DictGeneric,
             error::{QueryError, QueryResult},
             ql::{
-                ast::{Entity, QueryData, State},
+                ast::{QueryData, State},
                 lex::Ident,
             },
         },
-        util::{compiler, MaybeInit},
+        util::compiler,
     },
 };
 
@@ -85,7 +86,7 @@ impl<'a> CreateSpace<'a> {
 /// A model definition
 pub struct CreateModel<'a> {
     /// the model name
-    pub(in crate::engine) model_name: Entity<'a>,
+    pub(in crate::engine) model_name: EntityIDRef<'a>,
     /// the fields
     pub(in crate::engine) fields: Vec<FieldSpec<'a>>,
     /// properties
@@ -101,7 +102,11 @@ pub struct CreateModel<'a> {
 
 impl<'a> CreateModel<'a> {
     #[cfg(test)]
-    pub fn new(model_name: Entity<'a>, fields: Vec<FieldSpec<'a>>, props: DictGeneric) -> Self {
+    pub fn new(
+        model_name: EntityIDRef<'a>,
+        fields: Vec<FieldSpec<'a>>,
+        props: DictGeneric,
+    ) -> Self {
         Self {
             model_name,
             fields,
@@ -114,8 +119,7 @@ impl<'a> CreateModel<'a> {
             return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
         }
         // model name; ignore errors
-        let mut model_uninit = MaybeInit::uninit();
-        Entity::parse_from_state_len_unchecked(state, &mut model_uninit);
+        let model_uninit = state.try_entity_buffered_into_state_uninit();
         state.poison_if_not(state.cursor_eq(Token![() open]));
         state.cursor_ahead();
         // fields
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index 15634118..2497e518 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -25,9 +25,10 @@
 */
 
 use crate::engine::{
+    core::EntityIDRef,
     error::{QueryError, QueryResult},
     ql::{
-        ast::{Entity, QueryData, State, Statement},
+        ast::{QueryData, State, Statement},
         lex::{Ident, Token},
     },
 };
@@ -66,17 +67,17 @@ impl<'a> DropSpace<'a> {
 
 #[derive(Debug, PartialEq)]
 pub struct DropModel<'a> {
-    pub(in crate::engine) entity: Entity<'a>,
+    pub(in crate::engine) entity: EntityIDRef<'a>,
     pub(in crate::engine) force: bool,
 }
 
 impl<'a> DropModel<'a> {
     #[inline(always)]
-    pub fn new(entity: Entity<'a>, force: bool) -> Self {
+    pub fn new(entity: EntityIDRef<'a>, force: bool) -> Self {
         Self { entity, force }
     }
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
-        let e = Entity::parse_from_state_rounded_result(state)?;
+        let e = state.try_entity_ref_result()?;
         let force = state.cursor_rounded_eq(Token::Ident(Ident::from("force")));
         state.cursor_ahead_if(force);
         Ok(DropModel::new(e, force))
diff --git a/server/src/engine/ql/ddl/ins.rs b/server/src/engine/ql/ddl/ins.rs
deleted file mode 100644
index fc69e888..00000000
--- a/server/src/engine/ql/ddl/ins.rs
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Created on Wed Feb 01 2023
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use crate::{
-    engine::{
-        error::{QueryError, QueryResult},
-        ql::{
-            ast::{Entity, QueryData, State, Statement},
-            lex::Token,
-        },
-    },
-    util::compiler,
-};
-
-pub fn parse_inspect<'a, Qd: QueryData<'a>>(
-    state: &mut State<'a, Qd>,
-) -> QueryResult<Statement<'a>> {
-    /*
-        inpsect model <entity>
-        inspect space <entity>
-        inspect spaces
-
-        min length -> (<model> | <space>) <model> = 2
-    */
-
-    if compiler::unlikely(state.remaining() < 1) {
-        return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
-    }
-
-    match state.fw_read() {
-        Token![model] => {
-            Entity::parse_from_state_rounded_result(state).map(Statement::InspectModel)
-        }
-        Token![space] if state.cursor_has_ident_rounded() => {
-            Ok(Statement::InspectSpace(unsafe {
-                // UNSAFE(@ohsayan): Safe because of the match predicate
-                state.fw_read().uck_read_ident()
-            }))
-        }
-        Token::Ident(id) if id.eq_ignore_ascii_case("spaces") && state.exhausted() => {
-            Ok(Statement::InspectSpaces)
-        }
-        _ => {
-            state.cursor_back();
-            Err(QueryError::QLExpectedStatement)
-        }
-    }
-}
-
-#[cfg(test)]
-pub use impls::InspectStatementAST;
-mod impls {
-    use crate::engine::{
-        error::QueryResult,
-        ql::ast::{traits::ASTNode, QueryData, State, Statement},
-    };
-    #[derive(sky_macros::Wrapper, Debug)]
-    pub struct InspectStatementAST<'a>(Statement<'a>);
-    impl<'a> ASTNode<'a> for InspectStatementAST<'a> {
-        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
-        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
-        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
-            state: &mut State<'a, Qd>,
-        ) -> QueryResult<Self> {
-            super::parse_inspect(state).map(Self)
-        }
-    }
-}
diff --git a/server/src/engine/ql/ddl/mod.rs b/server/src/engine/ql/ddl/mod.rs
index 639106d0..4f0e0c21 100644
--- a/server/src/engine/ql/ddl/mod.rs
+++ b/server/src/engine/ql/ddl/mod.rs
@@ -29,4 +29,37 @@ pub(in crate::engine) mod syn;
 pub(in crate::engine) mod alt;
 pub(in crate::engine) mod crt;
 pub(in crate::engine) mod drop;
-pub(in crate::engine) mod ins;
+
+use {
+    super::{
+        ast::traits::ASTNode,
+        lex::{Ident, Token},
+    },
+    crate::engine::error::QueryError,
+};
+
+#[derive(Debug, PartialEq)]
+pub enum Use<'a> {
+    Space(Ident<'a>),
+    Null,
+}
+
+impl<'a> ASTNode<'a> for Use<'a> {
+    const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+    const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = true;
+    fn __base_impl_parse_from_state<Qd: super::ast::QueryData<'a>>(
+        state: &mut super::ast::State<'a, Qd>,
+    ) -> crate::engine::error::QueryResult<Self> {
+        /*
+            should have either an ident or null
+        */
+        if state.remaining() != 1 {
+            return Err(QueryError::QLInvalidSyntax);
+        }
+        Ok(match state.fw_read() {
+            Token![null] => Self::Null,
+            Token::Ident(id) => Self::Space(id.clone()),
+            _ => return Err(QueryError::QLInvalidSyntax),
+        })
+    }
+}
diff --git a/server/src/engine/ql/dml/del.rs b/server/src/engine/ql/dml/del.rs
index ed38ca56..7ecb0127 100644
--- a/server/src/engine/ql/dml/del.rs
+++ b/server/src/engine/ql/dml/del.rs
@@ -30,10 +30,11 @@ use {
     super::WhereClause,
     crate::{
         engine::{
+            core::EntityIDRef,
             error::{QueryError, QueryResult},
-            ql::ast::{Entity, QueryData, State},
+            ql::ast::{QueryData, State},
         },
-        util::{compiler, MaybeInit},
+        util::compiler,
     },
 };
 
@@ -46,12 +47,12 @@ use {
 
 #[derive(Debug, PartialEq)]
 pub struct DeleteStatement<'a> {
-    pub(super) entity: Entity<'a>,
+    pub(super) entity: EntityIDRef<'a>,
     pub(super) wc: WhereClause<'a>,
 }
 
 impl<'a> DeleteStatement<'a> {
-    pub const fn entity(&self) -> Entity<'a> {
+    pub const fn entity(&self) -> EntityIDRef<'a> {
         self.entity
     }
     pub fn clauses_mut(&mut self) -> &mut WhereClause<'a> {
@@ -62,12 +63,12 @@ impl<'a> DeleteStatement<'a> {
 impl<'a> DeleteStatement<'a> {
     #[inline(always)]
     #[cfg(test)]
-    pub(super) fn new(entity: Entity<'a>, wc: WhereClause<'a>) -> Self {
+    pub(super) fn new(entity: EntityIDRef<'a>, wc: WhereClause<'a>) -> Self {
         Self { entity, wc }
     }
     #[inline(always)]
     #[cfg(test)]
-    pub fn new_test(entity: Entity<'a>, wc: WhereClauseCollection<'a>) -> Self {
+    pub fn new_test(entity: EntityIDRef<'a>, wc: WhereClauseCollection<'a>) -> Self {
         Self::new(entity, WhereClause::new(wc))
     }
     #[inline(always)]
@@ -84,8 +85,7 @@ impl<'a> DeleteStatement<'a> {
         // from + entity
         state.poison_if_not(state.cursor_eq(Token![from]));
         state.cursor_ahead(); // ignore errors (if any)
-        let mut entity = MaybeInit::uninit();
-        Entity::parse_from_state_len_unchecked(state, &mut entity);
+        let entity = state.try_entity_buffered_into_state_uninit();
         // where + clauses
         state.poison_if_not(state.cursor_eq(Token![where]));
         state.cursor_ahead(); // ignore errors
diff --git a/server/src/engine/ql/dml/ins.rs b/server/src/engine/ql/dml/ins.rs
index eaf70aba..9bd4069b 100644
--- a/server/src/engine/ql/dml/ins.rs
+++ b/server/src/engine/ql/dml/ins.rs
@@ -27,14 +27,15 @@
 use {
     crate::{
         engine::{
+            core::EntityIDRef,
             data::cell::Datacell,
             error::{QueryError, QueryResult},
             ql::{
-                ast::{Entity, QueryData, State},
+                ast::{QueryData, State},
                 lex::{Ident, Token},
             },
         },
-        util::{compiler, MaybeInit},
+        util::compiler,
     },
     std::{
         collections::HashMap,
@@ -321,17 +322,17 @@ impl<'a> From<HashMap<Ident<'static>, Datacell>> for InsertData<'a> {
 
 #[derive(Debug, PartialEq)]
 pub struct InsertStatement<'a> {
-    pub(super) entity: Entity<'a>,
+    pub(super) entity: EntityIDRef<'a>,
     pub(super) data: InsertData<'a>,
 }
 
 impl<'a> InsertStatement<'a> {
     #[inline(always)]
     #[cfg(test)]
-    pub fn new(entity: Entity<'a>, data: InsertData<'a>) -> Self {
+    pub fn new(entity: EntityIDRef<'a>, data: InsertData<'a>) -> Self {
         Self { entity, data }
     }
-    pub fn entity(&self) -> Entity<'a> {
+    pub fn entity(&self) -> EntityIDRef<'a> {
         self.entity
     }
     pub fn data(self) -> InsertData<'a> {
@@ -353,8 +354,7 @@ impl<'a> InsertStatement<'a> {
         state.cursor_ahead(); // ignore errors
 
         // entity
-        let mut entity = MaybeInit::uninit();
-        Entity::parse_from_state_len_unchecked(state, &mut entity);
+        let entity = state.try_entity_buffered_into_state_uninit();
         let mut data = None;
         match state.fw_read() {
             Token![() open] if state.not_exhausted() => {
diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs
index a17d58d6..2efd189f 100644
--- a/server/src/engine/ql/dml/sel.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -30,13 +30,14 @@ use {
     super::WhereClause,
     crate::{
         engine::{
+            core::EntityIDRef,
             error::{QueryError, QueryResult},
             ql::{
-                ast::{Entity, QueryData, State},
+                ast::{QueryData, State},
                 lex::{Ident, Token},
             },
         },
-        util::{compiler, MaybeInit},
+        util::compiler,
     },
 };
 
@@ -47,7 +48,7 @@ use {
 #[derive(Debug, PartialEq)]
 pub struct SelectStatement<'a> {
     /// the entity
-    pub(super) entity: Entity<'a>,
+    pub(super) entity: EntityIDRef<'a>,
     /// fields in order of querying. will be zero when wildcard is set
     pub(super) fields: Vec<Ident<'a>>,
     /// whether a wildcard was passed
@@ -60,7 +61,7 @@ impl<'a> SelectStatement<'a> {
     #[inline(always)]
     #[cfg(test)]
     pub(crate) fn new_test(
-        entity: Entity<'a>,
+        entity: EntityIDRef<'a>,
         fields: Vec<Ident<'a>>,
         wildcard: bool,
         clauses: WhereClauseCollection<'a>,
@@ -70,7 +71,7 @@ impl<'a> SelectStatement<'a> {
     #[inline(always)]
     #[cfg(test)]
     fn new(
-        entity: Entity<'a>,
+        entity: EntityIDRef<'a>,
         fields: Vec<Ident<'a>>,
         wildcard: bool,
         clauses: WhereClauseCollection<'a>,
@@ -82,7 +83,7 @@ impl<'a> SelectStatement<'a> {
             clause: WhereClause::new(clauses),
         }
     }
-    pub fn entity(&self) -> Entity<'a> {
+    pub fn entity(&self) -> EntityIDRef<'a> {
         self.entity
     }
     pub fn clauses_mut(&mut self) -> &mut WhereClause<'a> {
@@ -128,8 +129,7 @@ impl<'a> SelectStatement<'a> {
         }
         state.poison_if_not(state.cursor_eq(Token![from]));
         state.cursor_ahead(); // ignore errors
-        let mut entity = MaybeInit::uninit();
-        Entity::parse_from_state_rounded(state, &mut entity);
+        let entity = state.try_entity_buffered_into_state_uninit();
         let mut clauses = <_ as Default>::default();
         if state.cursor_rounded_eq(Token![where]) {
             state.cursor_ahead();
diff --git a/server/src/engine/ql/dml/upd.rs b/server/src/engine/ql/dml/upd.rs
index 9252ba06..8eaf8ea3 100644
--- a/server/src/engine/ql/dml/upd.rs
+++ b/server/src/engine/ql/dml/upd.rs
@@ -28,15 +28,15 @@ use {
     super::{u, WhereClause},
     crate::{
         engine::{
-            core::query_meta::AssignmentOperator,
+            core::{query_meta::AssignmentOperator, EntityIDRef},
             data::lit::Lit,
             error::{QueryError, QueryResult},
             ql::{
-                ast::{Entity, QueryData, State},
+                ast::{QueryData, State},
                 lex::Ident,
             },
         },
-        util::{compiler, MaybeInit},
+        util::compiler,
     },
 };
 
@@ -124,13 +124,13 @@ impl<'a> AssignmentExpression<'a> {
 
 #[derive(Debug, PartialEq)]
 pub struct UpdateStatement<'a> {
-    pub(super) entity: Entity<'a>,
+    pub(super) entity: EntityIDRef<'a>,
     pub(super) expressions: Vec<AssignmentExpression<'a>>,
     pub(super) wc: WhereClause<'a>,
 }
 
 impl<'a> UpdateStatement<'a> {
-    pub fn entity(&self) -> Entity<'a> {
+    pub fn entity(&self) -> EntityIDRef<'a> {
         self.entity
     }
     pub fn expressions(&self) -> &[AssignmentExpression<'a>] {
@@ -148,7 +148,7 @@ impl<'a> UpdateStatement<'a> {
     #[inline(always)]
     #[cfg(test)]
     pub fn new(
-        entity: Entity<'a>,
+        entity: EntityIDRef<'a>,
         expressions: Vec<AssignmentExpression<'a>>,
         wc: WhereClause<'a>,
     ) -> Self {
@@ -170,8 +170,7 @@ impl<'a> UpdateStatement<'a> {
             return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
         }
         // parse entity
-        let mut entity = MaybeInit::uninit();
-        Entity::parse_from_state_len_unchecked(state, &mut entity);
+        let entity = state.try_entity_buffered_into_state_uninit();
         if !(state.has_remaining(6)) {
             unsafe {
                 // UNSAFE(@ohsayan): Obvious from above, max 3 fw
diff --git a/server/src/engine/ql/mod.rs b/server/src/engine/ql/mod.rs
index 02c1368f..a05c924c 100644
--- a/server/src/engine/ql/mod.rs
+++ b/server/src/engine/ql/mod.rs
@@ -27,9 +27,6 @@
 #[macro_use]
 mod macros;
 pub(super) mod ast;
-#[cfg(feature = "nightly")]
-#[cfg(test)]
-mod benches;
 pub(super) mod dcl;
 pub(super) mod ddl;
 pub(super) mod dml;
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index c856f19d..25c4b3a1 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -35,8 +35,8 @@ use {
 
 mod dcl;
 mod dml_tests;
-mod entity;
 mod lexer_tests;
+mod misc;
 mod schema_tests;
 mod structure_syn;
 
diff --git a/server/src/engine/ql/tests/dcl.rs b/server/src/engine/ql/tests/dcl.rs
index 5e15170a..44596d99 100644
--- a/server/src/engine/ql/tests/dcl.rs
+++ b/server/src/engine/ql/tests/dcl.rs
@@ -32,7 +32,7 @@ use crate::engine::ql::{
 
 #[test]
 fn report_status_simple() {
-    let query = lex_insecure(b"sysctl status").unwrap();
+    let query = lex_insecure(b"sysctl report status").unwrap();
     let q = ast::parse_ast_node_full::<dcl::SysctlCommand>(&query[1..]).unwrap();
     assert_eq!(q, SysctlCommand::ReportStatus)
 }
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index eeb7ec3a..15ad9c44 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -295,7 +295,7 @@ mod stmt_insert {
     use {
         super::*,
         crate::engine::ql::{
-            ast::{parse_ast_node_full, Entity},
+            ast::parse_ast_node_full,
             dml::{self, ins::InsertStatement},
             lex::Ident,
         },
@@ -311,7 +311,7 @@ mod stmt_insert {
         .unwrap();
         let r = parse_ast_node_full::<InsertStatement>(&x[1..]).unwrap();
         let e = InsertStatement::new(
-            Entity::Full(Ident::from("twitter"), Ident::from("users")),
+            ("twitter", "users").into(),
             into_array_nullable!["sayan"].to_vec().into(),
         );
         assert_eq!(e, r);
@@ -333,7 +333,7 @@ mod stmt_insert {
         .unwrap();
         let r = parse_ast_node_full::<InsertStatement>(&x[1..]).unwrap();
         let e = InsertStatement::new(
-            Entity::Full(Ident::from("twitter"), Ident::from("users")),
+            ("twitter", "users").into(),
             into_array_nullable!["sayan", "Sayan", "sayan@example.com", true, 12345, 67890]
                 .to_vec()
                 .into(),
@@ -360,7 +360,7 @@ mod stmt_insert {
         .unwrap();
         let r = parse_ast_node_full::<InsertStatement>(&x[1..]).unwrap();
         let e = InsertStatement::new(
-            Entity::Full(Ident::from("twitter"), Ident::from("users")),
+            ("twitter", "users").into(),
             into_array_nullable![
                 "sayan",
                 "Sayan",
@@ -387,7 +387,7 @@ mod stmt_insert {
         .unwrap();
         let r = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let e = InsertStatement::new(
-            Entity::Full(Ident::from("jotsy"), Ident::from("app")),
+            ("jotsy", "app").into(),
             dict_nullable! {
                 Ident::from("username") => "sayan"
             }
@@ -412,7 +412,7 @@ mod stmt_insert {
         .unwrap();
         let r = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let e = InsertStatement::new(
-            Entity::Full(Ident::from("jotsy"), Ident::from("app")),
+            ("jotsy", "app").into(),
             dict_nullable! {
                 Ident::from("username") => "sayan",
                 Ident::from("name") => "Sayan",
@@ -445,7 +445,7 @@ mod stmt_insert {
         .unwrap();
         let r = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let e = InsertStatement::new(
-            Entity::Full(Ident::from("jotsy"), Ident::from("app")),
+            ("jotsy", "app").into(),
             dict_nullable! {
                 Ident::from("username") => "sayan",
                 "password" => "pass123",
@@ -467,7 +467,7 @@ mod stmt_insert {
             lex_insecure(br#"insert into jotsy.app(@uuidstr(), "sayan", @timesec())"#).unwrap();
         let ret = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let expected = InsertStatement::new(
-            Entity::Full(Ident::from("jotsy"), Ident::from("app")),
+            ("jotsy", "app").into(),
             into_array_nullable![dml::ins::T_UUIDSTR, "sayan", dml::ins::T_TIMESEC]
                 .to_vec()
                 .into(),
@@ -481,7 +481,7 @@ mod stmt_insert {
         ).unwrap();
         let ret = parse_ast_node_full::<InsertStatement>(&tok[1..]).unwrap();
         let expected = InsertStatement::new(
-            Entity::Full(Ident::from("jotsy"), Ident::from("app")),
+            ("jotsy", "app").into(),
             dict_nullable! {
                 "uuid" => dml::ins::T_UUIDSTR,
                 Ident::from("username") => "sayan",
@@ -499,7 +499,7 @@ mod stmt_select {
         crate::engine::{
             data::lit::Lit,
             ql::{
-                ast::{parse_ast_node_full, Entity},
+                ast::{parse_ast_node_full, parse_ast_node_full_with_space},
                 dml::{sel::SelectStatement, RelationalExpr},
                 lex::Ident,
             },
@@ -513,9 +513,9 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = parse_ast_node_full::<SelectStatement>(&tok[1..]).unwrap();
+        let r = parse_ast_node_full_with_space::<SelectStatement>(&tok[1..], "apps").unwrap();
         let e = SelectStatement::new_test(
-            Entity::Single(Ident::from("users")),
+            ("apps", "users").into(),
             [].to_vec(),
             true,
             dict! {
@@ -534,9 +534,9 @@ mod stmt_select {
             "#,
         )
         .unwrap();
-        let r = parse_ast_node_full::<SelectStatement>(&tok[1..]).unwrap();
+        let r = parse_ast_node_full_with_space::<SelectStatement>(&tok[1..], "apps").unwrap();
         let e = SelectStatement::new_test(
-            Entity::Single(Ident::from("users")),
+            ("apps", "users").into(),
             [Ident::from("field1")].to_vec(),
             false,
             dict! {
@@ -557,7 +557,7 @@ mod stmt_select {
         .unwrap();
         let r = parse_ast_node_full::<SelectStatement>(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
-            Entity::Full(Ident::from("twitter"), Ident::from("users")),
+            ("twitter", "users").into(),
             [Ident::from("field1")].to_vec(),
             false,
             dict! {
@@ -578,7 +578,7 @@ mod stmt_select {
         .unwrap();
         let r = parse_ast_node_full::<SelectStatement>(&tok[1..]).unwrap();
         let e = SelectStatement::new_test(
-            Entity::Full(Ident::from("twitter"), Ident::from("users")),
+            ("twitter", "users").into(),
             [Ident::from("field1"), Ident::from("field2")].to_vec(),
             false,
             dict! {
@@ -672,7 +672,7 @@ mod update_statement {
             core::query_meta::AssignmentOperator,
             data::lit::Lit,
             ql::{
-                ast::{parse_ast_node_full, Entity},
+                ast::{parse_ast_node_full, parse_ast_node_full_with_space},
                 dml::{
                     upd::{AssignmentExpression, UpdateStatement},
                     RelationalExpr, WhereClause,
@@ -689,9 +689,9 @@ mod update_statement {
             "#,
         )
         .unwrap();
-        let r = parse_ast_node_full::<UpdateStatement>(&tok[1..]).unwrap();
+        let r = parse_ast_node_full_with_space::<UpdateStatement>(&tok[1..], "apps").unwrap();
         let e = UpdateStatement::new(
-            Entity::Single(Ident::from("app")),
+            ("apps", "app").into(),
             vec![AssignmentExpression::new(
                 Ident::from("notes"),
                 Lit::new_str("this is my new note"),
@@ -723,7 +723,7 @@ mod update_statement {
         .unwrap();
         let r = parse_ast_node_full::<UpdateStatement>(&tok[1..]).unwrap();
         let e = UpdateStatement::new(
-            Entity::Full(Ident::from("jotsy"), Ident::from("app")),
+            ("jotsy", "app").into(),
             vec![
                 AssignmentExpression::new(
                     Ident::from("notes"),
@@ -753,7 +753,7 @@ mod delete_stmt {
         crate::engine::{
             data::lit::Lit,
             ql::{
-                ast::{parse_ast_node_full, Entity},
+                ast::{parse_ast_node_full, parse_ast_node_full_with_space},
                 dml::{del::DeleteStatement, RelationalExpr},
                 lex::Ident,
             },
@@ -769,7 +769,7 @@ mod delete_stmt {
         )
         .unwrap();
         let e = DeleteStatement::new_test(
-            Entity::Single(Ident::from("users")),
+            ("apps", "users").into(),
             dict! {
                 Ident::from("username") => RelationalExpr::new(
                     Ident::from("username"),
@@ -779,7 +779,7 @@ mod delete_stmt {
             },
         );
         assert_eq!(
-            parse_ast_node_full::<DeleteStatement>(&tok[1..]).unwrap(),
+            parse_ast_node_full_with_space::<DeleteStatement>(&tok[1..], "apps").unwrap(),
             e
         );
     }
@@ -792,7 +792,7 @@ mod delete_stmt {
         )
         .unwrap();
         let e = DeleteStatement::new_test(
-            Entity::Full(Ident::from("twitter"), Ident::from("users")),
+            ("twitter", "users").into(),
             dict! {
                 Ident::from("username") => RelationalExpr::new(
                     Ident::from("username"),
diff --git a/server/src/engine/ql/tests/entity.rs b/server/src/engine/ql/tests/misc.rs
similarity index 59%
rename from server/src/engine/ql/tests/entity.rs
rename to server/src/engine/ql/tests/misc.rs
index 7999d36e..f9837884 100644
--- a/server/src/engine/ql/tests/entity.rs
+++ b/server/src/engine/ql/tests/misc.rs
@@ -25,17 +25,51 @@
 */
 
 use super::*;
-use crate::engine::ql::{ast::Entity, lex::Ident};
+use crate::engine::ql::{
+    ast::{traits::ASTNode, State},
+    ddl::Use,
+};
+
+/*
+    entity
+*/
+
 #[test]
 fn entity_current() {
     let t = lex_insecure(b"hello").unwrap();
-    let r = Entity::parse_from_tokens_len_checked(&t, &mut 0).unwrap();
-    assert_eq!(r, Entity::Single(Ident::from("hello")))
+    let mut state = State::new_inplace(&t);
+    state.set_space("apps");
+    let r = state.try_entity_ref().unwrap();
+    assert_eq!(r, ("apps", "hello").into());
 }
 
 #[test]
 fn entity_full() {
     let t = lex_insecure(b"hello.world").unwrap();
-    let r = Entity::parse_from_tokens_len_checked(&t, &mut 0).unwrap();
-    assert_eq!(r, Entity::Full(Ident::from("hello"), Ident::from("world")))
+    let mut state = State::new_inplace(&t);
+    assert_eq!(
+        state.try_entity_ref().unwrap(),
+        (("hello"), ("world")).into()
+    )
+}
+
+/*
+    use
+*/
+
+#[test]
+fn use_new() {
+    let t = lex_insecure(b"use myspace").unwrap();
+    let mut state = State::new_inplace(&t);
+    assert_eq!(
+        Use::test_parse_from_state(&mut state).unwrap(),
+        Use::Space("myspace".into())
+    );
+}
+
+#[test]
+fn use_null() {
+    let t = lex_insecure(b"use null").unwrap();
+    let mut state = State::new_inplace(&t);
+    assert_eq!(Use::test_parse_from_state(&mut state).unwrap(), Use::Null);
 }
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index bb0e1acf..6ee020b8 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -28,44 +28,6 @@ use {
     super::{super::lex::Ident, lex_insecure, *},
     crate::engine::data::lit::Lit,
 };
-mod inspect {
-    use {
-        super::*,
-        crate::engine::ql::{
-            ast::{parse_ast_node_full, Entity, Statement},
-            ddl::ins::InspectStatementAST,
-        },
-    };
-    #[test]
-    fn inspect_space() {
-        let tok = lex_insecure(b"inspect space myspace").unwrap();
-        assert_eq!(
-            parse_ast_node_full::<InspectStatementAST>(&tok[1..]).unwrap(),
-            Statement::InspectSpace(Ident::from("myspace"))
-        );
-    }
-    #[test]
-    fn inspect_model() {
-        let tok = lex_insecure(b"inspect model users").unwrap();
-        assert_eq!(
-            parse_ast_node_full::<InspectStatementAST>(&tok[1..]).unwrap(),
-            Statement::InspectModel(Entity::Single(Ident::from("users")))
-        );
-        let tok = lex_insecure(b"inspect model tweeter.users").unwrap();
-        assert_eq!(
-            parse_ast_node_full::<InspectStatementAST>(&tok[1..]).unwrap(),
-            Statement::InspectModel(Entity::Full(Ident::from("tweeter"), Ident::from("users")))
-        );
-    }
-    #[test]
-    fn inspect_spaces() {
-        let tok = lex_insecure(b"inspect spaces").unwrap();
-        assert_eq!(
-            parse_ast_node_full::<InspectStatementAST>(&tok[1..]).unwrap(),
-            Statement::InspectSpaces
-        );
-    }
-}
 
 mod alter_space {
     use {
@@ -417,7 +379,7 @@ mod fields {
 mod schemas {
     use super::*;
     use crate::engine::ql::{
-        ast::{parse_ast_node_full, Entity},
+        ast::parse_ast_node_full_with_space,
         ddl::{
             crt::CreateModel,
             syn::{FieldSpec, LayerSpec},
@@ -437,12 +399,12 @@ mod schemas {
         let tok = &tok[2..];
 
         // parse model
-        let model = parse_ast_node_full::<CreateModel>(tok).unwrap();
+        let model = parse_ast_node_full_with_space::<CreateModel>(tok, "apps").unwrap();
 
         assert_eq!(
             model,
             CreateModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 vec![
                     FieldSpec::new(
                         Ident::from("username"),
@@ -476,12 +438,12 @@ mod schemas {
         let tok = &tok[2..];
 
         // parse model
-        let model = parse_ast_node_full::<CreateModel>(tok).unwrap();
+        let model = parse_ast_node_full_with_space::<CreateModel>(tok, "apps").unwrap();
 
         assert_eq!(
             model,
             CreateModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 vec![
                     FieldSpec::new(
                         Ident::from("username"),
@@ -526,12 +488,12 @@ mod schemas {
         let tok = &tok[2..];
 
         // parse model
-        let model = parse_ast_node_full::<CreateModel>(tok).unwrap();
+        let model = parse_ast_node_full_with_space::<CreateModel>(tok, "apps").unwrap();
 
         assert_eq!(
             model,
             CreateModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 vec![
                     FieldSpec::new(
                         Ident::from("username"),
@@ -595,12 +557,12 @@ mod schemas {
         let tok = &tok[2..];
 
         // parse model
-        let model = parse_ast_node_full::<CreateModel>(tok).unwrap();
+        let model = parse_ast_node_full_with_space::<CreateModel>(tok, "apps").unwrap();
 
         assert_eq!(
             model,
             CreateModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 vec![
                     FieldSpec::new(
                         Ident::from("username"),
@@ -768,18 +730,18 @@ mod dict_field_syntax {
 mod alter_model_remove {
     use super::*;
     use crate::engine::ql::{
-        ast::{parse_ast_node_full, Entity},
+        ast::parse_ast_node_full_with_space,
         ddl::alt::{AlterKind, AlterModel},
         lex::Ident,
     };
     #[test]
     fn alter_mini() {
         let tok = lex_insecure(b"alter model mymodel remove myfield").unwrap();
-        let remove = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+        let remove = parse_ast_node_full_with_space::<AlterModel>(&tok[2..], "apps").unwrap();
         assert_eq!(
             remove,
             AlterModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 AlterKind::Remove(Box::from([Ident::from("myfield")]))
             )
         );
@@ -787,11 +749,11 @@ mod alter_model_remove {
     #[test]
     fn alter_mini_2() {
         let tok = lex_insecure(b"alter model mymodel remove (myfield)").unwrap();
-        let remove = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+        let remove = parse_ast_node_full_with_space::<AlterModel>(&tok[2..], "apps").unwrap();
         assert_eq!(
             remove,
             AlterModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 AlterKind::Remove(Box::from([Ident::from("myfield")]))
             )
         );
@@ -801,11 +763,11 @@ mod alter_model_remove {
         let tok =
             lex_insecure(b"alter model mymodel remove (myfield1, myfield2, myfield3, myfield4)")
                 .unwrap();
-        let remove = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+        let remove = parse_ast_node_full_with_space::<AlterModel>(&tok[2..], "apps").unwrap();
         assert_eq!(
             remove,
             AlterModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 AlterKind::Remove(Box::from([
                     Ident::from("myfield1"),
                     Ident::from("myfield2"),
@@ -819,7 +781,7 @@ mod alter_model_remove {
 mod alter_model_add {
     use super::*;
     use crate::engine::ql::{
-        ast::{parse_ast_node_full, Entity},
+        ast::parse_ast_node_full_with_space,
         ddl::{
             alt::{AlterKind, AlterModel},
             syn::{ExpandedField, LayerSpec},
@@ -834,9 +796,9 @@ mod alter_model_add {
         )
         .unwrap();
         assert_eq!(
-            parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap(),
+            parse_ast_node_full_with_space::<AlterModel>(&tok[2..], "apps").unwrap(),
             AlterModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 AlterKind::Add(
                     [ExpandedField::new(
                         Ident::from("myfield"),
@@ -856,11 +818,11 @@ mod alter_model_add {
             ",
         )
         .unwrap();
-        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+        let r = parse_ast_node_full_with_space::<AlterModel>(&tok[2..], "apps").unwrap();
         assert_eq!(
             r,
             AlterModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 AlterKind::Add(
                     [ExpandedField::new(
                         Ident::from("myfield"),
@@ -882,11 +844,11 @@ mod alter_model_add {
             ",
         )
         .unwrap();
-        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+        let r = parse_ast_node_full_with_space::<AlterModel>(&tok[2..], "apps").unwrap();
         assert_eq!(
             r,
             AlterModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 AlterKind::Add(
                     [ExpandedField::new(
                         Ident::from("myfield"),
@@ -922,11 +884,11 @@ mod alter_model_add {
             ",
         )
         .unwrap();
-        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+        let r = parse_ast_node_full_with_space::<AlterModel>(&tok[2..], "apps").unwrap();
         assert_eq!(
             r,
             AlterModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 AlterKind::Add(
                     [
                         ExpandedField::new(
@@ -967,7 +929,7 @@ mod alter_model_add {
 mod alter_model_update {
     use super::*;
     use crate::engine::ql::{
-        ast::{parse_ast_node_full, Entity},
+        ast::parse_ast_node_full_with_space,
         ddl::{
             alt::{AlterKind, AlterModel},
             syn::{ExpandedField, LayerSpec},
@@ -982,11 +944,11 @@ mod alter_model_update {
             ",
         )
         .unwrap();
-        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+        let r = parse_ast_node_full_with_space::<AlterModel>(&tok[2..], "apps").unwrap();
         assert_eq!(
             r,
             AlterModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 AlterKind::Update(
                     [ExpandedField::new(
                         Ident::from("myfield"),
@@ -1006,11 +968,11 @@ mod alter_model_update {
             ",
         )
         .unwrap();
-        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+        let r = parse_ast_node_full_with_space::<AlterModel>(&tok[2..], "apps").unwrap();
         assert_eq!(
             r,
             AlterModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 AlterKind::Update(
                     [ExpandedField::new(
                         Ident::from("myfield"),
@@ -1035,11 +997,11 @@ mod alter_model_update {
             ",
         )
         .unwrap();
-        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+        let r = parse_ast_node_full_with_space::<AlterModel>(&tok[2..], "apps").unwrap();
         assert_eq!(
             r,
             AlterModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 AlterKind::Update(
                     [ExpandedField::new(
                         Ident::from("myfield"),
@@ -1069,11 +1031,11 @@ mod alter_model_update {
             ",
         )
         .unwrap();
-        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+        let r = parse_ast_node_full_with_space::<AlterModel>(&tok[2..], "apps").unwrap();
         assert_eq!(
             r,
             AlterModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 AlterKind::Update(
                     [
                         ExpandedField::new(
@@ -1112,11 +1074,11 @@ mod alter_model_update {
             ",
         )
         .unwrap();
-        let r = parse_ast_node_full::<AlterModel>(&tok[2..]).unwrap();
+        let r = parse_ast_node_full_with_space::<AlterModel>(&tok[2..], "apps").unwrap();
         assert_eq!(
             r,
             AlterModel::new(
-                Entity::Single(Ident::from("mymodel")),
+                ("apps", "mymodel").into(),
                 AlterKind::Update(
                     [
                         ExpandedField::new(
@@ -1147,7 +1109,7 @@ mod ddl_other_query_tests {
     use {
         super::*,
         crate::engine::ql::{
-            ast::{parse_ast_node_full, Entity, Statement},
+            ast::{parse_ast_node_full, parse_ast_node_full_with_space, Statement},
             ddl::drop::{DropModel, DropSpace, DropStatementAST},
             lex::Ident,
         },
@@ -1172,19 +1134,16 @@ mod ddl_other_query_tests {
     fn drop_model() {
         let src = lex_insecure(br"drop model mymodel").unwrap();
         assert_eq!(
-            parse_ast_node_full::<DropStatementAST>(&src[1..]).unwrap(),
-            Statement::DropModel(DropModel::new(
-                Entity::Single(Ident::from("mymodel")),
-                false
-            ))
+            parse_ast_node_full_with_space::<DropStatementAST>(&src[1..], "apps").unwrap(),
+            Statement::DropModel(DropModel::new(("apps", "mymodel").into(), false))
         );
     }
     #[test]
     fn drop_model_force() {
         let src = lex_insecure(br"drop model mymodel force").unwrap();
         assert_eq!(
-            parse_ast_node_full::<DropStatementAST>(&src[1..]).unwrap(),
-            Statement::DropModel(DropModel::new(Entity::Single(Ident::from("mymodel")), true))
+            parse_ast_node_full_with_space::<DropStatementAST>(&src[1..], "apps").unwrap(),
+            Statement::DropModel(DropModel::new(("apps", "mymodel").into(), true))
         );
     }
 }
diff --git a/server/src/engine/storage/v1/inf/tests.rs b/server/src/engine/storage/v1/inf/tests.rs
index 9e1ce4fc..c8e2f7b0 100644
--- a/server/src/engine/storage/v1/inf/tests.rs
+++ b/server/src/engine/storage/v1/inf/tests.rs
@@ -99,7 +99,7 @@ fn model() {
     let model = Model::new_restore(
         uuid,
         "username".into(),
-        TagSelector::Str.into_full(),
+        TagSelector::String.into_full(),
         into_dict! {
             "password" => Field::new([Layer::bin()].into(), false),
             "profile_pic" => Field::new([Layer::bin()].into(), true),
diff --git a/server/src/engine/storage/v1/tests/batch.rs b/server/src/engine/storage/v1/tests/batch.rs
index 0cb4d24c..75ab5578 100644
--- a/server/src/engine/storage/v1/tests/batch.rs
+++ b/server/src/engine/storage/v1/tests/batch.rs
@@ -131,7 +131,7 @@ fn empty_multi_open_reopen() {
     let mdl = Model::new_restore(
         uuid,
         "username".into(),
-        TagSelector::Str.into_full(),
+        TagSelector::String.into_full(),
         into_dict!(
             "username" => Field::new([Layer::str()].into(), false),
             "password" => Field::new([Layer::bin()].into(), false)
@@ -149,7 +149,7 @@ fn unskewed_delta() {
     let mdl = Model::new_restore(
         uuid,
         "username".into(),
-        TagSelector::Str.into_full(),
+        TagSelector::String.into_full(),
         into_dict!(
             "username" => Field::new([Layer::str()].into(), false),
             "password" => Field::new([Layer::bin()].into(), false)
@@ -219,7 +219,7 @@ fn skewed_delta() {
     let mdl = Model::new_restore(
         uuid,
         "catname".into(),
-        TagSelector::Str.into_full(),
+        TagSelector::String.into_full(),
         into_dict!(
             "catname" => Field::new([Layer::str()].into(), false),
             "is_good" => Field::new([Layer::bool()].into(), false),
@@ -302,7 +302,7 @@ fn skewed_shuffled_persist_restore() {
     let model = Model::new_restore(
         uuid,
         "username".into(),
-        TagSelector::Str.into_full(),
+        TagSelector::String.into_full(),
         into_dict!("username" => Field::new([Layer::str()].into(), false), "password" => Field::new([Layer::str()].into(), false)),
     );
     let mongobongo = Row::new(
diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
index 191d663f..35be0bfb 100644
--- a/server/src/engine/txn/gns/tests/full_chain.rs
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -185,7 +185,7 @@ fn create_model() {
                         &Model::new_restore(
                             uuid_model,
                             "username".into(),
-                            TagSelector::Str.into_full(),
+                            TagSelector::String.into_full(),
                             into_dict! {
                                 "username" => Field::new([Layer::str()].into(), false),
                                 "password" => Field::new([Layer::bin()].into(), false),
diff --git a/server/src/engine/txn/gns/tests/io.rs b/server/src/engine/txn/gns/tests/io.rs
index 99ffc200..9a03c5dd 100644
--- a/server/src/engine/txn/gns/tests/io.rs
+++ b/server/src/engine/txn/gns/tests/io.rs
@@ -107,7 +107,7 @@ mod model_tests {
         let model = Model::new_restore(
             Uuid::new(),
             "username".into(),
-            TagSelector::Str.into_full(),
+            TagSelector::String.into_full(),
             into_dict!(
                 "password" => Field::new([Layer::bin()].into(), false),
                 "profile_pic" => Field::new([Layer::bin()].into(), true),
diff --git a/sky-bench/src/bench.rs b/sky-bench/src/bench.rs
index 4b9d435f..dc52f662 100644
--- a/sky-bench/src/bench.rs
+++ b/sky-bench/src/bench.rs
@@ -36,6 +36,9 @@ use {
     std::{fmt, time::Instant},
 };
 
+pub const BENCHMARK_SPACE_ID: &'static str = "bench";
+pub const BENCHMARK_MODEL_ID: &'static str = "bench";
+
 /*
     task impl
 */
@@ -90,7 +93,7 @@ impl BombardTaskSpec {
         }
     }
     pub fn generate(&self, current: u64) -> (Query, Response) {
-        let mut q = query!(&self.base_query);
+        let mut q = query!(self.base_query.as_str());
         let resp = match self.kind {
             BombardTaskKind::Insert(second_column) => {
                 self.push_pk(&mut q, current);
@@ -146,7 +149,9 @@ impl rookie::ThreadedBombardTask for BombardTask {
     type WorkerInitError = Error;
     type WorkerTaskError = BombardTaskError;
     fn worker_init(&self) -> Result<Self::Worker, Self::WorkerInitError> {
-        self.config.connect()
+        let mut db = self.config.connect()?;
+        db.query_parse::<()>(&skytable::query!(format!("use {BENCHMARK_SPACE_ID}")))
+            .map(|_| db)
     }
     fn generate_task(spec: &Self::WorkerTaskSpec, current: u64) -> Self::WorkerTask {
         spec.generate(current)
@@ -180,7 +185,9 @@ pub fn run(bench: BenchConfig) -> error::BenchResult<()> {
     info!("running preliminary checks and creating model `bench.bench` with definition: `{{un: string, pw: uint8}}`");
     let mut main_thread_db = bench_config.config.connect()?;
     main_thread_db.query_parse::<()>(&query!("create space bench"))?;
-    main_thread_db.query_parse::<()>(&query!("create model bench.bench(un: string, pw: uint8)"))?;
+    main_thread_db.query_parse::<()>(&query!(format!(
+        "create model {BENCHMARK_SPACE_ID}.{BENCHMARK_MODEL_ID}(un: string, pw: uint8)"
+    )))?;
     let stats = match bench.engine {
         BenchEngine::Rookie => bench_rookie(bench_config, bench),
         BenchEngine::Fury => bench_fury(bench),
@@ -278,23 +285,20 @@ fn prepare_bench_spec(bench: &BenchConfig) -> Vec<BenchItem> {
     vec![
         BenchItem::new(
             "INSERT",
-            BombardTaskSpec::insert("insert into bench.bench(?, ?)".into(), bench.key_size, 0),
+            BombardTaskSpec::insert("insert into bench(?, ?)".into(), bench.key_size, 0),
             bench.query_count,
         ),
         BenchItem::new(
             "UPDATE",
             BombardTaskSpec::update(
-                "update bench.bench set pw += ? where un = ?".into(),
+                "update bench set pw += ? where un = ?".into(),
                 bench.key_size,
             ),
             bench.query_count,
         ),
         BenchItem::new(
             "DELETE",
-            BombardTaskSpec::delete(
-                "delete from bench.bench where un = ?".into(),
-                bench.key_size,
-            ),
+            BombardTaskSpec::delete("delete from bench where un = ?".into(), bench.key_size),
             bench.query_count,
         ),
     ]
diff --git a/sky-bench/src/runtime/fury.rs b/sky-bench/src/runtime/fury.rs
index 53f0b2df..1d06e5de 100644
--- a/sky-bench/src/runtime/fury.rs
+++ b/sky-bench/src/runtime/fury.rs
@@ -26,7 +26,7 @@
 
 use {
     super::{RuntimeStats, WorkerLocalStats, WorkerTask},
-    crate::bench::BombardTaskSpec,
+    crate::bench::{BombardTaskSpec, BENCHMARK_SPACE_ID},
     skytable::Config,
     std::{
         fmt,
@@ -232,9 +232,9 @@ async fn worker_svc(
             return;
         }
     };
-    // warm up connection
+    // set DB in connections
     match db
-        .query_parse::<()>(&skytable::query!("sysctl report status"))
+        .query_parse::<()>(&skytable::query!(format!("use {BENCHMARK_SPACE_ID}")))
         .await
     {
         Ok(()) => {}

From c8a457c226c185aa1a1df21b9b7847edec9cc757 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 27 Nov 2023 21:17:24 +0530
Subject: [PATCH 294/310] Implement benches for SELECT

Also cleaned up unused AST impls
---
 server/src/engine/core/exec.rs             |   4 +-
 server/src/engine/net/protocol/mod.rs      |   7 +-
 server/src/engine/ql/ast/mod.rs            |  75 +---------
 server/src/engine/ql/ddl/drop.rs           |  29 +---
 server/src/engine/ql/tests/misc.rs         |   4 +-
 server/src/engine/ql/tests/schema_tests.rs |  20 +--
 sky-bench/src/bench.rs                     | 153 ++++++++++-----------
 sky-bench/src/runtime/fury.rs              |  79 +++++++----
 8 files changed, 150 insertions(+), 221 deletions(-)

diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 0ad1c5ca..8daf9dae 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -107,8 +107,8 @@ async fn run_blocking_stmt(
         &ClientLocalState,
         &mut State<'static, InplaceData>,
     ) -> QueryResult<()>; 8] = [
-        |_, _, _| Err(QueryError::QLUnknownStatement), // unknown
-        blocking_exec_sysctl,                          // sysctl
+        |_, _, _| Err(QueryError::QLUnknownStatement),
+        blocking_exec_sysctl,
         |g, _, t| _call(&g, t, Space::transactional_exec_create),
         |g, _, t| _call(&g, t, Model::transactional_exec_create),
         |g, _, t| _call(&g, t, Space::transactional_exec_alter),
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index cc2c5eec..c745086c 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -86,7 +86,12 @@ pub struct ClientLocalState {
 
 impl ClientLocalState {
     pub fn new(username: Box<str>, root: bool, hs: handshake::CHandshakeStatic) -> Self {
-        Self { username, root, hs, cs: None }
+        Self {
+            username,
+            root,
+            hs,
+            cs: None,
+        }
     }
     pub fn is_root(&self) -> bool {
         self.root
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index c2c29c99..7627ca2f 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -32,10 +32,7 @@ pub use traits::{
 };
 
 use {
-    super::{
-        ddl, dml,
-        lex::{Ident, Keyword, KeywordStmt, Token},
-    },
+    super::lex::{Keyword, KeywordStmt, Token},
     crate::{
         engine::{
             core::EntityIDRef,
@@ -374,73 +371,3 @@ impl<'a> QueryData<'a> for InplaceData {
         true
     }
 }
-
-#[derive(Debug, PartialEq)]
-#[allow(dead_code)] // TODO(@ohsayan): get rid of this
-/// A [`Statement`] is a fully BlueQL statement that can be executed by the query engine
-// TODO(@ohsayan): Determine whether we actually need this
-pub enum Statement<'a> {
-    /// DDL query to create a model
-    CreateModel(ddl::crt::CreateModel<'a>),
-    /// DDL query to create a space
-    CreateSpace(ddl::crt::CreateSpace<'a>),
-    /// DDL query to alter a space (properties)
-    AlterSpace(ddl::alt::AlterSpace<'a>),
-    /// DDL query to alter a model (properties, field types, etc)
-    AlterModel(ddl::alt::AlterModel<'a>),
-    /// DDL query to drop a model
-    ///
-    /// Conditions:
-    /// - Model view is empty
-    /// - Model is not in active use
-    DropModel(ddl::drop::DropModel<'a>),
-    /// DDL query to drop a space
-    ///
-    /// Conditions:
-    /// - Space doesn't have any other structures
-    /// - Space is not in active use
-    DropSpace(ddl::drop::DropSpace<'a>),
-    /// DDL query to inspect a space (returns a list of models in the space)
-    InspectSpace(Ident<'a>),
-    /// DDL query to inspect all spaces (returns a list of spaces in the database)
-    InspectSpaces,
-    /// DML insert
-    Insert(dml::ins::InsertStatement<'a>),
-    /// DML select
-    Select(dml::sel::SelectStatement<'a>),
-    /// DML update
-    Update(dml::upd::UpdateStatement<'a>),
-    /// DML delete
-    Delete(dml::del::DeleteStatement<'a>),
-}
-
-#[inline(always)]
-#[cfg(test)]
-#[allow(dead_code)] // TODO(@ohsayan): get rid of this
-pub fn compile<'a, Qd: QueryData<'a>>(tok: &'a [Token<'a>], d: Qd) -> QueryResult<Statement<'a>> {
-    use {self::traits::ASTNode, crate::util::compiler};
-    if compiler::unlikely(tok.len() < 2) {
-        return Err(QueryError::QLUnexpectedEndOfStatement);
-    }
-    let mut state = State::new(tok, d);
-    match state.fw_read() {
-        // DDL
-        Token![create] => match state.fw_read() {
-            Token![model] => ASTNode::test_parse_from_state(&mut state).map(Statement::CreateModel),
-            Token![space] => ASTNode::test_parse_from_state(&mut state).map(Statement::CreateSpace),
-            _ => compiler::cold_rerr(QueryError::QLUnknownStatement),
-        },
-        Token![alter] => match state.fw_read() {
-            Token![model] => ASTNode::test_parse_from_state(&mut state).map(Statement::AlterModel),
-            Token![space] => ASTNode::test_parse_from_state(&mut state).map(Statement::AlterSpace),
-            _ => compiler::cold_rerr(QueryError::QLUnknownStatement),
-        },
-        Token![drop] if state.remaining() >= 2 => ddl::drop::parse_drop(&mut state),
-        // DML
-        Token![insert] => ASTNode::test_parse_from_state(&mut state).map(Statement::Insert),
-        Token![select] => ASTNode::test_parse_from_state(&mut state).map(Statement::Select),
-        Token![update] => ASTNode::test_parse_from_state(&mut state).map(Statement::Update),
-        Token![delete] => ASTNode::test_parse_from_state(&mut state).map(Statement::Delete),
-        _ => compiler::cold_rerr(QueryError::QLUnknownStatement),
-    }
-}
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index 2497e518..1e77a8d2 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -28,7 +28,7 @@ use crate::engine::{
     core::EntityIDRef,
     error::{QueryError, QueryResult},
     ql::{
-        ast::{QueryData, State, Statement},
+        ast::{QueryData, State},
         lex::{Ident, Token},
     },
 };
@@ -84,26 +84,12 @@ impl<'a> DropModel<'a> {
     }
 }
 
-// drop (<space> | <model>) <ident> [<force>]
-/// ## Panic
-///
-/// If token stream length is < 2
-pub fn parse_drop<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Statement<'a>> {
-    match state.fw_read() {
-        Token![model] => DropModel::parse(state).map(Statement::DropModel),
-        Token![space] => return DropSpace::parse(state).map(Statement::DropSpace),
-        _ => Err(QueryError::QLUnknownStatement),
-    }
-}
-
-#[cfg(test)]
-pub use impls::DropStatementAST;
 mod impls {
     use {
         super::{DropModel, DropSpace},
         crate::engine::{
             error::QueryResult,
-            ql::ast::{traits::ASTNode, QueryData, State, Statement},
+            ql::ast::{traits::ASTNode, QueryData, State},
         },
     };
     impl<'a> ASTNode<'a> for DropModel<'a> {
@@ -124,15 +110,4 @@ mod impls {
             Self::parse(state)
         }
     }
-    #[derive(sky_macros::Wrapper, Debug)]
-    pub struct DropStatementAST<'a>(Statement<'a>);
-    impl<'a> ASTNode<'a> for DropStatementAST<'a> {
-        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
-        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
-        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
-            state: &mut State<'a, Qd>,
-        ) -> QueryResult<Self> {
-            super::parse_drop(state).map(Self)
-        }
-    }
 }
diff --git a/server/src/engine/ql/tests/misc.rs b/server/src/engine/ql/tests/misc.rs
index f9837884..70b379cd 100644
--- a/server/src/engine/ql/tests/misc.rs
+++ b/server/src/engine/ql/tests/misc.rs
@@ -60,7 +60,7 @@ fn entity_full() {
 #[test]
 fn use_new() {
     let t = lex_insecure(b"use myspace").unwrap();
-    let mut state = State::new_inplace(&t);
+    let mut state = State::new_inplace(&t[1..]);
     assert_eq!(
         Use::test_parse_from_state(&mut state).unwrap(),
         Use::Space("myspace".into())
@@ -70,6 +70,6 @@ fn use_new() {
 #[test]
 fn use_null() {
     let t = lex_insecure(b"use null").unwrap();
-    let mut state = State::new_inplace(&t);
+    let mut state = State::new_inplace(&t[1..]);
     assert_eq!(Use::test_parse_from_state(&mut state).unwrap(), Use::Null);
 }
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 6ee020b8..9a32fe31 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -1109,8 +1109,8 @@ mod ddl_other_query_tests {
     use {
         super::*,
         crate::engine::ql::{
-            ast::{parse_ast_node_full, parse_ast_node_full_with_space, Statement},
-            ddl::drop::{DropModel, DropSpace, DropStatementAST},
+            ast::{parse_ast_node_full, parse_ast_node_full_with_space},
+            ddl::drop::{DropModel, DropSpace},
             lex::Ident,
         },
     };
@@ -1118,32 +1118,32 @@ mod ddl_other_query_tests {
     fn drop_space() {
         let src = lex_insecure(br"drop space myspace").unwrap();
         assert_eq!(
-            parse_ast_node_full::<DropStatementAST>(&src[1..]).unwrap(),
-            Statement::DropSpace(DropSpace::new(Ident::from("myspace"), false))
+            parse_ast_node_full::<DropSpace>(&src[2..]).unwrap(),
+            DropSpace::new(Ident::from("myspace"), false)
         );
     }
     #[test]
     fn drop_space_force() {
         let src = lex_insecure(br"drop space myspace force").unwrap();
         assert_eq!(
-            parse_ast_node_full::<DropStatementAST>(&src[1..]).unwrap(),
-            Statement::DropSpace(DropSpace::new(Ident::from("myspace"), true))
+            parse_ast_node_full::<DropSpace>(&src[2..]).unwrap(),
+            DropSpace::new(Ident::from("myspace"), true)
         );
     }
     #[test]
     fn drop_model() {
         let src = lex_insecure(br"drop model mymodel").unwrap();
         assert_eq!(
-            parse_ast_node_full_with_space::<DropStatementAST>(&src[1..], "apps").unwrap(),
-            Statement::DropModel(DropModel::new(("apps", "mymodel").into(), false))
+            parse_ast_node_full_with_space::<DropModel>(&src[2..], "apps").unwrap(),
+            DropModel::new(("apps", "mymodel").into(), false)
         );
     }
     #[test]
     fn drop_model_force() {
         let src = lex_insecure(br"drop model mymodel force").unwrap();
         assert_eq!(
-            parse_ast_node_full_with_space::<DropStatementAST>(&src[1..], "apps").unwrap(),
-            Statement::DropModel(DropModel::new(("apps", "mymodel").into(), true))
+            parse_ast_node_full_with_space::<DropModel>(&src[2..], "apps").unwrap(),
+            DropModel::new(("apps", "mymodel").into(), true)
         );
     }
 }
diff --git a/sky-bench/src/bench.rs b/sky-bench/src/bench.rs
index dc52f662..e155e6a1 100644
--- a/sky-bench/src/bench.rs
+++ b/sky-bench/src/bench.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use skytable::response::Value;
+
 use crate::args::BenchEngine;
 
 use {
@@ -56,70 +58,6 @@ impl BombardTask {
     }
 }
 
-#[derive(Debug, Clone)]
-pub enum BombardTaskKind {
-    Insert(u8),
-    Update,
-    Delete,
-}
-
-#[derive(Debug, Clone)]
-pub struct BombardTaskSpec {
-    kind: BombardTaskKind,
-    base_query: String,
-    pk_len: usize,
-}
-
-impl BombardTaskSpec {
-    pub fn insert(base_query: String, pk_len: usize, second_column: u8) -> Self {
-        Self {
-            kind: BombardTaskKind::Insert(second_column),
-            base_query,
-            pk_len,
-        }
-    }
-    pub fn update(base_query: String, pk_len: usize) -> Self {
-        Self {
-            kind: BombardTaskKind::Update,
-            base_query,
-            pk_len,
-        }
-    }
-    pub fn delete(base_query: String, pk_len: usize) -> Self {
-        Self {
-            kind: BombardTaskKind::Delete,
-            base_query,
-            pk_len,
-        }
-    }
-    pub fn generate(&self, current: u64) -> (Query, Response) {
-        let mut q = query!(self.base_query.as_str());
-        let resp = match self.kind {
-            BombardTaskKind::Insert(second_column) => {
-                self.push_pk(&mut q, current);
-                q.push_param(second_column);
-                Response::Empty
-            }
-            BombardTaskKind::Update => {
-                q.push_param(1u64);
-                self.push_pk(&mut q, current);
-                Response::Empty
-            }
-            BombardTaskKind::Delete => {
-                self.push_pk(&mut q, current);
-                Response::Empty
-            }
-        };
-        (q, resp)
-    }
-    fn push_pk(&self, q: &mut Query, current: u64) {
-        q.push_param(self.get_primary_key(current));
-    }
-    fn get_primary_key(&self, current: u64) -> String {
-        format!("{:0>width$}", current, width = self.pk_len)
-    }
-}
-
 /// Errors while running a bombard
 #[derive(Debug)]
 pub enum BombardTaskError {
@@ -144,8 +82,8 @@ impl From<Error> for BombardTaskError {
 
 impl rookie::ThreadedBombardTask for BombardTask {
     type Worker = Connection;
-    type WorkerTask = (Query, Response);
-    type WorkerTaskSpec = BombardTaskSpec;
+    type WorkerTask = (Query, (BenchmarkTask, u64));
+    type WorkerTaskSpec = BenchmarkTask;
     type WorkerInitError = Error;
     type WorkerTaskError = BombardTaskError;
     fn worker_init(&self) -> Result<Self::Worker, Self::WorkerInitError> {
@@ -154,16 +92,16 @@ impl rookie::ThreadedBombardTask for BombardTask {
             .map(|_| db)
     }
     fn generate_task(spec: &Self::WorkerTaskSpec, current: u64) -> Self::WorkerTask {
-        spec.generate(current)
+        (spec.generate_query(current), (*spec, current))
     }
     fn worker_drive_timed(
         worker: &mut Self::Worker,
-        (query, response): Self::WorkerTask,
+        (query, (spec, current)): Self::WorkerTask,
     ) -> Result<u128, Self::WorkerTaskError> {
         let start = Instant::now();
         let ret = worker.query(&query)?;
         let stop = Instant::now();
-        if ret == response {
+        if spec.verify_response(current, ret) {
             Ok(stop.duration_since(start).as_nanos())
         } else {
             Err(BombardTaskError::Mismatch)
@@ -182,11 +120,11 @@ pub fn run(bench: BenchConfig) -> error::BenchResult<()> {
         "root",
         &bench.root_pass,
     ));
-    info!("running preliminary checks and creating model `bench.bench` with definition: `{{un: string, pw: uint8}}`");
+    info!("running preliminary checks and creating model `bench.bench` with definition: `{{un: binary, pw: uint8}}`");
     let mut main_thread_db = bench_config.config.connect()?;
     main_thread_db.query_parse::<()>(&query!("create space bench"))?;
     main_thread_db.query_parse::<()>(&query!(format!(
-        "create model {BENCHMARK_SPACE_ID}.{BENCHMARK_MODEL_ID}(un: string, pw: uint8)"
+        "create model {BENCHMARK_SPACE_ID}.{BENCHMARK_MODEL_ID}(un: binary, pw: uint8)"
     )))?;
     let stats = match bench.engine {
         BenchEngine::Rookie => bench_rookie(bench_config, bench),
@@ -252,21 +190,51 @@ fn print_table(data: Vec<(&'static str, RuntimeStats)>) {
     bench runner
 */
 
+#[derive(Clone, Copy, Debug)]
+pub struct BenchmarkTask {
+    gen_query: fn(&Self, u64) -> Query,
+    check_resp: fn(&Self, u64, Response) -> bool,
+    pk_len: usize,
+}
+
+impl BenchmarkTask {
+    fn new(
+        pk_len: usize,
+        gen_query: fn(&Self, u64) -> Query,
+        check_resp: fn(&Self, u64, Response) -> bool,
+    ) -> Self {
+        Self {
+            gen_query,
+            check_resp,
+            pk_len,
+        }
+    }
+    fn fmt_pk(&self, current: u64) -> Vec<u8> {
+        format!("{:0>width$}", current, width = self.pk_len).into_bytes()
+    }
+    pub fn generate_query(&self, current: u64) -> Query {
+        (self.gen_query)(self, current)
+    }
+    pub fn verify_response(&self, current: u64, resp: Response) -> bool {
+        (self.check_resp)(self, current, resp)
+    }
+}
+
 struct BenchItem {
     name: &'static str,
-    spec: BombardTaskSpec,
+    spec: BenchmarkTask,
     count: usize,
 }
 
 impl BenchItem {
-    fn new(name: &'static str, spec: BombardTaskSpec, count: usize) -> Self {
+    fn new(name: &'static str, spec: BenchmarkTask, count: usize) -> Self {
         Self { name, spec, count }
     }
     fn print_log_start(&self) {
         info!(
             "benchmarking `{}`. average payload size = {} bytes. queries = {}",
             self.name,
-            self.spec.generate(0).0.debug_encode_packet().len(),
+            self.spec.generate_query(0).debug_encode_packet().len(),
             self.count
         )
     }
@@ -285,20 +253,49 @@ fn prepare_bench_spec(bench: &BenchConfig) -> Vec<BenchItem> {
     vec![
         BenchItem::new(
             "INSERT",
-            BombardTaskSpec::insert("insert into bench(?, ?)".into(), bench.key_size, 0),
+            BenchmarkTask::new(
+                bench.key_size,
+                |me, current| query!("insert into bench(?, ?)", me.fmt_pk(current), 0u64),
+                |_, _, actual_resp| actual_resp == Response::Empty,
+            ),
+            bench.query_count,
+        ),
+        BenchItem::new(
+            "SELECT",
+            BenchmarkTask::new(
+                bench.key_size,
+                |me, current| query!("select * from bench where un = ?", me.fmt_pk(current)),
+                |me, current, resp| match resp {
+                    Response::Row(r) => {
+                        r.into_values() == vec![Value::Binary(me.fmt_pk(current)), Value::UInt8(0)]
+                    }
+                    _ => false,
+                },
+            ),
             bench.query_count,
         ),
         BenchItem::new(
             "UPDATE",
-            BombardTaskSpec::update(
-                "update bench set pw += ? where un = ?".into(),
+            BenchmarkTask::new(
                 bench.key_size,
+                |me, current| {
+                    query!(
+                        "update bench set pw += ? where un = ?",
+                        1u64,
+                        me.fmt_pk(current)
+                    )
+                },
+                |_, _, resp| resp == Response::Empty,
             ),
             bench.query_count,
         ),
         BenchItem::new(
             "DELETE",
-            BombardTaskSpec::delete("delete from bench where un = ?".into(), bench.key_size),
+            BenchmarkTask::new(
+                bench.key_size,
+                |me, current| query!("delete from bench where un = ?", me.fmt_pk(current)),
+                |_, _, resp| resp == Response::Empty,
+            ),
             bench.query_count,
         ),
     ]
diff --git a/sky-bench/src/runtime/fury.rs b/sky-bench/src/runtime/fury.rs
index 1d06e5de..02fa7a36 100644
--- a/sky-bench/src/runtime/fury.rs
+++ b/sky-bench/src/runtime/fury.rs
@@ -26,7 +26,7 @@
 
 use {
     super::{RuntimeStats, WorkerLocalStats, WorkerTask},
-    crate::bench::{BombardTaskSpec, BENCHMARK_SPACE_ID},
+    crate::bench::{BenchmarkTask, BENCHMARK_SPACE_ID},
     skytable::Config,
     std::{
         fmt,
@@ -121,7 +121,7 @@ impl fmt::Display for FuryWorkerError {
 
 #[derive(Debug)]
 pub struct Fury {
-    tx_task: broadcast::Sender<WorkerTask<BombardTaskSpec>>,
+    tx_task: broadcast::Sender<WorkerTask<BenchmarkTask>>,
     rx_task_result: mpsc::Receiver<FuryResult<WorkerLocalStats>>,
     client_count: usize,
 }
@@ -131,12 +131,14 @@ impl Fury {
         let (tx_task, rx_task) = broadcast::channel(1);
         let (tx_task_result, rx_task_result) = mpsc::channel(client_count);
         let (tx_ack, mut rx_ack) = mpsc::channel(1);
-        for _ in 0..client_count {
+        for id in 0..client_count {
             let rx_task = tx_task.subscribe();
             let tx_task_result = tx_task_result.clone();
             let tx_ack = tx_ack.clone();
             let config = config.clone();
-            tokio::spawn(async move { worker_svc(rx_task, tx_task_result, tx_ack, config).await });
+            tokio::spawn(
+                async move { worker_svc(id, rx_task, tx_task_result, tx_ack, config).await },
+            );
         }
         drop((tx_ack, rx_task));
         match rx_ack.recv().await {
@@ -150,16 +152,14 @@ impl Fury {
             client_count,
         })
     }
-    pub async fn bombard(
-        &mut self,
-        count: usize,
-        task: BombardTaskSpec,
-    ) -> FuryResult<RuntimeStats> {
+    pub async fn bombard(&mut self, count: usize, task: BenchmarkTask) -> FuryResult<RuntimeStats> {
         // pause workers and set target
         let start_guard = GLOBAL_START.write().await;
         gset_target(count);
         // send tasks
-        self.tx_task.send(WorkerTask::Task(task)).unwrap();
+        if self.tx_task.send(WorkerTask::Task(task)).is_err() {
+            return Err(FuryError::Dead);
+        }
         // begin work
         drop(start_guard);
         // init stats
@@ -220,7 +220,8 @@ impl Fury {
 }
 
 async fn worker_svc(
-    mut rx_task: broadcast::Receiver<WorkerTask<BombardTaskSpec>>,
+    id: usize,
+    mut rx_task: broadcast::Receiver<WorkerTask<BenchmarkTask>>,
     tx_task_result: mpsc::Sender<FuryResult<WorkerLocalStats>>,
     tx_ack: mpsc::Sender<skytable::error::Error>,
     connection_cfg: Config,
@@ -228,7 +229,9 @@ async fn worker_svc(
     let mut db = match connection_cfg.connect_async().await {
         Ok(c) => c,
         Err(e) => {
-            tx_ack.send(e).await.unwrap();
+            if tx_ack.send(e).await.is_err() {
+                error!("worker-{id} failed to ack because main thread exited");
+            }
             return;
         }
     };
@@ -239,16 +242,22 @@ async fn worker_svc(
     {
         Ok(()) => {}
         Err(e) => {
-            tx_ack.send(e).await.unwrap();
+            if tx_ack.send(e).await.is_err() {
+                error!("worker-{id} failed to report error because main thread exited");
+            }
             return;
         }
     }
     // we're connected and ready to server
     drop(tx_ack);
     'wait: loop {
-        let task = match rx_task.recv().await.unwrap() {
-            WorkerTask::Exit => return,
-            WorkerTask::Task(t) => t,
+        let task = match rx_task.recv().await {
+            Err(_) => {
+                error!("worked-{id} is exiting because main thread exited");
+                return;
+            }
+            Ok(WorkerTask::Exit) => return,
+            Ok(WorkerTask::Task(t)) => t,
         };
         // received a task; ready to roll; wait for begin signal
         let permit = GLOBAL_START.read().await;
@@ -262,7 +271,7 @@ async fn worker_svc(
         let mut local_tail = 0u128;
         while (current != 0) && !exit_now {
             // prepare query
-            let (query, response) = task.generate(current as u64);
+            let query = task.generate_query(current as _);
             // execute timed
             let start = Instant::now();
             let ret = db.query(&query).await;
@@ -271,20 +280,32 @@ async fn worker_svc(
             let resp = match ret {
                 Ok(resp) => resp,
                 Err(e) => {
-                    tx_task_result
+                    gset_exit();
+                    if tx_task_result
                         .send(Err(FuryError::Worker(FuryWorkerError::DbError(e))))
                         .await
-                        .unwrap();
-                    gset_exit();
+                        .is_err()
+                    {
+                        error!(
+                            "worker-{id} failed to report worker error because main thread exited"
+                        );
+                        return;
+                    }
                     continue 'wait;
                 }
             };
-            if resp != response {
-                tx_task_result
+            if !task.verify_response(current as _, resp.clone()) {
+                gset_exit();
+                if tx_task_result
                     .send(Err(FuryError::Worker(FuryWorkerError::Mismatch)))
                     .await
-                    .unwrap();
-                gset_exit();
+                    .is_err()
+                {
+                    error!(
+                        "worker-{id} failed to report mismatch error because main thread exited"
+                    );
+                    return;
+                }
                 continue 'wait;
             }
             // update stats
@@ -306,7 +327,7 @@ async fn worker_svc(
             continue 'wait;
         }
         // good! send these results
-        tx_task_result
+        if tx_task_result
             .send(Ok(WorkerLocalStats::new(
                 local_start.unwrap(),
                 local_elapsed,
@@ -314,13 +335,17 @@ async fn worker_svc(
                 local_tail,
             )))
             .await
-            .unwrap();
+            .is_err()
+        {
+            error!("worker-{id} failed to send results because main thread exited");
+            return;
+        }
         drop(permit);
     }
 }
 
 impl Drop for Fury {
     fn drop(&mut self) {
-        self.tx_task.send(WorkerTask::Exit).unwrap();
+        let _ = self.tx_task.send(WorkerTask::Exit);
     }
 }

From 87d11d6ddcbd93c34a3504caad7834ef83d2e7f6 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 28 Nov 2023 23:16:00 +0530
Subject: [PATCH 295/310] Simplify proto impls

Also removed perl from CI steps
---
 .github/workflows/test-pr.yml              |   4 -
 .github/workflows/test-push.yml            |  12 -
 server/src/engine/mem/scanner.rs           |   1 +
 server/src/engine/net/protocol/exchange.rs | 423 +++++++++------------
 server/src/engine/net/protocol/mod.rs      |  39 +-
 server/src/engine/net/protocol/tests.rs    | 278 +++++++++-----
 server/src/util/test_utils.rs              |   8 +-
 7 files changed, 391 insertions(+), 374 deletions(-)

diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml
index 6aebd0b6..1dc798c7 100644
--- a/.github/workflows/test-pr.yml
+++ b/.github/workflows/test-pr.yml
@@ -39,10 +39,6 @@ jobs:
           rustup update
           rustup default ${{ matrix.rust }}
         if: env.BUILD == 'true'
-      - name: Install perl modules
-        uses: perl-actions/install-with-cpanm@v1
-        with:
-          install: "HTML::Entities"
       - name: Run Tests
         run: make test
         env:
diff --git a/.github/workflows/test-push.yml b/.github/workflows/test-push.yml
index e794039d..2684d037 100644
--- a/.github/workflows/test-push.yml
+++ b/.github/workflows/test-push.yml
@@ -43,10 +43,6 @@ jobs:
           brew install gnu-tar
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
         if: runner.os == 'macOS'
-      - name: Install perl modules
-        uses: perl-actions/install-with-cpanm@v1
-        with:
-          install: "HTML::Entities"
       - name: Setup environment
         run: |
           chmod +x ci/setvars.sh
@@ -93,10 +89,6 @@ jobs:
         run: |
           chmod +x ci/setvars.sh
           ci/setvars.sh
-      - name: Install perl modules
-        uses: perl-actions/install-with-cpanm@v1
-        with:
-          install: "HTML::Entities"
 
       - name: Restore cache
         uses: actions/cache@v3
@@ -133,10 +125,6 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 2
-      - name: Install perl modules
-        uses: perl-actions/install-with-cpanm@v1
-        with:
-          install: "HTML::Entities"
 
       - name: Setup environment
         run: |
diff --git a/server/src/engine/mem/scanner.rs b/server/src/engine/mem/scanner.rs
index 48332b64..5c3b925c 100644
--- a/server/src/engine/mem/scanner.rs
+++ b/server/src/engine/mem/scanner.rs
@@ -173,6 +173,7 @@ impl<'a, T> Scanner<'a, T> {
 }
 
 impl<'a> Scanner<'a, u8> {
+    #[cfg(test)]
     /// Attempt to parse the next byte
     pub fn try_next_byte(&mut self) -> Option<u8> {
         if self.eof() {
diff --git a/server/src/engine/net/protocol/exchange.rs b/server/src/engine/net/protocol/exchange.rs
index 1a1a1547..388ac87b 100644
--- a/server/src/engine/net/protocol/exchange.rs
+++ b/server/src/engine/net/protocol/exchange.rs
@@ -26,123 +26,27 @@
 
 use crate::engine::mem::BufferedScanner;
 
-pub const EXCHANGE_MIN_SIZE: usize = b"S1\nh".len();
-pub(super) const STATE_READ_INITIAL: QueryTimeExchangeResult<'static> =
-    QueryTimeExchangeResult::ChangeState {
-        new_state: QueryTimeExchangeState::Initial,
-        expect_more: EXCHANGE_MIN_SIZE,
-    };
-pub(super) const STATE_ERROR: QueryTimeExchangeResult<'static> = QueryTimeExchangeResult::Error;
-
-#[derive(Debug, PartialEq)]
-/// State of a query time exchange
-pub enum QueryTimeExchangeState {
-    /// beginning of exchange
-    Initial,
-    /// SQ (part of packet size)
-    SQ1Meta1Partial { packet_size_part: u64 },
-    /// SQ (part of Q window)
-    SQ2Meta2Partial {
-        size_of_static_frame: usize,
-        packet_size: usize,
-        q_window_part: u64,
-    },
-    /// SQ waiting for block
-    SQ3FinalizeWaitingForBlock {
-        dataframe_size: usize,
-        q_window: usize,
-    },
-}
-
-impl Default for QueryTimeExchangeState {
-    fn default() -> Self {
-        Self::Initial
-    }
-}
-
-#[derive(Debug, PartialEq)]
-/// Result after attempting to complete (or terminate) a query time exchange
-pub enum QueryTimeExchangeResult<'a> {
-    /// We completed the exchange and yielded a [`SQuery`]
-    SQCompleted(SQuery<'a>),
-    /// We're changing states
-    ChangeState {
-        new_state: QueryTimeExchangeState,
-        expect_more: usize,
-    },
-    /// We hit an error and need to terminate this exchange
-    Error,
-}
-
-/// Resume a query time exchange
-pub fn resume<'a>(
-    scanner: &mut BufferedScanner<'a>,
-    state: QueryTimeExchangeState,
-) -> QueryTimeExchangeResult<'a> {
-    match state {
-        QueryTimeExchangeState::Initial => SQuery::resume_initial(scanner),
-        QueryTimeExchangeState::SQ1Meta1Partial { packet_size_part } => {
-            SQuery::resume_at_sq1_meta1_partial(scanner, packet_size_part)
-        }
-        QueryTimeExchangeState::SQ2Meta2Partial {
-            packet_size,
-            q_window_part,
-            size_of_static_frame,
-        } => SQuery::resume_at_sq2_meta2_partial(
-            scanner,
-            size_of_static_frame,
-            packet_size,
-            q_window_part,
-        ),
-        QueryTimeExchangeState::SQ3FinalizeWaitingForBlock {
-            dataframe_size,
-            q_window,
-        } => SQuery::resume_at_final(scanner, q_window, dataframe_size),
-    }
+pub(super) unsafe fn resume<'a>(
+    buf: &'a [u8],
+    last_cursor: usize,
+    last_state: QExchangeState,
+) -> (usize, QExchangeResult<'a>) {
+    let mut scanner = BufferedScanner::new_with_cursor(buf, last_cursor);
+    let ret = last_state.resume(&mut scanner);
+    (scanner.cursor(), ret)
 }
 
 /*
     SQ
 */
 
-enum LFTIntParseResult {
+#[derive(Debug, PartialEq)]
+pub(super) enum LFTIntParseResult {
     Value(u64),
     Partial(u64),
     Error,
 }
 
-fn parse_lf_separated(
-    scanner: &mut BufferedScanner,
-    previously_buffered: u64,
-) -> LFTIntParseResult {
-    let mut ret = previously_buffered;
-    let mut okay = true;
-    while scanner.rounded_cursor_not_eof_matches(|b| *b != b'\n') & okay {
-        let b = unsafe { scanner.next_byte() };
-        okay &= b.is_ascii_digit();
-        ret = match ret.checked_mul(10) {
-            Some(r) => r,
-            None => return LFTIntParseResult::Error,
-        };
-        ret = match ret.checked_add((b & 0x0F) as u64) {
-            Some(r) => r,
-            None => return LFTIntParseResult::Error,
-        };
-    }
-    let payload_ok = okay;
-    let lf_ok = scanner.rounded_cursor_not_eof_matches(|b| *b == b'\n');
-    unsafe { scanner.incr_cursor_if(lf_ok) }
-    if payload_ok & lf_ok {
-        LFTIntParseResult::Value(ret)
-    } else {
-        if payload_ok {
-            LFTIntParseResult::Partial(ret)
-        } else {
-            LFTIntParseResult::Error
-        }
-    }
-}
-
 #[derive(Debug, PartialEq)]
 pub struct SQuery<'a> {
     q: &'a [u8],
@@ -166,155 +70,204 @@ impl<'a> SQuery<'a> {
         &self.payload()[self.q_window()..]
     }
     #[cfg(test)]
-    pub fn query_str(&self) -> Option<&'a str> {
-        core::str::from_utf8(self.query()).ok()
+    pub fn query_str(&self) -> &str {
+        core::str::from_utf8(self.query()).unwrap()
     }
     #[cfg(test)]
-    pub fn params_str(&self) -> Option<&'a str> {
-        core::str::from_utf8(self.params()).ok()
+    pub fn params_str(&self) -> &str {
+        core::str::from_utf8(self.params()).unwrap()
     }
 }
 
-impl<'a> SQuery<'a> {
-    /// We're touching this packet for the first time
-    fn resume_initial(scanner: &mut BufferedScanner<'a>) -> QueryTimeExchangeResult<'a> {
-        if cfg!(debug_assertions) {
-            if !scanner.has_left(EXCHANGE_MIN_SIZE) {
-                return STATE_READ_INITIAL;
+/*
+    utils
+*/
+
+/// scan an integer:
+/// - if just an LF:
+///     - if disallowed single byte: return an error
+///     - else, return value
+/// - if no LF: return upto limit
+/// - if LF: return value
+pub(super) fn scanint(
+    scanner: &mut BufferedScanner,
+    first_run: bool,
+    prev: u64,
+) -> LFTIntParseResult {
+    let mut current = prev;
+    // guard a case where the buffer might be empty and can potentially have invalid chars
+    let mut okay = !((scanner.rounded_cursor_value() == b'\n') & first_run);
+    while scanner.rounded_cursor_not_eof_matches(|b| b'\n'.ne(b)) & okay {
+        let byte = unsafe { scanner.next_byte() };
+        okay &= byte.is_ascii_digit();
+        match current
+            .checked_mul(10)
+            .map(|new| new.checked_add((byte & 0x0f) as u64))
+        {
+            Some(Some(int)) => {
+                current = int;
             }
-        } else {
-            assert!(scanner.has_left(EXCHANGE_MIN_SIZE));
-        }
-        // attempt to read atleast one byte
-        if cfg!(debug_assertions) {
-            match scanner.try_next_byte() {
-                Some(b'S') => {}
-                Some(_) => return STATE_ERROR,
-                None => return STATE_READ_INITIAL,
+            _ => {
+                okay = false;
             }
+        }
+    }
+    let lf = scanner.rounded_cursor_not_eof_equals(b'\n');
+    unsafe {
+        // UNSAFE(@ohsayan): within buffer range
+        scanner.incr_cursor_if(lf);
+    }
+    if lf & okay {
+        LFTIntParseResult::Value(current)
+    } else {
+        if okay {
+            LFTIntParseResult::Partial(current)
         } else {
-            match unsafe { scanner.next_byte() } {
-                b'S' => {}
-                _ => return STATE_ERROR,
-            }
+            LFTIntParseResult::Error
         }
-        Self::resume_at_sq1_meta1_partial(scanner, 0)
     }
-    /// We found some part of meta1, and need to resume
-    fn resume_at_sq1_meta1_partial(
-        scanner: &mut BufferedScanner<'a>,
-        prev: u64,
-    ) -> QueryTimeExchangeResult<'a> {
-        match parse_lf_separated(scanner, prev) {
-            LFTIntParseResult::Value(packet_size) => {
-                // we got the packet size; can we get the q window?
-                Self::resume_at_sq2_meta2_partial(
-                    scanner,
-                    scanner.cursor(),
-                    packet_size as usize,
-                    0,
-                )
-            }
-            LFTIntParseResult::Partial(partial_packet_size) => {
-                // we couldn't get the packet size
-                QueryTimeExchangeResult::ChangeState {
-                    new_state: QueryTimeExchangeState::SQ1Meta1Partial {
-                        packet_size_part: partial_packet_size,
-                    },
-                    expect_more: 3, // 1LF + 1ASCII + 1LF
-                }
-            }
-            LFTIntParseResult::Error => STATE_ERROR,
+}
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub(super) enum QExchangeStateInternal {
+    Initial,
+    PendingMeta1,
+    PendingMeta2,
+    PendingData,
+}
+
+impl Default for QExchangeStateInternal {
+    fn default() -> Self {
+        Self::Initial
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub(super) struct QExchangeState {
+    state: QExchangeStateInternal,
+    target: usize,
+    md_packet_size: u64,
+    md_q_window: u64,
+}
+
+impl Default for QExchangeState {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[derive(Debug, PartialEq)]
+/// Result after attempting to complete (or terminate) a query time exchange
+pub(super) enum QExchangeResult<'a> {
+    /// We completed the exchange and yielded a [`SQuery`]
+    SQCompleted(SQuery<'a>),
+    /// We're changing states
+    ChangeState(QExchangeState),
+    /// We hit an error and need to terminate this exchange
+    Error,
+}
+
+impl QExchangeState {
+    fn _new(
+        state: QExchangeStateInternal,
+        target: usize,
+        md_packet_size: u64,
+        md_q_window: u64,
+    ) -> Self {
+        Self {
+            state,
+            target,
+            md_packet_size,
+            md_q_window,
         }
     }
-    /// We found some part of meta2, and need to resume
-    fn resume_at_sq2_meta2_partial(
+    #[cfg(test)]
+    pub(super) fn new_test(
+        state: QExchangeStateInternal,
+        target: usize,
+        md_packet_size: u64,
+        md_q_window: u64,
+    ) -> Self {
+        Self::_new(state, target, md_packet_size, md_q_window)
+    }
+}
+
+impl QExchangeState {
+    pub const MIN_READ: usize = b"S\x00\n\x00\n".len();
+    pub fn new() -> Self {
+        Self::_new(QExchangeStateInternal::Initial, Self::MIN_READ, 0, 0)
+    }
+    pub fn has_reached_target(&self, new_buffer: &[u8]) -> bool {
+        new_buffer.len() >= self.target
+    }
+    fn resume<'a>(self, scanner: &mut BufferedScanner<'a>) -> QExchangeResult<'a> {
+        debug_assert!(scanner.has_left(Self::MIN_READ));
+        match self.state {
+            QExchangeStateInternal::Initial => self.start_initial(scanner),
+            QExchangeStateInternal::PendingMeta1 => self.resume_at_md1(scanner, false),
+            QExchangeStateInternal::PendingMeta2 => self.resume_at_md2(scanner, false),
+            QExchangeStateInternal::PendingData => self.resume_data(scanner),
+        }
+    }
+    fn start_initial<'a>(self, scanner: &mut BufferedScanner<'a>) -> QExchangeResult<'a> {
+        if unsafe { scanner.next_byte() } != b'S' {
+            // has to be a simple query!
+            return QExchangeResult::Error;
+        }
+        self.resume_at_md1(scanner, true)
+    }
+    fn resume_at_md1<'a>(
+        mut self,
         scanner: &mut BufferedScanner<'a>,
-        static_size: usize,
-        packet_size: usize,
-        prev_qw_buffered: u64,
-    ) -> QueryTimeExchangeResult<'a> {
-        let start = scanner.cursor();
-        match parse_lf_separated(scanner, prev_qw_buffered) {
-            LFTIntParseResult::Value(q_window) => {
-                // we got the q window; can we complete the exchange?
-                let df_size = Self::compute_df_size(scanner, static_size, packet_size);
-                if df_size == 0 {
-                    return QueryTimeExchangeResult::Error;
-                }
-                Self::resume_at_final(scanner, q_window as usize, df_size)
-            }
-            LFTIntParseResult::Partial(q_window_partial) => {
-                // not enough bytes for getting Q window
-                QueryTimeExchangeResult::ChangeState {
-                    new_state: QueryTimeExchangeState::SQ2Meta2Partial {
-                        packet_size,
-                        q_window_part: q_window_partial,
-                        size_of_static_frame: static_size,
-                    },
-                    // we passed cursor - start bytes out of the packet, so expect this more
-                    expect_more: packet_size - (scanner.cursor() - start),
-                }
+        first_run: bool,
+    ) -> QExchangeResult<'a> {
+        let packet_size = match scanint(scanner, first_run, self.md_packet_size) {
+            LFTIntParseResult::Value(v) => v,
+            LFTIntParseResult::Partial(p) => {
+                // if this is the first run, we read 5 bytes and need atleast one more; if this is a resume we read one or more bytes and
+                // need atleast one more
+                self.target += 1;
+                self.md_packet_size = p;
+                self.state = QExchangeStateInternal::PendingMeta1;
+                return QExchangeResult::ChangeState(self);
             }
-            LFTIntParseResult::Error => STATE_ERROR,
-        }
+            LFTIntParseResult::Error => return QExchangeResult::Error,
+        };
+        self.md_packet_size = packet_size;
+        self.target = scanner.cursor() + packet_size as usize;
+        // hand over control to md2
+        self.resume_at_md2(scanner, true)
     }
-    /// We got all our meta and need the dataframe
-    fn resume_at_final(
+    fn resume_at_md2<'a>(
+        mut self,
         scanner: &mut BufferedScanner<'a>,
-        q_window: usize,
-        dataframe_size: usize,
-    ) -> QueryTimeExchangeResult<'a> {
-        if scanner.has_left(dataframe_size) {
-            // we have sufficient bytes for the dataframe
+        first_run: bool,
+    ) -> QExchangeResult<'a> {
+        let q_window = match scanint(scanner, first_run, self.md_q_window) {
+            LFTIntParseResult::Value(v) => v,
+            LFTIntParseResult::Partial(p) => {
+                self.md_q_window = p;
+                self.state = QExchangeStateInternal::PendingMeta2;
+                return QExchangeResult::ChangeState(self);
+            }
+            LFTIntParseResult::Error => return QExchangeResult::Error,
+        };
+        self.md_q_window = q_window;
+        // hand over control to data
+        self.resume_data(scanner)
+    }
+    fn resume_data<'a>(mut self, scanner: &mut BufferedScanner<'a>) -> QExchangeResult<'a> {
+        let df_size = self.target - scanner.cursor();
+        if scanner.remaining() == df_size {
             unsafe {
-                // UNSAFE(@ohsayan): +lenck
-                QueryTimeExchangeResult::SQCompleted(SQuery::new(
-                    scanner.next_chunk_variable(dataframe_size),
-                    q_window,
+                QExchangeResult::SQCompleted(SQuery::new(
+                    scanner.next_chunk_variable(df_size),
+                    self.md_q_window as usize,
                 ))
             }
         } else {
-            // not enough bytes for the dataframe
-            QueryTimeExchangeResult::ChangeState {
-                new_state: QueryTimeExchangeState::SQ3FinalizeWaitingForBlock {
-                    dataframe_size,
-                    q_window,
-                },
-                expect_more: Self::compute_df_remaining(scanner, dataframe_size), // dataframe
-            }
+            self.state = QExchangeStateInternal::PendingData;
+            QExchangeResult::ChangeState(self)
         }
     }
 }
-
-impl<'a> SQuery<'a> {
-    fn compute_df_size(scanner: &BufferedScanner, static_size: usize, packet_size: usize) -> usize {
-        (packet_size + static_size) - scanner.cursor()
-    }
-    fn compute_df_remaining(scanner: &BufferedScanner<'_>, df_size: usize) -> usize {
-        (scanner.cursor() + df_size) - scanner.buffer_len()
-    }
-}
-
-#[cfg(test)]
-pub(super) fn create_simple_query<const N: usize>(query: &str, params: [&str; N]) -> Vec<u8> {
-    let mut buf = vec![];
-    let query_size_as_string = query.len().to_string();
-    let size_of_variable_section = query.len()
-        + params.iter().map(|l| l.len()).sum::<usize>()
-        + query_size_as_string.len()
-        + 1;
-    // segment 1
-    buf.push(b'S');
-    buf.extend(size_of_variable_section.to_string().as_bytes());
-    buf.push(b'\n');
-    // segment
-    buf.extend(query_size_as_string.as_bytes());
-    buf.push(b'\n');
-    // dataframe
-    buf.extend(query.as_bytes());
-    params
-        .into_iter()
-        .for_each(|param| buf.extend(param.as_bytes()));
-    buf
-}
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index c745086c..a8b58074 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -34,7 +34,7 @@ pub use exchange::SQuery;
 
 use {
     self::{
-        exchange::{QueryTimeExchangeResult, QueryTimeExchangeState},
+        exchange::{QExchangeResult, QExchangeState},
         handshake::{
             AuthMode, CHandshake, DataExchangeMode, HandshakeResult, HandshakeState,
             HandshakeVersion, ProtocolError, ProtocolVersion, QueryMode,
@@ -140,42 +140,41 @@ pub(super) async fn query_loop<S: Socket>(
     // done handshaking
     con.write_all(b"H\x00\x00\x00").await?;
     con.flush().await?;
-    let mut exchg_state = QueryTimeExchangeState::default();
-    let mut expect_more = exchange::EXCHANGE_MIN_SIZE;
+    let mut state = QExchangeState::default();
     let mut cursor = 0;
     loop {
-        let read_many = con.read_buf(buf).await?;
-        if read_many == 0 {
+        if con.read_buf(buf).await? == 0 {
             if buf.is_empty() {
                 return Ok(QueryLoopResult::Fin);
             } else {
                 return Ok(QueryLoopResult::Rst);
             }
         }
-        if read_many < expect_more {
+        if !state.has_reached_target(buf) {
             // we haven't buffered sufficient bytes; keep working
             continue;
         }
-        let mut buffer = unsafe { BufferedScanner::new_with_cursor(&buf, cursor) };
-        let sq = match exchange::resume(&mut buffer, exchg_state) {
-            QueryTimeExchangeResult::ChangeState {
-                new_state,
-                expect_more: _more,
-            } => {
-                exchg_state = new_state;
-                expect_more = _more;
-                cursor = buffer.cursor();
+        let sq = match unsafe {
+            // UNSAFE(@ohsayan): we store the cursor from the last run
+            exchange::resume(buf, cursor, state)
+        } {
+            (_, QExchangeResult::SQCompleted(sq)) => sq,
+            (new_cursor, QExchangeResult::ChangeState(new_state)) => {
+                cursor = new_cursor;
+                state = new_state;
                 continue;
             }
-            QueryTimeExchangeResult::SQCompleted(sq) => sq,
-            QueryTimeExchangeResult::Error => {
+            (_, QExchangeResult::Error) => {
+                // respond with error
                 let [a, b] =
                     (QueryError::NetworkSubsystemCorruptedPacket.value_u8() as u16).to_le_bytes();
                 con.write_all(&[ResponseType::Error.value_u8(), a, b])
                     .await?;
                 con.flush().await?;
+                // reset buffer, cursor and state
                 buf.clear();
-                exchg_state = QueryTimeExchangeState::default();
+                cursor = 0;
+                state = QExchangeState::default();
                 continue;
             }
         };
@@ -198,8 +197,10 @@ pub(super) async fn query_loop<S: Socket>(
             }
         }
         con.flush().await?;
+        // reset buffer, cursor and state
         buf.clear();
-        exchg_state = QueryTimeExchangeState::default();
+        cursor = 0;
+        state = QExchangeState::default();
     }
 }
 
diff --git a/server/src/engine/net/protocol/tests.rs b/server/src/engine/net/protocol/tests.rs
index c32ca03e..9eab9549 100644
--- a/server/src/engine/net/protocol/tests.rs
+++ b/server/src/engine/net/protocol/tests.rs
@@ -24,17 +24,46 @@
  *
 */
 
-use crate::engine::{
-    mem::BufferedScanner,
-    net::protocol::{
-        exchange::{self, create_simple_query, QueryTimeExchangeResult, QueryTimeExchangeState},
-        handshake::{
-            AuthMode, CHandshake, CHandshakeAuth, CHandshakeStatic, DataExchangeMode,
-            HandshakeResult, HandshakeState, HandshakeVersion, ProtocolVersion, QueryMode,
+use {
+    super::{
+        exchange::{self, scanint, LFTIntParseResult, QExchangeResult, QExchangeState},
+        SQuery,
+    },
+    crate::{
+        engine::{
+            mem::BufferedScanner,
+            net::protocol::handshake::{
+                AuthMode, CHandshake, CHandshakeAuth, CHandshakeStatic, DataExchangeMode,
+                HandshakeResult, HandshakeState, HandshakeVersion, ProtocolVersion, QueryMode,
+            },
         },
+        util::test_utils,
     },
+    rand::Rng,
 };
 
+pub(super) fn create_simple_query<const N: usize>(query: &str, params: [&str; N]) -> Vec<u8> {
+    let mut buf = vec![];
+    let query_size_as_string = query.len().to_string();
+    let total_packet_size = query.len()
+        + params.iter().map(|l| l.len()).sum::<usize>()
+        + query_size_as_string.len()
+        + 1;
+    // segment 1
+    buf.push(b'S');
+    buf.extend(total_packet_size.to_string().as_bytes());
+    buf.push(b'\n');
+    // segment
+    buf.extend(query_size_as_string.as_bytes());
+    buf.push(b'\n');
+    // dataframe
+    buf.extend(query.as_bytes());
+    params
+        .into_iter()
+        .for_each(|param| buf.extend(param.as_bytes()));
+    buf
+}
+
 /*
     client handshake
 */
@@ -155,110 +184,155 @@ fn parse_auth_with_state_updates() {
 
 const SQ: &str = "select * from myspace.mymodel where username = ?";
 
-#[test]
-fn qtdex_simple_query() {
-    let query = create_simple_query(SQ, ["sayan"]);
-    let mut fin = 52;
-    for i in 0..query.len() {
-        let mut scanner = BufferedScanner::new(&query[..i + 1]);
-        let result = exchange::resume(&mut scanner, Default::default());
-        match scanner.buffer_len() {
-            1..=3 => assert_eq!(result, exchange::STATE_READ_INITIAL),
-            4 => assert_eq!(
-                result,
-                QueryTimeExchangeResult::ChangeState {
-                    new_state: QueryTimeExchangeState::SQ2Meta2Partial {
-                        size_of_static_frame: 4,
-                        packet_size: 56,
-                        q_window_part: 0,
-                    },
-                    expect_more: 56,
-                }
-            ),
-            5 => assert_eq!(
-                result,
-                QueryTimeExchangeResult::ChangeState {
-                    new_state: QueryTimeExchangeState::SQ2Meta2Partial {
-                        size_of_static_frame: 4,
-                        packet_size: 56,
-                        q_window_part: 4,
-                    },
-                    expect_more: 55,
+fn parse_staged<const N: usize>(
+    query: &str,
+    params: [&str; N],
+    eq: impl Fn(SQuery),
+    rng: &mut impl Rng,
+) {
+    let __query_buffer = create_simple_query(query, params);
+    for _ in 0..__query_buffer.len() {
+        let mut __read_total = 0;
+        let mut cursor = 0;
+        let mut state = QExchangeState::default();
+        loop {
+            let remaining = __query_buffer.len() - __read_total;
+            let read_this_time = {
+                let mut cnt = 0;
+                if remaining == 1 {
+                    1
+                } else {
+                    let mut last = test_utils::random_number(1, remaining, rng);
+                    loop {
+                        if cnt >= 10 {
+                            break last;
+                        }
+                        // if we're reading exact, try to keep it low
+                        if last == remaining {
+                            cnt += 1;
+                            last = test_utils::random_number(1, remaining, rng);
+                        } else {
+                            break last;
+                        }
+                    }
                 }
-            ),
-            6 => assert_eq!(
-                result,
-                QueryTimeExchangeResult::ChangeState {
-                    new_state: QueryTimeExchangeState::SQ2Meta2Partial {
-                        size_of_static_frame: 4,
-                        packet_size: 56,
-                        q_window_part: 48,
-                    },
-                    expect_more: 54,
+            };
+            __read_total += read_this_time;
+            let buffered = &__query_buffer[..__read_total];
+            if !state.has_reached_target(buffered) {
+                continue;
+            }
+            match unsafe { exchange::resume(buffered, cursor, state) } {
+                (new_cursor, QExchangeResult::ChangeState(new_state)) => {
+                    cursor = new_cursor;
+                    state = new_state;
+                    continue;
                 }
-            ),
-            7 => assert_eq!(
-                result,
-                QueryTimeExchangeResult::ChangeState {
-                    new_state: QueryTimeExchangeState::SQ3FinalizeWaitingForBlock {
-                        dataframe_size: 53,
-                        q_window: 48,
-                    },
-                    expect_more: 53,
+                (_, QExchangeResult::SQCompleted(q)) => {
+                    eq(q);
+                    break;
                 }
-            ),
-            8..=59 => {
-                assert_eq!(
-                    result,
-                    QueryTimeExchangeResult::ChangeState {
-                        new_state: QueryTimeExchangeState::SQ3FinalizeWaitingForBlock {
-                            dataframe_size: 53,
-                            q_window: 48
-                        },
-                        expect_more: fin,
-                    }
-                );
-                fin -= 1;
+                _ => panic!(),
             }
-            60 => match result {
-                QueryTimeExchangeResult::SQCompleted(sq) => {
-                    assert_eq!(sq.query_str().unwrap(), SQ);
-                    assert_eq!(sq.params_str().unwrap(), "sayan");
-                }
-                _ => unreachable!(),
-            },
-            _ => unreachable!(),
         }
     }
 }
 
 #[test]
-fn qtdex_simple_query_update_state() {
-    let query = create_simple_query(SQ, ["sayan"]);
-    let mut state = QueryTimeExchangeState::default();
-    let mut cursor = 0;
-    let mut expected = 0;
-    let mut rounds = 0;
-    loop {
-        rounds += 1;
-        let buf = &query[..expected + cursor];
-        let mut scanner = unsafe { BufferedScanner::new_with_cursor(buf, cursor) };
-        match exchange::resume(&mut scanner, state) {
-            QueryTimeExchangeResult::SQCompleted(sq) => {
-                assert_eq!(sq.query_str().unwrap(), SQ);
-                assert_eq!(sq.params_str().unwrap(), "sayan");
-                break;
-            }
-            QueryTimeExchangeResult::ChangeState {
-                new_state,
-                expect_more,
-            } => {
-                expected = expect_more;
-                state = new_state;
-            }
-            QueryTimeExchangeResult::Error => panic!("hit error!"),
+fn staged_randomized() {
+    let mut rng = test_utils::rng();
+    parse_staged(
+        SQ,
+        ["sayan"],
+        |q| {
+            assert_eq!(q.query_str(), SQ);
+            assert_eq!(q.params_str(), "sayan");
+        },
+        &mut rng,
+    );
+}
+
+#[test]
+fn stages_manual() {
+    let query = create_simple_query("select * from mymodel where username = ?", ["sayan"]);
+    assert_eq!(
+        unsafe { exchange::resume(&query[..QExchangeState::MIN_READ], 0, Default::default()) },
+        (
+            5,
+            QExchangeResult::ChangeState(QExchangeState::new_test(
+                exchange::QExchangeStateInternal::PendingMeta2,
+                52,
+                48,
+                4
+            ))
+        )
+    );
+    assert_eq!(
+        unsafe {
+            exchange::resume(
+                &query[..QExchangeState::MIN_READ + 1],
+                0,
+                Default::default(),
+            )
+        },
+        (
+            6,
+            QExchangeResult::ChangeState(QExchangeState::new_test(
+                exchange::QExchangeStateInternal::PendingMeta2,
+                52,
+                48,
+                40
+            ))
+        )
+    );
+    assert_eq!(
+        unsafe {
+            exchange::resume(
+                &query[..QExchangeState::MIN_READ + 2],
+                0,
+                Default::default(),
+            )
+        },
+        (
+            7,
+            QExchangeResult::ChangeState(QExchangeState::new_test(
+                exchange::QExchangeStateInternal::PendingData,
+                52,
+                48,
+                40
+            ))
+        )
+    );
+    // the cursor should never change
+    for upper_bound in QExchangeState::MIN_READ + 2..query.len() {
+        assert_eq!(
+            unsafe { exchange::resume(&query[..upper_bound], 0, Default::default()) },
+            (
+                7,
+                QExchangeResult::ChangeState(QExchangeState::new_test(
+                    exchange::QExchangeStateInternal::PendingData,
+                    52,
+                    48,
+                    40
+                ))
+            )
+        );
+    }
+    match unsafe { exchange::resume(&query, 0, Default::default()) } {
+        (l, QExchangeResult::SQCompleted(q)) if l == query.len() => {
+            assert_eq!(q.query_str(), "select * from mymodel where username = ?");
+            assert_eq!(q.params_str(), "sayan");
         }
-        cursor = scanner.cursor();
+        e => panic!("expected end, got {e:?}"),
     }
-    assert_eq!(rounds, 3);
+}
+
+#[test]
+fn scanint_impl() {
+    let mut s = BufferedScanner::new(b"\n");
+    assert_eq!(scanint(&mut s, true, 0), LFTIntParseResult::Error);
+    let mut s = BufferedScanner::new(b"12");
+    assert_eq!(scanint(&mut s, true, 0), LFTIntParseResult::Partial(12));
+    let mut s = BufferedScanner::new(b"12\n");
+    assert_eq!(scanint(&mut s, true, 0), LFTIntParseResult::Value(12));
 }
diff --git a/server/src/util/test_utils.rs b/server/src/util/test_utils.rs
index e3cf35ab..a12110a1 100644
--- a/server/src/util/test_utils.rs
+++ b/server/src/util/test_utils.rs
@@ -38,6 +38,10 @@ use {
     },
 };
 
+pub fn rng() -> ThreadRng {
+    rand::thread_rng()
+}
+
 pub fn shuffle_slice<T>(slice: &mut [T], rng: &mut impl Rng) {
     slice.shuffle(rng)
 }
@@ -73,8 +77,8 @@ pub fn random_bool(rng: &mut impl Rng) -> bool {
     rng.gen_bool(0.5)
 }
 /// Generate a random number within the given range
-pub fn random_number<T: SampleUniform + PartialOrd>(max: T, min: T, rng: &mut impl Rng) -> T {
-    rng.gen_range(max..min)
+pub fn random_number<T: SampleUniform + PartialOrd>(min: T, max: T, rng: &mut impl Rng) -> T {
+    rng.gen_range(min..max)
 }
 
 pub fn random_string(rng: &mut impl Rng, l: usize) -> String {

From bed0d93628f013d985848127e248879c1cc80f24 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 29 Nov 2023 09:08:47 +0530
Subject: [PATCH 296/310] Refine permissions

---
 server/src/engine/core/dcl.rs  |   3 +
 server/src/engine/core/exec.rs |   4 +-
 server/src/engine/ql/dcl.rs    |   6 +
 sky-macros/src/dbtest_fn.rs    | 376 ---------------------------------
 sky-macros/src/dbtest_mod.rs   | 116 ----------
 sky-macros/src/lib.rs          |  53 -----
 6 files changed, 11 insertions(+), 547 deletions(-)
 delete mode 100644 sky-macros/src/dbtest_fn.rs
 delete mode 100644 sky-macros/src/dbtest_mod.rs

diff --git a/server/src/engine/core/dcl.rs b/server/src/engine/core/dcl.rs
index 7bf0d6c8..cfb3356f 100644
--- a/server/src/engine/core/dcl.rs
+++ b/server/src/engine/core/dcl.rs
@@ -39,6 +39,9 @@ pub fn exec<G: GlobalInstanceLike>(
     current_user: &ClientLocalState,
     cmd: SysctlCommand,
 ) -> QueryResult<()> {
+    if cmd.needs_root() & !current_user.is_root() {
+        return Err(QueryError::SysPermissionDenied);
+    }
     match cmd {
         SysctlCommand::CreateUser(new) => create_user(&g, new),
         SysctlCommand::DropUser(drop) => drop_user(&g, current_user, drop),
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 8daf9dae..29586c2e 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -78,8 +78,8 @@ async fn run_blocking_stmt(
     mut state: State<'_, InplaceData>,
     stmt: KeywordStmt,
 ) -> Result<Response, QueryError> {
-    if !cstate.is_root() {
-        // all the actions here need root permission
+    if !(cstate.is_root() | (stmt == KeywordStmt::Sysctl)) {
+        // all the actions here need root permission (but we do an exception for sysctl which allows status to be called by anyone)
         return Err(QueryError::SysPermissionDenied);
     }
     state.ensure_minimum_for_blocking_stmt()?;
diff --git a/server/src/engine/ql/dcl.rs b/server/src/engine/ql/dcl.rs
index ec4be180..b04737c3 100644
--- a/server/src/engine/ql/dcl.rs
+++ b/server/src/engine/ql/dcl.rs
@@ -46,6 +46,12 @@ pub enum SysctlCommand<'a> {
     ReportStatus,
 }
 
+impl<'a> SysctlCommand<'a> {
+    pub fn needs_root(&self) -> bool {
+        !matches!(self, Self::ReportStatus)
+    }
+}
+
 impl<'a> traits::ASTNode<'a> for SysctlCommand<'a> {
     const MUST_USE_FULL_TOKEN_RANGE: bool = true;
     const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
diff --git a/sky-macros/src/dbtest_fn.rs b/sky-macros/src/dbtest_fn.rs
deleted file mode 100644
index 2b4ef21a..00000000
--- a/sky-macros/src/dbtest_fn.rs
+++ /dev/null
@@ -1,376 +0,0 @@
-/*
- * Created on Wed Mar 09 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {crate::util, proc_macro::TokenStream, quote::quote, syn::AttributeArgs};
-
-type OptString = Option<String>;
-
-pub struct DBTestFunctionConfig {
-    table_decl: String,
-    port: u16,
-    host: String,
-    tls_cert: OptString,
-    login: (OptString, OptString),
-    testuser: bool,
-    rootuser: bool,
-    norun: bool,
-    skip_cfg: quote::__private::TokenStream,
-}
-
-impl DBTestFunctionConfig {
-    pub fn default() -> Self {
-        Self {
-            table_decl: "(string, string)".to_owned(),
-            port: 2003,
-            host: "127.0.0.1".to_owned(),
-            tls_cert: None,
-            login: (None, None),
-            testuser: false,
-            rootuser: false,
-            norun: false,
-            skip_cfg: quote! {},
-        }
-    }
-    pub fn get_connection_tokens(&self) -> impl quote::ToTokens {
-        let DBTestFunctionConfig {
-            port,
-            host,
-            tls_cert,
-            ..
-        } = &self;
-        match tls_cert {
-            Some(cert) => {
-                quote! {
-                    let certpath = ::std::format!("{}/{}", crate::ROOT_DIR, #cert);
-                    let mut con = skytable::aio::TlsConnection::new(
-                        #host, #port, &certpath
-                    ).await.unwrap();
-                }
-            }
-            None => quote! {
-                let mut con = skytable::AsyncConnection::new(#host, #port).await.unwrap();
-            },
-        }
-    }
-    pub fn get_create_table_tokens(&self, table_name: &str) -> impl quote::ToTokens {
-        let Self { table_decl, .. } = self;
-        quote! {
-            con.run_query_raw(
-                &skytable::query!(format!("create model {}{} volatile", #table_name, #table_decl))
-            ).await.unwrap()
-        }
-    }
-    pub fn get_skip_cfg_tokens(&self) -> &impl quote::ToTokens {
-        &self.skip_cfg
-    }
-    pub fn get_login_tokens(&self) -> Option<impl quote::ToTokens> {
-        let Self {
-            login,
-            testuser,
-            rootuser,
-            ..
-        } = self;
-        let conflict = (*rootuser && *testuser)
-            || ((*rootuser || *testuser) && (login.0.is_some() || login.1.is_some()));
-        if conflict {
-            panic!("Expected either of `username` and `password`, or `auth_rootuser`, or `auth_testuser`");
-        }
-        let ret;
-        if *testuser {
-            ret = quote! {
-                let __username__ = crate::auth::provider::testsuite_data::TESTSUITE_TEST_USER;
-                let __password__ = crate::auth::provider::testsuite_data::TESTSUITE_TEST_TOKEN;
-            };
-        } else if *rootuser {
-            ret = quote! {
-                let __username__ = crate::auth::provider::testsuite_data::TESTSUITE_ROOT_USER;
-                let __password__ = crate::auth::provider::testsuite_data::TESTSUITE_ROOT_TOKEN;
-            };
-        } else {
-            let (username, password) = login;
-            match (username, password) {
-                (Some(username), Some(password)) => {
-                    ret = quote! {
-                        let __username__ = #username;
-                        let __password__ = #password;
-                    }
-                }
-                (None, None) => return None,
-                _ => panic!("Expected both `username` and `password`"),
-            }
-        }
-        Some(quote! {
-            #ret
-            let __loginquery__ = ::skytable::query!("auth", "login", __username__, __password__);
-            assert_eq!(
-                con.run_query_raw(&__loginquery__).await.unwrap(),
-                ::skytable::Element::RespCode(::skytable::RespCode::Okay),
-                "Failed to login"
-            );
-        })
-    }
-}
-
-pub fn parse_dbtest_func_args(
-    arg: &str,
-    lit: &syn::Lit,
-    span: proc_macro2::Span,
-    fcfg: &mut DBTestFunctionConfig,
-) {
-    match arg {
-        "table" => {
-            // check if the user wants some special table declaration
-            fcfg.table_decl =
-                util::parse_string(lit, span, "table").expect("Expected a value for `table`");
-        }
-        "port" => {
-            // check if we need a custom port
-            fcfg.port = util::parse_number(lit, span, "port").expect("Expected a u16");
-        }
-        "host" => {
-            fcfg.host = util::parse_string(lit, span, "host").expect("Expected a string");
-        }
-        "tls_cert" => {
-            fcfg.tls_cert = Some(util::parse_string(lit, span, "host").expect("Expected a string"));
-        }
-        "username" => {
-            fcfg.login.0 =
-                Some(util::parse_string(lit, span, "username").expect("Expected a string"))
-        }
-        "password" => {
-            fcfg.login.1 =
-                Some(util::parse_string(lit, span, "password").expect("Expected a string"))
-        }
-        "auth_testuser" => {
-            fcfg.testuser = util::parse_bool(lit, span, "auth_testuser").expect("Expected a bool")
-        }
-        "auth_rootuser" => {
-            fcfg.rootuser = util::parse_bool(lit, span, "auth_testuser").expect("Expected a bool")
-        }
-        "norun" => fcfg.norun = util::parse_bool(lit, span, "norun").expect("Expected a bool"),
-        "run_if_cfg" => {
-            let cfg_name = util::parse_string(lit, span, "run_if_cfg").expect("Expected a string");
-            fcfg.skip_cfg = quote! {
-                #[cfg_attr(not(feature = #cfg_name), ignore)]
-            };
-        }
-        "skip_if_cfg" => {
-            let cfg_name = util::parse_string(lit, span, "run_if_cfg").expect("Expected a string");
-            fcfg.skip_cfg = quote! {
-                #[cfg_attr(feature = #cfg_name, ignore)]
-            };
-        }
-        x => panic!("unknown attribute {x} specified"),
-    }
-}
-
-/// This parses a function within a `dbtest` module
-///
-/// This accepts an `async` function and returns a non-`async` version of it - by
-/// making the body of the function use the `tokio` runtime
-fn generate_dbtest(
-    mut input: syn::ItemFn,
-    rng: &mut impl rand::Rng,
-    fcfg: &DBTestFunctionConfig,
-) -> Result<TokenStream, syn::Error> {
-    let sig = &mut input.sig;
-    let fname = sig.ident.to_string();
-    let testbody = &input.block;
-    let attrs = &input.attrs;
-    let vis = &input.vis;
-    let header = quote! {
-        #[::core::prelude::v1::test]
-    };
-    if sig.asyncness.is_none() {
-        let msg = "`dbtest` functions need to be async";
-        return Err(syn::Error::new_spanned(sig.fn_token, msg));
-    }
-    sig.asyncness = None;
-    let rand_string = util::get_rand_string(rng);
-    let mut body = quote! {};
-
-    // first add connection tokens
-    let connection_tokens = fcfg.get_connection_tokens();
-    body = quote! {
-        #body
-        #connection_tokens
-    };
-
-    // check if we need to log in
-    if let Some(login_tokens) = fcfg.get_login_tokens() {
-        body = quote! {
-            #body
-            #login_tokens
-        };
-    }
-
-    if !fcfg.norun {
-        // now create keyspace
-        body = quote! {
-            #body
-            let __create_ks =
-                con.run_query_raw(
-                    &skytable::query!("create space testsuite")
-                ).await.unwrap();
-            if !(
-                __create_ks == skytable::Element::RespCode(skytable::RespCode::Okay) ||
-                __create_ks == skytable::Element::RespCode(
-                    skytable::RespCode::ErrorString(
-                        skytable::error::errorstring::ERR_ALREADY_EXISTS.to_owned()
-                    )
-                )
-            ) {
-                panic!("Failed to create keyspace: {:?}", __create_ks);
-            }
-        };
-        // now switch keyspace
-        body = quote! {
-            #body
-            let __switch_ks =
-                con.run_query_raw(
-                    &skytable::query!("use testsuite")
-                ).await.unwrap();
-            if (__switch_ks != skytable::Element::RespCode(skytable::RespCode::Okay)) {
-                panic!("Failed to switch keyspace: {:?}", __switch_ks);
-            }
-        };
-        // now create table
-        let create_table_tokens = fcfg.get_create_table_tokens(&rand_string);
-        body = quote! {
-            #body
-            assert_eq!(
-                #create_table_tokens,
-                skytable::Element::RespCode(skytable::RespCode::Okay),
-                "Failed to create table"
-            );
-        };
-        // now generate the __MYENTITY__ string
-        body = quote! {
-            #body
-            let mut __concat_entity = std::string::String::new();
-            __concat_entity.push_str("testsuite.");
-            __concat_entity.push_str(&#rand_string);
-            let __MYTABLE__ : String = #rand_string.to_string();
-            let __MYKS__: String = "testsuite".to_owned();
-            let __MYENTITY__: String = __concat_entity.clone();
-        };
-        // now switch to the temporary table we created
-        body = quote! {
-            #body
-            let __switch_entity =
-                con.run_query_raw(
-                    &skytable::query!(format!("use {}", __concat_entity))
-                ).await.unwrap();
-            assert_eq!(
-                __switch_entity, skytable::Element::RespCode(skytable::RespCode::Okay), "Failed to switch"
-            );
-        };
-    }
-    // now give the query ghost variable
-    body = quote! {
-        #body
-        let mut query = skytable::Query::new();
-    };
-    // IMPORTANT: now append the actual test body
-    body = quote! {
-        #body
-        #testbody
-    };
-    if !fcfg.norun {
-        // now we're done with the test so flush the table
-        body = quote! {
-            #body
-            {
-                let mut __flush__ = skytable::Query::from("flushdb");
-                std::assert_eq!(
-                    con.run_query_raw(&__flush__).await.unwrap(),
-                    skytable::Element::RespCode(skytable::RespCode::Okay)
-                );
-            }
-        };
-    }
-    let skip_cfg = fcfg.get_skip_cfg_tokens();
-    let result = quote! {
-        #skip_cfg
-        #header
-        #(#attrs)*
-        #vis #sig {
-            tokio::runtime::Builder::new_multi_thread()
-            .worker_threads(4)
-            .thread_name(#fname)
-            .thread_stack_size(3 * 1024 * 1024)
-            .enable_all()
-            .build()
-            .unwrap()
-            .block_on(async { #body });
-        }
-    };
-    Ok(result.into())
-}
-
-/// This function checks if the current function is eligible to be a test and if so, returns
-/// the generated test
-pub fn generate_test(
-    input: syn::ItemFn,
-    rng: &mut impl rand::Rng,
-    fcfg: &DBTestFunctionConfig,
-) -> TokenStream {
-    for attr in &input.attrs {
-        if attr.path.is_ident("test") {
-            let msg = "second test attribute is supplied";
-            return syn::Error::new_spanned(&attr, msg)
-                .to_compile_error()
-                .into();
-        }
-    }
-
-    if !input.sig.inputs.is_empty() {
-        let msg = "the test function cannot accept arguments";
-        return syn::Error::new_spanned(&input.sig.inputs, msg)
-            .to_compile_error()
-            .into();
-    }
-    generate_dbtest(input, rng, fcfg).unwrap_or_else(|e| e.to_compile_error().into())
-}
-
-fn parse_dbtest_func_attrs(attrs: AttributeArgs) -> DBTestFunctionConfig {
-    let mut fcfg = DBTestFunctionConfig::default();
-    attrs.iter().for_each(|arg| {
-        if let syn::NestedMeta::Meta(syn::Meta::NameValue(namevalue)) = arg {
-            let (ident, lit, span) = util::get_metanamevalue_data(namevalue);
-            parse_dbtest_func_args(&ident, lit, span, &mut fcfg)
-        }
-    });
-    fcfg
-}
-
-pub fn dbtest_func(args: TokenStream, item: TokenStream) -> TokenStream {
-    let input = syn::parse_macro_input!(item as syn::ItemFn);
-    let attrs = syn::parse_macro_input!(args as AttributeArgs);
-    let mut rng = rand::thread_rng();
-    let fcfg = parse_dbtest_func_attrs(attrs);
-    generate_dbtest(input, &mut rng, &fcfg).unwrap()
-}
diff --git a/sky-macros/src/dbtest_mod.rs b/sky-macros/src/dbtest_mod.rs
deleted file mode 100644
index d57a1754..00000000
--- a/sky-macros/src/dbtest_mod.rs
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Created on Wed Mar 09 2022
- *
- * This file is a part of Skytable
- * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
- * NoSQL database written by Sayan Nandan ("the Author") with the
- * vision to provide flexibility in data modelling without compromising
- * on performance, queryability or scalability.
- *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
-*/
-
-use {
-    crate::{
-        dbtest_fn::{self, DBTestFunctionConfig},
-        util,
-    },
-    proc_macro::TokenStream,
-    quote::quote,
-    std::collections::HashSet,
-    syn::{self, AttributeArgs},
-};
-
-struct DBTestModuleConfig {
-    fcfg: DBTestFunctionConfig,
-    skips: HashSet<String>,
-}
-
-impl DBTestModuleConfig {
-    fn default() -> Self {
-        Self {
-            skips: HashSet::new(),
-            fcfg: DBTestFunctionConfig::default(),
-        }
-    }
-}
-
-fn parse_dbtest_module_args(args: AttributeArgs) -> DBTestModuleConfig {
-    let mut modcfg = DBTestModuleConfig::default();
-    for arg in args {
-        if let syn::NestedMeta::Meta(syn::Meta::NameValue(namevalue)) = arg {
-            let (ident, lit, span) = util::get_metanamevalue_data(&namevalue);
-            match ident.as_str() {
-                "skip" => {
-                    modcfg.skips = util::parse_string(lit, span, "skip")
-                        .expect("Expected a value for argument `skip`")
-                        .split_whitespace()
-                        .map(|val| val.to_string())
-                        .collect();
-                }
-                possibly_func_arg => dbtest_fn::parse_dbtest_func_args(
-                    possibly_func_arg,
-                    lit,
-                    span,
-                    &mut modcfg.fcfg,
-                ),
-            }
-        }
-    }
-    modcfg
-}
-
-/// This function accepts an entire **inline** module which comprises of `dbtest` functions.
-/// It takes each function in turn, and generates `#[test]`able functions for them
-pub fn parse_test_module(args: TokenStream, item: TokenStream) -> TokenStream {
-    let input = syn::parse_macro_input!(item as syn::ItemMod);
-    let content = match input.content {
-        Some((_, c)) => c,
-        None => {
-            return syn::Error::new_spanned(&input, "Couldn't get the module content")
-                .to_compile_error()
-                .into()
-        }
-    };
-    let modcfg = parse_dbtest_module_args(syn::parse_macro_input!(args as AttributeArgs));
-    let mut result = quote! {};
-    let mut rng = rand::thread_rng();
-    for item in content {
-        match item {
-            // We just care about functions, so parse functions and ignore everything
-            // else
-            syn::Item::Fn(function) if !modcfg.skips.contains(&function.sig.ident.to_string()) => {
-                let generated_fn = dbtest_fn::generate_test(function, &mut rng, &modcfg.fcfg);
-                let __tok: syn::ItemFn = syn::parse_macro_input!(generated_fn as syn::ItemFn);
-                let tok = quote! {
-                    #__tok
-                };
-                result = quote! {
-                    #result
-                    #tok
-                };
-            }
-            token => {
-                result = quote! {
-                    #result
-                    #token
-                };
-            }
-        }
-    }
-    result.into()
-}
diff --git a/sky-macros/src/lib.rs b/sky-macros/src/lib.rs
index ad2356fe..cb04fb94 100644
--- a/sky-macros/src/lib.rs
+++ b/sky-macros/src/lib.rs
@@ -50,61 +50,8 @@ use {
     syn::{parse_macro_input, Data, DataStruct, DeriveInput, Fields, Lit, Meta, NestedMeta},
 };
 
-mod dbtest_fn;
-mod dbtest_mod;
 mod util;
 
-#[proc_macro_attribute]
-/// The `dbtest_module` function accepts an inline module of `dbtest_func` compatible functions,
-/// unpacking each function into a dbtest
-pub fn dbtest_module(args: TokenStream, item: TokenStream) -> TokenStream {
-    dbtest_mod::parse_test_module(args, item)
-}
-
-/// The `dbtest_func` macro starts an async server in the background and is meant for
-/// use within the `skyd` or `WORKSPACEROOT/server/` crate. If you use this compiler
-/// macro in any other crate, you'll simply get compilation errors
-///
-/// All tests will clean up all values once a single test is over
-///
-/// ## Arguments
-/// - `table -> str`: Custom table declaration
-/// - `port -> u16`: Custom port
-/// - `host -> str`: Custom host
-/// - `tls_cert -> str`: TLS cert (makes the connection a TLS one)
-/// - `username -> str`: Username for authn
-/// - `password -> str`: Password for authn
-/// - `auth_testuser -> bool`: Login as the test user
-/// - `auth_rootuser -> bool`: Login as the root user
-/// - `norun -> bool`: Don't execute anything on the connection
-///
-/// ## _Ghost_ values
-/// This macro gives:
-/// - `con`: a `skytable::AsyncConnection`
-/// - `query`: a mutable `skytable::Query`
-/// - `__MYENTITY__`: the entity set on launch
-/// - `__MYTABLE__`: the table set on launch
-/// - `__MYKS__`: the keyspace set on launch
-///
-/// ## Requirements
-///
-/// The `#[dbtest]` macro expects several things. The calling crate:
-/// - should have the `tokio` crate as a dependency and should have the
-/// `features` set to full
-/// - should have the `skytable` crate as a dependency and should have the `features` set to `async` and version
-/// upstreamed to `next` on skytable/client-rust
-///
-/// ## Collisions
-///
-/// The sample space for table name generation is so large (in the order of 4.3 to the 50) that collisions
-/// are practially impossible. Hence we do not bother with a global random string table and instead proceed
-/// to generate tables randomly at the point of invocation
-///
-#[proc_macro_attribute]
-pub fn dbtest_func(args: TokenStream, item: TokenStream) -> TokenStream {
-    dbtest_fn::dbtest_func(args, item)
-}
-
 #[proc_macro]
 /// Get a compile time respcode/respstring array. For example, if you pass: "Unknown action",
 /// it will return: `!14\nUnknown Action\n`

From 5c62a842b451c4bfaaa8ddfb797133d61ae3723f Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 29 Nov 2023 11:39:40 +0530
Subject: [PATCH 297/310] Implement new dbtest macros

---
 Cargo.lock                               |  11 +-
 ci/server1.yaml                          |   2 +-
 libsky/src/lib.rs                        |   9 +-
 server/Cargo.toml                        |   1 +
 server/src/engine/tests/cfg.rs           | 301 +++++++++++++++++++++++
 server/src/engine/tests/client/mod.rs    |  27 ++
 server/src/engine/tests/client/sysctl.rs |  41 +++
 server/src/engine/tests/mod.rs           | 280 +--------------------
 sky-macros/Cargo.toml                    |   6 +-
 sky-macros/src/dbtest.rs                 | 290 ++++++++++++++++++++++
 sky-macros/src/lib.rs                    |  82 +-----
 sky-macros/src/util.rs                   |  91 +++----
 12 files changed, 724 insertions(+), 417 deletions(-)
 create mode 100644 server/src/engine/tests/cfg.rs
 create mode 100644 server/src/engine/tests/client/mod.rs
 create mode 100644 server/src/engine/tests/client/sysctl.rs
 create mode 100644 sky-macros/src/dbtest.rs

diff --git a/Cargo.lock b/Cargo.lock
index 58bda0d7..0e30641b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -876,18 +876,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.66"
+version = "1.0.70"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
+checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.32"
+version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965"
+checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
 dependencies = [
  "proc-macro2",
 ]
@@ -1189,9 +1189,9 @@ dependencies = [
 name = "sky_macros"
 version = "0.8.0"
 dependencies = [
+ "libsky",
  "proc-macro2",
  "quote",
- "rand",
  "syn 1.0.109",
 ]
 
@@ -1214,6 +1214,7 @@ dependencies = [
  "serde",
  "serde_yaml",
  "sky_macros",
+ "skytable",
  "tokio",
  "tokio-openssl",
  "uuid",
diff --git a/ci/server1.yaml b/ci/server1.yaml
index fc3aaa18..ccf52e5e 100644
--- a/ci/server1.yaml
+++ b/ci/server1.yaml
@@ -3,7 +3,7 @@ system:
 
 auth:
   plugin: pwd
-  root_pass: password12345678
+  root_pass: mypassword12345678
 
 endpoints:
   insecure:
diff --git a/libsky/src/lib.rs b/libsky/src/lib.rs
index e5c93208..0532bc3a 100644
--- a/libsky/src/lib.rs
+++ b/libsky/src/lib.rs
@@ -31,13 +31,18 @@
 //!
 //! This contains modules which are shared by both the `cli` and the `server` modules
 
-/// The size of the read buffer in bytes
-pub const BUF_CAP: usize = 8 * 1024; // 8 KB per-connection
 /// The current version
 pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 /// The URL
 pub const URL: &str = "https://github.com/skytable/skytable";
 
+pub mod test_utils {
+    pub const DEFAULT_USER_NAME: &str = "root";
+    pub const DEFAULT_USER_PASS: &str = "mypassword12345678";
+    pub const DEFAULT_HOST: &str = "127.0.0.1";
+    pub const DEFAULT_PORT: u16 = 2003;
+}
+
 use std::{
     collections::{hash_map::Entry, HashMap},
     env,
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 15e2d358..5462c0f3 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -42,6 +42,7 @@ libc = "0.2.147"
 # external deps
 rand = "0.8.5"
 tokio = { version = "1.34.0", features = ["test-util"] }
+skytable = { branch = "octave", git = "https://github.com/skytable/client-rust.git" }
 
 [features]
 nightly = []
diff --git a/server/src/engine/tests/cfg.rs b/server/src/engine/tests/cfg.rs
new file mode 100644
index 00000000..dd8e2935
--- /dev/null
+++ b/server/src/engine/tests/cfg.rs
@@ -0,0 +1,301 @@
+/*
+ * Created on Wed Nov 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::{
+    engine::config::{
+        self, AuthDriver, CLIConfigParseReturn, ConfigAuth, ConfigEndpoint, ConfigEndpointTcp,
+        ConfigEndpointTls, ConfigMode, ConfigReturn, ConfigSystem, Configuration, ParsedRawArgs,
+    },
+    util::test_utils::with_files,
+};
+
+/*
+    CLI tests
+*/
+
+fn extract_cli_args(payload: &str) -> std::collections::HashMap<String, Vec<String>> {
+    extract_cli_args_raw(payload).into_config()
+}
+fn extract_cli_args_raw(
+    payload: &str,
+) -> CLIConfigParseReturn<std::collections::HashMap<String, Vec<String>>> {
+    config::parse_cli_args(payload.split_ascii_whitespace().map_while(|item| {
+        let mut item = item.trim();
+        if item.ends_with("\n") {
+            item = &item[..item.len() - 1];
+        }
+        if item.is_empty() {
+            None
+        } else {
+            Some(item)
+        }
+    }))
+    .unwrap()
+}
+#[test]
+fn parse_cli_args_simple() {
+    let payload = "skyd --mode dev --endpoint tcp@localhost:2003";
+    let cfg = extract_cli_args(payload);
+    let expected: ParsedRawArgs = into_dict! {
+        "--mode" => vec!["dev".into()],
+        "--endpoint" => vec!["tcp@localhost:2003".into()]
+    };
+    assert_eq!(cfg, expected);
+}
+#[test]
+fn parse_cli_args_packed() {
+    let payload = "skyd --mode=dev --endpoint=tcp@localhost:2003";
+    let cfg = extract_cli_args(payload);
+    let expected: ParsedRawArgs = into_dict! {
+        "--mode" => vec!["dev".into()],
+        "--endpoint" => vec!["tcp@localhost:2003".into()]
+    };
+    assert_eq!(cfg, expected);
+}
+#[test]
+fn parse_cli_args_multi() {
+    let payload = "skyd --mode=dev --endpoint tcp@localhost:2003";
+    let cfg = extract_cli_args(payload);
+    let expected: ParsedRawArgs = into_dict! {
+        "--mode" => vec!["dev".into()],
+        "--endpoint" => vec!["tcp@localhost:2003".into()]
+    };
+    assert_eq!(cfg, expected);
+}
+#[test]
+fn parse_validate_cli_args() {
+    with_files(
+        [
+            "__cli_args_test_private.key",
+            "__cli_args_test_cert.pem",
+            "__cli_args_test_passphrase.key",
+        ],
+        |[pkey, cert, pass]| {
+            let payload = format!(
+                "skyd --mode=dev \
+                --endpoint tcp@127.0.0.1:2003 \
+                --endpoint tls@127.0.0.2:2004 \
+                --service-window=600 \
+                --tlskey {pkey} \
+                --tlscert {cert} \
+                --tls-passphrase {pass} \
+                --auth-plugin pwd \
+                --auth-root-password password12345678
+                "
+            );
+            let cfg = extract_cli_args(&payload);
+            let ret = config::apply_and_validate::<config::CSCommandLine>(cfg)
+                .unwrap()
+                .into_config();
+            assert_eq!(
+                ret,
+                Configuration::new(
+                    ConfigEndpoint::Multi(
+                        ConfigEndpointTcp::new("127.0.0.1".into(), 2003),
+                        ConfigEndpointTls::new(
+                            ConfigEndpointTcp::new("127.0.0.2".into(), 2004),
+                            "".into(),
+                            "".into(),
+                            "".into()
+                        )
+                    ),
+                    ConfigMode::Dev,
+                    ConfigSystem::new(600),
+                    ConfigAuth::new(AuthDriver::Pwd, "password12345678".into())
+                )
+            )
+        },
+    );
+}
+#[test]
+fn parse_validate_cli_args_help_and_version() {
+    let pl1 = "skyd --help";
+    let pl2 = "skyd --version";
+    let ret1 = extract_cli_args_raw(pl1);
+    let ret2 = extract_cli_args_raw(pl2);
+    assert_eq!(ret1, CLIConfigParseReturn::Help);
+    assert_eq!(ret2, CLIConfigParseReturn::Version);
+    config::set_cli_src(vec!["skyd".into(), "--help".into()]);
+    let ret3 = config::check_configuration().unwrap();
+    config::set_cli_src(vec!["skyd".into(), "--version".into()]);
+    let ret4 = config::check_configuration().unwrap();
+    assert_eq!(
+        ret3,
+        ConfigReturn::HelpMessage(config::CLI_HELP.to_string())
+    );
+    assert_eq!(
+        ret4,
+        ConfigReturn::HelpMessage(format!(
+            "Skytable Database Server (skyd) v{}",
+            libsky::VERSION
+        ))
+    );
+}
+
+/*
+    env tests
+*/
+
+fn vars_to_args(variables: &[String]) -> ParsedRawArgs {
+    variables
+        .iter()
+        .map(|var| {
+            var.split("=")
+                .map(ToString::to_string)
+                .collect::<Vec<String>>()
+        })
+        .map(|mut v| {
+            let key = v.remove(0);
+            let values = v.remove(0).split(",").map(ToString::to_string).collect();
+            (key, values)
+        })
+        .collect()
+}
+#[test]
+fn parse_env_args_simple() {
+    let variables = [
+        format!("SKYDB_TLS_CERT=/var/skytable/keys/cert.pem"),
+        format!("SKYDB_TLS_KEY=/var/skytable/keys/private.key"),
+        format!("SKYDB_AUTH_PLUGIN=pwd"),
+        format!("SKYDB_AUTH_ROOT_PASSWORD=password12345678"),
+        format!("SKYDB_ENDPOINTS=tcp@localhost:8080"),
+        format!("SKYDB_RUN_MODE=dev"),
+        format!("SKYDB_SERVICE_WINDOW=600"),
+    ];
+    let expected_args = vars_to_args(&variables);
+    config::set_env_src(variables.into());
+    let args = config::parse_env_args().unwrap().unwrap();
+    assert_eq!(args, expected_args);
+}
+#[test]
+fn parse_env_args_multi() {
+    let variables = [
+        format!("SKYDB_TLS_CERT=/var/skytable/keys/cert.pem"),
+        format!("SKYDB_TLS_KEY=/var/skytable/keys/private.key"),
+        format!("SKYDB_AUTH_PLUGIN=pwd"),
+        format!("SKYDB_AUTH_ROOT_PASSWORD=password12345678"),
+        format!("SKYDB_ENDPOINTS=tcp@localhost:8080,tls@localhost:8081"),
+        format!("SKYDB_RUN_MODE=dev"),
+        format!("SKYDB_SERVICE_WINDOW=600"),
+    ];
+    let expected_args = vars_to_args(&variables);
+    config::set_env_src(variables.into());
+    let args = config::parse_env_args().unwrap().unwrap();
+    assert_eq!(args, expected_args);
+}
+#[test]
+fn parse_validate_env_args() {
+    with_files(
+        [
+            "__env_args_test_cert.pem",
+            "__env_args_test_private.key",
+            "__env_args_test_private.passphrase.txt",
+        ],
+        |[cert, key, pass]| {
+            let variables = [
+                format!("SKYDB_AUTH_PLUGIN=pwd"),
+                format!("SKYDB_AUTH_ROOT_PASSWORD=password12345678"),
+                format!("SKYDB_TLS_CERT={cert}"),
+                format!("SKYDB_TLS_KEY={key}"),
+                format!("SKYDB_TLS_PRIVATE_KEY_PASSWORD={pass}"),
+                format!("SKYDB_ENDPOINTS=tcp@localhost:8080,tls@localhost:8081"),
+                format!("SKYDB_RUN_MODE=dev"),
+                format!("SKYDB_SERVICE_WINDOW=600"),
+            ];
+            config::set_env_src(variables.into());
+            let cfg = config::check_configuration().unwrap().into_config();
+            assert_eq!(
+                cfg,
+                Configuration::new(
+                    ConfigEndpoint::Multi(
+                        ConfigEndpointTcp::new("localhost".into(), 8080),
+                        ConfigEndpointTls::new(
+                            ConfigEndpointTcp::new("localhost".into(), 8081),
+                            "".into(),
+                            "".into(),
+                            "".into()
+                        )
+                    ),
+                    ConfigMode::Dev,
+                    ConfigSystem::new(600),
+                    ConfigAuth::new(AuthDriver::Pwd, "password12345678".into())
+                )
+            )
+        },
+    );
+}
+const CONFIG_FILE: &str = "\
+system:
+  mode: dev
+  rs_window: 600
+
+auth:
+  plugin: pwd
+  root_pass: password12345678
+
+endpoints:
+  secure:
+    host: 127.0.0.1
+    port: 2004
+    cert: ._test_sample_cert.pem
+    private_key: ._test_sample_private.key
+    pkey_passphrase: ._test_sample_private.pass.txt
+  insecure:
+    host: 127.0.0.1
+    port: 2003
+    ";
+#[test]
+fn test_config_file() {
+    with_files(
+        [
+            "._test_sample_cert.pem",
+            "._test_sample_private.key",
+            "._test_sample_private.pass.txt",
+        ],
+        |_| {
+            config::set_cli_src(vec!["skyd".into(), "--config=config.yml".into()]);
+            config::set_file_src(CONFIG_FILE);
+            let cfg = config::check_configuration().unwrap().into_config();
+            assert_eq!(
+                cfg,
+                Configuration::new(
+                    ConfigEndpoint::Multi(
+                        ConfigEndpointTcp::new("127.0.0.1".into(), 2003),
+                        ConfigEndpointTls::new(
+                            ConfigEndpointTcp::new("127.0.0.1".into(), 2004),
+                            "".into(),
+                            "".into(),
+                            "".into()
+                        )
+                    ),
+                    ConfigMode::Dev,
+                    ConfigSystem::new(600),
+                    ConfigAuth::new(AuthDriver::Pwd, "password12345678".into())
+                )
+            )
+        },
+    )
+}
diff --git a/server/src/engine/tests/client/mod.rs b/server/src/engine/tests/client/mod.rs
new file mode 100644
index 00000000..fb6a3bec
--- /dev/null
+++ b/server/src/engine/tests/client/mod.rs
@@ -0,0 +1,27 @@
+/*
+ * Created on Wed Nov 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod sysctl;
diff --git a/server/src/engine/tests/client/sysctl.rs b/server/src/engine/tests/client/sysctl.rs
new file mode 100644
index 00000000..928d93d9
--- /dev/null
+++ b/server/src/engine/tests/client/sysctl.rs
@@ -0,0 +1,41 @@
+/*
+ * Created on Wed Nov 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod status {
+    use {sky_macros::dbtest, skytable::query};
+    #[dbtest]
+    fn check_status_root() {
+        let mut db = db!();
+        db.query_parse::<()>(&query!("sysctl report status"))
+            .unwrap();
+    }
+    #[dbtest(switch_user(username = "user1"))]
+    fn check_status_standard_user() {
+        let mut db = db!();
+        db.query_parse::<()>(&query!("sysctl report status"))
+            .unwrap();
+    }
+}
diff --git a/server/src/engine/tests/mod.rs b/server/src/engine/tests/mod.rs
index 1ac07cb3..e92542a3 100644
--- a/server/src/engine/tests/mod.rs
+++ b/server/src/engine/tests/mod.rs
@@ -24,281 +24,5 @@
  *
 */
 
-mod cfg {
-    use crate::{
-        engine::config::{
-            self, AuthDriver, CLIConfigParseReturn, ConfigAuth, ConfigEndpoint, ConfigEndpointTcp,
-            ConfigEndpointTls, ConfigMode, ConfigReturn, ConfigSystem, Configuration,
-            ParsedRawArgs,
-        },
-        util::test_utils::with_files,
-    };
-
-    /*
-        CLI tests
-    */
-
-    fn extract_cli_args(payload: &str) -> std::collections::HashMap<String, Vec<String>> {
-        extract_cli_args_raw(payload).into_config()
-    }
-    fn extract_cli_args_raw(
-        payload: &str,
-    ) -> CLIConfigParseReturn<std::collections::HashMap<String, Vec<String>>> {
-        config::parse_cli_args(payload.split_ascii_whitespace().map_while(|item| {
-            let mut item = item.trim();
-            if item.ends_with("\n") {
-                item = &item[..item.len() - 1];
-            }
-            if item.is_empty() {
-                None
-            } else {
-                Some(item)
-            }
-        }))
-        .unwrap()
-    }
-    #[test]
-    fn parse_cli_args_simple() {
-        let payload = "skyd --mode dev --endpoint tcp@localhost:2003";
-        let cfg = extract_cli_args(payload);
-        let expected: ParsedRawArgs = into_dict! {
-            "--mode" => vec!["dev".into()],
-            "--endpoint" => vec!["tcp@localhost:2003".into()]
-        };
-        assert_eq!(cfg, expected);
-    }
-    #[test]
-    fn parse_cli_args_packed() {
-        let payload = "skyd --mode=dev --endpoint=tcp@localhost:2003";
-        let cfg = extract_cli_args(payload);
-        let expected: ParsedRawArgs = into_dict! {
-            "--mode" => vec!["dev".into()],
-            "--endpoint" => vec!["tcp@localhost:2003".into()]
-        };
-        assert_eq!(cfg, expected);
-    }
-    #[test]
-    fn parse_cli_args_multi() {
-        let payload = "skyd --mode=dev --endpoint tcp@localhost:2003";
-        let cfg = extract_cli_args(payload);
-        let expected: ParsedRawArgs = into_dict! {
-            "--mode" => vec!["dev".into()],
-            "--endpoint" => vec!["tcp@localhost:2003".into()]
-        };
-        assert_eq!(cfg, expected);
-    }
-    #[test]
-    fn parse_validate_cli_args() {
-        with_files(
-            [
-                "__cli_args_test_private.key",
-                "__cli_args_test_cert.pem",
-                "__cli_args_test_passphrase.key",
-            ],
-            |[pkey, cert, pass]| {
-                let payload = format!(
-                    "skyd --mode=dev \
-                --endpoint tcp@127.0.0.1:2003 \
-                --endpoint tls@127.0.0.2:2004 \
-                --service-window=600 \
-                --tlskey {pkey} \
-                --tlscert {cert} \
-                --tls-passphrase {pass} \
-                --auth-plugin pwd \
-                --auth-root-password password12345678
-                "
-                );
-                let cfg = extract_cli_args(&payload);
-                let ret = config::apply_and_validate::<config::CSCommandLine>(cfg)
-                    .unwrap()
-                    .into_config();
-                assert_eq!(
-                    ret,
-                    Configuration::new(
-                        ConfigEndpoint::Multi(
-                            ConfigEndpointTcp::new("127.0.0.1".into(), 2003),
-                            ConfigEndpointTls::new(
-                                ConfigEndpointTcp::new("127.0.0.2".into(), 2004),
-                                "".into(),
-                                "".into(),
-                                "".into()
-                            )
-                        ),
-                        ConfigMode::Dev,
-                        ConfigSystem::new(600),
-                        ConfigAuth::new(AuthDriver::Pwd, "password12345678".into())
-                    )
-                )
-            },
-        );
-    }
-    #[test]
-    fn parse_validate_cli_args_help_and_version() {
-        let pl1 = "skyd --help";
-        let pl2 = "skyd --version";
-        let ret1 = extract_cli_args_raw(pl1);
-        let ret2 = extract_cli_args_raw(pl2);
-        assert_eq!(ret1, CLIConfigParseReturn::Help);
-        assert_eq!(ret2, CLIConfigParseReturn::Version);
-        config::set_cli_src(vec!["skyd".into(), "--help".into()]);
-        let ret3 = config::check_configuration().unwrap();
-        config::set_cli_src(vec!["skyd".into(), "--version".into()]);
-        let ret4 = config::check_configuration().unwrap();
-        assert_eq!(
-            ret3,
-            ConfigReturn::HelpMessage(config::CLI_HELP.to_string())
-        );
-        assert_eq!(
-            ret4,
-            ConfigReturn::HelpMessage(format!(
-                "Skytable Database Server (skyd) v{}",
-                libsky::VERSION
-            ))
-        );
-    }
-
-    /*
-        env tests
-    */
-
-    fn vars_to_args(variables: &[String]) -> ParsedRawArgs {
-        variables
-            .iter()
-            .map(|var| {
-                var.split("=")
-                    .map(ToString::to_string)
-                    .collect::<Vec<String>>()
-            })
-            .map(|mut v| {
-                let key = v.remove(0);
-                let values = v.remove(0).split(",").map(ToString::to_string).collect();
-                (key, values)
-            })
-            .collect()
-    }
-    #[test]
-    fn parse_env_args_simple() {
-        let variables = [
-            format!("SKYDB_TLS_CERT=/var/skytable/keys/cert.pem"),
-            format!("SKYDB_TLS_KEY=/var/skytable/keys/private.key"),
-            format!("SKYDB_AUTH_PLUGIN=pwd"),
-            format!("SKYDB_AUTH_ROOT_PASSWORD=password12345678"),
-            format!("SKYDB_ENDPOINTS=tcp@localhost:8080"),
-            format!("SKYDB_RUN_MODE=dev"),
-            format!("SKYDB_SERVICE_WINDOW=600"),
-        ];
-        let expected_args = vars_to_args(&variables);
-        config::set_env_src(variables.into());
-        let args = config::parse_env_args().unwrap().unwrap();
-        assert_eq!(args, expected_args);
-    }
-    #[test]
-    fn parse_env_args_multi() {
-        let variables = [
-            format!("SKYDB_TLS_CERT=/var/skytable/keys/cert.pem"),
-            format!("SKYDB_TLS_KEY=/var/skytable/keys/private.key"),
-            format!("SKYDB_AUTH_PLUGIN=pwd"),
-            format!("SKYDB_AUTH_ROOT_PASSWORD=password12345678"),
-            format!("SKYDB_ENDPOINTS=tcp@localhost:8080,tls@localhost:8081"),
-            format!("SKYDB_RUN_MODE=dev"),
-            format!("SKYDB_SERVICE_WINDOW=600"),
-        ];
-        let expected_args = vars_to_args(&variables);
-        config::set_env_src(variables.into());
-        let args = config::parse_env_args().unwrap().unwrap();
-        assert_eq!(args, expected_args);
-    }
-    #[test]
-    fn parse_validate_env_args() {
-        with_files(
-            [
-                "__env_args_test_cert.pem",
-                "__env_args_test_private.key",
-                "__env_args_test_private.passphrase.txt",
-            ],
-            |[cert, key, pass]| {
-                let variables = [
-                    format!("SKYDB_AUTH_PLUGIN=pwd"),
-                    format!("SKYDB_AUTH_ROOT_PASSWORD=password12345678"),
-                    format!("SKYDB_TLS_CERT={cert}"),
-                    format!("SKYDB_TLS_KEY={key}"),
-                    format!("SKYDB_TLS_PRIVATE_KEY_PASSWORD={pass}"),
-                    format!("SKYDB_ENDPOINTS=tcp@localhost:8080,tls@localhost:8081"),
-                    format!("SKYDB_RUN_MODE=dev"),
-                    format!("SKYDB_SERVICE_WINDOW=600"),
-                ];
-                config::set_env_src(variables.into());
-                let cfg = config::check_configuration().unwrap().into_config();
-                assert_eq!(
-                    cfg,
-                    Configuration::new(
-                        ConfigEndpoint::Multi(
-                            ConfigEndpointTcp::new("localhost".into(), 8080),
-                            ConfigEndpointTls::new(
-                                ConfigEndpointTcp::new("localhost".into(), 8081),
-                                "".into(),
-                                "".into(),
-                                "".into()
-                            )
-                        ),
-                        ConfigMode::Dev,
-                        ConfigSystem::new(600),
-                        ConfigAuth::new(AuthDriver::Pwd, "password12345678".into())
-                    )
-                )
-            },
-        );
-    }
-    const CONFIG_FILE: &str = "\
-system:
-  mode: dev
-  rs_window: 600
-
-auth:
-  plugin: pwd
-  root_pass: password12345678
-
-endpoints:
-  secure:
-    host: 127.0.0.1
-    port: 2004
-    cert: ._test_sample_cert.pem
-    private_key: ._test_sample_private.key
-    pkey_passphrase: ._test_sample_private.pass.txt
-  insecure:
-    host: 127.0.0.1
-    port: 2003
-    ";
-    #[test]
-    fn test_config_file() {
-        with_files(
-            [
-                "._test_sample_cert.pem",
-                "._test_sample_private.key",
-                "._test_sample_private.pass.txt",
-            ],
-            |_| {
-                config::set_cli_src(vec!["skyd".into(), "--config=config.yml".into()]);
-                config::set_file_src(CONFIG_FILE);
-                let cfg = config::check_configuration().unwrap().into_config();
-                assert_eq!(
-                    cfg,
-                    Configuration::new(
-                        ConfigEndpoint::Multi(
-                            ConfigEndpointTcp::new("127.0.0.1".into(), 2003),
-                            ConfigEndpointTls::new(
-                                ConfigEndpointTcp::new("127.0.0.1".into(), 2004),
-                                "".into(),
-                                "".into(),
-                                "".into()
-                            )
-                        ),
-                        ConfigMode::Dev,
-                        ConfigSystem::new(600),
-                        ConfigAuth::new(AuthDriver::Pwd, "password12345678".into())
-                    )
-                )
-            },
-        )
-    }
-}
+mod cfg;
+mod client;
diff --git a/sky-macros/Cargo.toml b/sky-macros/Cargo.toml
index b2052679..1f24cf3f 100644
--- a/sky-macros/Cargo.toml
+++ b/sky-macros/Cargo.toml
@@ -11,7 +11,7 @@ proc-macro = true
 
 [dependencies]
 # external deps
-proc-macro2 = "1.0.66"
-quote = "1.0.32"
-rand = "0.8.5"
+proc-macro2 = "1.0.70"
+quote = "1.0.33"
 syn = { version = "1.0.109", features = ["full"] }
+libsky = { path = "../libsky" }
diff --git a/sky-macros/src/dbtest.rs b/sky-macros/src/dbtest.rs
new file mode 100644
index 00000000..9cabd02c
--- /dev/null
+++ b/sky-macros/src/dbtest.rs
@@ -0,0 +1,290 @@
+/*
+ * Created on Wed Nov 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use quote::quote;
+
+use {
+    crate::util::{self, AttributeKind},
+    proc_macro::TokenStream,
+    std::collections::HashMap,
+    syn::{parse_macro_input, AttributeArgs, ItemFn},
+};
+
+/*
+    host setup
+*/
+
+#[derive(Debug)]
+enum DbTestClient {
+    Skyhash,
+    Tcp,
+}
+
+struct DbConfig {
+    client: DbTestClient,
+    port: u16,
+    host: String,
+}
+
+impl Default for DbConfig {
+    fn default() -> Self {
+        Self {
+            client: DbTestClient::Skyhash,
+            port: libsky::test_utils::DEFAULT_PORT,
+            host: libsky::test_utils::DEFAULT_HOST.into(),
+        }
+    }
+}
+
+/*
+    client setup
+*/
+
+#[derive(Debug)]
+struct ClientConfig {
+    username: String,
+    password: String,
+}
+
+impl Default for ClientConfig {
+    fn default() -> Self {
+        Self {
+            username: libsky::test_utils::DEFAULT_USER_NAME.into(),
+            password: libsky::test_utils::DEFAULT_USER_PASS.into(),
+        }
+    }
+}
+
+/*
+    test setup
+*/
+
+#[derive(Debug)]
+enum TestStrategy {
+    Standard,
+    Relogin { username: String, password: String },
+}
+impl TestStrategy {
+    fn is_relogin(&self) -> bool {
+        matches!(self, TestStrategy::Relogin { .. })
+    }
+}
+
+struct TestSetup {
+    client: ClientConfig,
+    db: DbConfig,
+    strategy: TestStrategy,
+}
+
+fn parse_attrs(attrs: AttributeArgs) -> TestSetup {
+    let mut db_config = DbConfig::default();
+    let mut client_config = ClientConfig::default();
+    let mut collected_attrs = HashMap::new();
+    let mut strategy = TestStrategy::Standard;
+    for attr in attrs {
+        match util::extract_attribute(&attr) {
+            AttributeKind::Pair(k, v) => {
+                assert!(
+                    collected_attrs.insert(k.to_string(), v).is_none(),
+                    "duplicate key: {}",
+                    k.to_string()
+                );
+                continue;
+            }
+            AttributeKind::NestedAttrs { name, attrs } => match name.to_string().as_str() {
+                "switch_user" => {
+                    if strategy.is_relogin() {
+                        panic!("already set `switch_user` strategy");
+                    }
+                    let mut username = None;
+                    let mut password = None;
+                    for (key, data) in attrs.into_iter().map(AttributeKind::into_pair) {
+                        match key.to_string().as_str() {
+                            "username" => {
+                                assert!(username.is_none(), "username already set");
+                                username = Some(util::extract_str_from_lit(&data).unwrap());
+                            }
+                            "password" => {
+                                assert!(password.is_none(), "password already set");
+                                password = Some(util::extract_str_from_lit(&data).unwrap());
+                            }
+                            unknown_subattr => panic!(
+                                "unknown sub-attribute for `switch_user`: `{unknown_subattr}`"
+                            ),
+                        }
+                    }
+                    assert!(username.is_some(), "username must be set");
+                    strategy = TestStrategy::Relogin {
+                        username: username.unwrap(),
+                        password: password.unwrap_or(libsky::test_utils::DEFAULT_USER_PASS.into()),
+                    };
+                }
+                unknown => panic!("unknown nested attribute `{unknown}`"),
+            },
+            AttributeKind::Path(_) | AttributeKind::Lit(_) => {
+                panic!("unexpected tokens")
+            }
+        }
+    }
+    for (attr_name, attr_val) in collected_attrs {
+        match attr_name.as_str() {
+            "client" => match util::extract_str_from_lit(&attr_val).unwrap().as_str() {
+                "skyhash" => db_config.client = DbTestClient::Skyhash,
+                "tcp" => db_config.client = DbTestClient::Tcp,
+                unknown_client => panic!("unknown client mode {unknown_client}"),
+            },
+            "port" => db_config.port = util::extract_int_from_lit(&attr_val).unwrap(),
+            "host" => db_config.host = util::extract_str_from_lit(&attr_val).unwrap(),
+            "username" => {
+                assert!(
+                    !strategy.is_relogin(),
+                    "`username` makes no sense when used with strategy `switch_user`. instead, set dbtest(switch_user(username = ...))"
+                );
+                client_config.username = util::extract_str_from_lit(&attr_val).unwrap()
+            }
+            "password" => {
+                assert!(
+                    !strategy.is_relogin(),
+                    "`password` makes no sense when used with strategy `switch_user`. instead, set dbtest(switch_user(password = ...))"
+                );
+                client_config.password = util::extract_str_from_lit(&attr_val).unwrap();
+            }
+            unknown_attr => panic!("unknown dbtest attribute `{unknown_attr}`"),
+        }
+    }
+    TestSetup {
+        client: client_config,
+        db: db_config,
+        strategy,
+    }
+}
+
+pub fn dbtest(attrs: TokenStream, item: TokenStream) -> TokenStream {
+    let attr_args = parse_macro_input!(attrs as AttributeArgs);
+    let input_fn = parse_macro_input!(item as ItemFn);
+    let TestSetup {
+        client:
+            ClientConfig {
+                username: login_username,
+                password: login_password,
+            },
+        db: DbConfig { client, port, host },
+        strategy,
+    } = parse_attrs(attr_args);
+
+    let function_attrs = &input_fn.attrs;
+    let function_vis = &input_fn.vis;
+    let function_sig = &input_fn.sig;
+    let function_block = &input_fn.block;
+
+    let retfn = quote!(
+        #(#function_attrs)* #function_vis #function_sig
+    );
+    let mut block = quote! {
+        const __DBTEST_HOST: &str = #host;
+        const __DBTEST_PORT: u16 = #port;
+    };
+    match strategy {
+        TestStrategy::Standard => {
+            block = quote! {
+                #block
+                /// username set by [`sky_macros::dbtest`]
+                const __DBTEST_USER: &str = #login_username;
+                /// password set by [`sky_macros::dbtest`]
+                const __DBTEST_PASS: &str = #login_password;
+            };
+        }
+        TestStrategy::Relogin {
+            username: ref new_username,
+            password: ref new_password,
+        } => {
+            // we need to create an user, log in and log out
+            block = quote! {
+                #block
+                /// username set by [`sky_macros::dbtest`] (relogin)
+                const __DBTEST_USER: &str = #new_username;
+                /// password set by [`sky_macros::dbtest`] (relogin)
+                const __DBTEST_PASS: &str = #new_password;
+                {
+                    let mut db = skytable::Config::new(#host, #port, #login_username, #login_password).connect().unwrap();
+                    db.query_parse::<()>(
+                        &skytable::query!("sysctl create user ? with { password: ? }", #new_username, #new_password)
+                    ).unwrap();
+                }
+            };
+        }
+    }
+    match client {
+        DbTestClient::Skyhash => {
+            block = quote! {
+                #block
+                /// Get a Skyhash connection the database (defined by [`sky_macros::dbtest`])
+                macro_rules! db {
+                    () => {{
+                        skytable::Config::new(__DBTEST_HOST, __DBTEST_PORT, __DBTEST_USER, __DBTEST_PASS).connect().unwrap()
+                    }}
+                }
+            };
+        }
+        DbTestClient::Tcp => {
+            block = quote! {
+                #block
+                /// Get a TCP connection the database (defined by [`sky_macros::dbtest`])
+                macro_rules! tcp {
+                    () => {{
+                        std::net::TcpStream::connect((__DBTEST_HOST, __DBTEST_PORT)).unwrap()
+                    }}
+                }
+            };
+        }
+    }
+    let mut ret_block = quote! {
+        #block
+        #function_block
+    };
+    match strategy {
+        TestStrategy::Relogin { ref username, .. } => {
+            let new_username = username;
+            ret_block = quote! {
+                #ret_block
+                {
+                    let mut db = skytable::Config::new(#host, #port, #login_username, #login_password).connect().unwrap();
+                    db.query_parse::<()>(
+                        &skytable::query!("sysctl drop user ?", #new_username)
+                    ).unwrap();
+                }
+            };
+        }
+        TestStrategy::Standard => {}
+    }
+    let ret = quote! {
+        #[core::prelude::v1::test]
+        #retfn {
+            #ret_block
+        }
+    };
+    ret.into()
+}
diff --git a/sky-macros/src/lib.rs b/sky-macros/src/lib.rs
index cb04fb94..3d517caf 100644
--- a/sky-macros/src/lib.rs
+++ b/sky-macros/src/lib.rs
@@ -47,87 +47,15 @@ use {
     proc_macro::TokenStream,
     proc_macro2::TokenStream as TokenStream2,
     quote::quote,
-    syn::{parse_macro_input, Data, DataStruct, DeriveInput, Fields, Lit, Meta, NestedMeta},
+    syn::{parse_macro_input, Data, DataStruct, DeriveInput, Fields, Meta, NestedMeta},
 };
 
+mod dbtest;
 mod util;
 
-#[proc_macro]
-/// Get a compile time respcode/respstring array. For example, if you pass: "Unknown action",
-/// it will return: `!14\nUnknown Action\n`
-pub fn compiled_eresp_array(tokens: TokenStream) -> TokenStream {
-    _get_eresp_array(tokens, false)
-}
-
-#[proc_macro]
-/// Get a compile time respcode/respstring array. For example, if you pass: "Unknown action",
-/// it will return: `!14\n14\nUnknown Action\n`
-pub fn compiled_eresp_array_v1(tokens: TokenStream) -> TokenStream {
-    _get_eresp_array(tokens, true)
-}
-
-fn _get_eresp_array(tokens: TokenStream, sizeline: bool) -> TokenStream {
-    let payload_str = match syn::parse_macro_input!(tokens as Lit) {
-        Lit::Str(st) => st.value(),
-        _ => panic!("Expected a string literal"),
-    };
-    let mut processed = quote! {
-        b'!',
-    };
-    if sizeline {
-        let payload_len = payload_str.as_bytes().len();
-        let payload_len_str = payload_len.to_string();
-        let payload_len_bytes = payload_len_str.as_bytes();
-        for byte in payload_len_bytes {
-            processed = quote! {
-                #processed
-                #byte,
-            };
-        }
-        processed = quote! {
-            #processed
-            b'\n',
-        };
-    }
-    let payload_bytes = payload_str.as_bytes();
-    for byte in payload_bytes {
-        processed = quote! {
-            #processed
-            #byte,
-        }
-    }
-    processed = quote! {
-        #processed
-        b'\n',
-    };
-    processed = quote! {
-        [#processed]
-    };
-    processed.into()
-}
-
-#[proc_macro]
-/// Get a compile time respcode/respstring slice. For example, if you pass: "Unknown action",
-/// it will return: `!14\nUnknown Action\n`
-pub fn compiled_eresp_bytes(tokens: TokenStream) -> TokenStream {
-    let ret = compiled_eresp_array(tokens);
-    let ret = syn::parse_macro_input!(ret as syn::Expr);
-    quote! {
-        &#ret
-    }
-    .into()
-}
-
-#[proc_macro]
-/// Get a compile time respcode/respstring slice. For example, if you pass: "Unknown action",
-/// it will return: `!14\nUnknown Action\n`
-pub fn compiled_eresp_bytes_v1(tokens: TokenStream) -> TokenStream {
-    let ret = compiled_eresp_array_v1(tokens);
-    let ret = syn::parse_macro_input!(ret as syn::Expr);
-    quote! {
-        &#ret
-    }
-    .into()
+#[proc_macro_attribute]
+pub fn dbtest(attrs: TokenStream, item: TokenStream) -> TokenStream {
+    dbtest::dbtest(attrs, item)
 }
 
 #[proc_macro_derive(Wrapper)]
diff --git a/sky-macros/src/util.rs b/sky-macros/src/util.rs
index b55c029b..540dcd7a 100644
--- a/sky-macros/src/util.rs
+++ b/sky-macros/src/util.rs
@@ -1,5 +1,5 @@
 /*
- * Created on Wed Mar 09 2022
+ * Created on Wed Nov 29 2023
  *
  * This file is a part of Skytable
  * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@@ -7,7 +7,7 @@
  * vision to provide flexibility in data modelling without compromising
  * on performance, queryability or scalability.
  *
- * Copyright (c) 2022, Sayan Nandan <ohsayan@outlook.com>
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published by
@@ -24,66 +24,55 @@
  *
 */
 
-use {
-    core::{fmt::Display, str::FromStr},
-    proc_macro2::Span,
-    rand::Rng,
-    syn::{Lit, MetaNameValue},
-};
+use proc_macro2::Ident;
+use syn::{Lit, Meta, MetaNameValue, NestedMeta, Path};
 
-const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
-
-pub fn get_rand_string(rng: &mut impl Rng) -> String {
-    (0..64)
-        .map(|_| {
-            let idx = rng.gen_range(0..CHARSET.len());
-            CHARSET[idx] as char
-        })
-        .collect()
+pub enum AttributeKind {
+    Lit(Lit),
+    NestedAttrs { name: Ident, attrs: Vec<Self> },
+    Pair(Ident, Lit),
+    Path(Path),
 }
 
-pub fn parse_string(int: &Lit, span: Span, field: &str) -> Result<String, syn::Error> {
-    match int {
-        syn::Lit::Str(s) => Ok(s.value()),
-        syn::Lit::Verbatim(s) => Ok(s.to_string()),
-        _ => Err(syn::Error::new(
-            span,
-            format!("Failed to parse {} into a string.", field),
-        )),
+impl AttributeKind {
+    pub fn into_pair(self) -> (Ident, Lit) {
+        match self {
+            Self::Pair(i, l) => (i, l),
+            _ => panic!("expected attribute name pair"),
+        }
     }
 }
 
-pub fn parse_number<T: FromStr<Err = E>, E: Display>(
-    int: &Lit,
-    span: Span,
-    field: &str,
-) -> Result<T, syn::Error> {
-    match int {
-        syn::Lit::Int(int) => int.base10_parse::<T>(),
-        _ => Err(syn::Error::new(
-            span,
-            format!("Failed to parse {} into an int.", field),
-        )),
+pub fn extract_attribute(attr: &NestedMeta) -> AttributeKind {
+    match attr {
+        NestedMeta::Lit(l) => AttributeKind::Lit(l.clone()),
+        NestedMeta::Meta(m) => match m {
+            Meta::List(l) => AttributeKind::NestedAttrs {
+                name: l.path.get_ident().unwrap().clone(),
+                attrs: l.nested.iter().map(extract_attribute).collect(),
+            },
+            Meta::NameValue(MetaNameValue { path, lit, .. }) => {
+                AttributeKind::Pair(path.get_ident().unwrap().clone(), lit.clone())
+            }
+            Meta::Path(p) => AttributeKind::Path(p.clone()),
+        },
     }
 }
 
-pub fn parse_bool(boolean: &Lit, span: Span, field: &str) -> Result<bool, syn::Error> {
-    match boolean {
-        Lit::Bool(boolean) => Ok(boolean.value),
-        _ => Err(syn::Error::new(
-            span,
-            format!("Failed to parse {} into a boolean.", field),
-        )),
+pub fn extract_str_from_lit(l: &Lit) -> Option<String> {
+    match l {
+        Lit::Str(s) => Some(s.value()),
+        _ => None,
     }
 }
 
-pub fn get_metanamevalue_data(namevalue: &MetaNameValue) -> (String, &Lit, Span) {
-    match namevalue
-        .path
-        .get_ident()
-        .map(|ident| ident.to_string().to_lowercase())
-    {
-        None => panic!("Must have specified ident!"),
-        Some(ident) => (ident, &namevalue.lit, namevalue.lit.span()),
+pub fn extract_int_from_lit<I>(l: &Lit) -> Option<I>
+where
+    I: std::str::FromStr,
+    I::Err: std::fmt::Display,
+{
+    match l {
+        Lit::Int(i) => i.base10_parse::<I>().ok(),
+        _ => None,
     }
 }

From d090e5ce37011781e278ba25f71f84b0f8c44e58 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 29 Nov 2023 19:29:26 +0530
Subject: [PATCH 298/310] Add basic dbtests for space DDL

---
 server/src/engine/ql/ast/mod.rs       | 22 +++++----
 server/src/engine/sync/cell.rs        |  8 +++-
 server/src/engine/tests/client/ddl.rs | 65 +++++++++++++++++++++++++++
 server/src/engine/tests/client/mod.rs |  1 +
 server/src/engine/tests/macros.rs     | 40 +++++++++++++++++
 server/src/engine/tests/mod.rs        |  2 +
 sky-macros/src/dbtest.rs              |  1 +
 7 files changed, 129 insertions(+), 10 deletions(-)
 create mode 100644 server/src/engine/tests/client/ddl.rs
 create mode 100644 server/src/engine/tests/macros.rs

diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 7627ca2f..9c3cbcfc 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -106,11 +106,11 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     }
     pub fn try_entity_ref(&mut self) -> Option<EntityIDRef<'a>> {
         let self_has_full = Self::_entity_signature_match_self_full(
-            &self.t[self.round_cursor()],
-            &self.t[self.round_cursor_up(1)],
-            &self.t[self.round_cursor_up(2)],
+            self.offset_current_r(0),
+            self.offset_current_r(1),
+            self.offset_current_r(2),
         );
-        let self_has_pre_full = self._entity_signature_match_cs(&self.t[self.round_cursor()]);
+        let self_has_pre_full = self._entity_signature_match_cs(self.offset_current_r(0));
         if self_has_full {
             unsafe {
                 // UNSAFE(@ohsayan): +branch condition
@@ -190,6 +190,10 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
         &self.t[self.i..]
     }
     #[inline(always)]
+    pub fn offset_current_r(&self, offset: usize) -> &Token<'a> {
+        &self.t[self.round_cursor_up(offset)]
+    }
+    #[inline(always)]
     /// Returns a count of the number of consumable tokens remaining
     pub fn remaining(&self) -> usize {
         self.t.len() - self.i
@@ -267,16 +271,16 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     }
     #[inline(always)]
     pub(crate) fn cursor_has_ident_rounded(&self) -> bool {
-        self.t[self.round_cursor()].is_ident() && self.not_exhausted()
+        self.offset_current_r(0).is_ident() && self.not_exhausted()
     }
     #[inline(always)]
     /// Check if the current token stream matches the signature of an arity(0) fn; rounded
     ///
     /// NOTE: Consider using a direct comparison without rounding
     pub(crate) fn cursor_signature_match_fn_arity0_rounded(&self) -> bool {
-        (self.t[self.round_cursor()].is_ident())
-            & (self.t[self.round_cursor_up(1)] == Token![() open])
-            & (self.t[self.round_cursor_up(2)] == Token![() close])
+        (self.offset_current_r(0).is_ident())
+            & (Token![() open].eq(self.offset_current_r(1)))
+            & (Token![() close].eq(self.offset_current_r(2)))
             & self.has_remaining(3)
     }
     #[inline(always)]
@@ -319,7 +323,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
         }
     }
     pub fn ensure_minimum_for_blocking_stmt(&self) -> QueryResult<()> {
-        if self.remaining() < 2 {
+        if self.exhausted() {
             return Err(QueryError::QLExpectedStatement);
         } else {
             Ok(())
diff --git a/server/src/engine/sync/cell.rs b/server/src/engine/sync/cell.rs
index 3960a2ee..7045db4f 100644
--- a/server/src/engine/sync/cell.rs
+++ b/server/src/engine/sync/cell.rs
@@ -39,7 +39,7 @@ use {
 
 /// A lazily intialized, or _call by need_ value
 #[derive(Debug)]
-pub struct Lazy<T, F> {
+pub struct Lazy<T, F = fn() -> T> {
     /// the value (null at first)
     value: AtomicPtr<T>,
     /// the function that will init the value
@@ -48,6 +48,12 @@ pub struct Lazy<T, F> {
     init_state: AtomicBool,
 }
 
+impl<T: Default> Default for Lazy<T> {
+    fn default() -> Self {
+        Self::new(T::default)
+    }
+}
+
 impl<T, F> Lazy<T, F> {
     pub const fn new(init_func: F) -> Self {
         Self {
diff --git a/server/src/engine/tests/client/ddl.rs b/server/src/engine/tests/client/ddl.rs
new file mode 100644
index 00000000..ce321a37
--- /dev/null
+++ b/server/src/engine/tests/client/ddl.rs
@@ -0,0 +1,65 @@
+/*
+ * Created on Wed Nov 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use skytable::error::Error;
+use {crate::engine::error::QueryError, sky_macros::dbtest, skytable::query};
+
+const INVALID_SYNTAX_ERR: u16 = QueryError::QLInvalidSyntax.value_u8() as u16;
+
+#[dbtest]
+fn ensure_create_space_end_of_tokens() {
+    let mut con = db!();
+    assert_err_eq!(
+        con.query_parse::<()>(&query!("create space myspace with {} this_should_fail")),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+    assert_err_eq!(
+        con.query_parse::<()>(&query!("create space myspace this_should_fail")),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+}
+
+#[dbtest]
+fn ensure_alter_space_end_of_tokens() {
+    let mut con = db!();
+    assert_err_eq!(
+        con.query_parse::<()>(&query!("alter space myspace with {} this_should_fail")),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+}
+
+#[dbtest]
+fn ensure_drop_space_end_of_tokens() {
+    let mut con = db!();
+    assert_err_eq!(
+        con.query_parse::<()>(&query!("drop space myspace this_should_fail")),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+    assert_err_eq!(
+        con.query_parse::<()>(&query!("drop space myspace force this_should_fail")),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+}
diff --git a/server/src/engine/tests/client/mod.rs b/server/src/engine/tests/client/mod.rs
index fb6a3bec..1ccfcc07 100644
--- a/server/src/engine/tests/client/mod.rs
+++ b/server/src/engine/tests/client/mod.rs
@@ -24,4 +24,5 @@
  *
 */
 
+mod ddl;
 mod sysctl;
diff --git a/server/src/engine/tests/macros.rs b/server/src/engine/tests/macros.rs
new file mode 100644
index 00000000..75cda79c
--- /dev/null
+++ b/server/src/engine/tests/macros.rs
@@ -0,0 +1,40 @@
+/*
+ * Created on Wed Nov 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+macro_rules! assert_err_eq {
+    ($me:expr, $target:pat) => {
+        match ::core::result::Result::unwrap_err($me) {
+            $target => {}
+            other => panic!(
+                "expected error `{}` but got {:?} at {}:{}",
+                stringify!($target),
+                other,
+                file!(),
+                line!()
+            ),
+        }
+    };
+}
diff --git a/server/src/engine/tests/mod.rs b/server/src/engine/tests/mod.rs
index e92542a3..9b2dbecb 100644
--- a/server/src/engine/tests/mod.rs
+++ b/server/src/engine/tests/mod.rs
@@ -24,5 +24,7 @@
  *
 */
 
+#[macro_use]
+mod macros;
 mod cfg;
 mod client;
diff --git a/sky-macros/src/dbtest.rs b/sky-macros/src/dbtest.rs
index 9cabd02c..6f81f23b 100644
--- a/sky-macros/src/dbtest.rs
+++ b/sky-macros/src/dbtest.rs
@@ -87,6 +87,7 @@ enum TestStrategy {
     Standard,
     Relogin { username: String, password: String },
 }
+
 impl TestStrategy {
     fn is_relogin(&self) -> bool {
         matches!(self, TestStrategy::Relogin { .. })

From acd7b3842cca55c0de5e10ccab3768349c43c6b5 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 29 Nov 2023 22:08:29 +0530
Subject: [PATCH 299/310] Add basic sec tests for all DDL, DCL and DML queries

---
 server/src/engine/ql/ast/mod.rs               |  14 +-
 server/src/engine/tests/client/mod.rs         |   2 +-
 .../tests/client/{ddl.rs => sec/dcl_sec.rs}   |  52 +++---
 server/src/engine/tests/client/sec/ddl_sec.rs | 160 ++++++++++++++++++
 server/src/engine/tests/client/sec/dml_sec.rs |  89 ++++++++++
 server/src/engine/tests/client/sec/mod.rs     |  54 ++++++
 server/src/engine/tests/macros.rs             |  13 ++
 7 files changed, 358 insertions(+), 26 deletions(-)
 rename server/src/engine/tests/client/{ddl.rs => sec/dcl_sec.rs} (58%)
 create mode 100644 server/src/engine/tests/client/sec/ddl_sec.rs
 create mode 100644 server/src/engine/tests/client/sec/dml_sec.rs
 create mode 100644 server/src/engine/tests/client/sec/mod.rs

diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index 9c3cbcfc..b0e2422e 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -39,7 +39,7 @@ use {
             data::{cell::Datacell, lit::Lit},
             error::{QueryError, QueryResult},
         },
-        util::MaybeInit,
+        util::{compiler, MaybeInit},
     },
 };
 
@@ -317,13 +317,17 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
         self.round_cursor_up(0)
     }
     pub fn try_statement(&mut self) -> QueryResult<KeywordStmt> {
-        match self.fw_read() {
-            Token::Keyword(Keyword::Statement(stmt)) => Ok(*stmt),
-            _ => Err(QueryError::QLExpectedStatement),
+        if compiler::unlikely(self.exhausted()) {
+            compiler::cold_call(|| Err(QueryError::QLExpectedStatement))
+        } else {
+            match self.fw_read() {
+                Token::Keyword(Keyword::Statement(stmt)) => Ok(*stmt),
+                _ => Err(QueryError::QLExpectedStatement),
+            }
         }
     }
     pub fn ensure_minimum_for_blocking_stmt(&self) -> QueryResult<()> {
-        if self.exhausted() {
+        if self.remaining() < 2 {
             return Err(QueryError::QLExpectedStatement);
         } else {
             Ok(())
diff --git a/server/src/engine/tests/client/mod.rs b/server/src/engine/tests/client/mod.rs
index 1ccfcc07..bff9291a 100644
--- a/server/src/engine/tests/client/mod.rs
+++ b/server/src/engine/tests/client/mod.rs
@@ -24,5 +24,5 @@
  *
 */
 
-mod ddl;
+mod sec;
 mod sysctl;
diff --git a/server/src/engine/tests/client/ddl.rs b/server/src/engine/tests/client/sec/dcl_sec.rs
similarity index 58%
rename from server/src/engine/tests/client/ddl.rs
rename to server/src/engine/tests/client/sec/dcl_sec.rs
index ce321a37..9db90b34 100644
--- a/server/src/engine/tests/client/ddl.rs
+++ b/server/src/engine/tests/client/sec/dcl_sec.rs
@@ -24,42 +24,54 @@
  *
 */
 
-use skytable::error::Error;
-use {crate::engine::error::QueryError, sky_macros::dbtest, skytable::query};
+use {
+    super::{INVALID_SYNTAX_ERR, UNKNOWN_STMT_ERR},
+    sky_macros::dbtest,
+    skytable::{error::Error, query},
+};
 
-const INVALID_SYNTAX_ERR: u16 = QueryError::QLInvalidSyntax.value_u8() as u16;
+#[dbtest]
+fn deny_unknown_sysctl() {
+    let mut db = db!();
+    for stmt in [
+        "sysctl magic moon",
+        "sysctl create wormhole",
+        "sysctl drop dem",
+    ] {
+        assert_err_eq!(
+            db.query_parse::<()>(&query!(stmt)),
+            Error::ServerError(UNKNOWN_STMT_ERR)
+        );
+    }
+}
 
 #[dbtest]
-fn ensure_create_space_end_of_tokens() {
-    let mut con = db!();
-    assert_err_eq!(
-        con.query_parse::<()>(&query!("create space myspace with {} this_should_fail")),
-        Error::ServerError(INVALID_SYNTAX_ERR)
-    );
+fn ensure_sysctl_status_end_of_tokens() {
+    let mut db = db!();
     assert_err_eq!(
-        con.query_parse::<()>(&query!("create space myspace this_should_fail")),
+        db.query_parse::<()>(&query!("sysctl report status blah")),
         Error::ServerError(INVALID_SYNTAX_ERR)
     );
 }
 
 #[dbtest]
-fn ensure_alter_space_end_of_tokens() {
-    let mut con = db!();
+fn ensure_sysctl_create_user() {
+    let mut db = db!();
     assert_err_eq!(
-        con.query_parse::<()>(&query!("alter space myspace with {} this_should_fail")),
+        db.query_parse::<()>(&query!(
+            "sysctl create user ? with { password: ? } blah",
+            "myuser",
+            "mypass"
+        )),
         Error::ServerError(INVALID_SYNTAX_ERR)
     );
 }
 
 #[dbtest]
-fn ensure_drop_space_end_of_tokens() {
-    let mut con = db!();
-    assert_err_eq!(
-        con.query_parse::<()>(&query!("drop space myspace this_should_fail")),
-        Error::ServerError(INVALID_SYNTAX_ERR)
-    );
+fn ensure_sysctl_drop_user() {
+    let mut db = db!();
     assert_err_eq!(
-        con.query_parse::<()>(&query!("drop space myspace force this_should_fail")),
+        db.query_parse::<()>(&query!("sysctl drop user ? blah", "myuser",)),
         Error::ServerError(INVALID_SYNTAX_ERR)
     );
 }
diff --git a/server/src/engine/tests/client/sec/ddl_sec.rs b/server/src/engine/tests/client/sec/ddl_sec.rs
new file mode 100644
index 00000000..2a4d8afd
--- /dev/null
+++ b/server/src/engine/tests/client/sec/ddl_sec.rs
@@ -0,0 +1,160 @@
+/*
+ * Created on Wed Nov 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::{INVALID_SYNTAX_ERR, UNKNOWN_STMT_ERR},
+    sky_macros::dbtest,
+    skytable::{error::Error, query},
+};
+
+#[dbtest]
+fn deny_unknown() {
+    let mut db = db!();
+    for stmt in [
+        "create magic blue",
+        "alter rainbow hue",
+        "drop sadistic view",
+    ] {
+        assert_err_eq!(
+            db.query_parse::<()>(&query!(stmt)),
+            Error::ServerError(UNKNOWN_STMT_ERR)
+        );
+    }
+}
+
+#[dbtest]
+fn ensure_create_space_end_of_tokens() {
+    let mut db = db!();
+    assert_err_eq!(
+        db.query_parse::<()>(&query!("create space myspace with {} blah")),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+    assert_err_eq!(
+        db.query_parse::<()>(&query!("create space myspace blah")),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+}
+
+#[dbtest]
+fn ensure_alter_space_end_of_tokens() {
+    let mut db = db!();
+    assert_err_eq!(
+        db.query_parse::<()>(&query!("alter space myspace with {} blah")),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+}
+
+#[dbtest]
+fn ensure_drop_space_end_of_tokens() {
+    let mut db = db!();
+    assert_err_eq!(
+        db.query_parse::<()>(&query!("drop space myspace blah")),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+    assert_err_eq!(
+        db.query_parse::<()>(&query!("drop space myspace force blah")),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+}
+
+#[dbtest]
+fn ensure_create_model_end_of_tokens() {
+    let mut db = db!();
+    assert_err_eq!(
+        db.query_parse::<()>(&query!(
+            "create model myspace.mymodel(username: string, password: binary) blah"
+        )),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+    assert_err_eq!(
+        db.query_parse::<()>(&query!(
+            "create model myspace.mymodel(username: string, password: binary) with {} blah"
+        )),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+}
+
+#[dbtest]
+fn ensure_alter_model_add_end_of_tokens() {
+    let mut db = db!();
+    assert_err_eq!(
+        db.query_parse::<()>(&query!(
+            "alter model myspace.mymodel add phone_number { type: uint64 } blah"
+        )),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+    assert_err_eq!(
+        db.query_parse::<()>(&query!(
+            "alter model myspace.mymodel add (phone_number { type: uint64 }, email_id { type: string }) with {} blah"
+        )),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+}
+
+#[dbtest]
+fn ensure_alter_model_update_end_of_tokens() {
+    let mut db = db!();
+    assert_err_eq!(
+        db.query_parse::<()>(&query!(
+            "alter model myspace.mymodel update password { type: string } blah"
+        )),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+    assert_err_eq!(
+        db.query_parse::<()>(&query!(
+            "alter model myspace.mymodel update (username {type: binary}, password { type: string }) blah"
+        )),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+}
+
+#[dbtest]
+fn ensure_alter_model_remove_end_of_tokens() {
+    let mut db = db!();
+    assert_err_eq!(
+        db.query_parse::<()>(&query!("alter model myspace.mymodel remove email_id blah")),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+    assert_err_eq!(
+        db.query_parse::<()>(&query!(
+            "alter model myspace.mymodel remove (email_id, phone_number) blah"
+        )),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+}
+
+#[dbtest]
+fn ensure_drop_model_end_of_tokens() {
+    let mut db = db!();
+    assert_err_eq!(
+        db.query_parse::<()>(&query!("drop model myspace.mymodel blah")),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+    assert_err_eq!(
+        db.query_parse::<()>(&query!("drop model myspace.mymodel force blah")),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+}
diff --git a/server/src/engine/tests/client/sec/dml_sec.rs b/server/src/engine/tests/client/sec/dml_sec.rs
new file mode 100644
index 00000000..301e91a0
--- /dev/null
+++ b/server/src/engine/tests/client/sec/dml_sec.rs
@@ -0,0 +1,89 @@
+/*
+ * Created on Wed Nov 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {
+    super::INVALID_SYNTAX_ERR,
+    sky_macros::dbtest,
+    skytable::{error::Error, query},
+};
+
+#[dbtest]
+fn insert_ensure_end_of_tokens() {
+    let mut db = db!();
+    assert_err_eq!(
+        db.query_parse::<()>(&query!(
+            "insert into myspace.mymodel(?, ?) blah",
+            "username",
+            "password"
+        )),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+    assert_err_eq!(
+        db.query_parse::<()>(&query!(
+            "insert into myspace.mymodel { username: ?, password: ? } blah",
+            "username",
+            "password"
+        )),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    );
+}
+
+#[dbtest]
+fn select_ensure_end_of_tokens() {
+    let mut db = db!();
+    assert_err_eq!(
+        db.query_parse::<()>(&query!(
+            "select * from myspace.mymodel where username = ? blah",
+            "username",
+        )),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    )
+}
+
+#[dbtest]
+fn update_ensure_end_of_tokens() {
+    let mut db = db!();
+    assert_err_eq!(
+        db.query_parse::<()>(&query!(
+            "update myspace.mymodel set counter += ? where username = ? blah",
+            1u64,
+            "username",
+        )),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    )
+}
+
+#[dbtest]
+fn delete_ensure_end_of_tokens() {
+    let mut db = db!();
+    assert_err_eq!(
+        db.query_parse::<()>(&query!(
+            "delete from myspace.mymodel where username = ? blah",
+            "username",
+        )),
+        Error::ServerError(INVALID_SYNTAX_ERR)
+    )
+}
diff --git a/server/src/engine/tests/client/sec/mod.rs b/server/src/engine/tests/client/sec/mod.rs
new file mode 100644
index 00000000..be1fa0bb
--- /dev/null
+++ b/server/src/engine/tests/client/sec/mod.rs
@@ -0,0 +1,54 @@
+/*
+ * Created on Wed Nov 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod dcl_sec;
+mod ddl_sec;
+mod dml_sec;
+
+use {
+    crate::engine::error::QueryError,
+    sky_macros::dbtest,
+    skytable::{error::Error, query},
+};
+
+const INVALID_SYNTAX_ERR: u16 = QueryError::QLInvalidSyntax.value_u8() as u16;
+const EXPECTED_STATEMENT_ERR: u16 = QueryError::QLExpectedStatement.value_u8() as u16;
+const UNKNOWN_STMT_ERR: u16 = QueryError::QLUnknownStatement.value_u8() as u16;
+
+#[dbtest]
+fn deny_unknown_tokens() {
+    let mut db = db!();
+    for token in [
+        "model", "space", "where", "force", "into", "from", "with", "set", "add", "remove", "*",
+        ",", "",
+    ] {
+        assert_err_eq!(
+            db.query_parse::<()>(&query!(token)),
+            Error::ServerError(EXPECTED_STATEMENT_ERR),
+            "{token}",
+        );
+    }
+}
diff --git a/server/src/engine/tests/macros.rs b/server/src/engine/tests/macros.rs
index 75cda79c..3f55138b 100644
--- a/server/src/engine/tests/macros.rs
+++ b/server/src/engine/tests/macros.rs
@@ -37,4 +37,17 @@ macro_rules! assert_err_eq {
             ),
         }
     };
+    ($me:expr, $target:pat, $($arg:tt)+) => {
+        match ::core::result::Result::expect_err($me, &format!($($arg)*)) {
+            $target => {}
+            other => panic!(
+                "expected error `{}` but got {:?} at {}:{}; {}",
+                stringify!($target),
+                other,
+                file!(),
+                line!(),
+                $($arg)*
+            ),
+        }
+    }
 }

From fce183afd9466fa217d060fdbc89e75776461fab Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 30 Nov 2023 00:13:05 +0530
Subject: [PATCH 300/310] Add `use $current`

---
 Cargo.lock                            | 16 ++++++++--------
 cli/src/query.rs                      | 26 ++++++++++++++++++++++++--
 cli/src/repl.rs                       | 25 +++++++++++++++++++++++--
 cli/src/resp.rs                       |  9 ++++++---
 server/src/engine/core/exec.rs        | 16 +++++++++++++++-
 server/src/engine/net/protocol/mod.rs |  2 ++
 server/src/engine/ql/ddl/mod.rs       | 16 +++++++++++++---
 server/src/engine/ql/tests/misc.rs    | 10 ++++++++++
 8 files changed, 101 insertions(+), 19 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 0e30641b..e644f78e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -647,9 +647,9 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
 
 [[package]]
 name = "memchr"
-version = "2.5.0"
+version = "2.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
 
 [[package]]
 name = "memoffset"
@@ -965,9 +965,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.9.3"
+version = "1.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a"
+checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -977,9 +977,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.3.6"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69"
+checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -988,9 +988,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.4"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
 
 [[package]]
 name = "rustc-demangle"
diff --git a/cli/src/query.rs b/cli/src/query.rs
index f4204131..eb9d0e08 100644
--- a/cli/src/query.rs
+++ b/cli/src/query.rs
@@ -75,6 +75,13 @@ pub struct Parameterizer {
     query: Vec<u8>,
 }
 
+#[derive(Debug, PartialEq)]
+pub enum ExecKind {
+    Standard(Query),
+    UseSpace(Query, String),
+    UseNull(Query),
+}
+
 impl Parameterizer {
     pub fn new(q: String) -> Self {
         Self {
@@ -84,7 +91,7 @@ impl Parameterizer {
             query: vec![],
         }
     }
-    pub fn parameterize(mut self) -> CliResult<Query> {
+    pub fn parameterize(mut self) -> CliResult<ExecKind> {
         while self.not_eof() {
             match self.buf[self.i] {
                 b if b.is_ascii_alphabetic() || b == b'_' => self.read_ident(),
@@ -111,7 +118,22 @@ impl Parameterizer {
                 self.params.into_iter().for_each(|p| {
                     q.push_param(p);
                 });
-                Ok(q)
+                Ok(if qstr.eq_ignore_ascii_case("use null") {
+                    ExecKind::UseNull(q)
+                } else {
+                    let mut splits = qstr.split_ascii_whitespace();
+                    let tok_use = splits.next();
+                    let tok_name = splits.next();
+                    match (tok_use, tok_name) {
+                        (Some(tok_use), Some(tok_name))
+                            if tok_use.eq_ignore_ascii_case("use")
+                                && !tok_name.eq_ignore_ascii_case("$current") =>
+                        {
+                            ExecKind::UseSpace(q, tok_name.into())
+                        }
+                        _ => ExecKind::Standard(q),
+                    }
+                })
             }
             Err(_) => Err(CliError::QueryError("query is not valid UTF-8".into())),
         }
diff --git a/cli/src/repl.rs b/cli/src/repl.rs
index 48ea52b6..8a122161 100644
--- a/cli/src/repl.rs
+++ b/cli/src/repl.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use crate::query::ExecKind;
+
 use {
     crate::{
         args::{ClientConfig, ClientConfigKind},
@@ -91,8 +93,9 @@ fn repl<C: IsConnection>(mut con: C) -> CliResult<()> {
         Ok(e) => e,
         Err(e) => fatal!("error: failed to init REPL. {e}"),
     };
+    let mut prompt = "> ".to_owned();
     loop {
-        match editor.readline("> ") {
+        match editor.readline(&prompt) {
             Ok(line) => match line.as_str() {
                 "!help" => println!("{TXT_WELCOME}"),
                 "exit" => break,
@@ -102,7 +105,25 @@ fn repl<C: IsConnection>(mut con: C) -> CliResult<()> {
                         continue;
                     }
                     match query::Parameterizer::new(line).parameterize() {
-                        Ok(q) => resp::format_response(con.execute_query(q)?)?,
+                        Ok(q) => {
+                            let mut new_prompt = None;
+                            let q = match q {
+                                ExecKind::Standard(q) => q,
+                                ExecKind::UseNull(q) => {
+                                    new_prompt = Some("> ".into());
+                                    q
+                                }
+                                ExecKind::UseSpace(q, space) => {
+                                    new_prompt = Some(format!("{space}> "));
+                                    q
+                                }
+                            };
+                            if resp::format_response(con.execute_query(q)?)? {
+                                if let Some(pr) = new_prompt {
+                                    prompt = pr;
+                                }
+                            }
+                        }
                         Err(e) => match e {
                             CliError::QueryError(e) => {
                                 eprintln!("[skysh error]: bad query. {e}");
diff --git a/cli/src/resp.rs b/cli/src/resp.rs
index c35f3374..163b2b85 100644
--- a/cli/src/resp.rs
+++ b/cli/src/resp.rs
@@ -34,10 +34,13 @@ use {
     std::io::{self, Write},
 };
 
-pub fn format_response(resp: Response) -> CliResult<()> {
+pub fn format_response(resp: Response) -> CliResult<bool> {
     match resp {
         Response::Empty => print_cyan("(Okay)\n")?,
-        Response::Error(e) => print_red(&format!("(server error code: {e})\n"))?,
+        Response::Error(e) => {
+            print_red(&format!("(server error code: {e})\n"))?;
+            return Ok(false);
+        }
         Response::Value(v) => {
             print_value(v)?;
             println!();
@@ -47,7 +50,7 @@ pub fn format_response(resp: Response) -> CliResult<()> {
             println!();
         }
     };
-    Ok(())
+    Ok(true)
 }
 
 fn print_row(r: Row) -> CliResult<()> {
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 29586c2e..79492c06 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -28,7 +28,7 @@ use crate::engine::{
     core::{dml, model::Model, space::Space},
     error::{QueryError, QueryResult},
     fractal::{Global, GlobalInstanceLike},
-    net::protocol::{ClientLocalState, Response, SQuery},
+    net::protocol::{ClientLocalState, Response, ResponseType, SQuery},
     ql::{
         ast::{traits::ASTNode, InplaceData, State},
         ddl::Use,
@@ -162,6 +162,20 @@ fn cstate_use(
             }
             cstate.set_cs(new_space.boxed_str());
         }
+        Use::RefreshCurrent => match cstate.get_cs() {
+            None => return Ok(Response::Null),
+            Some(space) => {
+                if !global.namespace().contains_space(space) {
+                    cstate.unset_cs();
+                    return Err(QueryError::QExecObjectNotFound);
+                }
+                return Ok(Response::Serialized {
+                    ty: ResponseType::String,
+                    size: space.len(),
+                    data: space.to_owned().into_bytes(),
+                });
+            }
+        },
     }
     Ok(Response::Empty)
 }
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index a8b58074..bf3e0b0b 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -113,6 +113,7 @@ impl ClientLocalState {
 #[derive(Debug, PartialEq)]
 pub enum Response {
     Empty,
+    Null,
     Serialized {
         ty: ResponseType,
         size: usize,
@@ -190,6 +191,7 @@ pub(super) async fn query_loop<S: Socket>(
                 con.write_u8(b'\n').await?;
                 con.write_all(&data).await?;
             }
+            Ok(Response::Null) => con.write_u8(ResponseType::Null.value_u8()).await?,
             Err(e) => {
                 let [a, b] = (e.value_u8() as u16).to_le_bytes();
                 con.write_all(&[ResponseType::Error.value_u8(), a, b])
diff --git a/server/src/engine/ql/ddl/mod.rs b/server/src/engine/ql/ddl/mod.rs
index 4f0e0c21..8cc1021f 100644
--- a/server/src/engine/ql/ddl/mod.rs
+++ b/server/src/engine/ql/ddl/mod.rs
@@ -41,24 +41,34 @@ use {
 #[derive(Debug, PartialEq)]
 pub enum Use<'a> {
     Space(Ident<'a>),
+    RefreshCurrent,
     Null,
 }
 
 impl<'a> ASTNode<'a> for Use<'a> {
     const MUST_USE_FULL_TOKEN_RANGE: bool = true;
-    const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = true;
+    const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
     fn __base_impl_parse_from_state<Qd: super::ast::QueryData<'a>>(
         state: &mut super::ast::State<'a, Qd>,
     ) -> crate::engine::error::QueryResult<Self> {
         /*
             should have either an ident or null
         */
-        if state.remaining() != 1 {
+        if state.exhausted() | (state.remaining() > 2) {
             return Err(QueryError::QLInvalidSyntax);
         }
         Ok(match state.fw_read() {
+            Token::Ident(new_space) => Self::Space(*new_space),
             Token![null] => Self::Null,
-            Token::Ident(id) => Self::Space(id.clone()),
+            Token![$] => {
+                if state.exhausted() {
+                    return Err(QueryError::QLInvalidSyntax);
+                }
+                match state.fw_read() {
+                    Token::Ident(id) if id.eq_ignore_ascii_case("current") => Self::RefreshCurrent,
+                    _ => return Err(QueryError::QLInvalidSyntax),
+                }
+            }
             _ => return Err(QueryError::QLInvalidSyntax),
         })
     }
diff --git a/server/src/engine/ql/tests/misc.rs b/server/src/engine/ql/tests/misc.rs
index 70b379cd..e396d326 100644
--- a/server/src/engine/ql/tests/misc.rs
+++ b/server/src/engine/ql/tests/misc.rs
@@ -73,3 +73,13 @@ fn use_null() {
     let mut state = State::new_inplace(&t[1..]);
     assert_eq!(Use::test_parse_from_state(&mut state).unwrap(), Use::Null);
 }
+
+#[test]
+fn use_current() {
+    let t = lex_insecure(b"use $current").unwrap();
+    let mut state = State::new_inplace(&t[1..]);
+    assert_eq!(
+        Use::test_parse_from_state(&mut state).unwrap(),
+        Use::RefreshCurrent
+    );
+}

From 8d6a047f02ba9cf4f7007c81d888d9a13ce12bd8 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 30 Nov 2023 19:36:32 +0530
Subject: [PATCH 301/310] Implement all inspect queries

Also cleaned up space handling
---
 CHANGELOG.md                                  |   4 +
 cli/src/query.rs                              |   7 ++
 cli/src/repl.rs                               |   7 +-
 cli/src/resp.rs                               |  84 +++++---------
 server/src/engine/core/ddl_misc.rs            | 104 ++++++++++++++++++
 server/src/engine/core/exec.rs                |  39 ++++---
 server/src/engine/core/mod.rs                 |  18 +--
 server/src/engine/core/model/mod.rs           |   3 +-
 server/src/engine/core/space.rs               |   6 +-
 server/src/engine/ql/ddl/mod.rs               |  50 ++++++++-
 server/src/engine/ql/tests/misc.rs            |  32 +++++-
 server/src/engine/storage/v1/loader.rs        |   1 -
 server/src/engine/tests/client/ddl.rs         |  41 +++++++
 server/src/engine/tests/client/mod.rs         |   1 +
 server/src/engine/txn/gns/model.rs            |  13 +--
 server/src/engine/txn/gns/space.rs            |   9 +-
 server/src/engine/txn/gns/tests/full_chain.rs |   5 +-
 17 files changed, 315 insertions(+), 109 deletions(-)
 create mode 100644 server/src/engine/core/ddl_misc.rs
 create mode 100644 server/src/engine/tests/client/ddl.rs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 35ea6ac1..9d6cead0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,10 @@ All changes in this project will be noted in this file.
     - `USE <space>`:
       - works just like SQL
       - **does not work with DDL queries**: the reason it works in this way is to prevent accidental deletes
+    - `INSPECT ...`:
+      - `INSPECT global`: can be used to inspect the global state, seeing all spaces currently present on the system, users and other information. Some information is limited to the root account only (as JSON)
+      - `INSPECT space <space>`: can be used to inspect a single space, returning a list of models and relevant information (as JSON)
+      - `INSPECT model <model>`: can be used to inspect a single model, returning declaration and relevant information (as JSON)
 - DML:
   - **All actions removed**: All the prior `SET`, `GET` and other actions have been removed in favor of the new query language
   - The following queries were added:
diff --git a/cli/src/query.rs b/cli/src/query.rs
index eb9d0e08..fa6ec7da 100644
--- a/cli/src/query.rs
+++ b/cli/src/query.rs
@@ -80,6 +80,7 @@ pub enum ExecKind {
     Standard(Query),
     UseSpace(Query, String),
     UseNull(Query),
+    PrintSpecial(Query),
 }
 
 impl Parameterizer {
@@ -121,6 +122,12 @@ impl Parameterizer {
                 Ok(if qstr.eq_ignore_ascii_case("use null") {
                     ExecKind::UseNull(q)
                 } else {
+                    if qstr.len() > 8 {
+                        let qstr = &qstr[..8];
+                        if qstr.eq_ignore_ascii_case("inspect ") {
+                            return Ok(ExecKind::PrintSpecial(q));
+                        }
+                    }
                     let mut splits = qstr.split_ascii_whitespace();
                     let tok_use = splits.next();
                     let tok_name = splits.next();
diff --git a/cli/src/repl.rs b/cli/src/repl.rs
index 8a122161..80d31500 100644
--- a/cli/src/repl.rs
+++ b/cli/src/repl.rs
@@ -107,6 +107,7 @@ fn repl<C: IsConnection>(mut con: C) -> CliResult<()> {
                     match query::Parameterizer::new(line).parameterize() {
                         Ok(q) => {
                             let mut new_prompt = None;
+                            let mut special = false;
                             let q = match q {
                                 ExecKind::Standard(q) => q,
                                 ExecKind::UseNull(q) => {
@@ -117,8 +118,12 @@ fn repl<C: IsConnection>(mut con: C) -> CliResult<()> {
                                     new_prompt = Some(format!("{space}> "));
                                     q
                                 }
+                                ExecKind::PrintSpecial(q) => {
+                                    special = true;
+                                    q
+                                }
                             };
-                            if resp::format_response(con.execute_query(q)?)? {
+                            if resp::format_response(con.execute_query(q)?, special) {
                                 if let Some(pr) = new_prompt {
                                     prompt = pr;
                                 }
diff --git a/cli/src/resp.rs b/cli/src/resp.rs
index 163b2b85..4fe25a40 100644
--- a/cli/src/resp.rs
+++ b/cli/src/resp.rs
@@ -25,51 +25,45 @@
 */
 
 use {
-    crate::error::CliResult,
-    crossterm::{
-        style::{Color, ResetColor, SetForegroundColor},
-        ExecutableCommand,
-    },
+    crossterm::style::Stylize,
     skytable::response::{Response, Row, Value},
-    std::io::{self, Write},
 };
 
-pub fn format_response(resp: Response) -> CliResult<bool> {
+pub fn format_response(resp: Response, print_special: bool) -> bool {
     match resp {
-        Response::Empty => print_cyan("(Okay)\n")?,
+        Response::Empty => println!("{}", "(Okay)".cyan()),
         Response::Error(e) => {
-            print_red(&format!("(server error code: {e})\n"))?;
-            return Ok(false);
+            println!("{}", format!("(server error code: {e})").red());
+            return false;
         }
         Response::Value(v) => {
-            print_value(v)?;
+            print_value(v, print_special);
             println!();
         }
         Response::Row(r) => {
-            print_row(r)?;
+            print_row(r);
             println!();
         }
     };
-    Ok(true)
+    true
 }
 
-fn print_row(r: Row) -> CliResult<()> {
+fn print_row(r: Row) {
     print!("(");
     let mut columns = r.into_values().into_iter().peekable();
     while let Some(cell) = columns.next() {
-        print_value(cell)?;
+        print_value(cell, false);
         if columns.peek().is_some() {
             print!(", ");
         }
     }
     print!(")");
-    Ok(())
 }
 
-fn print_value(v: Value) -> CliResult<()> {
+fn print_value(v: Value, print_special: bool) {
     match v {
-        Value::Null => print_gray("null")?,
-        Value::String(s) => print_string(&s),
+        Value::Null => print!("{}", "null".grey().italic()),
+        Value::String(s) => print_string(&s, print_special),
         Value::Binary(b) => print_binary(&b),
         Value::Bool(b) => print!("{b}"),
         Value::UInt8(i) => print!("{i}"),
@@ -86,7 +80,7 @@ fn print_value(v: Value) -> CliResult<()> {
             print!("[");
             let mut items = items.into_iter().peekable();
             while let Some(item) = items.next() {
-                print_value(item)?;
+                print_value(item, print_special);
                 if items.peek().is_some() {
                     print!(", ");
                 }
@@ -94,7 +88,6 @@ fn print_value(v: Value) -> CliResult<()> {
             print!("]");
         }
     }
-    Ok(())
 }
 
 fn print_binary(b: &[u8]) {
@@ -109,39 +102,22 @@ fn print_binary(b: &[u8]) {
     print!("]");
 }
 
-fn print_string(s: &str) {
-    print!("\"");
-    for ch in s.chars() {
-        if ch == '"' || ch == '\'' {
-            print!("\\{ch}");
-        } else if ch == '\t' {
-            print!("\\t");
-        } else if ch == '\n' {
-            print!("\\n");
-        } else {
-            print!("{ch}");
+fn print_string(s: &str, print_special: bool) {
+    if print_special {
+        print!("{}", s.italic().grey());
+    } else {
+        print!("\"");
+        for ch in s.chars() {
+            if ch == '"' {
+                print!("\\{ch}");
+            } else if ch == '\t' {
+                print!("\\t");
+            } else if ch == '\n' {
+                print!("\\n");
+            } else {
+                print!("{ch}");
+            }
         }
+        print!("\"");
     }
-    print!("\"");
-}
-
-fn print_gray(s: &str) -> std::io::Result<()> {
-    print_colored_text(s, Color::White)
-}
-
-fn print_red(s: &str) -> std::io::Result<()> {
-    print_colored_text(s, Color::Red)
-}
-
-fn print_cyan(s: &str) -> std::io::Result<()> {
-    print_colored_text(s, Color::Cyan)
-}
-
-fn print_colored_text(text: &str, color: Color) -> std::io::Result<()> {
-    let mut stdout = io::stdout();
-    stdout.execute(SetForegroundColor(color))?;
-    print!("{text}");
-    stdout.flush()?;
-    stdout.execute(ResetColor)?;
-    Ok(())
 }
diff --git a/server/src/engine/core/ddl_misc.rs b/server/src/engine/core/ddl_misc.rs
new file mode 100644
index 00000000..7d2a9511
--- /dev/null
+++ b/server/src/engine/core/ddl_misc.rs
@@ -0,0 +1,104 @@
+/*
+ * Created on Thu Nov 30 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use crate::engine::{
+    error::{QueryError, QueryResult},
+    fractal::GlobalInstanceLike,
+    net::protocol::{ClientLocalState, Response, ResponseType},
+    ql::ddl::Inspect,
+};
+
+pub fn inspect(
+    g: &impl GlobalInstanceLike,
+    c: &ClientLocalState,
+    stmt: Inspect,
+) -> QueryResult<Response> {
+    let ret = match stmt {
+        Inspect::Global => {
+            // collect spaces
+            let spaces = g.namespace().idx().read();
+            let mut spaces_iter = spaces.iter().peekable();
+            let mut ret = format!("{{\"spaces\":[");
+            while let Some((space, _)) = spaces_iter.next() {
+                ret.push('"');
+                ret.push_str(&space);
+                ret.push('"');
+                if spaces_iter.peek().is_some() {
+                    ret.push(',');
+                }
+            }
+            if c.is_root() {
+                // iff the user is root, show information about other users. if not, just show models and settings
+                ret.push_str("],\"users\":[");
+                drop(spaces_iter);
+                drop(spaces);
+                // collect users
+                let users = g.sys_store().system_store().auth_data().read();
+                let mut users_iter = users.users().iter().peekable();
+                while let Some((user, _)) = users_iter.next() {
+                    ret.push('"');
+                    ret.push_str(&user);
+                    ret.push('"');
+                    if users_iter.peek().is_some() {
+                        ret.push(',');
+                    }
+                }
+            }
+            ret.push_str("],\"settings\":{}}");
+            ret
+        }
+        Inspect::Model(m) => match g.namespace().idx_models().read().get(&m) {
+            Some(m) => format!(
+                "{{\"decl\":\"{}\",\"rows\":{},\"properties\":{{}}}}",
+                m.describe(),
+                m.primary_index().count()
+            ),
+            None => return Err(QueryError::QExecObjectNotFound),
+        },
+        Inspect::Space(s) => match g.namespace().idx().read().get(s.as_str()) {
+            Some(s) => {
+                let mut ret = format!("{{\"models\":[");
+                let mut models_iter = s.models().iter().peekable();
+                while let Some(mdl) = models_iter.next() {
+                    ret.push('\"');
+                    ret.push_str(&mdl);
+                    ret.push('\"');
+                    if models_iter.peek().is_some() {
+                        ret.push(',');
+                    }
+                }
+                ret.push_str("]}}");
+                ret
+            }
+            None => return Err(QueryError::QExecObjectNotFound),
+        },
+    };
+    Ok(Response::Serialized {
+        ty: ResponseType::String,
+        size: ret.len(),
+        data: ret.into_bytes(),
+    })
+}
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 79492c06..6f000374 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -25,7 +25,7 @@
 */
 
 use crate::engine::{
-    core::{dml, model::Model, space::Space},
+    core::{ddl_misc, dml, model::Model, space::Space},
     error::{QueryError, QueryResult},
     fractal::{Global, GlobalInstanceLike},
     net::protocol::{ClientLocalState, Response, ResponseType, SQuery},
@@ -63,7 +63,7 @@ pub async fn dispatch_to_executor<'a>(
 }
 
 #[inline(always)]
-fn _call<A: ASTNode<'static> + core::fmt::Debug, T>(
+fn _callgs<A: ASTNode<'static> + core::fmt::Debug, T>(
     g: &Global,
     state: &mut State<'static, InplaceData>,
     f: impl FnOnce(&Global, A) -> Result<T, QueryError>,
@@ -72,6 +72,17 @@ fn _call<A: ASTNode<'static> + core::fmt::Debug, T>(
     f(&g, cs)
 }
 
+#[inline(always)]
+fn _callgcs<A: ASTNode<'static> + core::fmt::Debug, T>(
+    g: &Global,
+    cstate: &ClientLocalState,
+    state: &mut State<'static, InplaceData>,
+    f: impl FnOnce(&Global, &ClientLocalState, A) -> Result<T, QueryError>,
+) -> QueryResult<T> {
+    let a = ASTNode::parse_from_state_hardened(state)?;
+    f(&g, cstate, a)
+}
+
 async fn run_blocking_stmt(
     global: &Global,
     cstate: &mut ClientLocalState,
@@ -109,12 +120,12 @@ async fn run_blocking_stmt(
     ) -> QueryResult<()>; 8] = [
         |_, _, _| Err(QueryError::QLUnknownStatement),
         blocking_exec_sysctl,
-        |g, _, t| _call(&g, t, Space::transactional_exec_create),
-        |g, _, t| _call(&g, t, Model::transactional_exec_create),
-        |g, _, t| _call(&g, t, Space::transactional_exec_alter),
-        |g, _, t| _call(&g, t, Model::transactional_exec_alter),
-        |g, _, t| _call(&g, t, Space::transactional_exec_drop),
-        |g, _, t| _call(&g, t, Model::transactional_exec_drop),
+        |g, _, t| _callgs(&g, t, Space::transactional_exec_create),
+        |g, _, t| _callgs(&g, t, Model::transactional_exec_create),
+        |g, _, t| _callgs(&g, t, Space::transactional_exec_alter),
+        |g, _, t| _callgs(&g, t, Model::transactional_exec_alter),
+        |g, _, t| _callgs(&g, t, Space::transactional_exec_drop),
+        |g, _, t| _callgs(&g, t, Model::transactional_exec_drop),
     ];
     let r = unsafe {
         // UNSAFE(@ohsayan): the only await is within this block
@@ -192,13 +203,13 @@ fn run_nb(
         &mut ClientLocalState,
         &mut State<'static, InplaceData>,
     ) -> QueryResult<Response>; 8] = [
-        cstate_use,                                    // use
-        |_, _, _| Err(QueryError::QLUnknownStatement), // inspect
+        cstate_use, // use
+        |g, c, s| _callgcs(g, c, s, ddl_misc::inspect),
         |_, _, _| Err(QueryError::QLUnknownStatement), // describe
-        |g, _, s| _call(g, s, dml::insert_resp),
-        |g, _, s| _call(g, s, dml::select_resp),
-        |g, _, s| _call(g, s, dml::update_resp),
-        |g, _, s| _call(g, s, dml::delete_resp),
+        |g, _, s| _callgs(g, s, dml::insert_resp),
+        |g, _, s| _callgs(g, s, dml::select_resp),
+        |g, _, s| _callgs(g, s, dml::update_resp),
+        |g, _, s| _callgs(g, s, dml::delete_resp),
         |_, _, _| Err(QueryError::QLUnknownStatement), // exists
     ];
     {
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index 8e416efc..d1d8de12 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -25,6 +25,7 @@
 */
 
 pub(in crate::engine) mod dcl;
+pub(super) mod ddl_misc;
 pub(in crate::engine) mod dml;
 pub(in crate::engine) mod exec;
 pub(in crate::engine) mod index;
@@ -58,7 +59,7 @@ type RWLIdx<K, V> = RwLock<IndexST<K, V>>;
 #[cfg_attr(test, derive(Debug))]
 pub struct GlobalNS {
     idx_mdl: RWLIdx<EntityID, Model>,
-    idx: RWLIdx<Box<str>, RwLock<Space>>,
+    idx: RWLIdx<Box<str>, Space>,
 }
 
 impl GlobalNS {
@@ -70,7 +71,7 @@ impl GlobalNS {
     }
     pub fn ddl_with_spaces_write<T>(
         &self,
-        f: impl FnOnce(&mut HashMap<Box<str>, RwLock<Space>>) -> T,
+        f: impl FnOnce(&mut HashMap<Box<str>, Space>) -> T,
     ) -> T {
         let mut spaces = self.idx.write();
         f(&mut spaces)
@@ -80,12 +81,11 @@ impl GlobalNS {
         space: &str,
         f: impl FnOnce(&mut Space) -> QueryResult<T>,
     ) -> QueryResult<T> {
-        let spaces = self.idx.read();
-        let Some(space) = spaces.get(space) else {
+        let mut spaces = self.idx.write();
+        let Some(space) = spaces.get_mut(space) else {
             return Err(QueryError::QExecObjectNotFound);
         };
-        let mut space = space.write();
-        f(&mut space)
+        f(space)
     }
     pub fn with_model_space_mut_for_ddl<'a, T, F>(
         &self,
@@ -100,8 +100,8 @@ impl GlobalNS {
             return Err(QueryError::QExecObjectNotFound);
         };
         let space_read = self.idx.read();
-        let space = space_read.get(entity.space()).unwrap().read();
-        f(&space, model)
+        let space = space_read.get(entity.space()).unwrap();
+        f(space, model)
     }
     pub fn with_model<'a, T, F>(&self, entity: EntityIDRef<'a>, f: F) -> QueryResult<T>
     where
@@ -116,7 +116,7 @@ impl GlobalNS {
     pub fn idx_models(&self) -> &RWLIdx<EntityID, Model> {
         &self.idx_mdl
     }
-    pub fn idx(&self) -> &RWLIdx<Box<str>, RwLock<Space>> {
+    pub fn idx(&self) -> &RWLIdx<Box<str>, Space> {
         &self.idx
     }
     #[cfg(test)]
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 6c67b55b..a779b7ca 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -121,7 +121,7 @@ impl Model {
         &self.decl
     }
     fn redescribe(&self) -> String {
-        let mut ret = format!("{{ ");
+        let mut ret = format!("{{");
         let mut it = self.fields().stseq_ord_kv().peekable();
         while let Some((field_name, field_decl)) = it.next() {
             // legend: * -> primary, ! -> not null, ? -> null
@@ -153,7 +153,6 @@ impl Model {
                 ret.push(' ');
             }
         }
-        ret.push(' ');
         ret.push('}');
         ret
     }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 8e5adb0f..7554037d 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -34,7 +34,6 @@ use {
         storage::v1::{loader::SEInitState, RawFSInterface},
         txn::gns as gnstxn,
     },
-    parking_lot::RwLock,
     std::collections::HashSet,
 };
 
@@ -174,7 +173,7 @@ impl Space {
                 }
             }
             // update global state
-            let _ = spaces.st_insert(space_name, RwLock::new(space));
+            let _ = spaces.st_insert(space_name, space);
             Ok(())
         })
     }
@@ -228,7 +227,6 @@ impl Space {
             let Some(space) = spaces.get(space_name.as_str()) else {
                 return Err(QueryError::QExecObjectNotFound);
             };
-            let space = space.read();
             if !space.models.is_empty() {
                 // nonempty, we can't do anything
                 return Err(QueryError::QExecDdlNotEmpty);
@@ -245,8 +243,6 @@ impl Space {
                     space.get_uuid(),
                 )));
             }
-            // good, we can get rid of this thing
-            drop(space);
             let _ = spaces.st_delete(space_name.as_str());
             Ok(())
         })
diff --git a/server/src/engine/ql/ddl/mod.rs b/server/src/engine/ql/ddl/mod.rs
index 8cc1021f..a1dd6643 100644
--- a/server/src/engine/ql/ddl/mod.rs
+++ b/server/src/engine/ql/ddl/mod.rs
@@ -32,10 +32,13 @@ pub(in crate::engine) mod drop;
 
 use {
     super::{
-        ast::traits::ASTNode,
+        ast::{traits::ASTNode, QueryData, State},
         lex::{Ident, Token},
     },
-    crate::engine::error::QueryError,
+    crate::engine::{
+        core::EntityIDRef,
+        error::{QueryError, QueryResult},
+    },
 };
 
 #[derive(Debug, PartialEq)]
@@ -48,9 +51,9 @@ pub enum Use<'a> {
 impl<'a> ASTNode<'a> for Use<'a> {
     const MUST_USE_FULL_TOKEN_RANGE: bool = true;
     const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
-    fn __base_impl_parse_from_state<Qd: super::ast::QueryData<'a>>(
-        state: &mut super::ast::State<'a, Qd>,
-    ) -> crate::engine::error::QueryResult<Self> {
+    fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+        state: &mut State<'a, Qd>,
+    ) -> QueryResult<Self> {
         /*
             should have either an ident or null
         */
@@ -73,3 +76,40 @@ impl<'a> ASTNode<'a> for Use<'a> {
         })
     }
 }
+
+#[derive(Debug, PartialEq)]
+pub enum Inspect<'a> {
+    Global,
+    Space(Ident<'a>),
+    Model(EntityIDRef<'a>),
+}
+
+impl<'a> ASTNode<'a> for Inspect<'a> {
+    const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+    const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+    fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+        state: &mut State<'a, Qd>,
+    ) -> QueryResult<Self> {
+        if state.exhausted() {
+            return Err(QueryError::QLUnexpectedEndOfStatement);
+        }
+        let me = match state.fw_read() {
+            Token::Ident(id) if id.eq_ignore_ascii_case("global") => Self::Global,
+            Token![space] => {
+                if state.exhausted() {
+                    return Err(QueryError::QLUnexpectedEndOfStatement);
+                }
+                match state.fw_read() {
+                    Token::Ident(space) => Self::Space(*space),
+                    _ => return Err(QueryError::QLInvalidSyntax),
+                }
+            }
+            Token![model] => {
+                let entity = state.try_entity_ref_result()?;
+                Self::Model(entity)
+            }
+            _ => return Err(QueryError::QLInvalidSyntax),
+        };
+        Ok(me)
+    }
+}
diff --git a/server/src/engine/ql/tests/misc.rs b/server/src/engine/ql/tests/misc.rs
index e396d326..032f3e0a 100644
--- a/server/src/engine/ql/tests/misc.rs
+++ b/server/src/engine/ql/tests/misc.rs
@@ -27,7 +27,7 @@
 use super::*;
 use crate::engine::ql::{
     ast::{traits::ASTNode, State},
-    ddl::Use,
+    ddl::{Inspect, Use},
 };
 
 /*
@@ -83,3 +83,33 @@ fn use_current() {
         Use::RefreshCurrent
     );
 }
+
+#[test]
+fn inspect_global() {
+    let t = lex_insecure(b"inspect global").unwrap();
+    let mut state = State::new_inplace(&t[1..]);
+    assert_eq!(
+        Inspect::test_parse_from_state(&mut state).unwrap(),
+        Inspect::Global
+    );
+}
+
+#[test]
+fn inspect_space() {
+    let t = lex_insecure(b"inspect space myspace").unwrap();
+    let mut state = State::new_inplace(&t[1..]);
+    assert_eq!(
+        Inspect::test_parse_from_state(&mut state).unwrap(),
+        Inspect::Space("myspace".into())
+    );
+}
+
+#[test]
+fn inspect_model() {
+    let t = lex_insecure(b"inspect model myspace.mymodel").unwrap();
+    let mut state = State::new_inplace(&t[1..]);
+    assert_eq!(
+        Inspect::test_parse_from_state(&mut state).unwrap(),
+        Inspect::Model(("myspace", "mymodel").into())
+    );
+}
diff --git a/server/src/engine/storage/v1/loader.rs b/server/src/engine/storage/v1/loader.rs
index e4340b64..065a3f49 100644
--- a/server/src/engine/storage/v1/loader.rs
+++ b/server/src/engine/storage/v1/loader.rs
@@ -79,7 +79,6 @@ impl SEInitState {
                 let mut models = gns.idx_models().write();
                 // this is an existing instance, so read in all data
                 for (space_name, space) in gns.idx().read().iter() {
-                    let space = space.read();
                     let space_uuid = space.get_uuid();
                     for model_name in space.models().iter() {
                         let model = models
diff --git a/server/src/engine/tests/client/ddl.rs b/server/src/engine/tests/client/ddl.rs
new file mode 100644
index 00000000..433a70ef
--- /dev/null
+++ b/server/src/engine/tests/client/ddl.rs
@@ -0,0 +1,41 @@
+/*
+ * Created on Thu Nov 30 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+use {sky_macros::dbtest, skytable::query};
+
+#[dbtest]
+fn inspect_global_as_root_returns_user_info() {
+    let mut db = db!();
+    let inspect: String = db.query_parse(&query!("inspect global")).unwrap();
+    assert!(inspect.contains("\"users\":"));
+}
+
+#[dbtest]
+fn inspect_global_as_std_user_does_not_return_user_info() {
+    let mut db = db!();
+    let inspect: String = db.query_parse(&query!("inspect global")).unwrap();
+    assert!(!inspect.contains("\"users\":"));
+}
diff --git a/server/src/engine/tests/client/mod.rs b/server/src/engine/tests/client/mod.rs
index bff9291a..faaae0fe 100644
--- a/server/src/engine/tests/client/mod.rs
+++ b/server/src/engine/tests/client/mod.rs
@@ -24,5 +24,6 @@
  *
 */
 
+mod ddl;
 mod sec;
 mod sysctl;
diff --git a/server/src/engine/txn/gns/model.rs b/server/src/engine/txn/gns/model.rs
index 0ddbd4cb..fc0f6c8d 100644
--- a/server/src/engine/txn/gns/model.rs
+++ b/server/src/engine/txn/gns/model.rs
@@ -164,7 +164,6 @@ fn with_space<T>(
     let Some(space) = spaces.st_get(&space_id.name) else {
         return Err(TransactionError::OnRestoreDataMissing.into());
     };
-    let space = space.read();
     if space.get_uuid() != space_id.uuid {
         return Err(TransactionError::OnRestoreDataConflictMismatch.into());
     }
@@ -176,15 +175,14 @@ fn with_space_mut<T>(
     space_id: &super::SpaceIDRes,
     mut f: impl FnMut(&mut Space) -> RuntimeResult<T>,
 ) -> RuntimeResult<T> {
-    let spaces = gns.idx().read();
-    let Some(space) = spaces.st_get(&space_id.name) else {
+    let mut spaces = gns.idx().write();
+    let Some(space) = spaces.st_get_mut(&space_id.name) else {
         return Err(TransactionError::OnRestoreDataMissing.into());
     };
-    let mut space = space.write();
     if space.get_uuid() != space_id.uuid {
         return Err(TransactionError::OnRestoreDataConflictMismatch.into());
     }
-    f(&mut space)
+    f(space)
 }
 
 fn with_model_mut<T>(
@@ -318,12 +316,11 @@ impl<'a> GNSEvent for CreateModelTxn<'a> {
             (or well magnetic stuff arounding spinning disks). But we just want to be extra sure. Don't let the aliens (or
             rather, radiation) from the cosmos deter us!
         */
-        let spaces = gns.idx().write();
+        let mut spaces = gns.idx().write();
         let mut models = gns.idx_models().write();
-        let Some(space) = spaces.get(&space_id.name) else {
+        let Some(space) = spaces.get_mut(&space_id.name) else {
             return Err(TransactionError::OnRestoreDataMissing.into());
         };
-        let mut space = space.write();
         if space.models().contains(&model_name) {
             return Err(TransactionError::OnRestoreDataConflictAlreadyExists.into());
         }
diff --git a/server/src/engine/txn/gns/space.rs b/server/src/engine/txn/gns/space.rs
index 193b9344..9961378e 100644
--- a/server/src/engine/txn/gns/space.rs
+++ b/server/src/engine/txn/gns/space.rs
@@ -215,10 +215,9 @@ impl<'a> GNSEvent for AlterSpaceTxn<'a> {
         }: Self::RestoreType,
         gns: &crate::engine::core::GlobalNS,
     ) -> RuntimeResult<()> {
-        let gns = gns.idx().read();
-        match gns.st_get(&space_id.name) {
+        let mut gns = gns.idx().write();
+        match gns.st_get_mut(&space_id.name) {
             Some(space) => {
-                let mut space = space.write();
                 if !crate::engine::data::dict::rmerge_metadata(space.props_mut(), space_meta) {
                     return Err(TransactionError::OnRestoreDataConflictMismatch.into());
                 }
@@ -281,10 +280,8 @@ impl<'a> GNSEvent for DropSpaceTxn<'a> {
         let mut wgns = gns.idx().write();
         match wgns.entry(name) {
             std::collections::hash_map::Entry::Occupied(oe) => {
-                let space = oe.get().read();
-                if space.get_uuid() == uuid {
+                if oe.get().get_uuid() == uuid {
                     // NB(@ohsayan): we do not need to remove models here since they must have been already removed for this query to have actually executed
-                    drop(space);
                     oe.remove_entry();
                     Ok(())
                 } else {
diff --git a/server/src/engine/txn/gns/tests/full_chain.rs b/server/src/engine/txn/gns/tests/full_chain.rs
index 35be0bfb..acc894c1 100644
--- a/server/src/engine/txn/gns/tests/full_chain.rs
+++ b/server/src/engine/txn/gns/tests/full_chain.rs
@@ -62,7 +62,6 @@ fn init_space(global: &impl GlobalInstanceLike, space_name: &str, env: &str) ->
         .read()
         .get(name.as_str())
         .unwrap()
-        .read()
         .get_uuid()
 }
 
@@ -78,7 +77,7 @@ fn create_space() {
         multirun(|| {
             let global = TestGlobal::new_with_vfs_driver(log_name);
             let spaces = global.namespace().idx().read();
-            let space = spaces.get("myspace").unwrap().read();
+            let space = spaces.get("myspace").unwrap();
             assert_eq!(
                 &*space,
                 &Space::new_restore_empty(
@@ -108,7 +107,7 @@ fn alter_space() {
         multirun(|| {
             let global = TestGlobal::new_with_vfs_driver(log_name);
             let spaces = global.namespace().idx().read();
-            let space = spaces.get("myspace").unwrap().read();
+            let space = spaces.get("myspace").unwrap();
             assert_eq!(
                 &*space,
                 &Space::new_restore_empty(

From 01c684bd381288b68137a9e8fec709207499d678 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Thu, 30 Nov 2023 23:55:51 +0530
Subject: [PATCH 302/310] Allow non empty DDL drops

Drop even if non empty during benchmark
---
 server/src/engine/core/exec.rs             |  5 +-
 server/src/engine/core/mod.rs              |  8 +++
 server/src/engine/core/model/mod.rs        |  2 +-
 server/src/engine/core/space.rs            | 81 +++++++++++++++-------
 server/src/engine/core/util.rs             |  4 +-
 server/src/engine/macros.rs                |  9 +++
 server/src/engine/ql/ddl/drop.rs           | 43 +++++++++---
 server/src/engine/ql/lex/raw.rs            | 15 ++--
 server/src/engine/ql/tests/schema_tests.rs |  4 +-
 server/src/engine/tests/client/ddl.rs      |  2 +-
 server/src/engine/txn/gns/space.rs         | 11 ++-
 sky-bench/src/bench.rs                     |  3 +-
 12 files changed, 134 insertions(+), 53 deletions(-)

diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 6f000374..96f8664c 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -105,12 +105,13 @@ async fn run_blocking_stmt(
     let alter = stmt == KeywordStmt::Alter;
     let drop = stmt == KeywordStmt::Drop;
     let last_id = b.is_ident();
+    let last_allow = Token![allow].eq(b);
     let c_s = (create & Token![space].eq(a) & last_id) as u8 * 2;
     let c_m = (create & Token![model].eq(a) & last_id) as u8 * 3;
     let a_s = (alter & Token![space].eq(a) & last_id) as u8 * 4;
     let a_m = (alter & Token![model].eq(a) & last_id) as u8 * 5;
-    let d_s = (drop & Token![space].eq(a) & last_id) as u8 * 6;
-    let d_m = (drop & Token![model].eq(a) & last_id) as u8 * 7;
+    let d_s = (drop & Token![space].eq(a) & (last_id | last_allow)) as u8 * 6;
+    let d_m = (drop & Token![model].eq(a) & (last_id | last_allow)) as u8 * 7;
     let fc = sysctl as u8 | c_s | c_m | a_s | a_m | d_s | d_m;
     state.cursor_ahead_if(!sysctl);
     static BLK_EXEC: [fn(
diff --git a/server/src/engine/core/mod.rs b/server/src/engine/core/mod.rs
index d1d8de12..83a4e8c9 100644
--- a/server/src/engine/core/mod.rs
+++ b/server/src/engine/core/mod.rs
@@ -69,6 +69,14 @@ impl GlobalNS {
             idx: RWLIdx::default(),
         }
     }
+    pub fn ddl_with_all_mut<T>(
+        &self,
+        f: impl FnOnce(&mut HashMap<Box<str>, Space>, &mut HashMap<EntityID, Model>) -> T,
+    ) -> T {
+        let mut spaces = self.idx.write();
+        let mut models = self.idx_mdl.write();
+        f(&mut spaces, &mut models)
+    }
     pub fn ddl_with_spaces_write<T>(
         &self,
         f: impl FnOnce(&mut HashMap<Box<str>, Space>) -> T,
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index a779b7ca..f87ae8e4 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -335,7 +335,7 @@ impl Model {
                 .get(&EntityIDRef::new(&space_name, &model_name))
                 .unwrap();
             // the model must be empty for us to clean it up! (NB: consistent view + EX)
-            if model.primary_index().count() != 0 {
+            if (model.primary_index().count() != 0) & !(stmt.force) {
                 // nope, we can't drop this
                 return Err(QueryError::QExecDdlNotEmpty);
             }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 7554037d..80f7fe32 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use super::EntityIDRef;
+
 use {
     crate::engine::{
         data::{dict, uuid::Uuid, DictEntryGeneric, DictGeneric},
@@ -218,33 +220,60 @@ impl Space {
         global: &G,
         DropSpace {
             space: space_name,
-            force: _,
+            force,
         }: DropSpace,
     ) -> QueryResult<()> {
-        // TODO(@ohsayan): force remove option
-        // TODO(@ohsayan): should a drop space block the entire global table?
-        global.namespace().ddl_with_spaces_write(|spaces| {
-            let Some(space) = spaces.get(space_name.as_str()) else {
-                return Err(QueryError::QExecObjectNotFound);
-            };
-            if !space.models.is_empty() {
-                // nonempty, we can't do anything
-                return Err(QueryError::QExecDdlNotEmpty);
-            }
-            // okay, it's empty; good riddance
-            if G::FS_IS_NON_NULL {
-                // prepare txn
-                let txn = gnstxn::DropSpaceTxn::new(gnstxn::SpaceIDRef::new(&space_name, &space));
-                // commit txn
-                global.namespace_txn_driver().lock().try_commit(txn)?;
-                // request cleanup
-                global.taskmgr_post_standard_priority(Task::new(GenericTask::delete_space_dir(
-                    &space_name,
-                    space.get_uuid(),
-                )));
-            }
-            let _ = spaces.st_delete(space_name.as_str());
-            Ok(())
-        })
+        if force {
+            global.namespace().ddl_with_all_mut(|spaces, models| {
+                let Some(space) = spaces.remove(space_name.as_str()) else {
+                    return Err(QueryError::QExecObjectNotFound);
+                };
+                // commit drop
+                if G::FS_IS_NON_NULL {
+                    // prepare txn
+                    let txn =
+                        gnstxn::DropSpaceTxn::new(gnstxn::SpaceIDRef::new(&space_name, &space));
+                    // commit txn
+                    global.namespace_txn_driver().lock().try_commit(txn)?;
+                    // request cleanup
+                    global.taskmgr_post_standard_priority(Task::new(
+                        GenericTask::delete_space_dir(&space_name, space.get_uuid()),
+                    ));
+                }
+                for model in space.models.into_iter() {
+                    let e: EntityIDRef<'static> = unsafe {
+                        // UNSAFE(@ohsayan): I want to try what the borrow checker has been trying
+                        core::mem::transmute(EntityIDRef::new(space_name.as_str(), &model))
+                    };
+                    let _ = models.st_delete(&e);
+                }
+                let _ = spaces.st_delete(space_name.as_str());
+                Ok(())
+            })
+        } else {
+            global.namespace().ddl_with_spaces_write(|spaces| {
+                let Some(space) = spaces.get(space_name.as_str()) else {
+                    return Err(QueryError::QExecObjectNotFound);
+                };
+                if !space.models.is_empty() {
+                    // nonempty, we can't do anything
+                    return Err(QueryError::QExecDdlNotEmpty);
+                }
+                // okay, it's empty; good riddance
+                if G::FS_IS_NON_NULL {
+                    // prepare txn
+                    let txn =
+                        gnstxn::DropSpaceTxn::new(gnstxn::SpaceIDRef::new(&space_name, &space));
+                    // commit txn
+                    global.namespace_txn_driver().lock().try_commit(txn)?;
+                    // request cleanup
+                    global.taskmgr_post_standard_priority(Task::new(
+                        GenericTask::delete_space_dir(&space_name, space.get_uuid()),
+                    ));
+                }
+                let _ = spaces.st_delete(space_name.as_str());
+                Ok(())
+            })
+        }
     }
 }
diff --git a/server/src/engine/core/util.rs b/server/src/engine/core/util.rs
index 02afa907..b6c9bc84 100644
--- a/server/src/engine/core/util.rs
+++ b/server/src/engine/core/util.rs
@@ -150,8 +150,8 @@ impl<'a> Borrow<EntityIDRef<'a>> for EntityID {
     }
 }
 
-impl From<(&'static str, &'static str)> for EntityIDRef<'static> {
-    fn from((s, e): (&'static str, &'static str)) -> Self {
+impl<'a> From<(&'a str, &'a str)> for EntityIDRef<'a> {
+    fn from((s, e): (&'a str, &'a str)) -> Self {
         Self::new(s, e)
     }
 }
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index f2f94739..08383ebf 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -317,6 +317,15 @@ macro_rules! Token {
     (null) => {
         __kw_misc!(Null)
     };
+    (not) => {
+        __kw_misc!(Not)
+    };
+    (return) => {
+        __kw_misc!(Return)
+    };
+    (allow) => {
+        __kw_misc!(Allow)
+    };
 }
 
 macro_rules! union {
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index 1e77a8d2..75e3d69d 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -29,7 +29,7 @@ use crate::engine::{
     error::{QueryError, QueryResult},
     ql::{
         ast::{QueryData, State},
-        lex::{Ident, Token},
+        lex::Ident,
     },
 };
 
@@ -47,24 +47,42 @@ impl<'a> DropSpace<'a> {
         Self { space, force }
     }
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<DropSpace<'a>> {
+        /*
+            either drop space <myspace> OR drop space allow not empty <myspace>
+        */
         if state.cursor_is_ident() {
             let ident = state.fw_read();
-            // should we force drop?
-            let force = state.cursor_rounded_eq(Token::Ident(Ident::from("force")));
-            state.cursor_ahead_if(force);
             // either `force` or nothing
             return Ok(DropSpace::new(
                 unsafe {
                     // UNSAFE(@ohsayan): Safe because the if predicate ensures that tok[0] (relative) is indeed an ident
                     ident.uck_read_ident()
                 },
-                force,
+                false,
             ));
+        } else {
+            if ddl_allow_non_empty(state) {
+                state.cursor_ahead_by(3);
+                let space_name = unsafe {
+                    // UNSAFE(@ohsayan): verified in branch
+                    state.fw_read().uck_read_ident()
+                };
+                return Ok(DropSpace::new(space_name, true));
+            }
         }
         Err(QueryError::QLInvalidSyntax)
     }
 }
 
+#[inline(always)]
+fn ddl_allow_non_empty<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> bool {
+    let tok_allow = Token![allow].eq(state.offset_current_r(0));
+    let tok_not = Token![not].eq(state.offset_current_r(1));
+    let tok_empty = state.offset_current_r(2).ident_eq("empty");
+    let name = state.offset_current_r(3).is_ident();
+    (tok_allow & tok_not & tok_empty & name) & (state.remaining() >= 4)
+}
+
 #[derive(Debug, PartialEq)]
 pub struct DropModel<'a> {
     pub(in crate::engine) entity: EntityIDRef<'a>,
@@ -77,10 +95,17 @@ impl<'a> DropModel<'a> {
         Self { entity, force }
     }
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
-        let e = state.try_entity_ref_result()?;
-        let force = state.cursor_rounded_eq(Token::Ident(Ident::from("force")));
-        state.cursor_ahead_if(force);
-        Ok(DropModel::new(e, force))
+        if state.cursor_is_ident() {
+            let e = state.try_entity_ref_result()?;
+            return Ok(DropModel::new(e, false));
+        } else {
+            if ddl_allow_non_empty(state) {
+                state.cursor_ahead_by(3); // allow not empty
+                let e = state.try_entity_ref_result()?;
+                return Ok(DropModel::new(e, true));
+            }
+        }
+        Err(QueryError::QLInvalidSyntax)
     }
 }
 
diff --git a/server/src/engine/ql/lex/raw.rs b/server/src/engine/ql/lex/raw.rs
index 959116dd..15c45eac 100644
--- a/server/src/engine/ql/lex/raw.rs
+++ b/server/src/engine/ql/lex/raw.rs
@@ -370,6 +370,7 @@ flattened_lut! {
                 Remove,
                 Transform,
                 Set,
+                Return,
                 // sort related
                 Order,
                 Sort,
@@ -397,6 +398,7 @@ flattened_lut! {
                 Else,
                 Where,
                 When,
+                Allow,
                 // value
                 Auto,
                 Default,
@@ -421,6 +423,7 @@ flattened_lut! {
 }
 
 impl Keyword {
+    #[inline(always)]
     pub fn get(k: &[u8]) -> Option<Self> {
         if (k.len() > Self::SIZE_MAX) | (k.len() < Self::SIZE_MIN) {
             None
@@ -429,13 +432,13 @@ impl Keyword {
         }
     }
     fn compute(key: &[u8]) -> Option<Self> {
-        static G: [u8; 64] = [
-            0, 27, 13, 56, 18, 0, 26, 30, 33, 56, 20, 41, 56, 39, 23, 34, 36, 23, 17, 40, 38, 45,
-            8, 25, 26, 24, 53, 59, 30, 14, 9, 60, 12, 29, 6, 47, 3, 38, 19, 5, 13, 51, 41, 34, 0,
-            22, 43, 13, 46, 33, 11, 12, 36, 58, 40, 0, 36, 2, 19, 49, 53, 23, 55, 0,
+        static G: [u8; 69] = [
+            0, 0, 9, 64, 16, 43, 7, 49, 24, 8, 41, 37, 19, 66, 18, 0, 17, 0, 12, 63, 34, 56, 3, 24,
+            55, 14, 0, 67, 7, 0, 39, 60, 56, 0, 51, 23, 31, 19, 30, 12, 10, 58, 20, 39, 32, 0, 6,
+            30, 26, 58, 52, 62, 39, 27, 24, 9, 4, 21, 24, 68, 10, 38, 40, 21, 62, 27, 53, 27, 44,
         ];
-        static M1: [u8; 11] = *b"RtEMxHylmiZ";
-        static M2: [u8; 11] = *b"F1buDOZ2nzz";
+        static M1: [u8; 11] = *b"D8N5FwqrxdA";
+        static M2: [u8; 11] = *b"FsIPJv9hsXx";
         let h1 = Self::_sum(key, M1) % G.len();
         let h2 = Self::_sum(key, M2) % G.len();
         let h = (G[h1] + G[h2]) as usize % G.len();
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index 9a32fe31..e22b6666 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -1124,7 +1124,7 @@ mod ddl_other_query_tests {
     }
     #[test]
     fn drop_space_force() {
-        let src = lex_insecure(br"drop space myspace force").unwrap();
+        let src = lex_insecure(br"drop space allow not empty myspace").unwrap();
         assert_eq!(
             parse_ast_node_full::<DropSpace>(&src[2..]).unwrap(),
             DropSpace::new(Ident::from("myspace"), true)
@@ -1140,7 +1140,7 @@ mod ddl_other_query_tests {
     }
     #[test]
     fn drop_model_force() {
-        let src = lex_insecure(br"drop model mymodel force").unwrap();
+        let src = lex_insecure(br"drop model allow not empty mymodel").unwrap();
         assert_eq!(
             parse_ast_node_full_with_space::<DropModel>(&src[2..], "apps").unwrap(),
             DropModel::new(("apps", "mymodel").into(), true)
diff --git a/server/src/engine/tests/client/ddl.rs b/server/src/engine/tests/client/ddl.rs
index 433a70ef..481596fd 100644
--- a/server/src/engine/tests/client/ddl.rs
+++ b/server/src/engine/tests/client/ddl.rs
@@ -33,7 +33,7 @@ fn inspect_global_as_root_returns_user_info() {
     assert!(inspect.contains("\"users\":"));
 }
 
-#[dbtest]
+#[dbtest(switch_user(username = "sneaking_user_info"))]
 fn inspect_global_as_std_user_does_not_return_user_info() {
     let mut db = db!();
     let inspect: String = db.query_parse(&query!("inspect global")).unwrap();
diff --git a/server/src/engine/txn/gns/space.rs b/server/src/engine/txn/gns/space.rs
index 9961378e..c4199463 100644
--- a/server/src/engine/txn/gns/space.rs
+++ b/server/src/engine/txn/gns/space.rs
@@ -28,7 +28,7 @@ use {
     super::GNSEvent,
     crate::{
         engine::{
-            core::{space::Space, GlobalNS},
+            core::{space::Space, EntityIDRef, GlobalNS},
             data::DictGeneric,
             error::{RuntimeResult, TransactionError},
             idx::STIndex,
@@ -278,10 +278,17 @@ impl<'a> GNSEvent for DropSpaceTxn<'a> {
         gns: &GlobalNS,
     ) -> RuntimeResult<()> {
         let mut wgns = gns.idx().write();
+        let mut wmodel = gns.idx_models().write();
         match wgns.entry(name) {
             std::collections::hash_map::Entry::Occupied(oe) => {
                 if oe.get().get_uuid() == uuid {
-                    // NB(@ohsayan): we do not need to remove models here since they must have been already removed for this query to have actually executed
+                    for model in oe.get().models() {
+                        let id: EntityIDRef<'static> = unsafe {
+                            // UNSAFE(@ohsayan): I really need a pack of what the borrow checker has been reveling on
+                            core::mem::transmute(EntityIDRef::new(oe.key(), &model))
+                        };
+                        let _ = wmodel.st_delete(&id);
+                    }
                     oe.remove_entry();
                     Ok(())
                 } else {
diff --git a/sky-bench/src/bench.rs b/sky-bench/src/bench.rs
index e155e6a1..bfe18fd0 100644
--- a/sky-bench/src/bench.rs
+++ b/sky-bench/src/bench.rs
@@ -160,8 +160,7 @@ pub fn run(bench: BenchConfig) -> error::BenchResult<()> {
 
 fn cleanup(mut main_thread_db: Connection) -> Result<(), error::BenchError> {
     trace!("dropping space and table");
-    main_thread_db.query_parse::<()>(&query!("drop model bench.bench"))?;
-    main_thread_db.query_parse::<()>(&query!("drop space bench"))?;
+    main_thread_db.query_parse::<()>(&query!("drop space allow not empty bench"))?;
     Ok(())
 }
 

From c2cb63e4664442fff918cf69dc5dad5dfef4729a Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 4 Dec 2023 00:55:33 +0530
Subject: [PATCH 303/310] Add `select all`

---
 Cargo.lock                                 |   2 +-
 cli/src/resp.rs                            |  11 ++
 server/src/engine/core/dml/del.rs          |   1 +
 server/src/engine/core/dml/ins.rs          |   1 +
 server/src/engine/core/dml/mod.rs          |   9 +-
 server/src/engine/core/dml/sel.rs          | 139 ++++++++++++++++++++-
 server/src/engine/core/exec.rs             |  12 +-
 server/src/engine/core/index/key.rs        |   3 +
 server/src/engine/core/index/mod.rs        |  34 +++++
 server/src/engine/core/model/mod.rs        |   1 +
 server/src/engine/core/space.rs            |  10 +-
 server/src/engine/core/tests/dml/mod.rs    |  26 ++++
 server/src/engine/core/tests/dml/select.rs |  63 +++++++++-
 server/src/engine/data/cell.rs             |  25 ++--
 server/src/engine/fractal/mod.rs           |  10 +-
 server/src/engine/fractal/test_utils.rs    |   9 +-
 server/src/engine/idx/mod.rs               |  15 +++
 server/src/engine/idx/mtchm/imp.rs         |  28 +++++
 server/src/engine/idx/mtchm/iter.rs        |  46 +++++++
 server/src/engine/idx/mtchm/mod.rs         |   2 +-
 server/src/engine/macros.rs                |   3 +
 server/src/engine/net/protocol/mod.rs      |   1 +
 server/src/engine/ql/ast/mod.rs            |   8 +-
 server/src/engine/ql/dml/sel.rs            |  89 ++++++++++++-
 server/src/engine/ql/macros.rs             |   3 +
 server/src/engine/ql/tests/dml_tests.rs    |  47 +++++++
 26 files changed, 566 insertions(+), 32 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e644f78e..306ce0da 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1234,7 +1234,7 @@ dependencies = [
 [[package]]
 name = "skytable"
 version = "0.8.0"
-source = "git+https://github.com/skytable/client-rust.git?branch=octave#62e4f3152d56127e406b9d5eeea60696acd77031"
+source = "git+https://github.com/skytable/client-rust.git?branch=octave#091aa94a696e22abd2ce10845e3040c48ea30720"
 dependencies = [
  "async-trait",
  "bb8",
diff --git a/cli/src/resp.rs b/cli/src/resp.rs
index 4fe25a40..276a9fd0 100644
--- a/cli/src/resp.rs
+++ b/cli/src/resp.rs
@@ -44,6 +44,17 @@ pub fn format_response(resp: Response, print_special: bool) -> bool {
             print_row(r);
             println!();
         }
+        Response::Rows(rows) => {
+            if rows.is_empty() {
+                println!("{}", "[0 rows returned]".grey().italic());
+            } else {
+                for (i, row) in rows.into_iter().enumerate().map(|(i, r)| (i + 1, r)) {
+                    print!("{} ", format!("({i})").grey().bold());
+                    print_row(row);
+                    println!();
+                }
+            }
+        }
     };
     true
 }
diff --git a/server/src/engine/core/dml/del.rs b/server/src/engine/core/dml/del.rs
index 119d1c92..b40d7e64 100644
--- a/server/src/engine/core/dml/del.rs
+++ b/server/src/engine/core/dml/del.rs
@@ -45,6 +45,7 @@ pub fn delete(global: &impl GlobalInstanceLike, mut delete: DeleteStatement) ->
     core::with_model_for_data_update(global, delete.entity(), |model| {
         let g = sync::atm::cpin();
         let delta_state = model.delta_state();
+        let _idx_latch = model.primary_index().acquire_cd();
         // create new version
         let new_version = delta_state.create_new_data_delta_version();
         match model
diff --git a/server/src/engine/core/dml/ins.rs b/server/src/engine/core/dml/ins.rs
index 24cc73b8..441dacaf 100644
--- a/server/src/engine/core/dml/ins.rs
+++ b/server/src/engine/core/dml/ins.rs
@@ -49,6 +49,7 @@ pub fn insert_resp(
 pub fn insert(global: &impl GlobalInstanceLike, insert: InsertStatement) -> QueryResult<()> {
     core::with_model_for_data_update(global, insert.entity(), |mdl| {
         let (pk, data) = prepare_insert(mdl, insert.data())?;
+        let _idx_latch = mdl.primary_index().acquire_cd();
         let g = cpin();
         let ds = mdl.delta_state();
         // create new version
diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index ea01a691..9f82f3e5 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -43,10 +43,15 @@ use crate::{
 pub use {
     del::delete,
     ins::insert,
-    sel::select_custom,
+    sel::{select_custom, select_all},
     upd::{collect_trace_path as update_flow_trace, update},
 };
-pub use {del::delete_resp, ins::insert_resp, sel::select_resp, upd::update_resp};
+pub use {
+    del::delete_resp,
+    ins::insert_resp,
+    sel::{select_all_resp, select_resp},
+    upd::update_resp,
+};
 
 impl Model {
     pub(self) fn resolve_where<'a>(
diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
index 7199f404..c7e95428 100644
--- a/server/src/engine/core/dml/sel.rs
+++ b/server/src/engine/core/dml/sel.rs
@@ -25,17 +25,22 @@
 */
 
 use crate::engine::{
-    core::index::DcFieldIndex,
+    core::{
+        index::{
+            DcFieldIndex, IndexLatchHandleExclusive, PrimaryIndexKey, Row, RowData, RowDataLck,
+        },
+        model::Model,
+    },
     data::{
         cell::{Datacell, VirtualDatacell},
         tag::{DataTag, TagClass},
     },
     error::{QueryError, QueryResult},
     fractal::GlobalInstanceLike,
-    idx::{STIndex, STIndexSeq},
+    idx::{IndexMTRaw, MTIndexExt, STIndex, STIndexSeq},
     mem::IntegerRepr,
     net::protocol::{Response, ResponseType},
-    ql::dml::sel::SelectStatement,
+    ql::dml::sel::{SelectAllStatement, SelectStatement},
     sync,
 };
 
@@ -56,6 +61,83 @@ pub fn select_resp(
     })
 }
 
+pub fn select_all_resp(
+    global: &impl GlobalInstanceLike,
+    select: SelectAllStatement,
+) -> QueryResult<Response> {
+    let mut ret_buf = Vec::new();
+    let i = self::select_all(
+        global,
+        select,
+        &mut ret_buf,
+        |buf, mdl| {
+            IntegerRepr::scoped(mdl.fields().len() as u64, |repr| buf.extend(repr));
+            buf.push(b'\n');
+        },
+        |buf, data, _| encode_cell(buf, data),
+    )?;
+    Ok(Response::Serialized {
+        ty: ResponseType::MultiRow,
+        size: i,
+        data: ret_buf,
+    })
+}
+
+pub fn select_all<Fm, F, T>(
+    global: &impl GlobalInstanceLike,
+    select: SelectAllStatement,
+    serialize_target: &mut T,
+    mut f_mdl: Fm,
+    mut f: F,
+) -> QueryResult<usize>
+where
+    Fm: FnMut(&mut T, &Model),
+    F: FnMut(&mut T, &Datacell, usize),
+{
+    global.namespace().with_model(select.entity, |mdl| {
+        let g = sync::atm::cpin();
+        let mut i = 0;
+        f_mdl(serialize_target, mdl);
+        if select.wildcard {
+            for (key, data) in RowIteratorAll::new(&g, mdl, select.limit as usize) {
+                let vdc = VirtualDatacell::new_pk(key, mdl.p_tag());
+                for key in mdl.fields().stseq_ord_key() {
+                    let r = if key.as_str() == mdl.p_key() {
+                        &*vdc
+                    } else {
+                        data.fields().get(key).unwrap()
+                    };
+                    f(serialize_target, r, mdl.fields().len());
+                }
+                i += 1;
+            }
+        } else {
+            // schema check
+            if select.fields.len() > mdl.fields().len()
+                || select
+                    .fields
+                    .iter()
+                    .any(|f| !mdl.fields().st_contains(f.as_str()))
+            {
+                return Err(QueryError::QExecUnknownField);
+            }
+            for (key, data) in RowIteratorAll::new(&g, mdl, select.limit as usize) {
+                let vdc = VirtualDatacell::new_pk(key, mdl.p_tag());
+                for key in select.fields.iter() {
+                    let r = if key.as_str() == mdl.p_key() {
+                        &*vdc
+                    } else {
+                        data.fields().st_get(key.as_str()).unwrap()
+                    };
+                    f(serialize_target, r, select.fields.len());
+                }
+                i += 1;
+            }
+        }
+        Ok(i)
+    })
+}
+
 fn encode_cell(resp: &mut Vec<u8>, item: &Datacell) {
     resp.push((item.tag().tag_selector().value_u8() + 1) * (item.is_init() as u8));
     if item.is_null() {
@@ -100,7 +182,7 @@ where
 {
     global.namespace().with_model(select.entity(), |mdl| {
         let target_key = mdl.resolve_where(select.clauses_mut())?;
-        let pkdc = VirtualDatacell::new(target_key.clone());
+        let pkdc = VirtualDatacell::new(target_key.clone(), mdl.p_tag().tag_unique());
         let g = sync::atm::cpin();
         let mut read_field = |key, fields: &DcFieldIndex| {
             match fields.st_get(key) {
@@ -128,3 +210,52 @@ where
         Ok(())
     })
 }
+
+struct RowIteratorAll<'g> {
+    _g: &'g sync::atm::Guard,
+    mdl: &'g Model,
+    iter: <IndexMTRaw<Row> as MTIndexExt<Row, PrimaryIndexKey, RowDataLck>>::IterEntry<'g, 'g, 'g>,
+    _latch: IndexLatchHandleExclusive<'g>,
+    limit: usize,
+}
+
+impl<'g> RowIteratorAll<'g> {
+    fn new(g: &'g sync::atm::Guard, mdl: &'g Model, limit: usize) -> Self {
+        let idx = mdl.primary_index();
+        let latch = idx.acquire_exclusive();
+        Self {
+            _g: g,
+            mdl,
+            iter: idx.__raw_index().mt_iter_entry(g),
+            _latch: latch,
+            limit,
+        }
+    }
+    fn _next(
+        &mut self,
+    ) -> Option<(
+        &'g PrimaryIndexKey,
+        parking_lot::RwLockReadGuard<'g, RowData>,
+    )> {
+        if self.limit == 0 {
+            return None;
+        }
+        self.limit -= 1;
+        self.iter.next().map(|row| {
+            (
+                row.d_key(),
+                row.resolve_schema_deltas_and_freeze(self.mdl.delta_state()),
+            )
+        })
+    }
+}
+
+impl<'g> Iterator for RowIteratorAll<'g> {
+    type Item = (
+        &'g PrimaryIndexKey,
+        parking_lot::RwLockReadGuard<'g, RowData>,
+    );
+    fn next(&mut self) -> Option<Self::Item> {
+        self._next()
+    }
+}
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 96f8664c..daba30c0 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -195,15 +195,15 @@ fn cstate_use(
 fn run_nb(
     global: &Global,
     cstate: &mut ClientLocalState,
-    state: State<'_, InplaceData>,
+    mut state: State<'_, InplaceData>,
     stmt: KeywordStmt,
 ) -> QueryResult<Response> {
-    let stmt = stmt.value_u8() - KeywordStmt::Use.value_u8();
+    let stmt_c = stmt.value_u8() - KeywordStmt::Use.value_u8();
     static F: [fn(
         &Global,
         &mut ClientLocalState,
         &mut State<'static, InplaceData>,
-    ) -> QueryResult<Response>; 8] = [
+    ) -> QueryResult<Response>; 9] = [
         cstate_use, // use
         |g, c, s| _callgcs(g, c, s, ddl_misc::inspect),
         |_, _, _| Err(QueryError::QLUnknownStatement), // describe
@@ -212,12 +212,16 @@ fn run_nb(
         |g, _, s| _callgs(g, s, dml::update_resp),
         |g, _, s| _callgs(g, s, dml::delete_resp),
         |_, _, _| Err(QueryError::QLUnknownStatement), // exists
+        |g, _, s| _callgs(g, s, dml::select_all_resp),
     ];
     {
+        let n_offset_adjust = (stmt == KeywordStmt::Select) & state.cursor_rounded_eq(Token![all]);
+        state.cursor_ahead_if(n_offset_adjust);
+        let corrected_offset = (n_offset_adjust as u8 * 8) | (stmt_c * (!n_offset_adjust as u8));
         let mut state = unsafe {
             // UNSAFE(@ohsayan): this is a lifetime issue with the token handle
             core::mem::transmute(state)
         };
-        F[stmt as usize](global, cstate, &mut state)
+        F[corrected_offset as usize](global, cstate, &mut state)
     }
 }
diff --git a/server/src/engine/core/index/key.rs b/server/src/engine/core/index/key.rs
index 8fd7d804..de82d9bc 100644
--- a/server/src/engine/core/index/key.rs
+++ b/server/src/engine/core/index/key.rs
@@ -80,6 +80,9 @@ impl PrimaryIndexKey {
 }
 
 impl PrimaryIndexKey {
+    pub unsafe fn data(&self) -> SpecialPaddedWord {
+        core::mem::transmute_copy(&self.data)
+    }
     pub unsafe fn read_uint(&self) -> u64 {
         self.data.load()
     }
diff --git a/server/src/engine/core/index/mod.rs b/server/src/engine/core/index/mod.rs
index 348ee017..a6df689a 100644
--- a/server/src/engine/core/index/mod.rs
+++ b/server/src/engine/core/index/mod.rs
@@ -38,17 +38,27 @@ pub use {
     row::{DcFieldIndex, Row, RowData},
 };
 
+pub type RowDataLck = parking_lot::RwLock<RowData>;
+
 #[derive(Debug)]
 pub struct PrimaryIndex {
     data: IndexMTRaw<row::Row>,
+    latch: IndexLatch,
 }
 
 impl PrimaryIndex {
     pub fn new_empty() -> Self {
         Self {
             data: IndexMTRaw::idx_init(),
+            latch: IndexLatch::new(),
         }
     }
+    pub fn acquire_cd(&self) -> IndexLatchHandleShared {
+        self.latch.gl_handle_shared()
+    }
+    pub fn acquire_exclusive(&self) -> IndexLatchHandleExclusive {
+        self.latch.gl_handle_exclusive()
+    }
     pub fn select<'a, 'v, 't: 'v, 'g: 't>(&'t self, key: Lit<'a>, g: &'g Guard) -> Option<&'v Row> {
         self.data.mt_get_element(&key, g)
     }
@@ -59,3 +69,27 @@ impl PrimaryIndex {
         self.data.mt_len()
     }
 }
+
+#[derive(Debug)]
+pub struct IndexLatchHandleShared<'t>(parking_lot::RwLockReadGuard<'t, ()>);
+#[derive(Debug)]
+pub struct IndexLatchHandleExclusive<'t>(parking_lot::RwLockWriteGuard<'t, ()>);
+
+#[derive(Debug)]
+struct IndexLatch {
+    glck: parking_lot::RwLock<()>,
+}
+
+impl IndexLatch {
+    fn new() -> Self {
+        Self {
+            glck: parking_lot::RwLock::new(()),
+        }
+    }
+    fn gl_handle_shared(&self) -> IndexLatchHandleShared {
+        IndexLatchHandleShared(self.glck.read())
+    }
+    fn gl_handle_exclusive(&self) -> IndexLatchHandleExclusive {
+        IndexLatchHandleExclusive(self.glck.write())
+    }
+}
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index f87ae8e4..8b60b5ce 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -356,6 +356,7 @@ impl Model {
                     space.get_uuid(),
                     model_name,
                     model.get_uuid(),
+                    false,
                 );
             }
             // update global state
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 80f7fe32..730d932a 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -240,12 +240,20 @@ impl Space {
                         GenericTask::delete_space_dir(&space_name, space.get_uuid()),
                     ));
                 }
+                let space_uuid = space.get_uuid();
                 for model in space.models.into_iter() {
                     let e: EntityIDRef<'static> = unsafe {
                         // UNSAFE(@ohsayan): I want to try what the borrow checker has been trying
                         core::mem::transmute(EntityIDRef::new(space_name.as_str(), &model))
                     };
-                    let _ = models.st_delete(&e);
+                    let mdl = models.st_delete_return(&e).unwrap();
+                    global.purge_model_driver(
+                        &space_name,
+                        space_uuid,
+                        &model,
+                        mdl.get_uuid(),
+                        true,
+                    );
                 }
                 let _ = spaces.st_delete(space_name.as_str());
                 Ok(())
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index 4418a943..1736ca6a 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -155,6 +155,32 @@ pub(self) fn exec_select(
     _exec_only_select(global, select)
 }
 
+pub(self) fn exec_select_all(
+    global: &impl GlobalInstanceLike,
+    model: &str,
+    inserts: &[&str],
+    select: &str,
+) -> QueryResult<Vec<Vec<Datacell>>> {
+    _exec_only_create_space_model(global, model)?;
+    for insert in inserts {
+        _exec_only_insert(global, insert, |_| {})?;
+    }
+    let lex_sel = lex_insecure(select.as_bytes()).unwrap();
+    let select = parse_ast_node_full(&lex_sel[2..]).unwrap();
+    let mut r: Vec<Vec<Datacell>> = Vec::new();
+    dml::select_all(
+        global,
+        select,
+        &mut r,
+        |_, _| {},
+        |rows, dc, col_cnt| match rows.last_mut() {
+            Some(row) if row.len() != col_cnt => row.push(dc.clone()),
+            _ => rows.push(vec![dc.clone()]),
+        },
+    )?;
+    Ok(r)
+}
+
 pub(self) fn exec_select_only(
     global: &impl GlobalInstanceLike,
     select: &str,
diff --git a/server/src/engine/core/tests/dml/select.rs b/server/src/engine/core/tests/dml/select.rs
index 2af7dfd6..1e693f43 100644
--- a/server/src/engine/core/tests/dml/select.rs
+++ b/server/src/engine/core/tests/dml/select.rs
@@ -24,7 +24,10 @@
  *
 */
 
-use crate::engine::{data::cell::Datacell, error::QueryError, fractal::test_utils::TestGlobal};
+use {
+    crate::engine::{data::cell::Datacell, error::QueryError, fractal::test_utils::TestGlobal},
+    std::collections::HashMap,
+};
 
 #[test]
 fn simple_select_wildcard() {
@@ -100,3 +103,61 @@ fn select_nonexisting() {
         QueryError::QExecDmlRowNotFound
     );
 }
+
+/*
+    select all
+*/
+
+#[test]
+fn select_all_wildcard() {
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
+    let ret = super::exec_select_all(
+        &global,
+        "create model myspace.mymodel(username: string, password: string)",
+        &[
+            "insert into myspace.mymodel('sayan', 'password123')",
+            "insert into myspace.mymodel('robot', 'robot123')",
+            "insert into myspace.mymodel('douglas', 'galaxy123')",
+            "insert into myspace.mymodel('hgwells', 'timemachine')",
+            "insert into myspace.mymodel('orwell', '1984')",
+        ],
+        "select all * from myspace.mymodel LIMIT 100",
+    )
+    .unwrap();
+    let ret: HashMap<String, Vec<Datacell>> = ret
+        .into_iter()
+        .map(|mut d| (d.swap_remove(0).into_str().unwrap(), d))
+        .collect();
+    assert_eq!(ret.get("sayan").unwrap(), &intovec!["password123"]);
+    assert_eq!(ret.get("robot").unwrap(), &intovec!["robot123"]);
+    assert_eq!(ret.get("douglas").unwrap(), &intovec!["galaxy123"]);
+    assert_eq!(ret.get("hgwells").unwrap(), &intovec!["timemachine"]);
+    assert_eq!(ret.get("orwell").unwrap(), &intovec!["1984"]);
+}
+
+#[test]
+fn select_all_onefield() {
+    let global = TestGlobal::new_with_tmp_nullfs_driver();
+    let ret = super::exec_select_all(
+        &global,
+        "create model myspace.mymodel(username: string, password: string)",
+        &[
+            "insert into myspace.mymodel('sayan', 'password123')",
+            "insert into myspace.mymodel('robot', 'robot123')",
+            "insert into myspace.mymodel('douglas', 'galaxy123')",
+            "insert into myspace.mymodel('hgwells', 'timemachine')",
+            "insert into myspace.mymodel('orwell', '1984')",
+        ],
+        "select all username from myspace.mymodel LIMIT 100",
+    )
+    .unwrap();
+    let ret: HashMap<String, Vec<Datacell>> = ret
+        .into_iter()
+        .map(|mut d| (d.swap_remove(0).into_str().unwrap(), d))
+        .collect();
+    assert_eq!(ret.get("sayan").unwrap(), &intovec![]);
+    assert_eq!(ret.get("robot").unwrap(), &intovec![]);
+    assert_eq!(ret.get("douglas").unwrap(), &intovec![]);
+    assert_eq!(ret.get("hgwells").unwrap(), &intovec![]);
+    assert_eq!(ret.get("orwell").unwrap(), &intovec![]);
+}
diff --git a/server/src/engine/data/cell.rs b/server/src/engine/data/cell.rs
index f8a3dfd3..c81a7c26 100644
--- a/server/src/engine/data/cell.rs
+++ b/server/src/engine/data/cell.rs
@@ -24,6 +24,10 @@
  *
 */
 
+use crate::engine::core::index::PrimaryIndexKey;
+
+use super::tag::TagUnique;
+
 use {
     crate::engine::{
         self,
@@ -508,7 +512,8 @@ pub struct VirtualDatacell<'a> {
 }
 
 impl<'a> VirtualDatacell<'a> {
-    pub fn new(lit: Lit<'a>) -> Self {
+    pub fn new(lit: Lit<'a>, tag: TagUnique) -> Self {
+        debug_assert_eq!(lit.kind().tag_unique(), tag);
         Self {
             dc: ManuallyDrop::new(unsafe {
                 // UNSAFE(@ohsayan): this is a "reference" to a "virtual" aka fake DC. this just works because of memory layouts
@@ -517,11 +522,14 @@ impl<'a> VirtualDatacell<'a> {
             _lt: PhantomData,
         }
     }
-}
-
-impl<'a> From<Lit<'a>> for VirtualDatacell<'a> {
-    fn from(l: Lit<'a>) -> Self {
-        Self::new(l)
+    pub fn new_pk(pk: &'a PrimaryIndexKey, tag: FullTag) -> Self {
+        debug_assert_eq!(pk.tag(), tag.tag_unique());
+        Self {
+            dc: ManuallyDrop::new(unsafe {
+                Datacell::new(tag, DataRaw::word(pk.data().dwordqn_promote()))
+            }),
+            _lt: PhantomData,
+        }
     }
 }
 
@@ -547,5 +555,8 @@ impl<'a> Clone for VirtualDatacell<'a> {
 #[test]
 fn virtual_dc_damn() {
     let dc = Lit::new_str("hello, world");
-    assert_eq!(VirtualDatacell::from(dc), Datacell::from("hello, world"));
+    assert_eq!(
+        VirtualDatacell::new(dc, TagUnique::Str),
+        Datacell::from("hello, world")
+    );
 }
diff --git a/server/src/engine/fractal/mod.rs b/server/src/engine/fractal/mod.rs
index 168593cb..cc8321b5 100644
--- a/server/src/engine/fractal/mod.rs
+++ b/server/src/engine/fractal/mod.rs
@@ -126,6 +126,7 @@ pub trait GlobalInstanceLike {
         space_uuid: Uuid,
         model_name: &str,
         model_uuid: Uuid,
+        skip_delete: bool,
     );
     // taskmgr
     fn taskmgr_post_high_priority(&self, task: Task<CriticalTask>);
@@ -187,6 +188,7 @@ impl GlobalInstanceLike for Global {
         space_uuid: Uuid,
         model_name: &str,
         model_uuid: Uuid,
+        skip_delete: bool,
     ) {
         let id = ModelUniqueID::new(space_name, model_name, model_uuid);
         self.get_state()
@@ -194,9 +196,11 @@ impl GlobalInstanceLike for Global {
             .write()
             .remove(&id)
             .expect("tried to remove non existent driver");
-        self.taskmgr_post_standard_priority(Task::new(GenericTask::delete_model_dir(
-            space_name, space_uuid, model_name, model_uuid,
-        )));
+        if !skip_delete {
+            self.taskmgr_post_standard_priority(Task::new(GenericTask::delete_model_dir(
+                space_name, space_uuid, model_name, model_uuid,
+            )));
+        }
     }
     fn initialize_model_driver(
         &self,
diff --git a/server/src/engine/fractal/test_utils.rs b/server/src/engine/fractal/test_utils.rs
index 91e28852..ed555cd1 100644
--- a/server/src/engine/fractal/test_utils.rs
+++ b/server/src/engine/fractal/test_utils.rs
@@ -126,15 +126,18 @@ impl<Fs: RawFSInterface> GlobalInstanceLike for TestGlobal<Fs> {
         space_uuid: Uuid,
         model_name: &str,
         model_uuid: Uuid,
+        skip_delete: bool,
     ) {
         let id = ModelUniqueID::new(space_name, model_name, model_uuid);
         self.model_drivers
             .write()
             .remove(&id)
             .expect("tried to remove non-existent model");
-        self.taskmgr_post_standard_priority(Task::new(GenericTask::delete_model_dir(
-            space_name, space_uuid, model_name, model_uuid,
-        )));
+        if !skip_delete {
+            self.taskmgr_post_standard_priority(Task::new(GenericTask::delete_model_dir(
+                space_name, space_uuid, model_name, model_uuid,
+            )));
+        }
     }
     fn initialize_model_driver(
         &self,
diff --git a/server/src/engine/idx/mod.rs b/server/src/engine/idx/mod.rs
index 8543112b..a7678741 100644
--- a/server/src/engine/idx/mod.rs
+++ b/server/src/engine/idx/mod.rs
@@ -150,6 +150,9 @@ pub trait MTIndex<E, K, V>: IndexBaseSpec {
         't: 'v,
         V: 'v,
         Self: 't;
+    fn mt_iter_kv<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterKV<'t, 'g, 'v>;
+    fn mt_iter_key<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterKey<'t, 'g, 'v>;
+    fn mt_iter_val<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterVal<'t, 'g, 'v>;
     /// Returns the length of the index
     fn mt_len(&self) -> usize;
     /// Attempts to compact the backing storage
@@ -217,6 +220,18 @@ pub trait MTIndex<E, K, V>: IndexBaseSpec {
         'g: 't + 'v;
 }
 
+pub trait MTIndexExt<E, K, V>: MTIndex<E, K, V> {
+    type IterEntry<'t, 'g, 'v>: Iterator<Item = &'v E>
+    where
+        'g: 't + 'v,
+        't: 'v,
+        K: 'v,
+        V: 'v,
+        E: 'v,
+        Self: 't;
+    fn mt_iter_entry<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterEntry<'t, 'g, 'v>;
+}
+
 /// An unordered STIndex
 pub trait STIndex<K: ?Sized, V>: IndexBaseSpec {
     /// An iterator over the keys and values
diff --git a/server/src/engine/idx/mtchm/imp.rs b/server/src/engine/idx/mtchm/imp.rs
index f9b1edc6..6497b6e7 100644
--- a/server/src/engine/idx/mtchm/imp.rs
+++ b/server/src/engine/idx/mtchm/imp.rs
@@ -24,6 +24,8 @@
  *
 */
 
+use crate::engine::idx::MTIndexExt;
+
 #[cfg(debug_assertions)]
 use super::CHTRuntimeLog;
 use {
@@ -63,6 +65,20 @@ impl<E, C: Config> IndexBaseSpec for Raw<E, C> {
     }
 }
 
+impl<E: TreeElement, C: Config> MTIndexExt<E, E::Key, E::Value> for Raw<E, C> {
+    type IterEntry<'t, 'g, 'v> = super::iter::IterEntry<'t, 'g, 'v, E, C>
+    where
+        'g: 't + 'v,
+        't: 'v,
+        E::Key: 'v,
+        E::Value: 'v,
+        E: 'v,
+        Self: 't;
+    fn mt_iter_entry<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterEntry<'t, 'g, 'v> {
+        super::iter::IterEntry::new(self, g)
+    }
+}
+
 impl<E: TreeElement, C: Config> MTIndex<E, E::Key, E::Value> for Raw<E, C> {
     type IterKV<'t, 'g, 'v> = IterKV<'t, 'g, 'v, E, C>
     where
@@ -86,6 +102,18 @@ impl<E: TreeElement, C: Config> MTIndex<E, E::Key, E::Value> for Raw<E, C> {
         E::Value: 'v,
         Self: 't;
 
+    fn mt_iter_kv<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterKV<'t, 'g, 'v> {
+        super::iter::IterKV::new(self, g)
+    }
+
+    fn mt_iter_key<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterKey<'t, 'g, 'v> {
+        super::iter::IterKey::new(self, g)
+    }
+
+    fn mt_iter_val<'t, 'g, 'v>(&'t self, g: &'g Guard) -> Self::IterVal<'t, 'g, 'v> {
+        super::iter::IterVal::new(self, g)
+    }
+
     fn mt_len(&self) -> usize {
         self.len()
     }
diff --git a/server/src/engine/idx/mtchm/iter.rs b/server/src/engine/idx/mtchm/iter.rs
index 8081a009..c7838fb0 100644
--- a/server/src/engine/idx/mtchm/iter.rs
+++ b/server/src/engine/idx/mtchm/iter.rs
@@ -74,6 +74,44 @@ where
     }
 }
 
+pub struct IterEntry<'t, 'g, 'v, T, C>
+where
+    't: 'v,
+    'g: 'v + 't,
+    C: Config,
+    T: TreeElement,
+{
+    i: RawIter<'t, 'g, 'v, T, C, CfgIterEntry>,
+}
+
+
+impl<'t, 'g, 'v, T, C> IterEntry<'t, 'g, 'v, T, C>
+where
+    't: 'v,
+    'g: 'v + 't,
+    C: Config,
+    T: TreeElement,
+{
+    pub fn new(t: &'t RawTree<T, C>, g: &'g Guard) -> Self {
+        Self {
+            i: RawIter::new(t, g),
+        }
+    }
+}
+
+impl<'t, 'g, 'v, T, C> Iterator for IterEntry<'t, 'g, 'v, T, C>
+where
+    't: 'v,
+    'g: 'v + 't,
+    C: Config,
+    T: TreeElement,
+{
+    type Item = &'v T;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.i.next()
+    }
+}
+
 pub struct IterKey<'t, 'g, 'v, T, C>
 where
     't: 'v,
@@ -157,6 +195,14 @@ trait IterConfig<T> {
     fn some<'a>(v: &'a T) -> Option<Self::Ret<'a>>;
 }
 
+struct CfgIterEntry;
+impl<T: TreeElement> IterConfig<T> for CfgIterEntry {
+    type Ret<'a> = &'a T where T: 'a;
+    fn some<'a>(v: &'a T) -> Option<Self::Ret<'a>> {
+        Some(v)
+    }
+}
+
 struct CfgIterKV;
 impl<T: TreeElement> IterConfig<T> for CfgIterKV {
     type Ret<'a> = (&'a T::Key, &'a T::Value) where T: 'a;
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index da78271b..e8c03ef9 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -26,7 +26,7 @@
 
 mod access;
 pub mod imp;
-mod iter;
+pub(super) mod iter;
 pub mod meta;
 mod patch;
 #[cfg(test)]
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index 08383ebf..716010ad 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -326,6 +326,9 @@ macro_rules! Token {
     (allow) => {
         __kw_misc!(Allow)
     };
+    (all) => {
+        __kw_misc!(All)
+    }
 }
 
 macro_rules! union {
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index bf3e0b0b..16436f3c 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -74,6 +74,7 @@ pub enum ResponseType {
     Error = 0x10,
     Row = 0x11,
     Empty = 0x12,
+    MultiRow = 0x13,
 }
 
 #[derive(Debug, PartialEq)]
diff --git a/server/src/engine/ql/ast/mod.rs b/server/src/engine/ql/ast/mod.rs
index b0e2422e..5446f584 100644
--- a/server/src/engine/ql/ast/mod.rs
+++ b/server/src/engine/ql/ast/mod.rs
@@ -224,7 +224,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     /// Check if the current cursor can read a lit (with context from the data source); rounded
     pub fn can_read_lit_rounded(&self) -> bool {
         let mx = self.round_cursor();
-        Qd::can_read_lit_from(&self.d, &self.t[mx]) && mx == self.i
+        Qd::can_read_lit_from(&self.d, &self.t[mx]) & (mx == self.i)
     }
     #[inline(always)]
     /// Check if a lit can be read using the given token with context from the data source
@@ -252,7 +252,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     /// Check if the cursor equals the given token; rounded
     pub fn cursor_rounded_eq(&self, tok: Token<'a>) -> bool {
         let mx = self.round_cursor();
-        self.t[mx] == tok && mx == self.i
+        (self.t[mx] == tok) & (mx == self.i)
     }
     #[inline(always)]
     /// Check if the cursor equals the given token
@@ -271,7 +271,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     }
     #[inline(always)]
     pub(crate) fn cursor_has_ident_rounded(&self) -> bool {
-        self.offset_current_r(0).is_ident() && self.not_exhausted()
+        self.offset_current_r(0).is_ident() & self.not_exhausted()
     }
     #[inline(always)]
     /// Check if the current token stream matches the signature of an arity(0) fn; rounded
@@ -296,7 +296,7 @@ impl<'a, Qd: QueryData<'a>> State<'a, Qd> {
     #[inline(always)]
     /// Loop condition for tt and non-poisoned state only
     pub fn loop_tt(&self) -> bool {
-        self.not_exhausted() && self.okay()
+        self.not_exhausted() & self.okay()
     }
     #[inline(always)]
     /// Returns the position of the cursor
diff --git a/server/src/engine/ql/dml/sel.rs b/server/src/engine/ql/dml/sel.rs
index 2efd189f..e379db28 100644
--- a/server/src/engine/ql/dml/sel.rs
+++ b/server/src/engine/ql/dml/sel.rs
@@ -152,9 +152,87 @@ impl<'a> SelectStatement<'a> {
     }
 }
 
+#[derive(Debug, PartialEq)]
+pub struct SelectAllStatement<'a> {
+    pub entity: EntityIDRef<'a>,
+    pub fields: Vec<Ident<'a>>,
+    pub wildcard: bool,
+    pub limit: u64,
+}
+
+impl<'a> SelectAllStatement<'a> {
+    #[cfg(test)]
+    pub fn test_new(
+        entity: EntityIDRef<'a>,
+        fields: Vec<Ident<'a>>,
+        wildcard: bool,
+        limit: u64,
+    ) -> Self {
+        Self::new(entity, fields, wildcard, limit)
+    }
+    fn new(entity: EntityIDRef<'a>, fields: Vec<Ident<'a>>, wildcard: bool, limit: u64) -> Self {
+        Self {
+            entity,
+            fields,
+            wildcard,
+            limit,
+        }
+    }
+    fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        /*
+            smallest query: select all * from mymodel limit 10
+        */
+        if state.remaining() < 5 {
+            return Err(QueryError::QLUnexpectedEndOfStatement);
+        }
+        let mut select_fields = Vec::new();
+        let is_wildcard = state.cursor_eq(Token![*]);
+        state.cursor_ahead_if(is_wildcard);
+        while state.not_exhausted() && state.okay() && !is_wildcard {
+            match state.read() {
+                Token::Ident(id) => select_fields.push(*id),
+                _ => break,
+            }
+            state.cursor_ahead();
+            let nx_comma = state.cursor_rounded_eq(Token![,]);
+            let nx_from = state.cursor_rounded_eq(Token![from]);
+            state.poison_if_not(nx_comma | nx_from);
+            state.cursor_ahead_if(nx_comma);
+        }
+        state.poison_if_not(is_wildcard | !select_fields.is_empty());
+        if state.remaining() < 4 {
+            return Err(QueryError::QLUnexpectedEndOfStatement);
+        }
+        state.poison_if_not(state.cursor_eq(Token![from]));
+        state.cursor_ahead(); // ignore error
+        let entity = state.try_entity_buffered_into_state_uninit();
+        state.poison_if_not(state.cursor_rounded_eq(Token![limit]));
+        state.cursor_ahead_if(state.okay()); // we did read limit
+        state.poison_if(state.exhausted()); // we MUST have the limit
+        if state.okay() {
+            let lit = unsafe { state.fw_read().uck_read_lit() };
+            match lit.try_uint() {
+                Some(limit) => {
+                    return unsafe {
+                        // UNSAFE(@ohsayan): state guarantees this works
+                        Ok(Self::new(
+                            entity.assume_init(),
+                            select_fields,
+                            is_wildcard,
+                            limit,
+                        ))
+                    };
+                }
+                _ => {}
+            }
+        }
+        Err(QueryError::QLInvalidSyntax)
+    }
+}
+
 mod impls {
     use {
-        super::SelectStatement,
+        super::{SelectAllStatement, SelectStatement},
         crate::engine::{
             error::QueryResult,
             ql::ast::{traits::ASTNode, QueryData, State},
@@ -169,4 +247,13 @@ mod impls {
             Self::parse_select(state)
         }
     }
+    impl<'a> ASTNode<'a> for SelectAllStatement<'a> {
+        const MUST_USE_FULL_TOKEN_RANGE: bool = true;
+        const VERIFIES_FULL_TOKEN_RANGE_USAGE: bool = false;
+        fn __base_impl_parse_from_state<Qd: QueryData<'a>>(
+            state: &mut State<'a, Qd>,
+        ) -> QueryResult<Self> {
+            Self::parse(state)
+        }
+    }
 }
diff --git a/server/src/engine/ql/macros.rs b/server/src/engine/ql/macros.rs
index fdad0dd5..78ef4830 100644
--- a/server/src/engine/ql/macros.rs
+++ b/server/src/engine/ql/macros.rs
@@ -106,6 +106,9 @@ macro_rules! into_vec {
     ($ty:ty => ($($v:expr),* $(,)?)) => {{
         let v: Vec<$ty> = std::vec![$($v.into(),)*];
         v
+    }};
+    ($($v:expr),*) => {{
+        std::vec![$($v.into(),)*]
     }}
 }
 
diff --git a/server/src/engine/ql/tests/dml_tests.rs b/server/src/engine/ql/tests/dml_tests.rs
index 15ad9c44..4be9b1ae 100644
--- a/server/src/engine/ql/tests/dml_tests.rs
+++ b/server/src/engine/ql/tests/dml_tests.rs
@@ -957,3 +957,50 @@ mod where_clause {
         assert!(parse_ast_node_full::<WhereClause>(&tok).is_err());
     }
 }
+
+mod select_all {
+    use {
+        super::lex_insecure,
+        crate::engine::{
+            error::QueryError,
+            ql::{ast::parse_ast_node_full_with_space, dml::sel::SelectAllStatement},
+        },
+    };
+
+    #[test]
+    fn select_all_wildcard() {
+        let tok = lex_insecure(b"select all * from mymodel limit 100").unwrap();
+        assert_eq!(
+            parse_ast_node_full_with_space::<SelectAllStatement>(&tok[2..], "myspace").unwrap(),
+            SelectAllStatement::test_new(("myspace", "mymodel").into(), vec![], true, 100)
+        );
+    }
+
+    #[test]
+    fn select_all_multiple_fields() {
+        let tok = lex_insecure(b"select all username, password from mymodel limit 100").unwrap();
+        assert_eq!(
+            parse_ast_node_full_with_space::<SelectAllStatement>(&tok[2..], "myspace").unwrap(),
+            SelectAllStatement::test_new(
+                ("myspace", "mymodel").into(),
+                into_vec!["username", "password"],
+                false,
+                100
+            )
+        );
+    }
+
+    #[test]
+    fn select_all_missing_limit() {
+        let tok = lex_insecure(b"select all * from mymodel").unwrap();
+        assert_eq!(
+            parse_ast_node_full_with_space::<SelectAllStatement>(&tok[2..], "myspace").unwrap_err(),
+            QueryError::QLUnexpectedEndOfStatement
+        );
+        let tok = lex_insecure(b"select all username, password from mymodel").unwrap();
+        assert_eq!(
+            parse_ast_node_full_with_space::<SelectAllStatement>(&tok[2..], "myspace").unwrap_err(),
+            QueryError::QLUnexpectedEndOfStatement
+        );
+    }
+}

From 0ade763b78f1d37a7a7819c0949c4a58137195c6 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 4 Dec 2023 13:13:11 +0530
Subject: [PATCH 304/310] Clarify errors, fix responses

---
 CHANGELOG.md                                  |  2 +-
 server/src/engine/core/dml/sel.rs             | 11 ++++---
 server/src/engine/core/model/mod.rs           |  4 +--
 server/src/engine/core/tests/dml/mod.rs       |  2 +-
 server/src/engine/error.rs                    | 33 +++++++++----------
 server/src/engine/net/protocol/mod.rs         |  2 +-
 server/src/engine/ql/lex/mod.rs               | 32 +++++++++---------
 server/src/engine/ql/tests/lexer_tests.rs     |  6 ++--
 server/src/engine/tests/client/mod.rs         |  5 +--
 .../tests/{client => client_misc}/ddl.rs      |  0
 server/src/engine/tests/client_misc/mod.rs    | 29 ++++++++++++++++
 .../{client => client_misc}/sec/dcl_sec.rs    |  0
 .../{client => client_misc}/sec/ddl_sec.rs    |  0
 .../{client => client_misc}/sec/dml_sec.rs    |  0
 .../tests/{client => client_misc}/sec/mod.rs  |  0
 .../tests/{client => client_misc}/sysctl.rs   |  0
 server/src/engine/tests/mod.rs                |  1 +
 17 files changed, 75 insertions(+), 52 deletions(-)
 rename server/src/engine/tests/{client => client_misc}/ddl.rs (100%)
 create mode 100644 server/src/engine/tests/client_misc/mod.rs
 rename server/src/engine/tests/{client => client_misc}/sec/dcl_sec.rs (100%)
 rename server/src/engine/tests/{client => client_misc}/sec/ddl_sec.rs (100%)
 rename server/src/engine/tests/{client => client_misc}/sec/dml_sec.rs (100%)
 rename server/src/engine/tests/{client => client_misc}/sec/mod.rs (100%)
 rename server/src/engine/tests/{client => client_misc}/sysctl.rs (100%)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9d6cead0..fc77a7d0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -49,7 +49,7 @@ All changes in this project will be noted in this file.
           - `@uuidstr`: returns a string with a randomly generated v4 UUID
           - `@uuidbin`: same as `@uuidstr` but returns it as a blob
           - `@timesec`: returns a 64-bit integer with the current time in seconds
-    - `SELECT field1, field2, ... FROM <space>.<model> WHERE <primary_key_column> = <value>`
+    - `SELECT [ALL] field1, field2, ... FROM <space>.<model> WHERE <primary_key_column> = <value> [LIMIT n]`
     - New data manipulation via `UPDATE` allows arithmetic operations, string manipulation and more! Examples:
       - `UPDATE <space>.<model> SET col_num += 1 WHERE <primary_key_column> = <value>`
       - `UPDATE <space>.<model> SET mystring += " last part of string" WHERE ...`
diff --git a/server/src/engine/core/dml/sel.rs b/server/src/engine/core/dml/sel.rs
index c7e95428..c85d73b9 100644
--- a/server/src/engine/core/dml/sel.rs
+++ b/server/src/engine/core/dml/sel.rs
@@ -70,8 +70,8 @@ pub fn select_all_resp(
         global,
         select,
         &mut ret_buf,
-        |buf, mdl| {
-            IntegerRepr::scoped(mdl.fields().len() as u64, |repr| buf.extend(repr));
+        |buf, _, col_c| {
+            IntegerRepr::scoped(col_c as u64, |repr| buf.extend(repr));
             buf.push(b'\n');
         },
         |buf, data, _| encode_cell(buf, data),
@@ -91,14 +91,14 @@ pub fn select_all<Fm, F, T>(
     mut f: F,
 ) -> QueryResult<usize>
 where
-    Fm: FnMut(&mut T, &Model),
+    Fm: FnMut(&mut T, &Model, usize),
     F: FnMut(&mut T, &Datacell, usize),
 {
     global.namespace().with_model(select.entity, |mdl| {
         let g = sync::atm::cpin();
         let mut i = 0;
-        f_mdl(serialize_target, mdl);
         if select.wildcard {
+            f_mdl(serialize_target, mdl, mdl.fields().len());
             for (key, data) in RowIteratorAll::new(&g, mdl, select.limit as usize) {
                 let vdc = VirtualDatacell::new_pk(key, mdl.p_tag());
                 for key in mdl.fields().stseq_ord_key() {
@@ -121,6 +121,7 @@ where
             {
                 return Err(QueryError::QExecUnknownField);
             }
+            f_mdl(serialize_target, mdl, select.fields.len());
             for (key, data) in RowIteratorAll::new(&g, mdl, select.limit as usize) {
                 let vdc = VirtualDatacell::new_pk(key, mdl.p_tag());
                 for key in select.fields.iter() {
@@ -146,7 +147,7 @@ fn encode_cell(resp: &mut Vec<u8>, item: &Datacell) {
     unsafe {
         // UNSAFE(@ohsayan): +tagck
         match item.tag().tag_class() {
-            TagClass::Bool => resp.push(item.read_bool() as _),
+            TagClass::Bool => return resp.push(item.read_bool() as _),
             TagClass::UnsignedInt => IntegerRepr::scoped(item.read_uint(), |b| resp.extend(b)),
             TagClass::SignedInt => IntegerRepr::scoped(item.read_sint(), |b| resp.extend(b)),
             TagClass::Float => resp.extend(item.read_float().to_string().as_bytes()),
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 8b60b5ce..8d42482d 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -134,7 +134,6 @@ impl Model {
             }
             ret.push_str(&field_name);
             ret.push(':');
-            ret.push(' ');
             // TODO(@ohsayan): it's all lists right now, so this is okay but fix it later
             if field_decl.layers().len() == 1 {
                 ret.push_str(field_decl.layers()[0].tag().tag_selector().name_str());
@@ -150,7 +149,6 @@ impl Model {
             }
             if it.peek().is_some() {
                 ret.push(',');
-                ret.push(' ');
             }
         }
         ret.push('}');
@@ -651,7 +649,7 @@ impl Layer {
     pub fn new_empty_props(tag: FullTag) -> Self {
         Self::new(tag)
     }
-    const fn new(tag: FullTag) -> Self {
+    pub const fn new(tag: FullTag) -> Self {
         Self { tag }
     }
     const fn empty(tag: FullTag) -> Self {
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index 1736ca6a..77f2b8ba 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -172,7 +172,7 @@ pub(self) fn exec_select_all(
         global,
         select,
         &mut r,
-        |_, _| {},
+        |_, _, _| {},
         |rows, dc, col_cnt| match rows.last_mut() {
             Some(row) if row.len() != col_cnt => row.push(dc.clone()),
             _ => rows.push(vec![dc.clone()]),
diff --git a/server/src/engine/error.rs b/server/src/engine/error.rs
index 6e41fc2d..9c31013e 100644
--- a/server/src/engine/error.rs
+++ b/server/src/engine/error.rs
@@ -47,29 +47,26 @@ pub enum QueryError {
     SysTransactionalError = 4,
     /// insufficient permissions error
     SysPermissionDenied = 5,
-    // exchange
-    NetworkSubsystemCorruptedPacket = 24,
+    SysNetworkSystemIllegalClientPacket = 6,
     // QL
     /// something like an integer that randomly has a character to attached to it like `1234q`
-    LexInvalidLiteral = 25,
-    /// something like an invalid 'string" or a safe string with a bad length etc
-    LexInvalidParameter = 26,
+    LexInvalidInput = 25,
     /// unexpected byte
-    LexUnexpectedByte = 27,
+    LexUnexpectedByte = 26,
     /// expected a longer statement
-    QLUnexpectedEndOfStatement = 28,
+    QLUnexpectedEndOfStatement = 27,
     /// incorrect syntax for "something"
-    QLInvalidSyntax = 29,
+    QLInvalidSyntax = 28,
     /// invalid collection definition definition
-    QLInvalidCollectionSyntax = 30,
+    QLInvalidCollectionSyntax = 29,
     /// invalid type definition syntax
-    QLInvalidTypeDefinitionSyntax = 31,
+    QLInvalidTypeDefinitionSyntax = 30,
     /// expected a full entity definition
-    QLExpectedEntity = 32,
+    QLExpectedEntity = 31,
     /// expected a statement, found something else
-    QLExpectedStatement = 33,
+    QLExpectedStatement = 32,
     /// unknown statement
-    QLUnknownStatement = 34,
+    QLUnknownStatement = 33,
     // exec
     /// the object to be used as the "query container" is missing (for example, insert when the model was missing)
     QExecObjectNotFound = 100,
@@ -89,15 +86,15 @@ pub enum QueryError {
     QExecDdlModelAlterIllegal = 107,
     // exec DML
     /// violated the uniqueness property
-    QExecDmlDuplicate = 150,
+    QExecDmlDuplicate = 108,
     /// the data could not be validated for being accepted into a field/function/etc.
-    QExecDmlValidationError = 151,
+    QExecDmlValidationError = 109,
     /// the where expression has an unindexed column essentially implying that we can't run this query because of perf concerns
-    QExecDmlWhereHasUnindexedColumn = 152,
+    QExecDmlWhereHasUnindexedColumn = 110,
     /// the row matching the given match expression was not found
-    QExecDmlRowNotFound = 153,
+    QExecDmlRowNotFound = 111,
     /// this query needs a lock for execution, but that wasn't explicitly allowed anywhere
-    QExecNeedLock = 154,
+    QExecNeedLock = 112,
 }
 
 impl From<super::fractal::error::Error> for QueryError {
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index 16436f3c..02ed0a1f 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -169,7 +169,7 @@ pub(super) async fn query_loop<S: Socket>(
             (_, QExchangeResult::Error) => {
                 // respond with error
                 let [a, b] =
-                    (QueryError::NetworkSubsystemCorruptedPacket.value_u8() as u16).to_le_bytes();
+                    (QueryError::SysNetworkSystemIllegalClientPacket.value_u8() as u16).to_le_bytes();
                 con.write_all(&[ResponseType::Error.value_u8(), a, b])
                     .await?;
                 con.flush().await?;
diff --git a/server/src/engine/ql/lex/mod.rs b/server/src/engine/ql/lex/mod.rs
index 3ca0b428..c4dac3aa 100644
--- a/server/src/engine/ql/lex/mod.rs
+++ b/server/src/engine/ql/lex/mod.rs
@@ -221,13 +221,13 @@ mod insecure_impl {
                 .token_buffer
                 .try_next_ascii_u64_lf_separated_or_restore_cursor()
             else {
-                self.l.set_error(QueryError::LexInvalidLiteral);
+                self.l.set_error(QueryError::LexInvalidInput);
                 return;
             };
             let len = len as usize;
             match self.l.token_buffer.try_next_variable_block(len) {
                 Some(block) => self.l.push_token(Lit::new_bin(block)),
-                None => self.l.set_error(QueryError::LexInvalidLiteral),
+                None => self.l.set_error(QueryError::LexInvalidInput),
             }
         }
         pub(crate) fn scan_quoted_string(&mut self, quote_style: u8) {
@@ -260,7 +260,7 @@ mod insecure_impl {
                                 // UNSAFE(@ohsayan): we move the cursor ahead, now we're moving it back
                                 self.l.token_buffer.decr_cursor()
                             }
-                            self.l.set_error(QueryError::LexInvalidLiteral);
+                            self.l.set_error(QueryError::LexInvalidInput);
                             return;
                         }
                     }
@@ -278,7 +278,7 @@ mod insecure_impl {
             }
             match String::from_utf8(buf) {
                 Ok(s) if ended_with_quote => self.l.push_token(Lit::new_string(s)),
-                Err(_) | Ok(_) => self.l.set_error(QueryError::LexInvalidLiteral),
+                Err(_) | Ok(_) => self.l.set_error(QueryError::LexInvalidInput),
             }
         }
         pub(crate) fn scan_unsigned_integer(&mut self) {
@@ -299,7 +299,7 @@ mod insecure_impl {
                         .token_buffer
                         .rounded_cursor_not_eof_matches(u8::is_ascii_alphanumeric),
             ) {
-                self.l.set_error(QueryError::LexInvalidLiteral);
+                self.l.set_error(QueryError::LexInvalidInput);
             } else {
                 self.l.push_token(Lit::new_uint(int))
             }
@@ -323,7 +323,7 @@ mod insecure_impl {
                 {
                     self.l.push_token(Lit::new_sint(int))
                 } else {
-                    self.l.set_error(QueryError::LexInvalidLiteral)
+                    self.l.set_error(QueryError::LexInvalidInput)
                 }
             } else {
                 self.l.push_token(Token![-]);
@@ -425,7 +425,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             let nb = slf.param_buffer.next_byte();
             slf.l.push_token(Token::Lit(Lit::new_bool(nb == 1)));
             if nb > 1 {
-                slf.l.set_error(QueryError::LexInvalidParameter);
+                slf.l.set_error(QueryError::LexInvalidInput);
             }
         },
         // uint
@@ -434,7 +434,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             .try_next_ascii_u64_lf_separated_or_restore_cursor()
         {
             Some(int) => slf.l.push_token(Lit::new_uint(int)),
-            None => slf.l.set_error(QueryError::LexInvalidParameter),
+            None => slf.l.set_error(QueryError::LexInvalidInput),
         },
         // sint
         |slf| {
@@ -442,7 +442,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             if okay {
                 slf.l.push_token(Lit::new_sint(int))
             } else {
-                slf.l.set_error(QueryError::LexInvalidParameter)
+                slf.l.set_error(QueryError::LexInvalidInput)
             }
         },
         // float
@@ -456,12 +456,12 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
                         .map(core::str::FromStr::from_str)
                     {
                         Ok(Ok(f)) => slf.l.push_token(Lit::new_float(f)),
-                        _ => slf.l.set_error(QueryError::LexInvalidParameter),
+                        _ => slf.l.set_error(QueryError::LexInvalidInput),
                     }
                     return;
                 }
             }
-            slf.l.set_error(QueryError::LexInvalidParameter)
+            slf.l.set_error(QueryError::LexInvalidInput)
         },
         // binary
         |slf| {
@@ -469,7 +469,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
                 .param_buffer
                 .try_next_ascii_u64_lf_separated_or_restore_cursor()
             else {
-                slf.l.set_error(QueryError::LexInvalidParameter);
+                slf.l.set_error(QueryError::LexInvalidInput);
                 return;
             };
             match slf
@@ -477,7 +477,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
                 .try_next_variable_block(size_of_body as usize)
             {
                 Some(block) => slf.l.push_token(Lit::new_bin(block)),
-                None => slf.l.set_error(QueryError::LexInvalidParameter),
+                None => slf.l.set_error(QueryError::LexInvalidInput),
             }
         },
         // string
@@ -486,7 +486,7 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
                 .param_buffer
                 .try_next_ascii_u64_lf_separated_or_restore_cursor()
             else {
-                slf.l.set_error(QueryError::LexInvalidParameter);
+                slf.l.set_error(QueryError::LexInvalidInput);
                 return;
             };
             match slf
@@ -496,10 +496,10 @@ static SCAN_PARAM: [unsafe fn(&mut SecureLexer); 8] = unsafe {
             {
                 // TODO(@ohsayan): obliterate this alloc
                 Some(Ok(s)) => slf.l.push_token(Lit::new_string(s.to_owned())),
-                _ => slf.l.set_error(QueryError::LexInvalidParameter),
+                _ => slf.l.set_error(QueryError::LexInvalidInput),
             }
         },
         // ecc
-        |s| s.l.set_error(QueryError::LexInvalidParameter),
+        |s| s.l.set_error(QueryError::LexInvalidInput),
     ]
 };
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index 558bfc5c..ce6b5859 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -145,7 +145,7 @@ fn lex_string_bad_escape() {
     let wth = br#" '\a should be an alert on windows apparently' "#;
     assert_eq!(
         lex_insecure(wth).unwrap_err(),
-        QueryError::LexInvalidLiteral
+        QueryError::LexInvalidInput
     );
 }
 #[test]
@@ -153,12 +153,12 @@ fn lex_string_unclosed() {
     let wth = br#" 'omg where did the end go "#;
     assert_eq!(
         lex_insecure(wth).unwrap_err(),
-        QueryError::LexInvalidLiteral
+        QueryError::LexInvalidInput
     );
     let wth = br#" 'see, we escaped the end\' "#;
     assert_eq!(
         lex_insecure(wth).unwrap_err(),
-        QueryError::LexInvalidLiteral
+        QueryError::LexInvalidInput
     );
 }
 #[test]
diff --git a/server/src/engine/tests/client/mod.rs b/server/src/engine/tests/client/mod.rs
index faaae0fe..cbecb1bd 100644
--- a/server/src/engine/tests/client/mod.rs
+++ b/server/src/engine/tests/client/mod.rs
@@ -1,5 +1,5 @@
 /*
- * Created on Wed Nov 29 2023
+ * Created on Mon Dec 04 2023
  *
  * This file is a part of Skytable
  * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
@@ -24,6 +24,3 @@
  *
 */
 
-mod ddl;
-mod sec;
-mod sysctl;
diff --git a/server/src/engine/tests/client/ddl.rs b/server/src/engine/tests/client_misc/ddl.rs
similarity index 100%
rename from server/src/engine/tests/client/ddl.rs
rename to server/src/engine/tests/client_misc/ddl.rs
diff --git a/server/src/engine/tests/client_misc/mod.rs b/server/src/engine/tests/client_misc/mod.rs
new file mode 100644
index 00000000..faaae0fe
--- /dev/null
+++ b/server/src/engine/tests/client_misc/mod.rs
@@ -0,0 +1,29 @@
+/*
+ * Created on Wed Nov 29 2023
+ *
+ * This file is a part of Skytable
+ * Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
+ * NoSQL database written by Sayan Nandan ("the Author") with the
+ * vision to provide flexibility in data modelling without compromising
+ * on performance, queryability or scalability.
+ *
+ * Copyright (c) 2023, Sayan Nandan <ohsayan@outlook.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+*/
+
+mod ddl;
+mod sec;
+mod sysctl;
diff --git a/server/src/engine/tests/client/sec/dcl_sec.rs b/server/src/engine/tests/client_misc/sec/dcl_sec.rs
similarity index 100%
rename from server/src/engine/tests/client/sec/dcl_sec.rs
rename to server/src/engine/tests/client_misc/sec/dcl_sec.rs
diff --git a/server/src/engine/tests/client/sec/ddl_sec.rs b/server/src/engine/tests/client_misc/sec/ddl_sec.rs
similarity index 100%
rename from server/src/engine/tests/client/sec/ddl_sec.rs
rename to server/src/engine/tests/client_misc/sec/ddl_sec.rs
diff --git a/server/src/engine/tests/client/sec/dml_sec.rs b/server/src/engine/tests/client_misc/sec/dml_sec.rs
similarity index 100%
rename from server/src/engine/tests/client/sec/dml_sec.rs
rename to server/src/engine/tests/client_misc/sec/dml_sec.rs
diff --git a/server/src/engine/tests/client/sec/mod.rs b/server/src/engine/tests/client_misc/sec/mod.rs
similarity index 100%
rename from server/src/engine/tests/client/sec/mod.rs
rename to server/src/engine/tests/client_misc/sec/mod.rs
diff --git a/server/src/engine/tests/client/sysctl.rs b/server/src/engine/tests/client_misc/sysctl.rs
similarity index 100%
rename from server/src/engine/tests/client/sysctl.rs
rename to server/src/engine/tests/client_misc/sysctl.rs
diff --git a/server/src/engine/tests/mod.rs b/server/src/engine/tests/mod.rs
index 9b2dbecb..c59d29c0 100644
--- a/server/src/engine/tests/mod.rs
+++ b/server/src/engine/tests/mod.rs
@@ -28,3 +28,4 @@
 mod macros;
 mod cfg;
 mod client;
+mod client_misc;

From 9e1ce8c3e28d4d41c339ad61554cf4c4dae913ff Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Mon, 4 Dec 2023 18:25:43 +0530
Subject: [PATCH 305/310] Support `if exists` and `if not exists`

---
 server/src/engine/core/exec.rs             |  40 +-
 server/src/engine/core/model/mod.rs        |  24 +-
 server/src/engine/core/space.rs            |  38 +-
 server/src/engine/core/tests/dml/mod.rs    |   2 +-
 server/src/engine/macros.rs                |   5 +-
 server/src/engine/net/protocol/mod.rs      |   9 +-
 server/src/engine/ql/ddl/crt.rs            |  27 +-
 server/src/engine/ql/ddl/drop.rs           |  43 +-
 server/src/engine/ql/tests.rs              |  40 +-
 server/src/engine/ql/tests/schema_tests.rs | 488 +++++++++++----------
 10 files changed, 429 insertions(+), 287 deletions(-)

diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index daba30c0..7e7d6ac3 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -62,6 +62,16 @@ pub async fn dispatch_to_executor<'a>(
     }
 }
 
+fn _callgs_map<A: ASTNode<'static> + core::fmt::Debug, T>(
+    g: &Global,
+    state: &mut State<'static, InplaceData>,
+    f: impl FnOnce(&Global, A) -> Result<T, QueryError>,
+    map: impl FnOnce(T) -> Response,
+) -> QueryResult<Response> {
+    let cs = ASTNode::parse_from_state_hardened(state)?;
+    Ok(map(f(&g, cs)?))
+}
+
 #[inline(always)]
 fn _callgs<A: ASTNode<'static> + core::fmt::Debug, T>(
     g: &Global,
@@ -106,27 +116,28 @@ async fn run_blocking_stmt(
     let drop = stmt == KeywordStmt::Drop;
     let last_id = b.is_ident();
     let last_allow = Token![allow].eq(b);
-    let c_s = (create & Token![space].eq(a) & last_id) as u8 * 2;
-    let c_m = (create & Token![model].eq(a) & last_id) as u8 * 3;
+    let last_if = Token![if].eq(b);
+    let c_s = (create & Token![space].eq(a) & (last_id | last_if)) as u8 * 2;
+    let c_m = (create & Token![model].eq(a) & (last_id | last_if)) as u8 * 3;
     let a_s = (alter & Token![space].eq(a) & last_id) as u8 * 4;
     let a_m = (alter & Token![model].eq(a) & last_id) as u8 * 5;
-    let d_s = (drop & Token![space].eq(a) & (last_id | last_allow)) as u8 * 6;
-    let d_m = (drop & Token![model].eq(a) & (last_id | last_allow)) as u8 * 7;
+    let d_s = (drop & Token![space].eq(a) & (last_id | last_allow | last_if)) as u8 * 6;
+    let d_m = (drop & Token![model].eq(a) & (last_id | last_allow | last_if)) as u8 * 7;
     let fc = sysctl as u8 | c_s | c_m | a_s | a_m | d_s | d_m;
     state.cursor_ahead_if(!sysctl);
     static BLK_EXEC: [fn(
         Global,
         &ClientLocalState,
         &mut State<'static, InplaceData>,
-    ) -> QueryResult<()>; 8] = [
+    ) -> QueryResult<Response>; 8] = [
         |_, _, _| Err(QueryError::QLUnknownStatement),
         blocking_exec_sysctl,
-        |g, _, t| _callgs(&g, t, Space::transactional_exec_create),
-        |g, _, t| _callgs(&g, t, Model::transactional_exec_create),
-        |g, _, t| _callgs(&g, t, Space::transactional_exec_alter),
-        |g, _, t| _callgs(&g, t, Model::transactional_exec_alter),
-        |g, _, t| _callgs(&g, t, Space::transactional_exec_drop),
-        |g, _, t| _callgs(&g, t, Model::transactional_exec_drop),
+        |g, _, t| _callgs_map(&g, t, Space::transactional_exec_create, Response::Bool),
+        |g, _, t| _callgs_map(&g, t, Model::transactional_exec_create, Response::Bool),
+        |g, _, t| _callgs_map(&g, t, Space::transactional_exec_alter, |_| Response::Empty),
+        |g, _, t| _callgs_map(&g, t, Model::transactional_exec_alter, |_| Response::Empty),
+        |g, _, t| _callgs_map(&g, t, Space::transactional_exec_drop, Response::Bool),
+        |g, _, t| _callgs_map(&g, t, Model::transactional_exec_drop, Response::Bool),
     ];
     let r = unsafe {
         // UNSAFE(@ohsayan): the only await is within this block
@@ -135,8 +146,7 @@ async fn run_blocking_stmt(
         let static_state: &'static mut State<'static, InplaceData> =
             core::mem::transmute(&mut state);
         tokio::task::spawn_blocking(move || {
-            BLK_EXEC[fc as usize](c_glob, static_cstate, static_state)?;
-            Ok(Response::Empty)
+            BLK_EXEC[fc as usize](c_glob, static_cstate, static_state)
         })
         .await
     };
@@ -147,9 +157,9 @@ fn blocking_exec_sysctl(
     g: Global,
     cstate: &ClientLocalState,
     state: &mut State<'static, InplaceData>,
-) -> QueryResult<()> {
+) -> QueryResult<Response> {
     let r = ASTNode::parse_from_state_hardened(state)?;
-    super::dcl::exec(g, cstate, r)
+    super::dcl::exec(g, cstate, r).map(|_| Response::Empty)
 }
 
 /*
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index 8d42482d..cb921505 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -210,6 +210,7 @@ impl Model {
             model_name: _,
             fields,
             props,
+            ..
         }: CreateModel,
     ) -> QueryResult<Self> {
         let mut private = ModelPrivate::empty();
@@ -266,13 +267,18 @@ impl Model {
     pub fn transactional_exec_create<G: GlobalInstanceLike>(
         global: &G,
         stmt: CreateModel,
-    ) -> QueryResult<()> {
+    ) -> QueryResult<bool> {
         let (space_name, model_name) = (stmt.model_name.space(), stmt.model_name.entity());
+        let if_nx = stmt.if_not_exists;
         let model = Self::process_create(stmt)?;
         global.namespace().ddl_with_space_mut(&space_name, |space| {
             // TODO(@ohsayan): be extra cautious with post-transactional tasks (memck)
             if space.models().contains(model_name) {
-                return Err(QueryError::QExecDdlObjectAlreadyExists);
+                if if_nx {
+                    return Ok(false);
+                } else {
+                    return Err(QueryError::QExecDdlObjectAlreadyExists);
+                }
             }
             // since we've locked this down, no one else can parallely create another model in the same space (or remove)
             if G::FS_IS_NON_NULL {
@@ -314,18 +320,22 @@ impl Model {
                 .idx_models()
                 .write()
                 .insert(EntityID::new(&space_name, &model_name), model);
-            Ok(())
+            Ok(true)
         })
     }
     pub fn transactional_exec_drop<G: GlobalInstanceLike>(
         global: &G,
         stmt: DropModel,
-    ) -> QueryResult<()> {
+    ) -> QueryResult<bool> {
         let (space_name, model_name) = (stmt.entity.space(), stmt.entity.entity());
         global.namespace().ddl_with_space_mut(&space_name, |space| {
             if !space.models().contains(model_name) {
-                // the model isn't even present
-                return Err(QueryError::QExecObjectNotFound);
+                if stmt.if_exists {
+                    return Ok(false);
+                } else {
+                    // the model isn't even present
+                    return Err(QueryError::QExecObjectNotFound);
+                }
             }
             // get exclusive lock on models
             let mut models_idx = global.namespace().idx_models().write();
@@ -360,7 +370,7 @@ impl Model {
             // update global state
             let _ = models_idx.remove(&EntityIDRef::new(&space_name, &model_name));
             let _ = space.models_mut().remove(model_name);
-            Ok(())
+            Ok(true)
         })
     }
 }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index 730d932a..de2ab239 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -51,6 +51,7 @@ pub struct Space {
 struct ProcedureCreate {
     space_name: Box<str>,
     space: Space,
+    if_not_exists: bool,
 }
 
 impl Space {
@@ -106,6 +107,7 @@ impl Space {
         CreateSpace {
             space_name,
             mut props,
+            if_not_exists,
         }: CreateSpace,
     ) -> QueryResult<ProcedureCreate> {
         let space_name = space_name.to_string().into_boxed_str();
@@ -137,6 +139,7 @@ impl Space {
         Ok(ProcedureCreate {
             space_name,
             space: Space::new_empty_auto(dict::rflatten_metadata(props)),
+            if_not_exists,
         })
     }
 }
@@ -145,13 +148,21 @@ impl Space {
     pub fn transactional_exec_create<G: GlobalInstanceLike>(
         global: &G,
         space: CreateSpace,
-    ) -> QueryResult<()> {
+    ) -> QueryResult<bool> {
         // process create
-        let ProcedureCreate { space_name, space } = Self::process_create(space)?;
+        let ProcedureCreate {
+            space_name,
+            space,
+            if_not_exists,
+        } = Self::process_create(space)?;
         // lock the global namespace
         global.namespace().ddl_with_spaces_write(|spaces| {
             if spaces.st_contains(&space_name) {
-                return Err(QueryError::QExecDdlObjectAlreadyExists);
+                if if_not_exists {
+                    return Ok(false);
+                } else {
+                    return Err(QueryError::QExecDdlObjectAlreadyExists);
+                }
             }
             // commit txn
             if G::FS_IS_NON_NULL {
@@ -176,7 +187,7 @@ impl Space {
             }
             // update global state
             let _ = spaces.st_insert(space_name, space);
-            Ok(())
+            Ok(true)
         })
     }
     #[allow(unused)]
@@ -221,12 +232,17 @@ impl Space {
         DropSpace {
             space: space_name,
             force,
+            if_exists,
         }: DropSpace,
-    ) -> QueryResult<()> {
+    ) -> QueryResult<bool> {
         if force {
             global.namespace().ddl_with_all_mut(|spaces, models| {
                 let Some(space) = spaces.remove(space_name.as_str()) else {
-                    return Err(QueryError::QExecObjectNotFound);
+                    if if_exists {
+                        return Ok(false);
+                    } else {
+                        return Err(QueryError::QExecObjectNotFound);
+                    }
                 };
                 // commit drop
                 if G::FS_IS_NON_NULL {
@@ -256,12 +272,16 @@ impl Space {
                     );
                 }
                 let _ = spaces.st_delete(space_name.as_str());
-                Ok(())
+                Ok(true)
             })
         } else {
             global.namespace().ddl_with_spaces_write(|spaces| {
                 let Some(space) = spaces.get(space_name.as_str()) else {
-                    return Err(QueryError::QExecObjectNotFound);
+                    if if_exists {
+                        return Ok(false);
+                    } else {
+                        return Err(QueryError::QExecObjectNotFound);
+                    }
                 };
                 if !space.models.is_empty() {
                     // nonempty, we can't do anything
@@ -280,7 +300,7 @@ impl Space {
                     ));
                 }
                 let _ = spaces.st_delete(space_name.as_str());
-                Ok(())
+                Ok(true)
             })
         }
     }
diff --git a/server/src/engine/core/tests/dml/mod.rs b/server/src/engine/core/tests/dml/mod.rs
index 77f2b8ba..8385004c 100644
--- a/server/src/engine/core/tests/dml/mod.rs
+++ b/server/src/engine/core/tests/dml/mod.rs
@@ -50,7 +50,7 @@ fn _exec_only_create_space_model(global: &impl GlobalInstanceLike, model: &str)
         .insert("myspace".into(), Space::new_auto_all().into());
     let lex_create_model = lex_insecure(model.as_bytes()).unwrap();
     let stmt_create_model = parse_ast_node_full(&lex_create_model[2..]).unwrap();
-    Model::transactional_exec_create(global, stmt_create_model)
+    Model::transactional_exec_create(global, stmt_create_model).map(|_| ())
 }
 
 fn _exec_only_insert<T>(
diff --git a/server/src/engine/macros.rs b/server/src/engine/macros.rs
index 716010ad..6bb0fc1a 100644
--- a/server/src/engine/macros.rs
+++ b/server/src/engine/macros.rs
@@ -328,7 +328,10 @@ macro_rules! Token {
     };
     (all) => {
         __kw_misc!(All)
-    }
+    };
+    (exists) => {
+        __kw_stmt!(Exists)
+    };
 }
 
 macro_rules! union {
diff --git a/server/src/engine/net/protocol/mod.rs b/server/src/engine/net/protocol/mod.rs
index 02ed0a1f..1dd6d62c 100644
--- a/server/src/engine/net/protocol/mod.rs
+++ b/server/src/engine/net/protocol/mod.rs
@@ -120,6 +120,7 @@ pub enum Response {
         size: usize,
         data: Vec<u8>,
     },
+    Bool(bool),
 }
 
 pub(super) async fn query_loop<S: Socket>(
@@ -168,8 +169,8 @@ pub(super) async fn query_loop<S: Socket>(
             }
             (_, QExchangeResult::Error) => {
                 // respond with error
-                let [a, b] =
-                    (QueryError::SysNetworkSystemIllegalClientPacket.value_u8() as u16).to_le_bytes();
+                let [a, b] = (QueryError::SysNetworkSystemIllegalClientPacket.value_u8() as u16)
+                    .to_le_bytes();
                 con.write_all(&[ResponseType::Error.value_u8(), a, b])
                     .await?;
                 con.flush().await?;
@@ -192,6 +193,10 @@ pub(super) async fn query_loop<S: Socket>(
                 con.write_u8(b'\n').await?;
                 con.write_all(&data).await?;
             }
+            Ok(Response::Bool(b)) => {
+                con.write_all(&[ResponseType::Bool.value_u8(), b as u8])
+                    .await?
+            }
             Ok(Response::Null) => con.write_u8(ResponseType::Null.value_u8()).await?,
             Err(e) => {
                 let [a, b] = (e.value_u8() as u16).to_le_bytes();
diff --git a/server/src/engine/ql/ddl/crt.rs b/server/src/engine/ql/ddl/crt.rs
index 4827146f..693cc8b9 100644
--- a/server/src/engine/ql/ddl/crt.rs
+++ b/server/src/engine/ql/ddl/crt.rs
@@ -40,6 +40,13 @@ use {
     },
 };
 
+fn sig_if_not_exists<'a, Qd: QueryData<'a>>(state: &State<'a, Qd>) -> bool {
+    Token![if].eq(state.offset_current_r(0))
+        & Token![not].eq(state.offset_current_r(1))
+        & Token![exists].eq(state.offset_current_r(2))
+        & (state.remaining() >= 3)
+}
+
 #[derive(Debug, PartialEq)]
 /// A space
 pub struct CreateSpace<'a> {
@@ -47,6 +54,7 @@ pub struct CreateSpace<'a> {
     pub space_name: Ident<'a>,
     /// properties
     pub props: DictGeneric,
+    pub if_not_exists: bool,
 }
 
 impl<'a> CreateSpace<'a> {
@@ -57,6 +65,13 @@ impl<'a> CreateSpace<'a> {
         if compiler::unlikely(state.remaining() < 1) {
             return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
         }
+        // check for `if not exists`
+        let if_not_exists = sig_if_not_exists(state);
+        state.cursor_ahead_by(if_not_exists as usize * 3);
+        // get space name
+        if state.exhausted() {
+            return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
+        }
         let space_name = state.fw_read();
         state.poison_if_not(space_name.is_ident());
         // either we have `with` or nothing. don't be stupid
@@ -65,7 +80,7 @@ impl<'a> CreateSpace<'a> {
         state.cursor_ahead_if(has_more_properties); // +WITH
         let mut d = DictGeneric::new();
         // properties
-        if has_more_properties && state.okay() {
+        if has_more_properties & state.okay() {
             syn::rfold_dict(DictFoldState::OB, state, &mut d);
         }
         if state.okay() {
@@ -75,6 +90,7 @@ impl<'a> CreateSpace<'a> {
                     space_name.uck_read_ident()
                 },
                 props: d,
+                if_not_exists,
             })
         } else {
             Err(QueryError::QLInvalidSyntax)
@@ -91,6 +107,8 @@ pub struct CreateModel<'a> {
     pub(in crate::engine) fields: Vec<FieldSpec<'a>>,
     /// properties
     pub(in crate::engine) props: DictGeneric,
+    /// if not exists
+    pub(in crate::engine) if_not_exists: bool,
 }
 
 /*
@@ -106,18 +124,22 @@ impl<'a> CreateModel<'a> {
         model_name: EntityIDRef<'a>,
         fields: Vec<FieldSpec<'a>>,
         props: DictGeneric,
+        if_not_exists: bool,
     ) -> Self {
         Self {
             model_name,
             fields,
             props,
+            if_not_exists,
         }
     }
-
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         if compiler::unlikely(state.remaining() < 10) {
             return compiler::cold_rerr(QueryError::QLUnexpectedEndOfStatement);
         }
+        // if not exists?
+        let if_not_exists = sig_if_not_exists(state);
+        state.cursor_ahead_by(if_not_exists as usize * 3);
         // model name; ignore errors
         let model_uninit = state.try_entity_buffered_into_state_uninit();
         state.poison_if_not(state.cursor_eq(Token![() open]));
@@ -150,6 +172,7 @@ impl<'a> CreateModel<'a> {
                 },
                 fields,
                 props,
+                if_not_exists,
             })
         } else {
             Err(QueryError::QLInvalidSyntax)
diff --git a/server/src/engine/ql/ddl/drop.rs b/server/src/engine/ql/ddl/drop.rs
index 75e3d69d..b164b16c 100644
--- a/server/src/engine/ql/ddl/drop.rs
+++ b/server/src/engine/ql/ddl/drop.rs
@@ -33,23 +33,33 @@ use crate::engine::{
     },
 };
 
+fn sig_if_exists<'a, Qd: QueryData<'a>>(state: &State<'a, Qd>) -> bool {
+    Token![if].eq(state.offset_current_r(0)) & Token![exists].eq(state.offset_current_r(1))
+}
+
 #[derive(Debug, PartialEq)]
 /// A generic representation of `drop` query
 pub struct DropSpace<'a> {
     pub(in crate::engine) space: Ident<'a>,
     pub(in crate::engine) force: bool,
+    pub(in crate::engine) if_exists: bool,
 }
 
 impl<'a> DropSpace<'a> {
     #[inline(always)]
     /// Instantiate
-    pub const fn new(space: Ident<'a>, force: bool) -> Self {
-        Self { space, force }
+    pub const fn new(space: Ident<'a>, force: bool, if_exists: bool) -> Self {
+        Self {
+            space,
+            force,
+            if_exists,
+        }
     }
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<DropSpace<'a>> {
         /*
             either drop space <myspace> OR drop space allow not empty <myspace>
         */
+        let if_exists = check_if_exists(state)?;
         if state.cursor_is_ident() {
             let ident = state.fw_read();
             // either `force` or nothing
@@ -59,6 +69,7 @@ impl<'a> DropSpace<'a> {
                     ident.uck_read_ident()
                 },
                 false,
+                if_exists,
             ));
         } else {
             if ddl_allow_non_empty(state) {
@@ -67,13 +78,25 @@ impl<'a> DropSpace<'a> {
                     // UNSAFE(@ohsayan): verified in branch
                     state.fw_read().uck_read_ident()
                 };
-                return Ok(DropSpace::new(space_name, true));
+                return Ok(DropSpace::new(space_name, true, if_exists));
             }
         }
         Err(QueryError::QLInvalidSyntax)
     }
 }
 
+fn check_if_exists<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> Result<bool, QueryError> {
+    if state.exhausted() {
+        return Err(QueryError::QLUnexpectedEndOfStatement);
+    }
+    let if_exists = sig_if_exists(state);
+    state.cursor_ahead_by((if_exists as usize) << 1);
+    if state.exhausted() {
+        return Err(QueryError::QLUnexpectedEndOfStatement);
+    }
+    Ok(if_exists)
+}
+
 #[inline(always)]
 fn ddl_allow_non_empty<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> bool {
     let tok_allow = Token![allow].eq(state.offset_current_r(0));
@@ -87,22 +110,28 @@ fn ddl_allow_non_empty<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> bool
 pub struct DropModel<'a> {
     pub(in crate::engine) entity: EntityIDRef<'a>,
     pub(in crate::engine) force: bool,
+    pub(in crate::engine) if_exists: bool,
 }
 
 impl<'a> DropModel<'a> {
     #[inline(always)]
-    pub fn new(entity: EntityIDRef<'a>, force: bool) -> Self {
-        Self { entity, force }
+    pub fn new(entity: EntityIDRef<'a>, force: bool, if_exists: bool) -> Self {
+        Self {
+            entity,
+            force,
+            if_exists,
+        }
     }
     fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
+        let if_exists = check_if_exists(state)?;
         if state.cursor_is_ident() {
             let e = state.try_entity_ref_result()?;
-            return Ok(DropModel::new(e, false));
+            return Ok(DropModel::new(e, false, if_exists));
         } else {
             if ddl_allow_non_empty(state) {
                 state.cursor_ahead_by(3); // allow not empty
                 let e = state.try_entity_ref_result()?;
-                return Ok(DropModel::new(e, true));
+                return Ok(DropModel::new(e, true, if_exists));
             }
         }
         Err(QueryError::QLInvalidSyntax)
diff --git a/server/src/engine/ql/tests.rs b/server/src/engine/ql/tests.rs
index 25c4b3a1..ecdd34f4 100644
--- a/server/src/engine/ql/tests.rs
+++ b/server/src/engine/ql/tests.rs
@@ -25,7 +25,10 @@
 */
 
 use {
-    super::lex::{InsecureLexer, SecureLexer, Symbol, Token},
+    super::{
+        ast::{self, traits::ASTNode},
+        lex::{InsecureLexer, SecureLexer, Symbol, Token},
+    },
     crate::{
         engine::{data::cell::Datacell, error::QueryResult},
         util::test_utils,
@@ -138,3 +141,38 @@ fn fuzz_tokens(src: &[u8], fuzzverify: impl Fn(bool, &[Token]) -> bool) {
         }
     }
 }
+
+pub(self) fn fullparse_verify<'a, A: ASTNode<'a> + 'a>(q: &'a str, offset: usize, v: impl Fn(A)) {
+    let tok = lex_insecure(q.as_bytes()).unwrap();
+    unsafe {
+        let q: &'a [_] = core::mem::transmute(tok.as_slice());
+        let a: A = ASTNode::from_insecure_tokens_full(&q[offset..]).unwrap();
+        v(a);
+    }
+}
+
+pub(self) fn fullparse_verify_substmt<'a, A: ASTNode<'a> + 'a>(q: &'a str, v: impl Fn(A)) {
+    fullparse_verify(q, 2, v)
+}
+
+pub(self) fn fullparse_verify_with_space<'a, A: ASTNode<'a> + 'a>(
+    q: &'a str,
+    space_name: &'static str,
+    offset: usize,
+    v: impl Fn(A),
+) {
+    let tok = lex_insecure(q.as_bytes()).unwrap();
+    unsafe {
+        let q: &'static [_] = core::mem::transmute(&tok.as_slice()[offset..]);
+        let a: A = ast::parse_ast_node_full_with_space(q, space_name).unwrap();
+        v(a);
+    }
+}
+
+pub(self) fn fullparse_verify_substmt_with_space<'a, A: ASTNode<'a> + 'a>(
+    q: &'a str,
+    space_name: &'static str,
+    v: impl Fn(A),
+) {
+    fullparse_verify_with_space(q, space_name, 2, v)
+}
diff --git a/server/src/engine/ql/tests/schema_tests.rs b/server/src/engine/ql/tests/schema_tests.rs
index e22b6666..2ecf25e3 100644
--- a/server/src/engine/ql/tests/schema_tests.rs
+++ b/server/src/engine/ql/tests/schema_tests.rs
@@ -32,38 +32,34 @@ use {
 mod alter_space {
     use {
         super::*,
-        crate::engine::{
-            data::lit::Lit,
-            ql::{ast::parse_ast_node_full, ddl::alt::AlterSpace},
-        },
+        crate::engine::{data::lit::Lit, ql::ddl::alt::AlterSpace},
     };
     #[test]
     fn alter_space_mini() {
-        let tok = lex_insecure(b"alter model mymodel with {}").unwrap();
-        let r = parse_ast_node_full::<AlterSpace>(&tok[2..]).unwrap();
-        assert_eq!(r, AlterSpace::new(Ident::from("mymodel"), null_dict! {}));
+        fullparse_verify_substmt("alter model mymodel with {}", |r: AlterSpace| {
+            assert_eq!(r, AlterSpace::new(Ident::from("mymodel"), null_dict! {}));
+        })
     }
     #[test]
     fn alter_space() {
-        let tok = lex_insecure(
-            br#"
-                alter model mymodel with {
-                    max_entry: 1000,
-                    driver: "ts-0.8"
-                }
-            "#,
-        )
-        .unwrap();
-        let r = parse_ast_node_full::<AlterSpace>(&tok[2..]).unwrap();
-        assert_eq!(
-            r,
-            AlterSpace::new(
-                Ident::from("mymodel"),
-                null_dict! {
-                    "max_entry" => Lit::new_uint(1000),
-                    "driver" => Lit::new_string("ts-0.8".into())
-                }
-            )
+        fullparse_verify_substmt(
+            r#"
+            alter model mymodel with {
+                max_entry: 1000,
+                driver: "ts-0.8"
+            }"#,
+            |r: AlterSpace| {
+                assert_eq!(
+                    r,
+                    AlterSpace::new(
+                        Ident::from("mymodel"),
+                        null_dict! {
+                            "max_entry" => Lit::new_uint(1000),
+                            "driver" => Lit::new_string("ts-0.8".into())
+                        }
+                    )
+                );
+            },
         );
     }
 }
@@ -378,233 +374,221 @@ mod fields {
 }
 mod schemas {
     use super::*;
-    use crate::engine::ql::{
-        ast::parse_ast_node_full_with_space,
-        ddl::{
-            crt::CreateModel,
-            syn::{FieldSpec, LayerSpec},
-        },
+    use crate::engine::ql::ddl::{
+        crt::CreateModel,
+        syn::{FieldSpec, LayerSpec},
     };
     #[test]
     fn schema_mini() {
-        let tok = lex_insecure(
-            b"
-                create model mymodel(
-                    primary username: string,
-                    password: binary
-                )
-            ",
-        )
-        .unwrap();
-        let tok = &tok[2..];
-
-        // parse model
-        let model = parse_ast_node_full_with_space::<CreateModel>(tok, "apps").unwrap();
-
-        assert_eq!(
-            model,
-            CreateModel::new(
-                ("apps", "mymodel").into(),
-                vec![
-                    FieldSpec::new(
-                        Ident::from("username"),
-                        vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
-                        false,
-                        true,
-                    ),
-                    FieldSpec::new(
-                        Ident::from("password"),
-                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
-                        false,
-                        false,
-                    )
-                ],
-                null_dict! {}
-            )
-        )
+        let mut ret = CreateModel::new(
+            ("apps", "mymodel").into(),
+            vec![
+                FieldSpec::new(
+                    Ident::from("username"),
+                    vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
+                    false,
+                    true,
+                ),
+                FieldSpec::new(
+                    Ident::from("password"),
+                    vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
+                    false,
+                    false,
+                ),
+            ],
+            null_dict! {},
+            false,
+        );
+        fullparse_verify_substmt_with_space(
+            "create model mymodel(
+                primary username: string,
+                password: binary
+            )",
+            "apps",
+            |r: CreateModel| assert_eq!(r, ret),
+        );
+        ret.if_not_exists = true;
+        fullparse_verify_substmt_with_space(
+            "create model if not exists mymodel(
+                primary username: string,
+                password: binary
+            )",
+            "apps",
+            |r: CreateModel| assert_eq!(r, ret),
+        );
     }
     #[test]
     fn schema() {
-        let tok = lex_insecure(
-            b"
-                create model mymodel(
-                    primary username: string,
-                    password: binary,
-                    null profile_pic: binary
-                )
-            ",
-        )
-        .unwrap();
-        let tok = &tok[2..];
-
-        // parse model
-        let model = parse_ast_node_full_with_space::<CreateModel>(tok, "apps").unwrap();
-
-        assert_eq!(
-            model,
-            CreateModel::new(
-                ("apps", "mymodel").into(),
-                vec![
-                    FieldSpec::new(
-                        Ident::from("username"),
-                        vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
-                        false,
-                        true,
-                    ),
-                    FieldSpec::new(
-                        Ident::from("password"),
-                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
-                        false,
-                        false,
-                    ),
-                    FieldSpec::new(
-                        Ident::from("profile_pic"),
-                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
-                        true,
-                        false,
-                    )
-                ],
-                null_dict! {}
-            )
-        )
+        let mut ret = CreateModel::new(
+            ("apps", "mymodel").into(),
+            vec![
+                FieldSpec::new(
+                    Ident::from("username"),
+                    vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
+                    false,
+                    true,
+                ),
+                FieldSpec::new(
+                    Ident::from("password"),
+                    vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
+                    false,
+                    false,
+                ),
+                FieldSpec::new(
+                    Ident::from("profile_pic"),
+                    vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
+                    true,
+                    false,
+                ),
+            ],
+            null_dict! {},
+            false,
+        );
+        fullparse_verify_substmt_with_space(
+            "create model mymodel(
+            primary username: string,
+            password: binary,
+            null profile_pic: binary
+        )",
+            "apps",
+            |r: CreateModel| assert_eq!(r, ret),
+        );
+        ret.if_not_exists = true;
+        fullparse_verify_substmt_with_space(
+            "create model if not exists mymodel(
+            primary username: string,
+            password: binary,
+            null profile_pic: binary
+        )",
+            "apps",
+            |r: CreateModel| assert_eq!(r, ret),
+        );
     }
 
     #[test]
     fn schema_pro() {
-        let tok = lex_insecure(
-            b"
-                create model mymodel(
-                    primary username: string,
-                    password: binary,
-                    null profile_pic: binary,
-                    null notes: list {
-                        type: string,
-                        unique: true,
-                    }
-                )
-            ",
-        )
-        .unwrap();
-        let tok = &tok[2..];
-
-        // parse model
-        let model = parse_ast_node_full_with_space::<CreateModel>(tok, "apps").unwrap();
-
-        assert_eq!(
-            model,
-            CreateModel::new(
-                ("apps", "mymodel").into(),
-                vec![
-                    FieldSpec::new(
-                        Ident::from("username"),
-                        vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
-                        false,
-                        true
-                    ),
-                    FieldSpec::new(
-                        Ident::from("password"),
-                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
-                        false,
-                        false
-                    ),
-                    FieldSpec::new(
-                        Ident::from("profile_pic"),
-                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
-                        true,
-                        false
-                    ),
-                    FieldSpec::new(
-                        Ident::from("notes"),
-                        vec![
-                            LayerSpec::new(Ident::from("string"), null_dict! {}),
-                            LayerSpec::new(
-                                Ident::from("list"),
-                                null_dict! {
-                                    "unique" => Lit::new_bool(true)
-                                }
-                            )
-                        ],
-                        true,
-                        false
-                    )
-                ],
-                null_dict! {}
-            )
+        let mut ret = CreateModel::new(
+            ("apps", "mymodel").into(),
+            vec![
+                FieldSpec::new(
+                    Ident::from("username"),
+                    vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
+                    false,
+                    true,
+                ),
+                FieldSpec::new(
+                    Ident::from("password"),
+                    vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
+                    false,
+                    false,
+                ),
+                FieldSpec::new(
+                    Ident::from("profile_pic"),
+                    vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
+                    true,
+                    false,
+                ),
+                FieldSpec::new(
+                    Ident::from("notes"),
+                    vec![
+                        LayerSpec::new(Ident::from("string"), null_dict! {}),
+                        LayerSpec::new(
+                            Ident::from("list"),
+                            null_dict! {
+                                "unique" => Lit::new_bool(true)
+                            },
+                        ),
+                    ],
+                    true,
+                    false,
+                ),
+            ],
+            null_dict! {},
+            false,
+        );
+        ret.if_not_exists = true;
+        fullparse_verify_substmt_with_space(
+            "
+        create model if not exists mymodel(
+            primary username: string,
+            password: binary,
+            null profile_pic: binary,
+            null notes: list {
+                type: string,
+                unique: true,
+            }
         )
+        ",
+            "apps",
+            |r: CreateModel| assert_eq!(ret, r),
+        );
     }
-
     #[test]
     fn schema_pro_max() {
-        let tok = lex_insecure(
-            b"
-                create model mymodel(
-                    primary username: string,
-                    password: binary,
-                    null profile_pic: binary,
-                    null notes: list {
-                        type: string,
-                        unique: true,
-                    }
-                ) with {
-                    env: {
-                        free_user_limit: 100,
-                    },
-                    storage_driver: \"skyheap\"
-                }
-            ",
-        )
-        .unwrap();
-        let tok = &tok[2..];
-
-        // parse model
-        let model = parse_ast_node_full_with_space::<CreateModel>(tok, "apps").unwrap();
-
-        assert_eq!(
-            model,
-            CreateModel::new(
-                ("apps", "mymodel").into(),
-                vec![
-                    FieldSpec::new(
-                        Ident::from("username"),
-                        vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
-                        false,
-                        true
-                    ),
-                    FieldSpec::new(
-                        Ident::from("password"),
-                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
-                        false,
-                        false
-                    ),
-                    FieldSpec::new(
-                        Ident::from("profile_pic"),
-                        vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
-                        true,
-                        false
-                    ),
-                    FieldSpec::new(
-                        Ident::from("notes"),
-                        vec![
-                            LayerSpec::new(Ident::from("string"), null_dict! {}),
-                            LayerSpec::new(
-                                Ident::from("list"),
-                                null_dict! {
-                                    "unique" => Lit::new_bool(true)
-                                }
-                            )
-                        ],
-                        true,
-                        false
-                    )
-                ],
-                null_dict! {
-                    "env" => null_dict! {
-                        "free_user_limit" => Lit::new_uint(100),
-                    },
-                    "storage_driver" => Lit::new_string("skyheap".into()),
+        let mut ret = CreateModel::new(
+            ("apps", "mymodel").into(),
+            vec![
+                FieldSpec::new(
+                    Ident::from("username"),
+                    vec![LayerSpec::new(Ident::from("string"), null_dict! {})],
+                    false,
+                    true,
+                ),
+                FieldSpec::new(
+                    Ident::from("password"),
+                    vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
+                    false,
+                    false,
+                ),
+                FieldSpec::new(
+                    Ident::from("profile_pic"),
+                    vec![LayerSpec::new(Ident::from("binary"), null_dict! {})],
+                    true,
+                    false,
+                ),
+                FieldSpec::new(
+                    Ident::from("notes"),
+                    vec![
+                        LayerSpec::new(Ident::from("string"), null_dict! {}),
+                        LayerSpec::new(
+                            Ident::from("list"),
+                            null_dict! {
+                                "unique" => Lit::new_bool(true)
+                            },
+                        ),
+                    ],
+                    true,
+                    false,
+                ),
+            ],
+            null_dict! {
+                "env" => null_dict! {
+                    "free_user_limit" => Lit::new_uint(100),
+                },
+                "storage_driver" => Lit::new_string("skyheap".into()),
+            },
+            false,
+        );
+        ret.if_not_exists = true;
+        fullparse_verify_substmt_with_space(
+            "
+            create model if not exists mymodel(
+                primary username: string,
+                password: binary,
+                null profile_pic: binary,
+                null notes: list {
+                    type: string,
+                    unique: true,
                 }
-            )
-        )
+            ) with {
+                env: {
+                    free_user_limit: 100,
+                },
+                storage_driver: \"skyheap\"
+            }",
+            "apps",
+            |r: CreateModel| assert_eq!(r, ret),
+        );
     }
 }
 mod dict_field_syntax {
@@ -1119,7 +1103,12 @@ mod ddl_other_query_tests {
         let src = lex_insecure(br"drop space myspace").unwrap();
         assert_eq!(
             parse_ast_node_full::<DropSpace>(&src[2..]).unwrap(),
-            DropSpace::new(Ident::from("myspace"), false)
+            DropSpace::new(Ident::from("myspace"), false, false)
+        );
+        let src = lex_insecure(br"drop space if exists myspace").unwrap();
+        assert_eq!(
+            parse_ast_node_full::<DropSpace>(&src[2..]).unwrap(),
+            DropSpace::new(Ident::from("myspace"), false, true)
         );
     }
     #[test]
@@ -1127,7 +1116,12 @@ mod ddl_other_query_tests {
         let src = lex_insecure(br"drop space allow not empty myspace").unwrap();
         assert_eq!(
             parse_ast_node_full::<DropSpace>(&src[2..]).unwrap(),
-            DropSpace::new(Ident::from("myspace"), true)
+            DropSpace::new(Ident::from("myspace"), true, false)
+        );
+        let src = lex_insecure(br"drop space if exists allow not empty myspace").unwrap();
+        assert_eq!(
+            parse_ast_node_full::<DropSpace>(&src[2..]).unwrap(),
+            DropSpace::new(Ident::from("myspace"), true, true)
         );
     }
     #[test]
@@ -1135,7 +1129,12 @@ mod ddl_other_query_tests {
         let src = lex_insecure(br"drop model mymodel").unwrap();
         assert_eq!(
             parse_ast_node_full_with_space::<DropModel>(&src[2..], "apps").unwrap(),
-            DropModel::new(("apps", "mymodel").into(), false)
+            DropModel::new(("apps", "mymodel").into(), false, false)
+        );
+        let src = lex_insecure(br"drop model if exists mymodel").unwrap();
+        assert_eq!(
+            parse_ast_node_full_with_space::<DropModel>(&src[2..], "apps").unwrap(),
+            DropModel::new(("apps", "mymodel").into(), false, true)
         );
     }
     #[test]
@@ -1143,7 +1142,12 @@ mod ddl_other_query_tests {
         let src = lex_insecure(br"drop model allow not empty mymodel").unwrap();
         assert_eq!(
             parse_ast_node_full_with_space::<DropModel>(&src[2..], "apps").unwrap(),
-            DropModel::new(("apps", "mymodel").into(), true)
+            DropModel::new(("apps", "mymodel").into(), true, false)
+        );
+        let src = lex_insecure(br"drop model if exists allow not empty mymodel").unwrap();
+        assert_eq!(
+            parse_ast_node_full_with_space::<DropModel>(&src[2..], "apps").unwrap(),
+            DropModel::new(("apps", "mymodel").into(), true, true)
         );
     }
 }

From bbcb7acb95915008d5e636c80fe82ed8e930de65 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 5 Dec 2023 01:30:27 +0530
Subject: [PATCH 306/310] Add sysctl alter user

We will now also only output server logs to the terminal in CI, iff
there is an error. Otherwise, it plagues output.
---
 CHANGELOG.md                                  |  9 ++-
 harness/src/main.rs                           |  7 +-
 harness/src/test/mod.rs                       |  1 +
 harness/src/test/svc.rs                       | 77 +++++++++++++++++--
 server/src/engine/core/dcl.rs                 | 31 ++++++--
 server/src/engine/fractal/sys_store.rs        | 17 ++++
 server/src/engine/ql/dcl.rs                   | 61 +++++++--------
 server/src/engine/ql/tests/dcl.rs             | 26 +++++--
 .../engine/tests/client_misc/sec/dcl_sec.rs   |  7 +-
 sky-macros/src/dbtest.rs                      | 10 +--
 10 files changed, 175 insertions(+), 71 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fc77a7d0..93d1ac18 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,15 +14,15 @@ All changes in this project will be noted in this file.
   - `space`s are the equivalent of the `keyspace` from previous versions
   - `model`s are the equivalent of `table`s from previous version
   - The following queries were added:
-    - `CREATE SPACE ...`
-    - `CREATE MODEL ...`
+    - `CREATE SPACE [IF NOT EXISTS] ...`
+    - `CREATE MODEL [IF NOT EXISTS] ...`
       - Nested lists are now supported
       - Type definitions are now supported
       - Multiple fields are now supported
     - `ALTER SPACE ...`
     - `ALTER MODEL ...`
-    - `DROP SPACE ...`
-    - `DROP MODEL ...`
+    - `DROP SPACE [IF EXISTS] ...`
+    - `DROP MODEL [IF EXISTS] ...`
     - `USE <space>`:
       - works just like SQL
       - **does not work with DDL queries**: the reason it works in this way is to prevent accidental deletes
@@ -56,6 +56,7 @@ All changes in this project will be noted in this file.
     - `DELETE FROM <space>.<model> WHERE <primary_key_column> = <value>`
 - DCL:
   - `SYSCTL CREATE USER <name> WITH { password: <password> }`
+  - `SYSCTL ALTER USER <name> WITH { password: <new password> }`
   - `SYSCTL DROP USER <name>`
 
 #### Fractal engine
diff --git a/harness/src/main.rs b/harness/src/main.rs
index 70011fce..77e1087e 100644
--- a/harness/src/main.rs
+++ b/harness/src/main.rs
@@ -52,10 +52,13 @@ fn main() {
     Builder::new()
         .parse_filters(&env::var("SKYHARNESS_LOG").unwrap_or_else(|_| "info".to_owned()))
         .init();
-    // avoid verbose logging
-    env::set_var("SKY_LOG", "error");
+    env::set_var("SKY_LOG", "trace");
     if let Err(e) = runner() {
         error!("harness failed with: {}", e);
+        error!("fetching logs from server processes");
+        for ret in test::get_children() {
+            ret.print_logs();
+        }
         process::exit(0x01);
     }
 }
diff --git a/harness/src/test/mod.rs b/harness/src/test/mod.rs
index 2695e109..46c2a0f1 100644
--- a/harness/src/test/mod.rs
+++ b/harness/src/test/mod.rs
@@ -46,6 +46,7 @@ use {
     std::{fs, io::Write},
 };
 mod svc;
+pub use svc::get_children;
 
 /// Run the test suite
 pub fn run_test() -> HarnessResult<()> {
diff --git a/harness/src/test/svc.rs b/harness/src/test/svc.rs
index 1203f81a..55d5f57b 100644
--- a/harness/src/test/svc.rs
+++ b/harness/src/test/svc.rs
@@ -33,12 +33,73 @@ use {
         HarnessError, HarnessResult, ROOT_DIR,
     },
     std::{
+        cell::RefCell,
         io::ErrorKind,
         path::Path,
-        process::{Child, Command},
+        process::{Child, Command, Output, Stdio},
     },
 };
 
+thread_local! {
+    static CHILDREN: RefCell<Vec<(&'static str, Child)>> = RefCell::default();
+}
+
+pub struct ChildStatus {
+    id: &'static str,
+    stdout: String,
+    stderr: String,
+    exit_code: i32,
+}
+
+impl ChildStatus {
+    pub fn new(id: &'static str, stdout: String, stderr: String, exit_code: i32) -> Self {
+        Self {
+            id,
+            stdout,
+            stderr,
+            exit_code,
+        }
+    }
+    pub fn print_logs(&self) {
+        println!(
+            "######################### LOGS FROM {} #########################",
+            self.id
+        );
+        println!("-> exit code: `{}`", self.exit_code);
+        if !self.stdout.is_empty() {
+            println!("+++++++++++++++++++++ STDOUT +++++++++++++++++++++");
+            println!("{}", self.stdout);
+            println!("++++++++++++++++++++++++++++++++++++++++++++++++++");
+        }
+        if !self.stderr.is_empty() {
+            println!("+++++++++++++++++++++ STDERR +++++++++++++++++++++");
+            println!("{}", self.stderr);
+            println!("++++++++++++++++++++++++++++++++++++++++++++++++++");
+        }
+        println!("######################### ############ #########################");
+    }
+}
+
+pub fn get_children() -> Vec<ChildStatus> {
+    CHILDREN.with(|c| {
+        let mut ret = vec![];
+        for (name, child) in c.borrow_mut().drain(..) {
+            let Output {
+                status,
+                stdout,
+                stderr,
+            } = child.wait_with_output().unwrap();
+            ret.push(ChildStatus::new(
+                name,
+                String::from_utf8(stdout).unwrap(),
+                String::from_utf8(stderr).unwrap(),
+                status.code().unwrap(),
+            ))
+        }
+        ret
+    })
+}
+
 #[cfg(windows)]
 /// The powershell script hack to send CTRL+C using kernel32
 const POWERSHELL_SCRIPT: &str = include_str!("../../../ci/windows/stop.ps1");
@@ -69,6 +130,8 @@ pub fn get_run_server_cmd(server_id: &'static str, target_folder: impl AsRef<Pat
     ];
     let mut cmd = util::assemble_command_from_slice(&args);
     cmd.current_dir(server_id);
+    cmd.stdout(Stdio::null());
+    cmd.stderr(Stdio::piped());
     #[cfg(windows)]
     cmd.creation_flags(CREATE_NEW_CONSOLE);
     cmd
@@ -174,15 +237,15 @@ fn wait_for_shutdown() -> HarnessResult<()> {
 }
 
 /// Start the servers returning handles to the child processes
-fn start_servers(target_folder: impl AsRef<Path>) -> HarnessResult<Vec<Child>> {
-    let mut ret = Vec::with_capacity(SERVERS.len());
+fn start_servers(target_folder: impl AsRef<Path>) -> HarnessResult<()> {
     for (server_id, _ports) in SERVERS {
         let cmd = get_run_server_cmd(server_id, target_folder.as_ref());
         info!("Starting {server_id} ...");
-        ret.push(util::get_child(format!("start {server_id}"), cmd)?);
+        let child = util::get_child(format!("start {server_id}"), cmd)?;
+        CHILDREN.with(|c| c.borrow_mut().push((server_id, child)));
     }
     wait_for_startup()?;
-    Ok(ret)
+    Ok(())
 }
 
 pub(super) fn run_with_servers(
@@ -191,14 +254,12 @@ pub(super) fn run_with_servers(
     run_what: impl FnOnce() -> HarnessResult<()>,
 ) -> HarnessResult<()> {
     info!("Starting servers ...");
-    let children = start_servers(target_folder.as_ref())?;
+    start_servers(target_folder.as_ref())?;
     run_what()?;
     if kill_servers_when_done {
         kill_servers()?;
         wait_for_shutdown()?;
     }
-    // just use this to avoid ignoring the children vector
-    assert_eq!(children.len(), SERVERS.len());
     Ok(())
 }
 
diff --git a/server/src/engine/core/dcl.rs b/server/src/engine/core/dcl.rs
index cfb3356f..d3e3476e 100644
--- a/server/src/engine/core/dcl.rs
+++ b/server/src/engine/core/dcl.rs
@@ -29,7 +29,7 @@ use crate::engine::{
     error::{QueryError, QueryResult},
     fractal::GlobalInstanceLike,
     net::protocol::ClientLocalState,
-    ql::dcl::{SysctlCommand, UserAdd, UserDel},
+    ql::dcl::{SysctlCommand, UserDecl, UserDel},
 };
 
 const KEY_PASSWORD: &str = "password";
@@ -45,22 +45,41 @@ pub fn exec<G: GlobalInstanceLike>(
     match cmd {
         SysctlCommand::CreateUser(new) => create_user(&g, new),
         SysctlCommand::DropUser(drop) => drop_user(&g, current_user, drop),
+        SysctlCommand::AlterUser(usermod) => alter_user(&g, current_user, usermod),
         SysctlCommand::ReportStatus => Ok(()),
     }
 }
 
-fn create_user(global: &impl GlobalInstanceLike, mut user_add: UserAdd<'_>) -> QueryResult<()> {
-    let username = user_add.username().to_owned();
-    let password = match user_add.options_mut().remove(KEY_PASSWORD) {
+fn alter_user(
+    global: &impl GlobalInstanceLike,
+    cstate: &ClientLocalState,
+    user: UserDecl,
+) -> QueryResult<()> {
+    if cstate.is_root() {
+        // the root password can only be changed by shutting down the server
+        return Err(QueryError::SysAuthError);
+    }
+    let (username, password) = get_user_data(user)?;
+    global.sys_store().alter_user(username, password)
+}
+
+fn create_user(global: &impl GlobalInstanceLike, user: UserDecl) -> QueryResult<()> {
+    let (username, password) = get_user_data(user)?;
+    global.sys_store().create_new_user(username, password)
+}
+
+fn get_user_data(mut user: UserDecl) -> Result<(String, String), QueryError> {
+    let username = user.username().to_owned();
+    let password = match user.options_mut().remove(KEY_PASSWORD) {
         Some(DictEntryGeneric::Data(d))
-            if d.kind() == TagClass::Str && user_add.options().is_empty() =>
+            if d.kind() == TagClass::Str && user.options().is_empty() =>
         unsafe { d.into_str().unwrap_unchecked() },
         None | Some(_) => {
             // invalid properties
             return Err(QueryError::QExecDdlInvalidProperties);
         }
     };
-    global.sys_store().create_new_user(username, password)
+    Ok((username, password))
 }
 
 fn drop_user(
diff --git a/server/src/engine/fractal/sys_store.rs b/server/src/engine/fractal/sys_store.rs
index 5dd7ac44..f20820fa 100644
--- a/server/src/engine/fractal/sys_store.rs
+++ b/server/src/engine/fractal/sys_store.rs
@@ -183,6 +183,23 @@ impl<Fs: RawFSInterface> SystemStore<Fs> {
             Entry::Occupied(_) => Err(QueryError::SysAuthError),
         }
     }
+    pub fn alter_user(&self, username: String, password: String) -> QueryResult<()> {
+        let mut auth = self.system_store().auth_data().write();
+        match auth.users.get_mut(username.as_str()) {
+            Some(user) => {
+                let last_pass_hash = core::mem::replace(
+                    &mut user.key,
+                    rcrypt::hash(password, rcrypt::DEFAULT_COST)
+                        .unwrap()
+                        .into_boxed_slice(),
+                );
+                self._try_sync_or(&mut auth, |auth| {
+                    auth.users.get_mut(username.as_str()).unwrap().key = last_pass_hash;
+                })
+            }
+            None => Err(QueryError::SysAuthError),
+        }
+    }
     pub fn drop_user(&self, username: &str) -> QueryResult<()> {
         let mut auth = self.system_store().auth_data().write();
         if username == SysAuthUser::USER_ROOT {
diff --git a/server/src/engine/ql/dcl.rs b/server/src/engine/ql/dcl.rs
index b04737c3..a8ddcba2 100644
--- a/server/src/engine/ql/dcl.rs
+++ b/server/src/engine/ql/dcl.rs
@@ -25,23 +25,23 @@
 */
 
 use crate::engine::{
-    data::{
-        tag::{DataTag, TagClass},
-        DictGeneric,
-    },
+    data::DictGeneric,
     error::{QueryError, QueryResult},
     ql::{
         ast::{traits, QueryData, State},
         ddl::syn,
+        lex::Ident,
     },
 };
 
 #[derive(Debug, PartialEq)]
 pub enum SysctlCommand<'a> {
     /// `sysctl create user ...`
-    CreateUser(UserAdd<'a>),
+    CreateUser(UserDecl<'a>),
     /// `sysctl drop user ...`
     DropUser(UserDel<'a>),
+    /// `systcl alter user ...`
+    AlterUser(UserDecl<'a>),
     /// `sysctl status`
     ReportStatus,
 }
@@ -62,16 +62,19 @@ impl<'a> traits::ASTNode<'a> for SysctlCommand<'a> {
             return Err(QueryError::QLUnexpectedEndOfStatement);
         }
         let (a, b) = (state.fw_read(), state.fw_read());
+        let alter = Token![alter].eq(a) & b.ident_eq("user");
         let create = Token![create].eq(a) & b.ident_eq("user");
         let drop = Token![drop].eq(a) & b.ident_eq("user");
         let status = a.ident_eq("report") & b.ident_eq("status");
-        if !(create | drop | status) {
+        if !(create | drop | status | alter) {
             return Err(QueryError::QLUnknownStatement);
         }
         if create {
-            UserAdd::parse(state).map(SysctlCommand::CreateUser)
+            UserDecl::parse(state).map(SysctlCommand::CreateUser)
         } else if drop {
             UserDel::parse(state).map(SysctlCommand::DropUser)
+        } else if alter {
+            UserDecl::parse(state).map(SysctlCommand::AlterUser)
         } else {
             Ok(SysctlCommand::ReportStatus)
         }
@@ -89,7 +92,7 @@ fn parse<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<UserMe
     }
     let token_buffer = state.current();
     // initial sig
-    let signature_okay = token_buffer[0].is_lit()
+    let signature_okay = token_buffer[0].is_ident()
         & token_buffer[1].eq(&Token![with])
         & token_buffer[2].eq(&Token![open {}]);
     // get props
@@ -100,45 +103,38 @@ fn parse<'a, Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<UserMe
     };
     let maybe_username = unsafe {
         // UNSAFE(@ohsayan): the dict parse ensures state correctness
-        token_buffer[0].uck_read_lit()
+        token_buffer[0].uck_read_ident()
     };
-    state.poison_if_not(maybe_username.kind().tag_class() == TagClass::Str);
     if state.not_exhausted() | !state.okay() {
         // we shouldn't have more tokens
         return Err(QueryError::QLInvalidSyntax);
     }
     Ok(UserMeta {
-        username: unsafe {
-            // UNSAFE(@ohsayan): +tagck in state
-            maybe_username.str()
-        },
+        username: maybe_username,
         options: dict,
     })
 }
 
 struct UserMeta<'a> {
-    username: &'a str,
+    username: Ident<'a>,
     options: DictGeneric,
 }
 
 #[derive(Debug, PartialEq)]
-pub struct UserAdd<'a> {
-    username: &'a str,
+pub struct UserDecl<'a> {
+    username: Ident<'a>,
     options: DictGeneric,
 }
 
-impl<'a> UserAdd<'a> {
-    pub(in crate::engine::ql) fn new(username: &'a str, options: DictGeneric) -> Self {
+impl<'a> UserDecl<'a> {
+    pub(in crate::engine::ql) fn new(username: Ident<'a>, options: DictGeneric) -> Self {
         Self { username, options }
     }
-    /// Parse a `user add` DCL command
-    ///
-    /// MUSTENDSTREAM: YES
     pub fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
         parse(state).map(|UserMeta { username, options }: UserMeta| Self::new(username, options))
     }
     pub fn username(&self) -> &str {
-        self.username
+        self.username.as_str()
     }
     pub fn options_mut(&mut self) -> &mut DictGeneric {
         &mut self.options
@@ -150,33 +146,28 @@ impl<'a> UserAdd<'a> {
 
 #[derive(Debug, PartialEq)]
 pub struct UserDel<'a> {
-    username: &'a str,
+    username: Ident<'a>,
 }
 
 impl<'a> UserDel<'a> {
-    pub(in crate::engine::ql) fn new(username: &'a str) -> Self {
+    pub(in crate::engine::ql) fn new(username: Ident<'a>) -> Self {
         Self { username }
     }
     /// Parse a `user del` DCL command
     ///
     /// MUSTENDSTREAM: YES
     pub fn parse<Qd: QueryData<'a>>(state: &mut State<'a, Qd>) -> QueryResult<Self> {
-        if state.can_read_lit_rounded() & (state.remaining() == 1) {
-            let lit = unsafe {
+        if state.cursor_has_ident_rounded() & (state.remaining() == 1) {
+            let username = unsafe {
                 // UNSAFE(@ohsayan): +boundck
-                state.read_cursor_lit_unchecked()
+                state.read().uck_read_ident()
             };
             state.cursor_ahead();
-            if lit.kind().tag_class() == TagClass::Str {
-                return Ok(Self::new(unsafe {
-                    // UNSAFE(@ohsayan): +tagck
-                    lit.str()
-                }));
-            }
+            return Ok(Self::new(username));
         }
         Err(QueryError::QLInvalidSyntax)
     }
     pub fn username(&self) -> &str {
-        self.username
+        self.username.as_str()
     }
 }
diff --git a/server/src/engine/ql/tests/dcl.rs b/server/src/engine/ql/tests/dcl.rs
index 44596d99..d7c8a169 100644
--- a/server/src/engine/ql/tests/dcl.rs
+++ b/server/src/engine/ql/tests/dcl.rs
@@ -39,12 +39,25 @@ fn report_status_simple() {
 
 #[test]
 fn create_user_simple() {
-    let query = lex_insecure(b"sysctl create user 'sayan' with { password: 'mypass123' }").unwrap();
+    let query = lex_insecure(b"sysctl create user sayan with { password: 'mypass123' }").unwrap();
     let q = ast::parse_ast_node_full::<dcl::SysctlCommand>(&query[1..]).unwrap();
     assert_eq!(
         q,
-        SysctlCommand::CreateUser(dcl::UserAdd::new(
-            "sayan",
+        SysctlCommand::CreateUser(dcl::UserDecl::new(
+            "sayan".into(),
+            into_dict!("password" => lit!("mypass123"))
+        ))
+    )
+}
+
+#[test]
+fn alter_user_simple() {
+    let query = lex_insecure(b"sysctl alter user sayan with { password: 'mypass123' }").unwrap();
+    let q = ast::parse_ast_node_full::<dcl::SysctlCommand>(&query[1..]).unwrap();
+    assert_eq!(
+        q,
+        SysctlCommand::AlterUser(dcl::UserDecl::new(
+            "sayan".into(),
             into_dict!("password" => lit!("mypass123"))
         ))
     )
@@ -52,7 +65,10 @@ fn create_user_simple() {
 
 #[test]
 fn delete_user_simple() {
-    let query = lex_insecure(b"sysctl drop user 'monster'").unwrap();
+    let query = lex_insecure(b"sysctl drop user monster").unwrap();
     let q = ast::parse_ast_node_full::<dcl::SysctlCommand>(&query[1..]).unwrap();
-    assert_eq!(q, SysctlCommand::DropUser(dcl::UserDel::new("monster")));
+    assert_eq!(
+        q,
+        SysctlCommand::DropUser(dcl::UserDel::new("monster".into()))
+    );
 }
diff --git a/server/src/engine/tests/client_misc/sec/dcl_sec.rs b/server/src/engine/tests/client_misc/sec/dcl_sec.rs
index 9db90b34..961203a3 100644
--- a/server/src/engine/tests/client_misc/sec/dcl_sec.rs
+++ b/server/src/engine/tests/client_misc/sec/dcl_sec.rs
@@ -57,12 +57,9 @@ fn ensure_sysctl_status_end_of_tokens() {
 #[dbtest]
 fn ensure_sysctl_create_user() {
     let mut db = db!();
+    let query = format!("sysctl create user myuser with {{ password: ? }} blah");
     assert_err_eq!(
-        db.query_parse::<()>(&query!(
-            "sysctl create user ? with { password: ? } blah",
-            "myuser",
-            "mypass"
-        )),
+        db.query_parse::<()>(&query!(query, "mypass")),
         Error::ServerError(INVALID_SYNTAX_ERR)
     );
 }
diff --git a/sky-macros/src/dbtest.rs b/sky-macros/src/dbtest.rs
index 6f81f23b..52c882a7 100644
--- a/sky-macros/src/dbtest.rs
+++ b/sky-macros/src/dbtest.rs
@@ -230,10 +230,9 @@ pub fn dbtest(attrs: TokenStream, item: TokenStream) -> TokenStream {
                 /// password set by [`sky_macros::dbtest`] (relogin)
                 const __DBTEST_PASS: &str = #new_password;
                 {
+                    let query_assembled = format!("sysctl create user {} with {{ password: ? }}", #new_username);
                     let mut db = skytable::Config::new(#host, #port, #login_username, #login_password).connect().unwrap();
-                    db.query_parse::<()>(
-                        &skytable::query!("sysctl create user ? with { password: ? }", #new_username, #new_password)
-                    ).unwrap();
+                    db.query_parse::<()>(&skytable::query!(query_assembled, #new_password)).unwrap();
                 }
             };
         }
@@ -272,10 +271,9 @@ pub fn dbtest(attrs: TokenStream, item: TokenStream) -> TokenStream {
             ret_block = quote! {
                 #ret_block
                 {
+                    let query_assembled = format!("sysctl drop user {}", #new_username);
                     let mut db = skytable::Config::new(#host, #port, #login_username, #login_password).connect().unwrap();
-                    db.query_parse::<()>(
-                        &skytable::query!("sysctl drop user ?", #new_username)
-                    ).unwrap();
+                    db.query_parse::<()>(&skytable::query!(query_assembled)).unwrap();
                 }
             };
         }

From 8c4009f2b817a01f49dee63c4dd5e268c64f78c5 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Tue, 5 Dec 2023 16:52:01 +0530
Subject: [PATCH 307/310] Update docs and fix dpkg scripts [skip ci]

---
 .github/workflows/deploy-docs.yml      |  34 ---
 .github/workflows/docker-image.yml     |   2 +-
 ACKNOWLEDGEMENTS.txt                   |  18 ++
 CONTRIBUTING.md                        |   9 +-
 README.md                              | 149 ++++++----
 actiondoc.yml                          | 380 -------------------------
 assets/logo.jpg                        | Bin 42591 -> 184696 bytes
 examples/config-files/dpkg/config.yaml |  14 +
 harness/src/linuxpkg.rs                |  12 +
 pkg/common/skyd.service                |   2 +-
 pkg/debian/postinst                    |  30 +-
 pkg/debian/postrm                      |  27 ++
 pkg/debian/preinst                     |  29 +-
 pkg/debian/prerm                       |   6 +
 server/Cargo.toml                      |  13 +-
 server/src/engine/idx/mtchm/mod.rs     |   2 +-
 16 files changed, 228 insertions(+), 499 deletions(-)
 delete mode 100644 .github/workflows/deploy-docs.yml
 create mode 100644 ACKNOWLEDGEMENTS.txt
 delete mode 100644 actiondoc.yml
 mode change 100755 => 100644 assets/logo.jpg
 create mode 100644 examples/config-files/dpkg/config.yaml
 mode change 100644 => 100755 pkg/debian/postinst
 create mode 100755 pkg/debian/postrm
 create mode 100755 pkg/debian/prerm

diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml
deleted file mode 100644
index c2316934..00000000
--- a/.github/workflows/deploy-docs.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-name: Rebuild docs
-
-on:
-  push:
-    branches:
-      - next
-env:
-  IS_ACTIONS_DOC: "false"
-
-jobs:
-  rebuild-docs:
-    name: Build new actiondoc
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout source code
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 2
-
-      - name: Setup environment
-        run: |
-          chmod +x ci/setvars.sh
-          ci/setvars.sh
-
-      - name: Publish docs
-        env:
-          BOT_USER: ${{ secrets.BOT_INIT_USER }}
-          BOT_MAIL: ${{ secrets.BOT_INIT_MAIL }}
-          BOT_API: ${{ secrets.BOT_API_CALL }}
-          GIT_SHA: ${{ env.GITHUB_SHA }}
-        run: |
-          chmod +x ci/doc.sh
-          ci/doc.sh
-        if: env.IS_ACTIONS_DOC == 'true'
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
index f74c86d3..8249413f 100644
--- a/.github/workflows/docker-image.yml
+++ b/.github/workflows/docker-image.yml
@@ -11,7 +11,7 @@ on:
       - v*
 
 env:
-  IMAGE_NAME: sdb
+  IMAGE_NAME: skytable
   BUILD: "false"
 
 jobs:
diff --git a/ACKNOWLEDGEMENTS.txt b/ACKNOWLEDGEMENTS.txt
new file mode 100644
index 00000000..200517e9
--- /dev/null
+++ b/ACKNOWLEDGEMENTS.txt
@@ -0,0 +1,18 @@
+----
+Notes for v0.8.0
+----
+
+I'd like to thank a lot of the people that have very indirectly had some influence on Skytable's design. Here's a little list (in no
+particular order):
+
+- First of all I'd like to thank Raymond F. Boyce and Donald D. Chamberlin for their work on SQL. While I'm not fortunate enough to have
+any connection to them, a lot of their work have laid out guiding principles for my work.
+- I'd also like to thank several people from the Rust community (listed in no particular order):
+    - Aaron Turon: Aaron's work on concurrency libraries have greatly helped in parts of "design thinking"
+    - Carl Lerche (@carllerche): For the immense amount of work Carl done on Tokio and related systems.
+    - Michael Vaner (@vorner): For their work on designing concurrency primitives. I'm a great admirer of Michael's work but
+      unfortunately haven't had the opportunity to directly talk.
+    - Amanieu d'Antras (@Amanieu): Amanieu's work on parking_lot and hashbrown have been eye openers for several things that I've designed
+      and implemented, both in and out of Skytable
+
+-- Sayan N. (Dec, 2023 <ohsayan@outlook.com>)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7dab9166..480bc4e4 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -37,13 +37,6 @@ The main parts (ignorning CI scripts, stress test suite, test harness and custom
 - `cli`: REPL shell
 - `server`: database server
 - `sky-bench`: benchmark tool
-- `sky-migrate`: migration tool
-
-### Jargon
-
-Each project has its own jargon — and so do we!
-
-- _actiondoc_ and _actions docs_ : This refers to the `actiondoc.yml` file, which is used by the Skytable documentation website for automatically building documentation for the actions
 
 ### Branches
 
@@ -103,4 +96,4 @@ Testing is simple: just run this:
 make test
 ```
 
-> **NOTE**: Make sure port 2003 and 2004 are not used by any applications. Also, make sure your _own instance_ isn't running on any of these ports; if that is the case, you might end up losing data due to conflicting entity names! The test suite creates a `testsuite` keyspace and some tables within it to run all the tests.
+> **NOTE**: Make sure port 2003 and 2004 are not used by any applications. Also, make sure your _own instance_ isn't running on any of these ports; if that is the case, you might end up losing data due to conflicting entity names! The test suite creates multiple spaces and some models within it to run all the tests.
diff --git a/README.md b/README.md
index 9bcd5571..095b6755 100644
--- a/README.md
+++ b/README.md
@@ -1,82 +1,125 @@
-<html>
-<div align="center">
-<img src="assets/logo.jpg" height=64 width=64>
-<h1>Skytable</h1><h3>Your next NoSQL database</h3>
+<br/><p align="center">
+    <img width="150" src="assets/logo.jpg">
+</p>
+<h2 align = "center">
+    Skytable — A modern database for building powerful experiences
+</h2>
+<h3 align="center">
+    Performance, scalability and flexibility. Choose three.
+</h3>
+</p>
+<p align="center">
+<a href="https://github.com/skytable/skytable/releases"><img src="https://img.shields.io/github/v/release/skytable/skytable?style=flat" alt="GitHub release (with filter)"></a> <a href="https://github.com/skytable/skytable/actions"><img src="https://img.shields.io/github/actions/workflow/status/skytable/skytable/test-push.yml?style=flat" alt="GitHub Workflow Status (with event)"></a> <a href="https://discord.gg/QptWFdx"><img src="https://img.shields.io/discord/729378001023926282?logo=discord&style=flat" alt="Discord"></a> <a href="https://docs.skytable.io"><img src="https://img.shields.io/badge/read%20the%20docs-here-blue?style=flat" alt="Docs"></a> <a href="https://github.com/skytable/skytable/discussions?style=flat"><img src="https://img.shields.io/badge/discuss-here-8A3324?style=flat&logo=github&labelColor=C34723" alt="Static Badge"></a>
+</p>
 
-![GitHub Workflow Status](<https://img.shields.io/github/workflow/status/skybasedb/skybase/Test%20(push)>) ![Development](https://img.shields.io/badge/development-regular-32CD32?style=flat-square) ![GitHub release (latest SemVer including pre-releases)](https://img.shields.io/github/v/release/skybasedb/skybase?include_prereleases&sort=semver&style=flat-square)
-[![Docs](https://img.shields.io/badge/readthedocs-here-blueviolet?style=flat-square)](https://docs.skytable.io) [![Contribute Now](https://img.shields.io/badge/%F0%9F%8C%9Fcontribute-now-a94064)](https://ohsayan.github.io/skythanks) [![Discord](https://img.shields.io/badge/talk-on%20discord-7289DA?logo=discord&style=flat-square")](https://discord.gg/QptWFdx)
+## 💡 What is Skytable?
 
-</div>
-</html>
+Skytable is a **modern NoSQL database** that focuses on **performance, flexibility and scalability**. Our goal is to deliver **a rock-solid database** that you can use as the foundation for your next application — **minus the gimmicks**.
 
-## What is Skytable?
+Skytable makes every attempt to **remove the gotchas from SQL-systems**. For example, nonempty `model`s and `space`s cannot be dropped and **BlueQL is designed to greatly deter SQL injection vulnerabilities** with a new **mandatory parameterization** design and several other **secure query language design principles**.
 
-Skytable is a free and open-source NoSQL database that aims to provide flexible data modeling at
-scale. For us simplicity, performance and flexibility are our guiding design principles.
-We were previously known as TerrabaseDB or Skybase and are nicknamed Sky, SDB or STable by the community.
+Every component in Skytable has been **engineered from the ground up** to meet our design goals. **Skytable uses BlueQL<sup>TM</sup>** which is our own **new in-house query language** designed from the ground up for a **clean, powerful, modern and secure querying experience** and is **generally more secure than SQL**.
 
-Features like keyspaces, tables, data types, authn+authz, snapshots and more are ready for you to use while we're working on [several new data models and features](https://github.com/skytable/skytable/issues/203). Skytable's key/value store is performant, secure and ready for you to deploy.
+Skytable works with **structured and semi-structured data**. We're currently working on supporting unstructured data.
 
-## Getting started 🚀
+> **You can read more about Skytable's architecture, including information on the clustering and HA implementation that we're currently working on, and limitations [on this page](https://docs.skytable.io/architecture).**
 
-1. Download a bundle for your platform from [here ⬇️ ](https://github.com/skytable/skytable/releases)
-2. Unzip the bundle
-3. Make the files executable (run `chmod +x skyd skysh` on \*nix systems)
-4. First run `skyd` to start the database server and then run `skysh` to start the interactive shell
-5. Run commands like: `SET foo bar` , `GET bar` , `UPDATE cat mitten` or `DEL proprietary` on `skysh`!
+## 🎨 Features
 
-You can learn more about installation [here](https://docs.skytable.io/getting-started)
+- **Spaces, models and more**: For flexible data definition
+- **Powerful querying with BlueQL**: A modern query language, designed for the 21<sup>st</sup> century
+- **Rich data modeling**: Use `model`s to define data with complex types, collections and more
+- **Performant, in and out of the box**: Heavily multithreaded and optimized
+- **Secure, query in and response out**: BlueQL is designed to strongly deter query injection pathways
+- **SQL minus the gotchas**: Ever done a `DROP TABLE users` and lost all data? **That won't happen in Skytable**.
+- **Designed to scale by enforcing best practices**: If you're building with Skytable today, the practices you'll learn here will let you easily take on the job of building large scale systems
 
-## Features
+> Learn more about [Skytable's features here](https://docs.skytable.io).
 
-- **Insanely fast**: Scale to millions of queries per second per node. See [benchmarks here](https://github.com/ohsayan/sky-benches).
-- **Multiple keyspaces/tables**: Seamlessly integrates with actions to provide a SQL-like experience
-- **Key/value store**: `GET` , `SET` , `UPDATE` and [all that stuff](https://docs.skytable.io/all-actions). With the `str` and `binstr` types.
-- **Authn/Authz**: Simple and secure authentication/authorization
-- **Volatile tables**: For all the caching you need
-- **Snapshots**: Automated (and tunable) snapshots for stress-free backups
-- **Secure**: Secure connections are built into Skytable with SSL/TLS
-- **Multithreaded**: Designed to exploit all CPU cores
-- **Resource friendly**: The database server doesn't need more than 1MB to run
-- **Convenient**: Without the setup hassle and system-specific dependencies
+## 🚀 Getting started
 
-**🛣️ There's a lot more coming! View our [roadmap](https://github.com/skytable/skytable/issues/203)**
+1. **Set up Skytable on your machine**: You'll need to download a bundled release file [from the releases page](https://github.com/skytable/skytable/releases). Unzip the files and you're ready to go.
+2. Start the database server: `./skyd --auth-root-password <password>` with your choice of a password for the `root` account. The `root` account is just like a `root` account on Unix based systems that has control over everything.
+3. Start the interactive client REPL: `./skysh` and then enter your password.
+4. You're ready to run queries!
 
-## Clients 🔌
+> **For a more detailed guide on installation and deployment, [follow the guide here.](https://docs.skytable.io/installation)**
 
-The project currently maintains an official [Rust driver](https://github.com/skytable/client-rust) and we have plans
-to support more languages along the way!
-We also maintain a list of [community supported drivers here](https://github.com/skytable/skytable/wiki/Drivers).
 
-If you want to use a different language, for now you'll just need to implement the simple and performant [Skyhash Protocol](https://docs.skytable.io/protocol/skyhash).
+## ⚡ Using Skytable
 
-## Community 👐
+Skytable has `SPACE`s instead of `DATABASE`s due to signficant operational differences (and because `SPACE`s store a lot more than tabular data). 
 
-A project which is powered by the community believes in the power of community! If you get stuck anywhere - here are your options!
+**With the REPL started, follow this guide**:
 
-<html>
-<a href="https://gitter.im/skytable/community"><img src="https://img.shields.io/badge/chat%20on-gitter-ed1965?logo=gitter&style=flat-square"></img>
-</a><a href="https://discord.gg/QptWFdx"><img src="https://img.shields.io/badge/talk-on%20discord-7289DA?logo=discord&style=flat-square"></img></a>
-</html>
+1. Create a `space` and switch to it:
+   ```sql
+   CREATE SPACE myspace
+   USE myspace
+   ```
+2. Create a `model`:
+   ```sql
+   CREATE MODEL myspace.mymodel(username: string, password: string, notes: list { type: string })
+   ```
+   The rough representation for this in Rust would be:
+   ```rust
+   pub struct MyModel {
+    username: String,
+    password: Strin,
+    notes: Vec<String>,
+   }
+   ```
+3. `INSERT` some data:
+   ```sql
+   INSERT INTO mymodel('sayan', 'pass123', [])
+   ```
+4. `UPDATE` some data:
+   ```sql
+   UPDATE mymodel SET notes += "my first note" WHERE username = 'sayan'
+   ```
+5. `SELECT` some data
+   ```sql
+   SELECT * FROM mymodel WHERE username = 'sayan'
+   ```
+6. Poke around! **And then make sure you [read the documentation learn BlueQL](https://docs.skytable.io/blueql/overview).**
 
-## Platforms 💻
+> **For a complete guide on Skytable, it's architecture, BlueQL, queries and more we strongly recommend you to [read the documentation here.](https://docs.skytable.io)**
+>
+> While you're seeing strings and other values being used here, this is so because the REPL client smartly parameterizes queries behind the scenes. **BlueQL has mandatory parameterization**. (See below to see how the Rust client handles this)
 
-![Linux supported](https://img.shields.io/badge/Linux%2032--bit%2F64--bit-Supported%20✓-%23228B22?logo=linux) ![macOS supported](https://img.shields.io/badge/macOS%20x86__64%2Farm64-supported%20✓-228B22?style=flat-square&logo=apple) ![Windows supported](https://img.shields.io/badge/Windows%2032--bit%2F64--bit-supported%20✓-228B22?style=flat-square&logo=windows)
+## 🧩 Find a client driver
 
-## Versioning
+You need a client driver to use Skytable in your programs. Officially, we maintain a regularly updated [Rust client driver](https://github.com/skytable/client-rust) which is liberally license under the Apache-2.0 license so that you can use it anywhere.
 
-This project strictly follows semver, however, since this project is currently in the development phase (0.x.y), the API may change unpredictably
+Using the Rust client driver, it's very straightforward to run queries thanks to Rust's powerful type system and macros:
 
-## Contributing
+```rust
+use skytable::{Config, query};
+
+fn main() {
+    let mut db = Config::new_default("username", "password").connect().unwrap();
+    let query = query!("select username, password from myspace.mymodel where username = ?", "sayan");
+    let (username, password): (String, Vec<u8>) = db.query_parse(&query).unwrap();
+    // do something with it!
+}
+```
+
+> **You can find more information on client drivers on [this page](https://docs.skytable.io/libraries). If you want to help write a client driver for your language of choice, *we're here to support your work*. Please reach out to: hey@skytable.io or leave a message on our Discord server!**
+
+## 🙋 Getting help
 
-[![Contribute Now](https://img.shields.io/badge/%F0%9F%8C%9Fcontribute-now-a94064?style=for-the-badge)](https://ohsayan.github.io/skythanks)
+We exclusively use [Discord](https://discord.gg/QptWFdx) for most real-time communications — you can chat with developers, maintainers, and our amazing users! Outside that, we recommend that you use our [GitHub Discussions page](https://github.com/skytable/skytable/discussions) for any questions or open a new issue if you think you've found a bug.
+
+*We're here to help!*
+
+## Contributing
 
-You are welcome to contribute to Skytable! Beginner friendly issues are marked with the [<img src=https://img.shields.io/badge/L--easy-C71585>](https://github.com/skytable/skytable/labels/L-easy) label. Read the guide [here](./CONTRIBUTING.md).
+Please read the [contributing guide here](./CONTRIBUTING.md).
 
-## Contributors
+## Acknowledgements
 
-You can see a full list of contributors [here](https://ohsayan.github.io/skythanks)
+Please read the [acknowledgements](./ACKNOWLEDGEMENTS.txt) document.
 
 ## License
 
-This project is licensed under the [AGPL-3.0 License](./LICENSE).
+Skytable is distributed under the [AGPL-3.0 License](./LICENSE). **You may not use Skytable's logo for other projects.**
diff --git a/actiondoc.yml b/actiondoc.yml
deleted file mode 100644
index f12a48be..00000000
--- a/actiondoc.yml
+++ /dev/null
@@ -1,380 +0,0 @@
-#
-#  Created on Thu Aug 27 2020
-#
-#  This file is a part of Skytable
-#  Skytable (formerly known as TerrabaseDB or Skybase) is a free and open-source
-#  NoSQL database written by Sayan Nandan ("the Author") with the
-#  vision to provide flexibility in data modelling without compromising
-#  on performance, queryability or scalability.
-#
-#  Copyright (c) 2020, Sayan Nandan <ohsayan@outlook.com>
-#
-#  This program is free software: you can redistribute it and/or modify
-#  it under the terms of the GNU Affero General Public License as published by
-#  the Free Software Foundation, either version 3 of the License, or
-#  (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-#  GNU Affero General Public License for more details.
-#
-#  You should have received a copy of the GNU Affero General Public License
-#  along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-#
-#
-#  This file is used by Skytable's documentation website for automatically
-#  generating documentation for the actions. It will also be used by the Skytable
-#  server in the future
-
-# the docbuilder expects:
-# 'name': str, 'complexity': str, 'accept': [str]
-# 'return': [str], 'syntax': [str], 'desc': str
-
-global:
-  - name: HEYA
-    complexity: O(1)
-    accept: [AnyArray]
-    syntax: [HEYA, HEYA <message>]
-    desc: |
-      Either returns a "HEY!" or returns the provided argument as an `str`
-    return: [String]
-  - name: DBSIZE
-    complexity: O(1)
-    accept: [AnyArray]
-    syntax: [DBSIZE, DBSIZE <entity>]
-    desc: Check the number of entries stored in the current table or in the provided entity
-    return: [Integer]
-  - name: MKSNAP
-    complexity: O(n)
-    accept: [AnyArray]
-    syntax: [MKSNAP, MKSNAP <SNAPNAME>]
-    desc: |
-      This action can be used to create a snapshot. Do note that this action **requires
-      snapshotting to be enabled on the server side**, before it can create snapshots.
-      If you want to create snapshots **without** snapshots being enabled on the server-side,
-      pass a second argument `<SNAPNAME>` to specify a snapshot name and a snapshot will
-      be create in a folder called `rsnap` under your data directory. For more
-      information on snapshots, read [this document](/snapshots)
-    return: [Rcode 0, err-snapshot-disabled, err-snapshot-busy]
-  - name: FLUSHDB
-    complexity: O(n)
-    accept: [AnyArray]
-    syntax: [FLUSHDB, FLUSHDB <entity>]
-    desc: Removes all entries stored in the current table or in the provided entity
-    return: [Rcode 0, Rcode 5]
-  - name: WHEREAMI
-    complexity: O(1)
-    accept: [AnyArray]
-    syntax: [WHEREAMI]
-    desc: |
-      Returns an array with either the name of the current keyspace as the first element or if a default table
-      is set, then it returns the keyspace name as the first element and the table name as the second element
-    return: [Non-null array]
-  - name: AUTH
-    desc: Change global authn/authz settings
-    subactions:
-      - name: LOGIN
-        complexity: O(1)
-        accept: [AnyArray]
-        syntax: [AUTH LOGIN <username> <token>]
-        desc: Attempts to log in using the provided credentials
-        return: [Rcode 0, Rcode 10]
-      - name: CLAIM
-        complexity: O(1)
-        accept: [AnyArray]
-        syntax: [AUTH CLAIM <origin key>]
-        desc: Attempts to claim the root account using the origin key
-        return: [String, Rcode 10]
-      - name: LOGOUT
-        complexity: O(1)
-        accept: [AnyArray]
-        syntax: [AUTH LOGOUT]
-        desc: Attempts to log out the currently logged in user
-        return: [Rcode 0, Rcode 10]
-      - name: ADDUSER
-        complexity: O(1)
-        accept: [AnyArray]
-        syntax: [AUTH ADDUSER <username>]
-        desc: Attempts to create a new user with the provided username, returning the token
-        return: [String, Rcode 11]
-      - name: DELUSER
-        complexity: O(1)
-        accept: [AnyArray]
-        syntax: [AUTH DELUSER <username>]
-        desc: Attempts to delete the user with the provided username
-        return: [Rcode 0, Rcode 10, Rcode 11]
-      - name: RESTORE
-        complexity: O(1)
-        accept: [AnyArray]
-        syntax: [AUTH RESTORE <username>, AUTH RESTORE <origin-key> <username>]
-        desc: |
-          Attempts to restore the password for the provided user. This will regenerate the token
-          and return the newly issued token. However, if you aren't a root account, that is, you
-          lost your root password, then you'll need to run `AUTH RESTORE <origin-key> root`.
-        return: [String, Rcode 10, Rcode 11]
-      - name: LISTUSER
-        complexity: O(1)
-        accept: [AnyArray]
-        syntax: [AUTH LISTUSER]
-        desc: |
-          Attempts to return a list of users for the current database instance
-        return: [Non-null array]
-      - name: WHOAMI
-        complexity: O(1)
-        accept: [AnyArray]
-        syntax: [AUTH WHOAMI]
-        desc: |
-          Returns a string with the AuthID of the currently logged in user or errors if the user
-          is not logged in
-        return: [String]
-  - name: SYS
-    desc: |
-      Get system information and metrics
-    subactions:
-      - name: INFO
-        complexity: O(1)
-        accept: [AnyArray]
-        syntax: [sys info <property>]
-        return: [String, Float]
-        desc: |
-          Returns static properties of the system, i.e properties that do not change during runtime.
-          The following properties are available:
-            - `version`: Returns the server version (String)
-            - `protocol`: Returns the protocol version string (String)
-            - `protover`: Returns the protocol version (float)
-      - name: METRIC
-        complexity: O(1)
-        accept: [AnyArray]
-        syntax: [sys metric <metric>]
-        return: [String, Float]
-        desc: |
-          Returns dynamic properties of the system, i.e metrics are properties that can change during
-          runtime. The following metrics are available:
-            - `health`: Returns "good" or "critical" depending on the system state (String)
-            - `storage`: Returns bytes used for on-disk storage (uint64)
-
-keyvalue:
-  generic:
-    - name: DEL
-      complexity: O(n)
-      accept: [AnyArray]
-      syntax: [DEL <key1> <key2> ...]
-      desc: |
-        Delete 'n' keys from the current table. This will return the number of keys that were deleted
-        as an unsigned integer
-      return: [Integer, Rcode 5]
-    - name: EXISTS
-      complexity: O(n)
-      accept: [AnyArray]
-      syntax: [EXISTS <key1> <key2> ...]
-      desc: |
-        Check if 'n' keys exist in the current table. This will return the number of keys that exist
-        as an unsigned integer.
-      return: [Integer]
-    - name: LSKEYS
-      complexity: O(n)
-      accept: [AnyArray]
-      syntax: [LSKEYS <limit>, LSKEYS <entity>, LSKEYS <entity> <limit>]
-      desc: |
-        Returns a flat string array of keys present in the current table or in the provided entity.
-        If no `<limit>` is given, then a maximum of 10 keys are returned. If a limit is specified,
-        then a maximum of `<limit>` keys are returned. The order of keys is meaningless.
-      return: [Typed Array]
-  string:
-    - name: GET
-      complexity: O(1)
-      accept: [AnyArray]
-      syntax: [GET <key>]
-      desc: Get the value of a key from the current table, if it exists
-      return: [Rcode 1, String, Binstr]
-    - name: MGET
-      complexity: O(n)
-      accept: [AnyArray]
-      syntax: [MGET <key1> <key2> ...]
-      desc: Get the value of 'n' keys from the current table, if they exist
-      return: [Typed Array]
-    - name: SET
-      complexity: O(1)
-      accept: [AnyArray]
-      syntax: [SET <key> <value>]
-      desc: Set the value of a key in the current table, if it doesn't already exist
-      return: [Rcode 0, Rcode 2, Rcode 5]
-    - name: MSET
-      complexity: O(n)
-      accept: [AnyArray]
-      syntax: [MSET <key1> <value1> <key2> <value2> ...]
-      desc: |
-        Set the value of 'n' keys in the current table, if they don't already exist. This will
-        return the number of keys that were set as an unsigned integer.
-      return: [Integer, Rcode 5]
-    - name: UPDATE
-      complexity: O(1)
-      accept: [AnyArray]
-      syntax: [UPDATE <key> <value>]
-      desc: Update the value of an existing key in the current table
-      return: [Rcode 0, Rcode 1, Rcode 5]
-    - name: MUPDATE
-      complexity: O(n)
-      accept: [AnyArray]
-      syntax: [MUPDATE <key1> <value1> <key2> <value2> ...]
-      desc: |
-        Update the value of 'n' keys in the current table, if they already exist. This will return
-        the number of keys that were updated as an unsigned integer.
-      return: [Integer, Rcode 5]
-    - name: SSET
-      complexity: O(n)
-      accept: [AnyArray]
-      syntax: [SSET <key1> <value1> <key2> <value2> ...]
-      desc: Set all keys to the given values only if all of them don't exist in the current table
-      return: [Rcode 0, Rcode 2, Rcode 5]
-    - name: SDEL
-      complexity: O(n)
-      accept: [AnyArray]
-      syntax: [SDEL <key1> <key2> ...]
-      desc: |
-        Delete all keys if all of the keys exist in the current table. Do note that if a single key doesn't
-        exist, then a `Nil` code is returned.
-      return: [Rcode 0, Rcode 1, Rcode 5]
-    - name: SUPDATE
-      complexity: O(n)
-      accept: [AnyArray]
-      syntax: [SUPDATE <key1> <value1> <key2> <value2> ...]
-      desc: |
-        Update all keys if all of the keys exist in the current table. Do note that if a single key doesn't
-        exist, then a `Nil` code is returned.
-      return: [Rcode 0, Rcode 1, Rcode 5]
-    - name: USET
-      complexity: O(n)
-      accept: [AnyArray]
-      syntax: [USET <key1> <value1> <key2> <value2> ...]
-      desc: SET all keys if they don't exist, or UPDATE them if they do exist. This operation performs `USET`s in the current table
-      return: [Integer, Rcode 5]
-    - name: KEYLEN
-      complexity: O(1)
-      accept: [AnyArray]
-      syntax: [KEYLEN <key>]
-      desc: Returns the length of the UTF-8 string, if it exists in the current table
-      return: [Integer, Rcode 1]
-    - name: POP
-      complexity: O(1)
-      accept: [AnyArray]
-      syntax: [POP <key>]
-      desc: |
-        Deletes and return the value of the provided key from the current table.
-        If the database is poisoned, this will return a server error.
-      return: [String, Binstr, Rcode 5]
-    - name: MPOP
-      complexity: O(n)
-      accept: [AnyArray]
-      syntax: [MPOP <key1> <key2> ...]
-      desc: |
-        Deletes and returns the values of the provided 'n' keys from the current table.
-        If the database is poisoned, this will return a server error
-      return: [Typed Array, Rcode 5]
-  lists:
-    - name: LGET
-      desc: |
-        `LGET` can be used to access the items in a list. Through the sub-actions provided by `lget`,
-         you can access multiple or individual elements in lists.
-      subactions:
-        - name: LGET
-          complexity: O(n)
-          accept: [AnyArray]
-          syntax: [LGET <list>]
-          desc: |
-            Returns all the values contained in a the provided list, if it exists in the current
-            table.
-          return: [Typed Array, Rcode 1]
-        - name: limit
-          complexity: O(n)
-          accept: [AnyArray]
-          syntax: [LGET <list> limit <limit>]
-          desc: Returns a maximum of `limit` values from the provided list, if it exists in the current table
-          return: [Typed Array, Rcode 1]
-        - name: len
-          complexity: O(1)
-          accept: [AnyArray]
-          syntax: [LGET <list> len]
-          desc: Returns the length of the list
-          return: [Integer, Rcode 1]
-        - name: valueat
-          complexity: O(1)
-          accept: [AnyArray]
-          syntax: [LGET <list> valueat <index>]
-          desc: Returns the element present at the provided `index`, if it exists in the given list.
-          return: [String, binstr, Rcode 1, bad-list-index]
-        - name: first
-          complexity: O(1)
-          accept: [AnyArray]
-          syntax: [LGET <list> first]
-          desc: Returns the first element present in the list, if it exists.
-          return: [String, binstr, Rcode 1, list-is-empty]
-        - name: last
-          complexity: O(1)
-          accept: [AnyArray]
-          syntax: [LGET <list> last]
-          desc: Returns the last element present in the list, if it exists.
-          return: [String, binstr, Rcode 1, list-is-empty]
-        - name: range
-          complexity: O(n)
-          accept: [AnyArray]
-          syntax: [LGET <list> range <start>, LGET <list> range <start> <stop>]
-          desc: |
-            Returns items in the given range. If no value for `stop` is provided, all the elements from that
-            index are returned. If a value for `stop` is provided, then a subarray is returned
-          return: [Typed Array, Rcode 1, bad-list-index]
-    - name: LMOD
-      desc: |
-        `LMOD` can be used to mutate the elements in a list
-      subactions:
-        - name: push
-          complexity: O(1)
-          accept: [AnyArray]
-          syntax: [LMOD <list> push <v1> <v2> ...]
-          desc: Appends the elements to the end of the provided list, if it exists.
-          return: [Rcode 0, Rcode 1, Rcode 5]
-        - name: insert
-          complexity: O(1)
-          accept: [AnyArray]
-          syntax: [LMOD <list> insert <index> <value>]
-          desc: |
-            Inserts the element to the provided index, if it is valid while shifting elements
-            to the right if required
-          return: [Rcode 0, Rcode 1, Rcode 5, bad-list-index]
-        - name: pop
-          complexity: O(1)
-          accept: [AnyArray]
-          syntax: [LMOD <list> pop, LMOD <list> pop <index>]
-          desc: |
-            Removes the element from the end of the list if no index is provided or from the provided
-            index while shifting elements to the right if required.
-          return: [String, Binstr, Rcode 1, Rcode 5, bad-list-index]
-        - name: remove
-          complexity: O(1)
-          accept: [AnyArray]
-          syntax: [LMOD <list> remove <index>]
-          desc: |
-            Removes the element at the provided index from the list, shifting elements to the right
-            if required.
-          return: [Rcode 0, Rcode 1, Rcode 5, bad-list-index]
-        - name: clear
-          complexity: O(n)
-          accept: [AnyArray]
-          syntax: [LMOD <list> clear]
-          desc: |
-            Removes all the elements present in the list
-          return: [Rcode 0, Rcode 1, Rcode 5]
-    - name: LSET
-      desc: |
-        `LSET` can be used to create empty lists or lists with the provided values.
-      subactions:
-        - name: LSET
-          complexity: O(n)
-          accept: [AnyArray]
-          syntax: [LSET <list>, LSET <list> <value1> <value2> ...]
-          desc: |
-            Creates a list with the provided values, or simply creates an empty list if it doesn't
-            already exist in the table.
-          return: [Rcode 0, Rcode 2, Rcode 5]
diff --git a/assets/logo.jpg b/assets/logo.jpg
old mode 100755
new mode 100644
index e0886a724c62cfd0106e2cb4305b371b6bf5a313..3bccc2497b61acec9c198885e8a74fb466d309fe
GIT binary patch
literal 184696
zcmeFZdED$|c`*F2i6DyT!KJk?^iXk0A<1O29f4zJO)`^AW=$qmXx5pmlSwj}Bvr6l
zmC{$KpmnKeiy*GGuB}xHY7wmq1yowpDlS#qQUOr`k@MZp%sIlb+Sd2?{qg<2_Y7wq
zX6C+g<+`r>y01O=fqy*k^_|D_Ev~h5)KNP-M}hx#4&1ThHgm_`+2Ocn?>u^EXXlum
zCmeOm?$G9q9(DB2Dd2Mjyq%XHyZNmFKJUJU^F_M0eE)RhYz#RYt?uj`y>s-Rov@j+
z_XGgx;OE~2pLbse?&weEX&UW=;V?N{GZ(sbwy_A{T#I1zYy{ppXMY}PhG(Ux>(<N(
z&)xg|k6gF+bjLh*uZ#<b5Y?@@!<Ml%EX9#gdIoLo-GAQc=j8kHAPTHhJ3SBlFxi)%
zyLSV8ANaicGrafo&6LzTcdsHyr`H#;b$a#e>e<Lyfb^`g7r=;3d*7hZItT%LJ9qCv
zRM+eE*=y|VMLdI1nx<i-0#_<$ff;8dMVM;&Sz&U<AqWjCF=8i5oke*1E<{aVtkQG$
zg8Od}GmW1?jaIS0foU4B<y(OjrU_sIx{K?Uh#>me<X_0GM@(#U!R)`DJcFHJ_DI-*
z@LyjL1V5Xdqyy_=cKkwIl2H*^@W4tItJtu>^fL}&m!!=MaNxgM_QNl3W`GqxyWH{p
zRg%V9x`=-<kVj0|$k3ttMvqv*!%zkTU?TQ2>cDp(FTV)C-I2qm0(oB~^TjV&nCQ^(
zVTfpknON3gci8;0k$fO3#$mH9uoVxR`?F25J9606)Ukib)ZLMTQ%z%^y@&%1EP)ef
zGYfVjYqrayXkR_HcG)^wEc|o#GQ0OVoqL$ur&sW^k<*_kJE6H)C(qhmA}mbDE*e@2
z=&2wT>?{QQBS#fvUxnO<qw2v~V9S|Ws{QAAmCZZ^LMr<ZWHVV1_6@Aibar=4-Pe-j
z+`Xn{Yb!rJG~A0Fpb9j<u`pJ<yM$|;yLYv6%zXsjBrqBwD5Bm%(MF9Snw17ck@Y51
zuT>H95P(Ns2<SfB1lO8XqH5q2eip6~Rq&^Zo~2<VeHK}*q7|}Y;v|M0x}FP@R0|Dj
z_r|-|JOAVBA0eCjhN%{(t4N9$w&Mf#$(9Z@+r#`mv>d~g{!3P5@997w_W=Or%CNw0
z*e;m_be8Yvu@)DnqhQGZTX*I55V(mICxEqJxqyN_dw1}VknUqG&GelR;4<`zRMUNH
z@9A4%-DhYU752!HUGdu|Il1L44F{~lL8ur;?LGbBvFYJsU=4O<aUZuUda`4tbAZU^
zl81k0EoU|d;t75~G#xBCSZHlmCeGc9?&?+`#;KF~)<H$h`02q<zO$<?_xV}~6kvYV
zA=x?a5J~62yK2~X!1T?h?UnE&;O%aO(R>x?q2~C>FTO#vt6=x74-$YA8`0SOHb00x
zN^IqM^W&i%$4kxj!&5i=@7%pG?ZQ=t=?^`XAIz8<Gix$Krp7T$119@SGH%Z^vrU&;
zn_hDbVI?q3weR^N10l@jRb#;X-B*)2^NgH%4%6Sg7`*yS4Th>r!)tkZh4-2EX!A2;
z!#cx(!CAjqW5&(eY}~5NymqZW<?HKNyEdD38_c3pUoScvfM6>L4Q9Rj?l+jK)MTh$
zqb7`6wf?A?ul07G7;H}2KYOp6Y;A3b4F;KlJ51a2@t`$WY{oYLe>U-v7yS=E!xSL2
zKVuevTxQ)IY)NLg0n---fV3u~ZK!9NUF!N^($8M~7PIEsfGupD@c?;XY{oTye%=;f
zpQ!_C0bdNR-d}XV)H=99gPHZ~;5Wb#04Nl|VhIDl{}wmk^8TdRZdRI(F>Q{8s6DUc
za^82O+6Le_W#>Y!I#hj7%(AB6F9#!U*(s)&R2Rqn&?9=KI5)>CJ8B>+j^h^sU!Tv^
zIS*L393qocoOEh&QH7=E1|ZrSA*v#`d`(7kQ}Jn8t;{1WNE2Pgvne(}&Csk`N<`W$
z3fWk%IiG5*!v!6uT8Qofs(biw;z!mrP&WXVddOF-5bwBAM-C#wiI&S%5^F@Z7%&V6
z$mbfhJ`k7wv{?rNq3`l`-fYzB&7m-Ew#EZ-&dt<(rZ$TWz=h7PTLL?%4#j$F!aGLR
zEcwQuF`H|{aUnZXvRkiV(onSc<}5eI-g>qnnRnGz+c7<}HO8oAuUr?iXaf*|q;k#=
zy;K;DacSV2Q+X2UN(;3Vbl|IjxzOY&*4yYxul#r9Zva9KpO<AHchn$S8KFwp-m>RK
znieiQ!E#9yX}p9oSObCv#Du9gcSWp0G2R3-HNx%*Y$QogsCE|vX9JKAt690M^${Lw
zX-$L}(+Np5g+79>vPM-Y;f7l_#MRKqT69Kah_IqcIAHMvaTQu~{cx6c>zK3hk(}KC
zOe?<Qd-Jp*w(`}q!ZmTjZne1OtXoCPkv2-onxFT%n&3<+nQkf_ml(kfg>z_ZX+Uw&
z54&D1lQAEo;SE4<Tp}@rk(0ERhzeRSJr^mAR3B=~;c|hN4ZST7J*6^>S1uQ3ffSG}
zzv?B%uqWu0<f&s3HV0kZCHq?p8Ggt{xPl|-)DCjHtmBs6kA{3VKogpuxt`PX{M?ia
zjG6o0Rga&@e0N-(>88;eB^I}+Pw}DP4k}I6*kCBSI@#0KC@x~`l98-|$xZCiM%yEq
zcN>ee<fpP98Y2Y_+OsBAOGqyt6-n7^Nv#1rA8H{<CDJshxvg>okZpDPOpj(Acgl82
znWYxPY8Hu1J5SRAHb!BoUL=*(WEE>Ntb#2jF4GlPEmD_VSA{;=Ee)z14A6C$)HeXf
z%j%GWJjH~^I@BM)ovCX&bV-FvF_T(`Oil7S*6FA%&(-tEe9V$=C=H#dH>1|%xSdZY
zg5y^@t9(+~D%j1ZU-eH%q6E!`87_tyVMxM?y|htwzQHgbxj8*Td!{8xK2*|i6igu9
zuSzp`O(`wf7p2*N8s)uoJR@UjW%DAsG)o{dUM^XfR`5nKF>3InpRYCmhqEbPs4|C-
zaSM{y<Opp}>r7y^rEXP+X9BNDO|?C6S7UaG;i{#?*vLjE+M?SS@uX9)qV0-Pn~ofe
z+46AF8*pyd7!>B9&@+F<rb@SNvt4w=aE?X588tI!>q6@^7Zg`<r&$>Z>40*eF+K2C
zxL%Gt&&#qQlqBs9hGspVv2A|7R;?~q>kDaPB-pLWJe)CH0R;pkiAfk4Mw}Q(9mXe6
zpB?tZly$2EX5MOg>sH$iiohjjDY^kzs}xl;COn<8rGh2EE@d0AIA~#zVtXr6l~-cN
zG5x@*)@r7Wd1=qC$gPR)bYeQJ6wMw&4_IHImXK220K};ZumX<M>vC&mQD1f#ooOs;
z0fTpwiRr|@%AXCu!>+J^Dh(T@ED4@KnWPaA-m2E3`?W<K<2qd6#CcRKJ1QoKwHXBz
zt~*h3J4&lkV@Z#;7^a2ab*n3d=va`tE;0ANM`=0GNF1kPqOla?2wGyx3W{1|VJp}z
zc{)~Cb>CTV<JP?2!~@MvU@>sizB_fQkTRK4#=JdP_~Ih%C%hWYoYgeqtCfz_jF-y9
zZw>1Uu|H6R>4xMg&d7aDp9@V?%f^#PvDZa|pLU&^vycZvS@mSMVS3B<*dnNr!W%s4
z$t`FwBwNaKdHA0Vz(c>uJ-8|+C62Md)Zr+7gtTTfXUvUOF)|DS)@Z5P^EDIb>k`Qd
zct*k#TZ&*Y#>iC{nz&VLYXWp5OUcSWo%P0t-|o<z4#Q(@moYS{_?9LKvPY!7o`g=a
z5z5q@c(~qR$hFO;?4%hnucj+$&SUXN%oaT~BJ5U=A*hUlE4EhcMTlS|)j}GDq?anx
zLSIS>ZYnNnj{BI4O_OSDZh5#SilMcjMulUek`s@nlm)Zq%y1!jX7uxbr9^pT594?q
z(^6Py87@g|ZYVMl)5}nU{SiL53R%OcEr!FKh(OrUGLt+ETAUKlyghb<)L9RBpRC7=
zcHB=I9b0d6!^JdTGr_8lqgLKrsca#dXuUTJCxNnrq!8a==w=iXv_z@U(0NyuTaqXg
z!}b7QsGj3D47}~D6+<+7os`HYPS{@3W~tOIOi;Z>wU19qc9;>8k7g!AZUFZDiD6eU
z3Kj`*fzHJU@@3x_jm#R!$S^OIv2Ph}r_*xBi3uAXgqH)AnuS6EeC<&|hU&B$2>rSq
zwYSdFxcn71EIy@5$O5KuPE{2TCWb3gvns2I8|Zm5+N!U)43^!I>lrhP`?_0QvT+tJ
zDjq2z(V8)YVWUk#Z9JpKR1~7}RITgW0(Mn`2Kv^wrfH8V<4DYbZHzZ0HwZ!Rpa_+W
z$`$Z6;sDc~Y0xx6GMg886P_0ko5U@tkvn5YmpZOY>$I6?Hje>ITe@C#$VfI?A<X7h
zY;YYG-Jv){vQZomj_+!Og{68xxkEo>Ev2b?G>fuYwnnO9-1nHUlTIo?a(aDdkg5#q
z2z_l8l#RH$^$(r8mDVBOA0aS>i;Wdd_xVI$_0}n;YcjSPv{g3LNE>M9WXO0Dr?rKN
z#7(3%vc`LB${Vogpc=I&?rN*PEHzuP%8DWfjkI0kD`Gu!?P_I=!M*mn7x&kF&9pL&
zQ4AAn&=XIr`nl$`3TarY4920vrB&PJOlMfFk2XB)NG(F6`&A7%Fcz$<HQ@BL`nA}O
zvT00fM3<_jjB8OAsYYELMU|1XO!SQJ>hNTt&+)J(ttnf<2)na2qYUk+!`hl6=t<~V
zoRcMEkWFb}MW(HYnT;f55?3l&+^f^6TI*_<pq2fhyC{a8_GC`24Fnq2=3MS4gJgpt
zPO0^Njpde=zS`UMCw^5g;@}mDk!2fzyziTk5wj&%b@-H1`s=h=FGmwLDgf4FdC^9V
zlCoAlJ1X)zJhtEn1-e0#gBnmL#+;Vn?x?_)K{C`f03%a04Llgi8Y-K)E{W}|-5cla
z1-|HodZn;-Ex(&nO^elInP!77Rq2fez;Sg%SW-J_AMQvaAjg>EZvesrzLLh$xSi%^
zY<R_DHj8Ad)v6jQR_zv|#;%0Iuyw%=+6ve8+eEB~h(x!RVZ~-=0*Kr-98$)XrePa^
z60SlrUv>v-wbs=uXfm1kn$RGezOGFyX()Hyfehj{tZciLIA|H}z%z3jwa1cS;c3sw
z!whHau21!68-R0YEcv-2EU`)y8zL`t=pwDG12vAONPrI+Bfz`$UPOkKNQb+!;;+}S
zuFos9h^7>9P{TEeD;Y`>@OT3dwQLW?+10{#t-9KB4XHsG(Ri_#yJ^{uBE><B4jPYX
zqB=EKn!~ShDiZ=Y7!S!_&PEV$Ng4v#E7N#umyvneozz--h3K-4b|-X}&a5>WhlGI&
zcwo33Rp1G*gA2p+QM-hh-pFLLFcC+nR!1^!9a;6hCopTgi)=7t4OYk+6*DaI>Y~p(
z2reqlO!GNrA;G@R;s#7B5DEcJr0OsSEnyJ9HYj)whcyiDRHOoj383{g*xD`%E!a4V
zDo|7?htFEqV|aJgFPOm8Qy2KM(9ciUVEA)jh>VVD(nfOFvnMtY@^%TakiptiJcYX<
zyoA@aUpip}aCa8d`Nd$`143Pz^QG^$Y!D}=rCQE{wkahEGmIjdD{DnHn&NF?KAzzM
zzx5pYNqg-WvC2<)JfgC}s*vowQ+V^)&~&6l*=ajP2sKwLNvj04MOj}%hScQ<!fnww
z9L-@4&+tfrCUkcj>oJXLJi-wwZp6#6goO~W?DG%~d>fY$jH0?Dyh=rGyfik05*6Yw
zJ|DqMJ&rQGDu*S8ODyowq)gh7ObcL+3g&>Dh+uct5y*!589}7AEe0Mo24*e<qtfo^
zZfMK1irlry5*{@GV09_&-l1cKTONjU%`REWZN#w%mfy08DZFNUB7~+B*zb<ua_zAJ
zOd%LjP+Ezs>b=P#=oDBJ=Awz68k7EV*rGH%5jOy96|tIF9hIYI#ApH`6BR^4AYX(7
zUg8s)D2oxK_t7ORuhAYVqh2VrE1}C*g`DUViQMt%wDeYpwT8EOn|Z#Lnzgdw(V>~L
zVs~ED9c$1)^qytS{wp5<mtiS}3pH5S)HK#%Z@o1EmJ5{`)EUq?MB}4WfnaG3;W!?U
zrtxzQBeb#$4PG~Rm1x1a<H5L;g@K}nkXX`HsJdMgg$`}3Y3HaK21^62PL^H>g_s1l
zCosgbJ#_6NE++YH!_Igcb!kE_hN{O1OtZ`L-nu=lH+{cYgFs{(JgU5qFzSpO4o6)E
zjJ7;)JF2S2?tlQJehV-2(I9K_4I=5z^MT9R^QaG1%ZX+$`ufVH+RI+h6E=$y@OZ^r
zxdG%;x!emY47(HsXUr^z7_jC^v+Sa5T^L705S2**9)vE3bGcilbYI1n(ClHGmh`t`
zBV?tt41IVoC4+U;E>{z874{{T?d!EA-BT-VT}j-*y2LvnwSu7>l{9EsPN##O1mfHh
zZbiIlhb_q2W|pd?EsG-E%qwUrnZx=RTC@YYi`IzFqB>siOG@?{7I2o!j$7fYGd?de
zbjVc5IJV$!P*X6T)p)0t=ZmdA>_TNkS+uB!R7GS`G9pb<BO3h{nrX|K(CwDhrR{Rl
zl$dhWAZ+=PEojmdlE=USg1aSdDIq=AaSdX&f<3myE)WIZ%zP>ZPFOf;uB!TkHR2T;
z$el@0H;TxG#F@CpShC+$ZHg%xeg$I@4_2p;g{?9_LE$27Y~@hu1&TGXMcKDhy3<~2
zQ$<YFwj(XO`Bd;bbbU>_S&*QC8%8+Qg!Eu4PzKms5-);v#4DZ5*C~)5<AcqjXmcQ=
zX$FnzI3bZBoQha1Br(5-+9U-x$S@4_6fF@zMHd9rco!v$Fm6X0YNZWRR;SGc%o16<
zSsQMx-qOl=;Mu23;6IS5lCtBeL2=V+g|{^euLKUV>hiAN8DMOMldCXaAr+9k(dKBg
zE$YJ_6*ei}88AhCxXlu4bDX!ascI$tSO%dMB<Cb>wX=C3QG<4#6>Ps@W3aZC)V$i`
z*vRfuz@-UbzAQUgG>{D0!StfYi+*cEGA2baOK_8-SgbiuEW&m;ozWd6=vAm5zEm20
z4Ah6sbx0Xipw_|>ncH+oATo@z(9{KKx4vRD5iA<?ww?pa4~Up=@EzBqYwaZGKqjRV
z_M1qNm|1n1%x1FfuWA%QE!8k7Or$R53Q?(Rc{IVJ_EM1pWmd-~A%|?Y-g-Ru@s8G>
zw{^D`xIU0veArsmygX>SO=>-((y0`OQ~;E3AzO2xt0D;+fso#>;IIoy(bB4c!YW1|
zOh{5-rSN4aT8tL+@T$K87rI>@&N&>Un5~&rl|U%<NKIogYeseK4(mgD;ZL!|L<)??
z_}Vt`NoFzE;JX}1meHbFOxHuLVe8@uvI<w~@NDGTmBtjqvw@qGbyI8ue7ylqdZd$e
z>H0)VA*HvdR9!LN<`27um2$|qQ*^kBnYV~#Cu~KNB{4&m>yls7%f4?nI|~J)HEJ~)
z`6L^*S4~WfSA>Fv!Q2B5tjv>$Okmr<XEj~7G+7<uOr<NW*`~4TIN6wi?V(D_xg#R-
z*edv7A~!`5nq)yM9z<21Q=NI(ZTNZ3oy)@9=doq(aT~E&^gD*Ng5&|I<i@1g1Q~;j
zRw_w`bH!x3px`8??TS$I{nBK5Dbx47KxTwBiwq#c_8mm;$8pK)E0CVv0Bm_nD1n<T
z)$7)nSdRxFxN9w;oE%r}q$?Ng{&Ykv3aF&a5)HN0eBR?h)wU0zF$KaND+Fx`bc%bS
zp=|eZOoYG(==HO9y_Js#`Fyyp!J&lMg1IsmWuZ14Hkvo76$R{mza+cJSTJf*$iY*Z
zFZ;d9!<KQom%*+{^$Mn8t3}>r@<}jQ%{u}+hWo_~@3n{HhA|pU3x}^eQK^lhF-0m(
zG^}-v-eMAk&LHe8wPihTX$K>Ex50`+ea@jI!m`bvVCJM<U<+}T6d;w%%T~WAmcWH?
zxos|8*_|24SJnnetJ1E>LsXV03P;*;M$)jAZ+O^lGeoy)cPq1g!p@Nbb*9ax;5D^i
zG3|-#Ohp^EWY@Z5d{p{f+~?M7mTTZ3@=CihI&Ze(y4UQW)qK#|7HGy3aWt4P?a3%m
z+lAik3H@4sV(eyqf2KJ9XF9V;HyAc71(aFxT8kTl3W40K_Dy!8G-owv#5Ft>RQ-6|
z_32{3&6*~`yC|%1=%9(#@%muNAtO}j%xhz#$BVtKO=<VAQ3u#56`r$&>UO296G?bT
ztxF|IiOQmkj8b4di!WPLUzv5fY(CL=ieuXA3Zb@pUS8+iwKJ#eY`DSDpb9;yBzD=t
zXQpq`qgBNey=hrA<Nl0f{dBc}$hs&xEG5`p(-MJWS@s2x$xYnhP*6O>2c`dE&J&)u
z0f-w6(`m>f4Xeil4)egF`frrfA^l|cDsf?xo}R(3$OGlO#jt1MGj_qQ8n_v<WT9jd
z7OCBC&<9Ft9A>0H9na;#yv1_r6s<)=P-sjVRb;Gk)7(U}LZA006KgHPtpeI$sEYaw
z*cp=+iDg<g1l+E!%CWXs_HuIChzwB5;-YN6G-e_-NSTHr%5%2erb9<@*9$LR_Qav4
zvk^5Ywn+@as^K|XjM%PVL@d6LZN*gVu*l;1YRXwj4CJvU*D!3^WTc`k4-l#b*Cnr{
zWR(c<XvFiukja^T;%)b`%!soF$xsLp;e%AKkxJP$DvoTfflVb-pYtYiEhcn5VbC6?
zNZpj5mxXUNht)aC64)xCsA+4&Eofq!S;94xEl`uto1sr=n6;jbdA6mG>x*n^)JCp?
ztL$W&AgYJ=EsiD(q1u)f%ZwL|JOFOP!|vQ_yY&uBTV(C5<YB(6z@idV2y|Iy@vvSu
zIUw;m+M5c-w2xL7Yr^d<$u4Q-n#~BtqTpN9lEk*FOq5AsDV)m8T^8=AY*4~WLT0fN
zI7JgloWqOd8i+(iso1fhKm#MRSyHj-I%4S(1fq1?7D4<srojBoDi0=eH~{*S*5IP+
zkyThDLG0IBP!*e2hHD7Y;brJ<z?Q15LaB|n3X`%^6GplNJb^*fx`m2vGfO#)Or|!T
zWC<*1l!Nymsj`BJ!yq!92zU*3CkAB)AGi%D>?u5!LnwAQO&bhpfGpjTSxE1a5(-K}
z^+R|rWVcjd)a*7$JcE{&1noj(8i(O&VOAgRq9|@s(Kc^mE>e{95IO+Xoo^|#A>vAD
z!AOGy?*doMl|tWR7DISUmcuFtvK*e^>7a=>iU0x`8>Q=zVKAiAS(3OL410_{UNU79
zFn(o<E~)3aQ(G4mx-8nGidhuiC?9|t1{BpLXb1-ksPwb6W>dZdX+py4mF_Na%|^x8
z<_`fNLaM;aF$?pJvWdYgygS<lvja(Mw6<JBV1hL=YCBVbxv=Cgc(_jB%!lzs(We)Q
zLRB^x?uvujsMrNuE?^ys24ML)sT9Ecd^3Mt*91?}q=O4m>66T6uC~YkGF8aZ8bEBi
zodVh_2L%pX-mUm+cq<iNZ*7FTs4TlMBlUV&;AfB@PGKN!<-nVhqv}DKDA=hrf=rg2
zv;Z5zHoPrm$^fHClcilrMpVv1+XojsnOArZ4hIm5=W!*~qPUVM?L?cA?dmFu)Vk|>
z%X+OhHik1eS+uLj7NAAgwMN5j_au(7^|CtyAztEv{7(tZ$COLp%hlXc^~R*zn?{`K
z4XU8SAa9nlA;^KPu?!=CUkUmSs%o+|^ot=o!r8@`P`X`K7_xLs@@0{d;^F)Lur2>c
zh49FM&7vHpE)I=mz+c6(0?njuWiiw;6$muTl-|N+l968Tqe8zS4(GVg!k8#U37W2S
zEAh5qEj9Z-Y=H~fN<fvB`NDOjrnG?YG4S)Llttmm)U>l{;DRiGFAp)Fuw{NBv$bZe
zj=G9V=}kS<^N3BiDbrM0PuW%zTJ<ps<Z$UVkmdzw@yJ7BK*vPCoMF*nMIRD$L+C5i
zQWwo&5cv>@S87vR)X-uZr_8xJQJFUQW-uFQu_x;ZC^ac?uU}aa*wiw+o%R}7^^}v5
z3SV(&fd|SKD!!Jv{%o?Sf!4HA4)YSB+|D)%P+3<J7@Es^bd0N@0fq6bkmfA~7%K_|
znYkqlQhZ*Z!$I7e!#vPTo=tbyi5&<)UX`-50;wF3im79b&7u(D5=2Q9GxY^}-IR)H
z3S5wpW(2Et3#tlP8`y^eQpXadSQ-%EC9ME~N4#AH3~Eb2I{{6lvM|$JZ0maP!5eie
z8O-qc3biNc0M1*G*R5C--6cXoL7fS1NnI3D*kCkFYf5iOF9m_K*md62tE+wtX=It`
z8w~l5!m2!OM`{SEDQs3|2uWlTP!zQ86M9QSXi&~;T0N~yT(+&WrkM}<Yf00^9y_N5
zmnBvM0Fa`CZ8;d8<VY5~0K0YuD<UO|yPp?O-%{FfX_q)iT`e?4&V+#@6qtz<eF8+D
zic8a)Dolk5I@am6Ud1U|*<hHDV$AQVnXp>9(Q<+U4dO<Cr$OA!BLV0nsQ`Xkl2H&a
z*kV?wAmL16Wz8O7CMFtO8W$5)Weo;eY=g%ollL?`#cK6-hj$ohf>|Sko%UoS@m8II
z6cJ%*Wq47dqhvB3S@LuxE%VTA<-=@2p&(Ici5lLW`4cqRVpyG<$|?qZ3_q8c0-B~V
z6#J-Kpthz*6k=u|PVFS)WIbJUdT~2rZG&-zc%Uw-Nun05t)XWPgmByHT9{JB5&{{Q
zOcbb$up^q}S<#e}PCgofa(r7I43|_&xl1ZB0(rb}X2Y<?<TJ*ajwHcML3pvj@ZtJ2
zYfE62WostV6C(%tL1zsuqD8Unmtvnz{U}(IVm=6cTH)&0077~;?;?3sbUAbiv;JsX
z%IuI{nCi<ODUgF!pPv=Y`3#@V#!;qB$K`6lfHn@)9^j1@3lxVIB3+Qj(DX^xfkJKG
zv&O*f!4uQT2hKLb$w8!{;Xtn1%rv1E4X+N(G*&Qzw&0fBC>F#51LnE(B9NMy0Fe@t
zRTo|YGnLsOQ{IYfw4@OwuqyVpl!<`C1LE^p0Ys$>c^Zo2t_$3JKaG2sMSxbH#n|%(
z-k{Y9$w`ARA}k6NU?$aSDjK`w+GYulfnlF6Hasi~DE4YPp+m9kfu5brk_iQLw8?XG
zV#>>zInVW~2o2&e#9Cat8mwZp#W+sa<`vu{_$k5TRa75!Yt}X^5;Zz;KZlFDEUFd@
ztw9oHQ6gmmlyx_%j(pG-fwe<VRGp@fEI_8C592vd3=n8BCj=YH-JVIu66M38y}^(~
zE7>%p>@_?DxneA=#*vcIQyBNv*jnfG?m(4I`w|J1M<b(&by$RIC5SAQ1i3-No`h!L
z*F<K}-nt?~V??qpD5Hb=QDU{nAmm=NLo=9ixIj${D5U!->3FSG*l8fB&{;-9q0^Ic
z-)E|{IAerrwcxSLq#Mm`MDJ>8IFs<DrcVh4kAr!#fcsA14$VcaUu#<nz)fzU8Cg`O
zR!${Jz_ew&wgjzSAmPZeA?I{zSOlETgE`o!rv&<V02eqYGVr}9=z^jqU9wOL(xE0P
zxir~@V#o)bMQkPuV_1|FkQE#i{4!W*S{tN2_*R3PEvMx+*5eY@u!#B61=41En$=du
zxhoMiTF07bMIH-dmkpp%xYE-i%?Yw=N*uf_Yh?kO^HJGlyddIv$-s&6mgKyR>Jo^v
zO)^_bR3O@DSBAMXm6k%Z(w&OurCB^``q9K)OjKuG;~Cu<TB$&hq=_Jc9PQHFLNRGf
zZ&o=})ltqA7*|otAT;Rz&S6+I6tk7kDpKepGO<_qV4}2opr;2lw;n9LHZr8L@aF+Y
zdd(MTFLZ<1bcnaAY6GvM;Y5-U*w-wT15#ASM9hx+es$QYI%2&i3G=l``xrrjRs(2j
z0t8DMHH2E$oi<7e?vhv_C@SBZRts~U>DX$Wbd?d=sTXOpIZ5>eSM^wR(&|M7p7(%1
z#B+te8qx}ImbWQ_L0jTF&?Grb1FUO@X#l3CDYOBbEj&y^h-PU87;XcSVWGExlvGV=
z8Noq8V8cV*gX<clOy&bmcP3)H79|F@S^{=M0@xJYIO%C}ukW(~i||VYxC|g+tE3qo
zI&(hjRx_=VdUdoJfQsP;;IKC;sxhI819~}QB1$#}4rmDMPT2a0n35i(_@F$8lM+Y*
zff(B5K~+TxyJLEonIjamebs4CKrwSDcDL0E&LRaZoSG7d^sC9(Fa~XI$yqvRweVCn
zP|^_Yb%Do#AQ<otrl2{xZTO^Y<6XE1d}{(2_z@&kfnA?$eQdK!O%NLc`7eqQVxf^R
zHgLP)$dD=lO9LQ02GN$%2Oawr5pP*?8T2gB0j0aiARG+yx-n`}4u#FbgL#{IM-n6@
zZcs(sQwmeUXRB$E_z_QxDv3JvBL_4Tm&USV^)a%6MPXO%6;T7EcO<kC;*wgmP{ybg
zp6+go3wTzC1tq9MXgJ2{L8jnk*|hn|csv=9Orw`0mB4jo+yJlPwix(+pcc3vN^no|
z!frWO0{v*XaAA(bEf2||dq@=!WNkN%0feH9#l5Z-^;b(mRkb)u%Lt|0ygnpcVO+4Q
z*v{Fps?I_Bx@=a-VWYt_BWLS<g`QISKpZ3`0pzZs38@)vY*o$T4sdD5HC>fO8`EkN
zbv=g4bc{&Dm5Di|?wB1TZ&Ra*WoIDV!|8A<*lO0OReCfuB3NMG8_{Z}4Rm4>C3YnH
zt9pwyd`DITP)}*E3{a67fDWHB&^={A$(>%0b6KysF6O|H##>k9zv+(~5^Mnixps=R
zR|DDMmnhoibRemgp2ztDm_TTq=_x3kjJ)1Pu!U)1_Z!oOEC>Ux1QreC5Oj9Bgg^qo
z01@YyNNlL0w1CVFnvhA7UAvlgK}BxmfS>_X#MuR?5l7uZ?`_)>%wZs40V_vzGo>5;
z3bTj0+%IJssQsp=``|%0g-%zn-Qfrw6n+cFN*qUCgS0(2x1+kETA*JkgA4`U3ij3-
zQcN-}-An^_tSgWSXNnCTxyj)M-Q}{fw4Id$!64obMnQ`NZ2>Ta(Om~ZX7CDZh#ri^
zMP1vv#J$LE;ftCEu$*uyA0SQ-S4}bWnh2V74B2aS1ak=|F3_e0(h$_TB-bXZ2}rc|
z1;!ql#IR@C-~og&vz5aU$o+?GXXN8CDgXoCb!=!<s3n&6BuuagjHr`jB5aBQHj*u*
z9%lunvs8wblD0gw=qgIu?8WQ6jcpdiUC~hA5l1<Jz?!6YbC5R6L)KpKt4xf0ZP0~=
z5?+2#F#c<q*Z_RQlp!{TDG^#_#0ZLDxyMyu*&gClC-8d~-9bw(x4O!rx};()qFT^0
zbLJ9cwqSAsnrxdSz5xhQr4lWHCSo#%T-wbZF=01X1CkpxiJZWujo6@(tOR)hEe5}X
z{1RLTodA^>BH#r)b<-}qEoCYsMz=r*LdBM$B?MZRLB~Ek8m`88)~r;aZWV+4WWKi7
zDAm$v7eOHwMW=G4Ah28y*D-LX+JiipLIJ#u0$fmAz*r@Z>H$V2W|i#1vuTo66Cf9)
z96M_pkrEk<E$J%Bf-%GDO7I9mt!@qOL%F8gg^RhnWx8!bvduPf6$ZFR)r6%RWOD$_
zz?|>o2;QBjmXrrrbE>;awgzEthng1?0_tSFO4&9{D9C0m1$y)k_Fisy_;bO;FaA1C
z9jbs7T&F5cYta|24%R@VvEs=DHGu;u5sF46Y(3a{9`IEQ3S0Bo5F&%~n*|M&AZSJg
zupjAM{|`*-219G4BA{ggUrBrxtSZP_3#CXzThrDMM(2y5J_iAWxt!YusX94YX^$ZW
zq`*ux1-iUHsJczWtEi(PU4ge^^K<Sk2*bDTEs0{-A<x#OPz89lVMc&pywNRz?79Rz
zhkO)E$GfiW?q)7!YlETPn}f~VuNmg9_f|ryTo%YmHXhFF{lPE=C}4$zyKT?_>-G3@
zs&zY4YU)@l*q32gif$`rR2&*0z`$$(f>wJNH$hfJ2*>3RJX!?0{NdPMbb%mK5|)6c
z6^K5)EIlO-r%R8@LbixOOBK=aKpVwq+?lm8VL6}WBD1x6*rH9ZR%##VP1n`Dt~4+N
z3CGjCJFm_)Lv_;f;o$9;eOfBelp~khlxd&DyJG<%`>W<eN?G0L!hTb4bZO9j-(?+K
zt_E3NhO@BEqHW&pP4ZT!5?3hGiM%X!CR5UCdh?au8+Nv-Dw39{JY0-bW?JU<u(qOF
z5onAC4he`6r6|B;RkwK6X$SQV-)8&isOm^jSvn?YeU%o7zEDcg$zRxFc&Oh7L~&~-
z0s$qx>%%b}ho?xaB7+B-|C0|lZQ%Th7C|>O<YIC(my@Z!;&I5X06z-iH;a-{{%B=j
zayIYD_34}*IGWKI4IRN7b$y+HR;Gdkpo=Fi2!9SrnXZcCl?7r)NR8Jk%N$g=%Cg>W
zs~)!vz-!eE$)>|%z$8?dtCpx|W|``paa!@F>jl_o;E4=~lw^)^xGbPXome!&6}#(`
zf&2~d$p$V^alj{=Z==zJju{qbRxVX{r4_XNtU*zFnAt=QmmOdL=UXve+uF>=fSCww
z62${Qucu`bbaM9RsNg`I*%lx$J_MJG?Ui1(IR>=$gb1r^u3E_J6=<0%wH8zL`e>i)
zfIvOX8Nly6te5$)Rvk8SDe{D5#5y&$4r$x$oG6lNNq{Ud>gUjMs0eYtH5d%%2GVhp
z2HG~^MRnNja3C>I?^-pKce`CKheAgba+n(e;})RJUE>zmTf013s>@6h?9~)TGo*zn
zQwFw*U1z|;@w`N}HrnfvK2jkapmTeqH}_=BgMF{#aTp@{0hc$|sy&*f?lwEu%tZss
zU6UBuk!6q7SRVG^NeMb{Qoij;HBdBK$#DwWM8-G~H^gRMoekDmYnTTFhuKrbA7UVk
zqY`bjHS~lhg94S-=S`z5ro(QVS*vrD1A!JrWvI6(%8?*Nf~%BPv99-X(1>l;c8fFU
ze~!5}a(H-SDX0;GbdhG_da{TM$PGYKwoiks%DA_dYEj1@7J=WChEWMRA;2?D-ZWpX
zyBIVYg=M~#L#Lo<(9Q}C=^h1q^uV_I%2)xcYxj`p5TIu8I8%<<T*w#%<72AKcugOx
zqR2L|HoUCF3kg}Ju9x=@sxRQ>19AdFj9Y%(4s(q56fxtJi9H@HDsej>%q9i{Yh%2G
zKoz&uo%<!Er-s!ZwOr3NA~ge5qL3fKTX(J(gY0Bi1>V?rAcYoGU{H4m$051M1UiAS
zt@p=B9n2q%Z6PJQYNhFc4tzn01$_>>GFfp-)PPo89<1x(D_V)UDd&viMUZ9z0oP`r
z2fY?jnNODCQu5@SiXBf)RVfw_#IHnitKDx^xxzuE_0pQgNo|xRBDam`M-j-y%X-$b
zN0~i_7odT^C(GOvB1>pgj%3iVX|ia2xX${s6}NUpLXUyl*J>>LMbjm_4H2G|`EI`^
zwcRNmQSz?P7~q)(eBj-LZ4_k?sGtE4)R%w<%G)@IrA(~C!z?8sNd)&mYbj{<0iG13
z%2RgAK)@v|ZDh5zDdeaDk#GqzGE|8W>y<dk=4yppcy5QBNA-#9X@0q4taS;V<?c_b
z)WEV3DC`ZDdV9Pky7*#Lqz0s*)oo`Q8QAF92k9v!ulp@^I9m?e9x*2InAhC8u`cK}
z9FQR9Yxl^U&;^0dO&kvO39L~hm1_f=5RFL3!id`1Wnhll(@^fW>iL+hj?z_B>48U!
z)G9tOte}jIRp>k7Pz9RI#rOi$P(2A!IG6%Pas(j@*vL0Va%HWz7(yUxHGzjTRXW&m
zoKQJ5?T-4Upzro|fhQSM@LZY4Ov1QR?YCWFwy2vl1o{Jf1W6F6&I6AfJoS)N#()_c
z9u6!+0X=`k2qf83TXZMFc(P1tj#e5YP?l&f6HHun+<6^zc2wz37-b*{J#LJ`8Y;w4
ztj!UKH}$UQP-AriQ0lG-8ZIo?QY0Y=ps6T9nPw9UX9AB*eo=8urM}F^^Lf?>xs;{E
z;0tpSsa4((mRTXLiZ#B#!J}LwVcQ<Fn~eQs2K>-~Hkpi#2{W-7y<s<fnc8*CaTLaj
zE>ey`GfS!t<uuv~w$uh40^_XJE@v!?m&4_ZBymhiN`DTT&TAYSOG@ETcqIqV(mHdv
z0eM+hLTWLPJyefl*c<H@{1F_nx2qfi`{hz=B3?&y4vln;jY-z4gqSW?gLXgE34uY|
zT~Kbr=XM=rIb)gu1>OxThHn<2)>gI49*73mEx^qz7MFAoVufvI4H8{2Qv=1yG}7&m
z9#*ZT%gbSmMg<cOSB6VYRIZ?;IVXW;I1~$~Nxj$072vyUGn_z1b<8LTkyhqj03s6*
z1uS=+{N-fP2-r%k!&OPq9zlMdX<fFX%n^|OgF%BFpNc4mPxvV)O?YGAv~Qgykn8e+
z(9S`&v=-rTIikFlq3N-1>K$C0Ejsy-otibIsPvRAcPeg8T1iE}S@A%nxmYU_%mpJT
zkhdvQQ0)NK#i?GKg&3fG2m)0rX2-f+j+P@uj74UiIG{~R^lH*FO6u#j)~NVJ-=DjK
z8m)jxFiRA+18P{EZL#&>CW%1IjVzbo`7nm@Jfwp4@E+eut3Us=6b#eguon2|9^3y<
z&+3C8zzGr!D|Rx=Yym}k`sT2bea<g9vq3Wu12@32Ev9XnXIUmV^MNKU>MR5tR$-W+
zfK#<Adk<c3KH{(maF5NI5x>gyi~sXOE4Bp0zvNJiU5;#tW}O6_-ccMpuwZB{4}JnC
zB5clF*{2!X0~8Vs@Mf1Va1hHcWQ;*z6&g3Kvv2~foJG@B>@3Zsjk5s3gn^NiiSC}v
zu)Tggw{{Pc01LM{`vV`L_&yBThfs&k^w>^I7IwPUVvCu9gB%VWRq<b)F0#GMn1gd1
zEO6Wpyjl9~(G9;UIsdb>J$@lM;3$)yJEG+OO>#B|P;3_p{`ocBj6LG?phJQWjwIUs
z__Gs^&O80c>j(#qaNr0Bj&R@z2aa&y|2_^pd`2R05zpPb1}7jMxN}F64xQ_DI(9a)
zL)AFp*kd<`E@DqRI^TKpq4ThwcC_=zFAVqsF5ErD@o7i5c8))EkmG?b@6>mWIr`|`
zf8cd2_;=i+k2~(zV~;!W_~Re-=#x%7>7>V;_?X8$_Hj>m{9_+?%HtmMn8!cq@uxg-
z&#9-LdeX^He#(>fJmm>{PTjK$<fvo7HOC%z!g0r)u;;OldF-BF`E}s>ol_ot!f&3i
z_n4#h?i_u}QOBHe)PcX-ISm|6dE8N(*M>;&=cuCrj*mM2(I=ev7%<_3kKZ}^sAG;f
z`q*Q3u?C|r1mio$o^sq1eiLOL^~An*{NClB%J0AG&mVn8?Y$p6b#U`tzlG`Xi%&T5
zNl$*tQ-AI0XFlVZ&w6$hCrFCkS8p_1ZI<JQqwz#mrm6uB8Jo==H%V97IxotX{J~3K
z_VSDW@RG|efAwqr<h8GR{Tr@)%T-su^_sW6{T=Un{|7$!p}+d;5C6@_Kk;`reDcOm
zefsZjx%CTQ{L*dz_)oXr@r`eO>)ZeGo$r3{U+=!>2lxK)M?b#r{s(sHI%?;bEm`32
zma?Ct>l8rO(Z?Qp%(2Js(sk6)>s`W6Irg~UL?878rhmM)eB$29?>~A^?NxvN-jAJd
z1~$0sR6V}=#3%h$^^T|Cy-V7Lvi})jFaAHG>}P~MOxNu@CmnMXSiECS+1bDI)8Unu
zJagyB`v?z?$iNX9I3fc_WZ;Ml9Fc(|GH^r&j>y0f88{*XM`YlL3>=YxBQkJA29C(U
z5g9ll1OFeDfy=A*i=K2z_vTkU_c6^g@A$V?bLXr*pZn(Rmm<UWKjZZ$UwQf7SA6_c
zwVSzD?fyP6rK4|d?%C-#&)og``aK^2A9ueVdH-L}1MXF=JO1stk3Hv>C*FAK(=NaA
zamvY`d@p|KQ|7lHs|%gM2j2U?@A<WVz2l5yFS`EXcf5k`+#%1ZcklVLYwuUS_So;W
zihnzQ`0j^pdGpEl#($6BaP6y~R%FX_KKuak>ldkC|KO!x-p{=VxwQ&?`9sHEdhgqx
zuyfH_-?{fagO~MQ`JM;&4eo#Eg|pXO@SN-a;l6)`FTd_vFMQTt4;IH<{etDYF8s%T
zKJz2?L!ZC--hVrF{|~<MY5KH_pS_>|=4$`#&-uoqu2g=bc=s8LuYc&pzjfn-j|sp3
zrE70L=NYg2;(eE0`pyG8(3_qf{`U2^-Uq{t7ykQgkABIY9R2?He6;h{?>T>S&jbJb
z#pM;ZeClochZkJ&z{@ZFsPK+&ou9vC_~GG$mGjxp+&??@lFxqrhS&cQO8?dyFZ2HK
z>W|@XzWdoP{LBx7|8>^+_$3dWy8qe(JFmIvA3EQE#d(ju^)toaKI7J>yz)xurGI?s
z$CB^;&JVwTC-e1R7e0m;;p@wHJ?XMHf8lMXz550Gu6*@Lx4ozL;yL_{(c(WYc*{BO
zc%yakYw!B{YybJc&ZmCzs2}|3y1U+W(&VxG#~;2|+v8sI=HZza@i%<*r|<vme>m-Z
z_qpE~e);nsRu1f3_Q4nY)l0;`c;E%U`Jz{T@4Cr>ozDMy!F}&R-gHanX823rx%ssh
ze5m#3dk^fqCAa_i?2EpA^*8sQ_Tsb5uUvTBdC=uQZC&x!=1q5Bb?J>?|Hemu>zFqV
z|Lud(-#-7ezx>-D9oQkFoBj}c>i7E(UUtEs|9kS#7tbeOP2Bv}k8@wYF}(3K;q&gh
z4(xp3woA|a@`oO}{b{Rz@~_$dI{e!^!|U&T_gg-3{|nO5H-G%JTW&&c`RA+t_DALD
zx)03$?y0W}Zks%Q|A)O3Zu*y({po?7$DVxmF%P}$%mX{uKkuT4$n)bzJ?pvezV0vY
z`SvyMz4R;Bed=@Hp}v3rPvC3c^ao%2Umv~jv&wy!NZd`2Zyf*a_g(zZdHLOsw(fq~
zRhNGMc_078J<mG-B2isD<NDy=-|}Pl9_*ppK6%Z9+zY?|;7zRqJ1=|5-3NAldQ$tq
z&Qb3^uyfy6x#RDA;y1th`(OQw-~OkEe)@rPE@Yw4UH`d#?<>x{;K0s#yO*CeqrQD$
z2ey9v{x6<&V8_}2>wk9@e&@Sx{NkNoFz&hG`7i&`TkgI}Tc3Hwna4lp&acc~dClYY
z^Y@&T@cV8%=Y3E6+vnYS!E$-i^5)wC$-no~_uTpXZ<t^H(JP+Qd+nQE@WEG64=P`J
z_g~$)%HMI`&3An2>E}N9k&nOmbB}$&eaAiL&F;5f``B;a^zZPs@A<;_UjBrCk#1l7
z%jFN=eD4jH-nr24`^n@b$;sXOvEP34`G4-&pZU?-@4EIAN8xY1+NC~o&NDx8VCOgD
zi}?E+CuY<Q(M>N!D!2dJ%?Ea_xT*HE>FesnGye6p-!uOG_j<QO*Z=tRW8QJjZ@0ht
zC%5;%{Do(o_U@Fp?$nPy>%!;0{O`)}o7c-9Ztj28>9>92jCb$*=mT#&t$ASQac{XG
zIk0oaKV0+4{Ne*UPdMR$=U?>t2TpL_dHeZqz3=QlzheKjdw&1r-<#a<@o&EP<Xf%-
z_3f9w<cvMbn_vC)OTPA(cYo~<(~qC`nZNi~>1_`kmHy<S?_ah&|M}r5@3`*e{r%&`
z56<O($2<ONH7K9+P<Gy3C%^f?&hz*EEqd#P=jC7e*lpQ6{_5iWFaDQf=?9<t*avQt
zuYTwYfAiH3AAMkld;dp{c`EXpzw3SHBlpXfKksGlz4xI#SAX}w&gXCZVEd1vQ_p+O
zjbFUzH8;KO52?4{ca*a`J6C-rc=GMf|LQy6^wzzXz3svH<nKNIWtZRmnUB5UZR212
zZ1(x`-M3Vg2ma(KKlN^S=bz}`eK7snr!TnY+`oV7mySQ>uF(bL;<NYm@4f07r@rQz
z{AHh*{L$y$_mLO>>;CV5;@N-w;K$yLo_@wl&QJdLm3KbfF8|>@C%p59uRakx^}>Gl
z#Ajalq9gDBpYz~Zm1lnH&yKtD67k9x?X{_M>=(t<gYP)=k59k<;s?HT=10EyiYI<z
z`KT}8kk>rpbm5tU3tw{G9rB6s+pjwHYp=TTc~5Si{N!i9uitrR_TcAVlHL5NPhSQ9
z{u}Pz|M-W#aN%#<T7T;mCq3}dYhV9F)BNf4-u$Uw`@zYlJap^%&-~&)ed*)R37&n%
zt@9sTecB)1f6?8y|LCrNdJ=icJ6`(QH@*3;-o>A#zVnRhzxsP(k?{w1UU}n}ZaKex
z;$`<d^TunJ4;<L}<jE)Ax4I(y;h*;RMmIMeGGFr0(I4J_^$War-FM^j?|$m#KWUFY
z@{!h0ZvMbktqU(cuYdpKtQVel+DrF+>FxJ)ez>2w@xHg5W&FWaul>{YgQxuPao7E1
z{M`?|>w8x#&vzfU@0{1Y47=}*fA!FF&cEm{9=i3RXMEu|Zg|DDAA0jC-+O80Q~RH=
z1c==B%8y*-PA@)j-{rTw_dOrH^8QXdzUVvG)GmC}o4<4Q3&khC40&%^|I^QX1StOV
zn7gyz`%~eQPyE)U&x4-+gRk8Ag3jHS-tgpaz5BL{;j1q{{>Ra$zxAF6PZ%C^*A2y;
z-@f<9Kls4=KXnUDf9BWz;samDF0fwqndl$$+mX)Azm?tc_KPpQR=w)#v+w*6P>C-+
zX3r1z-2A!ou6>;Hn$PdpS3GympP660SVDRF$~Qjh**j|Vl&kvU$KH0-bFaVpyU)D(
zM^{}hp`yx1r=E1&G1WUhx1)ne%47KB_q@oyV({FzTrOU9%$~1T+~<N>M_+cuE8U&*
zU;OHm21i|fN&eKg|L!xNa?Mek2>$=HTi=d;@>!qx($z2ev)>)Q=&Eauy5x`doOJ8k
zcW$5__bg=3<6iwA_g!`L8y^LZdfBtOW^mLc(No`bfB$zrd)WzBedtx@C9O-E-`a;B
zd*R-{x#HKqam^o9&-wKryz}Kh4o|!3g14SVJqW$}m@}_9|AoJE>WS~T^~IO{fV<)Y
z%lqB{zX5aZCocWno3C>&di=EVM?Zm{&m2Ga(<_bp?%99$?cc91Za0`~Z-2u}um0yZ
zeCM67dBP|6pZXoZoA<o_?IZKH-@oAH(dvW!Z(rEE_a7fPiQMzeyB?r_aQ5e(ao>Nu
z@a=#0VCSyyT>MW@`i%Uc%i7<2;(4Dw`N5<9@@rS8->|fM{t~|n55N1SeYZV$?gyOP
zKlsJJc@O*1kKcUheaBe93Vo%0;-|gVMQ2_&dg1v`e#PXAcl@At@9fJzJq}ymc#izp
zeg6-8?;X|D+J1`$5Ku(01O%ih0trPyK<NbCp-Kx)Q3yqmfKsJ*6w!cmqeM!CM0yiY
zkt$V1q*n#$y-RPNcftPl+28MX?iu&qbH_bp>^~T2%F4={tohDo&iOnr&+_L^e$Dj1
zXP2%*&+l>~spgjdaM46B{p`uXZ2d#t<CB=g^84P+N!C|)Z!v7dYiov8+X=awcBXb&
zctlWzzA==+-bo$KpN<=g@TzM8q-}b^v-4!<n$!66N4f)ks`!qlYvM9PrMSy_ZqWsb
z_yijR4PE!EiFgG0J}q1i_0P}$DQf^~Qz|N-Ed~#^v_AZHMYe@qzgVmMdHRG5Gv!+$
z0k<NleFNrvAK4Bc(gLh4{+W|x-l9t;KU?;9H|&#UF{|dWWcQ=FXPL7-F5xR|pI-b_
zD$5e~A+|J7GU-Fal*BKd*YusG-Zj5rr+iF@p4|A>-uSe5@<_zoBZcpB6^k>*ErS;r
z%><3P4aY0`iZ?p#>U)f^9{Mliw?xZ#BC<-g7B5xz9}NTi(KhnNd-kPwbCEkoZ%2%p
zX%0p7=ClfW6lz_#QC{W!=-BBsnrPLodp^B0t;Oz^-|G3Fd=~7CQs3<Aw(!6*Ykw<n
z_IKWTr^MGNulZH{QaG#7v8|u|?ulpChlhENoebA9=&AhSdC_jvYL|BHH*9fg=$rsy
z=nm178^<}C&|Pq1PD{1OX)T?O-fJ_1jZ=|ZzHlIVNp(#6>o`}0Psg-|PGd!-1|qqP
zrlut3<Xln=McLJpE8ctLE>i$g>ZC4qDyn<bZ@c@Gj!w2qK@<Pi;E{V@?cP3OZ|WyJ
z-n?0YNIcUaG|5v-IPZGMA%0HI>f0aY3rcmFF7s;A@X=gTG}=Ut$5%TH@$oY8&E>Yr
z%GZHJ$FWED(u|luBC^sl>a;cuJm6Ma#x>Yjk{TWIi_i@!RIM~4!&MGfHF{-mXJY$Q
zBnz(6IuiP<VVrG~uu2Mzlt>^dl}U?V2CYejtF(+{1P5TCl_QbsB}pzAngDPF3>DE8
z#Xu{~*s7mjCxil!4-f53z*OoY!Lb<i18R9$<LQvFq4;D{pj%sriH39g4_biPC<$$F
zczUWp4LTfhMFb!=64ogDDgY<|0f=P$75>7~_jw6--X>~#K7ST1C{#Uxc30$D9~>%!
zosnAlSy{34ZRHZDf4SJtIH}4OGgqIABY30q13P+?76l?ktT&s}x>6Lw7S?9ZQ~FpU
zjPsIoEFEt@-pG$|X$cuP;uRh|Y<j9@alUm$&b$1Akqq7FX0F~kejT60*m~p3n_52f
znfT0Cybq=h<`QZh%Q%QFF`-*s8%cM9hB?uvgcdP-tbP7=q}Y@a!v)pW@;?^3>gHKI
z9EB7n+V4G&5L$PT{tff7sJLL)DKE37X7eFM#H92D?)W%u)BKsHcE*T?#oX}Pk!xQs
zmX<a&Y`;_dMN`wwc{>HUq*>6m*uCnd*W9TXF?5Q{x&3C3X|?OIiG@#`ehE97hD_?Y
zw6(QMpRXtBGvuemlIG$>;SVUHd?c`Juz!C3cU*(Ww+L#{f&1j+&gk3Iif8hg?{Q4+
zE*(}oF|RHeXy7MMdZVtQh&##BjG4K&XuCSl=_2OX9KF$wV$sjwN=xzRQ3~Dmrtf&;
znCC7ww)*7>!}FrwH~9JmOYpRPe6%#_Qe$T4cSxlYgRgB~m2xC}SscASxVUn6mM76S
zbBu@8=jx(hSPxJcTxAr$_KA{qeq7n5R$SXYEKuxP=b8KVqho;);f(PTcFa#Semwzc
zmWVxke$0FO`9@;VP(`n1P4A*dVINVdz>&dk38T;*##~7=Z)WP7c5}*I%gNVC>c{LF
zCGql{ANiMIcZd?Ami>UU6ZT#(pq1Bpk81wfT2ngpwP3qiBI)9ptfXhlSbkD3z)JnG
z9Jn6(KrmdFP8)%=ttybxGQSZ67rbBgJq_#NVz^>@C+0|j?=Ai9in{!OTCe+JU2EfO
zJa*4UFBRD~wD`Xmy53!I{HwOQV2lQx$xCe-mZ>D0lux1n_Ry(R#GsHp@S&I7j7Blx
z+Ntom7i<71BF6BJY-teOMWIDeOf=NbNhzFdArt`^9kM}PoBCA%$vR$!1fZ=l-CPv0
z&s|B@32K1y{u^}#h(>bJrjdNX1XwK+*#VK>DS)bGMIy8`)bYvDfc>I$FW@RMh*VJl
z>fw0sBK*=!wrUWYJxP&fY!8?MXxc2HT`vVgu&3&2(AmOt#Wm3^ykJ>i|KaB}VY$)r
zH|&q%8=;4bXHw0gubjTJEIp-=*|sgZrDFF%?^@9{%Z>p~_LFOvz!e(Yb_K_hiSU=&
z+a7mK3I~#su%)YF*UKg7h1=4)p6e7(_<9cNg!yqS+JEX?ym#(p#G5|LOM=auvkoMI
zk`|#zF=edDI>(2m!#?_pf)8tOxHPHSi~(Qb^aa%j<v#y`v&J=69+rBgc}p8gzmzUF
ztT)+Mru>FEobTYVo*#$bV)|hm?xD0A+EV09*sM8ExydbGz4b}`H_XGi;&J6lu5sIn
zu&;K9U{P`S6gwT}TKt8)RlWyUmd78Bs~QMdCfns#cbMs44Cd#LrAVD`^h(jjE4LDg
zBMkll!2b;Zko<{4qQ)b091+wP*=N_=u0CYe7%SBpPT;FzeKc%Xfp4L63sI8T5(>Yz
zL+imDp@eIgMRRvmN4`XS@XZ*Yi`>d^PFZYfHqH!c5SQ*TiN8O$jP+dXsfXY0k#{gj
zEo&7v+BIb+m1++A@>IF`<2oFMio)(teHYpNl0s0d<XDI}wcCE;W@(;>^QX%RiaqY_
zTUl1iBK`wF)e28_&+7Z^nr7PQ>ezQHt|4YUvA;uydu5Y}<`JT`aFLmab^Z+t@;j91
z5fxV3PdMVNT0XnI$w6FuBBgYq|7A92*L$N_CT{6uk8yH@IV`LB{gfvuCc+jT_+4sM
za$$L*R)zS+G~(0ru&^b0$|xo_akk{$+S6pbF!@&%9syQn|EDni#fc<}`P628+=s(%
z8?y2b*2_ZP{o2K4*>T%X@H)h#yx%=<F}wX6mWcN1_K_;vNOFxGl_z8ydiR}lxCn7S
zK=tfe6{{^OZ>M`TOH)h&SQF`JB^ld7_E3rBf(8#d3;1CqBhgiJkM-38uD4ff*IrDd
z)6%3u4aG9*$`VZ?;S7%KWa>mv)&s;ui6K#q;7$Oe8qRnEs&KFpRQjMeUrxObe=!!z
zWQ%eH^*Y{$JvC65IuM{>?1GpO3Sy#O5QhX<sm03%AB6|#BD>+C+G)Hu-N3O>GO1wp
zJ#A_msu94S7$+Sqf*o+pO#&4}U#*|LI76^xL46h^vG9trJn<YB<@gl2)>2}{dnHa{
z_SvVV&ir%bh~8J&g6mqBa*0(+4W}|~ET-Dm0WhAqnf&g~J+$Z3rkZRG-73VrfddO}
z1dH{Uh}MF0KG>{2l5s^8YW}E!5;4#;sv)xi|0%-mL^tlN>yt||2j71ilV)4VJ@yh)
zQhM{Ina+hxbz!j`EhFj=e9tszbrzUyOSs#6SBY6PwyYVxj9-+p`8*z6)r(2LzI*bZ
zf3`<0i+_*K#*@`V)dK&jE0o``=F45XK>l8DWj;FjaCl3lYcWoBC6|*1UicgKrlQoi
za7p3C^ap#H_Pf7fhaW0+QXUu-7MthwT|w44M$>FR^#J>W!Ee}Yr7>pQh<DThD?WB|
z?orGaXYzxoBc@tbx=#!-tD^?@^(3s88gDHXXQ!^Se2cJ5E`7tpX1jPM_xoYQ?y{}=
zwiB@>WQb+y%|_n$PboRgUv!3}l{Cf))Ngvf=G^jPv0L}9?X3^Cm!B2ZB{n{3I1^v@
z8#aG>VE0VvRM(JH3E|hw=wr8~ZlzzfwkGTK>cxk5V@Bni-Wm!=q)%>ky~I)pMLA|`
zPi~lU88{gt)U{)YcGd}T%S{iIaMEO?!;qZ*;rm;rc_JZwF=^Wp%el8(<kf*1++lxz
zt2>w1KH>x&Y?J58jAw4}z1&x&Mm7TBuKGbO@6DBj+FOcduT*!1y1wpjyzN*~f5M8d
zIsBnF$NEkG!&UACuBC=|I_va5M@vo3p1#80Y?&_r7p-r{S~pk4Q>rDMU%5#>EL-ZD
zr|KPsG<|Y)ioTzpn%ps7%C&2j*LuB<ueC3n?#>xfsy%gwzB!z<#cekr?&&6YVAhD$
zbPemd=|&@6Wi{-jUox|7H{#HuCaV`^<}??IhU-%O^Yg#h8VI08Vek-&=xKS~16QP{
zBdUm|RwIiBE21jjufFxpj2nLUNJ^wt?V|xRMn~D~*y7>U_@MTN+k@{3OBNqXV~;T;
zt*X-QmNI9%4~j{L-Q7T5-t|41spxRA$x%k(oR1coyTNUTb-ZvrqIMJe=q|^!UOB$?
zblv#fTboqvNfwPh8;dlnaUVWJ&B{yhM4jaA|3=DR`(r{edxK%|o_meS^C!6+^F|ef
zRhJgBJYw096R`?2^5S~BGquBWZjdN>PyWi-bNkTkrtuu!#ZxR{Wt<EnA%3~z&BYnh
zCwn`7^<(=qy}_R3b^S&DrZ=-~dVbs3E3AX9PFZ_b&*d;2g-!UZp8}S}_=KQr&g@)&
zHni7}2LGdt!gvS&vC}m|OscHJr;MxKA8akV3AZ~}up*~CZj4XN$a(wvH?_f>?K-Lk
zcYo}v{usS(nr-rC8JoGcHvqU_ZYl)TzR61+PK-Ghu00f#_`HB^JF>kF$E}<V;A~5y
z0Sa+5bs%1^5k4TUOWhVwd#1G=Hshqt$$+|m^Tz{G*D{_~ut=KWD!nLzeUE0R?a^%g
zj6GrutCiVf*)(_!R}`&0qdsaZo(>1L3S=clKZA{d4lwyzp%t(dv85t_I2?$f4vf*J
zjzY`e#7<Htgtx~I$fWb?N@$bKVh1EJH0i3;=~J@ViO+cA+#Ms+C)k$!9;9yzKj|pv
z*F23({XWAb-?O10hu9_W2y1wF746cdI^UM!YPuXfBxlpBZff;Z<@%2S=DOlWogt>@
z7R8qCrLngxk1dV&ML0MSe`#GYD(xWubay#BavWt<xykfkA%kR`t0|;ulCv8km%W{O
zBh5#xJ?BN}E&OMRNEGm+FYEA7T%HqyJGN+CT|_@}$=8cUy!JA*HJmmplIIt~GGZ<`
z$G$UHz*V}FziZB#UcDShbme8E)Zvc6*i*q?Eu_KV+zMnF7@P`yuJ<sU5!0g0C3FqO
zmK<KViH5{kXq5si38XxXLC}SR%h{UP<5O?RGiA?yg>9sM1-ny0MAe(hMmdUwtmIhP
zs8fHvsof%j>)D&Vs=UK=0>TkZGC7&mJn$}bxn{PKZjweZ24*RCs(V;SwZO>?Q&===
z{AFSN@%t~bO4HV^n3|3herY+W4x6U8rBqv*%Fa>MxwNi=Hsyt<Kur*sCdgw5xLolh
zu`#j8_PNm|JyX<gn48+o3rRJ`qh+N_y`RoLGR7;nq*yYK2^TCl)G@=?&#c%>PA}0@
z4M*O-=_NMdmnzs=A=4<$ue~A`t-k8*U#W!t^8Q2iDgBxpo3*KmuRHud=;<G*$;@J9
zx5amImX%JZbA>z^Hf7ieXB~E%zO&rGZKX8tG~Zhu__hSU%0f8Adaawz_)4VmgfbV}
z&+uu(A{$5ivw@-CF#3trJ5w)2>X{=#6xh?x*LtN~#v^i0wExHJ>7W1qPqHbmv=U;{
zNC)A^>6J|MDktkH0`hrcj>3dR_XtUc1J_SwV-|Vo&qxYhkO5em_)~l3{-A8MZcvHu
zv0Cg2&BIz9*+9X*VgH%yj{*^^XrNH144tnpJN5C=4LNI$oU~%Pe)<jF->|pa!JE#$
znKip-e0)1gv1i|YxEn#4yc5Rfx2d<=#{5Cy`JzB+fs?Vq^t|)8Q!Yg}HGjk28Bc-4
z$u{2;dDSqPyba5e-8=6-yROJOu=%KT`M5e`4NFDooC%A!Ph*qEW9E7kSH3>lc!Ts*
zpk@5N*)>t<SeUn<M}bFAL{y3wKjIThn;H@r6)dqIwn@fM%ai_F`M|Bl`Rl~GG?BJ1
zkH-eflpcC@Jk`-NRphF%k9v<*XsRf3c2;YXlS}woe>QPt!oqQTKJf<6_1N2q{bv0c
zYeo&gKRnAX)XIeGm?>ChAwg0HP(nH*4~M&G$fQt&+F^wYVn|37Ay+!GQD}hP?4%Y6
zNxT<u9WWZAFE50|-3)Zd?xB}DGy-H=P6l0Z^=*RznOZdlS_K?Qzq0mUkOc5wKcJBx
zklQzqN{mir0%{30A(%bYiU@2zpeKXsWeNtV%h8GSKnWQTS1ZGHh|ub3(+EUDTMZeM
z8<k`eS7qwqBDr-f!K=}Bk&E;-Y<C-!Q-h1+9d9d5bxrQfkIva?+|Mw2F2YhXl9DO=
zatHHWBcRb$R+(+CJ~xj`r`eb}dt04oWK;NQI~G(}OR#G@^vXYT{24~h7oAGbc2Q_=
zlng0x;|{n_(`~?OS=P&b|8%w%cEDMq!2i(Mwc~vntxbH5e6F5s(f%i{CgWEP3|OKb
zyQb>WAzo8yb0NS^XpXr}E7V*#6-A;@M>4ny=;}fB7|@{Lddg~Gi%JfVP!BVtq)=qg
z>L|K{6rk@yYfFdgpbds&0>PRglAp7Uk!{4#vH{YKh>>PpQCYmuwPq^fveF}ePI<kB
z^1<U()3ZctKVR7yLTE0(+^gv?i&IBDS6IfZE^M;(+9!A*1zQr<@7}w6wteDroo0rT
zvLefFvmeL$OHQ{%ot?O>W4~}~cWGjN!>)c$iAYr>8D~7GXW7t)M=(A)Bf<BIF)Q`J
zvE~P6zhMh8tGON9LKbbMT4U3)=2h9vGVdRH$L!|$`Z>6?R&agukj>#<$#aPF7ydc2
znUpcgd4u6?N~n?l*S8xvA>R)S#kD(qUe58aGP~GP>iSyXRSLG&`|i)$Q_iU^&oh!N
zsd66L_swF7+9y&44`oVy@J>ppDilU*GfmF4NWt~gCjSB4|4D$mm&qYJe6XQD<LK3X
zqEEE&HrYIl9^-Vg;%WIAn_b$1OR@>|&)ZdxZ?|grvUu-!F^w2EV2}Gr*(oXzZO*v0
zE&B}I05LMj2c0VOyA#9Hon>F&3D#JR&Sk1*diz8K4KbbjIO6E!KuqZs_BZa~E?SgW
zjm~d1x~Eiaaeltos6=q>7>~nzQY!aO=J*ji({_3FG)>?1eogoLJGWfonNhM-GtyZr
z)5>SNrh5GJ9#z)0M3i-u{oLSbC|-jP7kqh_E~k8tB^i4zO9M#H!-GM86Jq`)$Uwho
z>wZ|iDlQK1WF=k+#%ibbzQ*vKYNF{ZHoar8+x*g!=f~*{Z+pd}w#%W}!uJ(^6wrv6
z-cSm*2fQwO^vCnABf>Eg^lPdVw3;j?+-~Byo{{Qg>||ZM0dH3101#CG^kF1f6T)2p
zto0$qm<nesqkt_(rUyxeztL#CUMGnb=*kl6KpBA*PN6AjRNX`I(!k_v{S9Lg02JF1
z9^j&#W-bGndJu2|3lBw3HHZXKGKfVux9>4+-f3+;VgQ2&B-${xnoENRg2#vf2|A5y
zU*f5iX3%gPBGb|c#M1&KKW}oCwq7RVLryL(`TIr}=75=7u94LQz5g#_!=hm5>@tgu
z{)zr)pCaVXd9m(#OfP@4`?_*OW!$h>qEF*?WZ11`nr5b|lDuxD@fwBfFF~Xj9$_7O
z_gLP-J%{i_{Vk}{6E!>?)*}2<H9i6W#Oi3oBI*eYpN8&z?!yD#s&^v{i6SwW1Tg4z
z22X3g4J7R?!z_^ZX;=d!R3x;yv;?l{p@DKloh=I2y#{GVFLCWKumm&|78(?-Vx$+Q
zS&RZSsR42IfI>YKd%B1&(q>PXH!(qw@_o}$wJ=KpCH$i{4IYKWF(SQ)8enGu@f9@O
zfMD+-SJyip=xW$5DYtPqhmIy#MqPEyREJTqBG}h=A!7AaOR0VH<cHs|=P7n~4j#TJ
zI_|K_w$rxTz;pGu-H=l&q5R6F!5;#5PQ71hA`?I$a$?gbQ=#^D>ze6c=Xr;9#vNVb
zUq_EBr(PUlqV*A7c(5ARS<b_De-_)=xy)Y`&=@z{wev{H!)pD;_XF-*JRkdITDINE
z0x8KqRlV<S9}i)Cy3?uuCCLi=*g8%?SU4dgvv!srtgL`v<Fh*+H@|TY7j`as|Au{3
z)kE`6YMN0kTl^zh{QoRkK%#`Pd$+TPiF~uuw5IeAX7)4rMSX4!)r1p2ZBLMo)?AX0
zz6j$Pf9K^C8l#6Vxg%w}P+j}Y{<H1(9zHW1;eg0j7FDlB9=v~{Xi^CGy-VmG-;8sH
zKV|SIc3XHnL)KKaM~q+kVAfashf-haw3yvEA~I@%EfM+ocnj}KFIaKYII?SkU5Ck~
zHwWo;{+ANXQ0~!U&%7SHC7>mo|0U@<f7F(nznxs(ah|`tG$t#W`$02PfHxyUBQMpw
z&aRNf;4ds$?-hfX!DQxHe}y4&DRDzCjW^CQ9dGBpuN=oCvcHJ|LdpU`sS((l1OGi2
zWOz?U_r@ug(;r=`PAg~`SdO&obzF{}ZZ{an=}8#kQf<J#sjHFj&*IBl-u8dqS-fxo
zXWFZJ$5K=LY|M{lI^l^)j6uqmhz_x9Y8EzWSH`yTah3Y2l2aWCLwf}=fU3xooMX?k
zxsXPbfcLeKR?sNzg86rns@$0X1Zt>&P8Ed&Z7cjec}5}%F@Oog>kOq*H3Et%<JG!P
zQMF_v88Pj!KPf8a)?olGcw}`p^u+scJml2z0(%j>jv|Q`L>h2_u3;dNgO@$kAP-VW
zAk0ytrG7up>d|xWa1U;V+wYGU*JE4E`rGAYLhBWEC8Z5qGP~Fk{;I_CSH2+We%@rr
z(8;X)1WlvuGvnmnFp2PNY{4<pUiHHzC*95#h<_GrZ$YyxyNya$#nS;+Sy+jf1~k%u
z6%Hsd{0NMVk{s)rZbh4_hniN)M*$KEMLa;#t`G*YAsW84!9uP28P{mf$-py^I5-^G
zj5rXZL)s*h^B@wB#ECIDPR^@;mmH98JX7_|hBHc=L>(Z3mL-95LstR?L<c>zvV}Gc
z5fJ`%7-pEWH4rbPp=`<=RO*twPIFC5^Gdn5OdSnrOvYHrwJU{r^gCUic&(^YS5;fh
z$F@xP(3*~ct6r|We20d8>kX;mHNlTRd(W$^M~&cmA2puG^6}hp+xanG{3HTv8WJ@h
z)>R~Yi&@8H`;Dqi-}OUA`jN^X>B^t3i*M=QoEYQctHXTuv=)o9y(TJRnva+o-pMVS
zE_pZXIe)0(h}XJ;Ri}b+s>_N`44%)ZUEIEj{6WdAzWWa%llbiQUbAewOcS){9eH=7
zHiX9*NL#0a`rl5?in>A)SKBk^7jvjCxfH-Aoa5iW&Htw)zWx_`AEzf(&~PeNG1WA(
z^)0O4RX4+Leps)?rqBwmtWti?5B5<)X13iz)iU8UQ|nXyJ==>vQx8AP4oypMefoVj
zr|E=5qhqn*A>$gIXXnexKXcuw;layds~Z)FtIDFo4@R!nsm38U-MF(nwmouVd|Vd~
z8?FbZp9<aZ$uw7g^yPxj`}*7xulyZ)me4HQwQ}Be!SePTv$-j&=Y%WKz9A&8fC0l3
z*sM_*7Lnun>GL`j4m!V-2+~PU^JQxKI4%V%hGzb9SmsbcBA1Q3cnEVBV(2>)Rd5Gu
zsW}uTZIYi=UPdAveQmgxD>abq4|S=*7Yk|xJ7Erh^Dy)c3&=A#8wneUX*NEcflatP
zl6Urx(ld$GokjAhX*0@p>>Dp`H<4zPnpSfB%Hp}W4QKhgtd=>SQ_Z}Zof5tME_!aC
z?&)3tK46|BoMa8f4qzs3GL2_eQty*1W!nPCCeADARF#$>u#iIZRp+G#F@>R6R}kB+
zg<@R+wJ=d>hQATaw)}56F&N0`Mne2jXeBi;1}FlRmXv@(63}4(O{S%ebWKGxq7FCf
zhiF`NfuQB@N^O7>$TVDI+RsPuBG_~2>&z}}sYh2;>GbXNr0TC#^yp7cdGE+}FXrM`
z>D#iBjxQ$~D!K1O%}-CZb#d~tOU;<)`E+LLd7W<`$~2y-fx82t;9_f6OFRgc*+78{
zL~<;{)jVnf-f$WNlLgRpv2_d7-O`M@bVw&+6e$(s)!H5l7A8Of!ZiftyRu~<-UMk(
zfe3RWzl;|KVFC&qD3t-L6;-y-pyh>6vC#^xyaHCHN>GF?c<Vs{kjM%cXdYE9_|R&n
zaFC~pY0oF6umi6X2?tr{LQzq*W&pWj;J%<BJYd%XUtplf$A{j$^J*Kh-j%a0Qg+C`
zZcR1nolfeP5u`#7^TMU~18ekQzWEDlL)IaSr&Kj(xu?Hz9x{(PEF!woWmf<5)r^*(
zb0D#I;T?X(fzCGHR#t1c_ecFl;V+-cI=WJj%TgO-o>j|E>7CWyMN0J3yLY~i{7}Df
zphLEQS<clgX~geEc&vRc*Sf^w+ST<qMmJ+Q$sN~|7p@ohys^qu(b4<KrTU&d*TRTV
z0{djhu_&*|RH$V%TIp8R(8Mq<qmXj{GmxIf<*NQK>Lu!bZvV^B|9`y}k;|0i0A6}~
zTC|Jv+t0<H3~A2B1PHw?%IvjyD`AG!;wmfEEbdq7C3zD!REO5;4_9zD2qe9hClD_r
zWc_rVQ|e#Fu6nO6bHBwBjOyD*Q^`j2$%+z>J)R$qY<A9=zf@>6za7=?gWVzZyCtbD
zl-8(Rh_OX@ZD#pzV@IdWU^Awn;vzL8(a!3xKD7GUt%_8fd-c3^B&aKMs%fP3ms0Jx
zM+l}c=L7og^IH`cK{Uym{&O(65cc-S)9m2v^IFB#b;#dQSr@}DWy{?=%|^P+<kf<P
z=TIM0*RJ+zmYKdv*P=~66L4bPMYq3?|5?G;9Ro)N9SeP9@!X#~uctrjZ@qsnJ=gae
zR{d)wCtl&ao#Y2sg{%Vl72>2)u<ec<`<pLqUzTKk%s6ZDvO{eFn_PuT+IYPPA=Y@1
zc_6wnzCtwu0;m5xTMcNJnw^0F-VE|!$N@`Y8E)WKdmeB~0)a0jO;!V-2P$KLfO}6M
z?5UFA7LAlL0h*928gz(x4ViR6`p%nZfVeCb>N8YQn*@sQ1tLJD_!$blf|uPVh9G#G
z4|t?3I4V#t>jnOu8$yESHHH+auO1DxG*=(?-cZ`S^?;!NRdaepp+f2Eq(_9A4)f#~
zhUVLEn226X`(~2G&)=|IGkqr~{`Ms;i8y|j;umXDlje?z9lx3eu}>OJQiVouM2(;Q
z6{Af<0flUN+5iIrEJ!2_N{@-;O>2`F_0*`rDT+sA%f=v(Mu8x*D+woIm<p$^0#>)D
zqT&(BL6Q4vFHRg^Sg0Rs5c}{uz^ng;2>I4lC726rXf$E1;c6t3i7VV7p9IhsL5AvV
z;6-V|09U+#V?_Wu3TP^ocsLG~94}`;INVAA^Dn{4LD;H{+Cxh*MoT<W)g(Gg6X&k*
zwB4AsP2Iui1bz6m7nyua$_MVBDbHCyy>%6x73xc08^d3=QFUWVG+4a}mD_{;`D#_c
zPn$AfH|FB=oH?$wu&eZ~-F@#3Ydh?@Ob-R_=@cd1lt}4MGH`NOSA7zVKTbG;QPUxL
zo#<ZN8h-rn*ajJNw%p!&<Gq4)YMTz#>~@t?%egmto%HNV{h6a*zNjc4a6Mo=P)sm>
z++gP|+n`!-Luoog$)Q0)gZ=3_8wI2349PSyyl9lW`_Ik%pU&?8FFV`Sm8D4s4-arl
zXPt+y7icdKVlKW)`yp~vMK)2ByON2Vq--efGMs(Ba`TVp#HZD#c|@Fe8$R_tQ+3(&
z>Z&hSSlmq?-Z7m&A74)$&0DCl^Fw&&Mn}mOBh|8uwvMq_LVx6q@?Ev<alCW^a};LP
z(bmwU#qF2>2+l`)JwM$CU4f{by`OeN%!F>rzr4nGY=_={WMNqkn>*+jn-}{`R{V|X
zlJ)JlAs6{SGL2rw5N3BZ-<VxKM${^t5-wb#i^6O83edtX;!O*W;bA~)%oxf8-;>Xv
zKHlqVJaO+7&W2>P#~5GQ96my>lS!Q=(In}kRE_RTf!+i20&D#z*ijv;>2?2Lr|H&0
zsh9Kb%<}bL#Ef2bU)%PtN;7UAO!pS@nrmuX%Hrur%gpsMsaV>lF9>$2cJC_z``Q<<
z-vN_5nJw+49sv0gNaJiZz{Nn63p7Dt)TkPPsX*a^GN-^>iH@fSI^zHUHCa#qTPAEv
zc5qGZXn|BLtu%vjqr7X{UTO@|$@XHgu~X5s002<ZsiJDY`l-YK84dv71g@GQAOoUa
zSAlxDAg%=-@(3!w;Z#Hd=qFJ1^kfOndMEL=JK6NBt;Vn1Q|lpH+S4<RVi)r+i6}kY
zN+L~eY#A*h<k4uBcltsxBgW!IWtOHcy3u3USngvIk)hrGi7S@_8>Pd#I_nsY>4ks#
zb)%$&6{1_VybvOhsACymlr#|lFekIm_0k0}Ef;zNWJXm23fK9(Ge*P&0oEq~bR*yq
zBFOG>$vyE9ByvSj&HHd@!v<B(kRS-uLdJFi0$zy8H6g&tfI6vc0hCC1p+SB(WYy4$
zo(?{*4{5Y7AhQM^UCq7+buze%u);#jK&uG&HuRdnz}dm96Lf(A1hWo+GngRKI2ge3
z!TB=48CwVRwl5PXu@BgDZ>$S10t0cdsNKttbgOJ(<ms;~@nyvdX7Z~m+97Elzxlji
z)H732I``^D%l&DU$=@(z)3{0V{AYPmh8maxY)4Ymrv6}`UwzG{VR=gGO;yi3vh#Sg
zm#0IP6&}k!h_t)4JXE2ydu-dX!;clb&}q9y4r?CUeCyw^`r+N_-H=x^s^9yMFNVhr
z`<^+Xqh5UDrFsY>6GP%ub(+P5-NMyj{@iDVUDp&Sn~Gm|N(8^9jTy`?G}>1fj@&OS
zj4k?A-?m2y{zVP``TalGEEfmAqyc*vE^ExPcw$H;Zr})i&Uwm(o9fRV^(s~Hd3#aS
z={Tgr>KA#2tdkbz)$CqA|20SX^4)~bV$@-R-sSs`Na2z5tR9(qi{S+qQ`5D^u5Jr4
z9haQJ%TaBv$P+j3CJ83W-dvFk+V+XsaA#fg<s`(s3IUk-{eH~U_V$y~+rtX_b(>#_
zk0|2yYWSl%&yW9xxvGxFw6A3`Aac9%+L42k8TUJ)dw9OEeZ76t^}12EAX2^WU0GEO
z+ZX{qDSh`M-W5)lVngXg9$Y3xg1VRMVI>*xjKCv&@Yv0(46u9f5*t4F98??634^cC
zRp>#mtUkZB83Mn-?$XWV9l4~2jm}8+9Eoe?C%e9VapBm0vdr}x#$D7COqmt3d!sOu
za%hD(g)y6P^-o6RYJey!xDMBPHgyXJ_CN^05dbT3ZD1*%lW9bZ?Q=sQl)cV1@nDyN
z{AKFsc-cS0vTYeKKxWzloV4}!I-Q`-sDQD^!`qT1bG`P~aF9>}tP+&I1~AY9K&An4
zr@bDjJ<#|BaYML%SGFdMI-$8$Kk!d#2sA<}A}BNU1LoCr(jiKOp4RI(t+lWdt!aFV
z=yKtiP}nl)`P_5pg_&d*h>70yYdSppA>RC`Cwk7GwCyY02ph_t`Vjk}Y`aU}cU=Db
zZLWY#hOFfB+(XK`sbg#MUt-3eRWe!<@WQ2ELD3o@u8E@MvH>p?RE0-6MP$=lh6-O*
z6M`5Za7=}v5D<YAX$9041~vzNP%~dE%!Hb7L0@ib8ifq-3fL^b76G)9KN#|^HPr+3
zLv*~fds3<DaIEl5)CeBRArX-sw6wsoCc+?C6*{~JZUPA?+H>NK2)1N083yzLdRr4Z
zNVH@T0hUf7AgBppz${O<$^*}%H*q}^w6AZzXl0-`0jD3Oj0ROZuz#d_eZ<#OGJVc<
zMK9I%+Q}R|X>_he^}^<s*i2v9N8ZIk<G9c&OdaN#Q2tnGpTxT(Ptt?aISMH_(YQ_I
z01x-1pHho?hh*IL1bJ}S{%r78N_dT5nTm3wD{8=a+@Tu#gd?|F>FDc}RHZnYLHn$}
ze6A`lp38EV930h@oP8A61IgQ}Eoa&bi`VXq5>5`OVLwRE_+M!^9_Z;*H5N{9<ieFD
zISBZ5Mc=vq>!7oaMyTu0My>?79#+iP9DJ$&;fMc&Yk=X7wSLeVAY3j@Lzum(Q&aKe
zA`kO@>-WVLQa_4sc;8Zz%u<TvytTR+d3cxK=gOM}rl2)<9);@L=7HMWD)q_}pv;^P
z@Kang#^-KZWof0`>az>@9g{ta>w&w`tW0@IEpI65iTG@clTQwhL^QlF2}O>yCvu0D
zT^E%@yZc!T<(`;QywEAwzM{T8{e+2nS*`00Vut=V?B>Wwrwm;J+OOpDL&Qh@*I9&5
zySlZO8I~CuMb~9Cv}en1SRqndj}a%OXD;Gza%dd*tF7nd9t}jprw%q!rS&4+T73~`
z0fV5oIM^uA>k2=JV<SUP{1-aX*_U{)3ezz-olQEk<ptmnKO(G&E!#5P5{j=s`KiKe
z{MYW~0=tQY!kuSQzCD}vhQXWVTmog2wzYf6r$J|zw)$r~W~Dj6E&V(sRV_pf-;aYP
zU`Ra!#t8`WfJaB8sR@Cte^&?vsN2%vAh-ijy`R-;^pbP0W1#K;yv?)@mn}$TA}d|g
zwf7sQ9uQsCp-d*!>@NY*nZT$%Z(<T~vIrza0Rn@DDd^#Q&8BP&^j@y;Owj5-WBs0o
zXFAbmI`wi-INt{hv+;})^O$dDc308umixu_+J_?uQc5-Bq}>XX-DhqUqKYS5Nxlfk
zG$rYq<zTNQnzh~#Ez>sn^=9-kC`Wmx@g|#B9`4gFumxiR)d?_6i$<9(iXLdAAXlj#
z3!+6RHp(<cj#f5?^KTL%9_iM6?L9>KGziqKfxxk6!ofuWlld!E3DP{kKQ{*_9z3nR
zY4DVmyWI-D$#4`9ObuGp41yC}^o(Y&!8QPjUQHw|8x)3BGY6@v%fP%pn$7{L^?RgO
zHehH`>s^~Y#8d&rIXG_t5H2TKlk@kL**pZ5O>#V~`kr~K13Dx?q;s!&C&3tv4Hu?|
z1Tq)C*BqPjx|M{ix_Lupr?1N2wQPr3*Z-)eD<y|fn>+DBe7eEYp=Dgci@WHnQfQ-?
zLc-Ub!|VRTvvNb1LIygPyS`j%I4im~80o6Q^>_!=zBy%%Xg|+w>Gh`>+Y()EdTmKQ
z%)uXu+-lqpC5AMIJNA`7e6V&haN8ku9Y{3`w!~Zgw=iL{oDNUw4a{Z5e#2;*@!Kcm
zKHJ`QwJ&<T>RTGOWu}>MMJ)BRf{<vJzoctB2GjMG=h|Nv!hh~{`;WbE|NQ?yt`X)I
zrkU3Lv}%!aztUn+L}1||sZ+%=`fsJ%x0i}1G8A+sE_fUMWc5g_ukzxrdzGff_lW<S
zT>cEn{p<p*kHavSUx5W5-wnI(sYvS4d)dF1s^WI8ttspVP%X2o7|K^=kCIvEE25QT
zhsqI@D2pG+o{nfxCf{~oU$4+xEHArANmKK)Kl6~2jO;PqOdKbEq<kdT39kylPlp7h
zYMBV%@;__kROHlg+o&*}BIwpX3*<smW9hr#V;6@?go7$LF<iSYD?3^-GOStnNb+;K
z=)VkPDt2&%qN%~K3<w@+PJx)S4&?x?eusl%wUtP5^{}mfsh5Ampt11tMXjH4E?jx?
zDwi&`|AwvPZxAP|+g6K?Pk8lMpPZI{ocg9?J50lH8q{bk%U^fop$2x)atZR|P?j2$
zb&)62Qmx`KOaPKdp#2G4vY<8T4SJss>O$d?2idY9i?c@`0Ud;fp3q1R$dIYQ<Sl6i
z?bpD4-8WYNhe0s_a|N|uRU<2JLFS483Q$lrdM~e)G;^4K7v&-Z3n_u%xB=FnqjazO
zmW<tugea3)(X*ka*sW7)gxk__zhUktx9TefF8DJzhQ`Ml%@AgCzMZe@ZCWK{ScHuo
zzDcTg+vVG|T3E}u&&UDS&zw%vHz2*@iW>c*tv!PPGMq3u5+Yid1|a*vYT*IMYEsI(
zGEo6tOg2rnEZGK$B2y;-AOxHY9soR$pjxv+K#~E+%hXg6N2~8;6d^fq53CO}ASl#;
zzv`g@sZ(nvS;m5>BC<M08C5t1SR4w<?x<PpyE2459E&msw*j9W=#m<=@RV#?yey^#
zbhhpRZyEwYBMH_A2HO(@gR($@j0Ox;Z10q>?VmDaVFP@mfO7=i5)2?D^yKCj9|1fE
zej@h(LObinx_!01oXsQH;hv_B)kW61Pj6WbR=C6Dc&cAX)wY<~OB~W<*D%ObStLB=
zdVg2zs_3<j*B~yVq$1BZ^pcs~#^^&%oHVg@(4A&m!#k7dEAs)PT?s6$59fM;_d>Pl
zkF7b$8wBdPc4rl((PS^N9aZ{#9>)oj+N8(#8+=u&T#Jj|E1z|)b7t4snCz{f{^slz
zXX<A<uzPY-Nz!jHW~}`egX!yc*o-CXTEGbvecArug#XVuG2noOad4`GX0&=8E8h1`
z>Ljkt-G0>1f1WRY5H)4n__Ydt=PWJFS>nc(;XAc)^0;)x=N1Bz-tHT5I@><dAapo8
z-POkZ_WE;!`pyv(wEY&}@cWp@Hn*-BV@w(+S%B{lEmkhHtyR1pl3jun$2p3#z8zmL
zIq9$(k!n$&F0TMLp6R|SCC`s&eVq;W=eG8I#3xzvtqA8LBlm9e+-Cr4rsD*GUZ4Xa
zsABaeOKhls*Ql<&C;!z#e(zD}bAM<Z1Nb791$e~4M(Vs?q$6(s4(QI$V>dj^jJF<B
z!etk49%z%QX$AABJiGJj({qw)js3KuQOu3PF1uh}+w|~~)R(T}z|;o%OYK`<pC6f-
zl+E?@gRWS>yGYh$e3Sd_@5o^aidHU-Y*si}D?k<Qqz!NZ3}w>RgJQ;K@$_CM^csK9
z8ooHJ4LUtRUk~W7@jt@L+4ccQdDH;2QnLT00YZ!jn(IUhEELdcvtKCFl<moYx@9<9
znDa6P<m$jEC<1lhKF~n}Lx2_f%nLGMjEU>RhPy4Dj0k@hyBC$eVRq{g%iF@mi>zsq
zPX#<b)&$&(*!k7AG`-HTbThg~@N8Emp`|pzxa|*T<7dwig#ZfdCYk;c`@p1;|El&q
zxF-sqVF-Lgi-5aQ>LqXHLnNy`000MDEu0lT;{xfl5TXUP;sZ&0r7s;=uIME70I<hG
z%!{&jmMl1%<e=(a@V;$8fvO)c1cn8i4h9ss7K&aX_OmmfS7}e@^FNv#WdY1DP;Nsl
zO>G(WdqQ6%1Fb0B;0P^LHxpfHA^P|NUL-<>Pr;G{_m3qBietQx${r%7iozH`DkrEa
z5#%r_X)qvC#tw)pTXtn_A6joZUp;#jM<izphS+1rKYxDf@*5@!n~u;`Oo@6h(rz5_
zVgPA1q45lT|2K?TYAiyPHZ(IZn*&U3*v;CGXniS{mOBt`!*x3x&6txaR2de;>||6~
znH%kYN9pOE>NQ;ttVz-?|I*v<!f~f|G-GzNBi7U>yiZJcY_@PpAGj~Iz4c;r#d&k?
zsg|*7AJ2!|aZg36g3^!h81}^Ip;^YbEC0c?|NEE*VOa<T_J_8RlElalCe#$}r#nrN
zCo|~1#ZM1%$t-6pqFgnf$!R35AUuBfHc4-9xpV|qwbvRx6|$WLny>uLwhONa7O9W3
zg;kWieUHoVL?p2`6qa+e-r4EiaL76O8#co0zkI%--8iT%p;y(~TI8#<y3E-&yVdx2
z=P7~7uXtY0Z6(!v?z&@TYrhn!zHTu+KJk32hEFnfO1N9m{2qMjGD4sVe5eOOO}aOA
z#=iHmxK45a;A=161{}v9to>80`z!hPH%6Cfdr8SMpXfJH>Xhik+>R&56E)NCr++BW
z4WI2Ek=m+wYwH~89JM>^&s33P8?f)@zNoz{B>ZZ#jKlV+^vAo;f<bQ|WFNe~`)^FG
zh<89gAZ<^ya^!^2gu#((5?M_-0LfE;H2j(4_}4@uSCH`IJ-H5saJWjQ?AafHi~;0>
zL1X5?)S@y#z>uiVKrQ@>s5!C&s{_6=sRhbg6oM1X0U?`!+x|?;OrZdHM0vFvci}8>
zQZ-4=XV0lP#k+Bjz5b^CiRt7JU7G#*;esnms7ul-F1s{5aXGq-2JcQXyE}<2akZ+|
zb`NvL5W~ys4{Pa>lRt19j5Z2X?de`ctth=tJQz(P0;rmPiFdOk4e7r|`{4f)>jD9=
z0-O<AA>EfGSq9ZoOf=lYF&}9W(;CTyeaS2a%+LcoOWgvZb(mIjfG+A`Ae)0^6%}fh
zk(|1^m+m8jO7gFCpW1$<jIM)|AlVRufq^9jf~779cmiJf1-3Vq1O^BzNFYMq2MEq6
zv5|El{$@#t;b4I6mALzd1nnunA7<RAjR42$4pL}9P8&QHib(=B41gpVGK%q0!3a2&
zCaO6l+xNzn%r<Z#9tQO*G1}>m>j#V5Igalt^3lx2^tJn$dj;uy(d^8+-rtnkf5p{K
z;`@)vA+^!4!KlDbyqgcO5(&2@3uKw()NPaQ?3y-t3xCZRU1=v|X3T-kKqUQ{7|!@E
z!uE;vV)~kUM^zm^a~IfB|EQGin>-USm(;vCnWVp#TPZH>^}_ZI#n41g$*I~BH#Dh|
zed+#Z@cplu{g-?9&+q?!2g>O?-;*?~x6?k2v`0)&UY8v!xHq^XJG8lE7=AFeq=Xv=
zABrc28Ge_^?0Cv1)7T?=V<y~v$hhV(7G`sy@5_OA=eJ5cs78NsGMmfBOwOT{xa~Ny
z`-{_mOr30FpG$RKl9}_RCvm}M@!!{jhe1eidI_uj_3h`_qKAomSAWAMRMVB38AccE
zR(?bXU6VCy_2+!Iv;;!68=7J7qpz0>cDMMKJZdPJ`Z>W5xVVX*6-3PT63o;=5`cld
z#39>tX>3v<Klg+M`0W1@>@I;!OKJ;>F*!-=JL(Yn+!1~`{6{ogpCzdH=gB@xY0(gj
zPu=i4<sE;aiOfe`=gPY>v@1sqX3U($q=&ng{5j@6ULj_0aqntg>FpZ#M*Hy}{gbPe
z2kgK=RSH;0qO1W-{{!#^K)W<(!>~1hTmk|J7z4`|sA^{d`w9Qn7_v{+xPToYuD#0v
zrfawY-Uje-KZdf`YYj5W-NP&*pxF*Iob1mYf>owc^@D~FFtrE<uE)_yXhNqYs70?a
zKxQvWLTAeM8kG74av8WuI2t4-Jum8Zr&x^Lo1@x2)_iQvf0<)C^NYM@`fr%+^rhpE
z`i>`^ljnSz5*CpwxR~?f$a$R6{O$B$oO4~@*AD<?*i(%=5*;Ci_2?SuUpky=;XMlZ
z3xgXi8@z}wj|wDDj1l`d6o!F8;G}Fh@K<Xy5Kqxe;IrC~>|oki_XRUX>i9xv9-Sm8
zVk=3cO2FQ1=`B!0AeCmZub5F{gYXa(RWj-mf+@SFp@)Kd;DBTO_w+Xz9J0m>V-BYI
zXP>DBuM11vLte<j1w|^D6;-KQSP0)cG$jn`8^LhEK?VsN6}+Xtqv75{CX<d-9Q4T4
zT`(%cWlXl-OQah;HTNZCp3f^tslLd!Z2^p#{or52eB`_B>N$q1Vv8pTc;&UIvid18
zuVdJxq6sXWBC=_rtH$=WweOjk4AUx^9)MkjO+yzH#5dNCkA$AooQ4}$@YK42Vbil7
z-j_UN%j+`N_}3N3`%)+etP5sx2ixzRDGovt4@IhTK6cI`-#Y4hFq>oBBK4%TUp+fA
z6M#2D<X^k}Kfm``&wpNrs1Lfdo`Tn&g>u4#M<c?29{*X>n_|;>VO*;yW{tvpLPO3w
zV>eYWFIQ!KWl_w-MO#)S3CTfe<$RN98S%=&(vxOUA#;qXwR|n)O^*L6F|)-lbs*lm
zX8Xx#H_rb_Zo_t6i_YsP{Eo=3#}5sy!b08C?}0vq{6VLu{ayWm0c4Xh0N~MutWp&1
z0c&tgTx``(BN?4SONX`IBN>6t43aX`@1q{5iH2Re#|Ban)Y&^--C3zg(g^Z{uwTu7
z?{k%Cq&n6=j5ieGk9Deus40Y4Dw6#Szl1-VAFyc9d0D&`C&hE;O0UAmZx}795bA0W
z^<vyShH8|J^3o5XKoL8j<tG5Z2w;F(fT^H)K>%6QwK;Y7@^#Q&zn`b5G{|F3c>9-!
z3IZroI8-B)MFEEegd_kC=CcKugU*WqaUk;`E6w*?F9O^_doUv{bz&K^aPTu|{H=v7
z1Zc1%3?gXi?QIz-2q6JpQj-{)j!!A|c!_=;eEZ4xmWQ-l1UvU|?x2}At(*TbhXzfY
zC3#RaemGw+cw4n`ql<&iZenT5C$_kM=GU~w(mM-Otti|$>ub;+#lgT*nPRQ*CRa%m
z6M7R@DA$XkBg7yeRSgN^F?e8?(gv-hV$1=*5^8M;1DLC>26|1v7V;<00sR>ov@oc?
zyJ<y?dIb&-EfSFdG-{wVIrf6VR~Yu7(I5{fYc!If&~34d1_4xfUFiR;h)K=;!vTs#
zB9Rzd0H1?Vf#Q>t3DuHTVD<JwEogN}X9IUYB_rTon&32nvj7OIDD;p+VaGHf8@I<2
zxWAC!3urCac^V)&FA@eX1m{b8C>&I&;O~`CXP={G`v@=M=iT!-GO(o*6aGC(LtQ;N
zd<%r^i)Wm*9)9zi=DvD%YX$2QK49F=7yfLc2%dW&UBtfSQy9CcHbYeNgK6xYZN~Mi
zievJMr%NJ_Sk@ezGY%YM9*ktZ67I|7eGD76sHnB#S!&8MSMkKnclXXzsm#l*Yda<-
zYtO5;a(?o4A;<?XUE|{u7XS3E{dfA-1khY)n98ZQ!p|1hLT5`^1~*oaKVQ@x7jtZt
znMj%5WK8U6)w$G@B2qu97p>(Jycjcd%456+bL9Ye(&!=iv<}nHTEXFFZ(l9%>nf%q
zey3GuvazWSMn0EV?ks$E)Ye>&ZWV5-NS2;;zwu#_Dc?1%|MS?7A2RL=A;}RaIv96Q
znjYG(MI|1otRDn&aZG~8<h9fd)8+-{zEAbfTxAeL+GK~{0REe~uJA$7B+(5oJa`)j
zG?&5GQvEFf?@k3_8{$1<a>Cmfy0ime3;L^_EP&x)BV3pI9vO5tWnIKO0Z#yx#Sh2G
zWvP7|w?#DYC~NZtxjYw=ZaQ%@^r>O{(LXef9@|K2k6Je}Rm2g8Jo){uL(-Yd>%c}|
z1-2z+)%?vw4kATE+>|94f*!5Z&Qw&UW}vFFW;4bdF`g}#4tbX1dm2B~T$_Ftw2gA=
zKc|3j2e^R1g#*(Afnm_*l}6>%3nFshF;k$K$`SAjA{}lxMi)dSK%bO`BilaXDl9q+
zU8w=)kafyL0YlGqX`WLMZ2y^VF_eJ@RcK%?ChaphtBD2ce`mrN^B@G7FY~d%DQZq4
zsVd%bl%J(ttLNq~U*~OGuiB1JGO9kO>n@}OZc-Vf(`%S)F*xZ4;U_$GjNSIhk*yeo
z%f_s*#D)W^-{h!C7J)#ApK(G%*FKQ=fZ`5hep;w<rcK?mIzlydC1EZ^NARw2?Tq^Z
zy#V-QM*pUa;bbw**Puw9D2s?2gCi#fK_TzgiQw%+4YUyIfOb29N>)O|5F8@{mM4}G
z>{Wjg$8G_50tFzD;P%jn^vWnf9pDt^i!uPYP)$@7#Mxp1r2^^_AFxKipddm3Y+oLO
zJQ`8xX#4sY=#GQl35qQ_C@7Y(88p1q3RHq;N1<iOWXRi~jsc203tyh`l>sbU|2j?0
zs#k5)j~1>sRoVyglS)ITH;*#gR?@(W>Yuz~)RNIp%Vo`K>3Wndd%}=+rZa7aapC#n
ztkN5=dZspg67Q9@TmGR?pOJK#Ntv6}*@^Od2L3Y2%;@4D>pC}OSzle0Oi)oCmEb-d
zzrdVlPqp@On7Fxt_2=C#+TMA2`&WaLc4+7MYvt1*!n4MiD*i}`>2*1OifNnzT+d7f
zGRgi217rVZKL6-5_Rq_&-+d`%@b_UD^T7?hkY^v9F4LZT|I|4>{-jk^+M3|@qV~~e
z>Ih}ivSam6a&?$B3^PhpkZ-rndgjg^T0Dr8y|%FPuwt@Hb2n1=7hEvRn{{%zsmQ0t
zc%{*DxQ3&spW=NwdT8wOoHsL?yV9I$LMW0gxA%?D0mo~uJVhnGKeuT%Xr2^&i0i!Q
z;-!pZ=@Bh=a0B7C(=2mIC!tc28e(O<0K%xm5TFnVyoMA5&6Nn2hal%R!(ejY0^W@l
z^hF$iyiK`)9~>IK@S6~+)5-wV&fjsbagn8|r!X$>yNOO;xa@e^c7cguqes<vsJCPP
zn+v-Epw2aHdlaH@Ev2;q=Y8`J<2(K(`b6kTlA+{M?o5K9Ehu05Xn9#<T{>f+OYFZz
z(Ix>ogIZ%9$Dr{6Ak-^&59l-da#?ktdV6aiF_IHx7Fr{n|J2LqHFSjza{dX0S^<TC
zy(K3A%#+g(xrza?HZ@RX1_DAiAOVhBE6)T?{3cT;0Tpkq2quId+U3KV_w)o^P>-uq
zqLuj#E6*(AG$yQPEcZ`$-Au^86#c5?-oS3L(o=Wy!N%)~bu0-fR@@-jo0}nxrU6&-
zcT}?rpfb~#y2F5J!K|Y!!2ke*0MNS&$}ST1Gq2_{oFhWS#1MjN0+ohr%LAZ#f$Row
z5w!wHAQ&g439y7rO^AV|TJe}%wIb@uwSvb3%{~pbN3dA_9%Exr-gh(4fhGqy2+2i9
zg267(oBC6|*9EshgvtusDRgZHRPYf%j8lUa93Yf{WDw}01=zYr$C@PpMxj|~pw|bg
zCJY3AfRBI#v<or^Gyy_^9R>r<@<OXsK?!D(4blmWXonXvLQaZ?n;05J2mAyo;^Dpg
z@IKdb6IX^95_Nd-a&Er=ou7D~g?Nv<*RWb8CT+22DNT)%zQ=W3y0^kW>?Ak5vi80r
z-D2=~V5#Pvtn1NNMW0xHJJ7;Y8ZxDHhH4heU&DUWg;DrjJ(vEk*@>o<&n`%FCuBw7
z#E#r%%JNZPWHnvhw)l{+(us-HO-jrY(K)N*d=Xc=H;+0WoKeoE@VfL?(8u1qHOy}v
z(K{$5osRCipn*t!;0V6m&^P>_pFh`ttboKZYrwK44t})iE!SJ093CD@I`;OZoxrS|
zjwKm6r0yl*4PuJBw9{sv7c1(=s}woKO+FXsmVVVc>Zu$7q5&`!g-^M$5{Wtj<<lV-
z=G+#gSbJ4h?hGpOE;1M38}gPqdK%i^Z9G-LfyUIycS?zLdJ$f3y&Nx&K=^c!oNCpN
z2hp;Bsr<A-f7XoD=6axLq|VzbZ2l8xJ5ref{Ixk65NR{le}4p!wwG6_-zOdav6EsH
zOZOu`kYllpCTb(bH2<`+$mKw2kn6bq*RD&)fi&l(a?WW=+pe`z<(Kn}ucErf3cvoF
zcl}}jdulHxnf0um0j)|T$^@8+@)~UB&=veQ;R~u@(x}>jMh5CLDli2A$ne%A+V4R2
z(r_Fnz6{t)R5VNIa1sgBApgtR3IeCl0bU1c1CMe~X#;7rO1;*2dfUCpf?m+yjv0X*
z1mNz0+O!g?Ux#8DfC2^n0mkGCg1YaR+09U4nK_2JV=ArpzDWee8$T^LY5CQ;W+bM+
zOt9V^Yu$Xn>Oj+(`mU($>mWn#wBGgdUaq%jaBS+&W$8@0!T<))X`1b=D$P_#vV2-N
z=R0{(Hx{Btlyr~|Moj{h{=fFIiqhhN-$;RMQ#>p=UiwTF3QXojfV3C*a}E2gse&es
zLyxGmP0-UCeL^b1lueBO1Ohk$Xd!%HH;8ZmLbaz;0V0J4G)&F1HJe{<Jy@+-4=vBA
zs|yljcvz!AU9cWd+aUEFw63L#?Xf|A@Y-PhdEG&%>w#i|4uvYO{}4LU4|s*MLDE7t
zs3sezf)QkZYS7=Cz+Lz^tb+m4i%Ekl6KFt-ei{umfaAUMA%j{LkGv=bK3-6@lc_W+
z?gJg|oyeL)Pt1_&s&$6EXqr~2=GzM<5{bvLL)u<+Q%b$u*;vC|!QKjK^Y?_-A>=?*
zGEsmj`gFB0O{DS)UHQ*-r9~#C$K1?s%iQ$R&|9JJpFe#=)y4IIQEYR#$WNkH#<*ij
zM2ml}L3LznyQxFk;lgTRKSxTZY_9fCz80`^uS?8B7XbA?KYy*k!Gi~uCk>?zf4aY=
zZZF`(sfWB{Ee)fJNvg{J9BgJ+tz?tSy%VDoTD|J<BnRakl=|#3H~sd(h)+BwvmY%4
z!)jDwy><_0D*DSdB1b)BUp!ehIX9oY)pTn8$<T(+y`j)-<tMBI8*UT+kHkE-b<S=P
z$I_B3?5kQHamV96Ugy8%&tc8><^6wQ@6F?(?%TKV&k#x?HHxy8$das;eJgwRilm5=
zrG-jKF+*vYWKWVUOQ;x%P+1E}vWB#feM{MwEWh*p8LsQPukUr;&-Z)(ey`W_d+z)B
z!%Jyqni=yxKgW5T$8nr}BUF>GrYExdKsV;b>*j+<dgkI2cp0MGCbR-EUP&(NwUm2H
zz;rK(sIMg(gQEgGgbA5uDNAGTEnC23=;HHFaM==j@4M3-1;=W)MsD>oD41xhn05A{
z=V){<wh5Ec4Cd;n;#o=`Q%~veXw37zmD2J;rCTVeeCNFvqz=df2@8&@Hcz-%T<h;q
zdlrrz930e1%7e;`6YU15UqzgDpz&dlq6hy1@(DOvH4Ra^<N;TTr9(&EHI$?ZttNj5
zG9Sn%nX-n)r>fJFfzp9`p_>lAEC32FJx-j<{p&LcXtZb2i+ZI7Ttp_9<c&|)$OI#N
zHB2lD%!5#FVS4Y$fYa6{na}92w@V-4`@wAo+P_)n;9kpok=$SA*Ck@spDXWdm;V)i
zs^4#J+imOpm#+zOQUxlZi6;&^A_M93CZF35JdPg()b%pS_P+_npdx_+)~Cn=DhS|Q
zP)yJh^T!1FAs7~#5E`yQ=p+`vAVoN_R6b?hTF&X(TEf>zMhD?F0EV_8?JHSIbAvD5
z87>wgGQJBz)Pk`YcrYG^Ehhh9Aw^|GjS>3H{M!r`aDP9TTnKaGNNfx3;V=MVWKgBt
zg%Sv2TBx4M2$2Dm3OI0ghFTloKs3dLqMsrSjiy=x2EYoW78<0bK+XvfAYZU_Y0{oh
zpHRd$CX^!anUfY79;Z|oh&Yt0zK0AlluY7sePHO!?_Xk_ymE=qZs9k4qV|U?L-l6g
z?#_@oI-?HDmlKJQ#Fw5ql(|{UsZXz^@&2dFuZ@;lZnGUTgvj0FPxx(4uTZiRdTBj=
zYVyWBw#?d>^yHM_m*n2V#$l?mfgq>77<4PCN7?yf5q%y>$xiZVvH5N0dnL28G4Ezu
zm?&4J{ls?K7bF;I63wN){$Ek;;^GV)3$X675&OgDc&gLJ)+f=4o!wfNTXuNR2=fTC
z=5)Vil{+MqeKEy~yYkEi&)hefpqeFPUL5_fCdJy{vagGe&9Z#>fzWeNkC@h-^Vog%
zx$o|tUNNuS+3a-N?DKk0fcb98oZ}F^(saWHo1TKxic<xX2pO?Q{IVOW072^s!6kp}
z=N}W<BOoXVH-IL&tn{mM?640xp-66)`T^&dJ;G-DLh;fB_o&+j;y;cbg@EY~qcGS7
z4~4#KJoj@b%WkVmLMwEF8`H582luN-WA^8YM8*1c*JEsbD~zehugbzg&4Z@*vGTsZ
zBGBOzRd0GC=`3gJ$0i`U#38N%SAo&f1+qGRfmM0{Q`9QeSQAa~yt#N7WD&nL#4-^m
zR<62w^1w@>vIfG5LVK7hx}wt3DhDm29P<f<1aCETic!)BSm0qbmH*X928eF{Gr*b#
zWaeEWjGD19`V_SUZeZ_?Wp?L8oj8#gRdnRW@^l^-Mgu)Y20QoK{()$o+zsPOWUc%u
z0fD)xFNTp9y5heCf4N;Hac*l#c;}6Bl?Q=tiHAb7#*;^k$<P);!GS7ZESzL=8lCA%
z9W;2Okz3GUA<?N$W}XuxnfaW2d;}Pgz_SDl6A0}6vjm~y7|Si>!)f*EtAm)UFcIm#
zQm{W$Thjoxbkm{O&27-!SR9S?5XJ-;T40qb4>}oOm5l*E-k+^o%ZPRxBHb5w?rl)>
z2&ideZS{hp0%x}PQ7r?eY*F~Nz--Znz^ef(mWd{M5#W)EjioXBf5mIj9wRX}Jq~#|
zpwwE5>@hYbpuz-`BJ@S5yCgT1Fh@V+7Dipu<-5Kj_ZF!$L44xQbbyQ4;CF@T6_)Rp
zS2(V5zEE?-f0z6|Q|YB3sfaRY&X1eD<F(;gQhe37o8^Subl#&Go5QX|YtJwQbQGSW
zzLU$|(nTpXvrOen(jHb-|K?ZrvGUtPW!^{i(`(SWmI5(V)BmfO0L26Z7E>_q>xFV%
z=}L#&#7HaLy(Ua@745P$n%*zx_14an<GXe0DW*$}oW~rfTsL3zoL%mydY)A<r>O7s
zrH5YR!5Q<~ye-*`Lpj&$4xEH6`nN;8&fJeAQufar5d4yt-(Kc+La*JTK8>1o;L^k@
zS^tYiB8R^B?iHzLw|-XhAf-e@QDCxvwWUSG`-Cr_K4fH?eh0O_T{s$VYRKq5i-MN#
zt;pyVdon0e&(#|1EPQm($A`H{ohy#QaCkc)Efd35SwB;=H(*gZPk;xB^)n;jj{z75
z$tFzM)wi!A@8D+5-0P`jwAtvn=O^aZuXh-_xK4i%d{?{eK9hX%`C~Dkcb`lXyqn|5
zor~ys6Mg^NV=nt{1=Kk{(s%<MnmkVIs{dCErP-@*OqZO*>IfozX6Wt;OJqIjuBF0q
zCCYhq1MUd-?o?cb>Pnj`qC_7Qbi@yF!e?{=aNR>=J%BLAg)TrR9SWsEsLK$efhLMS
zr4}ZNb>^vZX8^U}2ud!N;ZW9_OWL_dvz}zP9}kwjR?or7H_e__$YLJpKHX{G&>(2{
zEBUsChtwumLn8KwiPx#W!fai@mt)%zsn6rw_nzMp8Q`(Kz<382;{!$rXQLv2Is?N^
zJ$W!El!{s~2)EY!FkJ9lHZemdxRB7MSPL4i0lR)A;$rg9ys4Lo#PKw8ULD6cWPF_H
z69}ahAka-ea4<27IOzcjJDUJ-j0GQCN}~)TO&vgHzqu}zMVd;G?Go&&DN+j)MhfZ%
zWl_|;iKbdH5bDs&VW!`Ki83|?avseNQ|$2DP&A;}2&aHqUX7T!f<fXZ1&pFJ=0295
z4mFw#6aZFxXd-?^PE?|VX-jCtbpcC?!`23nge-nB{h)mOdF;kkC7w1<#r@AJwmrUo
zmrxdd?V5FoF5fLXV^-&Y)q2}{pPQK*?4OqL;5e$~l)w{lUc2Y&wTB0sJ@%&EcjOOl
z?h*{!!Sms&_DJf=c*<1wxW&+q=Ft~1%MX^{O+BNW)?R)+CW_R$1?02z<J9}{4BDwA
z#V$^Hn+%!wtpP^cYG%s@r_ODad#5B?_OP*hY!?(^7hafh{Z>sEtLlH$chhy2TD@xA
z6F1%ZNh=(7ctlm$in-sklly#;FldNPH<mO<C#g_VzF2?W-a9Dh$o(rMS~*XwndI<e
z?bOHtcdsv16*mSkrSnUZ#*SYUp)RM^<v248q$ms@8ZD5R+&p4*ovW-NL!ZguS9(ij
z-dro}ulQb7HUJ+Dx^#9lC|m-NXbDspP}KY315r9WaRXP{({%KITeaB1e~u_CXc_@*
ziMTivJ)qlXexuW3)Y1EWx9Fp>&uqhtS<W#-enrP)ukp#0zvQ!7I*)zyJ$txun+{c`
zx^QRo%e66`DdnspqhB)nrq6j@5$>pk6NX2kwMvR$T@a@MvL?erw_)IqI<{|xu?$FH
zn#5*F=4K-1v;}(9`WxsI>fzQ*%8su>Ko2kV8;TM}79)UBUBG)Kg2l)MdPb1z1BPO)
zw~H_-M{OkNw-_B;5&8lG#Tb|@H&Og3f*wi<X8k27>WHbwxqL?3rP1>HuY74isiC^%
z!vU~Et77A;)jHiy{9DSZS+m-u74-%N^kr7*Dpz{s9`~HmpPtK<f26#M3qUAzQPTp^
zg$yWUeDUbxa!oKKLbC^hQDX=s{-hIICYx!3f+3N0+^3i<qrE^&osbLA3j$+fl()J%
z@QA_?n1*Pg-?Uh_aPDZMFQ`_}4>1SD#!=ikW5Ep$8NG?F_W?&DG5mrPQvb3pU`l;N
ziR=5Z?<BY&YAm<}ZVWZ!CxB5Ar~Od<0M}tGW}*K@jYTNOpWW_1lr<g)3{meTq7+aG
zLoH<RRUk_Wr`Vt&A(5hnDsBiTPDkoMhzQZN`Y+MT_Lp337Xy~M#~U`B7(Ucs{2t~p
zk9WSJwk1%0b?elBA4^rJovT&fmi5kA)>J}k9~aZ`wyBTSrgIfu3JNjT29<WW^6yx8
zVT6ClaJ5tisnB9>cU*yV+dx6IR7sID4`2K|ws|Us^onyHJGRflF;DQ}m~~e`tV05i
zi1TpBm>Mdg|G8f~Ysmt3jg-m;H@UN(Tl>bpU2!Q|uk-FAB;Q<k^QJpuD5hDlq`<Z&
zHB-)dRm=7%$qVc$Ok;90x6aw5KPj=@pzB7uW>)n6>C&S)qTx5rT3*PGQ(FJJdEl#1
z<Bzceep7d(?`{u?dQ@Zn6JslWDjZjE>Q&vXbxx6cgRAFcOZG(zWj?GwAA0Y^#3>Un
zYOheB^vIQ(7$peE4nlD%N_ap3z6n=mJzipZ0)~JqN`;e*QAc+$GrR=9)@=YG14RG-
zZhZVNec~0==^e&OPJ7A5>{yfDJ}7c&)=+iavfuf}(2=eaR`)rF<M%*?Dj0YAQ6%=k
zDC2v_P#4v!JbZ2xWKzI&Ot1c4jA+@yGf<a6e2~x&(TCt|L>V_ANO%>YUYJCH)9lV|
zrkyJ1QERhRF3C(=C|Vvh5CD6K0EvLT1+4?vBJE?n)nMxb1feA~(Y6aaakA3gX%2KX
zDw9vZl+mai`a%dji>7UQr%_NyF%NntJ$E9gz)-X}u``*!a!9-|tD5@R?=rhlNxsQ+
z#IS2E@aT6|Fp}Re?y5~VPTCHy%enHJ-o%rpX4ZorE6rUY)f`Uk_yHL_<4g?^$iiD_
zjDBV~)`4~o?Fse$BkQCOGR9EWQ8KPZ0D?`Q=2Nsm`8}qkh{vHPnxc#!tWqHa&JvVU
z3+U8{v96|t$|7ozryHB0tp~(A9gfwq@R^|6az7JIiA42%fVJYGOshd90&gX{BW4RG
zNj%{PnJJ-YslfwNK(w#~r0;Pc#X5?3{W!F?9!(EJ;OUWNASD=Hh_IE5h+4^V6swUA
zt>mKcDn-f{^~pgfV@(QfCCps|{#AaYMrEL@1wF<hX!_%Kjt`e)hC(LUN}EqUUbb>m
z^ugkQHs2gl8k4|h=1$l54on+!q(TL5B#KI#o9-jYB(d!~E2{PQV5FsUhu1B2mphVM
zGSbhwa9MEsu<bPC%~9@uL$a@0$47Kuv1N1Mw!u)%r$>hEO6^nw#bEwKwYv3$fuvw3
zIZ|vMCGunKsYzAWUtd?R=HAP>$G_GbqW&%5A|C$ZKeqo95pkf)92B?_K1!D_Ga1lE
zn$%Z1OuFQ6`SCz`c6Wfd`x4I!rzSZ(R}Vccx%Mi0#dH7dB5vnYs(9Jt509{q9{fDr
zmT&UO+Vhp1!O`n?;%1GClr$?(m6IQgH(193&rPi{Dk@$WSZu+8(iN~4-Z_<<7`qc^
zp-_X?T5E=`6*Ns13$uoZL+nn6BolA76>*^yJ>1vxx9y)fJ=LhYf32aG<R{f1flQ-m
z(=YptBduDj-ZCx;e(F62wmkBoUtZd^M`~VnOKdXdMh5{*$IZtYAU7LDBs{O(1T9Zw
zo+xIF>pEY-N)uqV35L8`m>jE!YYG1tSg-5JnSY6ja0b~jObx1sd@Pb!C4juZAc<8J
zvvzXh?3M*M0vHPYKeW-^0xY6!?4=KjRHy}(9CTQ8p{x#I;7NER*K$Kfy6o^uRe`4&
zysen(sDPCuxB1?o73CjN+rQji$w*%vb{sOXtnxg!5>8j&s4#0oea^O?Kp5TQLMfc4
z2J;aNuM(5jX^hF{ba{Vpbl@~*L@7JF8Nkb@59I;U7l9%IUVQ0jVuA%6Y#Nya^bf^y
zCtk%+I+_X1p#uoY0Q}b4L(C#;h=^T6$|k6I)Ayj3sKJs0z)>9Z(0PGhJW5@Cft${V
z#h(N%4kDq{7d@O#F<!1(g)tLgf7UEuq=iP2;U*}vzz2B_2q&6Q3i>+ehvB6V`+@_J
zs-1cFVRRb+Ku}{LGDsNt+96k$NQCH&zb-k8<;P=3p#)dVvKh6f{^lZf$pKFcstDn8
z-wPNEt87a1oBgKEyD@w1eWrW8-v}gAU#Jk>Gu>N?*4aOPx9(f}@F4bGbcR}XJSgTq
z$K=FX8_y_<Hrwo)>CqWgfzi1&r6U>^7nHLOMUhtLLQNT|8~snkKy*S@6AzAv-KOtz
zu=6R1vYWhT8K6-kv5iZmgI)HDCqe0A^lcN!3b1=2pTS|LaW7@_k=reCD(<tfN``I$
zYlZHv?^>atB*}DHpf$T=d%)|)J9}?wjUUuE&V3m<7}2)x_M5p8?wIbq{o@1KEa^?|
z>SwJ3tY6P<uaX*hCpG3=_2}I20l^6dAqH`d-Wb)xFF;&iydl)=8K};YbQ6cA37-Zb
z+2(&$k%Wo>C$0IdBH;*$Hh^6sfCBl9#aOmxtoTt(-K=uE%f>w3$$FUeMfVboZq=Ww
z51myo&$>gs5wkZ`t2**!wOQ|)NTzLle7m4a65nhITrYG7c_QHKlQHD3Wd(&gP(>Nl
z#(5%dL0r%TqHVJ<u)i!^jcL&ia!ITn^($%3+fZSO4Q;Iz*p=6EL!7kP(0fOu>)uE~
z($7L$$V!C6Xi5V*9rX~V({RrSvJt>-2vDQKSj{fNrC$f-!3r^jPky*L8P;?1jT-sl
zC1@3-omn%tbxZe0c#FSS{rqxt(RIGJPy74R`+=?xmgjp7oU#bfv{82hkjhZ>d`e<Z
zJM^PMMi=UKUsGYh;)jpcait~b@?m0CMr;OQ5u|rDc$b5eQw``Qx4+_synQ7SF;_77
z!FClgR4O>}FkK8qpbBPUO(pzbP(BB;bnQe%u=m-a?kX-y_Z9<Y4|s;gO~uj@YmjXk
zJp$dG`(PTRFzZ0D7=W8ZKU{n+LgTMcm^Luj#-{Dwx_S!LAWS2grHf)93yx)}ws;7j
zD3&giB|;$rz|tSsAsw_fE0efbUH<Mn$B%+1*KW>LJa#&?KI*#vcdy2i%i7-=(|Zq^
zJI-U|bw5R$exASH`7?$hn`g@%_B86qQNGtvJS2`yUfV`KeqCng;wI40?2&3RaMU8{
za!IkL1U0@U=Kjq}g{PzaO<BuzwtjQnWVW$<Jypw8nytfGqaeo8@ng)3Vm-YcU_DpR
zEnI&8lfE-|oIEX70yK32dz9*~>{{DP$^-X0isrE}$IH_h)(v_0o3wd<=uNGMMT*x0
zuFstJYV8u+eLx!^<L2{oDhGa+c&^dDI$gR|{5y}Ar|RmFv~BOa;=}aVy+*&>iEsHS
z7kMl~3XBh4g(OjBEzs~%U@m6mVakr{Vjsh%ABWf=yzn4O2PENWz>k_P^tA@irndzk
zn#+c9H?JQhPHV^*w&J;4(Bo?^X@XxGU7_Z7V7u3AD)<P~%e-F2nRzkH%x8~8s}9pt
zBO{a3;LF`@%4@#qa(O;vO8Lw-n#Rdup6flg-s@OXCv*M!!ImCa0AdQ+sd)m;9Gc?9
z&R#U$MzjbwGff@tF`A5(sT+>p*WAjWv)qs_1Wo9{M+co$aEvKZCI_<A)<0pOm;F{r
zpNNl=2hQkZW?{(Z%;d2bAPXLy3Jn|FjR_^4pn)0eK7rKMKjNA@+|`-!i9!I7VtjPy
zQyzFzu9<{4_coeQzxoTgPSQD<zIZ}k_TYBX-40uQ#WkBIgg~T@RjP7QGZSZIuA_6>
z;7dls_AK;Y<{1dexs_}ryW!kU7wR9KmP0Afg)@E|Hl$o1O<4yyt4%~ACwflCig@BF
z<02KKgOkiGC=)>hqyng}sB3t`^eMvN7WACZc}e1)lEo=MYKfBUf_7rKXAfu|#Ie$(
zoGC|fnus5hGBg2d`5#?&m<*Z=WZqR*>p~?-7IGcn4Ppd{1{$b)I6VeHYQXk4W+!iB
zAYd$E7~><!e=U|s=Zin)e@pL^68wCf%B$H3B~W)QTl=bM%Tdq9bqDU*g_H;U(DJ%x
z*fP1oqtD-+8lY;g%E7}_)@^vMB>xfn&FE++x;G^8_BjS?TNZ+KRqh~LpL=hKxZuO7
zACsSF3j|7LPcbom&8V4qCJ30OPzA~+1;^w6Ip0GzlH~(<-MueKeDmCCvgss6AbrPq
zCE*zhZ)>h{0RsWWjPthp!y|)@%wYN8FP%y6`?Z;~8+VAFJSwe2`1STO+3WFJQtP&n
zF|T{C+)o`-Z*bnPT0|;Ry?06dooD`bIroBd%ZDz)2;SQH%rHPk5#-keaz;oQwLx%Q
zL$u7FOAGEtAs|d+Q;n>4i!?v3klcr+_M;}QU;@_Ba{SC#qe(1;-`BE?Ywg_gnEibo
zmbYxP!!6&e+}!4*Lsuwpj*Q6OCr{JQZF`eUJ;U&m&-wPF!;MuVpg9aRg@v2Fyvgv1
zM;nr%$zgP4qC<)Dj!j*@y^x(k_-@E;Y8MInI~F$#VX^1YVO;{X1pN0(XBT0LxIV$P
z5_%K2MEX2_TKXs%V2Ta=CjbL<7*ycg13|-t=F}piSbQp&d_hy}L(g4DuH*+55tJP6
zi2H52`BkwkxmA)+T<5Ak%{uR5*|)>*r;^OY$c}-s+3~Eiced#yeetpxSIR`}sptK-
zP_aNoc_>qo(102w2#k^^_TjV%;yDt-kOhvP=Hb|x@w)*I#HCQuWHGbkxWEOPIUGDf
z-Vwotz|BB*Oc(*K0Pup+7g-$$uXbW)Rsa$Ncfci%CW{s_IyRMXmcXrSy@8zG69yFk
zS{pJ{EH!vqP(t7V1ECv(GG~m*ZA9&%ZWw(kP|SZ2|G*TD3?`pAAKxNqh1AZ_h#2VC
zL>O>>9f^PR>+5gvLu5AIIH3gsQl?;u5v`U*>K~T>?s?5la5~fTJT?0M@lPkk?RD#B
zJ8iUwsntfTeLwq~#r@+?eu{nJy%~hsPu?z_bDR2RebB2SUuf>*PeR=hn`5=z4lW*3
zCq`0cD%d7hc-fV58albWE^%Ia48nj~=Z^JoOHjt$64k#8y7d*b@_qz0P^Wg^;Xhm(
z3IG0ohD;TWS7%&zpZdb8aDJNz4_nf<8*O<9WhGc!tmQtZYYewHVv%0TqW5_ng;PIP
z+(=HYy{$D6{9=kv>e7)$#@zP)VZU|jUBv5{e&s?QR6tmozzO{`N_WgxRf(+)+qZS#
zV(@7LZR_k-hsVlGHI!Fr52c;B>Hhhs<&!Vv1$p0xfzqY?P`ZfBY=M5pVt`D{au{H^
z&<fx~$^t$p3^oa6iQ<s$Nd&(xV~u`*o(7GZ*+RoLsQBQ4$lo)=WwFrThob}=Xs^IX
zA{!1~UiON~>ZoUS84~bWX_QtlH)%Hu&EUNXBBj_yNe0g~t0$$VU=2)XUz0nv-*peZ
zxTe8lqc`Y7U`0nEoJ^Gaz5*P)j$iA=Ar2W<paIhhB?3<jVDixO)TePADM#SS{VGHa
zl?Z-uMebU08h>0%R;*3Nr&8YD26WM&4`Kk?6W_4N1Zrw1Hy>czPZA!Yj^K>Y?@%S0
z(J|IGnvr2A(935?D-*=xw3xJ_fX!~%LP|XQP%LyZK=|PHX5P~yr)EL}W+R@x5^D%Z
zR=(Q}+HYt#s88iJq5v$>>l}jMJz3UAFmR^1EWoM{Y-}u0=OWwbs4#HHeP>~mFh!Vn
zGovk!I6#5LMhjuaP*ZT#BFb(87Rj_4mB;@g<5}=)Ff)4!$wN#5;$d#;qs|JoO?Y|}
zswQM0qR0%u2ZMA7h#K(g(&5i)LB!LACmsK-ipLH<ANX7z(zqaH4<hH&xgjY}FU|+#
z<Nqy~@|$cIJ1y3vYR|WtXMNkUx=kn2om^yXR(p2|9V8jF4^jEJ_XS)XyQ?JM#-q6R
zn#$40UP=1NXX%@~1OiH5?*7D_;jR!k7?tPYwK?UE?Ln5k^5f2feO@<G-XtoEsT$-_
zrr$3Qw^EYX_{LvBIw+7NTFoOi^;PfvyV=22-M6YUO*9$3|NW~Bt=rJ?Fs)sO-FW|b
zb;HY;ao>h{?CH#=FddhP+Q&y&Uzm5PCesg5&21CREBP|l<jy9x^=!FJ7iPi|pzUSU
zvnKCl#I*(~7o($8s546@y(CI!jl`1c8usaT7qnimk&pGOt+9Ucdd|=z1MhWo-io{i
zu^=W0%LIf83AVv8xM1fKS~Lb+aFPo&K2ZpY*ro$!VJ^}Qfbqr3S`fnl(|d<E^a}o@
z#g=(%!TmR#Tp_3uLPuV&sdz$Y1tthIBUC9h?ijxWJGYZD#VgaNL*}stj;iF;D$XN)
zuB=~;=j=1YdR<zIC@SsriJTq`O3(sPEPE7vc{5wV;{=?nNK;qfS=q6KLgkb~9JN4P
zI*qf0{s?9(@zGgJG8_?XaS^p3pLQH2+eSj$Q6`I9Iz+Hc#54engTY}%7Afd;FaR!j
zG~fbA|3Mwh)L|T=CW#0ZLbM~S^<iXC0U`(dW;%=v@=lm#+nk_}>eCCYciq8Urds;K
zSjK9{>l>G?EK1&+9L&q*Ck<qMykh=ZyL*cMW26lyh>MfV8=i)Ws1*@Ir!{fJfS6e{
z)&cH@pO_hrR4sbvnl&j35TN8hFPj5PGC;{wseqW*-{O-F(<%p<`CumD@3<9IBsBGT
zd_q}jdJRiOh@2%rfx`p+O?VPMP68^p6<Ns`_~EI-YYzkQps|cj!f)Y8VBdgE5~flx
zDR<!P6HbC)XB*QRHLlnUDEpc;kL*vHM+}q4=i$Mi<+il^S2~TWxwo$dWT_sbt_oOr
zX=@!v%cXI}#~r-)R6vEA&upH=Jau_oZ)ol96lY7Z>{VKKTsPl0%b3TM%csoi-d@-?
z^CR+`*CYQ&1}833qmu@6w{PyZP&nOpJElSdd@C={eEE-=6ya*XlygNJ?Ny7#+9f!h
z=CM!t!u8afr*yl|tS6<jt_F!dU4HW9om7&IyXu~o+QS_d`%UtA^7n$p=sGo^|A)J+
z@8^EiO$FP%xY~qdCf$#AwRHHvOsKl57$xk|p(?OcWND>15&Mz}2772|u7hR|+s%vu
zMd)=(1XcW)84ja1hn=znDgcgA6v=S65}R1R8hC_zj%p=D;|=SABefj2Ln{P_`4&#L
zKovoP!6mnfAfVjFSSD~Nj4}SCT0b_ARY#;dlS(b-YOZ*wsg4Rg=9jY8yfQ~LdR|Oo
z<n{5Um;eAtd*B9~gE12^{(cIyEs8KlHY|f>TVRuiKoLL^V2_Z21qn99J~bJAQ+)h`
za$<o>DU$_%`oc0-kZmZyI5qZ+5xo$CEepo2dfK`NWL$-{*icCTe!#{+L!VGq(f#1P
z#}^&~zl%B-AaY<yw(Fq_L!3d6ixoX1+2=9O9h2s9=Fu~|6tj}GRK7n6n7I%zmNvsS
z)EF<M*VCDT7De}EjS2Dp$(RL=6{zH>AZw{jaM)h=taibcg)3le;B93=3nO4-;1lF*
zcSVvFFpc1`MUX`VE-6OG?XvKs&=dz;2jE_~hk={`rccP1g%c$pRlbenFz{e>u($$9
zkgP3GgrHbL8qr%}x-cJninAw!f;Q3YRAWRnHZ0`mB6kY`A&K;|c!usn(UU-00ViT%
zmO;S^;}YQ)FULAclfYb+1d${-=|KBT=bMP?+`ptZq0gBVX(N0qxba-@mDI|L9UqUB
zX>4L(^t{ZW>g)L~IzG?q^iHL)x-wZqQ;Q~r^uDF?j0NhK#8S)JRM{M#hw5K>@yy?b
z>BL)BRhGu~&xfyeQVpN<MUcb`WJc$RuMb7`<lovh^7(6A;MqiKl~K)q3<nFl0uxSN
zB`cj8zxAQ=&eh`sYj!JlwB8=u#icqq{&uSA3}sBUdAa$Huj1KHR)l}&{}P~eZquB%
zbA{O0<EsG~#XDc7&CM3jPaU|GL9(nob}ZreopR^N>zaZJlxe$G+KxVCqe5@!KAca5
zPty`2(xhM~o58craMi>RW;6~2Ns;ZfHEIi6To?{#n!@g@B~X1(8R1aF%aE~_0}!$X
z;C+Pn+grM{`$6u^<$7Z#E?|hKn1L9!(w@8W!H-eQxuj*7Xg%GADu?B?lIqVk&#&tz
zSB_c-#sh`wsKD&by{}u%ETfXY--wAQ39UUyqCn#EJ3Rs%pfCp}j<P!HpJ1mQTLGn<
zu1|+cA0a}$1o5_xqaTPr(L5SqlOuS3P#tWIW<k+I@S|p2tr7)rLNPX(-|7r#gsbI}
zph_Tm26EHH)d1Asx;q=;{zH!K*~f-h9+quj=kUBLGgD68P0!K!bz9ZUrpD+`s-p~4
zZHb1bqaX?v%(%{otVN!>$OHfrkv*skK>9Wqqv8@6=GYBE5(J4qKw;>5BO?VKq4-CM
z3RbCWi(pjjp7a&qWAe{jgP4rp7we&ZL=A(mA?t4($QA`^TUd4&!AMJj{KtGCp||*0
z8--?Q3UkI{E<j_)rwXCS?SF^|ibWYhiE$L>wUe3((9_k2GbqxLQ92}WETBfL4=;up
z@j4CI^#BuZbs8a?0%a<cRd^i(+!40D{PVH#-~Btsxb{FDmE=4l`OG^!^VzC$r>{9o
z9`jgxTKrGyt^S@<9|f#FdQu%(Ast!Z`7N2cC68hnRNv;Wy(u}@xTH_Kt!`G;tUt%T
zAfr@E!cnNetdCiQNzZb|Vm1^$%>N4b`9JPHhs!#5hkw^{x<HpPqnq;Jc7oNH{3R~0
zho#2%Zl2y-ywy~fJBLEKEn;{;pp)-<bIf?QzuUdYpIce%c#1E@lsn)1WU4P#C04#c
zTp+{V?sK%k_5zopx;;ImVn>311+-s!|Ma!(!#!bx2D1bBoaLZ)IMTS?HphTy4LJ-B
za}%|z_`=V>Ehf-~Q0QI><5`Q?m$*PbM2dixu34DlYK@+j2%d{Q2)xx^46b>!%rkD1
zAJfcks2phm^O%OyEBd^X?yu%FTuF^^@xH&!wz1}0f$!?U^p?VFaWlBmi()L7`rg|W
z&UiJeh&HSSHH0i|WCb^tgn2N;oC4!oPlgLKG!oy+<G>fCW-Q`bxm-_<rho&gc27DF
zDi>yCT#1`VM{(&z8A04;YLIWi1U7O|X+$#8s9UhG1AnR336u%A7$iku@Z(s26A#qp
zHuX1vV`@y-81CLOr**L)fix}dT-RSQV_Wi4-`(gJ{XE8PIVRoGu4?pQwZxUzv_y*0
zcdwDva*^N+A(+EjJC)E*822K!$TG|r7vPg=^ajqhqFqgZurkA$GhxqCIu87iLJcZU
z{b#tKW0ik4DKV(r^J6!J%_Mul;SGTm0Ij5baZw4dC%}{x8e?%tcZYKw`Y>Qdi##D}
z1IiZ$yRk|@0>=zh0Wq&AZx+aPs1m{`RacT8t|WNs7;0pE0qMX-Xdck=f`AwiCP1b9
zHot@4_5bY8heCcS^;<X1V;r(s;?Kk~gFn0{y-W-xwdiGaeTe_|#kePWL+u!5G%fhf
ztd;ZxIu*U45lh|{rhPH)S(ChB*5*xZjulUhhE#V|cbHy|NIkLnE^|7+kMz<GcC$3+
zsSl6%@{2~a+I!+CTL1Rc22dI`YV2!DTRd9lr2F_jLXx<fw&{+yk1(eEab}VRNp!F!
znaVXA??za2Eor~l;lvz?0ON=1vliy@VYTB;4_yv6eoGm#+If9C|G>Va3CpQGc0_%8
zHPTP9#&-caCBB0zM(h#+*(<g%e37PjZr}nM6pp%R8vN?bgo<HTFi2+OqI5KWqa_~W
zKmzXj$9`rxOB~*o(4T|;#f^x8cj=zPAFBqFKs!Te(y}o!-^_(aEG!K_{0cfYFwMH~
zO|Rd_-)pUXb&gr()AAWkj}0CX#$d^&xVK029y5k<2|rJ71*l+Y3O2A}%NWv8_M#{Q
zuxC4tZ6Slp2naslE-+Tdrfy;=eApS1>h_ow(r#l)2r-0c2NZhKD(!?pkwEblx_}^P
zs3PSrP=#<Z!4Y}_g}R3ppl3x<5BeBYeCW=M-}xA5`KlQ_ZeK8`2#(CCdo-2uo$A2m
zZucLJDG3>#HIZ(rKzxO@mKp*Ue;_P2z#>jK?#7ZJv;aDm0Vg+T+u2a84I(<deYZAa
zXwo6vuLc6)!w5tCCI$HeT`{u}ssZ9FAj&4m)NWykFQ_wsbf5(pdeaQtkoC;X4i<*k
z1i%x3PySj9f*T5$kl<_&Mp_B0qao+j8)7a{;=NEXI4ZZGYc!KA(!!a-suGZ^;QSQg
zsQ%S2LfqwA%G<{mzS;KX@8|QlB=%t*%U0o?^qPuP3jX+d(<$@4%q^2Yw>4aT1F;5o
z#!h`_yK2%LR{rF91nb2kH<;%9sE5t`#ip|P^qbn8%nojv>yOFr9P><D6ZtD~q6sQv
znoQk)0)eq1^itMH_V=~Pp7F0EPK0-;pR9H3IS4e*6Z4u~KfP`b2b7A{T^yn}Ha(rb
z<kp2zW1~I;&mC-L-K8QV(c?3X{u#VwTU_Ss?|eyl&5p0i^-3ejz=?-Z2xJ06{TD;O
zW1ocv<e!&{vfTewvG~lc3z_v`Z3Old>_fC_K`5JixjF?jEO_HVVRSxUN<XWzx}{?7
zLHaez>D@bi+Isr4O-hYIENP*=H&B%4Q{UInuHryXz@;9Yo<uUpOJNpZ`UY_g_^Kb6
zqX>g0BMaLqUxF5q{~0*IB(fes35o-kY+oQZTtQLRDGEpRcg(N<DFwLUQSIUn@rAz7
zw*(RawjcsTZ8=hBnuoYwp4)jIRvmpTi1Mhspn6@kW5>mL?BKnr#*8r1szX=aR4w3!
zU62x=ki>dhKuychabJ|W1_8XmYRM223g?S<Avc92K)6UFrG=#)#GK;c#>Sa{VaF+i
zMJ0eBpbLQqarF_ATuTQNRgpC`X(%nx24=3P3c-VcjU0*q;*9Rmq8Xr0EXp?U72qpS
zuL5`hANZP>x!``Bjm1d73f?39acHuQ`+g#7y#OorkWIZcD3*-qlL3|Le+Co&;`TDV
zY#-V7i1S@P+oifuNhY=``RJ)x2It-_$6U4zO$=nEcm}lXdY15Q>*EgN+`hMO3+L1e
z6n-(Qj7_q!*NpGKr9IO?^8RJS`e8g=nA&*)A)2^+hv~q$e~m2x)ch~8EdM+1M;G-T
zd!zi+n%BMbXCJRv!^vfzc?S)WW}`aT>Nv)}rYbZyS?w<C^p}49ix4sST+8vuF}pBZ
zU6rx%mojW$4ZUto{|YF%q)Bz!(=%~mwn2Y{3#Qe*OyAbvY{PD5B>$qr7i5PMv1XB`
z(F6e^8ivwX&BX>zLq<mnnT~O-J9dI|LKem%4-wvwKE440_bx^VuWS~nzhb<o;`lu@
z*(cDiww5=P@Xozb{;6EY4=y0ENu{|j&m2^}#a4bK-?wV+vQ>3^#6)^QaX{6G%Bv+m
zwD<wWLmsjCIhh=^LX3=dcR_pss0)~VEHYt;5o5YRt(>O+G%W;2pua0bDIoUoED??v
zz6X&u)*#>sEnG0^Ks(3o!9*4}vp`vK(pk&h70K$zl!#Pei<|=i_ITq7RS)rA0ysTH
z*AdBL6_?DJU$w3OV5+lqeT*qV^?jb+%Q7fy!_r?p!GNL0(briuL9H9p2rHF?({`FN
zEh8lbzy=8;mJ`pSEAQXAVI*LF`mHvR3~D=+z`_K7#q&X-g82BLXihRFrAaI(7G+MS
zNy`7u5Ay_Wy^zJA$0&@Ifegb0R&b*S#Fgd%QB05e{uY}M8=C}l4486~NGpNBsJ}(i
z!uRclrbSdMFc!ug;6pYxmWSFYs)iPMQy9G+fn)Oh4IG$ax`iFTjOn^4RnHqo`vuQz
z8(=p67AR-WG+R??|9NskH(S82;xwDJsI(ODy|xc!yCtq}xh7Y3?(80O9@n9bUk=Rj
zzTa=&V7`>JyIy=qY;$Mb_36*Y=CPxxVe6j$67u>Qb0$0P<>bt+ys9<Tc0=>nx?sNR
z6NG?!$JYB7aZ@Khc$#M0dSw{peyy=GpV<<q#`iISxBENa>@BOCZ5-a}UFxR;J`3-o
zA9^#lm(};^xc=Uh{3J6?yHAr9x;esey(ZyjgY??_bjtcIT@IF<^=-W?q~>C|okeDh
zBjif_^NhIQuy<h;LLA!Q@xLq@^e-p*Qb<Q<%zut11O_N-Xs3-H;d{jV>$c$l`_}@c
z8Sx9VF>Rh}dLlier<a|X8=#O9y{;6!bMSmL6fK!_+)DKPzF#pjtH%t9DVLN|#?Sw%
zfD?9!ed60+)KtLm|F7%v|8E`x?}oVVynYn(#9d~8$ho?4T_WG<1h)FdHy_5<mMy=0
z>gR~Du*-Wp^~pE6voh9O8+$%WEjNvPQBV#VwCEX)N0m2yx7HMV1$_=c8fzynEum&T
zWJPvJvVklX7sQLu2-g2;$%8BLYx`T7ShLiiL)<?td1X7=H@&sot{}rQXZ_8pG0QC<
z7IX%lwH|9C^@v+RzUg_!AG&<|L6&|V3f-D07{M|+XGD`@Z6L82&Srf&GbF)eZSgKV
zpF{{j1&v>r7<)zo6l<_H7Ip<1nV{+foky!Q1R=99L;ecu2efShR#1TSj1?c|KrfXT
zh~y&>U^SXzBZ^dKZ8%}F5ab6#TmS=jN05o65a=*pi19N!c73oc?bSTyFYa;lLeec#
z$I<80Kp4k0<gu^~#U%B457>gz0~BnQ(>}XEzh#N1a2m&j1s3D^!PsxJT+CI^@He#w
z(vPB00@!qslB<p+s%pZ}L>bF&T_KF8b27tH3L#0z4|WjQM;2M+L;jGTAmTtk16CBB
zjHGG*;IRP+^cQT1EYiXW8OPc#kRfUyod?A`a&lw6uo=w(13V=R%nQ(yk7vyeWz<yP
zd*rKKy36j4>a}l;iLTAm*9UIg&|j+g{9KroOw6m;S!c_$PJ1W!T{U~O^yK^Ti`T!$
zOB;yp<dl;TV4xq`R@uj^QkN>tFL6OgZt3<k&+<8&bt9i%e7ZTc>yAvhVmtQcb*l%P
z-qk#2_w|=fb`vv<2I7<RCod;ktr8Db-1T0a{-RxVNLjDNj)+@bu~KK;wp2v=o)=Uf
z=V90zm&a+GWb{pY%R{{b=`d7^wGGSWho{bDL;DMqW7Br-YQnQfKRcJVB^IZb%54m$
zgpof!8M05Y*V?+fKk{}=cU61i@V0`(LyVgVr^K7Lm$h6UTr$h1J{i%bR<h)@Sg^aL
z*_ZWNkK60VUhr7fnVFtAx$5}oJ=~JK3B1~!`})fS?A9|PN0F$D=BeIj+59KPMmOJF
z<CA|CFu9ggwyk?lk)A@)=f10g%9{dq#XXfRSozUua%IIc>#5GoE9SAr-sa5yt0u1J
zf-{;Tf}aEUfKSa|t8M5fZ+TsJKX1G)?MBie`XJX++yxW0muH<v^7Fq>ZaXL8#pQm|
zTAwpsb+2*3FLAE}rDsQz%$_>`$e!Yhb4k=UJo_tQt{r)ohTBJf{x^#D{|;35|JnZu
z>81{`Y@259pJL%<&%REoaL&|F_IXOuo2vISe=%1xp(y)ZwB_l$s_(;Ts^d;S1Es8!
ze-2!5Ntrk`J;T=JFPHLNMpt{v)Y=yoVI&A_8`JI@Uk_Ine)Pu0(?N_MT6Y1Ve~@U7
z^0O9^A#<^f2~GAQg=B6(EE$$1qP%6eA8vqzla61W=uSMS>%p1kPiiaTdAlh8z&>$C
zi$jStB{jMM^J1BciBX4K=CK=2i$7qmZ_M(U+&^Z)HsL-qk4ZTe+Mr-QuC(?VXxTu$
zx{c1Zj{L(QsXka3+B`KWNQD3s;vMcbh%QCBe{?~ba?F7!b`-Yy&}1$!!-NEyzr=%e
zVS%u21IjKJioGdJ`U4uoX!#2$SrhHc5yq%8Ljf~><}p9nW(m%9V(*n465M@k)v<?x
zMqXld6)`_zlBSl0+t}h!f^MF|K4e7L5-14Lf!+kogE$y`VYL{%4j^NK^b}0~#wbK%
zhu}7{1+UbAh|Veb=j4zq!pp)0N7pxi>L6ADHy<s;$lSzxKONZEh}0}l0U>jSJ{ck;
zj0rOuv=kXefOyC#R10cpppbz%rr8tpnD@rAia20#aXC4Z+BIR0YlH)jLU)qJXEk}H
z@;T48bJU%#{IJ2~IosG+kC^42q`l3;@7)9pE|wj5*#FF9%Yi}rrLT|LaldfuQDw~&
zyV`K+;sp;&MX7N)kCWVQNW3lARL0&_%HCO(8T>+5fct4~x#}%x*ovgHbH7v7_6Hly
zY8-seJ90jivbPJ?Ef*q#c|1=5W=!xB=<w@67#WYsStAJVPmMmvw15^AA%ox#E~VJ^
z$V~PKV+7%#6J7?SoK7W%ko7^KfOf@4(iva6`nYG?Nbah<AV%%B>Z6#->^BT`64PDF
z^_F!=+47ZXcKx@J6N#%Nxac3wYM&f)dKp$b=)A1K`GTbg(~!t4Tk?2Xi~7)6^G`M}
zJ~5Q3eVpQ(VVfGKu7M4?vpszQT2XG|g7FFp!g0B*f7!WGGfDQp3NmLl)x6@)6jwS?
zmsu51O__Y2pFJBo_;Mb*W>z+jDM=&*Bs~-$4SblI;Cyv&8P~bsnY-hW;wn}L4%W@o
zDd&SQrOc?Y_-~3UoM?R5{D1uSpy#Hjt8BH-JnN(Xg;mKfZg*3aQs_L!JoIqPsF}5$
zbmMW{o2e5Mb_H8V#V^weEZ?+OT=5wu-JMA-T|Is`Rrf;T#3mnjx9+Rp9H@qjK@z@k
zOh?N>=#|6bGBO*Os{YTQB*T@sKi5F^h~4XO;{20?m|^Q-_ndj`^>e9D@A;;en{Q$`
zJ11$o@!9UyHFg13gt=|+yrCB+z0$6?A$jw~bb&+<9{`UvRH=#7lkLM21LXm{p>5a5
z@D$|egjt=Tz%DGHugKgCXzmVMF;J>8f`_57wTzw)*9ZfpPKIr=EX-kQnc#r}tugeR
zk6?0k8DuTcJL3^LGzyHyu+ca$=zh^e5HqK5?-a1OGF`|vr}Ux4oT<Ixea6z!Y#5{Q
zIt><)xCoVq@!DfvM@Fz~G0kBMz%E!?0obrJVY6&sP%P+dk|gw)iOit9twqWpaAvE6
zaRFM><KS+@ML7O23)6!=f_MrIU62jT)ztW*bRw>vHZFtBJf|soZVYglIz5`uRDuuB
z7E1JP^j~r&EX7Jt@6C)%3AZ&3z`hO!f}0>xf<$`M)BlzL&vICgNe@He`5@NQ<+0T|
zZ*x61%w7)jh+w?NE4KVKuUKYveW_85TQi5U>R080s55f{Lqj2M`Qds#l8u{e=CMNU
zQ`F&ps{>N!w!D728hL1Lv8<2DGdo$vVWnrOmiMSXJL2=F!J~reg0_d)RW47DJ%s0W
zsNOa<mE(T*_D?TXLBWnohHGYPCPs!U1A5B=?OKpM>XN}_0ZK=TB?j3P@FqaXf)|3B
z=y@>f!N5e{8_fU~1ke}~BQd3$J*x&8({diZYY<o$q<^tg`T*PQBYov_0%LP~Kb43Y
ztSeuCO^b^qiN7(}`@lM;>w-0j&1VE3@kJ<&s>HWU&Bd2feXWT*6h-!RP8`kX$=tIq
zTk4Qnh(?>v=OdCS&J~}uJl6kC0xV1lJk@^*STX+W%03k|eDD!y%uT%O=IE^@8f!=y
zV@2X^*L5)Z<L&j`l}uA?@AJhd@s)SV$DK!Xrv2C7*{UFx2fNgIpF;)=yQ=fQJ-H96
z#K6&5w!2=+&wlX4jz0DC^t!+{M%_WxnCzN6=HIoX`}k6(Y_nPCs-I%xTcVY6^t`SO
zpITLMPxj$F2A6^r;AeOioEOpw1?+d>5U!i}@6<6!`HCvkL-tMhyf7S=6s8#^IB|_3
zw9Iz3V5wO1ThEMIMD8lT!j_}`ciQKs($2Cle;K-T<!ceI;fDYhK2uV0I=Rx5k;~H0
z=y9=_@%2vN6h#}=YsKX?w}S~2XWZbNf)J1hjVuiDP#O*qRBv=N4+Osc2hyUZ)oa0@
zOy-8gh%o*e4M)>_GHQ->>DnVs*Rtukv>^rZw@S#jX%)qQ8bo0=Q0hWp4>tu&T#MKY
z3NV(ow(KEe3jKLWZ@8ZG4W;$voPsO+8?BbPT$0(r9MyJ-aPC4>ems$K7f{%uR}waj
znHchjS}bJ({tWga+9=>>5Us^Atw)16+M==ZKkGRTBkd20n3hp1;Q@n#h!aYrK!iV1
z6i3P>e1s#7N-Dmr5LGa#H?)izazES*NhZ-NKotu^E7#y)6Y=nBnCrL6XfcTE>cAtR
z+6W<laS4v-4FP2#UIRtdpd51COPlm*RL^daY7t{8W1j6+t-e_2XBD$$Uu?79Kvr_!
zzG;K_&sJ{)ZjrVtlX)sH#9V*5_Y76u&wjajEx*^q3()y&nEt?XVE^-mwB3!;$GDm=
z55|ttB{ybWD}LxTnsw~%z8BF!;N=Y!_t54?IwSayL9xP{eF&U}SGn*PSbPre5M*%x
z2E&(R!iyQl0G!o=Fbt~%?Gti+fLa>iT~wiC8G*|XCuWPGJ{D%d>serh6{L40FoxYo
z)sO#t)hd&Pbb*hWGeO;%_1u4ONyOS!TJ0NrFYI;hS=Zh(^8I+$qnIdVNUWd7s)ngD
z?AN-iYCiYIn}>Y4$a_t5yT}@c4>gwsus!~5QHM-={-IuZ5`Jy0e0p$biK5r+OYxZ9
zGAyL_mc&c**rCo}LT@YD#Sapu48nBAHfolQY#Y9`n|jBnb1J?^nyJG7!Awhsr=uo+
zufFvazTAI!RK%qGyL}0G*6ACNo;wJ}Y`Nsn-NrES3+9hKbUBm(udQLE>mE7pHNwf`
zv$c)fT4Lk7Qx7aNs`1j9)2;e4B3kvUY6L#Ed1tzp6fUN4;ZBIH&3dR*G1x2uB|4~L
zO9W(_J#lft6?>oK+u<j0c8m$?@P;mX796FC;sjs+naYFx^5DE!Y(e#(`ADg8pQj{_
zxEaKZyp=3?H;-ACAG&?<V5re!t0#U~5s9BLw^uJen9`ZV(c$UIQK2VR&Ir;*OUvr&
z(@0tbTpEt5JYYW}jdVkCrE)lI0K)`KB=K9ZLYE208H`T2lo1yn?Wd~)wIo*(n;yOt
zV^O9A3_Dt4mV%2K0JXKnG621<2CpA4*XX;L|Ijpf+`v2M+LHEQ73-mGO^saby=TrG
zTO+xb>*<)ljMxjN8Pxys%j+QsFl`SaWQe#*M-`<+!g4nQm<&pYH`g*oGHe_LGysQ=
z5aEfi(n9o=D5GzWBfd8cD%b!sR1SR0B3QHWxnX!U;0p*BUFf2~YQKbLuDV(KkLD^U
z%!>)hP{wu1nKbng6y8`<$lZb$vslx>S_N(P7L27WO7o1D=#ZsDLJ?uVM7WE>^`b$~
zS*G^7RE<?BTP+56C&Z;y@Cpv?UbpGM+SP|7r#4p6f5|I|=~F5lA3vedIk<I`^xN(&
zrFWHv5AGg*+oj9hc{gBFomh|Lq<f8d--W$S&ja2-rw7m7EgS>B98aW|F)S4(z{`Y<
z7!aIGD>(-<F;u|Jk=+ix1533W=;3)s`;b>b3kfN&pzjjy{z`_tX~10Qjb_S0?@MB?
zI?Qq4Pe^8q5&J;Z5Aba33hb;3N8_zXpZDK}=wpKJ#qCI{p2xrl<1`#7b+F7gi<<rE
z-h~4@*jbY%?=<!)cpVz+XzLlC8&;Kh=4+H**#}09y3EDTf!IgV(S7sS&5*N-q+|N?
zSiCKD9o2d^Eb3016`8aqMp)HgD5i_yQa1ae&M{@S^PA3<nvJQH;fm}cqb~>tpi}hk
z`n5z^gl{mytTt$_H`d6mQ{pCR*koJGMEpEfW74?cAgQEaO4y_7R}HBrr$3PT@pRRM
zp4ZJ`?O`oXfR)TGZKUukCXj48w)N83)gTH?D$Q~%^dNkhopJ<~aGJ{kiKK)W@c6(`
z)bsp4#6)F%sg`3VZtmz^TDhd@1pEYl(?eiM4;HSH8QKS1V*!=<@mbmk!astP%J`68
z_5LbRmzvV|Y&l#HZhyIX<-&uYBCE>rn<-tTEm9wcdSN|1#aOJ@=J8|C(=`@{LR%HV
z^Nu0?k^=0ffZaHd*<}VNH<6tA6E^Y5Yoc{I$jj6f1yfECQ1!w#R^;M9amp#M7ore4
zJCUYtj!h8$C7}mPQqj$zu+c-p1;gT|2zU@YZqVrfT!}@{Oa1N3oUOfJYS{r`Y$f`f
za{xr`irU99uz?A0^NOL(dk>dtpxmc5iV;QB1GE%TLJtDhK_(6VLXV#tgT;&+h~EIs
zq%5S@__;<HOX=Kq1vw|Qo-QI%_rYUjmhxv|)9>8Xea(OF#QGz5g=s-h31r+LLn^rA
zkns(gC-k!LikAC>V>0Ee+{|^!i8N^f7kj`DK9c&JE~j2Xs9mmfu+Iywwt_W60bJSH
zq7#o*;`lGdkklumrDC>+X-8;X;EHMrLyS|O!K$|-I<-0_9k%=L`s;4aeb88Ke1DzP
zh*HFD9&OkHX-^h$e*798Xpuq>)Pcd9yC~uWU;uRpLsou+LKx`=((6GyA<SPeA{beW
zk(?Zjh*8Wd`WX^)u(`nGgx0}Alh>=optBsxEJ#B`@pF#m!mCdpS$fLj!GtHz{oVZ=
zcLvRBx7Ws@89|7i-n-S$k`j9-g8H@xRdG#OotBv}a@{9J`5B!uu3z;eS7lH4@`2d`
zm6+S-@-#1vIeERkl*eWnIo0k0-1%xXc<vb0W;Sdd>v*tZ*1ZB0zk=?w7nugK=P^O4
zg8owL*=1r~lh@`nKPa=E5rpaPwTCJQ1v7#68=L*SSlIquzl%^O3=|^fv9r~?18%EG
zXxU9(ewfkIwVlsM9OC;FUIh3()DAdxV+mbQ(2rrtHwfr4AIP<bM(*aYYKY9_C#lrA
z%~Mc&4&5Dj1}MoJFEN_y;e)(yofb&{*!Q}mheC3^0I?kYnY<e4cLfTD%fFG$lrb1C
zQ40;6O@e@IrZF9%A0Rh&#r~dQSsvz5Ew%y$XCR>_$T{&(C327~yWGG3kl<b0+M`U0
zkq@bj@^fkpS!eR4BCn_3XOVn&@J<0e{ZzzPpRN?5+hMyZOX1GKq_cpJriDBuk!u6u
zmTSRmit|qA<nfpZZwh%K2crx6Iak9&nQ$8@SV*i1TV8>Q0tU#8+-v%V2tY_ap$ziB
zvKFjoXj+NnnrfVIexSKSb~Mmzk=LNf9r_6V@e8UEw&IWsZhrgv;$iDcN1<8glTw|f
zkH-2)spoBO_e*F5T}H}EYaL0%IKd@6D1it@|IXfpcpL(1p-dnn%Nuzr!1mRH3$1kA
zkeEtH65??}SwnB4-KS*0e5m6?qKd-v$o5reYIYqLS{(`vFZna$WGq65yTyn=X~C3o
z--A}%H>&9i@gE{iAhLp!thqrAArjcf6{M5ryyNw{4uusr!gkB11`$0!Lx$FmUA7(d
z``S)u9t?Dkc^%ZVFULyx^W_QGX;P+VK@7xLMqk)!f8pqvP5!6febG(&`qd!$psuS+
zm+leyEu<=#E`C;K(A#k_+Q2d`ewf)9vQ9ZZ+u+JkXs%&wh!7_r9SgF3466A0&tS`|
z6neYNTrepxE2ArcXf=kYaY!A)KzNIIIRHx#Wx4Qv7=ICwg4JhqAW~hZ@$itW!Wa%f
z;qFj(=~zQEp%D`1N?eX>N&TuZIVBg*N7bVSJAdgL`J0m5RbZ~{?lv<$x2?O-_Ss9Z
zWX}K-cB$DfrEBLHW>mjTHnNJwWL}g0!b3b(A$DHT36PRmEjk4&XWI{z{Mw@ZqRnbx
zDM7YZcVf0<=X9e#Xwolg4<xPUyWSWO5b9V<=lL$fZff1sp^fJ%9(s5s<pn)z7nzQ+
zMo-Q9GT(DJcmIB^+8<xjhF*Og4`ZfxByL-g_LOC<vg|>l<$TqRgaEx!Qo!=;(v0W}
znHr=YNmO}i<M6dhf;&1$BCeY*oKbl6t7@p;5vPL8v(9wBrk7<0tH~AHK+ucALG?W>
zoCqsceQn@iu0)PAxCm|FOv1NDy9qTv^1Tx0Bc&;=0oN$#=um<NA=C^I6H5PC_s6y`
zgm?6x)T~CXwi_?M9}Mx^MIrmo&STWxgGtztKy2`$d6d<=lP8aclOJe{dw&vo@zeF?
z=NG(bbZbNI*eW3BdjCgd6mA}PHVKdFQRHq5$r9#-{W&T$l_+foAp`=UF&}LH0wxJ}
zs6`nlguqfl_{{Ur02HGk0IxtdP(v=N!@OZk23KZaXct&58kI~W|M?;(R;6UQbco^2
zW~I$@<g6aIuCE*KeP7r4$wTG6@>SJDj|(2N?OxU$rZVJ81<KyuL;Xm&3AlxUq6i29
zgChy7E<T=Ph9?J7R3ge;h64%C^Y}z@Az+4(%?v+gh`QQt45$IsPazP#gFh9c4?>wf
z2^EkPq{+cgd<YE^jb@^mQGS<x*&s=U9+H{EPH1=;LFl~$^E_zS&Fmu-Ee`FEvb)9D
zW#7XfUU<{A-lq0=LT=89<an||#sE_;Rp`T*<JhC_+{j_~y*wNj3HxS@+L#)DWT%-n
zD_sv9D@&PB?QVFoy)n=C)i1$Hi5qiFcPy!jPnxiC=)?x)HQ+-5;AbiRS_W{~L&Ts^
zfoztZcp@<oW_VD40%?>t0a4y)ct`8TFqj|!m4piM!W$O*hyk-lgp)B;|IBcr4J-~L
zL_p#Y46)I>8Vfp}KB$3!v+zb1M`&0j!3O4pQkon)c)#hfvT83eg;Kq<Y1BY;)M<^B
zQE3;ajiO}hhv%J__%1x#pUU~O<>l_^S+*{zDf7M7s+PR(Pi{^zw5r(+Z{C|~vCjoZ
zdYUH7%tI#Eyb+ncyO!@w<JHD$)hAmBV=*fYE{FOMnX<QZgT9R#+;;x>^Al5E4$~~&
zXY68M0<EN@U|P!>nl-8WD;$B>f9Ee#-q1{l5V|xrG|dyupB^Ijg<$2m9uE4U{oYXG
zcfQsvrL^)S+M8Zv%MIwCG7m8C=h~E;?gC*u!#~!%3Xx5IS@)}Ur601SaEvmQHla=T
zu$+SH*{#erDP5XV)*y3`ju5f5WNWjAlvb-Cd{R)|62bdAekfq@xj>E<=2x&x=fuOD
z$AUz5ipX`EmH!^v%8nGTHYU#?y<_VNK}0U#vRM57jOVX!)l3VKf5XyVVSKUs^7N58
zj^alB_Vv{0uT86%q$LL~vdu>1rpp}4U!{~@aHdDnr6VE66<J%r*3NAnckU+ePUm+T
zV~mC(t@aU&BCWQuKq^5t4XcyIa(}SY2S+M^UL7vAut^6tU7_8n!I4aQkkSn9xJn`U
z5;?aNd|L%PAe!L88Ha#B*fozfSl~ZnVfZuGv{<Rc-nnf|O*tq%Tc;Q_`q>#GTaR%K
zI=|%%vo$g+r<zF|8jZQHvnC8G!NsdSt#uH``q#t~W>RKptPC1`1)t(5G>ZaTo3UIh
zTxkC*wRAoaa_hr<gk#fTZz@_Z^hakKU6h|G0m6RtK&Rho3!+pAOHSGYJ1k7S|B(Fy
z0o2JbuDuOmu|$@(mp@AL1r@G*w>!bVJt!vX&YY#>#3<h^Nv~<B;Kqt8+D6ZgTj>?N
zw7H<EoS!u1sZLd)%d_GIzWC*xikbJ^n${WYNS{B>rX-x$2N4VyUVB7~U_l1FmyG<-
zlfiaqcyr?@24vKeu}1{1gmMe)5%g`|5vDR~D^Tsw*UQ90q5DNg78o)Z!$JKq-d6;#
z1rQ(oYC1gdRSh~k1Y&5wvKo<Q_OMI{8bJr(ZA8=(PhSQuVb4t$0BNwVnh}38bfq!p
zj-NUld2DH6phD1Rul>|rR_sMfJSZauSzp{e`oCN5*)&`;H!$QMrMlbvuHV%<g`dM8
zUe`hoSE;CxA2=J(TI?Que0KbqVYH7K1m~Dnj*UqS1Q3jv`(;K(S}u6CY;huuJ55G!
ziP>r`_=Q@jT0Nb=E?~O?+Y{1l*n|;BnbDbzUHsVmJAWSvXcqd+Bcq=)c!xptN8)Bw
zq**~8puB%JzId6$%+FlyS>}0c;KRrlSWWn(tN7E?V<Xqrn{~X4EV#RQ+RGc1_4P4P
zIJSf<G^|c#kI;|;j~!%`dFyBi6J9vLGI|W{^a4PH_-qs@#1CV(q{C<sr;!c?AMex*
z0B_-ei3m|qicQox<L++Ih4Bzb5f+pLO+*L<5KjEGXWOT$^xmgBtc?OrXD)}7YS&S}
z?d8nf#!;W69jzypQs&gCF<}~Qtgg{ys_0#)nz$D>aoI;%YjY+i+}YV%YofaWfhDLd
z4@$SngDK44Clt2ZaKjZD;eV8n*pAQVzzGXCV|L-dP<27Je9+R%LA_$qgFa9fhH#K8
z3=6PuSOp?aNgybop5ZIN04+ht26<6T|5qQP)Q2xLAAHe?l-@PKR-;RmrtTWzPG!Au
zuA<SwY;+U-O#l1-j}`5M<x>&bHc70YgNQnW)TF?BJs~4j1x0#6{V*-{33Wq#pXPFW
zd;C@gAz5+AQ+C@3ZJb&qy&AkOJK`P|+E@)*8%s!06sDQzFo1RV`cX%sH$>qsF2?~R
z1V<giJ47``NlnBlqqB~$+JZ$Lc$PA}J4YvPS{X1FzEyK<>7+<|#F!1JzG}$adQ#v?
zcJPHrZK88-_&7($bVI68w%5d}6*nhLWTGB5nH#RV_U!i8H^o^-$9Q&#7>Iq|J^ZGm
z<MjUUF`>Jf9HaVQlmej`#d_3M^CPGO=|sy>#XvjkAx_{oLWwM;uR{l*n?adjIdssF
zoCL6l2LLQo@IUcMNNfOkCj8JDLfMM1^#Khbf>C;m4j2k%L@crt{4!~ejuGu(I_JWf
z=sEzjpK!`@+O#y86MmgIPLpMT4D5q!mj{<dw6R_=;qQLH%GMn;Wv*;~R(`JP;iv6{
zisK!vmp+p$yIyNY+|$Zz*;1$23@sg)_@?{E&7b~KeRb^)=P%9CC!Q|sq;boswUf6z
zQ-8@fd*1oROIi9#{%%#fO#5hw*Tqtq&f;R_jshW1D)Y{Jt2cjCt>nEAr>?!Mzi2^k
zNccDY&O|>H{3Y!rA^BmzT@BSb)`Ypc)Vt1oq#HkVRmoZ3n6E`0w|Bq&)Ti%CvRpS8
z{kGwAK`EM#9e*CuIK%xYhi&I)PxUu5VxwEvg`9oxeDl;i_KNR{>lkCk%G8?D7Fg2e
zV~Nbbo~c#kNWzr69WGYkKKRE0i3~Jk^fd{G6V^d0$<m`Rp4L&1Weke=EtQ0p{(KIQ
zmSG_vExlq?&1h{R!m;<D)B%5W0>&p<<{WsFC}klQI^_G~uF}4T?&^7k%2h^nQ;lY_
zH(5>zL~azZS0TGm6<*{ze@d|@y&O2e^<KGv`Fqi;h)Y%Xds<Vz8lJkpYIH#-|6S%I
zvh`s}XF2$>BW)0Shyl#ngJt$XULXW5=Cl(#VecQ}nlce91@b8xjSe*`F;E!v^k;OR
zMJmwt(<6=49zYl1exU;oA-NA~7hcQ$4oabNRx}iPi#NbmN$H`Va=Ot7U5?+nex)-T
zDkBf)2sTmcI8{^9Ogzm0ANIaHp6Yh(ej_P~28uG3$`lgHJXGe)W60P*B0`0XC1su!
z3b8{%WhxO7DrBC?JX6T9O}pQ9f7SE!Jm;L>dEbA}dH>iSLt#(f>t5Ho*0olhsZJW7
zfX;KIkYa32Soa_T$n#VF6k*>I27fWahpZ*a53WBmsQ4UAkaF<{{qnz|qjm#o;ovZS
zf^ZcWvvdKuhWQhqvE|<gDZFtGpTWq@^ZhiF{`)pi>3+4l_FVk&;+0V;7A3n|X$R@U
zIXQSQI1rD|?C$(3S!TPGcYAe`qQ_&JrFJ?~7eo>!%1SYKI@%lGV`DKb6O@-UXkl;{
zyi2iSld}$tNFk)j4@_gXl4Xc`ZX*MgNF6EA-$pQyM9?dO(g0_P=rgE55u^w@?Pl&r
zjWIa$-x)wSOP1RcD5?fbzK;VE6x?P>O)+@}6m<BNs2t*GQO|eD5E~FQ(A|cL26#;j
znp+r#0&#_~8QdM*1q8&#TKqT|Mx$oe9&Z}z%&(SSacE={I(XrRt=X~2Z4A}tl6FqS
zp`c0Cx3(HX;rGVNyARPV2AA%Yda-(JdQiASHh@<?@f?+hrgiW1v&p9k**iTcG_B{<
zPQ5sLTp>Hh$i|yCz+x?uY*}x}=)9P#s9UFpQ*b>&_2IZ`%DL#CA2=#2@b6jjTDJR7
zZuWjYHj;;Fo%rU!>Z4mf9=IGJAwv&QZM`ZN)MzGl?-A<j$l~5VLO4&c!CknriN;`U
zZ~U+(_ji;2w=v7??|<NE4ZO0eno@4eRM5q}B;M8=y~!MTDbnuS)Bz&2(;HT^1E<IF
z^eS^UZVMiX6OQqL4qrCU(9i6O${cjdWIRyTv~Z=8<&hir<E-4cgQ~VWx>>wG@op{m
zWQK!MhIG54GNHk10+@BEfI8)7a82`~T-K7GSOv^vfn&c<OMaWRLF+lE<u<hC9CAx~
zYo}bt^_H-QB^YA{i$=)jaUcETzn4IDS=xXTY33(V2gB^hai%v42r`y=pWK~;4BLSV
zHCfs-C|z#Y{OvP7Rq>c=^^~+L#~4v6P$$cSMy<W^Y;9taTKl_Gn2?ILmpd)iKgddY
zILI4m+!3Mu8&U%dgA7?w0@yF4cGRjw)0Be|464E4+83`#3Nm-9MwAB)bUp~<A!7&a
z0b)Enut%siXkb`^^k?n={UbG7g~Q>PmyhEj?rvoE#X?xAbHrticL>FTB}n>YC1l&r
zJzHMBEINdz8@&9}f=PD>fhdSWgQwFwBncr&kady4UL?wXMp5$XPy+2YDgxuqftpxg
z=a9@32Q3;jMCX4>rhou~e>m@hT-<O8aJ`espdC-ALq}39HXt%fnG#n+JANAOblM)Q
zio*^gYRj~}^fcf4D!+^xwzwT|k?G?kMN@0|C4<`q_{n^&?ZXNVUUk$ZZroQYV5qb3
za5UHJyEVx;^~quff+I_ChJ)vYqCzHSh40GOoFm5_-s-%BM$ogIRxkM~h7Y&EVFP6=
z+J?TeB`O@o7IyS$0IIgf7~m2%K*dQ(NKPQfDjl?rSR6LSy{k*Wi-;=H{YhHrhM`s+
z!*;lnNuWSQ+u*>D0g}*uN*WS27-Q5r5@G!0u5k3=wF3nc;|i(j_Tr#(MN$P|xq#^h
zDj#u#=&1*W9#vH*P~Cd@;l>Kt*mAD(j||FFQM{&*THciD{aTGuG&6Dki;B??8sk@`
zo>-@6Z7j&ub<sZC8<@56jE5!R&8b}OS88sg+dDVPQq0x3drQ-V6)O>L^&X2~OPsNg
zmS@l(4AYg2WxK$VO1HR!ib}7a(L^tnfzmCKxCoIHYu9}^J88HccE9ZhE1S35R!Sbj
ztNtH;Bi%2O<ym~`;eoMYhTJpuImTsMKfns~6Em9eJ8ysB!oRjw^yCc(<Po{17oJcY
zKNi*rhLlEK$BE5C3vp~FZNqlK-wzd=)DbedlBSOEntW<pqamGcDPs3^(*GJ_8?YLd
zvKTVI>Kcr94a}(4%bTg)a&Ld=NjB=<)>@&iRpA-Ewy(PtUt0`*>Um}EJ`mrx%rY{c
z)gf@0UED3_xZ$!#iBvql;Oo|--ONdO;Ts>y-mIx*gejv+j;9U#fGVi9O>q*IkNu=!
z&AT{|gBcbfK|P^kqzi59<3Z?k(S??<p(F`74$^lDV194}yt7&B=kX<2HX5|RTXDy2
z_FDCTT_a|ui?pXUv{TUH`r{&f`-_w(^US;V?x=nDZjjHTHf6+6OW}FNtVqY*E&Ow2
zX<6b;-#WG4hF$xxp%09!DHrIP@8GA{ufG&KlCv~Yy&!vj3>!+8WZhhl1@lRnG1XBn
z9Bvqxa-@oYg-K+G1eqS5C4h1wFzqSysX*o&3ZZ-)mjh9NT%g?Y91dqnLr`(u6Jj2O
zOQ7b$A@BS<WN9EMKuh9qQmn5_{yu5>^*P(o$5G~V@&_K3MrF^B5o?paNOrdeq^!-)
zf(R`&{Uh&yL3V9&=#$7ni>Rym@koFVxaRFsG5s#MGN~Fd_cu>kMoll&TLVRgn0ZSe
z(>OZ8nFSRvO$!A8GPK$O@N?h^UI%(n8R$h(sKFdQ6^9v7(C$Lc+}Mypvks=5Sdt_X
z9E4h7IUyj^!iw<UWLhLHEffK{-zNQO4(%@Q>h)or_3tiD_<@_y?e1>UJ=UNmu)$BL
zt8#TQ>tJ&(r;ytT>$DrZ?mC~Z@tK{yVIbLZ-IXn-NL8WT_LHd#Q0;Ephgkt_sg@*|
z!eao&>nt)Q1PTpi5=aO~k&01aAlX3o6f0Ml1QZ%oXhKQQ<KtJBF@w($rMWH(O#=(W
z60Gj+N5@jM4a`?D+N=hL4Jje!2x<uq0&ZdEl)9@A$|k6Y+g!2P0vu=Pm*Uo?^H66I
zM~8$O(lvXnn(}ZJ@I?sU=+(mp&&Zc|=*_TtEyqlydp~yyA~yGYJ^a*gEHvgDMN8pL
zgT+R%=ij<57d+YxRgOz-v;FF~n%AOsYqWZlVyxh8p>gE*lu2Lw(lzFNO$tA7hJw1|
zYAcl09&@HykKtPXp0&MY;kA#l<I7d4>a|v1E7!xcEZ@%(55QaNX5ajOa`XD@v28G^
zG!b#{FktT#Cx0@o@cP*GaeUGCW>9HYdh69^Oc-0N$c~oj6NLKx?cd$$nUWqzi73@t
z-nTl@_=$VXWUo@zVszJ>hgbhaT8X#R$>Om~85&Dt@+V+(AT4pF<Kp*-mXu7wmhvRy
z+!diRkCc*PeyQ1dmyOE{Qt>Vm=_55#>ha1&e&g-c4yQUJeP<;r@vc|yte(8muOf2s
zwoi!t;ZxuDdiW;7o8ONxS+;rJ+2NgQ*0lI#S!Vefroc-;T$K4Cwt;&yRxicw{)=h}
zlg}Kjv`yGpK|>aaIPL?m0;!oC;s*aHm7R*JP$@TR9#?W^)Y)pNp>nil<jVb^?H+~Q
zKEf&N=d{fS8|2vi=T#E)E8H#agpW1Rz&PaaM$Sg1#hv0|$40K62~JDSN=#JI^!Afj
zw$tu^?4tpCc-mg!jP`_A!x7YoPX_~?bt^5z==GXv0p3d{H-`Qg>|?;?*GpAHVWWyL
zB`8Uq4v~v7JyifZo~nkw!Nf|zWoXZ^-WXBf%5c5?RXp~452GhFAm$mh{RI7;u+sz0
z(j)WSXUXlq9DVhj@A2(pLqXyL_4h{RNDJ&{hMyMt2aQOgLh&}F2`<xh$yibb)Lcyt
zBXi`3rIp5yMN6YlG(Ih9R#X<s>BxbK0{uJ;OSdSIQ*v5@xfYU_Kh-ZqT;D#d5~Oi}
z^(P|p*ZDstS+LqjAS3wvQx<Ps8G4=&u77r?N|>q4gSQG1ixxejqCarUqZ$sExb2TU
z&irgr88gzszQ>Vg<<;jg1u5q@G-XR`+@UE7Q*6=K4o_ayj|+3N%AIqxdaAA=A1oqL
z^}q+aU+D=UuzUzoFX+RSdf{+F(-M+RH5}*&!Tv`od<Az53OmG~{M2ShFJe_IpZVQ-
zsf0{RD5yXTBa9cl(G%D_5PJNaJixRInTN<CRH*rCVH*fq)3%lZ5=4Uti966^!z%{B
zS-OT#MtUQ>57kMa$YMGT&3Qo_9FK6HpsIpD7{X<Yd*+OA3QQ!zhYH8vOtABvx2*}Q
z*%Kb0kj2KoxFdA5H*&Ck;mYL4<iO*7K5;82awliIw*SD9wn?3$n~)I^s<|oBAoaX(
zOQO}}D_2*JP(o!*j_sPmS@}Mbs{6}tdv<E3I3FjBdYtOzbUiF2d`)?l1=zfOf7R;$
zI$;09K2lL|pK)1;-IbMWe5P7p>wCT}!3$>;q!Kg42WZ2#ev;j0%Oh04pFt-qoIRVj
z{ORnm<>3aWPKm-mTK~~c3R0=5+;#3!mMqgjtHv`DFEe)-X60nMZ0EX1Ew!6%I!iKZ
zJRq|m>o#6%Da^w~DkhtfH)?;fiCk+}Y0j!x|CV6tOWY1LpHqgy_S-SJNdzZ8`@~(B
zzUedXQN7dOV^q$>OIsP_=H*(@HK%{`lj-rMv6HvZop@h(u>W--zw4WghKffH`bxM}
z!X~5Sm#Xuw8}Zh3H920rA+~zPKX7Xvmk0sIJM_=|z&TCU?$6y#;qC))@zrN_^sa*2
zdPeTLazwSCK~_AV!6r$1uPi5hGITz01t4!in7WFF_@ViSEUgh-I-GzXXmRc5w<U>f
zdzuiYCe2-&+kGb2y0j{k^yZ7Sx%uVz=?p$P>tf3bFK-j4)V}F^Y-f6VYiUhnc&4Ms
zh>2Qf>fq}Mv(#B$XIqqB3H3$q8Jfb9MuCJDM6wCcl!EgYd;^8ZQInwzmvvhq5-()d
zp~6ZhD*`N|srE}4)YcplgC16yL=TU)F~{|B3;;i|`70F#0$$JnVlgSd4!(c-(b)N!
z(OSB8K{#vRij47XYLZo->(Pr+=dxpN22}Xm`FeZ6YpFF>5APXV#JQ&gD`k`BMNjlK
zZ~1`~LQ@m1bAw)P;|AmyU888>*^j`{I>G;^VR?xrh)<Jw)-`U32E-5|;I7|yB?r8q
zB%u8wowQe!s2YcwLB#>H);zf92^T^YunkHyIYEOG663S|MORj=wJi2t=s28w^2Pj>
z$|duHp1l($?aLnR7Q&APO$VN)I1uJm*SJ5s#LQ1?7Y|J=F$(*|n{8gNWb;9+k3m`l
zyPQy^UXf#6trbq49A5rM-av$+Mil{Gh=Aa1LTxO(I%HkwI~Ibpa39dLIT;9=NLf~L
z+U>A&2Jlfi4<@Q;0}DZF@CHE@j{~4FvSmEZT!KKKj47I@m`6RVK=WNcY1%}@G{VoK
zHXG14dV>PVfOXP2u$20{5;04Q;F0_Tcg5P6x$)Xp!?cV(vrelHdLo(7P~;r@WKvR~
zMP<S1<O^c-7hapX*hjF03rqRjEZ)|lf0ItBD12oqrQh|n5TA{IirO2KM=4#q#&T(0
zD5DuQNxW1U{fHJ(DRbJpxt;?WWRw1Rsi4c{KlZk6U*T75BSjO-x7V_xG~nQT><Rns
z{+lBc_i(G+BbQb#E_|F`Y9cbcOM5&t_oi*|0$cLc3rD5CJyHAg_E6R+FU|Wk3*G?c
z=>iXSCxXDVJ7<|Im19hYu7^wZDxd|r#fuf>j?6n}&Jh_3I^I2b8A4px=1j3t^$qrE
z_KPfYQ|2TY@b1(*t?3t*EqVKSQ^vfQ{#Tpi9c8SaUjD%GWvCgm^aay)-?;ScrhIk2
z!%7?d^p}CUmDU%Q9YasvT*%)QW=nBLt8Cw#SzBxQz!8ViszPuGd@!h+(EBxy5AMMG
ztI#FGyZ6uAGoXcFqVkXB4@4%c=y=}-yUpOnHgZ#?pXFt`f#ES;`kSAnJUs#x`}wv%
zi3z+yw(*S_K~6%gIAL{-aP7i%SUcwbXns{OOMc6pk(=K;6T5vAJ=|r`^$Ua2zA?(2
zz7|BeC70-+DrF;183o+X+)v)H*B%U1$3swvHq1>ZJ;7QCEhZj{BR@o%AsWsI{%Xw!
zC@BPokaT~2z6-%@M?h&mS0|tf5)_Q7``iBIKY81d?JRGVRE-`YJ$|}RLBdYt!>aV^
zgkQ!F9DTQQ{*3fq?j7d>94`!lfBpu_#Un2UlMCmM8-Qk(R^x1WqP)R1eoWg&57+K`
z^f%VPeN0LN)=VV=o?p~kVzcZBfKYz0e<b+86ft0$2;{&c#P%%zDu6Sk{nk$llB<KP
zsC`BObg~L{2}q9&x!tH|1jl8qiw}8B!>kO?S>hiAn{v)q+*mn7c`xNuS03}_YfRD0
zM6#CL6qV)}zx`JS{X;)Z?1_u1ND`_V+$gM5+ihI>6jn?H%@e{5nZ>SOH<RTsFi6-*
zKa2)EX|J~8nXJ?_z|;fFxWnbVqBYg5BL&tI5_xgA5xLp6-V7_=MQfLwQK+7w<*_(X
zbyC!6V=@%Ll%#V|%1}ECb#eW@#)0~36l)HT{rNLE>!5c=v0MNK0VQJEW+XvLw1nkM
z#J*xHnK&Nc7rLP2VW;pPCzgqs`}=||yunY$zZQxZTE;R5z-C<y_dv%Q=~_ggxFg>!
z1F=u|3q@9Z^epn`JHJc&&ScdPMz&AdQ9hgxSDs&^dEF<<+{u{Iz*$_k5Cg3msnk^r
z$098IpMJw4X!?nqH%tk|ec$d1MyStrwSRa)R{M=~h-Ph$wskneJ-zDAiMNl7h2HWK
zt`N8L-E-g9Q0i=7d2RO8Q&G7eI8mCFAs=^qNn55tGsp3wnAP}EFh06*<r-adj7N;f
z9(YErr4C{4C%6QAgFTYD*AtC-%9G#F+kJ4>V-67WOH|ruO_|fwdN{IiV^)$xN}1%5
z?3n}mw3l{Yl61W$zNXityW*7fs6Vnd^ju(v2<;uclioC`LZi&|gfKeI9=+u)c$VdD
zEY<F3Cx%{}aZLN-n0vP1(Rm}GT(40_E%PXm{xkcX!j3WMoeYtGk)ju5hoW)T57obZ
zBz>+4Q$Unc(cyN~`tH<YtAkEKLJWs1aT@7$JZ_|~v$o&BU#>Dd^V~Hkkadv#`MlJp
z?A#x?-9p0?t~)GlB`Md(kBv&1;2oUC`=+B8$N8d{9{Dgz6pi^+H;7y~h*onB#2BE#
zZ&7w3D0>YM5+ji}xEjzvg1A8=6fOlN19HkSYVZ!3FF>iEKo2&nV5Y19vr^a|_=)pa
z=Tc0Yp3}Ji+GB<8G0`)@K{de+t~F(nrUVe_A)_JW7W;em`wyQ7#A%#aUTbX^e9+0}
zz+1jy;J{2}UpPVM1h1yXS-Hse_m_6dEi60FDqKHcI}ubiTP_3Qj~ONYm&bomf=~3d
ziti$$`K_Q3<b)daPgp^IWob%?DDQw|yY=k_7<9swua7~j_3jon%wZeL4wV&D<qe#q
z>+YjHBUq3Ch>ox$3d-^>g$Xs9hoj*t_xSGmu{?pZ%EU~eF6pnXH1B)eg$DEUubOl#
z+w*rH4meX>;tI)#QzoBZ1*-+VRe0+x(H2C}OEWQL6zHliY7{!+TKwVri=D&KT<zq%
z9sEI7){UV3h5PVRV}gS%%rHNk4x0EtUklhR6~jF6G6CKr*XKxL03W|Lj7<zdB40{P
zB;G{z@0X}I0GsE+#s;ct2r?oPQC6k$6=wCZJ{ZYSKZ2FH4R9C$OkP*HY#_ySEaMp0
zC=CoD-9LNV;?6W)zR)bR*O2>skG%ue$vw;2+i<Hj?nyTs<b0*;a=MQ;+2rTs2YulW
zFw;2^T&D*^$;rk(`~Knp7b@G>-j~<jf;r9gPYfr2;5@{OKNrigu`*J77n2V>Gs!70
z_4c5&S?yURn<rA2T~&X0eIZ79D9s$)X=-R1NfD&~Z@59)&STGgwiH4$zEkmiT2F4&
zDz*%qmQC%vZv^xp51WKq^I7@Tjc@DNuhYRoRc^kIR39rr%fW3=RJ_uwpc(0LJuybN
zBb~u3xpU$Ye#=kHlx#-3<7?HbmmcjaU9%?B<aW$_Ye=LhZPJkSP?7&4|NRTFDu{1?
zx(%8O;E}aGsi6fzsW5vw*zab|Z5FWRHD~9cic4~53V!u2ORjF!E#;DzW}i>sJ0gSO
zpp@L$Qh#Ao9EETe`*tt9q4u+t``Sr+V!yi1(+wCL_siwkxn;vjFUxGH$pb0sfvn4V
zHu#&0FJ6;eF?SUad%n~-H|TB>cysMC`6UB<bs=0OBX{DiHBF4B@(68lB`kKTF^-Df
zZ#yRWDs95&$-@Vq`dj*-KW2UWt0{<dX*?Lp8kii~BB&Ae*^Z*AZr7`E?)auc>y&m|
z{d;dh?C%nce&C9T44GP5GdjdnhAnsZNzFo|FHx(tT9p1YYDb2O&G})+G8&O9iEY-*
z+}REJ`|~?&DVPnIDVi3DPVX-jG`GAsG~v5M(H(e0XH_cgDGtKJ4iyy$D>*uKFG^54
zTBzx~T2BSb%(k}z1`kL$1v*(_CqzTSh&oAd(m}CK%U9`l)Cng1KhY#y^dKwbK>|K-
z@+i@nP-V{13|0ywCo2-MqA;+dW948PgT+=+tr9e>4uZg}{{`yAWm;6|c@*a7ZUCPM
zIi_GU5W%`THMTw)&o^IGe`Jc3ksFUDd3x+S`%1w<yVyd3`{hFptn>dG>mLB`i~W91
zhXmJ$wxKezkll{+1Vax{!+Co{mx_TY<OU#%lFl)%t@*GSYMDc@WFhScXe)u2BSUb!
zL!-qj2x`#Sy5AZHHu?r2E&aEZ)u`lzr!INPF>>)jRlN}`T;Yqc&mrC~ThHT8Au#0z
z#hrY*pN>`Qj_}<olY%)zN>j|2Kt5|B<~us<lLHP9V)R}`4u$%Em^^cTkv%Q>vEQ<o
z{!A%gPCPb^M!iVFg@}MUGI+H-;f%x~2P98^_%=LV7^uJ$`8NSGK(u7i2%s9)p(UIs
zqF9;ztA05l5g|z~@>c*kf1?`#m!mR{`gRa}!=GK>I)hD`PH5oq&!IEYgpv)c<v-kX
zomdX=8qu}SdrXS-N!LR2^C8|5f158jIgF--Yq^8Jq}=)9WP9Hgf~-6B=(d8(uL|yd
zsS3SSpjC8L;zR_CkINJOyy7T<LbX=@d218pb{8J0y)5-2hjx#*aXEfjcr))3RxeA}
z|A_Dp=k<T?`UMG#nLd%Le%}nWSE^MsnN!hFVbm<uMXqreuZnh%3AQk#FbIEC5=(mY
zj85TI)dhF`OEH(1kEOe&bY)h>Dn9z6Gv{djOzOxbyeq|&PM--uj>uzdR1s^Fs}MGS
zMEHxlJd080Hv*kAjv;IOa1!l{`&>F~&FtUD-<~GgRX_hm<DW)RpG!0W-&=^?-Dur@
zF$zX`WQU*z9R7(uu+hY6hZtiNsso)XB=Du0N-~`D)(K;`95g<;?;XoY!zf9?=E=O^
zoVCQ=go|g}mmd{pn30r&uI|9}+&G`JkuU9{{1uy-V@^g!`tocSpJ~$4FK_dBXeB?h
zzdYhf{%gIjblP5ouPmPm`{Oop-#O#9yP-#b;zaDI%WY$ch65XNUdjf4t8iY;R+AL2
zFAd{8bC<>ORJ#=3*^VJa_X;n*8+1GS-7`+lY0c26QLJfCeckRsc$#+ny%YKEV~5KK
zX9g>WaZ;aOx!BPso}Y^vIxHbnH)?y+;i-|ST)iPR(cq%gVvC1TWz2a0SecR9fof3(
z`aH=8sT3!bHgZ3ucX1@rw>8W06V#Jn%^&oOhEd4D4Gt-2WJw<)5o6QQb({){K8^`h
zjTmXpZgOb|ng*RS9%&fCVi^6eBM@pJjU}m<fG`M|2jddV<BF^}Az%d}WyO%O(u8J-
z4h+(Z{sg$lc22Xz3B|j?2G*|MS+GA2pI?X2`bW|``K}90N3BT8p0~yI4lcWiNcq^&
z)e}k=Z%%il^=(|lElQ9q(~KBRGh|)dOs!z@x}N;pmImmAVH-vtrIUEIB)=%JM|EJG
zxmXwOz^oQ_Z-K#lVTpR+^-yVe97m9D=7E_!(g%j1hyx=#4i2o0DM4Titbu8GDMKC;
zJUjoOcLdYB7&qjC89^ppuWZ=Vs?d1hwQyI`rj$3#w^kQ67C!98hdyk}esSQ1$bm=m
zjf4r^m9xw3743dK-%O|BJoOy-o@re4?4YEAgGBqVUcBJ50(h#S#kQ^!0p37O1}r9#
z+F*Ui0$dA*MgdsD1H%^kA&Ut<1?GVj5f^BlP4Ou1S5e&o!7QjT{<Ch%t(%AHty7pG
z&ygY>Je5+OmmtX_E4)@{sEw6SrCs2n4CooY^^<5tF=2mooI$hiUpH=#5@Y)C3U|SJ
zN8wrbZYM@wJ%W1v=N|7tBAu7X3HJ)goy1mx@AXV&X43{9zlty6%8~T&3~ct`HxY{;
zfNYL*&E?cLE=9ZhUC&7Fp3v{G8|^55+#G+o-QXa;P(v<}E${>vi6@R8r}H_7EQCY2
z+=KUcP1+ihDi`GkCt7ft`Nzut&rXB?jPGGq=PbwF+uu@)rg%*Xjaqc`M%w4ektJ*P
zDMB{FdyfZMfw*$_*c;!=G;ZX->!|QQw&8kS?XH2<!}41Rj4Y+af_#Tl8nbM9Y%MHp
zZ3ZpzfdxsN?<huP-5W~?Ut9#r6eo74fHTT1hnYuw?^X_<vfpfVFzadK>^^iYK&*RH
z@MjFbz?%s9L|%p95{tM75_uR&s!XV+!Y+e2RSZmq87aeLk-0cPVqDU?X73{kUJ?Q2
ziZ*<QD`<|y@A`CuDW8kRx+fbr?gHg#j>R=VCx!AM5vq%EQY6iQMR*8q2bJMDP}Z=Y
zL>6l>G6_-v=SnZXD7fO$eFM!L$|6ET>BgV64#w1d6S5q0X{_pEjnR142~|>9bS)Bw
z=myxnfM)G~F_cEfyw=$k!HX8T+B@_YNFVBd;VN3;joYtz^7FMp7wY0VcOks6NX@~f
z<jl#ddig5O&WESs-t6>t@M?1>^Yg=Gi*AQ^_`MXBMROEXUf@^$)Y-JlAQ}KVbBw2i
zCyG`>F$yX(az*+F*&;c$fx)GD<a+rqU^@UIh=xLYDu<v$<&yJ(1{GHhtvb{TkhBBo
zr_RB#D|NypY;4cvR`$ISX0(eIbsmq}(?|Sz#jmHkJzY9HaJiC#=hiz1BH8?*yhOE*
z8u`S?GqbYU_d`3*Fmy#-6S*KBW(n2>g5%4E%)(e#ObU1Wm-rgSN0=!wEsZIvvQ|dN
zk(wHiTm%Ra6Oc~Rj*9XKr<HUv7ET8x1sH}6?=ZU(5L|;~3CSD*SR&mbkMendbqc!h
zuwhxnstc}g&2{<#4O&pWgAo;0;mF~~1Sie?xNPBxTz6Lf7sm@Ot4QixFu6zR@#)83
zX0_X$(m%Ckgnl?VTOk>%4Nm(mZ&|0ad^5#$`fd2+^7>sT?oTzpc=+Z%X=NL!TAxX(
z=%LsvkF9j0cHJ8?G&~hjaf_g%&B5YHTFf#~=#;2Nu{-^`yZ@CUqWoAK)2)$o@l#Qu
zy2e=^ubN?%ZYF7d`d{6x|KRVFZkAIY!QZ4?(Y$e(pw*8bs<z6j)eSJ-N9H*)6ykQB
zv}c;%#B~RWWMr=tP;}nKeXP?P466CRy{>=qeKQ&R$M7wzs|&~LRov9JTD-;Im&yNH
zCI7v1=Eba$y!Y}KN?YzPLFq%qy*IT%!+_!-xtC=wyKjIx1g&AyZLawgYEJ&oR`P=P
z3oBtR34zvRbX4ZZ?ok6ANcMY`ueWJf1OmcEQBY-^(Dar|n)Z6iykLBp(nfLxa6$^o
zsr$KJ>nAM&3KkT!0EuyDNWHLP29gsvy%B!eitrPJSvx|hgxUzm3zpr!Bzlg6rWn%>
zI_(DoI6+U~6+un793@R-V8=8D88Wbm%LVHwUqDzlre2~h0$U8A>44nqNZEt`Ryzon
z1Y9oC<aTdai*|R!MzU7!{@N5o821n-(r0EFy(QK*;zM>EklJstQn`vyl@n~hjE@Ug
z@+PVh_d)T<I@eAvOldixnzp|6V1W&5>DUm%hf0YKKDiSFb$~d4Ut0$jTac`T4xR>(
ze88+Df~r0UAs7;+$37|ATR}6NnvjSp*e@)GPH9+7MsjlvnG>ma@PxsoJTu9RPN}`x
zCTTFFy=YtZz1&&6q3C_qfdFH9^Xkep_Au+Tnc=kIu5xqVq2L{g#2H4K5=QXLO^+lk
z3Sg4dN8rhvy@pq(4v05c@81Q6-nbfG5T@WXGuS2P#t}*ZLtE%cAzgb5)}8Lf90HK=
zh+{5+{4uuQrrG;zSd0WMKj*oCnaELR0qzOd6lAL>8YzRF0+b-2fZ?RUNI*{M1T^*l
zbit>k^wZ2z6UNFg8U8DC4L|l5Mq5_?!)w1MnY9VU7HZ@kwl`&7w_fmfdK<ct&0K0I
z=e15{<~GH-4_U8iO<M!qKM^<mz}=SWQ+;^&#pIi2nuXi18T(3RE7zuJ2VNX13Kyej
zD2^uhvaBWE*G`Qu$T9X`?QUP&9Mbz1?|s`~5uZMP4oIt;V{?iBO<?KgV<i(Cekm^d
z*l=Q*XCc<{(fQminhojW0eH1h>$7q$b(R5f24VF{dqQ;f_b*%=TD-R^;m~KYecVO1
zx4b;CtYn0~JE!-^0^0`HJvCX0QkPB0J<bt6RlPqVw_#7RvGyQ!xEyZ7ntCeB*F8(q
zRIPI@vO)TjTSfh^<<1)jEY%)+SIB%6fBdD3i>(jvS@z&+L|QaZL2D0})BO?JTpZLX
zuW)qcb1CjZcK<-a!&QR8ARMg7awMl{<G@0MLeSIv;i};MElmwUmZ(VSQ;K2}S852i
z62>#uE0N3mLOmgLvJx(hU=~0LV+yYt_Deq({Cfr9@4(WUXU?)E(^xYr{OMop8H_GI
zrf8aS`Pt1Umw*c;MxVK8Qs?w?Nds~!8IJjbKWm0UflY0x$^VUtZ=Dy>ZoYEop~Zpr
zr5CrEpSgGh_o;GzOuiG@9dxmL_Ya)a+ihu|WM<<R-~7PI5Ojs<lMdcv4|9z%Smv?^
zHDXXQSmCK=DC79y)kG!r7jcHJY6K5|uHPP%{25`n9&;Jy9YG0zFu{_uN(ExE=*Lkv
z<Vj@;Vv&UB=+aFEj}XC#X?|3gupC^dgUKXHImkpn-Nabl%!=oI%H};$;scUpU2|y@
z!7~d_bcHW#y!QB_7_3_M{bS|x49>ESnp2;Xm-e?;8W~A6zt8^Qa3J99i^fAqcw1PB
zTUN+CS<$70jZ&I%h{*b-8%FKix<a`VfDf3~!Ws(>Z-B!m;DVJ9vcZ7QuQn_Iuq+fX
zD7>IV5d+i<OSY+7OVI%2QCS;2O03wx+X?3d)m56D9vwDV!Z<QQ5d{2<V?|n(|Jkf`
z{O#@=$JAJEtemRT(Jyzh6Q3<Eqg-uZSs6S&;5TGF-qRLa;)-t(KWooIMm)H1J2*+H
zXBkJa)@Q=~{(<j;JE?=(mrGU6V+32bjf&pO!$-I0MZcLdD4vykk&~xvsVS&_*OTjM
z0FFYMt6hYat5++1DfG>_$H`K0Oc!hUw=((nviYC+wJ#ooO2bktH#h&lZ5}BL)chp8
zS~a?JO7}j4<RwyR=E>uHtCYo(dU*FI9}=5xH55xqN)lLa?CiggvVk-2)8txSXB}Jk
zYx$K-dW$;BwUgf2)JiW)RlYr4(;I|QFa4I({)EE#(ONMI+P-pj4b@8Fto8>>M-t2<
zT~JU5=I9tID@=LHPZXROBe2Z}Y@&fxgmn@JG`-y_?9+{&X|NeSkb$5LIIV#<0R=!O
zoCs7Bp!H!N7QT<<YsqBM89c1C^J5qRb0GxH3_&P|#jJp}D+O$BDqTZ*;wC?_Mo;8l
z4gQn|#UOr)V;+!DIAcGO(NPv5`?M5bp<uhS1bM<zkN`KK2>l;q$^ozZ?<QNXdUlSp
zTq>GXCw`?$TeSUwi&pzwAYrY($I1T#%{RlL0M}(AlgFA<>8-f%rsY<5>W4jI$5NOR
zVR@D;!<AjS7ZDwB*U?ad^5=-$JT52V)jQg)Ft-H<fpsS{Iq&sNtOqi{-J6H(1Ms8y
zn<mbblCufx_kicmJxq!e#`{AOpfls3hbKa&MiTM^WDj!H3Bm$~bcG7nyDcB4a<?lB
zFqcq;8*8MQ9K(A^R0S~XWo5P#qbV_eqsBOHa=tOFnWomxc|oIfzH-8dMrgb@tz9%Y
zZKVI=^-n%*!f2Yg(YjgKht!)B?#}u!7uYb%1WLompQva(M=8>h%+ClWx=8eb1N%Ln
zG3335DlF?yM@dwJxP#4*--+R!K1>jkU>^9pB>4CJS&*M#y|Q;5v~QP5{_S`hn&9=L
zf|i6Q<5_+yzoE=tW%18!K|<rzsyG%{iLLCr;=P~X)~mvDB7oCcANL?S=~ItjAOU>Z
z^eW<CdX)XZ5l*|apG-B9Emo`7KFcpb^}e5O*?SAXMec9+-)-Wa4mw`#mbYJ6L1JI}
zysKN)cPaBeDvIo+!>3zb*1PZ!7t}yJJj_0}@4pZs!7GH(V;hdzf0#1fykc;Jg1AGC
z?8zG&dg~gdfB<=2ujdC?yePlCGi)jP){u5-m-;1Lf%Y!esLe^1$$<q&L%nI5_Dq)a
zr+B|_%wp>QHeQhRYW6$T0sH$_4N(my4x_cztN?iE>Tk+U-+a4a=51D@k}^;7`%@$7
z?@I)2=DFnDz8g(>sF`t>?Nky<_?jGU96+=y*Nof~h8McHt8xCMXF1ZzM(8Pi;t_UC
zCM)8>0X2dKgaSydh2lIAY{L9&d&(3&i&Qu!q~Yq+EYWB|gi476IN3tO2c*&l$TUIu
zMX<su*4UaM2(&mzZJ~K7;*{=3Ypf_VxCDwDV5c!8lvv-0P2u>4Sfzd_1ZhVYDQJ^g
zJVo%8!fC7;6Y%>U-hkn6FECVLal$ZUY=A~-y#X9k{-1OM`fm?4m1h*UH?CbSqiA;}
zx=D!^*f|||?5Fg#Vpma`hvk=tzDFu&QxfN;f@jMwf+mdCJduvpDH$H3ES&}51!!(*
zW1)TX3jx&y2=PQ@!D;2OALr3>vQX6oN-k-rIH2xQjDq?C=HggbfQXe95OZLbWPz=|
z{@uqd1nS57?yDy}9B6bbccFTvA!|WM*C2kbALg<@P+Xu^nYNlu=#MiWI!M=~d5LSX
zZ&g0c<I;GeQbxhSwJ!Hqd5`ylM4?!Dm=U@OvL?~d)y)`Qy^gv`4P980h9_PE(a=y+
zH~`C#(VA*_X_D3}da07aMt;K-w(EuOl*TcQA#elgT@2hnAwp4p0B%$&p93Iz9HPJe
z2&Lbz6G`A837=qfMbis9GY~<b|9~vs;E(kJ%=p*ac<>xy9rYg{ErfbjDGBcGY^zEQ
zb067yQc8O+6@K_ItL_bBufl67tCWRZ?!5Hkf)h3Evd4(>w$!&@=zs_Dxdsz~NvU?%
zzNKltvb`d_`)B8Q)Ye1?*dnPYqI?@S=Lfydb>($?D*SlcsclpwCA8Ds+)G-!?>*uz
zoKSN<26>b$4fLnQ?IQNciOC)SgSUEFhllU}iz^g~f}AWsrP=(chrLl}1ypC;p6)t!
znTIbg(|$|-*FhL8yzcxq?5bB~eFx{giH$Yrw%*u9zry1e`)kNF8u-H99?rk%{FHPn
zH{w?C34svTJ*!nMIsUc2kNBF!h0>W`wi^j5bN1;w=`#C$I!Tk^Ui;43Hy3}HF}{dN
zFUA9Yzrc(!Z;31A%=>cJ?D{1kqGcXx)+{>WC|uT#tWIk*tQ!bnp0+Se3MbJ9;TOi?
zfbMJDO6pD=xdLuNjp>)iNUv}*S&Wl1Kq@30QBh_^u1H@nSs1l(4lJ0q;uUbPZotZj
zcI|&ERMFx<Q+ms~cns*QFQ{ZC*sGEGfi~A0XD0v>2<R=p7Q*H};>T?yL3V}&Xn?6e
zZU!6&XBi$DLjN1t8Aze%YCfIm{3v+)woG{NkkQtu#-46Pfwf@?2?{fVM+cgH2K&Dp
z+PEqBfX7ac`QnNf@=6PKInHd=ix*F`o#O#gi6KSLITfReKm81k!v<ZJB)U=I^!uZJ
zI(*~;tcOc~7vid+Q~}(gG@yj4CMOI9PMVy{Z`VV0LEv4;$tgF&mHBs8spvOW3E?&F
z+M8BsZrR$~k}h!D=idr_wf1dPit?+Fa&zM|_Sa^idj`qs9(lU6lNLUew@SWv^-Nos
z{ydn0YJX!tUN`XmQ&l7t;=Ri0rXIP!TW5q0?L!iTUjEZB;8lhD2yXwJO<;11)L$5x
zjYFffKUVF3lG&PG%34>wv3)ZA^+vD`7<%B!;Wx>VBQ_iIV18jNcmlA9fvEopee`=D
zL`J!CYeAZD>83!hv$%{ib<IfwGcH@kjDjDybe8G;XV%L2@Hb<plS*E$%FlcaPe?GM
zYPj;+iMVO^u*0KDGO{kewZkuRe3f(ZbMgYs^1XHm56JlFkl{#1aaonG1_WOQQN}h;
z@TvL_&r7NHF}~?g)lbV{OLjB<HfX8YpxHWbal-4rxhDO5?Bv%^v`@iMze{S1c&8!F
zUj5SRC-Z7GI;4gkNEJBDnQO-{L^r%G-crF$QNg@7d*-^Y)7IVU!PVJCUVV|SQ)U6d
zCq!?xc^N;^2!3Q3ka9QZ<M^ADf|Xzz;i|s-W?ULkQ<f`2*;0IhJSOg)XJUtY<tmF;
zq)e20G94<uY%iQ<_sg}RT=ht9`&_@PVuOD(qwCR-vbN@jGzKfuhOT$k7G)h)-79;R
z)?9wz)JoOgHw5kXFEl9r7+>QMkg8JmG=t;K&HB7`Qi-Z*7gP?6drHcHVHvJmj9_(S
zcN%ahjFf0MX<fCCC6n|cH2{qCz+e)SZqZWDtrJa{{UnERM1no&1E4EJu5R$X_+R@1
zzCdZkB^ANW6d9V{C6|EHchE*l6u_D6C7%K0j#K|bO)`Lfln~zZvv~wO1>>T}F!TF&
z6Q5^SwqN>zGxz;)?}hlSR3hE{a`Tk3daz2Kl}Xg-z3yPVU3TrkuG+)T&bAGPpKUV`
z|Mtj`O<!&~G$<lRE4@QhnhG@~fB=EwPDns>5!50KJfd7o406$7Qmu0he<z(FEFSR8
zzha$w`DM%p0Dv5Vju}zv(zNN2-zg)r{yu7F(AXkCuETI34y+T%LTA1vmKTM^FT+Dg
zWoezul+3skoub*q{#x8QFp}cs9NBESPF6x$<AP&@xJ&j(_)nUCeE&OH3<BlRgHQ%$
z{G7Vp5*#?@2;?4x!@t8svH-yl&Anccp^U8;ChXy23V^(y{*q=$kM?Ud>Tw*DEX)ri
zI&Iw#1Xad=qWk~1C4sJWm7U;WKO<1-a=~Y_Wq)L0)dTH|RX)zwmIu7P%GwXu(@sa6
zH%Q7ckG=Ad>4uWPd+k6D?oX;!&G8z<3UEIv?qiuQBytz}?k-6Ac;(gAs88_LE@`{i
z#kFTyiTCZivQp#JT<jL>z4}pm4R5q?B+2-k((t5sj-!0#U*!<jI-6$}@JNhKYrEEz
z#P0L;H$&GLqD=?9%T5Hie_H9$)rZ+#t%US{@Q4mxb98+D`&;rj&;~RiKX5i#w}-!}
z<j&;Y#(lKh)v?Mkl*qFJp$?ueI^5!}84$Yc8~xTUx8}H-Y`5ASbAD3U-pYB)yTdB=
zp?5Z|2D+Qx)R^AdQ>Mmv=@D~5SC!4731@weOt060gn`h2fy52hvbu!#3NJ)_Ax&RQ
zdUmt*O1RY40r8Z!7X44k-BDq)9@7efDpzVduh{hzjRtuHx2-^EPS)A_hQfsco1I4A
zy*s<|4*mHLCXM#=?k4UOBed2g-JylB_|l-2*ZSxhtRsDW5>Bux^U?%$%>bg`F~3O*
za63a@M(Hi!T?;y181Z?5t_k6{=d@_-H$MhH0LuX()SM-zh?kFVYLrgNQVQ7xOd4Fy
z;G+vkm@qy>LFJ5ECH=8#qzRskRE@qF;J7Q6MSCBXBYVwb)lwnO4yvRuvKhAm{U#7V
z|4t<JL`BrFboRqLHkmLLk^5ZW7b5-fKre~Wg*VS0D%pgF@zkvr<+ErRYCInQ)V(FZ
z`GKKsQ*yJda(>mG>3eJSL?KPMz+I>PykEABE0t#O(CQ#B5LB|yK<R<0DJ&~Rg+<fv
z_e#X1Qb#=zq(LGuwES~SA5a{v7xMpzW~4;Hn7}UqFAr`3Kw#^mh~&RTW0FzoOelz?
zG<j-f{9?9Bfv($dsca@Fae;?3N$VajO_T_crC%zVCD8T}pf<x)w|u=DP4T=vi>3~X
z#dilLDtozaii?bMPX=%+M^TkZpy?dl+{GxY4^&=n_FzMcoEhVsAXe!hN{0XC?Ex<w
zCObt73eYOjYHzVaVC@g#6LgS&c9XwKB8K83czAd`Q9BLl)&CiB{^$Sm@^Q;w*|*9g
zNjH7#M$`fe;lcR$9@cJ)JjR5DFYZ^K)G957_g*hkl;pfA{k)N3@5-^A3;tuc-cu=&
zwdR$SlCP&slA@wsw7+9ie_F|SiEg_b4rhuJqg=HP%yv!fwUW3Vog%v&pE4b#S99Dy
zHOD}{HJuL&CkT$u@cE-@{d*Jp-}m))#R>(+)|&Xd<E6m_<yA&;H8Xk}HCZ`l(trmx
z!AEHb6g|wHMb=|GD@9T-<}2SfW}rQKg!K-?2G`bWCV^>Q?@w;u+E1KszF9vwEKF}P
z-Iyei{C1>1K2#d^A`@3mu5x(X{(;-ZYxDSSW~f{2Lx~lxo!{l-wb@xzlIOGdvc@O#
z8PtxHohW2#ifo<lW~m~q@|@o4TpyP0pG_?MeCt%}LZtW9^P(MEoozpGfsRR!AfNy;
zRV51+gzuQ2yOsacO`@py<5Pn5O5IApm~Q?XwoGuEt~tu7X&I%^CVJ$9SzS|td0~EH
z8^_?!VW1obq;F_>0tp3c=OB@X)k_yU#Cw6xfm&rtLj*+l{t9(M4!zwfqM?vi2PH3#
zoc5_$6D|j}q@WU|06A<gq!QVG=2D<_0sfcWn6sG`RHPA_ka7;fDH+6V0f(0Q7$xHV
zV~hdn#L$g%Gv#J>+;zbnv^!LBgW*O|MOk}GTNbA7Qc#r<z#F4EmrFC9dw&wMnb4ZU
z)H9pj%Z9hUvwVx6ex}h7Kxwepx2ZAsuxfIcE&xx+h}8XMO!T{8ss!8Fd>nolNr|X_
z?ra|VE8PXP06EqD$#*?hhgGiia7;`1xe87IGAOygvm9D(Jlc{}_gDUnn^x9u+8CXp
zVyDoRew*SAbG*s)#=>Gyc&a!OMVbQg`OoYVGvC|gkt5Fn2W4IQeKMg*GtSw{2lL_`
z--jdLpr#M0Eu=@VNgjz31%wp*K}$utV3i7y?x+ifBiXA4?(t=?AcAfBWV4@~`=b){
zubfCU294-1&&otgQY^lF{FnXF|CZqTy*B9O5Ub)TgWvNpRi=J9SuESYa536$>3r-n
z`;~ye%62>V`5Q}5Bp;o<^U$hjf7V31OP<Ra_Z?5fwo@MYqDEv|4V_DwET7IM(v5gN
zeqH2S)hcv0|F~iF);$A@_AHAsADJG+6g}MhfxbUforBdeh4o$5(6bANo*XTlXmX!>
z2%MJ-+cMwgJ9l;jaJ;jkyFxes(uMzW*6VLjnp6Yq3Y=M{D5j&N{M|_la_*^H@A9)1
zT-r;v!)tLfaUwWkVboP`cy(d6qf2o^eU8?ta^~ao=S}Ton1Zb5n<H*bnNYj86;u4c
z^)b!tC%ieG*O@Y|_Hstuo~iPH$Q(7r+8nj}bXJovxA=0##aqwT+O1jGn{NXBH5+vF
zqg#<sz~iFPF+QRC)!>*hy^MomF+|U<RmJ@av1v6=sv^Xa*oOn=q<XEh8t3VqIP%~3
z99j9`lGVCueJjl-C9LIq6g$U6y^o!4;+~ichC(jkvAQ`?&?xvO6T-MNUUY?0y@Cxb
zpW-rbf4ElY7ZQ2giY-@B@hoLd4v=KlhcHxz+A(#cNoPtcdjcW_e&pna_R7@-)W)qa
zHY{WpY}kOg2DvB<H<4>^|04?6X^$h9!6B9{gc`(JklIo%PNgMp;C7Nk8|*NcBM+0K
zUT+zl{Qr8s7}{>3S^Z9%Dohne$JE@UtHE%=eh>BVtEr1d;aXSk-JkO66N`5s(iQF5
z@pP{3RmI@)@Y~k+?`)Mi=Xe4@_CTV4D}!T9j@-5GwpC>myOQp%NECzcECDrMih-Dg
zM)J>I?*A{qQXq$-#W4X?S%Nt%-)p8OTG>HjC;T1`4)r5YfU!+YsOS)?ncx7OIFgi7
z(pgzSKEog2=9c;!<-Llw2UbGU%mo5g+$FeN2F2~uR%i4R!EP|jVvePCHiJ`ZAzMQr
zjaMP>YU@sk?)TveGS2kb7ELiWD9=`&Nuu%NPEx4Y>inm1l$sI<5CP>5M&lbg;ZNc?
zD&Xh>=O9#D*U5X;wAX+<HR;eIl$W2ub(k&CdIBZ|bp#Ob>g2!tDN&KURtHEOw(;A5
z|3ijsFF%A-G$>RtS~s`#(xg2RGjgJfb?BGiZM^cpZOcWXbN+*+sn1+FM1h9=n!2Oa
z^3%zeH?NjFRS$39mG*J&*l4Dc`1L0Oo;Qa!4A`qu6mQJ%C^bn6;dj~0PCwh_F_|Ok
zaF$|nq^<37y5ab>>$dmEn|6w=H7Sx<5H1Q`c+#@(Nj^Mzq5J-JWs>~k*BZ~OTvQ{S
zGA~8NxJB?t40WY%;Lp;%CQSBSXjx|3zpp%{^apOEzGa9_*g2D<<97VR)je)+ANnTX
z^gM}`ESnaU*#mQhrHVGj1c_1ybF8U*l&qysM2wl#c9}hNyT0vtlgaC?X!~=)eb*=V
z4B7rmSuTdod<5hQ=YGxo4iF8$lZF5HIXoYy${I5qz6eY|i6022HW@95leCHKpied)
zsP|3WKuTjylIeA)Z!AXTzHLsTh*ns{*_dW2ftU>8X8+sGjrCb<vv2zOJm+cd5c!%M
zx1PFSt+&>dV0JNJHjM8D{W4=v^Jl#tmRyr}_A@M{7H1k+`TDnVo*B?)5qd9SEOk|K
zc4d&J$b_tQu9fPdMva}lCh>q{*!ORvwIh=IKMqCpcbII*^`D&@htwuJ*iuQ*>tNZ>
zL6?EQ;cdP3Mb!#HV)Ih;s7K1|v!~n)6#}|pf-2?R%(u?Fk*^w`<7K?g)U>=^SGpui
zH2;yZm99cuQkI;-SWXWen`iW`D%(ew0&dMS71t`wUh2bdwRkF;q#^OyM^-c@;3exh
z(@)M+DXc`nwcL?x4XcvtPZ}%n$|UlK(0l=cL%G1#2m~C$c%-2h{F(#w-D5~;@bUsi
zZ(2YRcCsLLl$92$j5O#rP;2RH5W)!kvZmHk9A~d4jS`xQet}RakU;#+C1U-;33x}F
zz3u5nMG0I2`O`g5vNuLEpA#Em8@i{)J~7J8r#qurS-^c*j5>RJh~%x})}_`XUuWNl
zy|^xMCP*=JTv(pZ#f443EdB8Kp19?>#9>>CZka{BF&!|uY)o(WR@YVN27H$Wk_~7?
z;UOT7AM=5OaW9q#4*@eF>w5gO%Vua3{NxEQ)HnWR3lr4%m1uCOjkDMQ9WuEm5=kHv
zHni-t1v#g49>K7xW&;k-IFD!INPx2b2gyRpogR^sA7-W6HlX>w^}g|0MUU$sHsfQv
z$S^i4c?<7IJl?$N#w43<vG^;g#Sg?+%X$Zm-%`hOjku{v7B1mG6>LV<?y&ZB)lU@c
zKTv;)?mwd8{w`pJGr{kdjR(f&p{=J|&;-L{cn{KfN>R|&fUO4)3`YMPWvmNY<;7<V
zN*ikSm+zUsC`|fV!lcrVOg81vKx1u)>Gjd#(2uKWr8{El8jeKdiMEgUbqE}2IG_>q
zY!VjR9b$Zg4`&b!^}mc27cK7oE>4UOwBOy~^Ev5wtb@SmHGgJ`=S;W-I^#EHca-}@
zljOJ^uC7$LJnTJxMz?*_JAFNtw2v%a&#b;4EC~51o+ap;{-ONds>)Hn(LmwKbB?L0
zkoF~O38NHFPn}L4SM|m`f7jRBcMG4m&RV;|&NlaS!*CeBluvHg>-I5d(1K3%7?Mhj
zBr`I{cS{ln#?GD(!ds`64;xfxb4=vSN+zX_eJS+D=f93>(TQSU=G>i>Wa>Bch?%di
zu0orQCr2X4Z_ts2@$isolRe|@UL$H+nEHyjxj1wb)ScvD9b}eanz&;Z!StX^SXe?9
zSVIB0Yk$1ne<9d1Tl;9r472=<g|q8>yO~}Ri3?6Ya8c^s$F~#fKWKcFzudZ)>ZsA_
z&{KhjUmnpu#c+Y$6UU^ORTtde*>}KW%T2l+T@}ThF`j3>#i!|3R?$n|rIy48yAX^y
zKSvq*IQIU)IZNywCz?E%FO-NBcR#3P-4&EIXClzxc$T?Sx{=3mMM1Npp+o2nM~ImQ
zttknCvZ>QYP%}X9d#tTW-u%UHNBq2p45{I%jrl(+;rF)a&(F6N*cvg<PpeH?Oj{7Q
z@xhy+^d$aWAdPNB^eOsXixdZ=+fAaF`+RYfE4`0gT15t~Y_cMXPuCEApURFHbXQnh
zb{mq=iC8$#y3I}2b&1(VJcVJ;ObeeP@5E}$=+d)A&O25<k;ju;Ubw}t?wl=%6V+3h
z*<B(UT&A|$#BxtqIM164oSvmM+gIScVF}&fL|5H9zj5bIs<Rx;Prfm5r*sP}z1e=^
zhD#V;$3Wwnb<NMxG${h_O`Qpu^KZ5)ji}x%+qWeyLOgAi`$^+XH5-T9uYTa<qo1gC
zNd;f2{GNFt()v!uCY`Y~`*3-D`bHA34``aM<$I`*SgBjkc;ESG$b*l8ueAv7OtOwH
z9{xPTB(~ZmYh71UmV;XgpWZhJX}%QL)Oft4Mx3h2ULjnL`S_V_bLT}UX4ACw_8vA+
z7f>LFv9wc)CKRt2Y2kI(C_&h(*hK5Nh#yl#n-sr;bLnr&vcfKihgBB?P%r&Y$vLh-
z;tT61GlCDoiOwlJOV?JdA2i+6a=ISryS7JaK~<k9)o0j{sF|5&Y2bfRKFvC>;DSzU
zbu7_&-^k3|51hCjdz$t2J+a%_F8hmY=Toa}$xf55ahZEwhtHgBUr%7|<v;6#1^JM~
zgq34+kag1&Jm!{!ZO7(Bys;z!4wZryETmBPZn+L>kvToM@NGh&jK!tcx~U)NxFI}o
zeb>|rusrfbh36(2NX*B68)?&9LLAhPI9N$10chEEMg_4dTzQEuQM+E&vUdeubz7t8
zzmYCfPH5Er{&h!%K!pEr$a0=Ai>FjpuC-izZp^)hinDSfVop~VqmR~dR(&WBl!HA7
zeAZ*74z`b7Cu>#VPTq%iNs^5y945azE2FqWeH<_maKbHO=M2SjPfSDeH-#XYUVHvI
z0<Yj8f%Axtsr<ceNMWPGi&M7_SKpu$i-`QlO>uU{DKET$<AsCc(POt}p~0{O>GxR2
z>ot!G&7NJ}A98lRH;YdyCeVEEp{v%j5to>M&`c?D&af%hI>X?~(P7Qi)%1G3Q7$~K
zo{J(kTS^@~A7JVK^8yu&t5dbYiZ4XHffgWny3(&l@qFIvgOm4q=lNZOuV^f}es|PX
zqP$f7Kvwb0qw1jS{RKxHO&QLp%22#Y(Ie^Y*|C>Fw_^OXp~zb2@zrdLZ*44f-zUzl
zISX+X*jCSuYuQ}`@c;7YAMMj0+;3(l6yVa8T7PE#+y>Tey2s|$oe5G+r8L6uDCfwU
zmdT9hC(n!1G3Y*>qEFAeW_DzrU&lfzrZ9OjbW-Z&@pnm?mtGL6bZ;Fy7Nn;t&oXhI
zk%52oA;k)84R3XeNw^hYf5u>=|HZx<u2$v7US&fu{V4iA)w>C%G{o0~EiZ?Q?+aZ1
z^yH#{r1R9vv(YD)cXC#x4H0fM1g+$E&mV%L>4U`ce|0wf-}gCi$pE5A`r>nf-w6@$
zee!Rf2ddo_>}@c}$5j^@?C|KQWElP88M6vUHoLc;pSI<o9M^{da)t3tlh*Su+E`Lt
z2m=d+guQF=4tzn5${uqt-m>g`Uu^%eB#!1}$(5VTgTYP0Qc}x!*<0_Wu9<9^TyYVY
z%{=_Ut<YG2x@c&IccS>i(m6DEasR%%+WjyWSmDE<rI`w=AkuVXCsrEM_-lB`4(7dy
zsb7@gKGnStcd>1`Wj@${ICzWGwg|^B@!W3qb;QWeBr$GRn-VGGckmnwTCOsFQABob
z0zxvcnnVPTdXQQxGxP#UogQV70fx%{zOqa^MNB@-7~h<hFeQGM<sC@^!<0<X)(Wi(
zesRv@^j*GIwP_P;+~z&yQ9CcX7E%NqP%VzK_IBi|1W0<JjXs6?(froj>f_F{7lJNR
z#q*R|O!(b*XnFl8hJQ-~RwIR-Qb|h8K`I4mO>%L{KMCX~y@2OfvCD~z^J`!3mJt<3
zu>$9y-BJ(FbrIgEdEN~Ao-Z!<>fPdJsng9BaYNJTkr!$T<n8WOzvoK*$nHsQzi7<=
z`u_GSJEZv+gFlxoyGI5(x!Sv|q&yfB`OtNPMW)W3u9b>T@uq4y$8|C8Z?x?Ow)$;d
zACB^~oL+NrKa=UcQocL-_U$X#^6eH<$NPKw{3`LJMU*!T9OrX;9#|dVU@o!*$K*F#
zWLEdDha%PSOQXSe6h`X;2v>)sr(W~OPw+5MdxW`TuZNawSx*wdGT&rArZoJPKSE9o
zfV>WgA2{uiOq)m13`^4mMxBlgdJoLD&=Q25Xe|*cDs6(g`Z_#cm+S)Jz-%u~>-133
zF+*a6jc-vy?L=h?fk-#}d5Syx&8}-=5AJ<Du$XaWCkO~SqK7oHuTbmo*F74UAxi7Y
zy6PJke`c(OP?D5dZk`aN)5;tm;jC&{XJ9maz+npuNCIUzZKgU2u;KZ;`rv=TmzPmY
zE}~)(<^FSf7ksFBkg5q7hu$1eHW-5b@*LsTdv!~@fqS-b?q3;+DRmxF)n4qFHTk6f
zp0_V-k_??2^Vx3)x_^(qGH(6Zsp#5>;j13b_H8R6X-nE3THk))#-HZrH;y0QXnk+b
z85#x&--NkT?nb=qNxj!Sl$V$jLAk96A~Uq{{FCcwaF7v62HFU=eSWWMS&Gxw@|<kM
zsvcCGmGLoGJ6sezw<<AFlde`4ld>R}MOEE)FF2a-Nn`75FOkm2(xI~Y(=(9)i)J`^
zXK4<g^W?W%G_OkK%r3!h(E;5ip2^DzuCZ?(?BXyBIRCJ%m*lI8yYK22g0jtY%Dd+s
z5qiy;-ZgGllfHcnE}&V|J3#Y(CT2FcouxOl!Ny`HFPCI@^!5Q2uS)(?@<-zr;_Rf_
zM+mQ$L#X^#Df}J2W$7$uP^`QvGPzDv{w{T)FCcn6hnRcDb=H8lsyLk;%KG=DIqBaY
z>HmHHPx~(t;r-c8Op;s88(fv|YFb;>Cq;>D)M*OXCT}N`A<6+n3d1MB1*=_zwdqR(
zeSH2g96Ez_Sq#Z9<1_2O9FqEC6rB2QzVbdv%$fw@!Sv#3LTsO5(N4`^{%QfnPe-Q}
zBq#m93J|<ydSh1mXKKBjq<p^|DykRK&ok;gUd6f+zHj9v=htKpb?1%69(|Im`r}eM
znb5M)EoKsnYTpSbvE88j@+EBXz}&1`5!D`@1c#QWVnxZ#6b9D?Z{1Q&08Ro{PLLj(
zBkizgqc6yZIFHb|G~_Gs2WRl?FU@+F{-u*{_A8!5jx_3^eD95?+^WwGZmLq;mvpJ8
zyyLcP{{58N#YTFtim=y9n5p*j{hY05TOo2RxM|N&Ns6A}^PKaAN==i!Gv#SNaA&Uw
zUL-!~D}F$Cl+aQVKC4PmKr@rMCxWU)jqJ31BD=rf%Y<0(<e)ppKCE$R^arvRjR&us
zXa9lQY3w9>VS1aJNw3L=xs&}h9E@9@t=#Qd<=QNMF(q-|sRq*lu3Pc$Dt4xzuYXS4
zYF9UPZl)+zt^1@vagvY5(qOZdxTDf`rl{7*nnUtK!4pPLI@Gyky&o6Qox7qkuy;wF
zd|RnU;ax?k2SQ42*N9{HTwlw{Z94gU`y4%;v?D$v$v1`R@f{`=Fh#JV&uBkU55zJo
zI6~G@nkmvULBZA6df|=Oko$F-86d&bT;F|$fD;7RN!KyGC(z`_8bIv@lSgXKOB?^W
zN>zT^&k0BSl#{a!j^}z%NzlR!7BNCFzXR0)xeu+n2rPKd14~3s`8X=n%JQ$2fJb>j
zahr{M4a3^p00VLSx>T@QZA0FbuLI9>!}_&u2b>!Kr{0O0tgLfQyo0Zz_jZktB{2Co
z^`)*TI!J`p7^iT({D16yc_5U1+xC>CU4=pzW#1A}2$Ra5WJ`#tkgXEgw?Uy0*`qAk
zDW-+&rm~0ZOZI)I8C!!fzTY*Y<?g<p_j}*xdB692p7-&Gdz!gtu5IQzf5&;8$9bIH
zVPo+Tx13X+(Tl`^`XCIlJp4fk$-+QS1J4y_7+|J&&j3LyWL8L;PYjB)q_zOE+?^XR
z_fY%<Z-9{o8eb0&d(R#JN>X*erlJ1BCTfS0=R;)QU{$7r^m(U_3?Ae%I#t@ksX{0D
z#EN5bmzUIDmeFPEipf!pI*yt;{!9aT@f)`j8x{Gg*%YSq+k<N=*kgv8ESgeOGz?5!
zSmUz5Vwt}iiqmo5BUW-q)b?5qXb}My?q|{sXn$~&>O!?JryOte8Z67^l%gW#$D3}}
z)`fRx?obcv+Ex<8a|I2Zw{C48(x%Ge2QAZI5W15uO4C?$vK+Mxt50}>#z~=d(C*>o
ztUdJwt60qC;J2co0qXc4#Zwe^dH5Vu4>)&rHRyxoP3nQlzTyJ}(dJ?v{JT!MS6jIe
zt@_<TVLA8Vk%4JbRMzq`!3a6R;eIK^`{msMkCM)?zRC8N^XpoE%QAIAQK|}Eom=qW
z03%$z1mmWx{RQ<@dJX3N=A>`hLR58^-%9Ti+wE<axId=uh`S0C@_ovr7Bns+ir2G8
znE7jJJ)Yo(T!jBk+#CPF^V~62@DR<s`n-5&C*gAa&W`dRi+4(OCU+cqyGln_GCJ$$
zn#32a9VgH8Q_Dn+eY|eo&nK77WhUFrL|@3kZ_J>Ujy{;TB!#+mH?jAMj)4qEMt8xJ
zByTO=%J+Sqr<j_h7Zxq2m9aaC`=5%tAmj8#wcnlZ(UU9eK9}$nCQ+F{T&7??O*JT$
zy(;i{RP*gk;owGP#iMnEg5yhVVxP>1pF~i-G<faWwzLFP_nV@Y!(GOJuvC_g!wZ-b
zg1TSGRodJNkr&DeiLA`yC7__X*~1SC^%pNfy)B@u2j+>=Y90WgD>xSNC3tRTF83H}
z)dU^PGLSkm1Wc=g)LI{*z?GC(?itbO&v;Qt{bFtzuP<n{<a-2m3^ZPGK=UaJ^ZE<w
zTTNL;0Yn21$z7W70DVct`53aY28RfYuH1nLwjM|mP5^^Kp=Kb}rz49LelZ{fDzVGC
zWzbDZ(u<j&=D7r%q@a$ZUm(*fNiAU5ZSrB!RPW&a00kl?n7T06k$&uMurH^&n)VGY
z-vx9|w`&Y%Z6Q~x<S8d2GXkOe0VpxF>(xC|=x7~<L!K7EAnS?Ohsj|Kb{agRB~2ws
zO1lYy2;fe(B#|xpLF{t>b!aC*r&~Z&B((y4AUjv{>VZ&Pi97h<Nd2@!Nw?T)m;4&+
zjC60^+~M-07b;WBs=n;S$CkcGkR2-VDYSYod7mX^nR?b`M9094MUkm6K>4^(75f5b
zxZ_Y><ZxJZ<tTqen#iYxG!4zR$gwj!c2t53&<+t4pm}s%NM<G;_pFiBZwDlGKRO>t
zol0u5fW3eL`XWAc&)fbcp_bdBCk{>xKUg>%N<CtMk!*X0-+FCNzNY8-S8sON4V|1v
z%b{8FSNlf?M4o_z;WL;f2)+Phy~ysfWsEK1VM(1*J<lCked#sQxSAJXM!^oS{SI_W
z+&yLo#2ONVDz$*jIL3X)O2Xm|{GQl7B0-tvIBy)?yrFfBTh+eNYWOtEN%&Q&GpZH1
zJ&vk8C8sP-CTiOg6_&3=aHmCHr@i!8w`vhN^0}@r;-enoz8kGFy=Pr&-9yXUxSBVH
zU<8=4)20O}MYMukB}f@sLoBI83B3={+B8}%sb6_cy>t8K^IUbelE^rp9B&m3ID??p
z9}xaKl;3zHCscpqI%qqTEDSN?OBNIHo0lo4EBxjU>37)FVlKXWfJ(jFss-=jN}amk
z#3J4B;dXLb&*d^#3wE0Fw`Y%pI&`1q<DqB0*SxQz$R$N2BWys+Ng^X^tj+L4t#!Um
z7}W(Qsiuj@B^7YHPouJs@5S91&8J_Q?JX>3C<qy*;J-P9n5_-`rnr)VmF&y$1>$s_
zNVS?*xQq+hs$W&p-F)WC5fs{jrApDVH!yI~p^pJTm{#V@^@PHgJH^oV@n$eO9E{aF
z06Dg7it_Z@3{{}O4GMYz(m;v~>Tsta7Br$U(G)oV<N#teWK$VMPuOu)(EFLqK?(33
znE45kW#E>>0|Rw9Ktt|2f7RC(Z18Whv!Hn!mIb}_Z_~56eHSkR^9yKb0O&#|Gqj-F
z@)!uNX@;amX<uM@4csKI$x=XOfFQ_f#hHSZbPul|phbf#QeeQ0{py1R{)$&|J32Cx
z4usv9tNJXa#jI1XWA5pTNXy&oF*mx|o`q=J7U?he6%k%;FEw?h)|j!7E_x;fH#t=F
zW#m%Y=c#v){e`xpy9U%z4WUP%)c?cm&Zbv4q0UA~s0#uK2y{R4_Ra3FtSksR#X<ic
zYX6Rh*cPp?U!OPKGRn<$eH}Ml#r2A!^*%bTvEcRA?3ZWbzc3`8dWK_i?OKERfHYuI
z`-wr13hFLL>1TlFxR$BSpI&ZBZWYvZyg+sCJl3yWi+6#$aY)-a#!HM_5Ud@1iOqjU
z!~>Wc!sE0-jrh%UAy@3Kg)r^<<hn-_b;P26(i3loUQM&<bBU3W6I(E51B`LzVxdzn
zjbxYdw~xNunRg$9sh|&5pP)?2akgF3JUHgpj7)nN8(XyoW=^}!QM4X&M(rDnW5tVS
zTAbxA`II65uKI;*(ByFYjxXLvMEY`8f@eM#ENZXD5^KVVA~<Hu*Dk82o5S#l=J`ku
zA{#;4eKb>O!E>5vhmlqK(H44&Y)RJ%qFK#d#xLQVmwoG8BhKZsyRC)~&%asT5ymmG
zoYNiLAWJvO^6Jk~{`aWA@%%5Y(Iy|XX}AGvqo7FlnVwTGPK63p?|Kntix@>4*1BxM
zRre)*`)IlMhG_VxOupgbsgK9A7G|FX_)9r-ZI!=+kbUbr>VoWNK3>7NyEX7dr$EcI
z$?UL*$IikrHPoyDVe>kiw%R<UcLlBzIhN5PE}723jkVl*W)&vO2u?SohqC^+a_alK
zu^*B9M3X8lq9v?rjh(2fc~Q1SOh!g-Z1=9v)1$|C?U4wV@V97)5HWs>h{au2%GG_7
zqAvL+;L&an$aMJ{#3`A;bYV(?)2vWF2AT2!2p0o<NC6y3e0`cUa0E>Jr2wHM@p3{2
zG)sdV0ZB7?cR=Hd!OY|0&?%s%P_Gx#)JQu94JjrmVabvM58v|u_5cAK$Z5gg0no8@
z9Y7pt)4<${^cG->xIVlDsP+>ezb-Jj6!e0-Lt@kG;faYLSn@PK0e2B;=J9J^fT<u<
z3EJnOc#*W()5FYk9lU%8F#|4YnXB*4no*2&Q`b}zObAp<)7k_L^n;+68L`4A_vaG#
zt@fhPOBHMe;Rk#Og%OnlXzi&|Sukjcn*n`!U@vLF63Nvjsq;5+{v!CFsqv4=4iu2w
z6;L@KV+Sz}`jvoX;bK<S`qWJbRP7tO4BS?Ex%q1<V4Iqw=2Z7vOlRo#3%`54pACsz
z49r_aNRB&HmtLQ)^?Do2LH+fLjU8#S*YPV!_Vr1YPV%<AF6m$IfVfpS1~&LJ>595w
z$|VvCA+rO2h9xRoA~2dLCGK?S3j@q7B|8wNUEpTuB8Ypnb}Xx73hdAAo{0-a;QIBw
zuHuh$$@I5h-hcjc<1wiKyEPaGU6Fx+yyM%-mtWPqXtmP`Tk(fnad-7-KBI5-SL?v?
z^O+c^sXqCCyDk($U>!|^;T^*b<prMewj>#0&_&T==Q(cPGHpsal(Xtuj-L98-8?!F
zkRFwp#`M_1yz%^7L$iWAs1_dL*Kbc;sU!mnvR1@wI>m0?>Q@lxS7VfA^wPX|3`XG&
zo7O5s#9zi+C}lU<te8$sWjvN=657SfJ-r5N`>LDEiFrErtmoaW2eu}O+8+x9=Ui2L
ztX<N>_2k1PYG+n3v*+g3WF4p)NWk;Q$_K4C8-M@lYrp~UPbGfxy@Y(J{^n7|R)Mmz
ztOVW?pNAFlXTSKMo0&Xi-P~L*;q}r9JUG7boW42>PJ_=_p6yof`D^d_I=l9&jvMR|
z>d1pXz);)7K6E=2tJo4ZcRgezK&|v~)tCB3wpljTAsVE{<I!yEs8)NM;r!{5!n((G
zZzLR}k2;vCpOPk?C)B50)xL&P?!SS?rj!OIPneB#Rz+kN$(GvlOi0mNg-y?F;|gTv
zTV|v3OY%=*%u1?dFpok|mR?K0X8tK!VfK@RNLiAa0;NO&G^!6Mj7VG>7)DHjbfCh~
zTgoHFV@B%<a#Si^TZ$cITf|QXfEElAcN)?HZ$<DiFl36>Z^xCZz->=i@ew62Fv+()
z0#(jyDM*+rl^z5xK-mG!zp`SzukH>R0OjswNRZi48FI;;#NZ_X1F+Mz^nKv{pc{!c
zbZQ6G+Y|<9xSwt0Mop=pQ0!5w3`+r8uD{^tUc&cHP9~RLy+4z0wFYymNc%Pq4{UXH
zzM{gpJN>M_*b+@s*NQe3KU(Aw+iMfUVAZ?&xi=A6F0<{!8QKxu0fUPCWe=->eQG?g
zsO*r$!9W8!m_{fEnO$T`-3O?b_v{^zvH~AeA=PW{P~9Igx`!ej@CE}|4_I%IR9c|l
z63SmetbsBD@-22ZMi586XpH%CtM{OFfWjo>@~Wz3U1PRrk%82K52Mz*qS^cbA8SXD
zvZ&6?Oe)NE9J$M$X<cKH>~++>W>LcI3~}xVh<q)l&VW?|EF@&w{68e&G!$!LKw#y|
zyROxhQ3S*s4HzRh5JLHYS%t#_f1z%=nDwo}nR23;SY7v!$;4^+qjSByH&wn|nV&pm
z`T9#|=gjV*^4BvIu)a;Pjmhyv@V<%<k2D5m3Xk}?CO^7eG99tvptf3Cr!5_wAM>6N
zj$oFjkkfjh9L{HFb?sG)Rah9WT4AChR=%(JInG(1`(%gdJOACoV@QH`WM3Y^eQ1&D
zYwr#`@&eZGp;@BYr59#~H=nXSq8d@0P?QXIWFLv_r`~23Lsg%V<;wC!;O4D1^Y{S8
znHuNUlEF@GvH?zQbKcG|mue?_%`N(oFXl{UA~{u*1rjpfI%66fslnH>6fzE0%(%@P
zfB*0`*dsUqFPY{W(Dq4b%yy8h?EO7%ZF0GQN4|97$FYq^MLC9ljmRzpQEA_Dpd6F2
zQ{WvLMACgD#v!*OE{H}qcD&u7VoTrLVKCZCe*c-(XzJ^TBf@)RL>F}SxN7sfFv&Ox
zf?r@joi(Fln6!hL+Rc@v<8$in)_kw<u~^Jyy!L>6ldZ_y)eu25C%*?X8qH3}2Kkzu
zJ+}`|E+3@w|0VQoNBg9CrSSr4nR~2>%+YCE)j$Xjq%xqbbHFPpjZp@&+W^@ll$Iry
zdqG80^<zn!-FeJ};*#be$!M@wIgh5g#w(sHAb|wWWg()T8}j-92u_Cdd7z0u0HP-C
z&(3oJe*k@sD?p7)pc(!mXk}t*dJVBmAd<lLWK96QKpX{>Q9~S^|8>v>V2}(Q7C$6D
zA{l4^eEN8Y76>grsVNBEeQI5(oOckm;wT9WMohS3Vn^@kwHY=HP&@>41cOFX@)n7x
z>U(*4;j{KpCc+A%9FJ_TVP_IX!Yf2|3!?-+4vZ-0^_V>?46v>-m%QPQ;JrhSw&mkv
z`%D_Xsqx2gn*0Yqz*hnwGZ)-Tz<&`mY>;g1FK!~c9zx_H;2go~a2nLUpMuPnY(>u#
zqRN~l$e+1i<WOYbX`Y1-`07%)MQD4c;!_($;kAZ#J1{C>o-RX7{8MG)-cdVYf=q^D
zhCx=6?|Tftg0mpH`P&){#>eZ+*XFBsNj^R-3*cR?pD6Ed)Q`c}0uBKXc(7-UYO;Wl
zfHH1V9@co@GghbHKhuCzQ~=xUxp^$dKGC+6S7!OZAaQBc;sH7pV}kNvOexLFxH+i(
zrg8dJ+qar0?W#3~$gc=77I&jXst>hujDTU$;!|qS!%WVKrku&-R7<&VGBit}9?yr>
z)52rWj%t=!4H@mOV78>?Hbic;Ur=X`Mmro+c(`x+`1=mJ%)o@sjEqP5(^_g(QeOh?
zxPn!6k=yWRm9ZzhZY7MhR#_Nsp4R;N|Hfc-t(b_nU5oE5^|D2=6$@@AQw%2*l?9UQ
zGvo_a4+aU)hGx%EHUFoP%m(WFPa$6>lTw6_F0vFwU7I*jzx29lPF5JlE^KEb^<Y&_
zrjoOG3qBeOMS24#n{3%ZC=#%MLJ?b?+E|G{+7b$QlGpPDAHb>SF%MZ-@)||;P{KUY
zMvtx71xaX6^4?<=*Gp$(T0Zf9nN*o6a7-AS>++2;R*hlQ)Pt~IYkUE<FMQ_$%FFGD
z>q&Lh6Fuj%lO~R`<pzRqSNUmyrO(rG?_FfQ!+CvvWA2uOqo&oG4fsoJ)EYb0HPhu9
z10F&05A|hjt506>zG|(Wq`(H|-jck}JU!iy%j*IYNI=1d?rIKo2MJv{&wH!WB}^dE
zr4Z%tC2>;Z`lvFw@(TK}fX{~-*;g?kgYzO($mIR*s|b0U0ZUh~7d7R3>!XH$NGD0e
z9IbluYlt)q0WtAZBY<+0z_D_H6{^c#@&w071~$;+OBGZpPck~}0L!?~g-R(^JSs#F
zi96FV(o=h{wjq(}{vt~XVO8v#UC@^&;WxzDp9NbKP<!9~Hu}2iQg%^$K$J;tSYl9h
z0aG|v2<UtO$5|8|0OEpB6sRqO8Nj@^Nx?q;<0}Z#ftJriz=;7|yA2xT0TpK<y*<FM
zBt{p2z(2YE=oqlJCDc#vyz`B^RBdHB_S*9Nw{%}?J6Uc}bM^5nlwFP7o|SXwXd?W$
z;N|8um`3N^JX$)X_IQ`lY-QPyH5l6j)gYQQTLR_w4P64E?Na&k@102?Zv|;9I0#?R
z5cXSE&kb-N3{+ofwSb!o#0_``@8C}4qTRK<>;&loE?~Y4E;_anr5s|bd}>}{R}=1~
znj1UFv{eU6irlwhh5_95spdE%gYEW9=%be_F0H{bi-Nv*muQwYYo_s41ruv~-}YqV
zn#yo5xm$Z!UWFmmnvKrK2NbKL9NX=t_NZP(X-abRO$)y7-~iFABPk;~|9ixeQj{_m
z+=M?Ui@(+HCqPlMC^Wuo9Ca%|gn{4rEz8T&S;e8K@z-@lsH3=pd)q!w_V%a1y%8_w
zP#()f(6#(tqoCE&ep&DcU}=lbErX!_^Cz+K@Bf}_kqGSi#4ryK)bnT%mtvIx4m126
zhhbGqd%YzvAM5r>N2ey^;Q|(RAhywvVV3JQuKBRnzC*>VcnW2FVE<$^msZZ)KH~48
zO%{YU^B<wD=GV~Hu{k{d>%%qJ&V@j)m?N&I)K!JI3%BMx$PMV1l7CJHjFREIQKzp7
z*}M)-xD=X~AfO@NpN-8vmd&A*c#9+dDo92*$vSbdlJiWG13V?kPmy=WnPm+UL3X+f
z)KGy4hos8{stGc!C+rDGv~`!ZP%_)@3B3{Y89{FW9Z_R>*B~qW(_~b(W5*qFQn&qP
z(m?Vwe&9#}wts#z#Kj$iW)rV-as7>8mM}0bq@>&oV%vH}NHd#Ry;s?<CK4B$?^k`Y
zUPH0!%l52n<GdS}9~ZnjXP_+fp00edhub1<aq<33F7JrvR%23Blk@3}#d%gQa;aBx
zHHWWL1*R>CMs2lddhVs%6EnJ7X?mR59gOIMoQdW?pN47;#{h2#0<XVjmoup>BSQ*o
zz#*61cyyXPGaaP2K=hqNN?0MUN&kyr7O^gH+WvWSc`h$ApJ-Kn*m=`b*s6Jj+Q|$l
zlVpEgKwHyC;jQz-JPI$hmZz`K$C?*-7Az|AqTfuvr-%=D@2t7plYJu3&iL`vBZmN1
z2bon>s4TJLGg|%8QCa)EF%6$1b0C=It9<;Pjr~2;9X*ZjEl5>KH;H;%F*TP#aKs@(
zs3WiSugFtfZ2pA3hmK%zFxxgJv8if`I`YX45#fpbFAT1{4gfW6eP^~8)IMnsQ`6M7
zLueKqR-+d%fd-tj*XH;O5vF|;jL!}B+RhHv?LN#*by{4?V^s|+=pkb1-9!JTUSuct
zlt_U{CzGl0<!?8nbRG_cREaoHzmF0uLG98pm4?3#RBhsKx@ncd5ol-F@+3x4GDKer
zC4f7)Py6%aVsbG(5nVTnxb{id*U|M<(T#7Y_*vi`$~UAM|K|w*d&J*({*PQEv0gYi
zPOUuRy7X?24aB-lrH?TtXWIiSXF<-1H7Wh-?SugxzWa)wrg1&p5^14adPV+DOhSVX
zgFymB+--%?Rk@tTqmqF)?rM2Zy39xgS1nV*W(D}$=Tq{NmEv34num0TuOw&0Wc5tu
zL`k2OtQ;Q^S?W)cefxE&u69BwYSN99AlyPc=<sO1TT!ywQFxIQ?T$A)?F$N<kiJjz
zX_GI}^W(s{A&*=J&s{m)<iL{G^iEHdf>SP?2yTG-N~XnO<f)X9`8<h%J3u>S%?;`j
z=gt0O22Mc?*f>6hA|;sq1Uk4OAqaHeL*tRb@HA6Kut6F6l#9ob)D$ngPh~vd%cFVm
zV%$|w?FTC1Q;lG1CnVCP5_tl^V}tAwFEwR_bpz>(gn_gP#-w+V2bP99V;+d#Sl%(1
z9k{>C+&SU=2g|1GF?tiIBul0H6GbP4qc4}-y?YsFbmDn@))7abZ<HQbg9R=vYahOA
zD>_umKg+;=`=vSW#p4ELyn7P^C2To>?QCP8dV@YE2*^NH?XHm;AE)O9qBN+JlC;eL
z%5RFYqg?mnfV|G4pk_%x$k{>d>X5h@j6nHOmsh*eEo*#x?(=}yW8)`_$s*`Def^&=
zv_(1i7=&CY;6KUnhDn}5N~$g|#lf#scrt>XA)tkNW{{ZDyX0mxq@5%e$W<ipdBSGN
z7DQV$eG>(!>H+-$-VM-hP}9EWk2v=|-udz*j<|2Wt0+r%8q{q7=8Q`OjOq3=ip^Fk
z;E1{7i!4D<im}B6N^?O$Ns<(lBL5ha=8o?w_trn7Wl<e2)Pk8r8-dJzK}eL2S84XT
zTOIsuaSm>>wCB9EafHt_6Z2V3zK?r~uQ>K6vc-*2alJyMOH*PwA7ML74(M+6FlP3S
zxbf8R(N<-asfrgufOF#v+=Qo*#ajaGM)ecB4ZWVJae7Nr1*W~Epofc(4Dc>>yEV6!
zz+0Ax@>Ak+YN~=M_Ah(B&KsOv2EQtFkn{uSE7<t^e}4@Ih^z$ZCTTVYJ6E{YgREO~
zvLX)qPY=%I)6@sLcnWW04MN5MZn=;x!-WZ_JkE0Dop7gO_?zbG<_DPx^b8-yO7Ibq
zH1(d_S5H*1w3Gu5&mzG^2ix-b;mgkLaaQ>}O%HDkRaCRz`r_?&-v|^rpp3QolZAD)
zuBB~fE&OoIkj)&U3$`)-%ePsRQM!zwgD>*8Us7wHmJCdtKrlvAWvJovuptI`p@H%y
zQPw&d8jkRUZG6n7S67Fno*3P}G6cUUBRL)DcQgb0HZ@PCi;&UOMvxyw*0<|6<FX8O
z?}q~DhuBrxkmj8_t`*3ikRWfUJg)Q3;PC6B_xqOwSHHL++mg($-t#=>%4dJ@;Ek)<
zyRMTwMUJYIu+QJ1>OcgEhjJ&fN#q}>y7O^yyMT2MNPPh=0+I1Fz?vX<l1DvW29*wa
z2r5G26yU%KAtw+;sg!;&b-d!4Z14Q5`2867yqxw}ZWB*)_$)RXt~ioIn16f?wW*mH
zmdNutAxmxlvugzsJTq+B50fHW#oAs7z0Wk!#Km%+q8kU(uoK~xDB$Ni+`UjnC^jz5
z;IpffnP;YyF?ht!ZuoLREaicv!kC0k%)nXpfJX6&8f%pkzJSM*<ztG26eOve!N7EK
z{lXm-W$6xI1*{_}*?kYG5Fx1Ro;3lx;0gFgSklhZBqMI1N})&tjU!-j_<g2i`h-Mu
zaoEkKuHqJE*h-BnY{o3fb)VviDW9I6l9W!#xm<;2+l57zg;AEnINFY`zN-~CeTMuH
zY2S#*nCoq#RSVUz!m7a>17aTi@(a?*Z!H><<Rf~*M{h~E<QIWpnOTzfJz{`B>3LTX
zqG|qWbY$b*)0@P>#N#+pi5<|bAbGw&;Nc;bOhVlTIV+NGbrQW5u{2A^mcm)!S$AxA
zpGR+#Sh5=RejIQwjw$B-Qlz)x*`v>SvxlqcCwdp!-blQ{J3Wc=4dZYWLYM@8>D9w8
z3|Ml)g{w;z1NJ;mr|YLe9+kv4?yI+<2x*76z&GP*<j5)R={Er-%5PO(m*4w|{N&vU
z+v5o*v1Q@+`d8@ZSA}a&R4`iq0FI)dccpiq6`jir22?HrCqSzj+agl}cS=MPIsHAn
zDl0&%T^W1s=gRdxpV)Z(e|b&r^wLN(j$!BsqIaK9Uh1*gzq{eWGvzH$mW&=19ILDM
zzxP-HU5oOI-Ik5H*Tdz?bxBpY<9t!{1kb)J6`=lVblV_q8N^aOF^Vp22HeU1;y|2U
z$w^(7>{}xrSi5PfJoxt;4hNr@DnXn9<&Go81wOf+?^Jagzoq4*lM5caKMNkSF1xg<
zrF3UXHgEr;vEma(2n24dNW6Roff;vpT;h#mYoqZLx<SErL~?<}U-+cpMOT?E*z8rd
z`(^bt+~pZOJOD+y(L5+=`PdvL@!BfX#&qJAq*03B3+t5FC>FP8a9l>UlzQtLZ1eVp
zb}WsUA)dwJ^ZTTXVIRNFvy^HXb*j3$b1N!#%QdiwU6sd9D9-KLQzC`f(UFCVW8eAI
z#oD_5*<jNSRCykU5147+qH;VLM8eNmZ7+}F-VN`29sI^6cX8CyEi9=i*1#f0F@qu9
z47fr7)6T8X2pUBtF_MoZuw!309*6YHfYAyu2E|Gbg}p!}!-KjH^wp5P$16HniIC0V
zK?=at6SQpThC_s%uo<lv7&Jk9@fd)9XkDYGGINiSpd%XZkA9FLjNjgpb!qpg|I4cb
zl+MXO(>#o<nOnX@`PKL93|{^~)i^cP+~m^r{KGzJ(%}ojVW#pKpIg6aWH-5=q<N_`
zwAos<PdwA|BJTB7^-ufrKs#yh0s%%$!De0#9tvj<X^zqw*Pq8e4tUc%9DYRg*7nS=
zM+G^0t@xC>RtMiN@3}X|%gdcP6si?e)9Lhz1Wj!(lAPk)K~<fbHd_F6qstz?OkyXL
zFAR2(ibbF%4k_#an1tPcT%kxf>e&I{A_w#~zw}^|Xu(Y9QUiV;Wo^&)hv?(?x(yfV
z(iRD;GeUrdgrw1!jLFtz_QOf1hOf<@Nuk8NcW!%T9+gp!wlT?dt-xP&ZDw(|OFf#I
z3`))^K}lhsl}|eP7D;G^NjXK=CqJJ6hi7vL;9D|Z{#(=b_gGDS1i3s6=y23Dfx#-Y
zdhAdD;~t(}JjO-a0o7mmz>f@s)lohC+yDxM)e=%zO$+;DSUrBM#?-i(<5FRsA+sJa
ze^LGtoLYlMO+69K$u)&rIW>u5DO4FM8PRcYYxvMCdZv(da=0Rf@r=YCcWY;g<%B-u
zX4Br?g+j()42D{}nVs1?v3jE)*wqvZ_YgsP6G1lbf-q!>P1Z?nAH47r`sO<k^#+a}
zK}WYutGEcQB4dZ#N<xikXP51rYG>A9vO|beK%6YmjCuY5^c$t+&HsbfV$Y!glM?-d
z13BKAHd41cYrV?HOwMn=HAL}{?RL%ad7z%z&o^f{(ECx!vhSO(QKPNV+^wPNF9zj_
zvbOY6@(H^hRW}2$iFn~`O`T+g+E0WceWB>xcniDu%g7O5<M-j)g-RcEKQS4Tfwnd2
zfO^0?wD59u!n)gTPQKD4k6H3ltcXbJJ6A&FlC1KlsA;003tOA?9(=qCSPCS*%r4sF
zGLPgz^_fBI1*+z~s{6sjQ*BDWgn$IbW}wC4OAh>iqc{Ja)hDx(CZK3rmD7n9O)Wkx
zj7;>CIwQoe+$;jxMHFXra;W^!gH$!MDRuVe_sy46ns?$qcjq{^&dQFz@~LY}vKG-k
z_56bVPK{ux&ArH`naf-9Wp5fSMV@yy>brE`n!n*8+f`gl0Pu?v9iguf8M^VPgpcl8
zC2ODW$IDs`QM^SB6BHQ|g|oZu)jYllbOkuMVZXvdZe)HKSixWP*>_$Qr;xaBY2TH)
z-J-AKU(mb*)4e{G4chM;P(Id(ay05d@WV$h4d9ZV2MV8+KzzHn6&@o$`!br*@40!R
zbM$VyKq}w)aW_H+^~#v)Y^@5IoObldWe*$b;&OP6rZW%vP4!6NT?Jro{L(4O-#5up
zpu9gQUkf>6HhSreT8Oi`TgY=_c699Z1}ScabWF!P_}q%+8VsdPFsn{)5Akrkh5wYW
zT;E(k*=F4$woA(}O((Uk^A>7*{TfWy({isF_5|aMuvP5i8KO>NXiTNOvYjcIw<Sp2
z?<&Z!a<MKYe2J)~N2F1a+`r=ri^P2-kvI13(++;|f&8cQ+DCiDRi+NqMmp0RZ6Z9<
z3@Oj?r{}-O>2zW{+d{?orB@5cGp<<&HQuHi-_{cSK+`~(#$7sl^sa-STJ&j)oQu?3
zkL@L1rO-5vQhvCznoVi+g*ba``Mb+Jn1_#XJL9Ex43<jRzAn)(uVC1Wh?V}r0FO4l
zg6}ydWO;MOs`bo4s<Gp-jd?I0g>k)#?UcpI7Ybv_%X)nmcE)KcyyBrhLRT?iE#GIs
zt7zcJsK}gnx9WyC=TlkR^3ZG87xdyvdap=$-`4!_!K6Q!PC?KgBuHvxxkKCu$UuI5
z$svi-A>IV4G+g9{?S%Sv)RddSOC;+mO%liPo$F9P_0sh7RI0DAP5_K0u|-N-C%fVK
z+)?<J)%Lb<%Q#~8X#VKKquDtf6`yjb_;$odr+_+G44;!tl16<8ZCCc#P5iy`a{YR<
z8a46VRt<qG#jysM^LH!NWS#EZzxpy@tV~u1qyV~%;6EEn(8dz<Ke*KVhs#<ps1NrO
zIS8T+ZhQ_mCeO7_-><JcDJy*Sd!j&i02pCkLMJ{48=`}k8F(lXeF*`o_eIgOP@({c
zWpBCR3F=hV*cSETXWGmlQF!+Vr(JSTf;0OjySGd}PsHZS>=paj?5-{Cdu+X(ZlMrS
zD*DEJT@)JlxXzlKCXh9uiSY$!Kg_lTkdkn_@@c7_=!Vvzw0-*WkR&wE>E6UPE*}cI
zRUVHDwep5Xy6PkTVNLS=V!m|8S~xDebTnpWghBm!J=T;*g=&uq$@00Atn)}M>;Sw0
zl(P;>fr1GtltF@l`psY{B`e@CUnPJwOk_e2_Jm|L5xN7?K5l_xg!(jK5+EcC!pfF%
zP*-;+dxmZ`Wd>pc_cB_|t^JUKUG>@&snQP2ML_yM3N#ltfuR==NlKDg!N3j*%>Fc`
z(8RoBI>sM^2@wto^Ab12+p&zKadrnOikm*#f5uZ4X04d1r;p3JRPg*2pJd^^lUTu~
zmG0Gi+e}LxoNP~~63vj_q8BytTcBG{PfMVFGj_4n<O+Av%ZImt2s1i4MlSr)N=*f4
ze7x_JQUzdm>0~zo(FPX&C&c_9Ed~*(7>Y=s4gU71EQm-!3MbE!BGSOQ+chL{<q^zD
zB0qtnmADt5G@qd**=b+>rMEmcVB#0{fSA5l=nAs5&btFqrn<XpEUNoeFvHfdlq=h>
zhA>I6tZxX=7I@EhD!O%K4wyM+NOO+&I|n>`;rO%?q_8_Ez25@0O;U|BES>A=%_V2$
zd36Ad0S@0oW-4%Wt6ZKHY87gabcM+g$svkvy}X|Q$;Ck$IVDaCKOo#><<i3TMM1IU
z25@{ZrJx0sd&qHzo23t8JHrRt$$7K_s1yO|WMU^|PYhs#m<<R*P$w@4CP3WN3?T_<
zKB3ZJJ1Bg`hxg}#W)+Z*(<gkFbpwtRl66BOP^dNMr&|uGWCM=pd$rtXuHqJrqKF@@
zMa#3k9fsKz%?IsUmt8pZA}`Tb*h#tF1HOGvS_saJTm4i6ovkI8w!*$qU<Q{M2ftco
zif*sT5zczUGUepFsoi<UxHpS>t1k22_?|Zsw=BzNFEBDoo#&~qDH%n`bYd`#d5Y|g
z`q5va+lM0q6s1R=$~x^$q32`E6jFoQI&>jd9U!LKfJqyf`hN&r{y#$~kijEM^bZZh
zf(%~&cDpj|J)7f4x4cpeV3(UJf!bd{0`E6a_fg-XA1EJ{S~O}r&le@lTYy>>B&%d2
z=hPDqKhTxCDB-OV2%6ZIRv%EQinq!aCt;@QBP`#kfXQ6c13sgLmkY_?^b6Y-(r{m9
zWL1Sy>UQ%A#G*Lew9rbr(;y(Sfq>}nd_LnlLOXgS|JVGz+hSRnG62fo*L!JpqP<9h
zayB!8CW?A>sMSqKKZO+pE1(O4BzfPX%{K^C0UNHoGa5ch5<8A@UlpKzLOV?Be)_nQ
z<2aB?17rk-;Q$p5lw^T={txVPnjsFP#J8oOx(%)8x*3c;z5Dg`sS=>)h8*0uK_&;)
z7{AM}*RdN^@zxUEeZB7dmF_F-=lt(aRA8AFCmW+wrqorY^ipSTUweDMTJi<0A^usP
zNz-J7(qJ|MuCS!M&E@qPOzH;1?e2qdU)w~h4UnoR!APs)HN?!j9zd3qZF`oQ3ukI0
z=jR+9kTTew(9P^`a2Hi)W_=LLtUBmBB)GtraRDhjawDvOqTZEC2Crq#_l`DUcfga6
zLhXPlYzHa=l8OyTMajj{jZ)7>sb^g(_fJ^b3eCDmwGYSDU;GHCT4FD8H2Wq8IXzP_
zs(?bxI<OLR<0o*o7VmS2CIKrT?ulrCc)(vs$8>(N%;N*vC6C|(5#BR4DmU6CFPxMY
z?gKgFz&WWWZs$nFbxUd-<;|$<zf=)@suxn$uPBSXAJz5|P9q(bp7PX^)1Yxt^ax*S
zh1HMfxR(;8c=dIbRhHp*)#T-8qUQw^)n9<&5%)o+#+^ABauo_TWdfY6(5S8C%b?R(
zmX4K^Qh;^>2B9LLU-=o0)=gZX8x~A>0A33}q|>1!70bGEdOJXDSrE{APPT1<ZgbEA
z4FgVy0o9DhU++j427}^Rl2fewWDR>A3Y1^7GZ5?h0>is333r<oGVQ)qM5yn6_c-ZQ
zMXzP_+n8?5S>5u`;Wr|hL!So}53rlqZ6h;HBd-?VjApH};hYwETjh?d<ki_BYHsB6
z6`dKq==$w=l@LSuVkKOr6Pv!!#w3iSIjI@k)zj=^mfACX-O7w%`e4swMQt16{H5*I
zHNcWyoP9wAsCI1SgaRtN+23t8jrV=?@t1WnZareb{?2~8r%-g4b?OBL3`tPHa8U&X
zjLYN#hVUN?7$Y|r9_`A%pWYV1Z}<Y$W>7g%(m!$CoL4h!sB|?3Vd82z*!HmSiA%L0
zb%&UQcdGV%W|!Q#5(RmgdXgsFWoMttq5d3i{;b`bKb_ir6e>FJ3I{zVm##aq&#JIc
zU^)G;y+HXx?K5TAG+zzWjAm7a-A@*7fF2a~HdSP`i)^XZT>fer8j@ra0^6<q<71E)
zzUmb9jurYW06>g9;=1=c{kQQ5eX;*^f164Oig?Qe7j+=w@yqCR1c{*RdYJi<2#S*u
zLAU)5F)DyB5{8(miKBmvS83-A$FJCz2Kq1}$^>_JnRe8^3KriwmQ2b76@1=OSCWe1
znf*l8jF8r>B2taSz@*Y7l1{QbR&u>aVt7w!D_;&Cjg8R7%MRE#E@+ZUGQDz%@^@LB
zzPAQSs-P75$RW$0EcABpmjWS61B5KoCTKv-QA)ch>-Dx<^$Z@CP}n^%B@L4G<pNEM
zEesG*@j&jfE(ifZXj2qSD2n93a8c?b)Rw#n=%Jx%D)0wXI`Dl?Dwy}Zvo0;}`X*(q
z+qTN5ga@2c4UJYRKc7VPN}-gh^KkDXHyviw3lQFA<`H|gfNDNnE%co%x8l^J2|nLE
z9AP@}ZHGLZo>NW`?{5gtFuH<|(8z~L4L#WE;J5w5Liurg>!<rY%Z%xQX=C3`hA;9x
zeK>e;xND35Cbo8a0w3eI3JKfXBl|d-2HDJ~uPpC(T;1)6V~=%G>Jc+tO*T!Fy={|I
zELHq0=oXXRi^nqMi_1n>6z|7^=A)MFqNTIvWyJP;p^(0p)2}s_F?d1f%OUpg51^4G
zVaIxt$Zz@?(DFk5xo)TwH&lw>0r)@rBM+EryN7K+zU0xNfk1$q^v$~zcRp%3pS%Ao
zE7Q+F^cn37s*Bl}`)*vWf`9`DvK%OJoG~Q#<3Z9GV#2;9Z8$)4pM!7+2+h5{$?huy
zC5{S4(?7sman5&g3#$5(A{z5A5zSOW%|jmpP*_LrM^)P5u(Xs|OHQM#qDQI`RxEDV
zc{YEvA>iBn^&F9jx2?vKF^R_<K3)DKgjP~g3r|bHcHqr(&4W|a1r$x|KV9Vcx+`c?
z5yPeh`eXsS8Gwv{D(?w|q(C|e^18$T9(D%J8w};AYx&v*tRPc=jSdFdzVd1?oh*H}
z`|QrpVADaX79IB^_rJbc(h;T8u|36k3v)Avs>c}K`cPDvUMmtUPzmo2yG%SiccI2y
z(D?x9S$<RTFzbE$$!o5QLh!g`)qx`6eWA~4KOYOq8LgzXRxI+V;;L_pe<lCu=CQlX
zz>VHlp6I0&TVeGwmLnwJ<b?w5;H;?mvy}sB)vudDwQw*~(*8KSOPAe{7$t-n_RX*w
z)IVdq>A*JXm!^tqu!@K2nm5YR6lPM<60xPrNu}2qr`?$cwazZN@79b}FUUU~(yb!A
z$B*61olvV@QvE{ki}Yy96&1Qv4J+Q$hn|Odj?Sgu#rvqdd(0m3+N4!7m(V6UlBM-h
zV&ao#P<QR%Er}!j=Vk|I-o)+bjij1?IZkjHkvw}Nb!nwhG1qVmMzpO8iBf90=qaK(
zkbkHN-=q;%Er;v$6nZE20ghi(F}^aP9bvlPIW7785MQMN)wot_h9(<xP`u(zz^h#b
zXfPQpDG;`W{z&pe018Q?3)b(RpY7RZ|0(Cd^@0lRV;}$$i@SP(2jUuOwKIr7!TmW0
zH(E*r3T_{gf?MN{3U28e*Phf@rz&MQT9-@YtlCZmyp(21=8R8LG&rvc8vpE}#=oba
z@$U_}@z3jzjeiUB?~AuI6d!W9UVXZrKjBs56zExr`-uV@`VgB<F}n#sLxXoKmqMI}
zl#S1Rf=HVMLQd%wS>!2nM-<+08H1=?_2K@+Gdw@D@U)xgSBX*sg``=o4X>H+T<pKX
zBZt7@@zYEP%T&Gc50+E!)OSE3vuN}xPo9;ys%*&(=MT!c!LUp!L@t@(68ee@u&QNm
zOqWaDY_sy5nUz2Sn(G0nAGea#kM~$9M`7C3IPHErFPzWE90Pb;#+KGb7u<y>`#V2T
z{VFC$?ZNE^`EK;^1ywDeCg*!Ll!crB-ZoQlRbe1#4Q5{{I)-F)+c~)gqm(pQKJgYF
zZE<w;b+e)Rk%9*-pq6^tm{d!ZyiF~-de*)CBB&^s3#&sl&VhD+(etA67t*uB#}3G;
zxtF@`7L4Zl);X7~X5dkYs0mq_UTTcELC>!WMubC_E%ygJtyD-Q&+kMP$YM!^73q<j
z0;<b;(W*i1f8^Tek4JKQ2M7|EzvC?M_<!}gCvP8wjlbzv0J3qk9+Ut^TMpJeB!C2v
znv?)0|1kk{sVNb99&s^DcXrs{)^d6B3-e_csq-qbO2LIsVnMUq*9?La?j&fIGbT05
zwe0v~v)n|Whs*W^mlHs=>=CCUe=X??kjKdcR}=_M)Tpw@C?>pj+tT262!Fj=2D|F-
z<Q55aj)?!#)jPRl`5@P~*d9rCDL@6S>`@EVs2=a0JPB+H9uwokH`?9))Xe=a(%Pq_
zd$!SCOwc!WXO^J90bK!>SY$CO!&L_q%>spSE^p+1h~{S7230?}qWWU}V&z&5=Q5JI
z7A5@6&uLpxK&Ay<MKV$!wh%vFy?aHl%+f0#UM{%33-4@@Gj|OY#nZG|&++uS)Y?;K
z%YMEohr1767-eS&n4JXj=)$_t&(`J+6W=S1g9-b&DXPC3Jj9{<-eIuHWwWNsHEy;q
zK}?dReUKSXl{$YdD9_0g*uu<x_Qs?h6YCK7;%-gi;I7gaaXW6{bzVwd=_TkT-f_lC
zMl89_vHTBJv|l4PN&Cth3@!Or!JB>~EXg)=zqNdA@J1WF(LW<U^miSTx_To^=%x=Y
z8hE`pf+04k&1Yf7Qlh(jECmQ~hbh)u8n#n`cC&B{qx3F|HP{w{HfP|=ic!7li1oD^
zUP$hci0Si0U420RVfqT93<*IF!5#Vz&IVa?8AFE#qK|Rpgi?nqbiUoSx{V!bI1Aik
zMRzkiJIP+LNZS&afZh$!sAwAX4m(hX2|+M@mfdo_h<XtD;T9Bg5*rqt5GKv67lD1-
zKK*mlJ><R!)jKlM_Q+;>CR=`2`qqT?0Dz>{*sa!>o&f>?cYFcVM*nlAhyJ~>+Hb7(
z|48Hc|C-O9Z47@rAGe2UAogST&cTCTw+RSz$5QL;?mr^>{h*@>Bj6_DLf=-r9iOiP
z*9okB7a?ev3IzD}d)I4WG@ze;>hOD}w%s_5{@eF(ejn`(SlVv6OxiE~^}Wx0GdzbM
z&8sp243dW5K8uQx{?0Drhn3_y<`Iujlwdror@>LzLpp(<J@xAhQ{#i^noqCyM6KWZ
zV$i+M{o~$$P|{qYX*2Oc@wO-EJRAf@ZNHYIHG{6n(rWwx`u_sc8~*4uyP><;=#cr5
z1OA>9{x5hQOuG4Z4y1gn-8o3*b<~D>uinGB$G3fL-iP;GVJ2gLIS8)LR+F)R1Xzc&
z*Vo~n*nd}%%)%C$l^s_8Ogs9CyafJw;xW9^V9D}oY7~RCDG5oyrpe8b+qdjhK)>oQ
zW<v*S4~Gpak=Ox&XG6`uumgLAF0kElqAo{!#<1c!2EH|9ht6dTSGP|H0qJuoHP8#O
z3X^(pVt}*8^Y??PgX4193_Nm5;|aKDaxBS=31IZDvW)=nZy2Q~S47f|eEcKVx54>=
z!tn;@x2~)FH6#4D`L{X=_-o?zp1T0x-wS|05&(ZZbh{?~5%{CXB;A%mJu>iD2Eac^
z!FPrg6nYsBMWjl{d7`d4R3!6x<6mHN&f?4Z`C_Rp7K?4Dfwkf<Oygb!Rsj2b0PJ@O
zE=qD#4{3qwFx4jFqLQxP&0uk#qPi{Wi6>lWaa11=2Ne@6gqB+03{Pd#17nMLs;|^?
z44X)MbeVgOo{-!l5g_ev-VhvWlo^FMg!YEKBw8*7cW(zHJL1R_dH(Q)f^Y!rlR;Gf
zl?L3<tZmp6{7(J+FMbYb-zWe`zu^HudW0LiZQ5nh<OfK<{3AhkyG$_03!q*0<8DTg
z$$107z<<B*!ClCe%NpD62AkiUYY7NM5jY?a^RYyXDQ6BU2J)BrBY$wM<e9u42ik;u
z-^*8zg4_Nhfc#OA!OWlseVo7cbHYtL51OzSXG7ZIt)m7N>c8*s6nf(PR_R~vddAXg
z6W7uuwCECZWS~Jkl|OmB4gPLJ_w`Sw<NCXJx?c)Uf7j>L)f-MTQG-HY6m_}k4t=~c
z@*QC1mg1wu!h0@C>li|P_N-7x;?gxx2n3|NJSYTqtds6?KMH~CHYgAJ<>Tl|T(t2z
zafIvGj5eR_6(P~AdoCI&zt;f)#|QHKqUd2V$5(9iE5~<eT%io${5nrJRy^CllE|$a
z2#Jm>MFB~SL3CZDG@pWGeX=x-QtPv77MeHI^$$(puxY^e059a-YE3^VzGLgg;2RnI
zhR4PKg3SECi*+Dp_|yaFkF`{T2k*}mN&$Lrv2i!{^X~<Fx63YOc)ssz{XtNO|840d
zVSGH?>&_aCuK2(hlD?3~wtw2K>GQkCmcBLvKkIuW_2}#V&mhCU^!8htBB1xs$A{;h
z#l&3uVSsMQM$3{thBzd>!{J_Epx+RQRL}wB{vSvMP*7azJ&SC1V6tTb?EZRr?<8RN
zsmZtg=DW~(U-NUl-)NQFFtyqI-*xl+D{qI?INT@AOW>_97IL^SS<=J$Zf<($P<)K^
zFIBiokdA{J-vWSqfVmCFI!fJKp(6Xw^?qIW1J-*exzE6&Wk|{Wp2@{{2lgN6Vcn(2
zb35Ilq<E?A8{Jm>FdiwWrdMti(gfA?LP&EDZjZ|=0a`zH3qc0pi?2{>zX(!$TiV|R
zKOnK+D(wws`>Bx=dnWXmWR`>!HNfhDo8A7I2higDLz-vUmI2VdvFjISRs)hVtBVw5
z7yX+v>+IX#oLTL_5TyT6XVxB)Gb?KNyW2GKSGQ?+GjM%$c{mS}pYN>Z#^b-+wNP*(
z1AK&+?t0n1^uTwQ!(WtsGQhng6W@Z>(+7aNtzqE-VG>$)Lx1Ra*17!scMdkumAYu+
zwINkNSe=iH%q@~EXuR~@tjBECA-ij_&$oDPW7lrKGrA9qw&~Cg<6HpUxRG-V<*KT1
zu+pnQqn&VXd{^+O527xo)?jD7fp5G!PJInllI<b|o(F*EZb#4^n6{~erC2cUwvw=P
zr}fm5b`Pcozu9+n6PSQ|)4`tTvs)Xeq6ZR)%;^5F%S@t30+ny<@)|634R(AD_8z%J
zwVV?`)cO`1+pIlCML4Bh5<7o#0)wZfgJY+Wdr<Yg>1(h)Nlo&8h?~KVP!m1Se%w$^
zinVU;v3Sl{A63F<#uOq&nLOc?Y6)gu%sdus0I{$J;~|=W<HpQVFPJbBUVN1wou*r+
z0CR?#v7@Vd7}4a9Xn=i+nUnT8Nz@?JfdfVj!6(+whCqzRoJRF`#4xSFo|kBWgLX&-
zCui<(_xs5u5i!Q_nl%{qAW;_e!l8E<>4ZR}qU+&!yA&V9*qs&>mW3Db`xcOI0!L8z
zoog`GHP|H_=2P*!8%hlshrmv&g46CNpEjx_TT&dl{4b!(ul93CGQ^BQRG917e{gi(
ztK>_+>h=i{2`-`_;ohoV`5KIOrD17R*3ARjwkX)P8)@4bsstL1k`$r|GwD+)M9eUP
zd`@Sf<BNrk&kYE+SHV}{V~PhS&#X-($bu^_4_&?q%(nqv^uh}Fj|*;jh<w4tc9Fi=
z^>=~q<rI-QslRubDI5AN1cEQ=`)4@rw4NmWfWDuU0Sma~)!>q2_Njp{tr7ceb&oE3
z6%Ri3V@G_ex}<YzTZ)ZM9C(V1i^Wb0QlsjJ!3FGnkOnTh!J+jp4kPr-1%%G(HMr^v
zM3d2Bx@9V`{RV6=_)$Q=qcdx;4<dzP4E7Esm-u2%d$KAlx1uH&)z*I-hz0V=eOiOP
z#eA1=ZVN6cX}$dywzSln@zip7i+(?53dok0B;Nkjme#{%XFTBqwkZ5woDPTgpA?+L
zY<%oq+#WVbwEavIau-T_bO|0eEUqF+`Rl}jz~}#Kgbn|ya?F2*;|9jjKoqh~wXSWZ
zo|fkQNU<Y6Z^Z<Ewy!O~Qd026xWsh#>Kl1BWn;^oXFk*)>o72u+C$AOlZha*sA91!
zHq<nO#G+f6r6Wa7I?~eFV$CT^7X<WA-45hTHug$7xxJP%%cM=5B{YvEY!*R~eA+*{
z3UdMXrh98IAxQrB7j)NP*KQ!^Mvzvqi|?q3ZfukOfl9(sd_yR_<H?e`Wf(pI#M%t7
zyTCuaFvVX@bg7=9o_2u~2=Z+VETKkD+La*4&RY7!&TqoW6Djmiv$F8tpSSBo5SAjk
z;2n6x5<W0fY@XI??0<5H!cyFo^Jt)u?;q-=MvPGm1QNEsTZ0YxBL4KktG9wnbAq{u
zY%FRH2ft$Eq!u)eWCKfVRWS>yo~jIv-+gNhRz-{Y(+lLQdIx3Ew+0&>kj|}~Phe{-
zmIskJ#6@xq7Dw%iz%q8O!GgK};w9q88q8=-aUZem!y2rv)YZykT#hQq#cvHp&5n0o
zgPn%cfYm}YXAQ;<zS%#$rm_C{e$w8f;rdsd)I7$KN1)@D1_8UCs&ozZ^T>aH3G>TY
ze-6yd1>etq?WEUW{~gyWaiCb?FRsmBy!405B_aLpJu9Q|rjcR=I946$<11b-c|mp6
z8Tu)j;#G+|DEd?n;!_mrGei*kIeO9@cZUiHWarAT2kcYcC(df@eSw<JfA*#;ci!jF
zVBMmf0cQ7~s%dDOwAg59htW>Vs$Ti7dD?g+Yrp<ouLBP?N#2hTM)k!@n0%3z=+NIZ
z+hd9qw_FU@c^pvei-~qALRz3coVSVP(#LYfACz`2>Mq#VX?U(AXH`UO^flVoVmZY#
zo%f5>iDk~>kp_ISGqs6-)EuTK1Ym{Yui;#uLCcJCFz}_3WeS3Hbwdw;Ta`TfX>iTO
zk8F~qRjb+y-Wj0a4k{Sd=KwjA@E339YeN~i9$K+q^u?Z|wBI(<GlY9T8}GP@P_G@N
z-Gsubx*l9n9_;LXGi)=>InXD>wACcwn1pTVqw)H-D%|-&pHDA9J0jmo#~Xsql<~#I
z+<dD!<h{IQyVyG#X;ezOL<NNHMtZzK4}jkzi5j6QM_|>vxs|bNu#ZvU;yEphz4j@Q
zJ9CJQt9)~<7*$MP9HabV#MFYT&88boU#D*iZyFRF25I0CV)oKKWTP~T)@U^oB3#8y
zbpdgBz`^=@=S;{d0tn6d7S0a+ml??7HXw5~F<n{cWit2AJ@&!u=A(duj!TxS2QW|2
zzI^i<#O-erb|SrE4mU@hcCOWrr)pvImUPQpSVFV3E$W<4Ak+$$A*zl~h}p$vm6t6R
z5U5{OD1lD+k+X{z!9DpGde@ED{{5~C==74bSj=mb%U+Ri80d2E<acYj7r10JDM+kB
z24$}(;*fIENASi2-GY>lM#5}w!Y1rLPIgWr{cKk--V3pBGvwc@e!a9xU5!sa@Zv&>
zO&j2B`at|g=D@f7QiS=fiR!PFG<3@;E4$376jq-VicER27dk(Upz5*LLinaf5niI(
z*I-}1$*sZeqxvE|UPjtUI!UDkS^1wj_Y&YjoG8BHA>GoQ`P8FtFkP6wSo@DzUv+Iy
z3D%-#r8s@L(eG88J^z*2`;BA&tpWERDd_?b1WRyb?9#3Ar(oluylk|uYajA~(@0du
z(VSGXuhFd$`}LkJzn~+At$44&gmW4(j6|t%uCw?rgD->&&P<-X?Kc+4CqeY*Tj<?m
zVLiyJF%z;9jnU{6OOGBu<N6foimB`yi}xiyKnVR;`4g_DDURl^(=YoDrR0WP^LKi?
zmukXd@ta(B;DyGi>FbBsJUJe1g?;q!XbSMeFcB$DD@qXzh5N4MSS!Blh&;&fi9fw|
z4Hl4w{$jhNB_D?#MelpjOQ#fQ)Z5dzO5G%ODLU<Qid{t4Wo8~T0D}lFt6TnEvGB%u
z|2Y@(sMw0(LfjJITj5&_!P&KRGFQFJL%O!&W~2~>j2N9ja{~Bzc|IKHtQd?An@Fj0
zOgJ-Kq;DLy4UBj^<7T@oxLJEq=S|?8EV_#5Q(+|{{_0tGl(as8LCfxhAFVh4*-QV;
zvOPrcmiFRfY{8(7o#I`aLFw=rKr9Cew@u8>qiet7x%ND;bUQkkA9*DyCZK_040xYW
z4<7;tBE`_BJ=?~ty_i>+Ku8m8TR<4}=es<Swj8-dRCr>Ol*r6uF*i5)aU-(+H%8Wf
zBr5-Z{_Nv<Fe{sn0%JP1+Xd7$C@Jt#_Jfub<3#?(TeW3?!Kv{c;4U{%mj7VKQn16j
z;+t|z%R)kOe;aME+?zDHrBJL)<`jlm<7jD9nOYC^?aB0|1Rn-3@1yrq>&)Dk150-B
z&Fc-GoTBXi%4n`J(X$tW+-Xj;f42QeVf9JFZq2J@cw<*u?falkv}FdLKwGwd@^s`W
zg`8k_6vZ~tRW|oQ|F`=Tm0pH&v`Dp$)m6P_*n0GSm*F1cZgUrmtShLd?U~1?087^_
zIAK@F8tjn<s=p_e7BG45Jl0?yZ&0}O1w<<$frvUcM64)Bv8@C~a-&ngn@~UA6b#;k
z`27Xcq&Fb!5b^MKeBBVrYL1R>{wv{B-9Rj1u^xdHRwaspYRw;>F87C`>c)H)rH?Pc
zyIji12RBDPIBN32fi3R&?G<9KF???M@M@qxQT`iTNDT=_$dnXG^8hyS7R4Hj!e|XP
zsEPX1D;V-Q#ZI{(XA!we!M@9PiOi!oWuns<0)L2W4aNwngwS(m)Ai>mKlbED+EWAa
z!p}WXfjzN9d+Np8r<5U<rsGh3H)9Fr5zD>MNrkN+p0FC}Lxsd2he!Gls^s@q)?jVB
zplgnJjf<@$4vz5hje{-6u5Z~0Y&r0^rwb*3UX5*_o^mmPPRQ4H*&aFpRd51uJj0I@
z`t|wholl6-dD)+r4mzRVkL1tChRS4+ClDW^+S@VYFLr?vO`yRH+;2)(c8t@j=#ejs
z#IPQgY^#TFL0rY`c9Cw5Yjx3flGIB7c=qh=YqL}Rw@aV1SIS>+XIky*YYPb+^hZ-G
z=BtlJoXbH;1+{rs3Tcfba9d}7J$rU>7ygtzcC<gcHSfil*5#ZBF{c6#moxGiQC00a
zbpQ;MHLhcAYgWn~hzfd854#hm{_xn4_S|g*gT}GaIvS0E5KM~ktS<az|FMRT4Ug!j
z>A){Y6^OkU+W)J0#>V^qI{*5Yzb-yW0<@bpiAiF4-mhE_)*50QCFl)wv%QaE{+x|Q
zmL-qd4WDbfYsBvSgm=iJT*rMmNR9rJx|(l+6MK%)CncUw7pj;K^j=fWY70V@=(vD8
z`uX5_fNMGl;Ewcui&q9^?cxY9lZqOzowE8g0NkczwE+`KzXVlRxjMG+cGmaI-HyUQ
zB4b7W(M>ET8?mTPODtLz3tZUh4w|-h6bHKQNl5NKiobnTTyd*HB|!%6W!?7i`kgAJ
zY;jrXada)CMycA%=?mtnRmr!ns(C2OUbo%uAqV_=!PJ$jK+>^qk5t;uXRBR}e8#Hg
z{e-)?ryr2JEd?@MR0jA2SVo3-y(+s?dOS{y{VL0GZKY_XrlFLBJa&pX-`IGXy}0L8
z5)0=~)nb%RtnyU`PMcsKqiS%;sNP#d+Ix#{@e1;UfMw+BVlP3(9(7|7HA*hl%b#q!
z(_05GV#$}zYKBKU%0;fI5CmGXcPj5hFqqjNj8IS9Tl$8n0CN646-?=rduwtbQ=*yg
zTGS&`v7GECWMsBP^9%i^;IM*I!F|w48QJ{dr1ox?auDfTl5X)Dq^36|Qr2gRK5`=N
z7F2dm4WH=ir@2dcWmU(WU{Tk(h{m7SCdxk&Bog7+Okc6{7mi5Ir+rekNbJB=j-)+I
zkezLgQmsvNA~wl28OR|l1n>J0O~Bh%EU}5de~p&_sqmzCX+e-cxdsc004F&&i*S;<
z6Ed!_t=!_^#~$t`N#l6kNeQVbC-0{=Nze1{53|D_D?s1cEsrX+RLX%tjvG1hSBw%v
z<7Xl{fLGM7U1S?9+rKx{xS9X1FD|h2;CaTJso_B}p#tM{Reu^{re;XaO4w8*W1qB?
z8Kx7lRnqdt!MnBbJ-j9e=StU<Ko{*Bo87AElw1>IyBaF|uCzP(<y&A8hd|my{QUev
zn39rwU)b&B!{f+AB=$n!A=GV`Bh8Ws{--JR+5e}#FOP?MZU3L5Y)OSAGC8&+M2Tz@
zC8;D^Bh*w#s1Odqe3B%4Sz3rm5)xC^EE%#)kz}2$W8b$ijM@Eeo#(uM-*cYt^ZK3h
zoag)ezRx-S@S6E}eeQei>$>0fc3to5y1BS9J$GGul}F{0P>F4eB1}O-ZF&{h2FW!^
z$8M!BhSz4Y#`{?Va7LdojZo=cZu?f$cI4rk{73cBhtVcCw%oT%g8^yj#G(OoLR4IS
zP?{p70?!3Dz3=xBY*rn{@iLpi88F-#euYYWX?^?0t}FL%bHjbplcL9q=CRG)%n!KT
z(DkGf)5Ltty^DYn#dl28?lq6xAUhX#sLxrV>p=99ITXV(i5W1O?^o<eqbIKeEBPYV
z+n4CWjZ%)4*3lHhh<vf>8gxD6s@orR;A3_&I-cQjjjWHE31g~YyRh67sLo!|=Gk16
z&TP9xIZ@hu_3K)MKD1QRY3xy#2DIV0xPp$Hz|reK1XENbCgr;TlH&$kX)u2$u)mik
zJ!#SA=x^k&k?rtu87Yo8BpIw-(Cr#J49El6MoqdQg+Y2klEa63z8kt+^fg;D|Jz1n
zgWF|~#p-+MtVU;A2`+=Gpd1C44~l6=lWEgbD<(>}+RxuUJKvL0#MB37$NXxubY9g5
zeP*#9hEpO_Rdy7CTM}#U-7d}db8<U{Uf~CIWVOcm4-t0EC7NtkuY&WQg~1A&r!THK
z3hUyH;_5#a4{fuPYrL1K-fqmAVv+eJtoKIS<0KbM4Lp)A{?#~*tMdLs9xunf1UyOc
zR@;6>{m-|KQ05J8yKNkn!VcQ(^vqDI9TOWrmFS)jGj{Rp`NWgVz0ZsMz0;Rr_SC`Z
zGji;6USjt&M`!Bl33IP^v)k$~zq%;j;Dto2t;Rphe@<WLh2ZnQL=<34`M6eJ411Xf
zZJ6)da`IxxV~s`QDpHPhm=#??@NUx9oEHh`XuXT+yDfzVOIlg=1xZR5j4Xo`b~#Ah
z*Bf(g_c*V`*i9v{1#!2~g01IGR7b>2?GEbrf8@CrJEOlI98-~41&6oes9LL6s#VA&
zQP-_4G%fi1i?3d_rkZK($Mml3_z$+P>{Rtfx2e(AG0Z2R&ShA>a?~h$nxWSMpJ=lK
zW6EW5W^hKm8Mjq;4+?Vfw;;v$>WOH!O{KyD$)`qBM&F~e`+E+Mc^dPtE#atKAKXt)
zhuqNu#Jd{@op(LQr8d^<<3K%*tV0ukn$G2E@|nIPc}9iIy-B_Qz6<FC?(8zm?%W9m
zlHxXJyA12OZKY|)vYj_vhUF3llBLa94cmLtT;D?#y5wL-=avvJs2EGusRc7arAezk
zFg3i;*|m13uAQY9U&hqb%Va|yap!YAQEP9h8Zmb_jkOH|I!bf|qMwVpW?*t5;L*NL
zF>W_E?=MHrO7tCcrSf}T+;u|me9|^od6}hBajURbk@8<`q&sLRl0c1_4-sE3N#X;S
zNQTBUmyTz*S$4;xDv!Z~J-|%ZsllsKb#d_c?oZN>o+H*{)y+~9dxCYjB_@3OL%ofp
zI4!@Z8dXf5%n9jlyVuH=^f~|emcf@y7oH8O8^<kfAhy@5VDdg##EvB@^GVz!MSD;Z
zj}_Xua7%K=MX4P!F@xdN{F2**-*jyW*nM>oJJ3;DH^rI{%ijX>^jReSyo#**MFjp|
zIpxA%S9bm#{96L54vjh4%p`D95Zj?E*VGwOgO8&6D#Z#CGv6H3Hj)JUq_)wvX-whW
z1kc${WbJMd9xa<w=~3P`M@&p8MQF<bFIG@h!NJ@TfwD{Nh3E=0_l-*vInB^xaE0SS
zNOHg=u%@+>GB8I-s9@RquKC5$rbYTUj(;q-eN0)upLxE#ds`1G4)<0)66)t?JzLNi
zj;+sc8+>#kF1FV8dW2oM!!i6ID(ko#3_Gm5uQI^oK+?n5<0u`>Kn@yTMUB38C*nNm
z8uUO1xI8xmtOTH}!9c2AXY2Pw7D(xT#=qf^CiGKMZqg%J4c(YYxd&<2J?giZAR^I?
z{%6~c2G6zBY>%48*taO<sITveykF|5Q{siKdBw`Aa)!#Xm6%yG#Wb9-iM87z1QJLK
zGx}*?k>g8KYcg4-6ra4JkKjm(D+#&fhXXdt0xjeSKl&b2c|dKA@`q*Emb2FzAH2?g
zH@Z_>Mc}1s$RhR=Q$Ix4`bCKmq5+-$&8?N@?&NKZ#-&gUK>NSQO{RbamQ!ZE%YZ9Y
z-Xww<y^(t21IegixV4D<Dp|ieMNqf8ju_^L?AqX4fd$iLS7NPH4U{jD?JAV)P#(#t
z>1i*a4zrIX2}mh3*OwE6+)%a+nLSRN!*u1Em;ObYF6rFeXlnB*jqTW&3Qp_DMR)2q
z<TG}orhA+2LF!f*k3zO}<NSR@TV1aAGVI4P`ujwR`Wq2qe){bnE^wdCkr}f0f7twv
zWr0>BKVi12mdr9D*&7yA$H)Vj__9YM(d|_BSeni{o{DD+!c2ek(vA?&Ct<(W3%C0I
ze@L?0e})}>xH(L97svcEjLplJ&X1r3l5bBlF2}m+x*e=Cl7Qyn&~e&xoXg@Y%*VAx
zJ=6QO4|KBadK;&U+VoQo&1cfV8|sdUN)L9oZJu+iz%A^$a+ux!1|kX5%gynwiXY5B
zl-PceS0MWNM+eo;4UsHIt@jxj^|m8KGO(S3b1oGj*q{YqH*%vw)C3gO91TnlDTtNW
z08a$d*bhP4f?`r3h{+MeGRH8PC=Z0#e#Oh9RV6ctS=I;YOZgtpq+D<hIl3X8ycXwN
zrkvA7O|2?9|JJ@gTaC{u=b5X;0}grSz1NmhoxaoW0AT{52kO3xU>bPR(U@y#XRDl?
zTz919@7|)Z^U%F8&2^>~5+BMr?vbgZEqB^<jgHcLmrmhZh7a1#&p60u%G;z{IDwgm
zgl}FaIex+b=Ccf&Hq2OtIiMTaT5;t)bc?T!KN1&%>cPH??mdpWzsLxDHGbe`&(n6z
z;1?nkvs=<d??UY(Ydz}j>kV-<c;Nepfk(NRlAW8EVehWOnFayGjqaKsFKRH<<>dU*
zuRkor*In*7Lf-u#Atvm2moFwl2stgnl*g?FG+;~$)K>y$%LW2P2BilrNCHKU0Yx%U
z)KCGoKLW>ILtchWJY8WhGoDaOQJzqx>g-Z_^lPiRY&%UQlY@d;aHbceN$>XGJ|{of
z&(8-AgDc&a-4DKuNo;W<8Z_-^$)s~MxF}flE3J1gy#fN5P}a21OA*Vk!Z$ZQl$=uR
zvYEb&ag}PXR~7AZsm)7pLY?qFqbb=8@?Mjrroqwg*@gf|^Pr<GO0zT4n>#x{kUv)k
zv|8>l8RAFp_Q|W>=Yy@W{!;AzKsU`#t?RbO-3Rp#=!EHj>8IP{U$hfcWNKc2w0kIO
zq5AQ~g)ELrPVr4HNr?4jd@bxv)11q&ZdC#H58!InePctGa>x#H-5HzCPtX(clk&6X
zs;46^KTky`b~DdC$6dko8~cc`gZTx3aDdxJ+7EqrGAD;18D0VV)>d7hNgROQy%?ac
zDhA;0z?IpV^=!0qF($lV0X4;!o^+TJ@+Ofp+*o(S?)G}s+qwZz`;XYB%hY{+XAtvp
zV<@U(5U3z40bbR}?{jIPcU%JAvcEJpZA1Hy?$wM_s_MIgqPQg5hDM&=B#X<ORDV0{
z5gs{P&{n$Rv94IM*k<0`1ZPE^(jBMAPZ*s*#bD6?@pkCHOB($rwYUE}<ODW-$!FY@
z8~kb-gW|mIk}k2-B+^=ZKNRSN5p;HBi5M3-g#`skc^(7V>n$aw<Xi4cp4#7smERvT
z(_3r))T!pnsZY*d@}x~;CsE0mxp|@x^fmP*K)OZT%99|NQ_@x*PsUSF6KIgjV7PGr
z_n9_3%@iQxiZLmr(b(~$gOhaR=p&|fQS~xxVb?g0Dk!7O)Gqz$2?L&x^rWXyBc=v=
zf|y*-MI&}Me$B?WN8;OYI*9(^a^OEmF$AuJiYMS($U|x>J9r0J0@m86v0skD$)hCH
zi1IX3<ogfe=9HVGsUqX3Wa#JCLVYLWpsM<nkQn_a;D88JH3j7m-^(EuL)I}HXB@{I
znS2YW>~)}_7|_rfIA|vXS_VQ{eiHe6C7H#IOd1nO3!zutW;-tJTTpOcOm$zxHlnD$
z-eHI_Q()PflRXaF-21)Fz~%A1&E|O+8kveEi%{5n-=%wxCoH6O2SCS}V7d)?<b(NH
zqExoTxdDhS<k`sNiHTm+(g0o(MA$M+Wf}H*oAM~+fzV%qTc7J)hNYo>ft#|v?7v6_
z0z;HYhAdu96r1ytAw3aH<10#i487h4UnzPR(T7Td0|nu<K`6|BuUF+2y{58^zd)la
zJ&9-+7a=1+M`Hd?uW2jY{(lt<cF_M-vGB=_d>{t3@hcAh3rSb{?x){aku(}eiu<Lc
zdzWEWdr@E5Bj~$!-Az<>JZ%`rr9eWkc&%Mm<YH>)f$ouaS^*AX5%IHzRB=Ufi0{QN
zJt6_taU)v-xuVJZMka4e0@Rm=-UTP`8V98ml@|~HTxu_ySg{P#VmzUXNiK>C*<a7$
z>%&)fxrPJ(=zf$?M@Go>*O4p2c|)IHybh)A)o3&g>Mc$EL?_@a(lmu-jz7<T-szHO
zgE@Tag>&VpjpJF*YlMK!S1$wXoByf2LaXKeTzUgZA@fgjuE`5TaY2<wb(LsEnn=LH
zxUqK7wDp4o^PUufN`5daip^(U8_0rl`q)_#3J!9f*&dMBeUe4V_yqHo+s0A66FKNu
zEwCNMJVMGT7%(;tKktlG)>675W&U9&#gaF*!vGyy(P2V(3kFZd`#L}0Pt!}TOK_2D
zpV^Cgra?<-xaY*#gLGh=OW%Vwc6<K&97`NiD5{)mO;ZXLYd*JQq1t+)qW9Z+ZN(4D
zvqBJVJJ`TneLuONas!5~R_6Bsq|Xn!k}YZt2h7*8lRS2`3XF|a-VM;J<(}Dx%EfFb
z^R*)>S9B_e<`1&NGrIOvdc=x(ddjs~sbaj;m@Rl;!`8Q1!p43D>WiomocZE)F}$Di
z#)#gxT}{Bt_zi5PX@QjfRa^Eyv}OMsl<`;i<Zur(ydL*tOhNg4d$zw$D$&`StXPor
zRKQ)kamg-HC{Kj~6Eu>Ojj_(PZ?)_o5t%~BfxZ^c9m@TzJ2lt6)+!gk(qC}n>kqRR
z!fe<b0@hUCQg}FV^9)<T|0u!V)YF!4^7gd>5BAG-v`f(~cX&5yK22W4BNs6|!wL%Z
z<26{{No6G4#y`4xpTA4ZK)Os}{mPwQsLMVVe=kN+?oj2ya;2$QB-I;IshVl^Y?xjU
zJc=L@U{K3p)JDSLIYzBFaihQMbDvmax#^n@&9=L*S3J<Gsx?WKchLRpGV~6k5asl!
z^#xnuysoh`?o;v?WQ5O23z-&dYBwvlQ~nT_Zb74uT+IRW{<+#mIpfpi^{0!XMz-o4
z*Srg(Z+^P@+iisu4o(JPe*d$6IZ*|*RD^4~nntGKWh0R1U2WB}^C!0K{6=e#q+BdY
z3aZp&D}jk#evVG4d?*;#>P2bzc(~45vfi=9ZyMz$j%*_oH958Cnd}J{SCRMj*04OS
zYNT^Zn95@4G0h$giqs@|R9`Z5;gIW?l0s{jTFpxh7S7m4Bh{AtMdKi+%PMh%VgX<J
z0t%g8=SZD>)&KMd*0|IBgl`2fqX@Gs;a*lu&)>>~{?&WBJ9XK>2k*r^r-=>mLWC@}
zY%OjHEpJ(pD4ePKtjC^;5MUKhumu+0v#4+)7h0;vUyd4^;`6G@FFUe=ab%a<<v?I7
zXo-^(9dq+TPTdX4>YSt-JWO{~?Y2JmA+}02phL=Q!)u*;I|=iR+lLVy9pRbAezAm6
z0+a1H*{u^r|034$iP6+D{IwmByZZ#Nt0U_#KC<-p9Xj~ZSniRiY~WS@8itR*%`z+)
znoC(wkY{GwRj0lFnnH`|wrg~)7LI0j`XNUQZUR%HCCQU{iCUHAyp>EVQ4D0PPYRlU
zeER7}1o1{g^1Izy$ot4e<@Lkxj(6Orj<;amD<{_vWs{4k<-U>h5!qA9JyGGqgb_gF
z$9@$^{HHO<5(lFx!2cXDmlpL86N0cW6*GN&Lms@(wJ*3I^KS85*Q0Mj!ZiH(aE3P1
zJhqQelPyqvb*AtKlRW3m>7>q2x0v2tRe=+?%SFQHgP%B%hmlVM#^^a1HIFgXtFCP-
zrq>u)!!eL&i1mQuhp4vWb;HURqO6ka4Nuoi+;}apGpO6Lw}6=?b4{SLuQmSb#IENL
z^nUN^yZZj0V@LTtbJr%j7=hz+0$o}Iqs-lyprX06UoN&UmW@+vOg;yO%-TyNn(y4)
z^hAN!HOxsz4x3D;ko?$Nqiaue`JISXKk`-GsXfr7u9<^D<7%tzj>_6jAh_jNkWbcb
ziYb$tJu#e1%NyNS=Gr=!JB(h)N9hnh&K(D*t8970j>GZrFhFD6UGb|$2i#Kp_Yp(?
z;D4dYPGTFm#*3!lBo{U?vfUw#dB=*XLHw#6#U2m+JYqS}yxS2B$61td=Epwdkn2<$
zcu`oO)>xK3;ITD213oo<L@;>sv7;RVj40te!kUnqEO+w~osLOVowf}~^yM(3YI^1l
zYnYCWML2wXOSGxlRWxg8NLT%Czwkf1(7!q!{v6}sclV==lsGSp1a)Q&%u)5`LXfSB
zqw@vfZz=p*a37PM8t&|YyAYKdxWnZxO`Q$BEGM13xKK3RPx7HI!v^q_b#d&=5Bi_A
zSIg7RPrn^iFYn&%@ip}(p+T6XT=B%P^$nk(+H7kZ-4pdAZXanz5+C5WrC($QZ!p=$
z4g>_Har~!0OmsoNLf1kf0yVBdKW^&z+&pP5yT7Y>bPq$w_Xhmjr-<0L_^kPy_hgD9
zwO|>>iyPb#pWR)nBT}v9*>(Svu!ABaRG{fgH)V?owt?EUi_Tta`3$Rh6Z>#-1Sb@5
zR4<DDQg*jVPjd8v<HWE~>$M4fP2A(?jupq%+SR|moDR5;)%d1Q^N=(VcMv>_%dkx|
zCWQt(VV~e!0aeVm8@$&<4IoOg&&N%pfDg9v7+W%*-WZ}S%o$Q2C}vS|$VaIrHAJ^>
z4}R>!`>BvMtklP5RRuFe-eUz28Xui7P@Z`=$)LQx4YRf9TqESI)Akvw^LeJZeox4r
zDZEVRjM2f2jOVyF8O@Q~k8ef@WGP>n0Zu??`#<anR1msQckIFq?g@UOGhR-+^J(_B
z0R&HUF&c1JwsO@p1|=wEY4={wx{ABldgBK1N(9V`Q4$x+<~m;$F<5~11j1_SO6tav
zxX#W+%nuOG6fi7!O^o`rZy`D(A7khv3JYgdCQ*2%0r1LP9<jV4bLBlUl`z*egeJxg
zB=6(ldEXx0wx!8Hn2ay{!_xda{B5hh@tbm0{MNqI$HOqknPIH#72h3?<22u()&p5;
z_Vt>WT?hGwo>x~u74e}<vN+qBjo;wkfV-z!i~NeIM^a;GJjhu*ky)SP*7fpoBf^@6
ztSaI4p|wGm<pXBfYv#(#O<!$F?<rbRrph6}nzIl{yYMgaysQ~R<>ABNmCG=3ZGl&k
zd_@*_dkyWH<)dQ(*`i$9nH6PTY(WoW9%z}hZ*iT*N@^tZTE=g5ziY-l6rixQrKz$g
zIFM8Lqttsy%O^7Abgfx-yAc);U=b$$){)Igx($G(vF)9vS@XuY0-ESj<?mkKSJM`)
z`489GI+s{RsBSzpe4yUkVIb*p$K!p6{q{SA+z^i*M-|27kcz~SgRawSp5gAI)qMs{
zyyWw&iXZMZWah;j{s9`yDWDTWgw1L#)7xqg#bB1!<qCV&v&$sUW!4a=2DxBLTgfi=
zFY#X@XJjL<D|*&j6<yzaB=f1A#<m9ZUVqglflys3D$?k*Nms!M8;`)J>*Fmj;I7l@
zlj28$I&m2PW3Q8CBV|gAS(KuoMv!=e|ER};!7FgY5=ZF$({_sAXyII~|9_t?Zl}_E
z7qU8=cXOZP+-PmaVs%9LIdt8@laDL*>2sdK-R3r7^5RZnlFN8$SPF@1+pA06r;BwF
zjWwGax=ULdk*61N5_jmG$1==za)N3jkvr8KZ=`kf{3H$F|1G(rS9st}tf@8La-KMc
z*qZ@D#ft6}51?fi02p5X3pVwcvBQy-aDDWLavs;M{sAqn);g3DnX-~QFQU}1%@4xQ
z&98^gY-gQ1f4kfS7}NZjiV(NIN-4>BNLGuu`Lp!By_Ggrxo;Q8iz6aucoXRO)JwS7
zccm(4qnyl6dq-Roo+pxu8M=pqeQc#|iyq?8#6WOgCu)1Z-hb6dUajX}Gb8>x^ca+8
zT9y}nx-fw!_3VZC<3?f;S@PR<M)ifgkKonc%g@u%v#G7^36;SxVv1n;LaJGYhF0BI
zmX7o~c@#-ze|sva)3NdNQlCVGLfzzIDdiEB-;qS@A1dTDbj|4ybbop7-7*ZRbnvLo
zj~Z)C((As-9KzP|`{}i6t7H?e(6e6@YDUIob5h6~R5i4t**N`!!YsAQGClG74R6}u
zvy!RgDL_s7%!IV-dbgB56)9FkK6(I7&)DY1_&+@P&a&6}%e|+zkmOFx^}O*a_>nxD
zvkA&5G}Fe@=Y^pBdfQV;>E|Z52Nwhf=NEYs%wF%csR&Lsv<tT6h%wrE31@+G5a^f@
z2t2=bJ;^SyX8ZlK1!BQNvbsaOP|1^nI^W)o-$G~(ZeWX^-`oUZqJ4F?Fp23C0ZJzx
zo)Eof7PBOkwG5N$STO#9aJWTrKSN_n=a*)o+uTyytOhN4U+LU}ok`#JY);+>UjQi=
z4$Ukf9PU^M-_JKSSoeqR9;<;t-@|Ny|NTGebF(^(f=xg;&vxh&1`R?9o(KgKQoxHN
zQn|q=7C7#u={&X@Mt4*Wu0ucJBE4{E@a%Jc7qQM~VDJ2J90Zr^*%vE(H!b|dxFzx=
zQ<Jb?$FR!!eea{JvkmDJXL7fV7AEm(U)9<KA>2GH6c&pH8>brjO`_-?Vq>Qgv3jB>
z&RuHbO4>Gb$9);5C@808axy8+NEUZcR~7w5Vn0>RcW1jN+4sfPYR#I*%2x-q*X!8X
zvD)#4Hwg>T0;Gpxs$uE({Npv#2~F&fAzHE_io3h7IHkvPQM+(uNK0w7exre`ncofK
z7r@bP=WiI|k3kn61IuoPmykCs%li5g-|UGqooI}U;`HL@aG|}2f_n|+)>v#bk}Yk#
z1HsB1Nm-|@m|AIX-Xi%9q<cM{zBKx@T=x>9<XKSxIs)BD%*6(muj}G_$3?+~zKFis
z`MwTTCtw-uJKxGO()l!o&}pRNc^>MuJf}@~o=DG{reaJU+Q*Y(%wmQ^yy@N=*bXU*
zVG-5bkt<Bw+;vA|gk!WywcGg16!BNdcl9@l;!j!nd~$ttI{C(~F7o;Y^*iET*pY7Z
zMU)D5_u{ZGqFAXu*Kl&%Ntrz|YNrx%QPGKeOpj@r5qP?P>tT5Hy+0=rd?($KZ*rHK
zn6G5Abh4;WD*lJNXUr=(CAhOH1|mpdf`u7RWE4fo$~6Mt5dE`Jy(hAN5G|9+C^vcX
zGLUkkH+v235Jf+ptQ-mTC8IW%X_Jjo&N=nU6~W!NpmMv_+w5svET$U$LPsr6>bCPw
z32hEc^8|<Sqe(V3o)2+zkYGiP{o|A7r6CG4`3Ug>Vn^eEcZjro*_=|zx8daGrkv9o
zeHFL%M`?}}7qi8wx$TF=dqvtW-uE)vPqT=w$VPvQ+W@jZUVF~&lN(1}G^w_|!Isg$
z9VO6`rK#58=%LO<kW2aj6%d5EMM807wr#k%^(eM%H1$v!NK&Ogk`1mZPzAviY79Gy
z4+v_(#mjXJvIp3ffbX|?CGWj1k`1CZKH^!HA!J-Bn}>?|e^=%3y(&yR{Bk%WoCeHC
zo25X`BV8n5PJcwSX@cbbEaIF=HSyPt01+PR23+YPYOc&Rz$zJv?NZJ&VAU`)82eG(
zsJdus=r$~EzI=(T$Idhd<m_k=D0~|&3#c%^D3;3Jq$~)1EzW}aHsF}X=#GE{<r(E+
z!;a@bFbk;qSPj$|zqE1(&T$Z>1bH(BN$S5W0-p=?Nn{5%02%<y0XSA%DD_iKkUO+n
zw?Sk{WImPs!%+pcKlc4H>^}6$0tBE8GC=0mP~5MIT>;KRzG~uvk~5LY0f^Uvs6y0F
zDxqA#0ae3ETU1cw;ec6i5sev$X-B|&K+q=>bj&Y{88UaC2U4bZfTS-Kq9A05mHRi1
z{G=Wb#PR^})G-RP5V)o%9^CW52I&O{Tz_2*(Z80Y&f-fbV#Df}Vbz4`M?b0LZ~9>#
zi9j-GD8KJF@K|JmT_S!7szeAvzbpoj@e3^Bu9p9#m7i3E7?8e%8?9py!A(YKCTuif
z1vH7jD)Zmq&*5)6`NNoj=cR)_dcn|!zGLPd6+@fH&B+1KysDxRziZemq@Nfz<6xCC
z_vP)L$R*P~GSV!V_1FTcV{ia<@7&h<iV_N3%>Lcm7|w&sFy}+55;W|$rnRHNsJi$|
zreAqSGY1Z6qN>t`C%V9P)5@(th(#-!KgO(^b7TsRu2ca(8?dix6TG}gM8dzqZ;wqJ
zTS;LYbr<mleoK3^wEE3a%pkwM>Nc?osRFLnL+PA;S{f8=M#o`Z$2Zs*Jn4+n<Mi$E
zzL)3*HY6fRr%cJ{&WCb`jgCE&)T`#r`B~>=zlhmrUCCDLKmDL;EoY>VCvWO4onR+7
zgB=;*c8|mx-j3xlVYypZuFm884NLy5)p!2)nO**;`b2PT1h4h3HG6kefV`3a{KNMY
zQ7=~N_@8R%YQ3wvUB$pE239e!ih)%OtYTml1FINV#lR{CRxz-OfmIBwVqg^ms~A|t
Wz$yk-F|dk(RSf+9!vJEr<Np8+`y8?W

literal 42591
zcmeFZcUV(d_b(pBGWJ2FgJ@_Hnt*_Sz>E?Q5MqENbVjNs6d}@^qk?oM^csc^2_Xgy
z5UNr`mlAprA@trmH@@#X@66oiTkd_n_xIm#<vBT@bIv|{t-bcnE^DowlYx`3fFHHg
zHPr#9P5}U?7$3mNx6^ktA3d^oss~Zmd;<Q*gVP)U2A%#90B~@8<)WwY@Rp&G@vZOo
z{-9@8=$C)s|KQBXcf0R<?f^iK_<!*Hzq@t8+6HaKNVLj0Ub!%sGlaEf;2U=T#2(+V
z<v(%QckE)I_lS|^cLu&=_ZMvW7wrD(<ts)S>i6_kFI~Rl5e9zXVDI{!*LV6Z@nsuF
zm_Fld$T)ZaXn-C-1MrZ6|DV#I`gF<w0Pb%A08GFAdCxKt04NOr062#JyeIHG0C4Rc
z08rBT@|nxCe+YAi@qOCX764e!1puxX0RSvt0Dy}oe+c^LyMNZje^S<4jCXh#<#b|v
z>;U!v8^A4qCcqJ31(0AMX~1286hQ7|81RVEJpO0&J(K@g4ve=?J^`42IQ`}H7p7A{
zz-i`FOw6ZF>H&O=>N|b<G}HH{{V%$3@%#^G&z)jAbNUh^f$@)kQ)f<}Vmi%q=IohE
zm(E{dRQPv%j`{o#*I8~~xFN;L2K<>{8v5*44nyZYc1{67nTL9qg!IoBxgO~wT;4>*
z#nWUBETc1u`-Oz>+>^6HzarK4tgbQgzRBQv>daqx1I{y@`{BarGiOg-WIQ)yW<37R
z`YhA=v*)idSpSRndFJc4e_)Zi0sI;Ij8)qC0>9p`n9r*m0uL=oY%(rUaa}aQd$Nz<
z2nMe=Z==&QCfJ4K^sNJ8i;6$i^{$-^0WLE!ip|W#3{VBseEOr{|Nr^_BMqDiclRrP
z<)7nsq%mKk<K-6;lvh!`g`ZE!ILJ5w_!q?}$0~nWgyWHm1fniE6kW6Y$e2?&cEoG5
ziN$=!FDJiN(<&fj&)WJ41VP;<1@BT?g3<_2Vy>7F#`$dI#T8?9SCvf2uDEblt<g`L
zN!~gx#2@+*p$@O|aMje1dYHZ{@}Mxe@~NY~*05`M4aXEJ8AIC`AiQ~-G(|6TsMM)&
zrSV{w3@}jrx#rQTH|>)gQBxoGOnn?DbF>EzTsf9~@B6Ga(q>{N16DdM-^sriggJP-
z+PQd@a5HP=?`GblE8PXGM=-jQ|M#TA69?fAOm`AZphD}v%|2XUFESE!o&Az)mPS;r
zh|3D8@-RBSSNyn9e#3<)p^3WP#BI7$)o#Vnh{x_wpahh0FASZ63+l@N;c!4<kPkYk
zk+R|<O8}!&=;~ww(xs50uArb094;>?xFzi+zZ1cN)vv4k;ep49`LH)WnbjJTX0usc
zz*%4-tZTvrDIFz8(6S0=GO~)kNqZ*$NMbz!2=m9uX|fzy?$TU09QJU$w|dADQC0%$
z_ME6Codc(Embjwli*y{6hTqGb<285C2hOH_Yc$0apy^cJYZLBX<e{IH#rdtyhNW^%
z)5#{spK+L)h+rdl9dAPMrQwwlV~<uTN?u6BP9lwYK73<X2UG;MdkT)NV88S6?&Slx
zU(V6&s3m(T@8xg{*RH{;OI;%wr7`Yoek9AP$3kRB4`d&aI~&^&Y;^g@`g(+lW)7x3
z;{E_bH)GS}ybxSoVByyJs^#u-i-lsRg%#arWRqqwtGk>z%ZUj$J3K>tBof*DMbF@y
zKI-d-FSLN<1HdBVz02SBSNdKEx*K?rs~uzMUd`KZD@&@9MZo-&3qi-R^2;xG)~CP*
zx#7RE8jm~Gj@_li?jzRy^-%orCd+5@vtZz)z}P0RQJC&XmZyCUSy2}eqoH(#T?;6w
zO_{i5x)W*wnrX_p7sE_aSYNU$)RakGM|^48yTHdwQJ>11nZ{=4loG=c_FNm&gdAiQ
zbpA@9rx)T@ouzBojYF5<hJt0YMCxIDz@gW8F(TGNVD;VsYIk}ISl`E;xheX%zJJ5T
zAR09&Gkke0P4(BlTlL@qDZZwgPeN+R?1KUp`DdEA<69?R-d`}XucAJmoKu@CFiNHN
zNf96!evaPpt3`4dRm%n_D9l6n@ci4Vz*^aAw>V0kFj7`IQ#rYL03E9$K~VQ`J&W-1
zPh4A7Q(FXcs{VLDjEl~=Ji<{zUFNo64$bBR(I*P5OP*2CG!LX;fC~|ai=q|zSXVT)
z3*f^@-AA&8>+r&=%<BFO>_RKTp?&FxYLPu^B*YPRpMHgXp4hljQE3oI8aS@9+1QKQ
zBXyyMI(1j?C+Pw4z6$Gx4iAf`e0;o{pkj}4A%g{AAU?={KG&?+JEhN=gVU}Kd8`{+
zD=&sv@5;SXWFCv3Vf9oYWOFBt-@IPiQb{g9)-oIoA8gY%j+Lu<$e-NwB<@}{aC2$$
z1n{B?-X7tra5&?-HPgHDrOi?3XKvKfg;aRei{6C!!Nu^$W#+8P=^_JxD!%@*+)$I@
zL}sr9Csl)E!O<O8-#Da$Z`W2*30sdqps%;lxSKNKf@Nh&=E7L^YpT_7bTGA2@j)m-
zy+)lxn8?rEb}*{2^mZ46IF2S-$cDmgqP*FJFfDUJ*cYaIf^ixk%KGCO<qjAEgj@GI
z<^S!&{RcO6-=7bxpGf!Tb+#n+u63#AE^le*aTqQum@9xMu}uV7JlxAShlbaxpC~N0
zL|ucV)yRy|VY@gSKE526rzDh5aoE3UThf%d<Smr=q1#0f>u%6NmmO)>0Io+6V(N`^
zz0LzOKw|nNs6JM=<eHXY23EfAfphz10$lS(fjx%WBz)CsU=v<dsN2#H%*eHQ7;QSO
zK(tC}B17BJFym>8QU;ltmzP)71@(N0C|6$lup7P^Lg0Di{kDB&|40L<_+D?dyl^8v
zPSP5#^m6c}f=-FX%d~*L4Yr}q44HthF&z>9<>!|9pWEMCp7eT@*qUjw!4aNj&}Jna
z4SZRo%Ijo0{GnIeuTu!>Jv3$}cxMl+g~L*_<1ScXF9-xa#|{GHgv%B9F(DCFlisX6
zO=N0Fzbq6AWo5k)>T<wd?AfF#q+$QjAX;li(S-Hmu@avxg5rsuDvo?pu&#`ttzgWy
zlto-6NY*d5-AKJ2dxm2`xQ)0GZj4#ccpjdeR7ASJ92Oz(bufN<jSJDFdaqSVqu2b`
zZ;J--HQ9Dir&8Tkv!&gP#u=&73``Ad)ekPat+uhVu0#FeC(Zl212Lq!Qxd9ZrBrDm
zb^y<y*jrf}oB-aCGslOPt8c=%=LZ^k49rb;2e~|BJZ4V-FLhJ5$f?`iUXg}h8yEFn
zj>}7awXrYCy)aE*vqzLcg#EaAD0;9FAxa@q0T+S7@G#12JdJY+3Bk66U}y6e^OxUg
z=J{|e>b~GxEaNFjN}Ov-d)+aHB>QW|Zd`5CL!JO&qWr4(wFl}>V=HqeA|`@S>51X&
zI7zaUtHtHD+zYMtR>|Q-Q%3@?nnhVX+<8NLUmcXIjO^#nYV;hJZ1&ht>>4cVVZ+X>
zC1*E|Aj<ByZFAiC1h<ho5#ccEbnZ74MyC02rwhg&!}tw#>CcBA=%md#dGYqVLeVq6
zD2`WIt&tyGzkJS0WT)6TL~b?ohT`-f8|o5_i1bjViTo@XB5T5L+Ne;rgI5*aT(XXS
z?%;oexKml_hi_73BQ{id`e>w$)7Zm+!WkKd*R?-+kb_OBTk6r?+{1;_=O(Q!7G6yj
z-9+T#(9st0eAicR5l#Rae)0#^GIZzBS92=tNZj}jxr62IUS>|x)ZwPsWUgu3Cpr8g
z4s)H+AR96+4;Mlm2qGBS%*NNNejKatAt<A997_o6kJB1A(|aQ|v^rHhnUxhkPEiN_
z&da=*$AuGLx6`xfh4j3jkg~)=9|fnI<8mm@D$d_Z!`|k1w0%`Xm(>SX7q<&3*peRd
zy@%_P1vlyWt)6MGs@InK3JoV3ky^3Kt+n#paq|skM(h#5;P?k+4w~c3-PjgMKUOcZ
z#-+ElAO+@j12_(t@ttz-<mE5s<u5a;{(aOnf$uBG{|9s)J}U34nyWYlc{wg;Dv^nw
zNH<zH8aaMgEzb)}Q<v%!s|xqos0qT+LVJrH-KDFZ+a#&0&(jtLN*~A~Kg%Y|30ku*
z^apQ19&}SUZ``fpHWMhK?(lPRbInw>gd;(?+qy3@h&Q4;#a@2C#4A?(WV`7Gl!TjU
z7KntC!mOaCGmPmEBU#4np(oF1<{w|-)Ay&6E#U6$?&`I&0*{5;$L!tK_3C?lVqcae
zaI07-c@`M>Q=jFHl^4+GPLo;nn3eK&JHdQI#_q@jwZxVvT0Gu6LSZQwSN&u!RY|w3
zkGiF6Bxs*A%;SKHdYnTOL^eGCwS6NmdECHFDcK$Bw)Eb0&`FX_MoCC-3_jl&fzBkH
ziOa%(^xn3&;~!@JlC7r#B%Fcr8zz>19J?T!L$I-L{ALpW=$YVbY(h>0Z#cq*bppm`
z3^p!PDIs2Q`&n!>c$ScSy%$b#j<?uN>hbdJvHBR`h73zqOjuappZL^|QF0kf2v*V0
z_SMU%Gg>k*T%kaXqnk$98~g5z`!fw&KuoL}tlvnayNX0fvjfrG1rwVo4hhp}#Gx{G
zV%vy>E=N4Mfxapr>Ka+LT;_TLxU)z^c*Pbw>T1Rz%PQipV@t!m@KtMyB~mfu5xvzF
zc9lTs3pQ%Vc7X^8{{<$?3QtnSGxA#fl5lojU6d7bWakw3HrCH(f_eEscSutl-dd}u
zwY!<CV;V##7-MF+ChIqg3U}Gb@mE3`>$>YZ%d!uUpc#_l5X-`X2v@sIp^_4NAiAx!
z<@G6XHim@*$C#|<1qCris9K<KISzLK1o|2<eQmM1^UQgBouVzJpRUf{X<Of+UUYvT
zg;<Vrnemnlht%(YWb<%&@0_;maC<5lbc@T<I{HT0GSS=-Gj)A%99hy7>HiLEaQMWE
zrgx5!wqG=2vVRP(gly5oO^8X3E<Twb)>o&mwOCuiP$`v?R|&AOC&7g?Ee5Yb!Y7zr
z%W!?GDLD82!wVLgbB#U0iK3*8k=VTXwJK)cBw?Vad0usNmD0unH!JsG%jQpjX^UI`
z{GBm5DzfxZL|XN=hY9%n#|6Z<tw#Q$aIk^B;ekt)=bk(93PFbED-TlN;b+Am?}H7@
zp&kZ1U<gifNd*MVz34gk#KxYCpcRd^gpjm?u-4a|mz%kW8M&Luv)k+aZhF=#H9w)w
zJB=E;HR+VQ`8CC%xKI1WJ0jK(V>zND!SgHb#(DK>Fqj4Wh6Z|SoW-sMlp=YC7t=J*
zXcCtix4PRu>gC&PWxg02x`yhhX~(yk)g|S<KD9cVZ1=@+=OAIJ!<fA9*;`Oh-fkzw
zo3Z5R*OlGM#L9~L9N0|Vnc$YZqG^wUuf}8h$e~W3AHZKKx$kyvX1XT@R`o9#cWR=e
z_g_kkzjJMm{Hgfah&+vA^jnQhpH&iq9~i!B{|ZmyRX%JtgmySsojSj)@!l+VWP2b@
zDF^?hUANyk|K7evP`f}OWMy0Mjck52Mww;FD}JbN+fx3WLG-5W>cZ01S<437?9#Ac
z<IuoX(RB`5c%>hbO=}Yy${uk7up7w3RgDya4J>Av@Jn9RD{s<^=PKA(Q2o*$vUW$d
zD`K(}>n#Ehk&2YdK7F8$aCdS4T+!G_uvJ$fSZ$_7_?1T4^2hC^4>Hf@*bb6uHoif*
z&CkAlDA@{ISIY~&>TgoI@NB)nu<Mz1A5xJHSMJC}7K}-k)KwMu{IU<}-D;dS^UK!H
z9IjlrVz?cYjcs~@hGPkKDWwG}_7QP=)-E`3YSb}XCA8c`Nln*VG^fnn>2t}%EZbn~
z9N4eA$C;n*O(L7IWu#y7U3PqCxHs-{e<&~1j#V&Mfm2|kC;PfvTWZZ3cb2SGdeaQ1
z93l%OhdB&(KLJzN-+imrYzrojZ7u&cZhZxewhU=cWUp*tn#p&}-X{~G#pX+*<O#Aa
zT{E*7K5D{TV!Vwq4CoB>+Nj^wzTBm}*2<l<t<dfQIy%RvP>6~x1f!C0{j$`&X$pG)
z7`z-3Itca7*w(+@1*JwHGsuit)pt?=gLPX%QoHnveh2Vf`~813;<#4TIn%uS3AXaQ
z!@tze50#wGT(OlItoZPy@k>Di3c6vK+U}H6RDECcr%oldg4Qd>UD}q<GklR1<GZ9K
zUjsLb9JA8l*5_67-Z|-U^Gt9b^R@C%{fY{lLGW@VFF%R{GmNVt(x#cVIqYSBk7c}8
zThifX^MdbGoA<fI%qh$Hmd0sW@oCMQ+$`K>E+v{tG!sM`*A}9Ywm_B?k5YSs)3&{{
z^U_`Bi9?6d(1u&(a1W_9qLzl{!&FbvncKomgU`Y#_migPsr(a$4RNj!7X<k$RfB6Z
zl<XWKy1i5RG;<O{6ITJR8$&J(W%uD4@fCIpVkBCzv6@#iVN%8J{v2@6=glwQNLSX2
zQa|r#^i|KeSyY-CkBHMJ2oJiyZXs`aj4ByMI2wQW>bjfli(D7b8W3zAb{!d=%?y?{
z(k5D&C+*HCbTI4!41>XpLJQ+S#G+$8VgDdHVia<2w8XtGXWV}2d4ZKkjuT8wYuKw)
z{0xMO$BrdhHPo%t1?DW9_6BCA+CEOfj8?cB_X}0avwM);zO@U;+ZW25iJT|q&T8|&
z$dDarJm~f=vM!JhBqlz)&Zf+|g@@0>^8k={Pyf*t`gOnVN0sNCm2HE)vvpTy85Es#
z*^hA!Z54`-!-)zOw=`Un(BSt;`!=1Qil!tPZ}agdQ>mDqMp|5N2W*YbkHlLciK-zQ
zyN%<H4^l#^M=(Yd2l-O*NZ}7tJoLEDWkX9C%mEq3INHF7mjf9SjE;330rD1U4{q65
zg)B)N-41*bP9rNR;`HDYsNZ6VlHwhjin!BGGfO#$-!Q-Kz%`QC3xEmz_?Ol@C1s>N
zU>Yk{BIn*$^|0Oz^`rq#iBhl3t$?u_PthaQ4A!ku#@k~2QkOrLwwnjK6p>4K>fZ_)
zu|B|!2j@jM=GTIx7WN;fHiafkslnFtK$qvfIR0LkOm2#(0F|Eq(9Ejze8!m<<!E4|
zog#TBMOjaTm8%fV6BH5@lFKeEvITkAOG~hB{$ey?Codw~tx@aU-LN30Dr?SCcyCvk
zm1QNL|MhAJ!wi>~m)`>N&d4}oW#0flh<W_{doiv|E<wJX>c{pj(Ha)^{Hs&%Z5^&`
zHrbPH8<{c9!0P()_Dv%vq{8{VR-k6sMgznalGrlgLbP^p_+80p1(kj@FkvDToN~bD
zmJ2PBIg%ZiB0}uz@y5L*+Y}Mt=MNKB7$|pAol3owwANC~qtFU@YEkri$=3ri<vA!$
z(gJ5nPSuBJga&`+<l3>~u$D=QPt{L{GW}9M4B;Q3I>SD+G-ZHtAAm9@Z*F-Qe=#BR
z7LXrjc6-rg_c-&MV-u&W?Iw%SyVGJ&YvvPxtCRYy^phRdKKn=4VLqcJ6?3>vrmyLi
z8Z(<cumU&WV#me{aHnhih>P+TYyK#xN-%51^ZE~2ULE7J*MGRZ=>KgZcu1eQdwJfe
ztA883DYdKeVgKdResq+A-ko2@ML!MTP@cldhG`!24Oc183ST7}-sKZJD4juySYq$Y
zym>qw_<T@IMs2(K2I+CHkBbIBtKsfggND=+43(5OjmyYLiX0&n^d&>EeoThn#&r>5
zkxf&-%%?FY61XwSL{{(gFcRNY%hpT7tSzDZ{1wKPh3oCVKCg1k+1eJNI4F6RaEt{G
z=Sn^?y8LPdY++Z0$nOX~N(sXhMpe!>seJ5FPu2JU(6IUXmp1x;1)0w6oF9Q`AK1<<
z9cQ{~$R!-U0?n8C$f~M0Hjt-TLw4_ajY%6V5DwNeJX>~J&x-llHhT*@rgGSP=)jL6
zTvKJ2WOC)?!<YLz=quStUb@lh_0So2u@Nc$?$mqUHhW@b;HV2SiE|TCSl~yJU&NUD
zm^`;%8gHY!Fim3C>n%D@8PZ0`@985DQ6J+BWvd=t-D6&x*jhx>V?{q<rON5eX>uB(
z&w^2cj%~*hjt0B0R(lf1M7w5+{dxk1TjadNr;NuXmc5>v)ep3BQJZA43&A1V&~~32
zBZYxN10{pNzzbvI^NmY1#5WRFR~s?(9(NI0<w}A+9&aCQd8NP}58R?<(CkZ?uWPb#
zEn7{sq<(d;qhVL%oDfz;OIEoY$wiR-SDh@5hzU{(|F`!d$)ri966ILh*et!(H1Ix1
zpyYPl(*}64E$7EFCTC;RlV9$+u|=fz+{Y!@%P9(kWM71Cs@HG%L@B_!2a07VQSJC^
zaqWpId%1C%<}-!m#YUAS8O=j&wkre8)`hBVBxLJ8mcEJN=vA|R+Bn8}`A70^bzE}y
zGp)G7e*)}4?);;Q&$B1D!aiF{8zmD_qE;{39bqoJ8Cg#dvMH;TVv%d%tiBl2l!I&f
z(Z(~B0!vZs4!?2l7CI#_Ty39WEaplb*<L3&@N&Hy5j}bkXZLwTG6Zy(iop5ln%lYn
zagsmu=YHvVlro4e*|uNSSMpr;4U26qAVwXKBE9yZKp+kXzAl?0zq%fh7u@_gw(Gtc
z1oL3Rnb2T`@6~-!@06$%y^n6LrPdfAR~t_NP{L0D6>Q#L8t1h;IGTnmtdKElpyEP8
zLR&!JyQ_nY+;Wkx08PXr_t?>U<@t^Veij!d!l*9!GyIm8f+iCC-t8BK<1Z{uAK-B7
zl!Wl=w~nFtPO`U(<#R(i`JLUN9pB`%`7tj#R{U6M9l0uj6zZlf^v9@&q0s^RlIa(a
zVRnOwMkd@|s6oh!x9=th8n``7E^l-zhPPHYlczHQz@z>0yO^g$eM5Y7O3jDq4X^rf
z#1^NjYg<6PPIboc&_0?aC0vu7SU?^BNS?)bl5ob@`wLR}DyQ2oaN7pLMNZkF2xqh9
z`5{9+iZv|f5S%uOiG5hJayd+)J%ZFXSl?UtxwqPMm7<t1Qzv6X6#LTb{K6N3?V@<d
zczGRulzg;Yk?r+6`80)B&?I(ruR=C^PPZ%zJV71gH=K@8;Pm$ucIz#V#Qm}p&aN6h
zB&!@1=E)<3W_ea3VvYw=LI^G|ckSNTY8um(E(g-Nf9uqm`qB8!bE6@x=iA_M_ov=J
z4-M<U#|FC=k9M<1K62W7_j@1SPPXY4NS!+;X+vZ(Sx_sC<o$pM1aUg)K6a)&fmQ5V
z(5|o-l*=*Yjmi&**bl`uK}$$`T#gf6RjOB#I4^dtks?|qr#J9XjUy4FF%B96Y(T5w
z&pEvNt0#bbIfxt<*&5$9s4(Toq$e#A<C3AM)@YMZzdlD#4`!wIeye`SF^-Hnh{Nj-
z3eg<<*wIp<^h~M!kgz9iCYNW{3A$}213L;{1y)lSsKSb@CeMIaZq=DSrCmVNH-<<2
z|DL#pN9EB+50*5t_iko5$(bMz^Odj2KnRxi(MmZ@vjwqh2F&fTD809FhV9eVfyM(~
zJZ-y+BS`_5s!jj|Uh6K!^F6-tIA4vH97&@-oL9HpJ!=XNc`v@Z@#_aNuK>zhDz1do
zn`y+y^L(egAs-JVB71MtEJspO56ZDP`aY(@IPM}!VDr_Io{~f|YYys4>N>P`vZt0!
z=b14fO)|z1NQ$SHo46F+sWvCr|GwHYF~7WFHyYDc(jb27*W&B8jxvQJXtpj+nU@@c
z(GiK(;cd}^NLj?wd;wq3upBP`yc2x6Z#Yc*`J~yERFl|%g@}UVM9zpp-0Wyv1)rZ`
zQ)Z&p`3$+yK-HL6vWvI-y^i{ivv!1geM<EeQxn`YBz@u4GMpB@q37`v{7FW47Rr~-
zHv^#!U7F^35k4i$gHeqdg0|CIqj>bQpRRoQw&t-Yq!`D~tX?9|9!_9YydhFc1@+>Y
zm@$veKr>_+mK$bfmRNHb%oK+w;QI@$myM{_;t;p3<}4DDmdGo%Egai2$1r!}C>`kd
zSeW@J=DwO5fT`N`r+?P(xk7iP(n5Ljrg^f>T5(FE{fIg)GdU7h$<H2O0PmE}H18Ll
zC$rudQqAoRyISNRTH>@<L820Kb)Aau91tU0yJ5km;WG_|K6zVSrCViS7;Hy@q+sr6
zF?{G^=B0Eq-eoa%X&NNH9q*DFXAb=!>sN&=67Fm@A91n#ddeX2Uvm7rwW&H&ceAWX
z6e9sAdrTF+E@4}NVQ=*pL}6Ij*9`<^vWK<hukfEw*gzE=7QYx*C&mX1J^!sO1Xqb8
zUh$6gpm$hUid`vNLLm*M6;dD8r)5@Lw~<;zQ!LEOJ&diG{bBfvSrL<1Q|xRS?As<?
z_38<rX<qE;MdWs+-XO{|?rpb$xdJTUP}+KF72*P(>Eeeijf}oNrS|^Qzf`um{b;OX
zZRp(L@ZIv|hVJk<y4x1hBh8-L`p03v%IP=@d!xP#t^Ee{(i2T5B;~U>D`*tm%k<{V
z{In{o9{*|K!By=i^QNyjO(l#g@TR%XDl?OC5fO%2$gJ2UL~H?;kZ0PwU>Kp0?k!+t
ziY>XM5ZmKvY*Z=U{M<V4Ni>{veU&j6RZ-xmEHV_bWc2Qs!jR|K-7HiB9bALZRSeH@
zMLq8K3hh~3QjUH)L|S9`XOWH;7c9}KH)x9wH|;lzZiqF>^290jI-53)6D?!QEJ<fD
zr4=`hc0G1Fla?OkF{8X476z{dI8rz=hgaw3ht2c_6~V^5D`#Q(0tLl7VS&nT!sSGZ
zJv_=Di%qQ!L~4y0rj*Dihc?dKrCjv1m$M9bk|Qh<L1HcE6m!^LW6Vfn%XoI}drOTC
z3lOAB<2Sxgn>l7=srW3N!-2z&=J|#ut+^sD3$fzC?!A7ECNj;cr#;=(u&?2!2~T8r
z+(11_IDFMC0MlgE{NDBeO4%K4`y@#1R`hK10-r;(^>0k+P7}f9R63%D&)IkffsXA6
zUW)q|;XSb-Fa<bsUF(;Bx3K>+pu&TN@CGza$tZ-<q&!k|xIG`_KyF)tVx$J<S*OTu
zTf_R4-{ARrAN2-`E$xc#JPTicJWayo@33y{es)h4&|2=<cZ!EAto1g#rin*<$XmH!
z-0I<n73fl6z5ckb8vQ)Hx-;a7oJcp`A|<29wreQ@<u2POs_|?;-R8Y`Wwnl#(e1hk
zlmd1XlbDwmfo5KOlO=(^-oC(!62oM*^4k+W_3e(<^ssNSX$13L6yVr@ylI`t6GL6G
z$>n0${phG{tQsngiv*F^uB?vNSgF^j@=0@3hWH5^QXGkDo0t0LX3NN1_+qO{S`=aQ
zKpd_X0j`LP?BL<`b}liMINSE?E&OlP>AqgK*G}FVFD_qw+hZ6moReP?!FL4~i&sk#
zE2@a~*<N}a?v&;>wH!sGjPx26X!GwPL9((Id0STzFxw)|Ls~2u)2ur;<Nqq+-UK(R
z2M2BiM1Gz`6|E1+=Ml5=gn?mzO9^ZLu9mCsdq=~|3nM?!54(yr)U~#l5s^x6UB3hn
zESkr|JDqw)tH%zRjd#j57`}PR3L3kk;K^8r!_cvac^t#gxra-}kE%D!PJZC3*8?d|
zt<%LxtT5iQ{#`#ss9D$vv=l-lJau6#8#;GDr2CNi_*Y;_y9ULc==Y<Eo#5TQyBjPl
ztB9;ZNM`gF;Joz4zXkgj;H~><z?!ym`P_>L1Dhw|uRf&glDBfaz)BX&DI|_9@94C(
zr*mI#Ob18TiAf7fwR2B;Nj4CX<mQ7L2~?X1j|{9Z|F<MLZSJ1g*(s!Q1Y~3y2lCfk
zlyX-u;j64t>R(oY=(QWShLu#VS68ua`h_kB&t>_bJ5>$LFK0Y8n1HFMD;pQ?m)OJo
z$KSjFy3?$@O4HukOvWtD{toc@=WG@LxG=E$WT$sYLp*%R39pdrOY1b-RQ&mawcR1t
zbe*5O@{lZr?tZz>9!`H%zK}9?p!!SVZoNHu!?n8g4SI(sGDKaQWt@jpRiq){2-oBu
z$eUJ<L}n&!Q<xc!NLJkU6|qJ|X33Dh>!zc=#ykh+c3Gzq-*46qz6El|Y&8Q#F|?MD
zsDAH}K`RYX2~N{8rEuN;aj5&OgI(Mg<A{iidS$rQ^7^s?dSpQQo&ef6CjiUr8DUH2
zX~LU+(52cpVy(qgf1l;QHW4~z;vGLH{rsQ=F|F9y-Ew1gQQ9Pt*WH<xRFE4mYBF6-
zzz5eGrsk*_nYPuTtQV^V29OAAEEjLTqNGA;IZ-xqi`Cz*<(Z?(M_jKv26<L3DtkZD
zMb<jk5C5$c4n~=NXzBGf0xE|O#l`m~RGIzSI4|~K%z-D>t*ssgk-MNDXPEzt08=-@
zD_Ikjm|@o+j}Xua1ee!mbG*O)?+xU?xIT>oGMp@o&=UM&Q1BMaqWMQAJ*9zVch;f!
zhjO~<%3ETN3OCG9IlJ1ReqTb_RC@f>Oj6CAIg-c4MH03RSnwn+^=wm-N1?SyRt&yl
zMuuAsB}rQ36ip1<_80b+Noutg@=YFmP^i^bDmhNE1S8sQNAt6>o~8_64}VIHzVERh
zZ%^|`%}dYccR7Vk=G?WjsV~aUyljbrHaB01ld&DSy#})mEw-N5Sch}v&o2zLT}yH9
z80_krcC4t4YcEB~ZC?(A<1`9fV_aSK8ZfDHQoX5MH7sxFP@abDGaV$?kZ>1k!eGFH
zfGeI--Zf5WbStN=_oOS<=*~5DMmA*fwjsnayC|1)Z4A7O;_e0FDR9D_TbR72cIi6+
z8|HJb|E<5Ps^XprZRQ&M{7HS#Y}=mLmZ0%35BHea)0|w9K;dcpr!5sSZbp`%Yf4wo
z@S!2zLHckS!sf*%0LLwUnXe}R#oes;tQUdn9mwKMqTS)%&1&;QQ0e;Jt{oeR=BQ>1
zs#f8v_(G$PbQfGj;<Qf+Bkt$;h6~V};U?j`FrZ5fUVn7^4%|Tzh9oQfY<?$xqBW^r
zL$awNwT0-q{5#<4z`tev1Dv9a;$|v;AC#r=wbbwLX|=m#c#sZ1%NVAmNve4t<FMmu
z`Y(e9X0JLe`ei>Gn8815DQXm7f=z@RFyrcH{49Q5q$?Px!4CW1XtqEO#zXuk4uVid
zz1#M^QNWkO&fsUNwG)=nTlh?bBwzM;4G3`lbnhR`0Wx-uN9bX1=lShaT9u+5VaW;D
zNtV)<NzX4r#ib0OhsHGCq0RKgf3^-NQdHnopF~<O$d=ye!^C^d#Y4&Fg|AKkPv>o;
zeWi^{7T#uK>jn=m$oxX#lkToy;RC{(tsLZh-1L&MMS9d;vYxr%ud^|n!v0@dN0nBU
zRP4xAH3BQ7lK#=5audOPe~ri?5ZpC2%_v}90rs2SnrroNx>+czhl@uFF{fG5u7>T$
z-JZdcg)2OxO|>u$q%sR^A=VzcApy~HEhs6cvBWpO!K_<nzzo#8qVI(Eod9m<M`O8m
zIEUP!5)sWEggWGlp%`BNvWPfO<AJYpbPtqt7)s=EP)Om&7Er4AhH1&q0IqPkAO5YL
z{~qvLeZ!1=tg6`_jjlyeShIUoNSbJHA0N}9gw@L^IBLfs@G4I;zw2f3pfy&9Pul~T
z2?NgZmNWDbLX=C2#r3go*2|a5uaE~?=TP9eu;|T6EqiItUJ~2&k%EB@l0qd|amEeu
zQHbbvZ@Te%$#p@WFameKkk1_lZ+B5wQjxW&z&=`|L|G@<K5YI?Tdb|s%iU&XsNrTs
z)H*%s0<)2dO6oSqhmB>t>Sej)&t^87abg`AhzaMArdO?(gu`1g<qcpBS(M&Fgo0t-
zEMYKjlYbT9IMVk$-uMr?{Byr)=fLQ_&MM++{4J`pGYCBuQ3?d&Jr27HTtGMwP_l+6
zBhja~W4@@BjoYM7YCaUX6<S=p?(<YVEOh6X6NcD;Onz?8Z;jqQt{dpts=~`e1n9>d
zXbZaU-bKq$_CYd{eJ_3l6hQvq`PcKQ)?cKrHQ25x+!-U4$Ejzh-7~*$uTSdSv9P8G
zO~DQ+f-Tu`rupy@|F27kH*J20#}_8;2|I@;fJ@Qi8@ESd#$xCQnVwPj!uSm6+r%5!
znOIHK#&z;{+dEg^k?*cv@)67KvGNZq7X8y=z8`fLqmauMnL~q|onJ{ct26V~cTYVD
z`}BuMrv_woGL^IJ_qXZzDZ6_6q(T(uhp$Dhsguf=OMp}Iv&NBFV|rj4bCuyHS^z03
zfJ&H=_1YC_zbtn<t|}-Kdtow)b0Ri8N+dlN*9?r4&mgT)n-n6gFpMY>Mzjv@{r#yZ
zPW~j}Y7t9s>yXDI1yKqdQ|O!DV|u)Kf~G0BkbWQt1Y##F5eWn4-^y?IbP%8U{!qRB
z%kR<h5x>rFPsk-*byIp?E18@sEyJCHce@y+8x}QsCf}0_GZc<ch$ZiW+*u?pd5kEb
zI?XJK<h}ZKQm-mx9m;Nx@ZL!*wwx!yU}h<O$fh`WhQQ0C-c%|!y9r+q^W4O*q)#V*
z<OrOd8NO7nXx%#3%-gb8b~!mKDHOWAljX}^;$ylF5>B*zx1u(OhPjc*#$8D}k0v$>
zy%Dzt`uP&Ape(Jej3~X2bfkBILr%G9mlUhltx^`47X}JA%_Hzn55*t4{x8n=Hfkg)
zYf8=g_c7I01wPd@%<t`@8s65Arz7o$xg`|i9v0z~=_;oqhfHLwbbZw)%o~uIG1**<
zIp05t2*Op*<M7Kk{Fa`ZpS<r2)9Rh><2RGyo_akbHpRCe8@N+QHNOnPP0g#|buj4l
z4&k%fx7}n>Ft+OMF>`#Y(U}~&5BqbrIo@<)TCo{_E2_1~&dxizzB#CnnEj@ZzG-O2
zBu2lrWu8(@u~08HH5b^jBA=Vk2@W)`^ouEIGu1(Ea7_uR8@IFsuVMsW-Lrw{IxkJA
zfB>9sR$u>8_A5-wZ|{`sByo&On1s~TrU-YjEsQE|9GESwtSqJom(J9@vve&6QY3O5
zu6ue)lN-CkN7h4mpC|OK9aPcoD-_fGuZ;!Fq^EDlT-VBPLF9{K{B$v0jcMdUFP{^D
zo=C{+vwCKKVfb%&>Q+_JRNo9_`u7>pw9wX}$Wc#tlMVL+hohjr-1rdxl*mkHZzz?B
z9SoisZj94tT1U8$DU;h7!hBs@hY^wo`28C4C4|w>Gr#1z^AJ4oZE27W0bPqV8-15g
zXRmYqn-KuOSXp;hXWH$9d&Bwihbr-G-rzx!l8`d%a)hX4qe+A(X*)I#H?N@Bp3Lhf
zs3C`h=LRgH&8G=V_jvm+FkX5(e~zr$hB_!AzZ-hFjw4x{y1DaO2*RSX9KDTf?-Z1X
zN_m3no<-F!^E3gU-X%MIt{_+po-av>d{QJ=WxoQJGg`abxU<(*?lvsg{?-%=hHZQk
z%W&XG;K(Zo#DcQy-9rSNzd|B;PmdIa^c)1}m5YyxV4mp0;qvd5&Zl@AnQNwvnzk6l
z!2-`Q;-qaBm9U2r@Ld=!B0b+_NWFdp)~RnA@WaS*b4;VGnJCcdbDwh|R?{>ESIT$M
zxz;6W8X&7>m+%*z{1@=r#k^jG0+fpptbq#(3QED@h(+IHHV%L{b!=8-@=gE)X=J|&
zC$SH)5|icbsg2JT@s)7i_b-CWb1FprLS{5l+9${QfEiULlP%~LuK^xq!lTsMuOh<o
zGKuzXH#1%qPWS0f-NWZ<F_6*VAbafQqx4wn=b(nEqCP~(KnuGx8+(lI?QC;x=^gXt
zW%cES!Hcim?P$4Mt(2=ISLG5v6d!4s@_1{`hc;WDbH#`_uLGu=ED^qu{QP*NY!H<E
zI*RO9rcFpdeb1+7r2As8{NSpW!ZU+2ufN2aJ#aO|!R@TocAf=<F9xMn+|I3xd)Q&Z
z?=RwfY*QIKj95~wsj-||H%uJev+&a7sXG1z;AQvtFDg@&^tQ4ZV+6pWu=JzvVNEB1
zGhpB&RQX~=ds;c2Zf~<v#g6rKYNPfM*HBP@_pS@Zoda44V(sygAwP#uwL4dAj7-PB
z6{@Kn&8r!7m%qqP94ClH#*S}fxm5Z0WvdJ0j*0J20OKQ%5B%AyqEq$D1)pR5SreDl
z4!uk|zrHU37zq6K7m5B2x%q~#Y!PdWvpX;RR%c9CFy|qg4(jhn-qgzPT$TOQrhKkq
z-51jDySrC9n7uWbW`5q=?*u@bi5xwEL~L4oh6pROrv^4_G(G4|Y-YG&rSSW)d0FwV
zFP^#hZyt>OIB>sfzNStPZ{mBcDx2GNTcP@`^XAg3wH0y?9<jXWx3BV4Gn%p$6@QmU
z18<}$4s<=h(jMnXU(<BQD@)GD>v}~(dQG@)uHly2A_ONkN-f&ETeMB)gQkr`@C%yU
z@wJcXK2Bh(lu#>2$LLTqhiy`~wA9VT<uqy!zLPJRdZ3ik&=FJ#EH6<9m2idRh0J1D
zp_ek%X2!G0ZYjN&UC3pOC9KR^i)+t@3GZx$yINB6Ot8mFT|dcp)#syQZ?j%5CjEh1
z-Z<WTNl8jl$}k@(1TpuZL0PSmI63)Qp$aIB$D?^wWeLV8E42it;0S0_Mucg*Icp(!
zf)QuRAXe7<`i`@CQyQ|+b{F73;$a!iS&*B7!E#7yEC8_U`xlk{r|^v8Axgd!byS4p
z(m`EfE&P2;<3{Um{AJqrc@fNUZ6ioBdwiBr|AO?(JOY?8AvZtMz*@LVUzRbQCq63o
zUyy8F7**+X6g{o<Zfl1(e83=f2|R)5>8b>xF|fR%OpgtJ1@?(YkkY3s9q&Ta`SITf
z7QHOET;;2ER^lT$n~)^yL``PzR~x;lUChsx4q7Gnk!0-wx_#-_{NT`l$$V)9zQ2fO
z5!LOQZLreGBWVMBd>6poAp81nMLG|jcOXW6ksE%nyYFG*cevJ+tfxvmQu!fYV7yR{
zt>%dwmOauu%e8+etGH?D(Bo^^8@w|6+7VT&{6dOVq2`)zM_Mn{iV-49o@bal4*r(;
zzb;OV?OUOy%74u_nu&zSBF3@zH(=quCxE%tSC3L@)2_bJr)2yJ+N==U-r{9R)hB>2
zxT{qqo@8;26F>n`Mztxt{b-892&H5MV3mV_8OQYfo4S;UuYcoZmib|HZ&Z8np!6ui
zXw<gXSG5VbGlQY-<xrnT4|T2RC{@MUVmqn?R?yMS3zCctD}LIgXMXl!lV_i)fDLSU
zvv_u9Y}r|+>s|FUe7QFwQSkxpez7J%sLi}{H3<8lu@DSHV+1!D(?}=tG{^YSs)o;v
z^${17Dr5!kcin$HYfce*bh92Z-%V!n)NO6LlLF#?KK!I;Mvf85YCT5xDY>^T6?YTg
zQcW{%%XetOkB=3f!H>f9v8;$*zz?6Q|5lp+<l+>xGjzIL$h*q7lx=5EUaPy@zo0&C
z&3WjPMj0h|xn_0Po@kL<(LcOl;v7@58#b*nvVtnpu6`k$Ue;5$C)}m7f-)5MuiLjg
zkm<1)YmjmOWUf%+j#l+unt$T>wpmPfa@nXf`4a+#@w-T_BGkxgNPfQ2sLJZj##>Y0
zi;87i#$`q7`fbXtCB85X-T3BY=l(72*K7cjqiyx4zX|<$w7%`bW}V0hU~IJMVfCT6
zrgZLsN?Ib%nfLCJ!jU?Hk;gJJp^Q-pv&_)w($UzIN37A@Zob5<?OGsaoZT$Z-}+^S
z-zq>M{y!W6uhrZ*0YF5;G0L&=b2VvuCH(FO*Iw_?+yjzUP5?q4sV4xqI`QCC!(1y*
znjA$H3A%m)&^*otMM=`OP*whOYEAwh_ao#luRKs^2uZhkUU5t|j6cd1aBV|-z1+!d
z?tl#7UIVV)`m<Q4o?ZJD-cobK|8dW}(tiV0)D@X#ZE<X5xlT0Lcc~e(&B*N!n3)ZD
zcd_B{n#|4jFMW+ZdE$g1nNjpA)iv`)p#_}2hU5~OMP<rq=GzXj>qc>UHXkR)cGvik
ziHu42NGh-FZxbx<5E5xkdYI;8o*3+mWWzk&d_JeOQo+=fkn>5MK5r~nY_PYxc|>$+
zW%mDU6Gv9jQP2GZxNIi;7g7Hd`hQ2zg(}fYb`@B<wZ)p(aBs*7AnE`%ROG5ZcaRPl
zO9SsnUWDQ+cBdjMu1=ePa1^g2+8#6FSi<ZCpem+$bb)0c_~5tB6F~D}Um>}_ldhPY
zohQrkjo@b#LWg9Ca(8|U6!IXuoB+^@I#s)-OQoPg1xICS8Ufnc^fjj(a3wE6;BPXW
ztJ=9c=H)o!ADzf>O)n`z#t&*NG1HITU6v<vib{vqY)SsbF%#>)s<ci1`hw2#G~$6l
z{HE%`#hUfO-4nodzA?ag;GY)Yzhr)j&T5=Dwa*B=-pp8k|N4|9*Fxnghx|ro93RKB
ze@@Ir^m!&cFo#UcI3Ny`)esrFc)x$uV}Zz+8t_dY*GqL+e|rH{xlT<Z%>w{Af64j!
z_^RoL4?CU-;k>mbvyC%m`q&4UimS1x234m^M!ZdnNFwc$VeErlS!s=(un4>PhHk>d
z`jP)<0>SIvK1g`ih<|@Qtpdv`9{8fQiJNP51qMOPU1l2HbkUBo^WQ5#9$&OKsXd5w
zYYNzO%xIOZdDEmhT4NHw_4?HNK=5BA`A^~L-%?j#zA7sfQ2~sB@W^V600cR=O4>P$
zhL3oAE5vb4S+ujK`FTog>q~xn_WlW<kFQ@|IJ~6M-Kb4?VogKB{~=8#(&c4PNT|@y
zhZL1U4VP-u#EGvLhm9zTEg?q>ioDxBs|+ho?aG1E#|9@h7;Q*K*Z4S0G4+Gwq#Y{;
zQ**@7<IQp*fZV5Fe)^l7Tq`5pxL0#5>!uy5X<vy3{3E|rPjh$bjI6Aq@2_lC&1$3@
zrMaqFZ0Fy7?!V3KPmfdGvji|<JpPu;9~VHEN@1GsZ#U<cHb_jWElENrfIe^iqu+P&
zwIJeWg`;<63*l@1A9Riq>@6iXv+pRJ0F2sF99gS5e6&@mja^Ph#w6i`tB)5FW<VQa
zA=ps<{%y)nj7`UX(rI(Qi?MuG6%I+Z#myfT`O1tiBi`zRa?5GmD9X$rY<sAFAD0<F
z903#8g}?vm@J@5Ty{{_rGtAjcs$t@~u5v=+5iWMivSQsxWr-i7s;YxH)N5@_I7AIQ
zr0%pnq|u%C{dIc%yE@aCW<%F^BuIX|@oD)*d{<aUuhPC?e8uS-a@<=e2X7FgM=h8-
z`UzlZr26_#0sm8UI+u@=SD(?zdTh&xuq2Jn<QS{B6iI@bZ89hhvd5rm>NFVlsb5OD
zi<M|_1OyH96cU>rX~*$~mL6w-FP8HJvb%P#3?L<cY<*u9*kO4aGveHyGb7^^zzp-q
ze+FY&-Lr{*Q`v1<I?fF37I5gj&-*CA9P!SpD9Uz_i<m1BeYqpXqVhN{R|1}X$|JUs
zaD=B0sBcN|i@tAm(}m?NSlB91Q$Pm=%?hUZvt!w?Iq)q3fTKs>*T0pCgPp<=r<*Wp
zu-_Edpm})Rj<KaREp71F38$ecsO(o*yjiEzba-jr&|(DBmU05vKRy9$XQ5b)=Jrnj
zZQ4n!BNNk3vt#kcn@{fmcv`>wE%Sd}oWC6uNo%zN?(ZjZLDx<IDs4K9oo7v{(Ym-&
zxhj_;bX#J9d(XnIxcb&HkNo(K%SC+(=jBdwDmBZ6*v!>nF`_Ma{|FRox$ourH8OH=
zg2m*xZWgeRFpqD+9nuinuK_B5;d?e*<!pIIf3MQJF+MJkt(*>v0Rcqzv|LE2dMoL5
zf4qrDa|eHO&N#GlHXR{6A3Xq${5<|78nM{PnjEKkcAgI^pIR%6_CSBlwG0g9BC5<c
z_{EDRb{1t;CNCbt^~0HW6WVgTeRn_$=23E2ek!wOgtN0lpt--IStgcMxuz>3=lJ8x
ziJ~8mVj2rYQTNPjoF29ZKZ(l7y;y@9B*miQXL^ST1{PfgyDwh@ezs=(9l(E2Kcw$c
zXC;S~9=dg!pY@GM++x=o5=sn1G~X1JNW21?(#i{+JQMHc0}D>A1uCZKBKVCfTh`ut
zznjZ4vz&E;EB>-OuvhAE$@=po>{#Wl?V^>P-|3mjiE0D1^W?I8Gqn1tFerFA{+Nag
zoULm9vY(@|=lY>Mnb~oJGI^x4UG}Y+;m|TyF%hGCs?F1(`lJ^NZCvwaocXIyoJ<GB
zgdScbOs37`o2^65Rg~ohnaZ}zl|Yg%LAQ>Yd46jakWt^w4qjn+)KQ57E>bQ1joS*$
zZQeYLRRoo>eT%Vvxb}IUnGsSA^<+d`V?)wppnllj|E3lzT@qriFmahV(<db%QNYrF
zPD9KQvIWGv(uU7LC6Ok)U|l_Xxu5SJRkLPANBTSZpD$hc8f>xU4=FkU+&By=*tBTc
zxh?XXQz6?=@(Y<KZ41NifA|vsE&26tjQ(}O$)6>w;+}8(4wO>5^n5CMNBYi`7je}9
zGT79=!!gp4TCTU(ku!;%UT=9N%|;vXmR+mwjofF2wFDn}sz_R!ANu-fq{c@#H&|P5
z&Sl%^uNj&Lp8$~Q{I5^{`ZtD5s&2#{cGvi_v9MP(S~pEREsx`e;+J@zLvo5k$#hBX
z?#sNb_a;p?w(1pu3RV-#D3~eM(N0AFd{E2uN=<9#`^K>rGYpKGKR*&LUU3a(&aC(>
zV3`hMzP~cCY?ap;Lw(u^b22LkZ~2IlDc^!Px*-B8g<a>P)N>t=4%qpYRMpHAw>!TO
zfX1zQ%h$4pylfUIq1+jrQ+KzDDlRI7FbtyT_@b*W3e3m4F#pMQxywMH@DC|KZ=i4&
z%Qn+SZc>Y3hpqG=-gn7X8c`JIjju_u7T0!k@`SRErzDQjm=Uz6BPw!X_bXj5JNVJo
zdU0KEi-f0<Kl6H22wU&)J6SJcUjsaI8Gq67e?lTwYZqo0yaY#U5V9ezeuz_P9sbr=
zj0z&V3lntcc&zcq!CH^rI-5m0c~^gwS?Xd?q1w8cxR}#g<!lT^*&@5@lO<9tYJNv-
zbKE)U{^67(x4gh$s@7D88aM>2D#PaXc5Dl6hPKY+hDNyT_1_xjdvE44`gw4PJvr)C
znjnIs?dl8IsJ@WNncxVl37-h@(G{Px3&X8dT&pRoN%%2diE<5RSs`(Zr$KJ{VxcT^
zl@#G}t~@Y;FU8e53i;qT6_G321Ol+U;C%gOfzP+QogwRv+Nc?jj6GP{M_C1f*H&xD
z?5!g~VITQKW1HcVk?~a1w~>Q$hEb}FPW0{>n5z=+Hk`3g>{iE{uvzvbFV%|LxIBq~
zSkqa}8LPfTXxfcq)y_j-%}YP&G=zom<zAKn`ZrLgB^etauLc1?*Z&a-_@@m2L|1T2
zCVUI_$ydSc+qofu9HX_8b7GLX;e#3`sORDC<Xkf@XJeZfwkcbA3|rc%h>7?it%)y_
za8V!}GQ1B4!;a6h`_hh81j%^Y^_TKWtHok>bKAY9Eh#L=R;Kt(uv``Zdi$rpuw^k?
z)tNhOKk7KX-|!nQ%4**%ujy@VJOBO1+g01zta|N6JZZsOvOc<Xx;^}d&|(E6YqW_d
zx0p!$g<a!K@F?dJpEenTBbS)t2hwnf^_UbLUbB#@$Ff?u8K4$M9-1CdIlrQwjoK4?
z3&ldbry86%h9=4%*%aU3wOcRp>Z3VG>JZ##*s=pmJo8zD^6*R~S^KWoan~Y?UkmBx
zedTVebJ_h*m*I*dB-ieqr#h)0=n(YKZ8c(;r2-TxsJ&75BK)AkMiW-~3YQi7`idAg
z=r85_FCo(<ch+tmfAey_-et^`FPqR3&RVK$SdJOa<JVxu8AV$!0?Ky4E!`9iE2EQo
z*S(Q=W~f;U)Lr>)D@viQqZFH|BBMaGak>AXL$1|FY;t`dIr=HJ>q7^k(JNlG{S1Nb
zV;$byvg-d+5rL1xamZnFoUpY&Q#i3$s=jbTsz%w3pmCJ^ejYA&h8T@UJ~t%KYk&N<
zKpOxFCUUAEgm3mdxQzLji<a5>6gRCA*gM!>+&o}!u%)ju_-T)*PAY%Pds@RS>p3oX
zdTLFo#2isxaplGnun#pfxLV|+_PXH{+mh^y)g2ecZRzFQt*OBjNtnqs1F(6B{1|C9
z6(cR_U1pEWzxs&@Y5T+LzvwC=Vv-d4C|_O}<q<0QFt(&3;=$A^TId+d)2r*>`58E!
zxMLI$P3x08r-r`{MsYp}*M$mfXfu0-1TVFEqF7<fh`8LE^K!l`4XSHL4gHy70-V$Y
zTu|JzB!(@V{Z$r0Z6BYZ2C47)@E=6J{X-B<_n+6aL+0X6@r<)ub6W{bO$Hn<#KIn6
ze7k%MLu-%6oh;?%4C0PjVtzrT$+q6@3nqYF3ZnlPYwrQo<hJhpvbL_}Qc)Ko(q&6k
zs-OYs7CHh_l0bm4AQVX`lF&n0>XK@}1On0*9TEbDYJh<B4pKrdBE8qp!TY-Rx##SA
z?j7Gb-*@giNXDBn0%J1g9P^#?|NNid(>gp>FSY@s6#;HQu*RIO`_NWfSlu_-utulW
z84?7;9(7&kFNDKPlxegvUv);&&~mNmn^X3!Gxg~F-&a;tA1r}2H$b7C^E;<p7JU)<
z+spCAr&q05Oa%XRmo*$TP{|+ssit;!D2r*EGQ4_L2Ww^^uZdRi=GbwrsTKliPqI9>
zMF<&r=_<a{Wkml5V0)1Y>vb10OtA{oE4?!mk}VGUE7vgFckGIP8A@XqHS}6kQ0|WH
z+jkLnhQk0@0t#?6{;F|s&zs@?tXF>Wn|G&bgyLkTycFW`v$wiri^xam!rZKJZW+&7
z$B&e~T}u<-_Q+ynViSsJA87SjO}F8}%*;~^H>E=LLAQK^Q{CFBpNC|Ck#>Vx&`f%g
zsxk-X+dZEKqwBL~`;N`S7^?!}jp3G%iR){!8Jr4ej>x-Yhv>i0)9G@W;e`b4i160@
zo`oc-$i+PROclFT*x~>StE$}J|7F(vFK>VP;-B8nFIJFvtF0fNJ}t4z6UB&*5r}Q+
zPL**NA3?!0ZP^?$W1B~#cn><?dbvg@kn0>T$m{ePo?Puxj@#N>o=SK0_sKU8nNar1
z4!DUNEfR8g-Q<^vv5$D+fUu9y0_XsvKN9e7WIEEZ#D#n$R{^8s#(VQ371ucKuL+TS
zRDIrF3t%7AjToK<<W#`7lP#j$ye8qb#z~4R?c6qs>)(-Z7we}m{=U2H%5#DNk@?Op
zE;5E+PL}y6U6tbChc&*pDkRu#n~y=f?o<Q10RN{K{JpzHpnm@CVy!VLuQez8qVwpv
z($mCW9UK*YZ&F;xSuOGlrMDVeWQ0n*OPo|{81A?z>DjO{^rYf!yfWpZXHnk46rtPJ
zLDzZmm58?(qTW0pKXT-re3pH%VpzL2`vhympb?}KLamr46EeNCcvCl9zTM2(bl$CZ
z%6VNr{kwR{nhtx$O{ctON_fK2Z@K!$-I#=1umUM9aQU7AyCB)E+8BkwmWKpZbp+-{
zL|K$Z!F;f+WQGtGk{pE}CZlD<DO#CnPy9GZP2QCXw4I@ZjfOwYiR=8w;BxJgU}Ob;
zA%!q4A&Q8Tbqgt1yrYcGA@a{@a6m6#&y5)(8hgzBtAMdDJV+C{(+2Z((WurvUqFoP
zxwXR=Wf&CMaw+WwSabDhEWYNhzzrPWq?+J+hkiM)8U>*h8nwY@&CE`S*Xll0F65o|
zPZdkOmeChTSde-evc)43D%05{ptiEuGdEA}`4&9%<|?tjQt+hnRW4GwOkylNu!Btw
zvcC|OH3zE>Fkgq&ifBG!xnTeQo{+=R<IHQ-yj600f;;Jy#tN|2)aT<q48In-_SM;?
zLj0;3c&2GwteC)OYxsKkGHKB>o64Du?@?Z}44Xe_9n=|c)v?z+nR&%#Yz*eO6%m5E
zL-(_3sMtDc)i$#a>g^cD@H2jPpZ~Z&$3Ycm($yP?@L8Kk|DZ=K9#a3Yg6M@dAU-Aq
zcDeR4@8>@p16Q)=%qGSO;zbJ1vRurj`0~k!Iirj0=M2CER|}<e?IXA73s1D*G~_z(
zFW#V$RK~=<X=alkyneR=l1N$4a(mtwR(4mln@~ghGtN<|dQ8nvkig@Lag5|NF2~4!
zrSj{%nEccoorO%1AMjecrd@9`1X6`^+pxW^pP7L`vh^xuF3tkNuB4GrR*Cmof+#I$
zbx?Xd6$=qfIMDvq(unom=U1CttZF+QpCe^Bpg~!hK`~9E(XkL6#>t@1?aO#)5NB7q
z@pqONRqb(BnAtYU>yd26kRW3;4p2_e2ZxLorU@im5B$|>Zznnk!(pOfR=h!TYY4<C
z%{VldeC*FP<u@r*7l4%U+(11xB=Wq}gk&ob?OZrW49Yo+EB=gW#O>F=nzv?z<D%rt
za2iHg=ZvE;$w2ZURI$Af$UZCt(hi|O_Tdlr0~ij*=6^KPKnlUaa2o%-3;~Nc5J<Gw
zSPf&j0Gwt2`&XazA@)84TeI};Q?}kics}yWD7`<h{RQ!Q_E1@|`OtlIeT#XLd5@%V
zgGYf&-?w!Ut1cZ$T@$22h3Boa_<_SN?mSulG=r%m?!<;shf4Wr_!`y^^(_uG7sDuF
zPlHc58-E$0NPc|Q(jt^tV~DvBIq)HASfxv6eFl9`iJrJE=78L!5ER7dH#S>wiPc+V
z``3<g=Ho+DQtZXa2e_h~G4F=-4KDr4`4O`Hs@?rhE^pZ4oUIG<+x=vFBAWbWJiVQ*
zD8d69{fYEZOb?{-zB(!H%R!7cyD5#LWtihR#tSL3E*(b^sy(v25(aSINy0=gmp{qV
zJWlx;H3m>IWqfL8R^6<y120i}n&g2_>GbOGJ$Y;S!2L^!s`5bLV=0v9I+(a=K}ME#
z*)vv$5~4$VMGztp5G6&Pp)zBINU9~{%_QaaO}n%h?dLNDp32O>5xjJ9HS;nC5o_2u
z3t5Y6(z?_^zQTP=DGPWxX;@?M>XK`rsjJuJtwEIz^T<fy0~Jz9h;`nOn3)OLn5%=S
zG{WC%np@eT;;JJS4xMLVxBZWuJ<r^DPUhZZ(ygG<AgCAZ)X!@0Obn5Ga+9rl-^^Yb
z(jFGRKDTf7ME&BMw&z8}mWk$3q#Av>gpmN?O;i<UJl)`&Bj6gi!71RDZ4I9&`UwFY
z-r?dkfVAQ0s@|mSnwOc3-|>8TsV@$|5ru}p)=;qMXNZiOn*K@9w#k<S0eQ+h&!{<O
z+QO5F+BLbNnt>xZIB<|No%Q<qQ)lfT|7reRz%J!bL}w~(@8val`ANxl&1oG9-C>v|
zHY7)`Qp_|EF?;G`<BK6{d$hchas{bRqmkk0>w!s8n6Y&tnWWTNu&n`+m^Z`mO*wN#
z9uV2zd@K5uyshH;`290SamJr;5sb<Q<$a6neOUg^qNZo$u3Zx1hH>4E=Nbb*I#_E4
zA{dNs9zgJ<(I`MzxE0HC=Qr)5ynk<{{DtzMafP$RBl6p&s8BG%Qrh)2%|7*_NaNB`
zBfI7;{omVZQFe+S<+bXfGYC!cP%+RmFCXfjs`=J}{t=?eD<)v@2-V*yf_vjSTFf@H
z<$J*1gq!0RN!l+Emo)S69W6)%625IMVK|<hk(9M4kjWf9d5@U27ui=?>G;#%J6sQA
z^JA8!mp}S@@8MoQv>LF#w96!|47<bp6ZfRRys+vxX~Us8ISUDh2e~ooI}2hDVStqe
zG1pP!Oh3z9YS9ug_BPnl#A(|DGfISJCK+DHJ<9e=eaeQa^M~yicCLZwtzCZmydc-R
z5eZQp6Yi2;G`8zncFwvj*!yL$mFpF;xfYWsqG((J#zFe^uFRvgjvS@$EP6*CxI@WX
z*G1yymiiPN%2nn<G!hJARFU%9I|%_<c-ej(oIM9h6UzI9=hNr|(&G=GA7sb00j7Fh
znhT|an>AH$VBdYRLMW#Mz%=no?sQ2otPX|o-56q!M7^M(Ag@%mUXDDlcf7PYkgfMC
z*^GSd`a{&v+i*T(jpj`pf@=UN7*MrM|ID(O^4Gr)*?;fmH{C7cgP_38{DgH(#-|6?
zCHyd#P{{6yb#PP346-pQ$**DHZEd7&GZfb|g4Ag8_7M;m-2&aeQJ91FZW&tE!P?OL
zW1N{m>QHCeL2B_KH)}u<*4fbmO^bBr^usNXxO;H{CYjbE`o?u`S4Ce%z<6dfZOisj
zD{^;?gOK(+NJ|{W^5Iw~ig1}>wV^8rm4G*y*>9@Bws-gnpB`?!Mge<<Ya6S^#I}9T
z-U9LaIf^T5NqIaua<#wh9;?1j;NF{*FO}LV%2{x0$Hi5_h5^)62%~7}8~Wi9z`#?;
zEFX)^RWQ;;cm*Yt%=cIJNKYv8O}RGCW}t3PO5ElJ2avt2_V$gshF@)t6)s7bhd`HS
zU@um?-wtp-*mJuT$&oSyZ(y73dY{AeDVfaM7xb!AhS6(I<jP??LZTsB4_IXC8UNPZ
zJlYdCGq*=nAkz?O!B)*ZH%-`-<49pDINK*e5F~J&7O{Y>F=S6`ooM)rsSJ+x=qvGc
zKv>%{tdXxteiAZfnu!-cYNQ1TaQN=NKQ=kBT097d!a4o)-pUa|A^NaMoFoaz2!gff
z(D`@pgolR!Cra~$I$B#;PM5X7d=60jw3J6W@lg<Zlu@uZje~PP{Xhc9J{Lzvw##^$
z?V26_@t<Z!;BaSPa`}@uxlE2d_mPkSDGjJha*viEN$gVIWI)P#+FK4kkx;YGPzG~c
z?H95!74f3M8vO7CooVcZ2dj=35#crZ?Ie%(qF>(cG4f8!D!M+7M7GD#t|d#OWg=z^
z1dFe63LdL*oy<*}zD41_v)rWzvNMaePW<uk%GSfduPs^|=*)l-JZEvwePdYU<{bWI
z{+1@bmE~u-hJR};Un=|M_>ucubgWG?ymJAQaLG4WO^6p}nZTNAbX&#C`FXa<FEyx}
z6!W-;zlVq_ZtLuo<&koE!U})D=Gp=JOT<6$FM`+~_!qIte5da$Z>JiLkFVOvp4Ks=
zFD@|=zyG!LAkazV+EOK^aF#5=kFSM6G6pB^56lx3r?Do@^w0#4z5+#+5P0^(Xn%!Z
zqt(IalDzHZ4hxA}4hD*^UGh2<3v%NJT4NYrddUNA#ZZ5~(J}@F<xr-ox6`WZT9M$|
zGVpwQYjN@J(_VyZEQaID%nRn`LQ$rDnZng5qFa{d_I4rU@r@nLkLuTMHEr?dEC&0@
zj}8=DV<xsfN>35(nvhbaoDvxoHzQw{_J~UPmuR0qzH-##=~vLm3Nb+N;u>iLwVyIo
zoeIi+sU4tMTjX!3ogrjw9aF(`AM1FHx#1}f!QFbh`SpX)d}~tv4cRgdtFzpy7yj;s
zcAlv>L(W|Umw5~ynB4i0EJ>gTF1ZZj*>0lbg`xT?g&NS-IWw(_>^zp6wDtcqi~d_5
zJq}$AWmF^Ee&6vuvIvm3!&SpAIq;M#ok9A%K^Mg*GX1kdbbn5aThSN3Dbd-|^uQ><
z9hEqznb*?@r9R6=S`JN4$Wb~&pX?_S-ww*iP>=>PGBWu{YhDHC72ycH2BE89*0r_7
zf%@ElwQr=&uK0e3J0g=QmdoP=eGnLvtNOT(F<i!9#4oKN8(Gen-cq#`zDeqw6wSt`
zJ3e$r$br)Ms{@ai>)<%WY1~?hJUe#B@-Eqe5ZY6ly<Hemo?j=s7>-Wj%jaKq#S-pp
z53g5r7-~!K-scD1&5GicCOoY0DsSPX`Cc%MX(uQDMKE~XP9t}ru4=519bggrUub=U
z=5|7H3c}t9#O&LGz@O_X=<|Eww7s_sWu7Qo7e~+JjV=czOl83>jy=g67$F;znM@><
zQapXrTzp}xb8C~JN8{-|O^Ru06HVu>?@x@pOn&LxrJGw+7gIkku8n>>x=d<IkBK$I
zSm6LmF1{$F%MbP=mkzem!q-d34c<arBYbv{u`OnbWD;5oj|Nxt!lI+JUIK?=(g?t`
z8?jAPP`aCr6$#(Kt)c(8Q0n*BKS!61Jmjjqo^$%%TyU2<QhE%FjtoF(CXB<v(-MzN
zcV+?4-oj1~EkqX}?ie5Y=s%nDtu8`g0y>C!sfI0``TRMw$(5PwWCxV7>cl!gWy%WR
z7_H7#&kw%t)^{jS>cKI-;ALQ;8s`QjLN&tPp?4<0TM~TaEp*l9BbFnpNB=s>+Ehgc
zCO&OEv_gC+%w=6`wdYH19gQ82Q6<%qJEKtMIG>da+Cv-9OVvJiEU5f?I~EE4h}UB6
z%ri0%DK8B?^XDNcgLkM`N)yC?nUl5mz#dIGw){IOSyi}nz*AQI*iW8Ig)Jg4@*qA&
zS|F~h0A=2s`Qv9WLkRZcHDF}_<7@%Ee?7gy;RlC)Lolo4tjbjLp$U?Dht|7L8ki?d
zEB6XAfQ=3CV|ji4NAvae8}vU~s<yd4XWEdsXw!?3biy|!Z(iZc16@(?aOf2#U+Q(~
zk|;den4y-KSf?s)(R2UNG%@$fPN)f=T<<A;2atE(C!(xxD^2Tu_F&PNzc{#gW*U>Q
z##yM7YQ^RS!Wq&Q2s~f99{Ajh?Hj>4rDW!AdJQ(M6u&I=x%)G0)y!hu63;ei0PBR)
zZ0c#yoI6RvecHTx>eJB86YJN_?3K65zK(woH?50ZTu+OA^JmvYkyyTEo?;?KA&t7&
z*{q+(Al6P<78tx*bCWL=eMW!RqlskXLy8Pje8o-k&x3(u+l`kH3hdXh8yqWj>=rlf
zJ!R+SGu9ex0mx@ugJpT)8xL%&k|=o<Z_KS2iVFqzju|gM_|AF8Zu;h^cE2VDWg&GX
zXgfug5ZQgH8eK46bGGhkZ~5_K+r16Lp*p*ydEL=~RLxbCi~R;IZ^WgezDUElB_2|#
z*j04SLcomRM`&eX^9%Ub8S}q$|I07y+7*H4XzRoUCF|M&qqrId?T?<J*LTrJQEl+M
z7Vm~PH}IA-j+xm35VAkMA*Q(p!|6OFg@)gKJAUo4&)JxN<_~=*R?S8xlql9>zfuft
zbUC;z6SGQ*`PojZV%{U+PG&G)hLP5+f~?O>5g{F2s!;x*f<D>pCB0E;BmGNH+q6<l
zL5utCe%C&8FqAWvAYX9stol85&ylgS!V21c-#lWUtt{>+UZ<Ljm9XeAx;!Z;=_=~+
zU^0vP;Mgy<obidVLL7A$k}H1DHGpI5hDB>p6C&^+8%b7&_O-yoNzG3e0=$FxKQ|Sk
zuCRI_PryDRUWOmW5_1Jihq!H58ce)NwBeO&W+)L1xpTfjJqaH7_qDP0(DK~|^ziPL
zm5UhbF|TLY@3b&%eqke|vBLa@Ogw4P)wJw#Mn!ZjOxJn#P7j=UcY$mN<a--*KuX|q
z$bE66E2Oxu5nYy`S>UV)O_lFBziGfzsR%4P<vLja=9Bt^@|l<O$uR`kuMclDd8~51
zjdk-`i#C8oVcP$Y@&x$s^g=+&(+0It(1IC%v2aBE{$qFp5-x$a&0J^JWQ8f-r3^<q
zfHR6|UYr1Eh3!H#Bnhj?7Az>{eHs*K>CpMr1q54lUHS_LCGNawv$kHGJN5K5bduII
zgZhc7WDrK3wLSf#)3NNI1|xTUxc3}&yENfqZ&gU`Oe4q4jCtXc@mWPGc5CNsYUW^9
z#+vQ@o=eP<>bR!A8c*b6oS>#>@XY;ACr0}Kh>Ha*^6_8hA1`C<YWkk{x!dKi$mVXO
zO>9=7`&BnJGTv%-PA$f$BXOQ2SXM2L<pIBG9O-eC{e*PEtj-obHW4jT1jvUx3xYvG
zNh(=pr#KUH<_X9Zdoi-!6lGz<t#=8ZX&ORm2J&3l0B@qMSj_2nmiC9LzJ_WJxI(m=
zCRd*9xUSc!NrsGP(WT6)Qvn%0u`aIJ!{pB_S^wt8k(Vf2AN5&ZRw9Qk<wRREzG5?5
zUB{j=r{pCJ1_cuzZp?4rk`6C<7Pd=?XwLf571!1Qk44w<E|ghzb~#dEKJ57J1sQ0w
zoh(?C{=U}v^fw6Q9C%xYIc+sJtz^N=v*dv9j1X1RrbbRQ&$!Ra^f%3jclq27YV<;a
zU(wM$Dur599FTX`zP&($wGKyN#|J^(&*;PP4pr(t^k)wGS8CrbNoU(pYBF6n@7FDz
zV>lF%zKX^v)X8@wBlwzCEhQgX`S*OF-RR)w3)1`M@10WMc?|yStmJqzs*1FH(Ecgl
zCMQ2<wy8jLE{`_rGfk+jWYn|+)zPN|IgtR+pPdW3B#V~#uPnyB85=SHO~!dn-S{wp
z{po_-aD0Ev-PDfmL1`nq?lF6hlRYPi``0Z^59ID3TyETHfbtp=l@4F_X@><Kv~NK2
zp`Xj|yr>o~d7AUGPewnL0S~{gC<<zCnUg7FGfu8i2?RSl<lD!Bldz!CE9CU@55g|_
z92EVSD8>7JH*-xutzq*BXH{dqQIHjRLmafXuv{Vzs?h*_K6gv^|Jz3NFVpOQb^A9N
zHDUg@z3Y*N@N~~ToL-=JAGNrGi=u;U8BqDzK45)7t{hHn?#%X)EM87fmJ_}Qi*qqJ
z6l|I0#d4U8?aJ6~SU1}*jkefR2UnlBA~v9Llu?EJ&ZE7hak8>dY8k1&<;O0rS)a8a
z04-aD%m{vL!de5E6tDHvmSXrjJ#8*UR@phLZJ04x9Xz$yxgze7v&TI+(w*ei9wy?y
zvji<sNvCf(&$&U3%JbT8`uwiOrlJD~?{Utq@XFfeRGQM`tP;rF^e5$V;r*8tj{ckD
znj>i^%|0XrUN_AU&ArQ4g4C~{JaOq6yEiiA56e~b@0F~4!)u+oXHxo1!;1J~G-aD3
z(|so|F?phJN;Z7$IoWmaBa5p_-Mt?@2bhWtDYvF`obE;0DfuDQnmEknEc{85W#a?D
z8ODlX7u=)#MJ;19uU2H1e7+Bwc=sDT;WnPoyb&{lRwN&w!`lN)saG+Y!q+>_cbd<I
zXg5jlug>f85~Be<2XJiH(yv)09_{Cy4o<msoQJ2Xt4x*UZ>CD3W_@s<KBRtU3AuZ?
z59pb}YEw@yar{&D&UB8?AFL1<r(PD(U@k$$s+*3e<J2I%2*Bg{e~ZlLI$#TmJU|{r
zI}Qbl0Hjf0V4LCvwgIvZz<ZCbq^(689~&WROMz2b;iVg|GiZvaV)+<mOzK0=-vsG}
znl?V-;z1Leve7yq(WN6%-dlS@ZliDIEH1o^m@DzPZu7Gb!}12Qxr}@iISUAB_TM7o
zi8i@*mk0a=N{a^_QI+Yq$sU{9yK|Zy{xQ>y(JL_(ODBPkSibrG4s`3>CKu}@Be@X;
zn-t%}{N7{l{O#VS00(EOSvf!Do#Ha_2P;p8^i?D8x3*^BZ~%)wO=HgRdIz6u+k-y2
zJ9db<I1BjjOvJ;{!T^G5`w#g8stO>*VLuq3Bs4-mXoSzjlb|yXusj*q$$b*200D~t
zlwvcjGBz;Sn2p11jechdvNhQK7b;M0sr#nO`bg=dt7M?gp@aWc*-gs?m6Re*xey^v
zW5Q(HRD!B_?AJJ`5Rs?Fs-skT;DKA864Xd@koTskL)%HYQ66sraqQ|-*WrZAABwSu
zx~3N*b)E8y>{PY2qw@}(i2ZXib23)-z_gEQ-b~5b1-zjtgi98NpWKFz)+b8vlHPVh
zd8)l*p_)Uh6>a%Z>GuWWLH<>rdK~}m8m}v@7Q!P=5O&^Zf`*MKs7r}P^eNoxs;wV8
zVScch7cFRq1YZn-b8rm%qd`G!g&W>JxuclOEU!CDF50$x9*MW!fTaH_972xA3M7SS
zXa~O1a?GG;`DfU>fP&LHQ@$aOS@quO%M`P4x!Aw>G3ozD7X<q)Ve|B?c|@`TZOv7m
zspOYfwPUR};Newm*INRe*Q#=Ag1oJ{w`@TsWY~)XJg~yKRS={1-pH>gTU(a1*M5?2
z%J-vcLpyqNV;kq3Va>B!375W%r@wERe8ns@t}<y5e$CMLnj>5W&EvKVSb<}UVw4#S
zf1-7K4Jh?crR>*xcZ~ieW$Lcn@42U-ga;~f(UAV#0($&m>oSVhF$vsef$xX6JO*7R
z^x&I~&}R%4!2|r+(u(QmgjaBiuGaZbJx5C0#=_!I^hBc-aiwGDfvpqB2~FPioPo5H
zgG#b$U*G!sp3|!0>Iy~5^c^T4@@!B37wo4bQ&q)Q2rhqphN834(>S2jS3HYRPwae0
z=4Lsk&T9ChW6!~lqXvSe`Z#4_GRmXlQ_3BtTQ%>hwOvVh5&M~JmO?^?rXBHEkG5O(
z4e;Dwo}`fEKM<faII>^Xn6q6m)6@MZBUeQcOs?O|i)22zqoC>?XzJAYRScOiv);w0
zmshP;lCogEhCZv5ltxPQg862BE`jZ~lQ9$OA7ed5pLS9ZJo{M2#KPyumD%=cogb<+
zKOkXx{Q-#5pZ_`bzd#pjGOB_O!zq%;Bd!tlyYJzgExmcO0UqR-k-|88q(Z830QjNo
z<66~{Wp#UGoVTT{@Az<zQ9LFlG)JC9w%dZU-Bg_0CaN<0j7WjkqTp=M1E$Iqef+Tk
zn<Jzu$2n%?uZ@Q4HoUP_+ATyhWwG{WWI$KkKO;37(`>4EHnp(uL%~~*?)>)hN23IC
z%6c&7Kp6@wzTgL^S+P9~I79qc*?>Sf2N=x8PUx*=y%u`xfBVnOp~8>0#&p{e+4Z2@
zB70lSq#XZwy-I4|@M|s74>e~FBd4Yl_wr%F(66;p-jiq=CGApC=(kyiIv?G^YMvam
zYT`DXyMOz=c))4c#4=-9-Y&Vz@GcaKNYba=hb}m2j6mcE-u!IG48LOUTe~H+*3;@L
z`lj}zT}&;fqT4L~)K7vl$MwFJ??g*+{Vc>u6j7Vp8P&8+{8rszQ8vBc7)UJLzW_En
zL+_P0e`p$ZI+|1vGxMu5R#mTUjfiS$o;~SA#vfSNfHzrwp}lMV=dk?G-aRMNJ&%o+
zbK<GH`$2X6!%Y4nUU>;nSS7=J6&l~P)xlZ-=`vvgV6S!M+Yp^X#KsWCJNe6K(T#%j
zM@0}uA_O&&Yln;%H{wVvC+i(ZLQ2l#;x2woRFK!n34zG#WaGL3t;T9d2!A76<+srg
z<e+Ut6P8W){N}Uf4%RR2P5;=pcLGVj3$m+NfVfV!iipTFNu$GiDz1L<&z3L2Tly8I
ztr}JkmTHy{RP4H@XNcFW;JRC~Wq*xqsuh4r%!ct<6haQHRQ7or1K~MN5B1=$UF7fB
ztx)glPB+%|7||K*7Px=jxJriP0{%w)M32AlApD3gL`@eSjk9`h|Dp@S;aB{Hj^w`k
zW*q8X^_|6K!n6lCrI`^_qNJ8k*Aggv`JtoCF&VrpR`j|vEE06NGki5p1f0hI{`I=}
z3WBcH`E0x=<l1k{3ZL_dSYsnr-q2ThO*K7eQKV#mHJgI6o@l1|*cBiM56?2tdThH|
za!!cs{2nrRcHnw`3`;z=xpsrF#9FYf7qy?|eQT^2{;5h%M%tXnw$aifNX~v+_!_F_
z%P%?nfW_g}-+(din_L13=nV>>$(+;0Q^mz@NNg5fEJYDsGY+ztyBtI+JWWeo;up)S
zthNkENvfmgE#>A552UmZTKuaZupYaY`SV5YZ~QbhpgS$pHgbLLs%pU1#~ulK<wW1e
zqV}}ekNWF6q_kU6CK_iYzePO9I)&)%7{DDRL|<FnCPo71EmZ_Q@_2NQ4Bby(mSF#p
zzw{3VeEtLnXRmS4oxfPF{q<vVVk*0@c6X#TekFmd<qdZ^Vxr0B`?;huUo)d$@#~s4
zZpLh2MUiNB93`~*d6OAZu3RmP<2K5eC-t}HLBE%&twm-~U`TY^oqbTFq1_JKNr8E^
zP)^Q~gMCHSxVgAPBO&9q2KpUio@n=~TXTlS&^Nw%M)1WnnMsY&8KE`qUl*z4ExO6I
z(VA2_p_HDU47HF`4)`A|FD3tAVYxoG*|mDf?VC63)ONXC@Wl~-{w#F$#P!nzyMJ0S
zVjfv2c$3D~xBM_+q`AW@Ewk0a4i!5g9GRHJ{;KNIY8qmFCtvDWq|9`sa_$%fk%zUC
zGS4b$7iR9B`uk9OJE)Eb4Q8%Z+s$(~59J0&gx8x8+wqqNm<H~egD+v;=^>d#$fH5I
z9NwSbHa9KyHP5_g97aK2CDXdM5Pk#;8{wVyg&sQ#CxR~jn;<C_9nn8C5pJHEvj)zw
z1-Py`@zzwwxMqv!*br{6>Za7zC%c}PA_z0is?r(Sc1_$CvApL>^huPb1zsSqHApMS
zBi6~ODusUTz(N=iv-5$32AP<dZLeD(2f+dN#&F}S_Zr|_<e`X%5_P;lWF6u1yn)p~
zOY?@YEKlnU4u=Au`8b3z&wz%;)8U`;0ifBxT>xbbP|%Bj%_R8#Dce6T_CJB3diY{k
z+IfJ+DDddVRQMlXza;yRN6ykF_Vt|x<s?im*RFJx0qeqw@TJ3-e&Isy<aez#qLrOT
z-PqpOg~%*tp$zk72h?M!69<U0N8P$JMK`gJdmNtKUoqe8_;GKB_GI-^*g>|_9l&y;
zY0p@|^}OzKh8;)_5nwyb2U%GDx%rR7T~8zEZ#&JR))j=#t&CE{(L>a6am7MIem<%y
zuOzz`CGKJZA62nfwovaPlRdX8RL{pXCKtPReUM&(^i$kYEC841Cd=QR?gi|B;E=h?
zWm8vH-JX=p<pNlAihpuHgnFZB$)WO0$U1bUD!(u2_(kc`mzDAu9Pyq!w3mF7`R2_)
zi74?y>Bg@k&cdE0%OlE0Ixp-U5^*$%nc_G%<NT{ey!Zxt_v!6jX}j%VayfNG#qmk$
zz{bq>jod&~tlPlH)&;T~B`oWQJIekefL+4luwZsy%PaIx*2{gea}tLSyyT@mG8hj<
z+dA?lFwl%SEqaIKhvevZ;Zg@Q9f5Jom5fd;i1z-t4|OTFbj}mda+e#tx!s|5S$uvo
zjtpuWf_ou_xP#Ixh01t@j2EdRI7);!sP;FO<gjWM7NOvxqv#3T-oyu?&6ZW5&{D;f
zErcVMSpLmUtXN3Y&~ySE!0<Inf78acE@6D!kXb7Mmr*_%uT^V8qW4Yxg}p~sBlZu<
zEQPp|a|sfk3^{bxB|v^g4OIChm*`QhFplndWNnw@A*nHZjuu=WZceJ@6#9c@*6Mc_
z7IlvnK*(f$W+6gcjgnWG=)tc8N`jHQkD@XAe*VFInI~lebnLpSQ{+I4t$NK6%{><R
zu_}V9J~qan&V0+S&nQ_|y-FD~?#s6uC$B@px5Y6nCS|IFxs3Jce5RQ~QT7q}!J5#;
z&`qt8L&x!&yogkvLQrux34kM0<_gT<z8KDlXMg<k7;vH-e)!H}=O@_%3Wt((X9J3C
zl2Yf;XNAz>fZtfIz5UH44LLz|rw=kMIr?#Ooc%3SyVmt27siyljKLtcvJj0(Yshqs
z>=&e1iHe9uxRmQ|)h!R^^N$T|)#@l|)0sVa{!`DCJVN3@WiCUOu!6gD{7M|=so5vZ
z@KM>d^w36zOFSBCk0S7FDGJ&5*Ki5bikt(<iz(T=g`q}FQEhy;^{yOtaslMnKp2<T
z2(7n_5sQt^5C1EgkaO<ph<zxxrqx1Cx8A(bZ7kP`-odNP5*TLnM1#1ta>8_Gzo0q6
zexJjlebFXQ?s?JO*+;urS>~Lf947*!Bus`aw<x%j_rb;kddls&r2<7FQX*qZQ;aK5
zTEm4rZe$s9I1ougR*9hb6li8;k_yxK@P5Vw&zH(`{ekY;bcMzW4ho7GC|_xGQBE=7
z-MG3)Y8N24NS$=UdWcDbM0|5Z0mX!;R05p;iy44eczPQv6*O-u=GJr)^`vv(-MUO-
z{vYx52LSzXxrkulfCb8%fvpzA^NUGD@h@Z;PJ9cTYxpY}OgYDU1GG}#7<I=Emxr_W
zpaM0}0QNxGA1t2TpZ~{8*}wmRU!NXipQx)B!<D7RqqXkk(Vx~o_bSXrnzt`4Z<s#|
z;Hz?Mg1*gL+B=bYh_6_dEsRGrKI)0n?3|gVBqn{f8FT)06oX`XD?^u(_-~Sw+i7}G
z_2J@!5v!(($W|?ekz83El@w*v4f)tAHH49bMre)t4{1%8O{URuOP(_y+txUsB(;tW
z)mC_Bi~KW()(tKv=TVMhG?q~9Q6UyQ8w)iTdk%6>)YZOCcKLvM1H=_{+bu2Wru|!9
zy#)huLyjni@t2Ih+X;c*mC@KnMm@^de~L|U|D0ww87Fyzi??@-zpX!slI1a4WnjdB
z_Xi~H3S%MbymxuIR(#2;?h{n&9@K|jjVOlm-z)xBd2t54ykNOl-TfK}w4WiRMPCK9
zmk$q#xZpTxZE&Wx@`k23kla3JbG|dKI&Z`d`<zAidaoLac$;q0B_XLjRB@h>W4Bpj
zY`~l8kIl$clZJD?o%S`kQ{jqsH9~g%?uabAW1a9`JR?IFylx?(N~+mFHqPg!^>(A;
z#})~l&e5KU(SHsHwil@NRjei_jZS3(<q`CSYQiEI0a6-sU_%%9NdRZ<3g8en+38ih
z=FnD|G|>o6_%dLlK^2h31*I75a5KS39)A{b!Xkj1ySUcy{P=>t)x?KtcNfA*Ke#pZ
zSxZxgco`=XFKh#7Mn+CAYv!YzD_4)<Jt3c=NE({cHUg{x=e8)^`M9sBZk&JFDiu`^
zYM)xP_&w9GMu`aHFvkz(!g;>PvavS<8l%g?pBi(<d4+J*aRBp@d!fip&EZwywi+0D
zFg($)TGf-nhRu?dVJi!Yj-UL_l5v#w98h*Y&ok-ns-IT5T%nLgc#jMAre3PR#kbAS
zz=9B<xRW<D-!O|Uqf$3FQHz?tI=Eek1*-nFHMH=>=mKGzLkHqo)SXOJnlM4Nbx@m^
zCt~486*t%qi770Ml=n90jEnw<_7|(4)jlyXSEp9{OA%I;XV*`3mi!Tv*}ou_Rnz`5
z9qShD1TqFAE|M&-<S>+Z=$F$YH?B;tYWZUXC!iKAN*(XC(nUHKT+(@$CL5zAn4uhH
zb5$wL3S+<S(SKgr-k3R7VsbIdx@0W5e0KQ`^?@;Si+NJTT|g=!NqUK(Gf0^AHgu1E
zMz&3=1D;>#2-RR`zO(T=|NGySwU)lYyPEW61}S}SEfglAgrgdL{g<huk=jEpo1{)r
zX>aNwuZOs@ZAX2=#i7~d>HL|j8|Sn}YoSie!OZq$ib2q`C?^?*K`FyC<9CK^?->+M
z@zLGoCqNwCB6XHyCQ5$KU80m<HFrzN@{0DkKS}Mr;{kVB*%YeAYyEW>V*d0TbkQkH
zZ0y;eBd%lVzK>YSqyOiM%fGKKKW@&=)jo+b!F4(UM)N2IA(JN75H+SRrH`&PE>q-|
z(q2-mBgrT^Pk3xs-CJ`v13QC{0w5PA8{wR`K2<j3HUoyKy(RAH%H<|z&6(FNuY2yO
zJKKM(3^DYPB&f41$vb;JO9?F~F^jM=I237p?yyQJ@6N7yQ5kRQc^dk7@}p<Dsy?#W
zD7@RYh-w3$(Cy{mC_<Ja2g%P`DPQO?9XG_rV;~yzwG7Ukpx|Q2T#$!Cwd3*^;Qa|A
zA%%ILv%=oB_M(O|y6pY4JRCgE>|>s%fWy}Fk<a!9%jED;OKUmz(Ywc6JBMDF8w86@
z3$T-Pu5*`Ro+u=PzANo1r|co9XquzUR_6Fx?$sX?YWr0W$FPr<13pMx#whf=rs^kL
zO<G4ch9#)jn}D3^<^@<?C|7%5>88ICV|9&?_^4|tW5OjdrB_z^aZ>ujz_0=0RnxMo
zVh~>{rz|*bar`Ybd#g6C1^`!S*V$c!9Y{fl1&L*0*+2Z4lLLb{-7m-V(=3Odb?M%<
zr}kIUNqrgUmWsEi;n}(31_{2A-xEQeruBRJ>?ypXbNmW<Je`(7=}C1u1D<jDtw&K?
z+Dgjr1v@NOHTSg1V_zc=g$C9i?VA>7oJ1>e+rVGs;{5GHwDej6(s=P*HlyJO?^o<D
zB70PwhYJ#``ySutTyC?&X5z`mqw}QTbA3*RU&{olEGLqP-+--{>}jq|Q{JF_ebL+2
zM4ZvhOo!<%Lq8t^&6xG3H~T*v)#<MBOM^6l3F>9xnO&>=4Qfz<zap9W=%@9f^X0iA
zrK}dBRLPf58jor$;DB53bRC0N4^!3yWLJRIlT0eI*DwNu4+MEVMkmvkVIAw<87H98
zPX&nNfqjCR_Bb(eqg_bF)@PZdyqA%b<yn)uyt#A~!u~U_u?WRPQChjBsn0qkUn-u?
zKGxGUc`2W{srhyN?FAWe^6{?%%I~dGO&D2l1oZHE0O419>w)I(iiQP;y6$o{%SHbK
zd+&2HHhVlNF3a8gYm2D>iEjGkeg=kn;53F`Ju+X|U4yy1DwW5e9jh_7hrU^(uUX?4
zpaS;mS8Mx|oU-NoYt1b6hF3iUv;hnXjX)Pv9+Ca9Xf=PhU#LoIO&A!|YHo9Ee9w68
zm$~80FZN-OwhY5wu<53GoYZK0iTHb&=l=-65BLxvRbYU}2UTrvwR4H-xZysc{ubJ>
z)AiE2k6H9f-e6@7G*ihNQoEw~FfP~R{*8HeH7T2mrHTDz9NikDs_Hj})|Bd<zCD}?
zk=JDB22NzuA3<}WD0vW1vn);n#nqevq5$4z;B2Pxbniv2w}QU$`g&0niOHG@Q%@Xz
zXJ(?fyZIq+askpZu&l+D@Q<^EfZqjnpbc!->{O}RrLO1FUycRlBENdDYD@Sg-3|o8
z=(8Jiip!ALH1tn+to?6H#j=&+hp+`t+6K$7U+9niAIr@DFCWfb-v8-~>p9x3_*GSj
zWl>cryZiSFhF4=Hqt{#!>TpF<j^eKcIzs{FrreT+7{B?;Wg-=kwfyf~SBN=pybf-x
zSrtWl9352<x4=t<njEHS4rT^z#vG>0VWhB{@uC6ht2Y&s3s}?=q(L;Bado<qw?D9u
zr_9};V1sV)rSvpJ912_Lqlt3b_RJJNBfH`-%fnvwAq}tvjr|O9!rr8N`O=!3(!7wy
zvY+gSj=>~vB>N2257t@!PA5(E#dc|7wsse<`PgO4cKuss5^}1&T9gt@l2lHGkp$dw
z^vPak=Rn@x{Z-G7WB)#T9Qtr;Iy=hZ>ucSIxSg`KS56r7c|TIr8z36M;Tqhn4FAqj
zWfjkC41PgIoq3GzpN`Jf778~9OT*i~v!rsOY?glRb6<IRW<`(r&Y~hcSlf3wYijV@
z<1_Qr+rbmr?%Sri-&xYtqo7MI+J`*O-&yuE25Xr-3rpWwuAE>FIM&vMPJ2S33MtrY
zQ0*1(R|%c{tu;<vDMR;_1|BRc$#5G-7rpPPS?<om=_C^^FI6zM@zBPBYx|}2iQx7r
z&F?JCW3e+XOYKASp3%u>73@2Uj&w}0W$5<YYTDtnciGu{Ytgfc^IMVM_yI0K4{|x}
zy!G9zDt~XodoQZJ7b;2naLy<;@s?>eCLYo9g5Cj?N6wF3nUO20V<++sQrid(Qx%6T
z=zY*2d;mK!SB5h~npK^y$@ti8SM&y-1_AvL!flzEwvIYtzMHr+uY?;?i~sg<IBmUs
zt$zCcF+Vwer2X;bWh43PTz}aW<sDi(cdUlBCeKRlpU|*kXTH{y;4DtvT1vxz`WK>X
zIs_WkkIM?^MD<-$-IGrX<!qd>OrxTHDmay0is^ySS$Fuv7iC|T(>N7z_ru;UEt-&e
zm69WvS+jfdw!e7pHKQkd-&sn*W#3sIu3^vC&^AMCrx}!@!xq%kK8kSmDTe#MXo7kN
zd!x*8DdkMhS$$V*tNrxh@^=;?1NF0A*w9&~)v5Y)ZB+2zM9QwcO3HWTx|~!2&tCxX
zB<RI?OG;nUriFAkY(3W-aYfvj%N^O!e3!WE3Y$_ZT~o>w@!YDD#8V^Hl$rqtwEsG%
z3FF@GJ)S`tE<dzx+T*=^8LOlGJE_+cy)Pvr)FyVGWSX^b)HVTHqfK8Q3ZCm2QQxUw
zL!JJ;(%{rhUJN$Er1zYQ90i{D7s^(++Y9J#hnWRstm_wK^)xXET4cw!RanccIK2xk
z*QC$>WDbQd0&9Mg8Mpp69yn;_zVDTn-O&`LThl}4mkg3izHYv>lMDM$8N1Tqu!cbF
zg=Z2R8>j9@yCCN&ZEr^Ol#Q;0FTMz9j(3EaPsgpp<D8>Q7tI%1AQB0w6}z%@4EDiw
z5Q%OA6sOQuNM>Ps^?zhb{I3_ZyRuuy^+wt?4@=}U_MWCLxg~9HE9Sk6U;R|TAOf{<
zecnvse8j6|DyZJ*0`+yA(o-+DUG~Bo%e3L${N$x|M_+S5?j3ZT_t+Uj3zmzj$%ubc
zY;VM$P-_e|zZFO{0QvG}fK%gBS6bVC9_sgMJ?G`YEEj`;4_9<^(=iOTo1LXe#3CNd
z@=|tVxu~EIl%AKGZT;wi%jtzb{@MI%i#<R|jAq@C)r#^?mjZ}dh5pN11{iQeYKh~=
zWT1$eqDx+_QIOx!-IG;jfk;Xc;xoz<51Fq^YO)mvV9e#{p}*4qCK!Y)9(KBTSVTRQ
zQ+k}A6W&*tw3JGyKef?jY{oRs&2^SVHJDOdIrmRGM8HNJsp^7%u8ia}!KcZ=$qc7C
zt<J}{Vn=c6^9rMsW&Xz>wrPf0Z}@Tfy2576CC>y+q)e`qnawNwqYLK!&;RRw|3CP~
zx#!0VYoVMIL$20O7U#+vZn+CZAKU1+KEr1qeE5`YP#2Y2cc<|NN>aK+t7ajw9sH#1
zrt8-qas2${h{)q>?gq<z<%9t)Yopht%N8l>G1>9WFek6qYnRzZ3YqTqSe4dZVO-#d
z5L6a0AElqvSBCaav}7w6w)A&>XF2zN(EHTrE{ZL=!G5I!1_UeZJ>AUgI}uq$_l-r#
zwv^ohC_ym;^I?y)3-s!5YHtEmDrq;xuddnGFCTYBxEcg|FtTg4Neo#W@r$-tYE^jm
zIQ>uEBuQ}1uC3e$pgdGP7$Nf7%klk-?<_yrgr=E-Mw7hX3>nVsrJ#?q90F1Wf|sNc
z*Ais}U@0NW8F%ecMkihLZ@9Hw8C{;w+N5?H`W2r-+n1MA^z9xj6MkuJ(Q91WP<nu-
zQ`S?)2be~QylGzqi0ln}-m&H>2*mB&{a@air{q287Kz<iVhw9O=_uni`OY%>{7XU)
zYQ|9yXOX_cd#SQsNK)VEk=4TKdWHRWmhJQjnbYzm1prQH6t{nzYS>JTrOcwOt=8QI
zqSN81_@c?F%Eu-|apH_;xs+uYA_HFsuV$0o?ot7w{F$iKeGmn#xANcx;M)#MVStqd
z0|xN?hnfK(*+T4gW!XCvxEoVS$@x)`enGLrmW^j%4h;6y&;+lVKDwDdpmsg@qQQ=v
zTR6$bV*IL+Q(S$2uI6#P;TEIVyZ7u0Ii%CU|0nIsiPZb2q`L6diMXlaL6^L|oYr9a
z;A+~LnR51bmhuSz!Sgj-i<cn@vVr^NqO2&GbUdMqFO>q7?bbjC7(-(ZZ#jx>fABT#
zsS`=QS<vN3etMtWg<${lSuC5monXyHhg|O$GYMgNMd97rWvZ~`+y~7+cLz_dcpH6Z
z5$Esw&NAi}6TGR>Vi{joF^w9_fnFf4$XFarOWsV|0eP+!`475^dKZ{T2oqd@vKIQ`
z6Hwk|&=-BIfd4y#_zxyy4%4}JU58#P=sze3k>6qn7rMB(OVp$H$*^QnhL||;#s|PS
zV#)RMm-$f=#2%#H1CsWT<o-+6md_C+T$F-k>#VrR+wE3L1c>U$6$cd0*2>t{XsL@S
zvHH&9VcofMYS{I4Ay}tU3Lk3U<4{mPKU18f>0MBHR&j0;=bAr-h^*i~pGwiW9pwaq
z%{|$1Jhw*X9vfuOI^dIW0eiNqN;DRfQU^-QGbO-PJX*)kI?IDMTE_b1bzE#>Y!^Vb
zC667)W?g!9P7aJM5|re1CWjhlHF2G2#X{r}@FH79G3Pi;XGuuQ#`v)O0KJt@Ov;@k
zQ=R_imUq3eLKQy`IC47>I*M;f4iI1%c)Nl4NbwOt7yfVzmSt-002z*DQRCYCKUhls
z8^6MD<T<%}mh!b=wW_8ma_}S3I8c8?S?X1>Zao>rh}<$lk0*<3nq-+;5Cb#A+qLC)
zwp9B({5ZcXEf|(yiQc+PqL!feXmFA3BJc<Z<Y7bHuUH5%o9VN}H4o33c9%dm<M!Mk
z&J&7X5|SzYXr1KZtHP;3A|HMUQi?HO1g{b#b-J-$XpN-!nq%#fwO0!4R(76DvZStr
z-I1HaeN5vVq}+G&1CqvCJcz-Y$E6sQRpMbKf*D)aRaXKx&dgGrj+b`QUIcE{?rC=&
zB?o?)PQ$1{dp1wfCZdZ^W%PH0;JNBq%qzzS8iA|FPYbF87RKFU1_S6uma}Sm{`;ru
zG23SghdtC&qf)v|zrVUeK+D%dM6i$bwZ8&AzC7s8@{#SIPykn<I1M&+%Ut!})P_dv
zq`2kqSl8nyCUdRA(3%0Rw(LXin$Gnok}l|NuuGqR_-K(nvpq2`wARxHJoJLL@rLuy
zF|YDRB^}>lEUChN+)i}=mNz%Mj$_Pe4(5zhKaRY7$6{;sx7{T|F^mtASW#5*Z;&>7
zG6H@5sb93GzN%#(G)`VzXKoqorD~o>rB3LyrlC$`mp4xGo2nlcq?}9=YEgIds1lY}
z^0-rL5~H#cgmOO=q4>`^OUl~!?h!2b(NK-yg>*e#_^`anm!fcm$&Fb`DFBw6iaQR5
z9ZzlFsvz%cF9;v@SbjMP7TGHro_gUI6go=}PB>d&8idEtZoO(bny~qSayr0D%iEj|
z^gMhQkG8Iu%68|CT@HCeXt5~*5-O#}-`v()(`T!c;BaLG+^=&i-BR5$I8Uv)ivS)R
zv_dl*OQ9m~y(VR_P0_BGSL(&vJDjG|JOvg!6iqYBwK?VEE&yoPj!xcoi2FD?fgqQr
zX7u0tTBdnn;p5Pj3*F}DSUfYuyJXsh)I5Z2<B{m^4!_kD#}V*i&x<KyKm{0iJG(*c
zhdpI#=!%hAr=uzLowE`(quC(K<IDv$ME{}t)7{C6(CmW~eFE?pau00pKVt^(rLC9O
z?ng6gxA#XtYQv|&Ex^j7hF}8NheP;DqDcOi0}oH2EP)%7d_Tu`B9&JHojsdo-vMd1
zgdV*sd)qK8(rWv%{O`fv$ZA6ThI+|@kbpa3A@%B2lVdZdZlr0>GY7tt+T-e{9hOF@
z!1B-86;=~h=umU>Ual_0D;HN6y|Y_$aVN+kbZrz{HHa;6!OQz?wTiFVgh7^p0RjX>
zl_eLfpKu{<OV2V1??7AY<$D$hds|-ifo*Z4B^?ey5nI&On4X0ocYD20`=|RQhY+Ur
zhEum+z`V@*Lg&$x%)ZfTgpFNb((+mK?5Vot;Gz4M)BL^Y_<ik8_8Bq1z*6m_3FE)2
zgPT^418R>cekZ_|z~*Rf0N!)_rrrSOsBX60VnRI_3s1L0A4L6y9FkMWM6Kmqx>6ja
zaL%njAwU#1AkwL+7w{&19z-w_ic23)p>%;GaDYW@Nd~LW4@u@}_{p(pX*?=IiMTx2
zs_)p8pC%|*`rA{<{$~Xd562JLqk|iLLMa{NS7^oUiSAIli5wSwQnn?Jk*Sg4gcV%=
z$ny1zMZ48e3k$fj7->;_)W3`8(R=ZpdF|ZqEWh^9zq5>-BMZpvq`eGUKm-kcXSuj7
zfIS<d>|>&W0iZ4-aCv+4R4j95IY>Y!rSFoG>{K#8fBG*e+8O>)wYktLl|7Hjn1UHO
z*D}F!#Z_3`K1I|>7-*VkReWmGkS3hVM!e;`i+!P|4NW@Et1q$2pj$Q}@3wsfGdT_S
zjA3XTFh08Bvu||;0L_~h099img%X<;Ni}<+j6_GKj_d;rYuqtJTM8xHX5vI!$Bei4
z4r?8DIZ2EvD6VX%b~>rvjlRXcPj)P>)V@-o<l{kh@Upg$=|0hdWv~8B%b_7r1V$aW
zqv>4H5M^F@f-GEqYd3MAj4WJZ=C{hC)9n1;So#0Azfy1QyHb<#MOCi3zodJNGe=iq
zWpwvCmBzQH%SCEujMnJ$`woSUf0{1<XUuhj(Pf40hpY4VVfAzJtX+9ab{_{^X5B}L
zT?j!6L`+|Bdmlt%Y8^B;p(SQ9=4s)r5bUS!iut3Bfp>*NL4FBuDU!pdrZ*O<dYf$p
ziIcD8Kw4$x(l6s&+rAFfXlL3(C11=%Rq$F?Xnz|X1&E06LajW7{Dl(EafJ~kh-rN;
z)&?%mY`k@o2jJ#yzctEmo{89P9;bheys$c0aSf84m<XqYl)sQAennYgjp2pl!{IxU
zR876=@VW6?A)ehp|Gm*<Bbfg}t=G_tB*cWAQK0WE@so8>0iojZA36InIF~@EqUExL
z?_ne%vtTqPzJ1sIolDomdry(m-<QjtmBTZ$`yW^UVLWL(dC|I-FYE*^(PqrUjPow{
z8*VArdddr@VEFPXy`o#j+o!VMwG0s$CvWrano_R&>8d8lm2(N&1p^zj!CVzVOgm6?
zCMG!USfRLvk-xHQeNKr;8}h_-4roxN>tZO~PiF9@1F!?7XxJvG=aet6=3Qj{?RTFd
z%|4v$eMRY;=>E}XHrKNtW$t4YU!XY&sB!+8_EdiWmw!(%Lw)>J-T*!f1^s;1jO@9Y
zD&#!x^(>yLssgc#!<x>bdkCZRC75BT?Uwg^Onza(l0;k3?w#=STviSdT~QBbUxgI<
zv<>btuLZe<zbYO=!fq7zpDp+Yls{DoU$Xuf)?_}xA)nhe1d)mU^~`*zv==k-pl>+N
zVgbKnpc0+vY>%$db-=g(U+tY~I9prW$Mv+QeLK*qsirzbX~l3*+8Cd6OjSjd#2h6-
z8$`t{hST<FD=Lbjrf5x(L`0C9ic?BNLTZ*8dK57QHP#e;_j6tE_j5hZ=XZbDANIP|
z{;-F&*1hig|NHMMW5!99n8|<JF_Y%hNL00$9#@n~18c>yhO=ZeHiY7>T6<R&XZAUi
z^(wx_Jt@3-WdNa9%*W28AR@IenF%NZLmPN4EcfGs3U^U@NVw}MW`9ato(RtdIy)_>
z?s;ad+akPVdHpH9{0Wxnwd+?BqP2j}-{7|h3rNYk)LDT-4zr=Fvw^jVRHM#r9%dAe
z<_rK%>#*4T_wl?#>%-u!?6U6rZlCM!*kZ2!WdHPo&Q;!Ha~>$$7rlSk)vdYklroxQ
zD3#T$gs`_jR9Xx%X2aJq@!#63Z=}f9Nk>0yqP)*SyX4y;U$sDOZY{`p+|eu>ys+R#
zyzEiSX*LsnUf-MZfE6)mS&=rzXxl!7x<U?%y0#LGc6YV1O$RupX<Y5>32#a1>>Rgz
z`j@=Vg|LJe47#;Y)!F!E*Ss$x1!4hDi)(ow|KPxJw7UTFX3dRZ@50!>d><`AZXOK?
z>5vO1_~>tlYhYxsnJUnC(r@j1HnrT{(ye6kcB{WRPTQl6)K>9lJyps$vtjygd!bJv
zYCFaY#x}PsPYJTDUa<h>)w-1?p~FHs$H>x?xlBY#5U;HkNAShIx6cz$J*UH|jyYS)
z12kdMJ4G+)UuZB(pg=EjW@iQt8xoX1V~QuK2d1@J0Z6@%g4|7vD$#qbSs5s4$!Dml
zF}?9pWRLu!Vu1Un2e_ZonOuiEf+4t$3x5TqklpF}<WdrQdJ#d}o^r~f^CK*4BS@}E
z6JyGf0k(1#u=$<I<at8elE}q(?KV=wa`F)t_Q+~JYRR%G!RpDEzn={^Dbgk4)G&~U
zRzoY+pYGOKi4?>`^LCMdG09LTwgYD=+P&VRS3in>w`+PX4)d#xEKocK>#$b=<>Et(
z<>gWibhLXQW(Hvc#z~Kd9dD0aB$lkwrT^@;ZZm%P{jO8?iJIYndh@zROlAG-XPsOU
zC3ge-scYNtG)kt--HKadP=;3NNN<}+FX>H*eN&?#oK9Ixf0(4Fo@jlk=~l#{G>Icl
z*^aIfRE>ZtuQeO7$`&|q<!AEc=dVYj>n_X1g;!TtKIB<tKfdl3FxH>+^4Vo^+rWob
zoaxZ@s;Ga_ry@W8w4#zofXmYN=`ZVun>{bk)-VlFg{zJSXJJ(J31KuXtK{~ua7FB`
z?ySpH*K)Q`eF2KI|FL*@k+vXPL;AW0*hf;6`qqRss!Q(=3^!O{-&>?5l`o(ikR<ik
zo)hW;eb)zxCar~RWCI(@pnjnx_e=#lcTMX^z-S!k#DnvI5-MXcpO8Gd;;M{_MeDIK
zx$isyZxpA!jsAQUey4@Io>)oSa@0<y(G|YS&XjwSSUiOsAx%aQiw-H67m3}~<{Jk3
zR4<$IH}PUxS}>niCJ0H-Hp5G{*x`o$SjnOA(7O<G-u>}(5{Rpt74}W<D`|TbuD)E^
z+L#r2!uHelxStP9R<mN@o52;8tbT%Ue5m-YcMyXvx_c^}5$>`;o>B4G7E;mm{`)_k
z4F93S@xn;1CR+3J_u#5A!)EJ{5MRNu$f*x4{c&#=V&h*DM3u3v&nCbJ*m=n>tgj&r
zeBT32zs%Sy&{ZhXd44RVdV~&*P)B!w<gi#Aa$oHs4=l=uHdD~QuX>uu8^+7}QPbwf
zMtlZCUGKjw6hqxtWFxiR3q&`J`<ch0V~s71d#&$5y|6d>!2v7)!z+1YQdl~VH9;)s
z_)YWoD@dIb>_foW{1kTvqly*(UW&6QEc4Qk+ZlTP{W{AToLhGArbX^M$P1}{eExo5
z(1g3?)1Thuu@?9N^8Q-MVbZc$0W%ACf^{ZKAD}_<gXLV+vYqqnK8lr%UOZ@O;nR-3
zEm>(eh;-bFJa4Tvy6%p=><;F&EvVF!6&2MlU?=Z3T9QXI!0=3B+tp+YamjqbZxLs&
zWEa^4{mZ~ZlXf<J4@EeJ&G@o5>!93aH<SsH<5fJ6p*hHvqvp^}`m*ovkC1{ReFTIw
zdqo4}s{uvpv$O$CJhvjB<D+9a%Jy(JJA(`L54@7UOP9Er@~dn|*89|xW@oD}7-~dv
zSE%KY$3LE()$r$=GebuG8_4Zb9ni1{U!V06yFFw%lWZ*fcf8FHp;O}9NB!1Z150ma
zJMnzdCzd>go$a$4yYgN)$mOHwLD3HPw5b;6ZZ7*`MG2<pa-n;)+Kf~tYheJWAKmdu
z=ob{$q>cgcQrJd~-#GD@^{#=%+|KokikuHq0Nq;N^LVx>k%w3x;on>tCkIr76(gxs
zVYgK$i)kEv`Ag<OWW0xPaOB;nfeafKX!z;Gav30%t0b*zf;V1{4~_W4A)@6AV+BMx
zY;Oq~>H})brQ8#;JB}COKF{ba(u=1xxF48mnmz+~#W%oae|#XV8764DwK8xbu_L%8
z_;lxr2!Sf*Kx4*f>Ji%%HFhpF0n85SNnY=vSY7aAg+_oeUQAgH0OEjn`Hg6B30@#C
zDY$CX-@u7Yn`igh-|7#=Z(3lo<)0G<R~AMFpU;I17Hb6#W|NwA{1|6-gA1;{6+W{s
zYE;2DHUicZ5uTHirOCOONs^f(8|sy9qu7re#D(D^AF=v*_rxBl2c3ZsAKW<P%jLi_
z{fON`ws&RcA~(=t6ylIE)O9!AMHy=chlG0YsHrCcka`2PDr72pV8%nzmePs5pfuZE
zzgr5=i@4Djx9`$0Ccn3Amu)j^;{Y3_*s{~6&$ip;t$9^_W<3af@%T*UBCh-X_;?z9
zePP)f2cK=5`5Gx7l;{s3yiCFQ*G}`fm9q<Jx8-inwT(#?P>EXsT1YUr60y-pzYQv}
z(mz>U-{@NBKU0aDs{Gnwc2vB&=2d9X&CKq32&G+Kjjz+{Sa(`B(EUt!j66C@=kw-}
zs`a*9>FH05@P{H6$k@ymM*U4^<2RSB*n8D3R0S=6T@c~HXWkRaSGtD1l$YqEc8IUr
zH8NdPlw!}g#+mNM=(gqr+ACJ1cLTeIE6?-(_^ct8j<PGOq$!7dou(SDp)HhrQ7}V#
z$z#w%kc7kN_YVf6kHej|6J5N-JY~on*6I_56M57+%m<Sx6ih!zUh+{LX>&_mk3jeg
zK4@Q@scq&SJf-r!>E-xM<kqRqNBsUVtc)O@8C?}|?@0R7@xT6I?E8<Kk9^=aiD)_g
z<)`m4k>%9keZ{qF5^tqPK}ZxklN1{qTF`3BAw{I<o7;7-X6A&gS7Y#&bHTE~+;v-p
z?y(M!=J{s+2E)wvE*rfQU#q#{pC`W~-=?_#%}oaISa<6;P;f?KA?F1>rUYGS?BF<Q
zc(hQ#5~FRoWe&MHv{uAqg!_O+9ip7-!|)e{e<NKx6zNGMduhF9zzJ2)vGYE4$SVkp
z14I;-5>pIb9wc&nB=%T?;{@lJq^cF}^;U*bP@m6Y1H|1-)U!8L;jVrI#O+0*=qi$&
zsf-z5I-Kixqkuot>7`W2Le%77?s|6nuDW*3Xmr14`b;njjn<p~F1QOw!R%xPrzqw~
zMU}s4^?OnIaq2LW<Iv(%^3MjD({|DlF>CLxye>4!(gUNvqt}7;G3k#Zyn=xwws|$w
ziY@_FEzT4>$lQh;USvQF_lf{BGR2~O0Gxs?q&dVBWHlA@an6d?G64K3FHLMmDR&b9
zB|S$#2T+GfH>U~-^q(;;J@UkgzNT9$E|&V;^bNq|w-jet>}$o6qbZYCfhk<J9~be=
z^lIx&^*FHICtKqjr*CRll5Lze@H+oEtzV|@Qt<Kph+jpSj#9ZFX5TVquG%V(jzHhW
zJhcv?JktK;N<p(Km@ttNk-qKA_B$+vt!ZFrObv9&4IzQub|s7l1|zQ?xD6t;Hr3S5
z(+kfToVwtptPnVsS>AlkDmBm!({hsMLK6?Ni}R=i7?}>0FcwAq#Ej7z<yH4!X_jaC
zz>S4cco0uV=4-$=$0tN1?s>e_>9l4)XbsUP@g*(YSTq$c3HA*p2%oAx(LFOg7&#?a
zwh!Jw=NBDrfiwQiD*_ITqT<8ng)~u<XBp@8U~dZ{NV&@RkJd-m^`X$eM#4+ZX}@^-
z-Z=I;#pm&j(Yeh!N*8b<|0%HBc`!O|`g9_BEG`4!jj^3RMIDD%@cb$>m3w@bKi9nK
z_01yL7(o2Yv|^TDt3_{6ZmnCm`sa0T$O%cVfzxe9r99(T_>u-@G|zACfuSaoRE?w?
z*sM^T|C<i<w_(I6{x(uiG*FkxK|V~@_66_|b*NRD#M_}|zT9mO_3$r8zsRxE&=Nm{
z<br*QM|szoleSJy=CISo!zpV}syH>Q+Txf4;37&5OGN|)OwKg`X^3&`Y-lE=n?!m~
ztSkJ<?YklI-S_LI)je_8Tp58uePHtBoN>sm%d#u#>io>gcp!ZyOvFmdHXy`#0sc!W
z85Xa0txW4Ac`au=<{e0;s5DPnEfA?kL_Jw*tIJ~`+$(aDKrxP_DCIn2Z>GTH+mU>p
z#vlLjF!2u^j#dWk^_qqbZG0|+B&nF1CfB7@<Ls3K0?jFi79&D>yVjq8{9S@=?gghk
zp`s|5p}_uz)9OxuI_({~yTE}1`-plPg+OkHUQSwp_r(FJ0D3f!Y$QqUnh%8Lg8$A;
z?%yd(Tu#4wfjA+@pjRRZMMYO#ddoKV!To|U<5K@6=Fj8CTpz5JQnO*L$DBdt{6&DB
z^OSw@tQAcHA0LFIrmFs~t}&hv<i_#(RU@=eQYxauO!1z+emgH;4b;-pP0KgwHSk~S
zE~*&HZ>caMf#iP{+*&Z%@4wBj7oIy3EarW2lMwk~zz0p!c22(VK0}6HX+AU6J^E$I
ztXt=!gEdRc3U<lJa4{&4GtuByP?7aVE6V!U=pA@$Uq9BLT=bC#>m)h0x=Jx|u4yk3
za36cuCDmGIYOVa+Nja72uOKL%Elv-*#rJa*QyP=yx2Pz2z3ikfCi|6iCo6Gf8r_1?
z9fmPR#4^S&M`cfz{NFn{LYHriC~etly5xLG&&FW*i5xpzP$r1{nC%^q=1ZY|P}Adr
z?2(}J?P4<l;Ang}=C0Iuqm9*~XLvq=a1DQv7oZ(JKS~-%@<^+BWyPiItO?(GSp;y$
z*=f#+_|0^^vr>1E>Y7;{T=v|$-rDO$inF!xvtr1~6r}2dOz7q0^C1HgQ7*eBk$&1v
zOyg&psLAEQ*L#oB$%+APO8soHu8x~1{&{V8yILf#A-&WOJglzg$Y6Acc5T|J94s`7
zBvynCKsz7o;f!DN^)<ErXEb@vIH#bE6EV~Md`U{ZHAgk)a0UHsFhFLQfWDqrCHtGZ
z@Ah6+PS~gMp7Da+IGdM`<Y%}nF7C>NgfGTjiP%Ryb8QK{3@;6|ub%%JyOPpaH$iF7
zLx;!ZkHOR5>AbgPOd)R(d1k4Js~AL%Or?(tT|M2UfD)RnmsUkz4YWo7actc3mw))|
z|C{C;!g)@^bw;)?Bu5^1@NZyjp`28f<I~wlpwjqeC!3+}sEdc=C2;o(031f8xzpT-
z-+SCf0($PwMrjPz$WGCqK$HxNLRbW**`A@qqC2-q^tI;7fHVjYu0S3~lWiri0Qe#b
z#=V3;sUY5Wvtu^hLLCVp`~fP&G8#<EFDgo#YKb*4@Ws=NdldXR_tV*xG4P7tF7_oR
zc9Y6Ke;cExvMgi-5-M!BwI4#F<%5<o)30`dw){>a_GKd$RxFz!ajG+_qigyIWSr5P
zCr7iNUMD3aZ=@o%>WGX6-8Y>A>#4KedY5xFO)cY+kp2zpt#-aTH*yy%SKGB|F7awj
z4$)3Q$X^NZQm@R-bE;Y{MS7h+4Z40ej&Ll<lGc!WZCy-J^(3p3c{;e8l+;!+T;i2V
zTK{djMP3kV;bhZnf>!*7Lv>_KCB#Gr^hBQC|D3#kXH7sAX-Sxi15oR=u385|`TuhL
zA=KpsH#KtxQVBqOQVi6=h-jM5+BABV5`b%Arr$k&S>1=p$NmuN-Ae!vq}5(KLWd*I
zyi7M*3V|fGdiJ4^-7h8ym*QRsguM^%Fb;8tU@yX1;0~sxgoGWlA3}oub8mLSw|M?L
z%vTE?0NTCNzREA3vjH7e2a2kyB6~1X#3p;)_^mBfXos_fMZyH%&)rlv<G`<OS!L<A
z54BNgWN><sMqSMK>8zQ;->KeLZ!-hEw~j+9(BOMrNE?3QL4w87i+uxBVc~5GWOEM3
zu=|qKGg2d)BWWe}Td&=Dx7TjCw8G)wccopsz;-*%p76FTPMhyv6lX#^ZV$S(dPf&B
z$@QtBSDg2Gli4!Aw{wct@w=UbXwU83nIW8c7z&<|fmXy`QN;a^BJP4W9FD#4dh{z+
zUr{fOAuCz|h~zB+Z0f(c(xT_Kk1E#2Kkmytda=Gox{PKJkvBxv)uu~}592>NA+jqC
zfi=4umJ(Q}igyZbOr55kMQXSb^@3`fiMp<5I<mHuQBB?)?D3|=P_>a@+Zb8{;9Ex#
czYSTT%+(7?g#`aM|J48Y`R6Ba;s^JC0hF&AZU6uP

diff --git a/examples/config-files/dpkg/config.yaml b/examples/config-files/dpkg/config.yaml
new file mode 100644
index 00000000..44408e3c
--- /dev/null
+++ b/examples/config-files/dpkg/config.yaml
@@ -0,0 +1,14 @@
+# this is the default configuration for the Debian package. Modify to use your own settings.
+system:
+  mode: prod
+  rs_window: 300
+
+auth:
+  plugin: pwd
+  # replace with your root password of choice
+  root_pass: rootpass
+
+endpoints:
+  insecure:
+    host: 127.0.0.1
+    port: 2003
diff --git a/harness/src/linuxpkg.rs b/harness/src/linuxpkg.rs
index 755afb41..1f8dbd44 100644
--- a/harness/src/linuxpkg.rs
+++ b/harness/src/linuxpkg.rs
@@ -76,6 +76,18 @@ pub fn create_linuxpkg(package_type: LinuxPackageType) -> HarnessResult<()> {
         LinuxPackageType::Deb => {
             // install cargo-deb
             util::handle_child("install cargo-deb", cmd!("cargo", "install", "cargo-deb"))?;
+            // make files executable
+            util::handle_child(
+                "make maintainer scripts executable",
+                cmd!(
+                    "chmod",
+                    "+x",
+                    "pkg/debian/postinst",
+                    "pkg/debian/preinst",
+                    "pkg/debian/postrm",
+                    "pkg/debian/prerm"
+                ),
+            )?;
             // assemble the command
             let mut build_args = vec!["cargo".into(), "deb".to_owned()];
             if let Some(t) = util::get_var(util::VAR_TARGET) {
diff --git a/pkg/common/skyd.service b/pkg/common/skyd.service
index 3990c09f..8182822d 100644
--- a/pkg/common/skyd.service
+++ b/pkg/common/skyd.service
@@ -8,7 +8,7 @@ Type=simple
 Restart=always
 RestartSec=1
 User=skytable
-ExecStart=/usr/bin/skyd --noart
+ExecStart=/usr/bin/skyd --config=/var/lib/skytable/config.yaml
 WorkingDirectory=/var/lib/skytable
 
 [Install]
diff --git a/pkg/debian/postinst b/pkg/debian/postinst
old mode 100644
new mode 100755
index 15592134..1bf534ea
--- a/pkg/debian/postinst
+++ b/pkg/debian/postinst
@@ -1,15 +1,35 @@
 #!/bin/sh -e
 
+SKY_DIR=/var/lib/skytable
 systemctl daemon-reload
 
-if [ $1 = "install" ]; then
+echo "Doing '$1'"
+if [ "$1" = "configure" ]; then
+    # Enable and start skyd on fresh install
     systemctl enable skyd
-fi
+    systemctl start skyd
+    echo "Generating password and configuration"
 
-if [ $1 = "upgrade" ]; then
+    if [ -f /var/lib/skytable/config.yaml ]; then
+        echo "Configuration already exists. Not updating configuration."
+    else
+        mv /var/lib/skytable/config.yaml.tmp /var/lib/skytable/config.yaml
+        # Generate and set password
+        if [ ! -f "$SKY_DIR/config.yaml" ]; then
+            echo "Error: The file $SKY_DIR/config.yaml does not exist."
+            exit 1  # Exit with an error code
+        fi
+        PASSWORD=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 16 ; echo '')
+        sed -i "s/rootpass/$PASSWORD/g" "$SKY_DIR/config.yaml"
+        echo "Your root password is: '$PASSWORD'. You can change this using the config file in $SKY_DIR/config.yaml"
+    fi
+elif [ "$1" = "upgrade" ]; then
+    # On upgrade, just restart skyd
+    echo "Not changing configuration. This is an upgrade."
     systemctl stop skyd
+    systemctl start skyd
 fi
 
-systemctl start skyd
+echo "Done executing post install scripts."
 
-#DEBHELPER#
\ No newline at end of file
+#DEBHELPER#
diff --git a/pkg/debian/postrm b/pkg/debian/postrm
new file mode 100755
index 00000000..cad6a545
--- /dev/null
+++ b/pkg/debian/postrm
@@ -0,0 +1,27 @@
+#!/bin/sh -e
+
+SKY_DIR=/var/lib/skytable
+SERVICE_NAME=skyd
+
+# Only perform cleanup on package removal, not on upgrade
+case "$1" in
+    remove)
+        echo "Removing the skytable user..."
+        # Remove the user and group, if they exist
+        # This will not remove the /var/lib/skytable directory
+        if getent passwd skytable > /dev/null; then
+            deluser --system skytable
+        fi
+        if getent group skytable > /dev/null; then
+            delgroup skytable
+        fi
+        echo "Removing the configuration file ..."
+        rm /var/lib/skytable/config.yaml
+        echo "Cleanup complete."
+        ;;
+    *)
+        # No action required for other cases (upgrade, failed-upgrade, etc.)
+        ;;
+esac
+
+#DEBHELPER#
diff --git a/pkg/debian/preinst b/pkg/debian/preinst
index 158b1666..106070c2 100755
--- a/pkg/debian/preinst
+++ b/pkg/debian/preinst
@@ -2,19 +2,22 @@
 
 SKY_DIR=/var/lib/skytable
 
-# create the data directory
-if [ ! -e $SKY_DIR ]; then
-    mkdir $SKY_DIR
-elif [ ! -d $SKY_DIR ]; then
-    echo "ERROR: /var/lib/skytable exists but it is not a directory" 1>&2
-    return 1
+# Create the data directory if it doesn't exist
+if [ ! -e "$SKY_DIR" ]; then
+    mkdir -p "$SKY_DIR"
+    echo "Created directory $SKY_DIR"
+elif [ ! -d "$SKY_DIR" ]; then
+    echo "ERROR: $SKY_DIR exists but it is not a directory" 1>&2
+    exit 1
 fi
 
-if [ $1 = "install" ]; then
-    # add the `skytable` user
-    adduser --system --group skytable
-    # change ownership
-    chown skytable:skytable /var/lib/skytable
+# On initial install, add the `skytable` user
+if [ "$1" = "install" ]; then
+    echo "Creating user 'skytable'"
+    if ! getent passwd skytable > /dev/null; then
+        adduser --system --group --no-create-home skytable
+    fi
+    chown -R skytable:skytable "$SKY_DIR"
+    echo "Created user 'skytable'"
 fi
-
-#DEBHELPER#
\ No newline at end of file
+#DEBHELPER#
diff --git a/pkg/debian/prerm b/pkg/debian/prerm
new file mode 100755
index 00000000..8fe8f64d
--- /dev/null
+++ b/pkg/debian/prerm
@@ -0,0 +1,6 @@
+#!/bin/sh -e
+
+echo "Stopping processes"
+systemctl stop skyd
+systemctl disable skyd
+echo "Stopped processes"
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 5462c0f3..8e6ed0f3 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -3,6 +3,8 @@ authors = ["Sayan Nandan <ohsayan@outlook.com>"]
 edition = "2021"
 name = "skyd"
 version = "0.8.0"
+description = "Skytable is a modern NoSQL database powered by BlueQL that aims to deliver performance, scalability and flexibility with data"
+license = "AGPL-3.0"
 
 [dependencies]
 # internal deps
@@ -61,19 +63,24 @@ priority = "optional"
 assets = [
   [
     "target/release/skyd",
-    "usr/bin/",
+    "/usr/bin/skyd",
     "755",
   ],
   [
     "target/release/skysh",
-    "usr/bin/",
+    "/usr/bin/skysh",
     "755",
   ],
   [
     "target/release/sky-bench",
-    "usr/bin/",
+    "/usr/bin/sky-bench",
     "755",
   ],
+  [
+    "../examples/config-files/dpkg/config.yaml",
+    "/var/lib/skytable/config.yaml.tmp",
+    "644"
+  ],
   [
     "../pkg/common/skyd.service",
     "/etc/systemd/system/skyd.service",
diff --git a/server/src/engine/idx/mtchm/mod.rs b/server/src/engine/idx/mtchm/mod.rs
index e8c03ef9..68023225 100644
--- a/server/src/engine/idx/mtchm/mod.rs
+++ b/server/src/engine/idx/mtchm/mod.rs
@@ -57,7 +57,7 @@ use {
 
 /*
     HACK(@ohsayan): Until https://github.com/rust-lang/rust/issues/76560 is stabilized which is likely to take a while,
-    we need to settle for trait objects
+    we need to settle for trait objects.
 */
 
 #[cfg(debug_assertions)]

From 179fb09544143429cbbd93f3575531119864f3ff Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 6 Dec 2023 00:18:21 +0530
Subject: [PATCH 308/310] DDL: return bool only if EX/NX used

---
 server/src/engine/core/dml/mod.rs         |  2 +-
 server/src/engine/core/exec.rs            | 30 ++++++++++++++++++++---
 server/src/engine/core/model/mod.rs       | 20 ++++++++++-----
 server/src/engine/core/space.rs           | 28 +++++++++++++++------
 server/src/engine/idx/mtchm/iter.rs       |  1 -
 server/src/engine/ql/tests/lexer_tests.rs | 15 +++---------
 server/src/engine/tests/client/mod.rs     |  1 -
 7 files changed, 64 insertions(+), 33 deletions(-)

diff --git a/server/src/engine/core/dml/mod.rs b/server/src/engine/core/dml/mod.rs
index 9f82f3e5..6463bbe0 100644
--- a/server/src/engine/core/dml/mod.rs
+++ b/server/src/engine/core/dml/mod.rs
@@ -43,7 +43,7 @@ use crate::{
 pub use {
     del::delete,
     ins::insert,
-    sel::{select_custom, select_all},
+    sel::{select_all, select_custom},
     upd::{collect_trace_path as update_flow_trace, update},
 };
 pub use {
diff --git a/server/src/engine/core/exec.rs b/server/src/engine/core/exec.rs
index 7e7d6ac3..d32c8f62 100644
--- a/server/src/engine/core/exec.rs
+++ b/server/src/engine/core/exec.rs
@@ -93,6 +93,14 @@ fn _callgcs<A: ASTNode<'static> + core::fmt::Debug, T>(
     f(&g, cstate, a)
 }
 
+#[inline(always)]
+fn translate_ddl_result(x: Option<bool>) -> Response {
+    match x {
+        Some(b) => Response::Bool(b),
+        None => Response::Empty,
+    }
+}
+
 async fn run_blocking_stmt(
     global: &Global,
     cstate: &mut ClientLocalState,
@@ -132,12 +140,26 @@ async fn run_blocking_stmt(
     ) -> QueryResult<Response>; 8] = [
         |_, _, _| Err(QueryError::QLUnknownStatement),
         blocking_exec_sysctl,
-        |g, _, t| _callgs_map(&g, t, Space::transactional_exec_create, Response::Bool),
-        |g, _, t| _callgs_map(&g, t, Model::transactional_exec_create, Response::Bool),
+        |g, _, t| {
+            _callgs_map(
+                &g,
+                t,
+                Space::transactional_exec_create,
+                translate_ddl_result,
+            )
+        },
+        |g, _, t| {
+            _callgs_map(
+                &g,
+                t,
+                Model::transactional_exec_create,
+                translate_ddl_result,
+            )
+        },
         |g, _, t| _callgs_map(&g, t, Space::transactional_exec_alter, |_| Response::Empty),
         |g, _, t| _callgs_map(&g, t, Model::transactional_exec_alter, |_| Response::Empty),
-        |g, _, t| _callgs_map(&g, t, Space::transactional_exec_drop, Response::Bool),
-        |g, _, t| _callgs_map(&g, t, Model::transactional_exec_drop, Response::Bool),
+        |g, _, t| _callgs_map(&g, t, Space::transactional_exec_drop, translate_ddl_result),
+        |g, _, t| _callgs_map(&g, t, Model::transactional_exec_drop, translate_ddl_result),
     ];
     let r = unsafe {
         // UNSAFE(@ohsayan): the only await is within this block
diff --git a/server/src/engine/core/model/mod.rs b/server/src/engine/core/model/mod.rs
index cb921505..0f9b9d8d 100644
--- a/server/src/engine/core/model/mod.rs
+++ b/server/src/engine/core/model/mod.rs
@@ -267,7 +267,7 @@ impl Model {
     pub fn transactional_exec_create<G: GlobalInstanceLike>(
         global: &G,
         stmt: CreateModel,
-    ) -> QueryResult<bool> {
+    ) -> QueryResult<Option<bool>> {
         let (space_name, model_name) = (stmt.model_name.space(), stmt.model_name.entity());
         let if_nx = stmt.if_not_exists;
         let model = Self::process_create(stmt)?;
@@ -275,7 +275,7 @@ impl Model {
             // TODO(@ohsayan): be extra cautious with post-transactional tasks (memck)
             if space.models().contains(model_name) {
                 if if_nx {
-                    return Ok(false);
+                    return Ok(Some(false));
                 } else {
                     return Err(QueryError::QExecDdlObjectAlreadyExists);
                 }
@@ -320,18 +320,22 @@ impl Model {
                 .idx_models()
                 .write()
                 .insert(EntityID::new(&space_name, &model_name), model);
-            Ok(true)
+            if if_nx {
+                Ok(Some(true))
+            } else {
+                Ok(None)
+            }
         })
     }
     pub fn transactional_exec_drop<G: GlobalInstanceLike>(
         global: &G,
         stmt: DropModel,
-    ) -> QueryResult<bool> {
+    ) -> QueryResult<Option<bool>> {
         let (space_name, model_name) = (stmt.entity.space(), stmt.entity.entity());
         global.namespace().ddl_with_space_mut(&space_name, |space| {
             if !space.models().contains(model_name) {
                 if stmt.if_exists {
-                    return Ok(false);
+                    return Ok(Some(false));
                 } else {
                     // the model isn't even present
                     return Err(QueryError::QExecObjectNotFound);
@@ -370,7 +374,11 @@ impl Model {
             // update global state
             let _ = models_idx.remove(&EntityIDRef::new(&space_name, &model_name));
             let _ = space.models_mut().remove(model_name);
-            Ok(true)
+            if stmt.if_exists {
+                Ok(Some(true))
+            } else {
+                Ok(None)
+            }
         })
     }
 }
diff --git a/server/src/engine/core/space.rs b/server/src/engine/core/space.rs
index de2ab239..5567d7ed 100644
--- a/server/src/engine/core/space.rs
+++ b/server/src/engine/core/space.rs
@@ -148,7 +148,7 @@ impl Space {
     pub fn transactional_exec_create<G: GlobalInstanceLike>(
         global: &G,
         space: CreateSpace,
-    ) -> QueryResult<bool> {
+    ) -> QueryResult<Option<bool>> {
         // process create
         let ProcedureCreate {
             space_name,
@@ -159,7 +159,7 @@ impl Space {
         global.namespace().ddl_with_spaces_write(|spaces| {
             if spaces.st_contains(&space_name) {
                 if if_not_exists {
-                    return Ok(false);
+                    return Ok(Some(false));
                 } else {
                     return Err(QueryError::QExecDdlObjectAlreadyExists);
                 }
@@ -187,7 +187,11 @@ impl Space {
             }
             // update global state
             let _ = spaces.st_insert(space_name, space);
-            Ok(true)
+            if if_not_exists {
+                Ok(Some(true))
+            } else {
+                Ok(None)
+            }
         })
     }
     #[allow(unused)]
@@ -234,12 +238,12 @@ impl Space {
             force,
             if_exists,
         }: DropSpace,
-    ) -> QueryResult<bool> {
+    ) -> QueryResult<Option<bool>> {
         if force {
             global.namespace().ddl_with_all_mut(|spaces, models| {
                 let Some(space) = spaces.remove(space_name.as_str()) else {
                     if if_exists {
-                        return Ok(false);
+                        return Ok(Some(false));
                     } else {
                         return Err(QueryError::QExecObjectNotFound);
                     }
@@ -272,13 +276,17 @@ impl Space {
                     );
                 }
                 let _ = spaces.st_delete(space_name.as_str());
-                Ok(true)
+                if if_exists {
+                    Ok(Some(true))
+                } else {
+                    Ok(None)
+                }
             })
         } else {
             global.namespace().ddl_with_spaces_write(|spaces| {
                 let Some(space) = spaces.get(space_name.as_str()) else {
                     if if_exists {
-                        return Ok(false);
+                        return Ok(Some(false));
                     } else {
                         return Err(QueryError::QExecObjectNotFound);
                     }
@@ -300,7 +308,11 @@ impl Space {
                     ));
                 }
                 let _ = spaces.st_delete(space_name.as_str());
-                Ok(true)
+                if if_exists {
+                    Ok(Some(true))
+                } else {
+                    Ok(None)
+                }
             })
         }
     }
diff --git a/server/src/engine/idx/mtchm/iter.rs b/server/src/engine/idx/mtchm/iter.rs
index c7838fb0..10b008f6 100644
--- a/server/src/engine/idx/mtchm/iter.rs
+++ b/server/src/engine/idx/mtchm/iter.rs
@@ -84,7 +84,6 @@ where
     i: RawIter<'t, 'g, 'v, T, C, CfgIterEntry>,
 }
 
-
 impl<'t, 'g, 'v, T, C> IterEntry<'t, 'g, 'v, T, C>
 where
     't: 'v,
diff --git a/server/src/engine/ql/tests/lexer_tests.rs b/server/src/engine/ql/tests/lexer_tests.rs
index ce6b5859..e1aeec6e 100644
--- a/server/src/engine/ql/tests/lexer_tests.rs
+++ b/server/src/engine/ql/tests/lexer_tests.rs
@@ -143,23 +143,14 @@ fn lex_string_escape_bs() {
 #[test]
 fn lex_string_bad_escape() {
     let wth = br#" '\a should be an alert on windows apparently' "#;
-    assert_eq!(
-        lex_insecure(wth).unwrap_err(),
-        QueryError::LexInvalidInput
-    );
+    assert_eq!(lex_insecure(wth).unwrap_err(), QueryError::LexInvalidInput);
 }
 #[test]
 fn lex_string_unclosed() {
     let wth = br#" 'omg where did the end go "#;
-    assert_eq!(
-        lex_insecure(wth).unwrap_err(),
-        QueryError::LexInvalidInput
-    );
+    assert_eq!(lex_insecure(wth).unwrap_err(), QueryError::LexInvalidInput);
     let wth = br#" 'see, we escaped the end\' "#;
-    assert_eq!(
-        lex_insecure(wth).unwrap_err(),
-        QueryError::LexInvalidInput
-    );
+    assert_eq!(lex_insecure(wth).unwrap_err(), QueryError::LexInvalidInput);
 }
 #[test]
 fn lex_unsafe_literal_mini() {
diff --git a/server/src/engine/tests/client/mod.rs b/server/src/engine/tests/client/mod.rs
index cbecb1bd..2ac2be22 100644
--- a/server/src/engine/tests/client/mod.rs
+++ b/server/src/engine/tests/client/mod.rs
@@ -23,4 +23,3 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.
  *
 */
-

From 6904224eb1697e01f9a094b0925382fc3737b1e2 Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 6 Dec 2023 01:07:32 +0530
Subject: [PATCH 309/310] Fix Docker images

---
 .dockerignore                      |  3 ++-
 .github/workflows/docker-image.yml | 17 ++++++++---------
 Dockerfile                         | 14 ++++++++------
 pkg/docker/start-server.sh         | 22 ++++++++++++++++++++++
 4 files changed, 40 insertions(+), 16 deletions(-)
 create mode 100644 pkg/docker/start-server.sh

diff --git a/.dockerignore b/.dockerignore
index 4cec9f56..73b3870e 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -4,4 +4,5 @@
 # Only include the skyd binary
 !target/release/skyd
 !target/release/skysh
-!examples/config-files/docker.toml
\ No newline at end of file
+!examples/config-files/dpkg/config.yaml
+!pkg/docker/start-server.sh
\ No newline at end of file
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
index 8249413f..a972a9d9 100644
--- a/.github/workflows/docker-image.yml
+++ b/.github/workflows/docker-image.yml
@@ -2,11 +2,8 @@ name: Docker image
 
 on:
   push:
-    # Publish `next` as Docker `latest` image.
     branches:
       - next
-
-    # Publish `v1.2.3` tags as releases.
     tags:
       - v*
 
@@ -34,14 +31,16 @@ jobs:
           command: build
           args: --release
       - name: Build image
-        run: docker build . --file Dockerfile --tag $IMAGE_NAME
-        if: env.BUILD == 'true' || github.event_name == 'create' && startsWith(github.ref, 'refs/tags/v')
+        run: docker build . --file Dockerfile --tag $IMAGE_NAME:${{ github.ref == 'refs/heads/next' && 'next' || github.ref_name }}
+        if: github.ref == 'refs/heads/next' || startsWith(github.ref, 'refs/tags/v')
       - name: Push to Docker Hub
         uses: docker/build-push-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: skytable/sdb
-          tags: latest
-          tag_with_ref: true
-        if: env.BUILD == 'true' || github.event_name == 'create' && startsWith(github.ref, 'refs/tags/v')
+          repository: skytable/skytable
+          tags: |
+            ${{ github.ref == 'refs/heads/next' && 'next' || '' }}
+            ${{ startsWith(github.ref, 'refs/tags/v') && github.ref_name || '' }}
+            ${{ startsWith(github.ref, 'refs/tags/v') && 'latest' || '' }}
+        if: github.ref == 'refs/heads/next' || startsWith(github.ref, 'refs/tags/v')
diff --git a/Dockerfile b/Dockerfile
index 03820a3b..c58a8ce8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,14 +1,16 @@
-#
 # The Dockerfile for the Skytable server sdb
-#
 
 FROM debian:stable
-
+# Copy the necessary binaries
 COPY target/release/skyd /usr/local/bin
 COPY target/release/skysh /usr/local/bin
-RUN mkdir /etc/skytable
+# Create necessary directories
 RUN mkdir /var/lib/skytable
-COPY examples/config-files/docker.toml /etc/skytable/skyd.toml
+COPY examples/config-files/dpkg/config.yaml /var/lib/skytable/config.yaml
+COPY pkg/docker/start-server.sh /usr/local/bin/start-server.sh
 WORKDIR /var/lib/skytable
-CMD ["skyd", "-c", "/etc/skytable/skyd.toml"]
+# Install uuidgen for generating a random password
+RUN apt-get update && apt-get install -y uuid-runtime
+RUN chmod +x /usr/local/bin/start-server.sh
+ENTRYPOINT ["/usr/local/bin/start-server.sh"]
 EXPOSE 2003/tcp
diff --git a/pkg/docker/start-server.sh b/pkg/docker/start-server.sh
new file mode 100644
index 00000000..6f16923f
--- /dev/null
+++ b/pkg/docker/start-server.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+CONFIG_FILE="/var/lib/skytable/config.yaml"
+PASSWORD_MARKER="rootpass"
+IP_MARKER="127.0.0.1"
+
+generate_password() {
+    uuidgen | cut -c -16
+}
+
+sed -i "s/$IP_MARKER/0.0.0.0/g" "$CONFIG_FILE"
+
+if grep -q "$PASSWORD_MARKER" "$CONFIG_FILE"; then
+    # Password not set, generate a new one
+    PASSWORD=$(generate_password)
+    sed -i "s/$PASSWORD_MARKER/$PASSWORD/g" "$CONFIG_FILE"
+    echo "Generated Password: $PASSWORD"
+else
+    echo "Using existing password in config file"
+fi
+
+exec skyd --config "$CONFIG_FILE"

From 3c57e4453f5dab177a7ff7d97fb3f2716161f80e Mon Sep 17 00:00:00 2001
From: Sayan Nandan <nandansayan@outlook.com>
Date: Wed, 6 Dec 2023 01:39:23 +0530
Subject: [PATCH 310/310] Upgrade deps and fix openssl certgen

---
 Cargo.lock              | 419 ++++++++++++++++++++++------------------
 cli/Cargo.toml          |   2 +-
 harness/Cargo.toml      |   6 +-
 harness/src/test/mod.rs |   2 +-
 server/Cargo.toml       |  16 +-
 5 files changed, 248 insertions(+), 197 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 306ce0da..b35e3b21 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4,9 +4,9 @@ version = 3
 
 [[package]]
 name = "addr2line"
-version = "0.20.0"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
+checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
 dependencies = [
  "gimli",
 ]
@@ -30,9 +30,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "1.0.3"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86b8f9420f797f2d9e935edf629310eb938a0d839f984e25327f3c7eed22300c"
+checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
 dependencies = [
  "memchr",
 ]
@@ -45,7 +45,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.28",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -56,9 +56,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "backtrace"
-version = "0.3.68"
+version = "0.3.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
+checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
 dependencies = [
  "addr2line",
  "cc",
@@ -102,9 +102,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.3.3"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
+checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
 
 [[package]]
 name = "block-buffer"
@@ -127,15 +127,15 @@ dependencies = [
 
 [[package]]
 name = "byteorder"
-version = "1.4.3"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.4.0"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
+checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
 
 [[package]]
 name = "bzip2"
@@ -160,9 +160,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.82"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01"
+checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
 dependencies = [
  "jobserver",
  "libc",
@@ -186,13 +186,11 @@ dependencies = [
 
 [[package]]
 name = "clipboard-win"
-version = "4.5.0"
+version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362"
+checksum = "c57002a5d9be777c1ef967e33674dac9ebd310d8893e4e3437b14d5f0f6372cc"
 dependencies = [
  "error-code",
- "str-buf",
- "winapi",
 ]
 
 [[package]]
@@ -203,9 +201,9 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
 
 [[package]]
 name = "core-foundation"
-version = "0.9.3"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"
+checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -213,15 +211,15 @@ dependencies = [
 
 [[package]]
 name = "core-foundation-sys"
-version = "0.8.4"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
+checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.9"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1"
+checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0"
 dependencies = [
  "libc",
 ]
@@ -237,9 +235,9 @@ dependencies = [
 
 [[package]]
 name = "crc-catalog"
-version = "2.2.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484"
+checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
 
 [[package]]
 name = "crc32fast"
@@ -288,7 +286,7 @@ version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df"
 dependencies = [
- "bitflags 2.3.3",
+ "bitflags 2.4.1",
  "crossterm_winapi",
  "libc",
  "mio",
@@ -319,9 +317,12 @@ dependencies = [
 
 [[package]]
 name = "deranged"
-version = "0.3.7"
+version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929"
+checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc"
+dependencies = [
+ "powerfmt",
+]
 
 [[package]]
 name = "digest"
@@ -361,34 +362,19 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 
 [[package]]
 name = "errno"
-version = "0.3.2"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
+checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"
 dependencies = [
- "errno-dragonfly",
- "libc",
- "windows-sys",
-]
-
-[[package]]
-name = "errno-dragonfly"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
-dependencies = [
- "cc",
  "libc",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "error-code"
-version = "2.3.1"
+version = "3.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21"
-dependencies = [
- "libc",
- "str-buf",
-]
+checksum = "281e452d3bad4005426416cdba5ccfd4f5c1280e10099e21db27f7c1c28347fc"
 
 [[package]]
 name = "fastrand"
@@ -398,20 +384,20 @@ checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
 
 [[package]]
 name = "fd-lock"
-version = "3.0.13"
+version = "4.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5"
+checksum = "b93f7a0db71c99f68398f80653ed05afb0b00e062e1a20c7ff849c4edfabbbcc"
 dependencies = [
  "cfg-if",
  "rustix",
- "windows-sys",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "flate2"
-version = "1.0.26"
+version = "1.0.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743"
+checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e"
 dependencies = [
  "crc32fast",
  "miniz_oxide",
@@ -434,30 +420,30 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
 
 [[package]]
 name = "futures-channel"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
+checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb"
 dependencies = [
  "futures-core",
 ]
 
 [[package]]
 name = "futures-core"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
+checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c"
 
 [[package]]
 name = "futures-task"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
+checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2"
 
 [[package]]
 name = "futures-util"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
+checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -479,9 +465,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.10"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
+checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
 dependencies = [
  "cfg-if",
  "libc",
@@ -490,9 +476,9 @@ dependencies = [
 
 [[package]]
 name = "gimli"
-version = "0.27.3"
+version = "0.28.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
+checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
 
 [[package]]
 name = "harness"
@@ -508,15 +494,15 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.14.0"
+version = "0.14.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
+checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
 
 [[package]]
 name = "hermit-abi"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
+checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
 
 [[package]]
 name = "hmac"
@@ -533,7 +519,7 @@ version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb"
 dependencies = [
- "windows-sys",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -544,9 +530,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "indexmap"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
+checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
 dependencies = [
  "equivalent",
  "hashbrown",
@@ -569,7 +555,7 @@ checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
 dependencies = [
  "hermit-abi",
  "rustix",
- "windows-sys",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -600,9 +586,9 @@ dependencies = [
 
 [[package]]
 name = "jobserver"
-version = "0.1.26"
+version = "0.1.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
+checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
 dependencies = [
  "libc",
 ]
@@ -625,15 +611,15 @@ version = "0.8.0"
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.5"
+version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
+checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456"
 
 [[package]]
 name = "lock_api"
-version = "0.4.10"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16"
+checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"
 dependencies = [
  "autocfg",
  "scopeguard",
@@ -678,7 +664,7 @@ dependencies = [
  "libc",
  "log",
  "wasi",
- "windows-sys",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -710,11 +696,11 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.26.4"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
+checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags 2.4.1",
  "cfg-if",
  "libc",
 ]
@@ -731,9 +717,9 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.31.1"
+version = "0.32.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
+checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
 dependencies = [
  "memchr",
 ]
@@ -746,11 +732,11 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
 [[package]]
 name = "openssl"
-version = "0.10.56"
+version = "0.10.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e"
+checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags 2.4.1",
  "cfg-if",
  "foreign-types",
  "libc",
@@ -767,7 +753,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.28",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -778,18 +764,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-src"
-version = "111.27.0+1.1.1v"
+version = "300.1.6+3.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06e8f197c82d7511c5b014030c9b1efeda40d7d5f99d23b4ceed3524a5e63f02"
+checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085"
 dependencies = [
  "cc",
 ]
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.91"
+version = "0.9.97"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac"
+checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b"
 dependencies = [
  "cc",
  "libc",
@@ -810,15 +796,15 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.8"
+version = "0.9.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
+checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
 dependencies = [
  "cfg-if",
  "libc",
  "redox_syscall",
  "smallvec",
- "windows-targets",
+ "windows-targets 0.48.5",
 ]
 
 [[package]]
@@ -846,9 +832,9 @@ dependencies = [
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.12"
+version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05"
+checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
 
 [[package]]
 name = "pin-utils"
@@ -862,6 +848,12 @@ version = "0.3.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
 
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
 [[package]]
 name = "powershell_script"
 version = "1.1.0"
@@ -956,9 +948,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.3.5"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
+checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
 dependencies = [
  "bitflags 1.3.2",
 ]
@@ -1000,24 +992,24 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
 
 [[package]]
 name = "rustix"
-version = "0.38.7"
+version = "0.38.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "172891ebdceb05aa0005f533a6cbfca599ddd7d966f6f5d4d9b2e70478e70399"
+checksum = "9470c4bf8246c8daf25f9598dca807fb6510347b1e1cfa55749113850c79d88a"
 dependencies = [
- "bitflags 2.3.3",
+ "bitflags 2.4.1",
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "rustyline"
-version = "12.0.0"
+version = "13.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "994eca4bca05c87e86e15d90fc7a91d1be64b4482b38cb2d27474568fe7c9db9"
+checksum = "02a2d683a4ac90aeef5b1013933f6d977bd37d51ff3f4dad829d4931a7e6be86"
 dependencies = [
- "bitflags 2.3.3",
+ "bitflags 2.4.1",
  "cfg-if",
  "clipboard-win",
  "fd-lock",
@@ -1027,7 +1019,6 @@ dependencies = [
  "memchr",
  "nix",
  "radix_trie",
- "scopeguard",
  "unicode-segmentation",
  "unicode-width",
  "utf8parse",
@@ -1046,7 +1037,7 @@ version = "0.1.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88"
 dependencies = [
- "windows-sys",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -1089,29 +1080,29 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.183"
+version = "1.0.193"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c"
+checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.183"
+version = "1.0.193"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816"
+checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.28",
+ "syn 2.0.39",
 ]
 
 [[package]]
 name = "serde_yaml"
-version = "0.9.25"
+version = "0.9.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574"
+checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c"
 dependencies = [
  "indexmap",
  "itoa",
@@ -1122,9 +1113,9 @@ dependencies = [
 
 [[package]]
 name = "sha1"
-version = "0.10.5"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"
+checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -1133,9 +1124,9 @@ dependencies = [
 
 [[package]]
 name = "sha2"
-version = "0.10.7"
+version = "0.10.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
+checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -1248,18 +1239,18 @@ dependencies = [
 
 [[package]]
 name = "slab"
-version = "0.4.8"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d"
+checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
 dependencies = [
  "autocfg",
 ]
 
 [[package]]
 name = "smallvec"
-version = "1.11.0"
+version = "1.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
+checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970"
 
 [[package]]
 name = "socket2"
@@ -1268,15 +1259,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9"
 dependencies = [
  "libc",
- "windows-sys",
+ "windows-sys 0.48.0",
 ]
 
-[[package]]
-name = "str-buf"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0"
-
 [[package]]
 name = "subtle"
 version = "2.5.0"
@@ -1296,9 +1281,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.28"
+version = "2.0.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567"
+checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1307,42 +1292,43 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.8.0"
+version = "3.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
+checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5"
 dependencies = [
  "cfg-if",
  "fastrand",
  "redox_syscall",
  "rustix",
- "windows-sys",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
 name = "termcolor"
-version = "1.2.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
+checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449"
 dependencies = [
  "winapi-util",
 ]
 
 [[package]]
 name = "time"
-version = "0.3.25"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea"
+checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5"
 dependencies = [
  "deranged",
+ "powerfmt",
  "serde",
  "time-core",
 ]
 
 [[package]]
 name = "time-core"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
+checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
 
 [[package]]
 name = "tokio"
@@ -1360,7 +1346,7 @@ dependencies = [
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "windows-sys",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -1371,7 +1357,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.28",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -1398,15 +1384,15 @@ dependencies = [
 
 [[package]]
 name = "typenum"
-version = "1.16.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
+checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.11"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
 
 [[package]]
 name = "unicode-segmentation"
@@ -1434,9 +1420,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
 
 [[package]]
 name = "uuid"
-version = "1.4.1"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d"
+checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560"
 dependencies = [
  "getrandom",
  "rand",
@@ -1445,13 +1431,13 @@ dependencies = [
 
 [[package]]
 name = "uuid-macro-internal"
-version = "1.4.1"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7e1ba1f333bd65ce3c9f27de592fcbc256dafe3af2717f56d7c87761fbaccf4"
+checksum = "f49e7f3f3db8040a100710a11932239fd30697115e2ba4107080d8252939845e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.28",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -1490,9 +1476,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
 [[package]]
 name = "winapi-util"
-version = "0.1.5"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"
 dependencies = [
  "winapi",
 ]
@@ -1509,65 +1495,131 @@ version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
 dependencies = [
- "windows-targets",
+ "windows-targets 0.48.5",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.0",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.5",
+ "windows_aarch64_msvc 0.48.5",
+ "windows_i686_gnu 0.48.5",
+ "windows_i686_msvc 0.48.5",
+ "windows_x86_64_gnu 0.48.5",
+ "windows_x86_64_gnullvm 0.48.5",
+ "windows_x86_64_msvc 0.48.5",
 ]
 
 [[package]]
 name = "windows-targets"
-version = "0.48.1"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
+checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
 dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
+ "windows_aarch64_gnullvm 0.52.0",
+ "windows_aarch64_msvc 0.52.0",
+ "windows_i686_gnu 0.52.0",
+ "windows_i686_msvc 0.52.0",
+ "windows_x86_64_gnu 0.52.0",
+ "windows_x86_64_gnullvm 0.52.0",
+ "windows_x86_64_msvc 0.52.0",
 ]
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.48.0"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.48.0"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.48.0"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
 
 [[package]]
 name = "zip"
@@ -1610,11 +1662,10 @@ dependencies = [
 
 [[package]]
 name = "zstd-sys"
-version = "2.0.8+zstd.1.5.5"
+version = "2.0.9+zstd.1.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c"
+checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656"
 dependencies = [
  "cc",
- "libc",
  "pkg-config",
 ]
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index acd78d13..a1fe8fbc 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -12,4 +12,4 @@ libsky = { path = "../libsky" }
 skytable = { git = "https://github.com/skytable/client-rust.git", branch = "octave" }
 # external deps
 crossterm = "0.27.0"
-rustyline = "12.0.0"
+rustyline = "13.0.0"
diff --git a/harness/Cargo.toml b/harness/Cargo.toml
index 1bd479a9..247cb9a4 100644
--- a/harness/Cargo.toml
+++ b/harness/Cargo.toml
@@ -9,8 +9,8 @@ edition = "2021"
 # internal deps
 libsky = { path = "../libsky" }
 # external deps
-env_logger = "0.10.0"
-log = "0.4.19"
+env_logger = "0.10.1"
+log = "0.4.20"
 zip = { version = "0.6.6", features = ["deflate"] }
 powershell_script = "1.1.0"
-openssl = { version = "0.10.55", features = ["vendored"] }
+openssl = { version = "0.10.61", features = ["vendored"] }
diff --git a/harness/src/test/mod.rs b/harness/src/test/mod.rs
index 46c2a0f1..de97c421 100644
--- a/harness/src/test/mod.rs
+++ b/harness/src/test/mod.rs
@@ -171,6 +171,6 @@ fn mk_ca_cert(password: &[u8]) -> Result<(Vec<u8>, Vec<u8>), ErrorStack> {
     cert_builder.append_extension(subject_key_identifier)?;
     cert_builder.sign(&key_pair, MessageDigest::sha256())?;
     let cert = cert_builder.build().to_pem().unwrap();
-    let key_pair = key_pair.private_key_to_pem_pkcs8_passphrase(Cipher::des_cbc(), password)?;
+    let key_pair = key_pair.private_key_to_pem_pkcs8_passphrase(Cipher::aes_256_cbc(), password)?;
     Ok((cert, key_pair))
 }
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 8e6ed0f3..064203ed 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -12,18 +12,18 @@ libsky = { path = "../libsky" }
 sky_macros = { path = "../sky-macros" }
 rcrypt = "0.4.0"
 # external deps
-bytes = "1.4.0"
-env_logger = "0.10.0"
-log = "0.4.19"
-openssl = { version = "0.10.56", features = ["vendored"] }
+bytes = "1.5.0"
+env_logger = "0.10.1"
+log = "0.4.20"
+openssl = { version = "0.10.61", features = ["vendored"] }
 crossbeam-epoch = { version = "0.9.15" }
 parking_lot = "0.12.1"
-serde = { version = "1.0.183", features = ["derive"] }
+serde = { version = "1.0.193", features = ["derive"] }
 tokio = { version = "1.34.0", features = ["full"] }
 tokio-openssl = "0.6.3"
-uuid = { version = "1.4.1", features = ["v4", "fast-rng", "macro-diagnostics"] }
+uuid = { version = "1.6.1", features = ["v4", "fast-rng", "macro-diagnostics"] }
 crc = "3.0.1"
-serde_yaml = "0.9"
+serde_yaml = "0.9.27"
 
 [target.'cfg(all(not(target_env = "msvc"), not(miri)))'.dependencies]
 # external deps
@@ -38,7 +38,7 @@ winapi = { version = "0.3.9", features = [
 
 [target.'cfg(unix)'.dependencies]
 # external deps
-libc = "0.2.147"
+libc = "0.2.150"
 
 [dev-dependencies]
 # external deps